1933 files changed, 79072 insertions, 25235 deletions

diff --git a/ChangeLog b/ChangeLog
index c3d503c9f00..abf3341c68c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2017-10-27  Martin Liska  <mliska@suse.cz>
+
+	* Makefile.tpl: Use proper name of folder as it was renamed
+	during transition to 4 stages.
+	* Makefile.in: Regenerate.
+
+2017-10-16  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+        * MAINTAINERS (write after approval): Add myself.
+
 2017-10-01  Gerald Pfeifer  <gerald@pfeifer.com>
 
 	* MAINTAINERS: Add a note that maintainership also includes web
diff --git a/MAINTAINERS b/MAINTAINERS
index 0d33f0b14ac..9c3a56ea094 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -603,6 +603,7 @@ Ilya Tocar					<tocarip@gmail.com>
 Philipp Tomsich					<philipp.tomsich@theobroma-systems.com>
 Konrad Trifunovic				<konrad.trifunovic@inria.fr>
 Markus Trippelsdorf				<markus@trippelsdorf.de>
+Igor Tsimbalist					<igor.v.tsimbalist@intel.com>
 Martin Uecker					<uecker@eecs.berkeley.edu>
 David Ung					<davidu@mips.com>
 Neil Vachharajani				<nvachhar@gmail.com>
diff --git a/Makefile.in b/Makefile.in
index 78db0982ba2..13d23915349 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -57174,8 +57174,8 @@ stageprofile-end::
 stagefeedback-start::
 	@r=`${PWD_COMMAND}`; export r; \
 	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	for i in prev-*; do \
-	  j=`echo $$i | sed s/^prev-//`; \
+	for i in stageprofile-*; do \
+	  j=`echo $$i | sed s/^stageprofile-//`; \
 	  cd $$r/$$i && \
 	  { find . -type d | sort | sed 's,.*,$(SHELL) '"$$s"'/mkinstalldirs "../'$$j'/&",' | $(SHELL); } && \
 	  { find . -name '*.*da' | sed 's,.*,$(LN) -f "&" "../'$$j'/&",' | $(SHELL); }; \
diff --git a/Makefile.tpl b/Makefile.tpl
index 5fcd7e358d9..1f23b79b4b2 100644
--- a/Makefile.tpl
+++ b/Makefile.tpl
@@ -1718,8 +1718,8 @@ stageprofile-end::
 stagefeedback-start::
 	@r=`${PWD_COMMAND}`; export r; \
 	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	for i in prev-*; do \
-	  j=`echo $$i | sed s/^prev-//`; \
+	for i in stageprofile-*; do \
+	  j=`echo $$i | sed s/^stageprofile-//`; \
 	  cd $$r/$$i && \
 	  { find . -type d | sort | sed 's,.*,$(SHELL) '"$$s"'/mkinstalldirs "../'$$j'/&",' | $(SHELL); } && \
 	  { find . -name '*.*da' | sed 's,.*,$(LN) -f "&" "../'$$j'/&",' | $(SHELL); }; \
diff --git a/config/ChangeLog b/config/ChangeLog
index 6732bf4f73f..2e3ead0a26b 100644
--- a/config/ChangeLog
+++ b/config/ChangeLog
@@ -1,3 +1,7 @@
+2017-10-24  H.J. Lu  <hongjiu.lu@intel.com>
+
+	* bootstrap-cet.mk: New file.
+
 2017-06-19  Martin Liska  <mliska@suse.cz>
 
 	* bootstrap-lto-noplugin.mk: Enable -flto in all PGO stages.
diff --git a/config/bootstrap-cet.mk b/config/bootstrap-cet.mk
new file mode 100644
index 00000000000..f09193a6dea
--- /dev/null
+++ b/config/bootstrap-cet.mk
@@ -0,0 +1,4 @@
+# This option enables -fcf-protection -mcet for stage2 and stage3.
+
+STAGE2_CFLAGS += -fcf-protection -mcet
+STAGE3_CFLAGS += -fcf-protection -mcet
diff --git a/fixincludes/ChangeLog b/fixincludes/ChangeLog
index ffd171843e1..1f29148e8e3 100644
--- a/fixincludes/ChangeLog
+++ b/fixincludes/ChangeLog
@@ -1,3 +1,8 @@
+2017-10-13  Mike Frysinger  <vapier@chromium.org>
+
+	* fixinc.in (dirname): Change sed from 's|[^/]*/||' to
+	's|[^/]*//*||'.
+
 2017-06-12  Doug Rupp  <rupp@adacore.com>
 
 	* inclhack.def (AAB_vxworks_stdint): Remove hack.
diff --git a/fixincludes/fixinc.in b/fixincludes/fixinc.in
index 15cbaa23544..cd0b458b8f8 100755
--- a/fixincludes/fixinc.in
+++ b/fixincludes/fixinc.in
@@ -344,7 +344,7 @@ if $LINKS; then
             mkdir $component >/dev/null 2>&1
             cd $component
             dirmade=$dirmade/$component
-            dirname=`echo $dirname | sed -e 's|[^/]*/||'`
+            dirname=`echo $dirname | sed -e 's|[^/]*//*||'`
           done
         fi
 
diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index cb9f1a392aa..806732359b6 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,3364 @@
+2017-11-03  Jeff Law  <law@redhat.com>
+
+	* config/i386/i386.c (ix86_emit_restore_reg_using_pop): Prototype.
+	(ix86_adjust_stack_and_probe_stack_clash): Use a push/pop sequence
+	to probe at the start of a noreturn function.
+
+2017-11-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/78821
+	* gimple-ssa-store-merging.c: Update the file comment.
+	(MAX_STORE_ALIAS_CHECKS): Define.
+	(struct store_operand_info): New type.
+	(store_operand_info::store_operand_info): New constructor.
+	(struct store_immediate_info): Add rhs_code and ops data members.
+	(store_immediate_info::store_immediate_info): Add rhscode, op0r
+	and op1r arguments to the ctor, initialize corresponding data members.
+	(struct merged_store_group): Add load_align_base and load_align
+	data members.
+	(merged_store_group::merged_store_group): Initialize them.
+	(merged_store_group::do_merge): Update them.
+	(merged_store_group::apply_stores): Pick the constant for
+	encode_tree_to_bitpos from one of the two operands, or skip
+	encode_tree_to_bitpos if neither operand is a constant.
+	(class pass_store_merging): Add process_store method decl.  Remove
+	bool argument from terminate_all_aliasing_chains method decl.
+	(pass_store_merging::terminate_all_aliasing_chains): Remove
+	var_offset_p argument and corresponding handling.
+	(stmts_may_clobber_ref_p): New function.
+	(compatible_load_p): New function.
+	(imm_store_chain_info::coalesce_immediate_stores): Terminate group
+	if there is overlap and rhs_code is not INTEGER_CST.  For
+	non-overlapping stores terminate group if rhs is not mergeable.
+	(get_alias_type_for_stmts): Change first argument from
+	auto_vec<gimple *> & to vec<gimple *> &.  Add IS_LOAD, CLIQUEP and
+	BASEP arguments.  If IS_LOAD is true, look at rhs1 of the stmts
+	instead of lhs.  Compute *CLIQUEP and *BASEP in addition to the
+	alias type.
+	(get_location_for_stmts): Change first argument from
+	auto_vec<gimple *> & to vec<gimple *> &.
+	(struct split_store): Remove orig_stmts data member, add orig_stores.
+	(split_store::split_store): Create orig_stores rather than orig_stmts.
+	(find_constituent_stmts): Renamed to ...
+	(find_constituent_stores): ... this.  Change second argument from
+	vec<gimple *> * to vec<store_immediate_info *> *, push pointers
+	to info structures rather than the statements.
+	(split_group): Rename ALLOW_UNALIGNED argument to
+	ALLOW_UNALIGNED_STORE, add ALLOW_UNALIGNED_LOAD argument and handle
+	it.  Adjust find_constituent_stores caller.
+	(imm_store_chain_info::output_merged_store): Handle rhs_code other
+	than INTEGER_CST, adjust split_group, get_alias_type_for_stmts and
+	get_location_for_stmts callers.  Set MR_DEPENDENCE_CLIQUE and
+	MR_DEPENDENCE_BASE on the MEM_REFs if they are the same in all stores.
+	(mem_valid_for_store_merging): New function.
+	(handled_load): New function.
+	(pass_store_merging::process_store): New method.
+	(pass_store_merging::execute): Use process_store method.  Adjust
+	terminate_all_aliasing_chains caller.
+
+2017-11-03  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_legitimate_constant_p):
+	Return true for more constants, symbols and label references.
+	(aarch64_valid_floating_const): Remove unused function.
+
+2017-11-03  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	PR target/82786
+	* config/aarch64/aarch64.c (aarch64_layout_frame):
+	Undo forcing of LR at bottom of frame.
+
+2017-11-03  Jeff Law  <law@redhat.com>
+
+	PR target/82823
+	* config/i386/i386.c (ix86_expand_prologue): Tighten assert
+	for int_registers_saved.
+
+	* cfganal.c (single_pred_edge_ignoring_loop_edges): New function
+	extracted from tree-ssa-dom.c.
+	* cfganal.h (single_pred_edge_ignoring_loop_edges): Prototype.
+	* tree-ssa-dom.c (single_incoming_edge_ignoring_loop_edges): Remove.
+	(record_equivalences_from_incoming_edge): Add additional argument
+	to single_pred_edge_ignoring_loop_edges call.
+	* tree-ssa-uncprop.c (single_incoming_edge_ignoring_loop_edges): Remove.
+	(uncprop_dom_walker::before_dom_children): Add additional argument
+	to single_pred_edge_ignoring_loop_edges call.
+	* tree-ssa-sccvn.c (sccvn_dom_walker::before_dom_children): Use
+	single_pred_edge_ignoring_loop_edges rather than open coding.
+	* tree-vrp.c (evrp_dom_walker::before_dom_children): Similarly.
+
+2017-11-03  Marc Glisse  <marc.glisse@inria.fr>
+
+	* match.pd (-(-A)): Rewrite.
+
+2017-11-03  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	* config/rs6000/rs60000-protos.h (rs6000_emit_sISEL): Delete.
+	(rs6000_emit_int_cmove): New declaration.
+	* config/rs6000/rs6000.c (rs6000_emit_int_cmove): Delete declaration.
+	(rs6000_emit_sISEL): Delete.
+	(rs6000_emit_int_cmove): Make non-static.
+	* config/rs6000/rs6000.md (cstore<mode>4): Use rs6000_emit_int_cmove
+	instead of rs6000_emit_sISEL.
+
+2017-11-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* asan.c (create_cond_insert_point): Maintain profile.
+	* ipa-utils.c (ipa_merge_profiles): Be sure only IPA profiles are
+	merged.
+	* basic-block.h (struct basic_block_def): Remove frequency.
+	(EDGE_FREQUENCY): Use to_frequency
+	* bb-reorder.c (push_to_next_round_p): Use only IPA counts for global
+	heuristics.
+	(find_traces): Update to use to_frequency.
+	(find_traces_1_round): Likewise; use only IPA counts.
+	(bb_to_key): Likewise.
+	(connect_traces): Use IPA counts only.
+	(copy_bb_p): Update to use to_frequency.
+	(fix_up_crossing_landing_pad): Likewise.
+	(sanitize_hot_paths): Likewise.
+	* bt-load.c (basic_block_freq): Likewise.
+	* cfg.c (init_flow): Set count_max to uninitialized.
+	(check_bb_profile): Remove frequencies; check counts.
+	(dump_bb_info): Do not dump frequencies.
+	(update_bb_profile_for_threading): Update counts only.
+	(scale_bbs_frequencies_int): Likewise.
+	(MAX_SAFE_MULTIPLIER): Remove.
+	(scale_bbs_frequencies_gcov_type): Update counts only.
+	(scale_bbs_frequencies_profile_count): Update counts only.
+	(scale_bbs_frequencies): Update counts only.
+	* cfg.h (struct control_flow_graph): Add count-max.
+	(update_bb_profile_for_threading): Update prototype.
+	* cfgbuild.c (find_bb_boundaries): Do not update frequencies.
+	(find_many_sub_basic_blocks): Likewise.
+	* cfgcleanup.c (try_forward_edges): Likewise.
+	(try_crossjump_to_edge): Likewise.
+	* cfgexpand.c (expand_gimple_cond): Likewise.
+	(expand_gimple_tailcall): Likewise.
+	(construct_init_block): Likewise.
+	(construct_exit_block): Likewise.
+	* cfghooks.c (verify_flow_info): Check consistency of counts.
+	(dump_bb_for_graph): Do not dump frequencies.
+	(split_block_1): Do not update frequencies.
+	(split_edge): Do not update frequencies.
+	(make_forwarder_block): Do not update frequencies.
+	(duplicate_block): Do not update frequencies.
+	(account_profile_record): Do not update frequencies.
+	* cfgloop.c (find_subloop_latch_edge_by_profile): Use IPA counts
+	for global heuristics.
+	* cfgloopanal.c (average_num_loop_insns): Update to use to_frequency.
+	(expected_loop_iterations_unbounded): Use counts only.
+	* cfgloopmanip.c (scale_loop_profile): Simplify.
+	(create_empty_loop_on_edge): Simplify
+	(loopify): Simplify
+	(duplicate_loop_to_header_edge): Simplify
+	* cfgrtl.c (force_nonfallthru_and_redirect): Update profile.
+	(update_br_prob_note): Take care of removing note when profile
+	becomes undefined.
+	(relink_block_chain): Do not dump frequency.
+	(rtl_account_profile_record): Use to_frequency.
+	* cgraph.c (symbol_table::create_edge): Convert count to ipa count.
+	(cgraph_edge::redirect_call_stmt_to_calle): Conver tcount to ipa count.
+	(cgraph_update_edges_for_call_stmt_node): Likewise.
+	(cgraph_edge::verify_count_and_frequency): Update.
+	(cgraph_node::verify_node): Temporarily disable frequency verification.
+	* cgraphbuild.c (compute_call_stmt_bb_frequency): Use
+	to_cgraph_frequency.
+	(cgraph_edge::rebuild_edges): Convert to ipa counts.
+	* cgraphunit.c (init_lowered_empty_function): Do not initialize
+	frequencies.
+	(cgraph_node::expand_thunk): Update profile.
+	* except.c (dw2_build_landing_pads): Do not update frequency.
+	* final.c (compute_alignments): Use to_frequency.
+	(dump_basic_block_info): Do not dump frequency.
+	* gimple-pretty-print.c (dump_profile): Do not dump frequency.
+	(dump_gimple_bb_header): Do not dump frequency.
+	* gimple-ssa-isolate-paths.c (isolate_path): Do not update frequency;
+	do update count.
+	* gimple-streamer-in.c (input_bb): Do not stream frequency.
+	* gimple-streamer-out.c (output_bb): Do not stream frequency.
+	* haifa-sched.c (sched_pressure_start_bb): Use to_freuqency.
+	(init_before_recovery): Do not update frequency.
+	(sched_create_recovery_edges): Do not update frequency.
+	* hsa-gen.c (convert_switch_statements): Do not update frequency.
+	* ipa-cp.c (ipcp_propagate_stage): Update search for max_count.
+	(ipa_cp_c_finalize): Set max_count to uninitialized.
+	* ipa-fnsummary.c (get_minimal_bb): Use counts.
+	(param_change_prob): Use counts.
+	* ipa-profile.c (ipa_profile_generate_summary): Do not summarize
+	local profiles.
+	* ipa-split.c (consider_split): Use to_frequency.
+	(split_function): Use to_frequency.
+	* ira-build.c (loop_compare_func): Likewise.
+	(mark_loops_for_removal): Likewise.
+	(mark_all_loops_for_removal): Likewise.
+	* loop-doloop.c (doloop_modify): Do not update frequency.
+	* loop-unroll.c (unroll_loop_runtime_iterations): Do not update
+	frequency.
+	* lto-streamer-in.c (input_function): Update count_max.
+	* omp-expand.c (expand_omp_taskreg): Update count_max.
+	* omp-simd-clone.c (simd_clone_adjust): Update profile.
+	* predict.c (maybe_hot_frequency_p): Use to_frequency.
+	(maybe_hot_count_p): Use ipa counts only.
+	(maybe_hot_bb_p): Simplify.
+	(maybe_hot_edge_p): Simplify.
+	(probably_never_executed): Do not take frequency argument.
+	(probably_never_executed_bb_p): Do not pass frequency.
+	(probably_never_executed_edge_p): Likewise.
+	(combine_predictions_for_bb): Check that profile is nonzero.
+	(propagate_freq): Do not set frequency.
+	(drop_profile): Simplify.
+	(counts_to_freqs): Simplify.
+	(expensive_function_p): Use to_frequency.
+	(propagate_unlikely_bbs_forward): Simplify.
+	(determine_unlikely_bbs): Simplify.
+	(estimate_bb_frequencies): Add hack to silence graphite issues.
+	(compute_function_frequency): Use ipa counts.
+	(pass_profile::execute): Update.
+	(rebuild_frequencies): Use counts only.
+	(force_edge_cold): Use counts only.
+	* profile-count.c (profile_count::dump): Dump new count types.
+	(profile_count::differs_from_p): Check compatiblity.
+	(profile_count::to_frequency): New function.
+	(profile_count::to_cgraph_frequency): New function.
+	* profile-count.h (struct function): Declare.
+	(enum profile_quality): Add profile_guessed_local and
+	profile_guessed_global0.
+	(class profile_proability): Decrease number of bits to 29;
+	update from_reg_br_prob_note and to_reg_br_prob_note.
+	(class profile_count: Update comment; decrease number of bits
+	to 61. Check compatibility.
+	(profile_count::compatible_p): New private member function.
+	(profile_count::ipa_p): New member function.
+	(profile_count::operator<): Handle global zero correctly.
+	(profile_count::operator>): Handle global zero correctly.
+	(profile_count::operator<=): Handle global zero correctly.
+	(profile_count::operator>=): Handle global zero correctly.
+	(profile_count::nonzero_p): New member function.
+	(profile_count::force_nonzero): New member function.
+	(profile_count::max): New member function.
+	(profile_count::apply_scale): Handle IPA scalling.
+	(profile_count::guessed_local): New member function.
+	(profile_count::global0): New member function.
+	(profile_count::ipa): New member function.
+	(profile_count::to_frequency): Declare.
+	(profile_count::to_cgraph_frequency): Declare.
+	* profile.c (OVERLAP_BASE): Delete.
+	(compute_frequency_overlap): Delete.
+	(compute_branch_probabilities): Do not use compute_frequency_overlap.
+	* regs.h (REG_FREQ_FROM_BB): Use to_frequency.
+	* sched-ebb.c (rank): Use counts only.
+	* shrink-wrap.c (handle_simple_exit): Use counts only.
+	(try_shrink_wrapping): Use counts only.
+	(place_prologue_for_one_component): Use counts only.
+	* tracer.c (find_best_predecessor): Use to_frequency.
+	(find_trace): Use to_frequency.
+	(tail_duplicate): Use to_frequency.
+	* trans-mem.c (expand_transaction): Do not update frequency.
+	* tree-call-cdce.c: Do not update frequency. 
+	* tree-cfg.c (gimple_find_sub_bbs): Likewise.
+	(gimple_merge_blocks): Likewise.
+	(gimple_split_edge): Likewise.
+	(gimple_duplicate_sese_region): Likewise.
+	(gimple_duplicate_sese_tail): Likewise.
+	(move_sese_region_to_fn): Likewise.
+	(gimple_account_profile_record): Likewise.
+	(insert_cond_bb): Likewise.
+	* tree-complex.c (expand_complex_div_wide): Likewise.
+	* tree-eh.c (lower_resx): Update profile.
+	* tree-inline.c (copy_bb): Simplify count scaling; do not scale
+	frequencies.
+	(initialize_cfun): Do not initialize frequencies
+	(freqs_to_counts): Delete.
+	(copy_cfg_body): Ignore count parameter.
+	(copy_body): Update.
+	(expand_call_inline): Update count_max.
+	(optimize_inline_calls): Update count_max.
+	(tree_function_versioning): Update count_max.
+	* tree-ssa-coalesce.c (coalesce_cost_bb): Use to_frequency.
+	* tree-ssa-ifcombine.c (update_profile_after_ifcombine): Do not update
+	frequency.
+	* tree-ssa-loop-im.c (execute_sm_if_changed): Use counts only.
+	* tree-ssa-loop-ivcanon.c (unloop_loops): Do not update freuqency.
+	(try_peel_loop): Likewise.
+	* tree-ssa-loop-ivopts.c (get_scaled_computation_cost_at): Use
+	to_frequency.
+	* tree-ssa-loop-manip.c (niter_for_unrolled_loop): Pass -1.
+	(tree_transform_and_unroll_loop): Do not use frequencies
+	* tree-ssa-loop-niter.c (estimate_numbers_of_iterations):
+	Use reliable prediction only.
+	* tree-ssa-loop-unswitch.c (hoist_guard): Do not use frequencies.
+	* tree-ssa-sink.c (select_best_block): Use to_frequency.
+	* tree-ssa-tail-merge.c (replace_block_by): Temporarily disable
+	probability scaling.
+	* tree-ssa-threadupdate.c (create_block_for_threading): Do
+	not update frequency
+	(any_remaining_duplicated_blocks): Likewise.
+	(update_profile): Likewise.
+	(estimated_freqs_path): Delete.
+	(freqs_to_counts_path): Delete.
+	(clear_counts_path): Delete.
+	(ssa_fix_duplicate_block_edges): Likewise.
+	(duplicate_thread_path): Likewise.
+	* tree-switch-conversion.c (gen_inbound_check): Use counts.
+	* tree-tailcall.c (decrease_profile): Do not update frequency.
+	(eliminate_tail_call): Likewise.
+	* tree-vect-loop-manip.c (vect_do_peeling): Likewise.
+	* tree-vect-loop.c (scale_profile_for_vect_loop): Likewise.
+	(optimize_mask_stores): Likewise.
+	* tree-vect-stmts.c (vectorizable_simd_clone_call): Likewise.
+	* ubsan.c (ubsan_expand_null_ifn): Update profile.
+	(ubsan_expand_ptr_ifn): Update profile.
+	* value-prof.c (gimple_ic): Simplify.
+	* value-prof.h (gimple_ic): Update prototype.
+	* ipa-inline-transform.c (inline_transform): Fix scaling conditoins.
+	* ipa-inline.c (compute_uninlined_call_time): Be sure that
+	counts are nonzero.
+	(want_inline_self_recursive_call_p): Likewise.
+	(resolve_noninline_speculation): Only cummulate defined counts.
+	(inline_small_functions): Use nonzero_p.
+	(ipa_inline): Do not access freed node.
+
+2017-11-03  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_override_options_internal):
+	Set PARAM_SCHED_PRESSURE_ALGORITHM to SCHED_PRESSURE_MODEL.
+
+2017-11-03  Kito Cheng  <kito.cheng@gmail.com>
+
+	* config/riscv/riscv.c (riscv_legitimize_move): Handle
+	non-legitimate address.
+
+2017-11-03  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	* config/rs6000/rs6000.md (*lt0_disi): Delete.
+	(*lt0_<mode>di, *lt0_<mode>si): New.
+
+2017-11-03  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	* config/rs6000/rs6000.md (move_from_CR_ov_bit): Change condition to
+	TARGET_PAIRED_FLOAT.
+
+2017-11-03  Siddhesh Poyarekar  <siddhesh.poyarekar@linaro.org>
+	    Jim Wilson  <jim.wilson@linaro.org>
+
+	* config/aarch64/aarch64-cores.def (saphira): New CPU.
+	* config/aarch64/aarch64-tune.md: Regenerated.
+	* doc/invoke.texi (AArch64 Options/-mtune): Add "saphira".
+	* gcc/config/aarch64/aarch64.c (saphira_tunings): New tuning table.
+
+2017-11-03  Cupertino Miranda  <cmiranda@synopsys.com>
+
+	* config/arc/arc.c (arc_save_restore): Corrected CFA note.
+	(arc_expand_prologue): Restore blink for millicode.
+	* config/arc/linux.h (LINK_EH_SPEC): Defined.
+
+2017-11-03  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	PR target/82809
+	* config/i386/i386.c (ix86_vector_duplicate_value): Use
+	gen_vec_duplicate after forcing the scalar into a register.
+
+2017-11-02  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	* combine (try_combine): Print the insns input to try_combine to the
+	dump file.
+
+2017-11-02  Steve Ellcey  <sellcey@cavium.com>
+
+	PR target/79868
+	* config/aarch64/aarch64-c.c (aarch64_pragma_target_parse):
+	Remove second argument from aarch64_process_target_attr call.
+	* config/aarch64/aarch64-protos.h (aarch64_process_target_attr):
+	Ditto.
+	* config/aarch64/aarch64.c (aarch64_attribute_info): Change
+	field type.
+	(aarch64_handle_attr_arch): Remove second argument.
+	(aarch64_handle_attr_cpu): Ditto.
+	(aarch64_handle_attr_tune): Ditto.
+	(aarch64_handle_attr_isa_flags): Ditto.
+	(aarch64_process_one_target_attr): Ditto.
+	(aarch64_process_target_attr): Ditto.
+	(aarch64_option_valid_attribute_p): Remove second argument.
+	on aarch64_process_target_attr call.
+
+2017-11-02  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.c: Include "selftest-diagnostic.h".
+	(selftest::assert_location_text): New function.
+	(selftest::test_diagnostic_get_location_text): New function.
+	(selftest::diagnostic_c_tests): Call it.
+
+2017-11-02  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add selftest-diagnostic.o.
+	* diagnostic-show-locus.c: Include "selftest-diagnostic.h".
+	(class selftest::test_diagnostic_context): Move to...
+	* selftest-diagnostic.c: New file.
+	* selftest-diagnostic.h: New file.
+
+2017-11-02  James Bowman  <james.bowman@ftdichip.com>
+
+	* config/ft32/ft32.c (ft32_addr_space_legitimate_address_p): increase
+	offset range for FT32B.
+	* config/ft32/ft32.h: option "mcompress" enables relaxation.
+	* config/ft32/ft32.md: Add TARGET_NOPM.
+	* config/ft32/ft32.opt: Add mft32b, mcompress, mnopm.
+	* gcc/doc/invoke.texi: Add mft32b, mcompress, mnopm.
+
+2017-11-02  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/aarch64/aarch64.h (MALLOC_ABI_ALIGNMENT): New define.
+
+2017-11-02  Jeff Law  <law@redhat.com>
+
+	* gimple-ssa-sprintf.c (sprintf_dom_walker): Remove
+	virtual keyword on FINAL OVERRIDE members.
+
+	* tree-ssa-propagate.h (ssa_propagation_engine): Group
+	virtuals together.  Add virtual destructor.
+	(substitute_and_fold_engine): Similarly.
+
+2017-11-02  Jan Hubicka  <hubicka@ucw.cz>
+
+	* x86-tune.def (X86_TUNE_USE_INCDEC): Enable for Haswell+.
+
+2017-11-02  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82795
+	* tree-if-conv.c (predicate_mem_writes): Remove bogus assert.
+
+2017-11-02  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* acinclude.m4 (gcc_AC_INITFINI_ARRAY): Don't require
+	gcc_SUN_LD_VERSION.
+	(gcc_GAS_CHECK_FEATURE): Remove.
+	* configure.ac (ld_vers) <*-*-solaris2*>: Move comments from
+	gcc_AC_INITFINI_ARRAY here.  Update for Solaris 11.4 changes.
+	* configure: Regenerate.
+
+2017-11-02  Claudiu Zissulescu <claziss@synopsys.com>
+
+	* config/arc/arc.c (hwloop_optimize): Account for empty
+	body loops.
+
+2017-11-02  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/82765
+	* varasm.c (decode_addr_const): Make offset HOST_WIDE_INT.
+	Truncate ARRAY_REF index and element size.
+
+2017-11-01  Palmer Dabbelt  <palmer@dabbelt.com>
+
+	* doc/invoke.texi (RISC-V Options): Use "@minus{}2 GB", not "-2 GB".
+
+2017-11-01  Jeff Law  <law@redhat.com>
+
+	* tree-ssa-ccp.c (ccp_folder): New class derived from
+	substitute_and_fold_engine.
+	(ccp_folder::get_value): New member function.
+	(ccp_folder::fold_stmt): Renamed from ccp_fold_stmt.
+	(ccp_fold_stmt): Remove prototype.
+	(ccp_finalize): Call substitute_and_fold from the ccp_class.
+	* tree-ssa-copy.c (copy_folder): New class derived from
+	substitute_and_fold_engine.
+	(copy_folder::get_value): Renamed from get_value.
+	(fini_copy_prop): Call substitute_and_fold from copy_folder class.
+	* tree-vrp.c (vrp_folder): New class derived from
+	substitute_and_fold_engine.
+	(vrp_folder::fold_stmt): Renamed from vrp_fold_stmt.
+	(vrp_folder::get_value): New member function.
+	(vrp_finalize): Call substitute_and_fold from vrp_folder class.
+	(evrp_dom_walker::before_dom_children): Similarly for replace_uses_in.
+	* tree-ssa-propagate.h (substitute_and_fold_engine): New class to
+	provide a class interface to folder/substitute routines.
+	(ssa_prop_fold_stmt_fn): Remove typedef.
+	(ssa_prop_get_value_fn): Likewise.
+	(subsitute_and_fold): Remove prototype.
+	(replace_uses_in): Likewise.
+	* tree-ssa-propagate.c (substitute_and_fold_engine::replace_uses_in):
+	Renamed from replace_uses_in.  Call the virtual member function
+	(substitute_and_fold_engine::replace_phi_args_in): Similarly.
+	(substitute_and_fold_dom_walker): Remove initialization of
+	data member entries for calbacks.  Add substitute_and_fold_engine
+	member and initialize it.
+	(substitute_and_fold_dom_walker::before_dom_children0: Use the
+	member functions for get_value, replace_phi_args_in c
+	replace_uses_in, and fold_stmt calls.
+	(substitute_and_fold_engine::substitute_and_fold): Renamed from
+	substitute_and_fold.  Remove assert.   Update ctor call.
+
+	* tree-ssa-propagate.h (ssa_prop_visit_stmt_fn): Remove typedef.
+	(ssa_prop_visit_phi_fn): Likewise.
+	(class ssa_propagation_engine): New class to provide an interface
+	into ssa_propagate.
+	* tree-ssa-propagate.c (ssa_prop_visit_stmt): Remove file scoped
+	variable.
+	(ssa_prop_visit_phi): Likewise.
+	(ssa_propagation_engine::simulate_stmt): Moved into class.
+	Call visit_phi/visit_stmt from the class rather than via
+	file scoped static variables.
+	(ssa_propagation_engine::simulate_block): Moved into class.
+	(ssa_propagation_engine::process_ssa_edge_worklist): Similarly.
+	(ssa_propagation_engine::ssa_propagate): Similarly.  No longer
+	set file scoped statics for the visit_stmt/visit_phi callbacks.
+	* tree-complex.c (complex_propagate): New class derived from
+	ssa_propagation_engine.
+	(complex_propagate::visit_stmt): Renamed from complex_visit_stmt.
+	(complex_propagate::visit_phi): Renamed from complex_visit_phi.
+	(tree_lower_complex): Call ssa_propagate via the complex_propagate
+	class.
+	* tree-ssa-ccp.c: (ccp_propagate): New class derived from
+	ssa_propagation_engine.
+	(ccp_propagate::visit_phi): Renamed from ccp_visit_phi_node.
+	(ccp_propagate::visit_stmt): Renamed from ccp_visit_stmt.
+	(do_ssa_ccp): Call ssa_propagate from the ccp_propagate class.
+	* tree-ssa-copy.c (copy_prop): New class derived from
+	ssa_propagation_engine.
+	(copy_prop::visit_stmt): Renamed from copy_prop_visit_stmt.
+	(copy_prop::visit_phi): Renamed from copy_prop_visit_phi_node.
+	(execute_copy_prop): Call ssa_propagate from the copy_prop class.
+	* tree-vrp.c (vrp_prop): New class derived from ssa_propagation_engine.
+	(vrp_prop::visit_stmt): Renamed from vrp_visit_stmt.
+	(vrp_prop::visit_phi): Renamed from vrp_visit_phi_node.
+	(execute_vrp): Call ssa_propagate from the vrp_prop class.
+
+2017-11-01  Jakub Jelinek  <jakub@redhat.com>
+
+	PR rtl-optimization/82778
+	PR rtl-optimization/82597
+	* compare-elim.c (struct comparison): Add in_a_setter field.
+	(find_comparison_dom_walker::before_dom_children): Remove killed
+	bitmap and df_simulate_find_defs call, instead walk the defs.
+	Compute last_setter and initialize in_a_setter.  Merge definitions
+	with first initialization for a few variables.
+	(try_validate_parallel): Use insn_invalid_p instead of
+	recog_memoized.  Return insn rather than just the pattern.
+	(try_merge_compare): Fix up comment.  Don't uselessly test if
+	in_a is a REG_P.  Use cmp->in_a_setter instead of walking UD
+	chains.
+	(execute_compare_elim_after_reload): Remove df_chain_add_problem
+	call.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_rtx_costs): Use
+	aarch64_hard_regno_nregs to get the number of registers
+	in a mode.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* config/aarch64/constraints.md (Upl): Rename to...
+	(Uaa): ...this.
+	* config/aarch64/aarch64.md
+	(*zero_extend<SHORT:mode><GPI:mode>2_aarch64, *addsi3_aarch64_uxtw):
+	Update accordingly.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_add_constant_internal)
+	(aarch64_add_constant, aarch64_add_sp, aarch64_sub_sp): Move
+	earlier in file.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_evpc_trn, aarch64_evpc_uzp)
+	(aarch64_evpc_zip, aarch64_evpc_ext, aarch64_evpc_rev)
+	(aarch64_evpc_dup): Generate rtl direcly, rather than using
+	named expanders.
+	(aarch64_expand_vec_perm_const_1): Explicitly check for permutes
+	of a single element.
+	* config/aarch64/iterators.md: Add a comment above the permute
+	unspecs to say that they are generated directly by
+	aarch64_expand_vec_perm_const.
+	* config/aarch64/aarch64-simd.md: Likewise the permute instructions.
+
+2017-11-01  Nathan Sidwell  <nathan@acm.org>
+
+	* tree-dump.c (dequeue_and_dump): Use HAS_DECL_ASSEMBLER_NAME_P.
+
+2017-11-01  Palmer Dabbelt  <palmer@dabbelt.com>
+
+	* doc/invoke.texi (RISC-V Options): Explicitly name the medlow
+	and medany code models, and describe what they do.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	Revert accidental duplicate:
+
+	* combine.c (can_change_dest_mode): Reject changes in
+	REGMODE_NATURAL_SIZE.
+
+2017-11-01  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	PR rtl-optimization/64682
+	PR rtl-optimization/69567
+	PR rtl-optimization/69737
+	PR rtl-optimization/82683
+	* combine.c (distribute_notes) <REG_DEAD>: If the new I2 sets the same
+	register mentioned in the note, drop the note, unless it came from I3,
+	in which case it should go to I3 again.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* tree-ssa-dse.c (normalize_ref): Check whether the ranges overlap
+	and return false if not.
+	(clear_bytes_written_by, live_bytes_read): Update accordingly.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* tree-ssa-alias.h (ranges_overlap_p): Return false if either
+	range is known to be empty.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* simplify-rtx.c (simplify_const_unary_operation): Use GET_MODE_NUNITS
+	and CONST_VECTOR_NUNITS instead of computing the number of units from
+	the byte sizes of the vector and element.
+	(simplify_binary_operation_1): Likewise.
+	(simplify_const_binary_operation): Likewise.
+	(simplify_ternary_operation): Likewise.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* var-tracking.c (INT_MEM_OFFSET): Replace with...
+	(int_mem_offset): ...this new function.
+	(var_mem_set, var_mem_delete_and_set, var_mem_delete)
+	(find_mem_expr_in_1pdv, dataflow_set_preserve_mem_locs)
+	(same_variable_part_p, use_type, add_stores, vt_get_decl_and_offset):
+	Update accordingly.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* lower-subreg.c (interesting_mode_p): New function.
+	(compute_costs, find_decomposable_subregs, decompose_register)
+	(simplify_subreg_concatn, can_decompose_p, resolve_simple_move)
+	(resolve_clobber, dump_choices): Use it.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* rtlhash.c (add_rtx): Use add_hwi for 'w' and add_int for 'i'.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* alias.c (find_base_value, find_base_term): Only process integer
+	truncations.  Check the precision rather than the size.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* machmode.h (is_narrower_int_mode): New function
+	* optabs.c (expand_float, expand_fix): Use it.
+	* dwarf2out.c (rotate_loc_descriptor): Likewise.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* rtl.h (narrower_subreg_mode): New function.
+	* ira-color.c (update_costs_from_allocno): Use it.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* optabs-query.h (convert_optab_p): New function, split out from...
+	(convert_optab_handler): ...here.
+	(widening_optab_handler): Delete.
+	(find_widening_optab_handler): Remove permit_non_widening parameter.
+	(find_widening_optab_handler_and_mode): Likewise.  Provide an
+	override that operates on mode class wrappers.
+	* optabs-query.c (widening_optab_handler): Delete.
+	(find_widening_optab_handler_and_mode): Remove permit_non_widening
+	parameter.  Assert that the two modes are the same class and that
+	the "from" mode is narrower than the "to" mode.  Use
+	convert_optab_handler instead of widening_optab_handler.
+	* expmed.c (expmed_mult_highpart_optab): Use convert_optab_handler
+	instead of widening_optab_handler.
+	* expr.c (expand_expr_real_2): Update calls to
+	find_widening_optab_handler.
+	* optabs.c (expand_widen_pattern_expr): Likewise.
+	(expand_binop_directly): Take the insn_code as a parameter.
+	(expand_binop): Only call find_widening_optab_handler for
+	conversion optabs; use optab_handler otherwise.  Update calls
+	to find_widening_optab_handler and expand_binop_directly.
+	Use convert_optab_handler instead of widening_optab_handler.
+	* tree-ssa-math-opts.c (convert_mult_to_widen): Update calls to
+	find_widening_optab_handler and use scalar_mode rather than
+	machine_mode.
+	(convert_plusminus_to_widen): Likewise.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* machmode.h (fixed_size_mode): New class.
+	* rtl.h (get_pool_mode): Return fixed_size_mode.
+	* gengtype.c (main): Add fixed_size_mode.
+	* target.def (get_raw_result_mode): Return a fixed_size_mode.
+	(get_raw_arg_mode): Likewise.
+	* doc/tm.texi: Regenerate.
+	* targhooks.h (default_get_reg_raw_mode): Return a fixed_size_mode.
+	* targhooks.c (default_get_reg_raw_mode): Likewise.
+	* config/ia64/ia64.c (ia64_get_reg_raw_mode): Likewise.
+	* config/mips/mips.c (mips_get_reg_raw_mode): Likewise.
+	* config/msp430/msp430.c (msp430_get_raw_arg_mode): Likewise.
+	(msp430_get_raw_result_mode): Likewise.
+	* config/avr/avr-protos.h (regmask): Use as_a <fixed_side_mode>
+	* dbxout.c (dbxout_parms): Require fixed-size modes.
+	* expr.c (copy_blkmode_from_reg, copy_blkmode_to_reg): Likewise.
+	* gimple-ssa-store-merging.c (encode_tree_to_bitpos): Likewise.
+	* omp-low.c (lower_oacc_reductions): Likewise.
+	* simplify-rtx.c (simplify_immed_subreg): Take fixed_size_modes.
+	(simplify_subreg): Update accordingly.
+	* varasm.c (constant_descriptor_rtx::mode): Change to fixed_size_mode.
+	(force_const_mem): Update accordingly.  Return NULL_RTX for modes
+	that aren't fixed-size.
+	(get_pool_mode): Return a fixed_size_mode.
+	(output_constant_pool_2): Take a fixed_size_mode.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* doc/rtl.texi (vec_series): Document.
+	(const): Say that the operand can be a vec_series.
+	* rtl.def (VEC_SERIES): New rtx code.
+	* rtl.h (const_vec_series_p_1): Declare.
+	(const_vec_series_p): New function.
+	* emit-rtl.h (gen_const_vec_series): Declare.
+	(gen_vec_series): Likewise.
+	* emit-rtl.c (const_vec_series_p_1, gen_const_vec_series)
+	(gen_vec_series): Likewise.
+	* optabs.c (expand_mult_highpart): Use gen_const_vec_series.
+	* simplify-rtx.c (simplify_unary_operation): Handle negations
+	of vector series.
+	(simplify_binary_operation_series): New function.
+	(simplify_binary_operation_1): Use it.  Handle VEC_SERIES.
+	(test_vector_ops_series): New function.
+	(test_vector_ops): Call it.
+	* config/powerpcspe/altivec.md (altivec_lvsl): Use
+	gen_const_vec_series.
+	(altivec_lvsr): Likewise.
+	* config/rs6000/altivec.md (altivec_lvsl, altivec_lvsr): Likewise.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* doc/rtl.texi (const): Update description of address constants.
+	Say that vector constants are allowed too.
+	* common.md (E, F): Use CONSTANT_P instead of checking for
+	CONST_VECTOR.
+	* emit-rtl.c (gen_lowpart_common): Use const_vec_p instead of
+	checking for CONST_VECTOR.
+	* expmed.c (make_tree): Use build_vector_from_val for a CONST
+	VEC_DUPLICATE.
+	* expr.c (expand_expr_real_2): Check for vector modes instead
+	of checking for CONST_VECTOR.
+	* rtl.h (const_vec_p): New function.
+	(const_vec_duplicate_p): Check for a CONST VEC_DUPLICATE.
+	(unwrap_const_vec_duplicate): Handle them here too.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    David Malcolm  <dmalcolm@redhat.com>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* rtl.h (vec_duplicate_p): New function.
+	* selftest-rtl.c (assert_rtx_eq_at): New function.
+	* selftest-rtl.h (ASSERT_RTX_EQ): New macro.
+	(assert_rtx_eq_at): Declare.
+	* selftest.h (selftest::simplify_rtx_c_tests): Declare.
+	* selftest-run-tests.c (selftest::run_tests): Call it.
+	* simplify-rtx.c: Include selftest.h and selftest-rtl.h.
+	(simplify_unary_operation_1): Recursively handle vector duplicates.
+	(simplify_binary_operation_1): Likewise.  Handle VEC_SELECTs of
+	vector duplicates.
+	(simplify_subreg): Handle subregs of vector duplicates.
+	(make_test_reg, test_vector_ops_duplicate, test_vector_ops)
+	(selftest::simplify_rtx_c_tests): New functions.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* emit-rtl.h (gen_const_vec_duplicate): Declare.
+	(gen_vec_duplicate): Likewise.
+	* emit-rtl.c (gen_const_vec_duplicate_1): New function, split
+	out from...
+	(gen_const_vector): ...here.
+	(gen_const_vec_duplicate, gen_vec_duplicate): New functions.
+	(gen_rtx_CONST_VECTOR): Use gen_const_vec_duplicate for constants
+	whose elements are all equal.
+	* optabs.c (expand_vector_broadcast): Use gen_const_vec_duplicate.
+	* simplify-rtx.c (simplify_const_unary_operation): Likewise.
+	(simplify_relational_operation): Likewise.
+	* config/aarch64/aarch64.c (aarch64_simd_gen_const_vector_dup):
+	Likewise.
+	(aarch64_simd_dup_constant): Use gen_vec_duplicate.
+	(aarch64_expand_vector_init): Likewise.
+	* config/arm/arm.c (neon_vdup_constant): Likewise.
+	(neon_expand_vector_init): Likewise.
+	(arm_expand_vec_perm): Use gen_const_vec_duplicate.
+	(arm_block_set_unaligned_vect): Likewise.
+	(arm_block_set_aligned_vect): Likewise.
+	* config/arm/neon.md (neon_copysignf<mode>): Likewise.
+	* config/i386/i386.c (ix86_expand_vec_perm): Likewise.
+	(expand_vec_perm_even_odd_pack): Likewise.
+	(ix86_vector_duplicate_value): Use gen_vec_duplicate.
+	* config/i386/sse.md (one_cmpl<mode>2): Use CONSTM1_RTX.
+	* config/ia64/ia64.c (ia64_expand_vecint_compare): Use
+	gen_const_vec_duplicate.
+	* config/ia64/vect.md (addv2sf3, subv2sf3): Use CONST1_RTX.
+	* config/mips/mips.c (mips_gen_const_int_vector): Use
+	gen_const_vec_duplicate.
+	(mips_expand_vector_init): Use CONST0_RTX.
+	* config/powerpcspe/altivec.md (abs<mode>2, nabs<mode>2): Likewise.
+	(define_split): Use gen_const_vec_duplicate.
+	* config/rs6000/altivec.md (abs<mode>2, nabs<mode>2): Use CONST0_RTX.
+	(define_split): Use gen_const_vec_duplicate.
+	* config/s390/vx-builtins.md (vec_genmask<mode>): Likewise.
+	(vec_ctd_s64, vec_ctd_u64, vec_ctsl, vec_ctul): Likewise.
+	* config/spu/spu.c (spu_const): Likewise.
+
+2017-11-01  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* combine.c (can_change_dest_mode): Reject changes in
+	REGMODE_NATURAL_SIZE.
+
+2017-10-31  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* configure.ac (--enable-libssp): New.
+	(gcc_cv_libc_provides_ssp): Check for explicit setting before
+	trying to determine target-specific default.  Adjust indentation.
+	* configure: Regenerated.
+	* doc/install.texi (Configuration): Expand --disable-libssp
+	documentation.
+
+2017-10-31  Daniel Santos  <daniel.santos@pobox.com>
+
+	config/i386/i386.c (ix86_expand_epilogue): Correct stack
+	calculation.
+
+2017-10-31  Martin Jambor  <mjambor@suse.cz>
+
+	PR c++/81702
+	* gimple-fold.c (gimple_get_virt_method_for_vtable): Remove assert.
+
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* auto-profile.c (autofdo_source_profile::read): Use
+	UNKNOWN_LOCATION rather than 0.
+	* diagnostic-core.h (warning_at_rich_loc): Rename to...
+	(warning_at): ...this overload.
+	(warning_at_rich_loc_n): Rename to...
+	(warning_n): ...this overload.
+	(error_at_rich_loc): Rename to...
+	(error_at): ...this overload.
+	(pedwarn_at_rich_loc): Rename to...
+	(pedwarn): ...this overload.
+	(permerror_at_rich_loc): Rename to...
+	(permerror): ...this overload.
+	(inform_at_rich_loc): Rename to...
+	(inform): ...this overload.
+	* diagnostic.c: (diagnostic_n_impl): Delete location_t-based decl.
+	(diagnostic_n_impl_richloc): Rename to...
+	(diagnostic_n_impl): ...this rich_location *-based decl.
+	(inform_at_rich_loc): Rename to...
+	(inform): ...this, and add an assertion.
+	(inform_n): Update for removal of location_t-based diagnostic_n_impl.
+	(warning_at_rich_loc): Rename to...
+	(warning_at): ...this, and add an assertion.
+	(warning_at_rich_loc_n): Rename to...
+	(warning_n): ...this, and add an assertion.
+	(warning_n): Update location_t-based implementation for removal of
+	location_t-based diagnostic_n_impl.
+	(pedwarn_at_rich_loc): Rename to...
+	(pedwarn): ...this, and add an assertion.
+	(permerror_at_rich_loc): Rename to...
+	(permerror): ...this, and add an assertion.
+	(error_n): Update for removal of location_t-based diagnostic_n_impl.
+	(error_at_rich_loc): Rename to...
+	(error_at): ...this, and add an assertion.
+	* gcc.c (do_spec_1): Use UNKNOWN_LOCATION rather than 0.
+	(driver::do_spec_on_infiles): Likewise.
+	* substring-locations.c (format_warning_va): Update for renaming
+	of inform_at_rich_loc.
+
+2017-10-31  Michael Meissner  <meissner@linux.vnet.ibm.com>
+
+	* builtins.def (DEF_FLOATN_BUILTIN): Change most _Float<N> and
+	_Float<N>X built-in functions so that the variant without the
+	"__builtin_" prefix is only enabled for the GNU C and Objective C
+	languages when they are in non-strict ANSI/ISO mode.
+	(DEF_EXT_LIB_FLOATN_NX_BUILTINS): Likewise.
+	* target.def (floatn_builtin_p): Add a target hook to control
+	whether _Float<N> and _Float<N>X built-in functions without the
+	"__builtin_" prefix are enabled, and return true for C and
+	Objective C in the default hook.  Include langhooks.h in
+	targhooks.c.
+	* targhooks.h (default_floatn_builtin_p): Likewise.
+	* targhooks.c (default_floatn_builtin_p): Likewise.
+	* doc/tm.texi.in (TARGET_FLOATN_BUILTIN_P): Document the
+	floatn_builtin_p target hook.
+	* doc/tm.texi (TARGET_FLOATN_BUILTIN_P): Likewise.
+
+2017-10-31  Matthew Fortune  <matthew.fortune@imgtec.com>
+            Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR rtl-optimization/81803
+	* lra-constraints.c (curr_insn_transform): Also reload the whole
+	register for a strict subreg no wider than a word if this is for
+	a WORD_REGISTER_OPERATIONS target.
+
+2017-10-31  Jason Merrill  <jason@redhat.com>
+
+	* gdbinit.in: Skip over inlines from timevar.h.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* doc/gcov.texi: Document new option.
+	* gcov.c (print_usage): Likewise print it.
+	(process_args): Support the argument.
+	(format_count): New function.
+	(format_gcov): Use the function.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* gcov.c (struct name_map): do not use typedef.
+	Define operator== and operator<.
+	(name_search): Remove.
+	(name_sort): Remove.
+	(main): Do not allocate names.
+	(process_file): Add vertical space.
+	(generate_results): Use std::find.
+	(release_structures): Do not release memory.
+	(find_source): Use std::find.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* gcov.c (struct line_info): Remove it's typedef.
+	(line_info::line_info): Add proper ctor.
+	(line_info::has_block): Do not use a typedef.
+	(struct source_info): Do not use typedef.
+	(circuit): Likewise.
+	(get_cycles_count): Likewise.
+	(output_intermediate_file): Iterate via vector iterator.
+	(add_line_counts): Use std::vector methods.
+	(accumulate_line_counts): Likewise.
+	(output_lines): Likewise.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* gcov.c (struct source_info): Remove typedef.
+	(source_info::source_info): Add proper ctor.
+	(accumulate_line_counts): Use struct, not it's typedef.
+	(output_gcov_file): Likewise.
+	(output_lines): Likewise.
+	(main): Do not allocate an array.
+	(output_intermediate_file): Use size of vector container.
+	(process_file): Resize the vector.
+	(generate_results): Do not preallocate, use newly added vector
+	lines.
+	(release_structures): Do not release sources.
+	(find_source): Use vector methods.
+	(add_line_counts): Do not use typedef.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* doc/gcov.texi: Document that.
+	* gcov.c (add_line_counts): Mark lines with a non-executed
+	statement.
+	(output_line_beginning): Handle such lines.
+	(output_lines): Pass new argument.
+	(output_intermediate_file): Print it in intermediate format.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* color-macros.h: New file.
+	* diagnostic-color.c: Factor out color related to macros to
+	color-macros.h.
+	* doc/gcov.texi: Document -k option.
+	* gcov.c (INCLUDE_STRING): Include string.h.
+	(print_usage): Add -k option.
+	(process_args): Parse it.
+	(pad_count_string): New function.
+	(output_line_beginning): Likewise.
+	(DEFAULT_LINE_START): New macro.
+	(output_lines): Support color output.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	PR gcov-profile/82633
+	* doc/gcov.texi: Document -fkeep-{static,inline}-functions and
+	their interaction with GCOV infrastructure.
+	* configure.ac: Add -fkeep-{inline,static}-functions to
+	coverage_flags.
+	* configure: Regenerate.
+
+2017-10-31  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82772
+	* config/alpha/sync.md (fetchop_constr) <and>: Change to "rINM".
+
+2017-10-31  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	PR target/82674
+	* config/rs6000/rs6000.md (allocate_stack): Force update interval
+	into a register if it does not fit into an immediate offset field.
+
+2017-10-31  Olivier Hainque  <hainque@adacore.com>
+
+        * gcc/Makefile.in (FLAGS_TO_PASS): Pass libsubdir as well.
+
+2017-10-31  Julia Koval  <julia.koval@intel.com>
+
+	* config.gcc: Add gfniintrin.h.
+	* config/i386/gfniintrin.h: New.
+	* config/i386/i386-builtin-types.def
+	(__builtin_ia32_vgf2p8affineinvqb_v64qi,
+	__builtin_ia32_vgf2p8affineinvqb_v64qi_mask,
+	__builtin_ia32_vgf2p8affineinvqb_v32qi,
+	__builtin_ia32_vgf2p8affineinvqb_v32qi_mask,
+	__builtin_ia32_vgf2p8affineinvqb_v16qi,
+	__builtin_ia32_vgf2p8affineinvqb_v16qi_mask): New builtins.
+	* config/i386/i386-builtin.def (V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI,
+	V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI,
+	V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI,
+	V64QI_FTYPE_V64QI_V64QI_INT): New types.
+	* config/i386/i386.c (ix86_expand_args_builtin): Handle new types.
+	* config/i386/immintrin.h: Include gfniintrin.h.
+	* config/i386/sse.md (vgf2p8affineinvqb_*) New pattern.
+
+2017-10-30  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc.c (HAVE_TARGET_EXECUTABLE_SUFFIX): Remove old kludge.
+
+2017-10-30  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/arm/arm.md (ashldi3): Remove shift by 1 expansion.
+	(arm_ashldi3_1bit): Remove pattern.
+	(ashrdi3): Remove shift by 1 expansion.
+	(arm_ashrdi3_1bit): Remove pattern.
+	(lshrdi3): Remove shift by 1 expansion.
+	(arm_lshrdi3_1bit): Remove pattern.
+	* config/arm/arm.c (arm_rtx_costs_internal): Slightly increase
+	cost of ashldi3 by 1.
+	* config/arm/neon.md (ashldi3_neon): Remove shift by 1 expansion.
+	(<shift>di3_neon): Likewise.
+
+2017-10-30  Dominik Infuehr  <dominik.infuehr@theobroma-systems.com>
+
+	* config/aarch64/aarch64-simd.md (*aarch64_simd_mov): Rename
+	both identically named patterns to (*aarch64_simd_mov<VD:mode>)
+	and (*aarch64_simd_mov<VQ:mode>).
+	(*aarch64_simd_mov<VD:mode>): Change type attribute to match
+	pattern alternative.
+	(*aarch64_simd_mov<VQ:mode>): Re-order and change type
+	attributes to match pattern alternative.
+
+2017-10-30  Steven Munroe  <munroesj@gcc.gnu.org>
+
+	* config.gcc (powerpc*-*-*): Add emmintrin.h.
+	* config/rs6000/emmintrin.h: New file.
+	* config/rs6000/x86intrin.h [__ALTIVEC__]: Include emmintrin.h.
+
+2017-10-30  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/arm/vfp.md (movdi_vfp): Merge changes from movdi_vfp_cortexa8.
+	* (movdi_vfp_cortexa8): Remove pattern.
+
+2017-10-30  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* doc/install.texi (Specific, alpha*-*-*): Remove DEC OSF/1
+	etc. reference.
+	(Specific, alpha*-dec-osf5.1): Remove.
+	(Specific, mips-sgi-irix5): Remove.
+	(Specific, mips-sgi-irix6): Remove.
+
+2017-10-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/22141
+	* gimple-ssa-store-merging.c (merged_store_group::apply_stores): Fix
+	arguments to clear_bit_region_be.
+
+2017-10-30  Jim Wilson  <wilson@tuliptree.org>
+
+	* gimplify.c: Include memmodel.h.
+
+2017-10-30  Martin Jambor  <mjambor@suse.cz>
+
+	* omp-grid.c (grid_attempt_target_gridification): Also insert a
+	condition whether loop should be executed at all.
+
+2017-10-30  Will Schmidt  <will_schmidt@vnet.ibm.com>
+
+	* config/rs6000/rs6000.c (rs6000_gimple_fold_builtin): Add support for
+	gimple folding of vec_madd() intrinsics.
+	* config/rs6000/altivec.md (mulv8hi3): Rename altivec_vmladduhm to
+	fmav8hi4.  (altivec_vmladduhm): Rename to fmav8hi4.
+	* config/rs6000/rs6000-builtin.def: Rename vmladduhm to fmav8hi4
+
+2017-10-30  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82762
+	Revert
+	2017-10-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82129
+	Revert
+	2017-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/81181
+	* tree-ssa-pre.c (compute_antic_aux): Defer clean() to ...
+	(compute_antic): ... end of iteration here.
+
+2017-10-30  Joseph Myers  <joseph@codesourcery.com>
+
+	* doc/invoke.texi (C Dialect Options): Document -std=c17,
+	-std=iso9899:2017 and -std=gnu17.
+	* doc/standards.texi (C Language): Document C17 support.
+	* doc/cpp.texi (Overview): Mention -std=c17.
+	(Standard Predefined Macros): Document C11 and C17 values of
+	__STDC_VERSION__.  Do not refer to C99 support as incomplete.
+	* doc/extend.texi (Inline): Do not list individual options for
+	standards newer than C99.
+	* dwarf2out.c (highest_c_language, gen_compile_unit_die): Handle
+	"GNU C17".
+	* config/rl78/rl78.c (rl78_option_override): Handle "GNU C17"
+	language name.
+
+2017-10-30  Maxim Ostapenko  <m.ostapenko@samsung.com>
+
+	* asan.c (asan_finish_file): Align asan globals array by shadow
+	granularity.
+
+2017-10-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/22141
+	* gimple-ssa-store-merging.c: Include rtl.h and expr.h.
+	(struct store_immediate_info): Add bitregion_start and bitregion_end
+	fields.
+	(store_immediate_info::store_immediate_info): Add brs and bre
+	arguments and initialize bitregion_{start,end} from those.
+	(struct merged_store_group): Add bitregion_start, bitregion_end,
+	align_base and mask fields.  Drop unnecessary struct keyword from
+	struct store_immediate_info.  Add do_merge method.
+	(clear_bit_region_be): Use memset instead of loop storing zeros.
+	(merged_store_group::do_merge): New method.
+	(merged_store_group::merge_into): Use do_merge.  Allow gaps in between
+	stores as long as the surrounding bitregions have no gaps.
+	(merged_store_group::merge_overlapping): Use do_merge.
+	(merged_store_group::apply_stores): Test that bitregion_{start,end}
+	is byte aligned, rather than requiring that start and width are
+	byte aligned.  Drop unnecessary struct keyword from
+	struct store_immediate_info.  Allocate and populate also mask array.
+	Make start of the arrays relative to bitregion_start rather than
+	start and size them according to bitregion_{end,start} difference.
+	(struct imm_store_chain_info): Drop unnecessary struct keyword from
+	struct store_immediate_info.
+	(pass_store_merging::gate): Punt if BITS_PER_UNIT or CHAR_BIT is not 8.
+	(pass_store_merging::terminate_all_aliasing_chains): Drop unnecessary
+	struct keyword from struct store_immediate_info.
+	(imm_store_chain_info::coalesce_immediate_stores): Allow gaps in
+	between stores as long as the surrounding bitregions have no gaps.
+	Formatting fixes.
+	(struct split_store): Add orig non-static data member.
+	(split_store::split_store): Initialize orig to false.
+	(find_constituent_stmts): Return store_immediate_info *, non-NULL
+	if there is exactly a single original stmt.  Change stmts argument
+	to pointer from reference, if NULL, don't push anything to it.  Add
+	first argument, use it to optimize skipping over orig stmts that
+	are known to be before bitpos already.  Simplify.
+	(split_group): Return unsigned int count how many stores are or
+	would be needed rather than a bool.  Add allow_unaligned argument.
+	Change split_stores argument from reference to pointer, if NULL,
+	only do a dry run computing how many stores would be produced.
+	Rewritten algorithm to use both alignment and misalign if
+	!allow_unaligned and handle bitfield stores with gaps.
+	(imm_store_chain_info::output_merged_store): Set start_byte_pos
+	from bitregion_start instead of start.  Compute allow_unaligned
+	here, if true, do 2 split_group dry runs to compute which one
+	produces fewer stores and prefer aligned if equal.  Punt if
+	new count is bigger or equal than original before emitting any
+	statements, rather than during that.  Remove no longer needed
+	new_ssa_names tracking.  Replace num_stmts with
+	split_stores.length ().  Use 32-bit stack allocated entries
+	in split_stores auto_vec.  Try to reuse original store lhs/rhs1
+	if possible.  Handle bitfields with gaps.
+	(pass_store_merging::execute): Ignore bitsize == 0 stores.
+	Compute bitregion_{start,end} for the stores and construct
+	store_immediate_info with that.  Formatting fixes.
+
+2017-10-30  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82725
+	* config/i386/i386.c (legitimate_pic_address_disp_p): Allow
+	UNSPEC_DTPOFF and UNSPEC_NTPOFF with SImode immediate offset.
+
+2017-10-29  Jim Wilson  <wilson@tuliptree.org>
+
+	* gimplify.c: Include tm_p.h.
+
+	* common.opt (gcoff): Re-add as ignored option.
+	(gcoff1, gcoff2, gcoff3): Likewise.
+
+	* Makefile.in (OBJS): Delete sdbout.o.
+	(GTFILES): Delete $(srcdir)/sdbout.c.
+	* debug.h: Delete sdb_debug_hooks.
+	* final.c: Delete sdbout.h include.
+	(final_scan_insn): Delete SDB_DEBUG check.
+	(rest_of_clean_state): Likewise.
+	* output.h: Delete sdb_begin_function_line.
+	* sdbout.c: Delete.
+	* sdbout.h: Delete.
+	* toplev.c: Delete sdbout.h include.
+	(process_options): Delete SDB_DEBUG check.
+	* tree-core.h (tree_type_common): Delete pointer field of
+	tree_type_symtab.
+	* tree.c (copy_node): Clear TYPE_SYMTAB_DIE instead of
+	TYPE_SYMTAB_POINTER.
+	* tree.h (TYPE_SYMTAB_POINTER): Delete.
+	(TYPE_SYMTAB_IS_POINTER): Delete.
+	(TYPE_SYMTAB_IS_DIE): Renumber.
+	* xcoffout.c: Refer to former sdbout.c file.
+	(xcoffout_begin_prologue): Use past tense for sdbout.c reference.
+
+	* doc/install.texi (--with-stabs): Delete COFF and ECOFF info.
+	* doc/invoke.texi (SEEALSO): Delete adb and sdb references.
+	(Debugging Options): Delete -gcoff.
+	(-gstabs): Delete SDB reference.
+	(-gcoff): Delete.
+	(-gcoff@var{level}): Delete.
+	* doc/passes.texi (Debugging information output): Delete SDB and
+	sdbout.c references.
+	* doc/tm.texi: Regenerate.
+	* doc/tm.texi.in (DWARF_CIE_DATA_ALIGNMENT): Delete SDB from xref.
+	(SDB and DWARF): Change node name to DWARF and delete SDB and COFF
+	references.
+	(DEBUGGER_AUTO_OFFSET): Delete COFF and SDB references.
+	(PREFERRED_DEBUGGING_TYPE): Delete SDB_DEBUG and -gcoff references.
+	(SDB_DEBUGGING_INFO): Delete.
+	(PUT_SDB_@dots{}, SDB_DELIM, SDB_ALLOW_UNKNOWN_REFERENCES)
+	SDB_ALLOW_FORWARD_REFERENCES, SDB_OUTPUT_SOURCE_LINE): Delete.
+	* target.def (output_source_filename): Delete COFF reference.
+
+	* common.opt (gcoff): Delete.
+	(gxcoff+): Update Negative chain.
+	* defaults.h: Delete all references to SDB_DEBUGGING_INFO and
+	SDB_DEBUG.
+	* dwarf2out.c (gen_array_type_die): Change SDB to debuggers.
+	* flag-types.h (enum debug_info_type): Delete SDB_DEBUG.
+	* function.c (number_blocks): Delete SDB_DEBUGGING_INFO, SDB_DEBUG,
+	and SDB references.
+	(expand_function_start): Change sdb reference to past tense.
+	(expand_function_end): Change sdb reference to past tense.
+	* gcc.c (cpp_unique_options): Delete gcoff3 reference.
+	* opts.c (debug_type_names): Delete coff entry.
+	(common_handle_option): Delete OPT_gcoff case.
+	* system.h (SDB_DEBUG, SDB_DEBUGGING_INFO): Poison.
+
+	* config/dbxcoff.h (PREFERRED_DEBUGGING_TYPE): Set to DBX_DEBUG.
+	* config/cris/cris.h: Delete SDB reference in comment.
+	* config/i386/cygming.h: Don't define SDB_DEBUGGING_INFO.
+	(ASM_DECLARE_FUNCTION_NAME): Delete SDB reference from comment.
+	* config/i386/gas.h: Don't define SDB_DEBUGGING_INFO.
+	* config/i386/i386.c (svr4_dbx_register_map): Change SDB references
+	to past tense.
+	(ix86_expand_prologue): Likewise.
+	* config/i386/winnt.c (i386_pe_start_function): Don't check SDB_DEBUG.
+	* config/ia64/ia64.h: Likewise.
+	* config/m68k/m68kelf.h (DBX_REGISTER_NUMBER): Delete SDB reference.
+	* config/mips/mips.h (SUBTARGET_ASM_DEBUGGING_SPEC): Delete gcoff*
+	support.
+	* config/mmix/mmix.h: Likewise.
+	* config/nds32/nds32.c: Likewise.
+	* config/stormy/storym16.h: Likewise.
+	* config/visium/visium.h: Likewise.
+	* config/vx-common.h (SDB_DEBUGGING_INFO): Delete undef.
+
+2017-10-28  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* config/nios2/nios2.h (FRAME_GROWS_DOWNWARD): Define to 1.
+	* config/nios2/nios2.c (nios2_initial_elimination_offset):  Make
+	FRAME_POINTER_REGNUM point at high end of local var area.
+
+2017-10-27  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* bb-reorder.c (find_traces_1_round): Fix off-by-one index.
+	Move comment around.  Do not reset best_edge for a copiable
+	destination if the copy would cause a partition change.
+	(better_edge_p): Remove redundant check.
+
+2017-10-27  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386-protos.h (ix86_fp_compare_mode): Remove prototype.
+
+2017-10-27  Michael Meissner  <meissner@linux.vnet.ibm.com>
+
+	* builtins.c (CASE_MATHFN_FLOATN): New helper macro to add cases
+	for math functions that have _Float<N> and _Float<N>X variants.
+	(mathfn_built_in_2): Add support for math functions that have
+	_Float<N> and _Float<N>X variants.
+	(DEF_INTERNAL_FLT_FLOATN_FN): New helper macro.
+	(expand_builtin_mathfn_ternary): Add support for fma with
+	_Float<N> and _Float<N>X variants.
+	(expand_builtin): Likewise.
+	(fold_builtin_3): Likewise.
+	* builtins.def (DEF_EXT_LIB_FLOATN_NX_BUILTINS): New macro to
+	create math function _Float<N> and _Float<N>X variants as external
+	library builtins.
+	(BUILT_IN_COPYSIGN _Float<N> and _Float<N>X variants) Use
+	DEF_EXT_LIB_FLOATN_NX_BUILTINS to make built-in functions using
+	the __builtin_ prefix and if not strict ansi, without the prefix.
+	(BUILT_IN_FABS _Float<N> and _Float<N>X variants): Likewise.
+	(BUILT_IN_FMA _Float<N> and _Float<N>X variants): Likewise.
+	(BUILT_IN_FMAX _Float<N> and _Float<N>X variants): Likewise.
+	(BUILT_IN_FMIN _Float<N> and _Float<N>X variants): Likewise.
+	(BUILT_IN_NAN _Float<N> and _Float<N>X variants): Likewise.
+	(BUILT_IN_SQRT _Float<N> and _Float<N>X variants): Likewise.
+	* builtin-types.def (BT_FN_FLOAT16_FLOAT16_FLOAT16_FLOAT16): New
+	function signatures for fma _Float<N> and _Float<N>X variants.
+	(BT_FN_FLOAT32_FLOAT32_FLOAT32_FLOAT32): Likewise.
+	(BT_FN_FLOAT64_FLOAT64_FLOAT64_FLOAT64): Likewise.
+	(BT_FN_FLOAT128_FLOAT128_FLOAT128_FLOAT128): Likewise.
+	(BT_FN_FLOAT32X_FLOAT32X_FLOAT32X_FLOAT32X): Likewise.
+	(BT_FN_FLOAT64X_FLOAT64X_FLOAT64X_FLOAT64X): Likewise.
+	(BT_FN_FLOAT128X_FLOAT128X_FLOAT128X_FLOAT128X): Likewise.
+	* gencfn-macros.c (print_case_cfn): Add support for math functions
+	that have _Float<N> and _Float<N>X variants.
+	(print_define_operator_list): Likewise.
+	(fltfn_suffixes): Likewise.
+	(main): Likewise.
+	* internal-fn.def (DEF_INTERNAL_FLT_FLOATN_FN): New helper macro
+	for math functions that have _Float<N> and _Float<N>X variants.
+	(SQRT): Add support for sqrt, copysign, fmin and fmax _Float<N>
+	and _Float<N>X variants.
+	(COPYSIGN): Likewise.
+	(FMIN): Likewise.
+	(FMAX): Likewise.
+	* fold-const.c (tree_call_nonnegative_warnv_p): Add support for
+	copysign, fma, fmax, fmin, and sqrt _Float<N> and _Float<N>X
+	variants.
+	(integer_valued_read_call_p): Likewise.
+	* fold-const-call.c (fold_const_call_ss): Likewise.
+	(fold_const_call_sss): Add support for copysign, fmin, and fmax
+	_Float<N> and _Float<N>X variants.
+	(fold_const_call_ssss): Add support for fma _Float<N> and
+	_Float<N>X variants.
+	* gimple-ssa-backprop.c (backprop::process_builtin_call_use): Add
+	support for copysign and fma _Float<N> and _Float<N>X variants.
+	(backprop::process_builtin_call_use): Likewise.
+	* tree-call-cdce.c (can_test_argument_range); Add support for
+	sqrt _Float<N> and _Float<N>X variants.
+	(edom_only_function): Likewise.
+	(get_no_error_domain): Likewise.
+	* tree-ssa-math-opts.c (internal_fn_reciprocal): Likewise.
+	* tree-ssa-reassoc.c (attempt_builtin_copysign): Add support for
+	copysign _Float<N> and _Float<N>X variants.
+	* config/rs6000/rs6000-builtin.def (SQRTF128): Delete, this is now
+	handled by machine independent code.
+	(FMAF128): Likewise.
+	* doc/cpp.texi (Common Predefined Macros): Document defining
+	__FP_FAST_FMAF<N> and __FP_FAST_FMAF<N>X if the backend supports
+	fma _Float<N> and _Float<N>X variants.
+
+2017-10-27  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82692
+	* config/i386/i386-modes.def (CCFPU): Remove definition.
+	* config/i386/i386.c (put_condition_mode): Remove CCFPU mode handling.
+	(ix86_cc_modes_compatible): Ditto.
+	(ix86_expand_carry_flag_compare): Ditto.
+	(ix86_expand_int_movcc): Ditto.
+	(ix86_expand_int_addcc): Ditto.
+	(ix86_reverse_condition): Ditto.
+	(ix86_unordered_fp_compare): Rename from ix86_fp_compare_mode.
+	Return true/false for unordered/ordered fp comparisons.
+	(ix86_cc_mode): Always return CCFPmode for float mode comparisons.
+	(ix86_prepare_fp_compare_args): Update for rename.
+	(ix86_expand_fp_compare): Update for rename.  Generate unordered
+	compare RTXes wrapped with UNSPEC_NOTRAP unspec.
+	(ix86_expand_sse_compare_and_jump): Ditto.
+	* config/i386/predicates.md (fcmov_comparison_operator):
+	Remove CCFPU mode handling.
+	(ix86_comparison_operator): Ditto.
+	(ix86_carry_flag_operator): Ditto.
+	* config/i386/i386.md (UNSPEC_NOTRAP): New unspec.
+	(*cmpu<mode>_i387): Wrap compare RTX with UNSPEC_NOTRAP unspec.
+	(*cmpu<mode>_cc_i387): Ditto.
+	(FPCMP): Remove mode iterator.
+	(unord): Remove mode attribute.
+	(unord_subst): New define_subst transformation
+	(unord): New define_subst attribute.
+	(unordered): Ditto.
+	(*cmpi<unord><MODEF:mode>): Rewrite using unord_subst transformation.
+	(*cmpi<unord>xf_i387): Ditto.
+	* config/i386/sse.md (<sse>_<unord>comi<round_saeonly_name>): Merge
+	from <sse>_comi<round_saeonly_name> and <sse>_ucomi<round_saeonly_name>
+	using unord_subst transformation.
+	* config/i386/subst.md (SUBST_A): Remove CCFP and CCFPU modes.
+	(round_saeonly): Also handle CCFP mode.
+	* reg-stack.c (subst_stack_regs_pat): Handle UNSPEC_NOTRAP unspec.
+	Remove UNSPEC_SAHF unspec handling.
+
+2017-10-27  Jan Hubicka  <hubicka@ucw.cz>
+
+	* x86-tune.def (X86_TUNE_INTER_UNIT_MOVES_TO_VEC): Disable for Zen.
+
+2017-10-27  Jeff Law  <law@redhat.com>
+
+	* gimple-ssa-sprintf.c: Include domwalk.h.
+	(class sprintf_dom_walker): New class, derived from dom_walker.
+	(sprintf_dom_walker::before_dom_children): New function.
+	(struct call_info): Moved into sprintf_dom_walker class
+	(compute_formath_length, handle_gimple_call): Likewise.
+	(sprintf_length::execute): Call the dominator walker rather
+	than walking the statements.
+
+	* tree-vrp.c (check_all_array_refs): Do not use wi->info to smuggle
+	gimple statement locations.
+	(check_array_bounds): Corresponding changes.  Get the statement's
+	location directly from wi->stmt.
+
+2017-10-27  Palmer Dabbelt  <palmer@dabbelt.com>
+
+	PR target/82717
+	* doc/invoke.texi (RISC-V) <-mabi>: Correct and improve.
+
+2017-10-27  Jan Hubicka  <hubicka@ucw.cz>
+
+	* config/i386/x86-tune.def (X86_TUNE_PARTIAL_REG_DEPENDENCY,
+	X86_TUNE_MOVX): Disable for Haswell and newer CPUs.
+
+2017-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82703
+	* config/i386/i386-protos.h (maybe_get_pool_constant): Removed.
+	* config/i386/i386.c (maybe_get_pool_constant): Removed.
+	(ix86_split_to_parts): Use avoid_constant_pool_reference instead of
+	maybe_get_pool_constant.
+	* config/i386/predicates.md (zero_extended_scalar_load_operand):
+	Likewise.
+
+2017-10-27  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* doc/install.texi (Specific, i?86-*-solaris2.10): Simplify gas
+	2.26 caveat.  Update gas and gld versions.
+	(Specific, *-*-solaris2*): Update binutils version.  Remove caveat
+	reference.
+
+2017-10-27  Prathamesh Kulkarni  <prathamesh.kulkarni@linaro.org>
+
+	* cgraph.h (set_malloc_flag): Declare.
+	* cgraph.c (set_malloc_flag_1): New function.
+	(set_malloc_flag): Likewise.
+	* ipa-fnsummary.h (ipa_call_summary): Add new field is_return_callee.
+	* ipa-fnsummary.c (ipa_call_summary::reset): Set is_return_callee to
+	false.
+	(read_ipa_call_summary): Add support for reading is_return_callee.
+	(write_ipa_call_summary): Stream is_return_callee.
+	* ipa-inline.c (ipa_inline): Remove call to ipa_free_fn_summary.
+	* ipa-pure-const.c: Add headers ssa.h, alloc-pool.h, symbol-summary.h,
+	ipa-prop.h, ipa-fnsummary.h.
+	(pure_const_names): Change to static.
+	(malloc_state_e): Define.
+	(malloc_state_names): Define.
+	(funct_state_d): Add field malloc_state.
+	(varying_state): Set malloc_state to STATE_MALLOC_BOTTOM.
+	(check_retval_uses): New function.
+	(malloc_candidate_p): Likewise.
+	(analyze_function): Add support for malloc attribute.
+	(pure_const_write_summary): Stream malloc_state.
+	(pure_const_read_summary): Add support for reading malloc_state.
+	(dump_malloc_lattice): New function.
+	(propagate_malloc): New function.
+	(warn_function_malloc): New function.
+	(ipa_pure_const::execute): Call propagate_malloc and
+	ipa_free_fn_summary.
+	(pass_local_pure_const::execute): Add support for malloc attribute.
+	* ssa-iterators.h (RETURN_FROM_IMM_USE_STMT): New macro.
+	* doc/invoke.texi: Document Wsuggest-attribute=malloc.
+
+2017-10-27  Martin Liska  <mliska@suse.cz>
+
+	PR gcov-profile/82457
+	* doc/invoke.texi: Document that one needs a non-strict ISO mode
+	for fork-like functions to be properly instrumented.
+
+2017-10-27  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/81659
+	* tree-eh.c (pass_lower_eh_dispatch::execute): Free dominator
+	info when we redirected EH.
+
+2017-10-26  Michael Collison  <michael.collison@arm.com>
+
+	* config/aarch64/aarch64.md(<optab>_trunc><vf><GPI:mode>2):
+	New pattern.
+	(<optab>_trunchf<GPI:mode>2: New pattern.
+	(<optab>_trunc<vgp><GPI:mode>2: New pattern.
+	* config/aarch64/iterators.md (wv): New mode attribute.
+	(vf, VF): New mode attributes.
+	(vgp, VGP): New mode attributes.
+	(s): Update attribute with SImode and DImode prefixes.
+
+2017-10-26  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* config/nios2/constraints.md ("S"): Match r0rel_constant_p too.
+	* config/nios2/nios2-protos.h (r0rel_constant_p): Declare.
+	* config/nios2/nios2.c: (nios2_r0rel_sec_regex): New.
+	(nios2_option_overide): Initialize it.  Don't allow R0-relative
+	addressing with PIC.
+	(nios2_rtx_costs): Handle r0rel_constant_p like gprel_constant_p.
+	(nios2_symbolic_constant_p): Likewise.
+	(nios2_legitimate_address_p): Likewise.
+	(nios2_r0rel_section_name_p): New.
+	(nios2_symbol_ref_in_r0rel_data_p): New.
+	(nios2_emit_move_sequence): Handle r0rel_constant_p.
+	(r0rel_constant_p): New.
+	(nios2_print_operand_address): Handle r0rel_constant_p.
+	(nios2_cdx_narrow_form_p): Likewise.
+	* config/nios2/nios2.opt (mr0rel-sec=): New option.
+	* doc/invoke.texi (Option Summary): Add -mr0rel-sec.
+	(Nios II Options): Document -mr0rel-sec.
+
+2017-10-26  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* config/nios2/nios2.c: Include xregex.h.
+	(nios2_gprel_sec_regex): New.
+	(nios2_option_overide): Initialize it.  Don't allow GP-relative
+	addressing with PIC.
+	(nios2_small_section_name_p): Check for regex match.
+	* config/nios2/nios2.opt (mgprel-sec=): New option.
+	* doc/invoke.texi (Option Summary): Add -mgprel-sec.
+	(Nios II Options): Document -mgprel-sec.
+
+2017-10-26  Jim Wilson  <wilson@tuliptree.org>
+
+	* doc/invoke.texi (-fdebug-prefix-map): Expand documentation.
+
+2017-10-26  Tom de Vries  <tom@codesourcery.com>
+
+	PR tree-optimization/82707
+	* gimple.c (gimple_copy): Fix unsharing of
+	GIMPLE_OMP_{SINGLE,TARGET,TEAMS}.
+
+2017-10-26  Olga Makhotina  <olga.makhotina@intel.com>
+
+	* config/i386/avx512fintrin.h (_mm512_cmpeq_pd_mask,
+	_mm512_cmple_pd_mask, _mm512_cmplt_pd_mask,
+	_mm512_cmpneq_pd_mask, _mm512_cmpnle_pd_mask,
+	_mm512_cmpnlt_pd_mask, _mm512_cmpord_pd_mask,
+	_mm512_cmpunord_pd_mask, _mm512_mask_cmpeq_pd_mask,
+	_mm512_mask_cmple_pd_mask, _mm512_mask_cmplt_pd_mask,
+	_mm512_mask_cmpneq_pd_mask, _mm512_mask_cmpnle_pd_mask,
+	_mm512_mask_cmpnlt_pd_mask, _mm512_mask_cmpord_pd_mask,
+	_mm512_mask_cmpunord_pd_mask, _mm512_cmpeq_ps_mask,
+	_mm512_cmple_ps_mask, _mm512_cmplt_ps_mask,
+	_mm512_cmpneq_ps_mask, _mm512_cmpnle_ps_mask,
+	_mm512_cmpnlt_ps_mask, _mm512_cmpord_ps_mask,
+	_mm512_cmpunord_ps_mask, _mm512_mask_cmpeq_ps_mask,
+	_mm512_mask_cmple_ps_mask, _mm512_mask_cmplt_ps_mask,
+	_mm512_mask_cmpneq_ps_mask, _mm512_mask_cmpnle_ps_mask,
+	_mm512_mask_cmpnlt_ps_mask, _mm512_mask_cmpord_ps_mask,
+	_mm512_mask_cmpunord_ps_mask): New intrinsics.
+
+2017-10-26  Michael Meissner  <meissner@linux.vnet.ibm.com>
+
+	* config/rs6000/aix.h (TARGET_IEEEQUAD_DEFAULT): Set long double
+	default to IBM.
+	* config/rs6000/darwin.h (TARGET_IEEEQUAD_DEFAULT): Likewise.
+	* config/rs6000/rs6000.opt (-mabi=ieeelongdouble): Move the
+	warning to rs6000.c.  Remove the Undocumented flag, since it has
+	been documented.
+	(-mabi=ibmlongdouble): Likewise.
+	* config/rs6000/rs6000.c (TARGET_IEEEQUAD_DEFAULT): If it is not
+	already set, set the default format for long double.
+	(rs6000_debug_reg_global): Print whether long double is IBM or
+	IEEE.
+	(rs6000_option_override_internal): Rework setting long double
+	format.  Only warn if the user is changing the long double default
+	and they did not use -Wno-psabi.
+	* doc/invoke.texi (PowerPC options): Update the documentation for
+	-mabi=ieeelongdouble and -mabi=ibmlongdouble.
+
+2017-10-26  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* rtl.h (wider_subreg_mode): New function.
+	* ira.h (ira_sort_regnos_for_alter_reg): Take a machine_mode *
+	rather than an unsigned int *.
+	* ira-color.c (regno_max_ref_width): Replace with...
+	(regno_max_ref_mode): ...this new variable.
+	(coalesced_pseudo_reg_slot_compare): Update accordingly.
+	Use wider_subreg_mode.
+	(ira_sort_regnos_for_alter_reg): Likewise.  Take a machine_mode *
+	rather than an unsigned int *.
+	* lra-constraints.c (uses_hard_regs_p): Use wider_subreg_mode.
+	(process_alt_operands): Likewise.
+	(invariant_p): Likewise.
+	* lra-spills.c (assign_mem_slot): Likewise.
+	(add_pseudo_to_slot): Likewise.
+	* lra.c (collect_non_operand_hard_regs): Likewise.
+	(add_regs_to_insn_regno_info): Likewise.
+	* reload1.c (regno_max_ref_width): Replace with...
+	(regno_max_ref_mode): ...this new variable.
+	(reload): Update accordingly.  Update call to
+	ira_sort_regnos_for_alter_reg.
+	(alter_reg): Update to use regno_max_ref_mode.  Call wider_subreg_mode.
+	(init_eliminable_invariants): Update to use regno_max_ref_mode.
+	(scan_paradoxical_subregs): Likewise.
+
+2017-10-26  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/aarch64/aarch64.h (EXIT_IGNORE_STACK): Set if alloca is used.
+	(aarch64_frame): Add emit_frame_chain boolean.
+	* config/aarch64/aarch64.c (aarch64_frame_pointer_required)
+	Move eh_return case to aarch64_layout_frame.
+	(aarch64_layout_frame): Initialize emit_frame_chain.
+	(aarch64_expand_prologue): Use emit_frame_chain.
+
+2017-10-26  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_layout_frame):
+	Ensure LR is always stored at the bottom of the callee-saves.
+	Remove rarely used frame layout which saves callee-saves at top of
+	frame, so the store of LR can be used as a valid probe in all cases.
+
+2017-10-26  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_legitimize_address_displacement):
+	Improve unaligned TImode/TFmode base/offset split.
+
+2017-10-26  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* caller-save.c (mark_referenced_regs):  Use read_modify_subreg_p.
+	* combine.c (find_single_use_1): Likewise.
+	(expand_field_assignment): Likewise.
+	(move_deaths): Likewise.
+	* lra-constraints.c (simplify_operand_subreg): Likewise.
+	(curr_insn_transform): Likewise.
+	* lra.c (collect_non_operand_hard_regs): Likewise.
+	(add_regs_to_insn_regno_info): Likewise.
+	* rtlanal.c (reg_referenced_p): Likewise.
+	(covers_regno_no_parallel_p): Likewise.
+
+2017-10-26  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* wide-int-print.cc (print_hex): Loop based on extract_uhwi.
+	Don't print any bits outside the precision of the value.
+	* wide-int.cc (test_printing): Add some new tests.
+
+2017-10-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* configure.ac (gcc_cv_as_ix86_xbrace_comment): Check if assembler
+	supports -xbrace_comment option.
+	* configure: Regenerate.
+	* config.in: Regenerate.
+	* config/i386/sol2.h (ASM_XBRACE_COMMENT_SPEC): Define.
+	(ASM_CPU_SPEC): Use it.
+
+2017-10-26  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* target.def (static_rtx_alignment): New hook.
+	* targhooks.h (default_static_rtx_alignment): Declare.
+	* targhooks.c (default_static_rtx_alignment): New function.
+	* doc/tm.texi.in (TARGET_STATIC_RTX_ALIGNMENT): New hook.
+	* doc/tm.texi: Regenerate.
+	* varasm.c (force_const_mem): Use targetm.static_rtx_alignment
+	instead of targetm.constant_alignment.  Remove call to
+	set_mem_attributes.
+	* config/cris/cris.c (TARGET_STATIC_RTX_ALIGNMENT): Redefine.
+	(cris_preferred_mininum_alignment): New function, split out from...
+	(cris_constant_alignment): ...here.
+	(cris_static_rtx_alignment): New function.
+	* config/i386/i386.c (ix86_static_rtx_alignment): New function,
+	split out from...
+	(ix86_constant_alignment): ...here.
+	(TARGET_STATIC_RTX_ALIGNMENT): Redefine.
+	* config/mmix/mmix.c (TARGET_STATIC_RTX_ALIGNMENT): Redefine.
+	(mmix_static_rtx_alignment): New function.
+	* config/spu/spu.c (spu_static_rtx_alignment): New function.
+	(TARGET_STATIC_RTX_ALIGNMENT): Redefine.
+
+2017-10-26  Tamar Christina  <tamar.christina@arm.com>
+
+	PR target/81800
+	* config/aarch64/aarch64.md (lrint<GPF:mode><GPI:mode>2):
+	Add flag_trapping_math and flag_fp_int_builtin_inexact.
+
+2017-10-25  Palmer Dabbelt  <palmer@dabbelt.com>
+
+	* config/riscv/riscv.md (ZERO_EXTEND_LOAD): Define.
+	* config/riscv/pic.md (local_pic_load): Rename to local_pic_load_s,
+	mark as a sign-extending load.
+	(local_pic_load_u): Define.
+
+2017-10-25  Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR middle-end/82062
+	* fold-const.c (operand_equal_for_comparison_p): Also return true
+	if ARG0 is a simple variant of ARG1 with narrower precision.
+	(fold_ternary_loc): Always pass unstripped operands to the predicate.
+
+2017-10-25  Jan Hubicka  <hubicka@ucw.cz>
+
+	* i386.c (ix86_builtin_vectorization_cost): Compute scatter/gather
+	cost correctly.
+	* i386.h (processor_costs): Add gather_static, gather_per_elt,
+	scatter_static, scatter_per_elt.
+	* x86-tune-costs.h: Add new cost entries.
+
+2017-10-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-ssa-sccvn.h (vn_eliminate): Declare.
+	* tree-ssa-pre.c (class eliminate_dom_walker, eliminate,
+	class pass_fre): Move to ...
+	* tree-ssa-sccvn.c (class eliminate_dom_walker, vn_eliminate,
+	class pass_fre): ... here and adjust for statistics.
+
+2017-10-25  Jakub Jelinek  <jakub@redhat.com>
+
+	PR libstdc++/81706
+	* attribs.c (attribute_value_equal): Use omp_declare_simd_clauses_equal
+	for comparison of OMP_CLAUSEs regardless of flag_openmp{,_simd}.
+	(duplicate_one_attribute, copy_attributes_to_builtin): New functions.
+	* attribs.h (duplicate_one_attribute, copy_attributes_to_builtin): New
+	declarations.
+
+2017-10-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-ssa-pre.c (need_eh_cleanup, need_ab_cleanup, el_to_remove,
+	el_to_fixup, el_todo, el_avail, el_avail_stack, eliminate_avail,
+	eliminate_push_avail, eliminate_insert): Move inside...
+	(class eliminate_dom_walker): ... this class in preparation
+	of move.
+	(fini_eliminate): Remove by merging with ...
+	(eliminate): ... this function.  Adjust for class changes.
+	(pass_pre::execute): Remove fini_eliminate call.
+	(pass_fre::execute): Likewise.
+
+2017-10-24  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82460
+	* config/i386/sse.md (UNSPEC_VPERMI2, UNSPEC_VPERMI2_MASK): Remove.
+	(VPERMI2, VPERMI2I): New mode iterators.
+	(<avx512>_vpermi2var<mode>3_maskz): Remove 3 define_expand patterns.
+	(<avx512>_vpermi2var<mode>3<sd_maskz_name>): Remove 3 define_insn
+	patterns.
+	(<avx512>_vpermi2var<mode>3_mask): New define_expand using VPERMI2
+	mode iterator.  Remove 3 old define_insn patterns.
+	(*<avx512>_vpermi2var<mode>3_mask): 2 new define_insn patterns.
+	(<avx512>_vpermt2var<mode>3_maskz): Adjust 1 define_expand to use
+	VPERMI2 mode iterator, remove the other two expanders.
+	(<avx512>_vpermt2var<mode>3<sd_maskz_name>): Adjust 1 define_insn
+	to use VPERMI2 mode iterator, add another alternative for vpermi2*
+	instructions, remove the other two patterns.
+	(<avx512>_vpermt2var<mode>3_mask): Adjust 1 define_insn to use VPERMI2
+	mode iterator, remove the other two patterns.
+	* config/i386/i386.c (ix86_expand_vec_perm_vpermi2): Renamed to ...
+	(ix86_expand_vec_perm_vpermt2): ... this.  Swap mask and op0
+	arguments, use gen_*vpermt2* expanders instead of gen_*vpermi2*
+	and adjust argument order accordingly.
+	(ix86_expand_vec_perm): Adjust caller.
+	(expand_vec_perm_1): Likewise.
+	(expand_vec_perm_vpermi2_vpshub2): Rename to ...
+	(expand_vec_perm_vpermt2_vpshub2): ... this.
+	(ix86_expand_vec_perm_const_1): Adjust caller.
+	(ix86_vectorize_vec_perm_const_ok): Adjust comments.
+
+	PR target/82370
+	* config/i386/sse.md (VIMAX_AVX2): Remove V4TImode.
+	(VIMAX_AVX2_AVX512BW, VIMAX_AVX512VL): New mode iterators.
+	(vec_shl_<mode>): Remove unused expander.
+	(avx512bw_<shift_insn><mode>3): New define_insn.
+	(<sse2_avx2>_ashl<mode>3, <sse2_avx2>_lshr<mode>3): Replaced by ...
+	(<sse2_avx2>_<shift_insn><mode>3): ... this.  New define_insn.
+
+2017-10-24  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82466
+	* doc/invoke.texi ([Wbuiltin-declaration-mismatch]): Extend
+	description.
+
+2017-10-24  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	PR rtl-optimization/82396
+	* gcc/haifa-sched.c (ready_sort_real): Remove qsort workaround.
+	(autopref_multipass_init): Simplify initialization.
+	(autopref_rank_data): Simplify sort order.
+	* gcc/sched-int.h (autopref_multipass_data_): Remove
+	multi_mem_insn_p, min_offset and max_offset.
+
+2017-10-24  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	PR middle-end/60580
+	* config/aarch64/aarch64.c (aarch64_frame_pointer_required)
+	Check special value of flag_omit_frame_pointer.
+	(aarch64_can_eliminate): Likewise.
+	(aarch64_override_options_after_change_1): Simplify handling of
+	-fomit-frame-pointer and -fomit-leaf-frame-pointer.
+
+2017-10-24  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82697
+	* tree-ssa-phiopt.c (cond_store_replacement): Use alias-set
+	zero for conditional load and unconditional store.
+
+2017-10-24  H.J. Lu  <hongjiu.lu@intel.com>
+
+	* doc/install.texi: Document bootstrap-cet.
+
+2017-10-24  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/82659
+	* config/i386/i386.c (rest_of_insert_endbranch): Don't insert
+	ENDBR instruction at function entrance if function is only
+	called directly.
+
+2017-10-24  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82628
+	* config/i386/i386.md (addcarry<mode>, subborrow<mode>): Change
+	patterns to better describe from which operation the CF is computed.
+	(addcarry<mode>_0, subborrow<mode>_0): New patterns.
+	* config/i386/i386.c (ix86_expand_builtin) <case handlecarry>: Pass
+	one LTU with [DT]Imode and another one with [SD]Imode.  If arg0
+	is 0, use _0 suffixed expanders instead of emitting a comparison
+	before it.
+
+2017-10-06  Sergey Shalnov  <Sergey.Shalnov@intel.com>
+
+	* config/i386/i386.md(*movsf_internal, *movdf_internal):
+	Avoid 512-bit AVX modes for TARGET_PREFER_AVX256.
+
+2017-10-24  Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR middle-end/82569
+	* tree-outof-ssa.h (always_initialized_rtx_for_ssa_name_p): Delete.
+	* expr.c (expand_expr_real_1) <expand_decl_rtl>: Revert latest change.
+	* loop-iv.c (iv_get_reaching_def): Likewise.
+	* cfgexpand.c (expand_one_ssa_partition): Initialize the RTX if the
+	variable is promoted and the partition contains undefined values.
+
+2017-10-23  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* config/nios2/nios2.c (nios2_rtx_costs): Make costs better
+	reflect reality.
+	(nios2_address_cost): Define.
+	(nios2_legitimize_address): Recognize (exp + constant) directly.
+	(TARGET_ADDRESS_COST): Define.
+
+2017-10-23  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* config/nios2/nios2-protos.h (nios2_large_constant_p): Declare.
+	(nios2_symbolic_memory_operand_p): Declare.
+	(nios2_split_large_constant): Declare.
+	(nios2_split_symbolic_memory_operand): Declare.
+	* config/nios2/nios2.c: Adjust includes.
+	(nios2_symbolic_constant_allowed): New.
+	(nios2_symbolic_constant_p): New.
+	(nios2_plus_symbolic_constant_p): New.
+	(nios2_valid_addr_expr_p): Recognize addresses involving
+	symbolic constants.
+	(nios2_legitimate_address_p): Likewise, also LO_SUM.
+	(nios2_symbolic_memory_operand_p): New.
+	(nios2_large_constant_p): New.
+	(nios2_split_large_constant): New.
+	(nios2_split_plus_large_constant): New.
+	(nios2_split_symbolic_memory_operand): New.
+	(nios2_legitimize_address): Code refactoring.  Handle addresses
+	involving symbolic constants.
+	(nios2_emit_move_sequence): Likewise.
+	(nios2_print_operand): Improve error output.
+	(nios2_print_operand_address): Handle LO_SUM.
+	(nios2_cdx_narrow_form_p): Likewise.
+	* config/nios2/nios2.md (movqi_internal): Add splitter for memory
+	operands involving symbolic constants.
+	(movhi_internal, movsi_internal): Likewise.
+	(zero_extendhisi2, zero_extendqi<mode>2): Likewise.
+	(extendhisi2, extendqi<mode>2): Likewise.
+
+2017-10-23  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* tree-pass.h (PROP_rtl_split_insns): Define.
+	* recog.c (pass_data_split_all_insns): Provide PROP_rtl_split_insns.
+
+2017-10-23  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* config/nios2/nios2.c (TARGET_LRA_P): Don't override.
+
+2017-10-23  Jakub Jelinek  <jakub@redhat.com>
+
+	PR debug/82630
+	* target.def (const_not_ok_for_debug_p): Default to
+	default_const_not_ok_for_debug_p instead of hook_bool_rtx_false.
+	* targhooks.h (default_const_not_ok_for_debug_p): New declaration.
+	* targhooks.c (default_const_not_ok_for_debug_p): New function.
+	* dwarf2out.c (const_ok_for_output_1): Only reject UNSPECs for
+	which targetm.const_not_ok_for_debug_p returned true.
+	* config/arm/arm.c (arm_const_not_ok_for_debug_p): Return true
+	for UNSPECs.
+	* config/powerpcspe/powerpcspe.c (rs6000_const_not_ok_for_debug_p):
+	Likewise.
+	* config/rs6000/rs6000.c (rs6000_const_not_ok_for_debug_p): Likewise.
+	* config/i386/i386.c (ix86_delegitimize_address_1): Don't delegitimize
+	UNSPEC_GOTOFF with addend into addend - _GLOBAL_OFFSET_TABLE_ + symbol
+	if !base_term_p.
+	(ix86_const_not_ok_for_debug_p): New function.
+	(i386_asm_output_addr_const_extra): Handle UNSPEC_GOTOFF.
+	(TARGET_CONST_NOT_OK_FOR_DEBUG_P): Redefine.
+
+2017-10-23  David Malcolm  <dmalcolm@redhat.com>
+
+	PR bootstrap/82610
+	* system.h: Conditionally include "unique-ptr.h" if
+	INCLUDE_UNIQUE_PTR is defined.
+	* unique-ptr-tests.cc: Remove include of "unique-ptr.h" in favor
+	of defining INCLUDE_UNIQUE_PTR before including "system.h".
+
+2017-10-23  Sebastian Perta  <sebastian.perta@renesas.com>
+
+	* config/rl78/rl78.md: New define_expand "subdi3".
+
+2017-10-23  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/82673
+	* config/i386/i386.c (ix86_finalize_stack_frame_flags): Skip
+	DF_REF_INSN if DF_REF_INSN_INFO is false.
+
+2017-10-23  Jan Hubicka  <hubicka@ucw.cz>
+
+	* i386.c (dimode_scalar_chain::compute_convert_gain): Use
+	xmm_move instead of sse_move.
+	(sse_store_index): New function.
+	(ix86_register_move_cost): Be more sensible about mismatch stall;
+	model AVX moves correctly; make difference between sse->integer and
+	integer->sse.
+	(ix86_builtin_vectorization_cost): Model correctly aligned and unaligned
+	moves; make difference between SSE and AVX.
+	* i386.h (processor_costs): Remove sse_move; add xmm_move, ymm_move
+	and zmm_move. Increase size of sse load and store tables;
+	add unaligned load and store tables; add ssemmx_to_integer.
+	* x86-tune-costs.h: Update all entries according to real
+	move latencies from Agner Fog's manual and chip documentation.
+
+2017-10-23  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82628
+	* config/i386/predicates.md (x86_64_dwzext_immediate_operand): New.
+	* config/i386/constraints.md (Wf): New constraint.
+	* config/i386/i386.md (UNSPEC_SBB): New unspec.
+	(cmp<dwi>_doubleword): Removed.
+	(sub<mode>3_carry_ccc, *sub<mode>3_carry_ccc_1): New patterns.
+	(sub<mode>3_carry_ccgz): Use unspec instead of compare.
+	* config/i386/i386.c (ix86_expand_branch) <case E_TImode>: Don't
+	expand with cmp<dwi>_doubleword.  For LTU and GEU use
+	sub<mode>3_carry_ccc instead of sub<mode>3_carry_ccgz and use CCCmode.
+
+	* common.opt (gcolumn-info): Enable by default.
+	* doc/invoke.texi (gcolumn-info): Document new default.
+
+2017-10-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82672
+	* graphite-isl-ast-to-gimple.c (graphite_copy_stmts_from_block):
+	Fold the stmt if we propagated into it.
+
+2017-10-23  Richard Biener  <rguenther@suse.de>
+
+	* tree-ssa-pre.c (bitmap_remove_from_set): Rename to...
+	(bitmap_remove_expr_from_set): ... this.  All callers call this
+	for non-constant values.
+	(bitmap_set_subtract): Rename to...
+	(bitmap_set_subtract_expressions): ... this.  Adjust and
+	optimize.
+	(bitmap_set_contains_value): Remove superfluous check.
+	(bitmap_set_replace_value): Inline into single caller ...
+	(bitmap_value_replace_in_set): ... here and simplify.
+	(dependent_clean): Merge into ...
+	(clean): ... this using an overload.  Adjust.
+	(prune_clobbered_mems): Adjust.
+	(compute_antic_aux): Likewise.
+	(compute_partial_antic_aux): Likewise.
+
+2017-10-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82129
+	Revert
+	2017-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/81181
+	* tree-ssa-pre.c (compute_antic_aux): Defer clean() to ...
+	(compute_antic): ... end of iteration here.
+
+2017-10-23  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* target.def (starting_frame_offset): New hook.
+	* doc/tm.texi (STARTING_FRAME_OFFSET): Remove in favor of...
+	(TARGET_STARTING_FRAME_OFFSET): ...this new hook.
+	* doc/tm.texi.in: Regenerate.
+	* hooks.h (hook_hwi_void_0): Declare.
+	* hooks.c (hook_hwi_void_0): New function.
+	* doc/rtl.texi: Refer to TARGET_STARTING_FRAME_OFFSET instead of
+	STARTING_FRAME_OFFSET.
+	* builtins.c (expand_builtin_setjmp_receiver): Likewise.
+	* reload1.c (reload): Likewise.
+	* cfgexpand.c (expand_used_vars): Use targetm.starting_frame_offset
+	instead of STARTING_FRAME_OFFSET.
+	* function.c (try_fit_stack_local): Likewise.
+	(assign_stack_local_1): Likewise
+	(instantiate_virtual_regs): Likewise.
+	* rtlanal.c (rtx_addr_can_trap_p_1): Likewise.
+	* config/avr/avr.md (nonlocal_goto_receiver): Likewise.
+	* config/aarch64/aarch64.h (STARTING_FRAME_OFFSET): Delete.
+	* config/alpha/alpha.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/arc/arc.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/arm/arm.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/bfin/bfin.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/c6x/c6x.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/cr16/cr16.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/cris/cris.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/fr30/fr30.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/frv/frv.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/ft32/ft32.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/h8300/h8300.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/i386/i386.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/ia64/ia64.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/m32c/m32c.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/m68k/m68k.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/mcore/mcore.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/mn10300/mn10300.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/moxie/moxie.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/msp430/msp430.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/nds32/nds32.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/nios2/nios2.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/nvptx/nvptx.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/pdp11/pdp11.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/riscv/riscv.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/rl78/rl78.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/rx/rx.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/s390/s390.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/sh/sh.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/sparc/sparc.c (sparc_compute_frame_size): Likewise.
+	* config/sparc/sparc.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/spu/spu.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/stormy16/stormy16.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/tilegx/tilegx.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/tilepro/tilepro.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/v850/v850.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/visium/visium.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/avr/avr.h (STARTING_FRAME_OFFSET): Likewise.
+	* config/avr/avr-protos.h (avr_starting_frame_offset): Likewise.
+	* config/avr/avr.c (avr_starting_frame_offset): Make static and
+	return a HOST_WIDE_INT.
+	(avr_builtin_setjmp_frame_value): Use it instead of
+	STARTING_FRAME_OFFSET.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/epiphany/epiphany.h (STARTING_FRAME_OFFSET): Delete.
+	* config/epiphany/epiphany.c (epiphany_starting_frame_offset):
+	New function.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/iq2000/iq2000.h (STARTING_FRAME_OFFSET): Delete.
+	* config/iq2000/iq2000.c (iq2000_starting_frame_offset): New function.
+	(TARGET_CONSTANT_ALIGNMENT): Redefine.
+	* config/lm32/lm32.h (STARTING_FRAME_OFFSET): Delete.
+	* config/lm32/lm32.c (lm32_starting_frame_offset): New function.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/m32r/m32r.h (STARTING_FRAME_OFFSET): Delete.
+	* config/m32r/m32r.c (m32r_starting_frame_offset): New function.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/microblaze/microblaze.h (STARTING_FRAME_OFFSET): Delete.
+	* config/microblaze/microblaze.c (microblaze_starting_frame_offset):
+	New function.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/mips/mips.h (STARTING_FRAME_OFFSET): Delete.
+	* config/mips/mips.c (mips_compute_frame_info): Refer to
+	TARGET_STARTING_FRAME_OFFSET instead of STARTING_FRAME_OFFSET.
+	(mips_starting_frame_offset): New function.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/mmix/mmix.h (STARTING_FRAME_OFFSET): Delete.
+	* config/mmix/mmix-protos.h (mmix_starting_frame_offset): Delete.
+	* config/mmix/mmix.c (mmix_starting_frame_offset): Make static
+	and return a HOST_WIDE_INT.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	(mmix_initial_elimination_offset): Refer to
+	TARGET_STARTING_FRAME_OFFSET instead of STARTING_FRAME_OFFSET.
+	* config/pa/pa.h (STARTING_FRAME_OFFSET): Delete.
+	* config/pa/pa.c (pa_starting_frame_offset): New function.
+	(pa_compute_frame_size): Use it instead of STARTING_FRAME_OFFSET.
+	(pa_expand_prologue): Likewise.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/powerpcspe/aix.h (STARTING_FRAME_OFFSET): Split out
+	!FRAME_GROWS_DOWNWARD handling to...
+	(RS6000_STARTING_FRAME_OFFSET): ...this new macro.
+	* config/powerpcspe/darwin.h (STARTING_FRAME_OFFSET): Split out
+	!FRAME_GROWS_DOWNWARD handling to...
+	(RS6000_STARTING_FRAME_OFFSET): ...this new macro.
+	* config/powerpcspe/powerpcspe.h (STARTING_FRAME_OFFSET): Split out
+	!FRAME_GROWS_DOWNWARD handling to...
+	(RS6000_STARTING_FRAME_OFFSET): ...this new macro.
+	* config/powerpcspe/powerpcspe.c (TARGET_STARTING_FRAME_OFFSET):
+	Redefine.
+	(rs6000_starting_frame_offset): New function.
+	* config/rs6000/aix.h (STARTING_FRAME_OFFSET): Split out
+	!FRAME_GROWS_DOWNWARD handling to...
+	(RS6000_STARTING_FRAME_OFFSET): ...this new macro.
+	* config/rs6000/darwin.h (STARTING_FRAME_OFFSET): Split out
+	!FRAME_GROWS_DOWNWARD handling to...
+	(RS6000_STARTING_FRAME_OFFSET): ...this new macro.
+	* config/rs6000/rs6000.h (STARTING_FRAME_OFFSET): Split out
+	!FRAME_GROWS_DOWNWARD handling to...
+	(RS6000_STARTING_FRAME_OFFSET): ...this new macro.
+	* config/rs6000/rs6000.c (TARGET_STARTING_FRAME_OFFSET): Refine.
+	(rs6000_starting_frame_offset): New function.
+	* config/vax/elf.h (STARTING_FRAME_OFFSET): Delete.
+	* config/vax/vax.h (STARTING_FRAME_OFFSET): Delete.
+	* config/vax/vax.c (vax_starting_frame_offset): New function.
+	(vax_expand_prologue): Use it instead of STARTING_FRAME_OFFSET.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* config/xtensa/xtensa.h (STARTING_FRAME_OFFSET): Delete.
+	* config/xtensa/xtensa.c (xtensa_starting_frame_offset): New function.
+	(TARGET_STARTING_FRAME_OFFSET): Redefine.
+	* system.h (STARTING_FRAME_OFFSET): Poison.
+
+2017-10-23  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* tree-vect-loop.c (vect_create_epilog_for_reduction): Use
+	SCALAR_TYPE_MODE instead of TYPE_MODE.
+
+2017-10-23  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* dwarf2out.c (loc_list_from_tree_1): Use SCALAR_INT_TYPE_MODE
+
+2017-10-23  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* expmed.c (expand_shift_1): Use scalar_mode for scalar_mode.
+
+2017-10-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82129
+	* tree-ssa-pre.c (bitmap_set_and): Remove.
+	(compute_antic_aux): Compute ANTIC_OUT intersection in a way
+	canonicalizing expressions in the set to those with lowest
+	ID rather than taking that from the first edge.
+
+2017-10-23  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* combine.c (rtx_equal_for_field_assignment_p): Use
+	byte_lowpart_offset.
+
+2017-10-22  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* internal-fn.c (expand_direct_optab_fn): Don't assign directly
+	to a SUBREG_PROMOTED_VAR.
+
+2017-10-22  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* cfgexpand.c (expand_debug_expr): Use GET_MODE_UNIT_PRECISION.
+	(expand_debug_source_expr): Likewise.
+	* combine.c (combine_simplify_rtx): Likewise.
+	* cse.c (fold_rtx): Likewise.
+	* optabs.c (expand_float): Likewise.
+	* simplify-rtx.c (simplify_unary_operation_1): Likewise.
+	(simplify_binary_operation_1): Likewise.
+
+2017-10-22  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* combine.c (simplify_comparison): Use HWI_COMPUTABLE_MODE_P.
+	(record_promoted_value): Likewise.
+	* expr.c (expand_expr_real_2): Likewise.
+	* ree.c (update_reg_equal_equiv_notes): Likewise.
+	(combine_set_extension): Likewise.
+	* rtlanal.c (low_bitmask_len): Likewise.
+	* simplify-rtx.c (neg_const_int): Likewise.
+	(simplify_binary_operation_1): Likewise.
+
+2017-10-22  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* lra-spills.c (assign_mem_slot): Use subreg_size_lowpart_offset.
+	* regcprop.c (maybe_mode_change): Likewise.
+	* reload1.c (alter_reg): Likewise.
+
+2017-10-22  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* inchash.h (inchash::hash::add_wide_int): New function.
+	* lto-streamer-out.c (hash_tree): Use it.
+
+2017-10-22  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* inchash.h (inchash::hash::add_wide_int): Rename to...
+	(inchash::hash::add_hwi): ...this.
+	* ipa-devirt.c (hash_odr_vtable): Update accordingly.
+	(polymorphic_call_target_hasher::hash): Likewise.
+	* ipa-icf.c (sem_function::get_hash, sem_function::init): Likewise.
+	(sem_item::add_expr, sem_item::add_type, sem_variable::get_hash)
+	(sem_item_optimizer::update_hash_by_addr_refs): Likewise.
+	* lto-streamer-out.c (hash_tree): Likewise.
+	* optc-save-gen.awk: Likewise.
+	* tree.c (add_expr): Likewise.
+
+2017-10-22  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/52451
+	* config/i386/i386.c (ix86_fp_compare_mode): Return CCFPmode
+	for ordered inequality comparisons even with TARGET_IEEE_FP.
+
+2017-10-22  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82628
+	* config/i386/i386.md (cmp<dwi>_doubleword): New pattern.
+	* config/i386/i386.c (ix86_expand_branch) <case E_TImode>:
+	Expand with cmp<dwi>_doubleword.
+
+2017-10-21  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+	* extend.texi: Add x86 specific to 'nocf_check' attribute.
+	List CET intrinsics.
+	* invoke.texi: Add -mcet, -mibt, -mshstk options.  Add x86
+	specific to -fcf-protection option.
+
+2017-10-21  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+	* common/config/i386/i386-common.c (OPTION_MASK_ISA_IBT_SET): New.
+	(OPTION_MASK_ISA_SHSTK_SET): Likewise.
+	(OPTION_MASK_ISA_IBT_UNSET): Likewise.
+	(OPTION_MASK_ISA_SHSTK_UNSET): Likewise.
+	(ix86_handle_option): Add -mibt, -mshstk, -mcet handling.
+	* config.gcc (extra_headers): Add cetintrin.h for x86 targets.
+	(extra_objs): Add cet.o for Linux/x86 targets.
+	(tmake_file): Add i386/t-cet for Linux/x86 targets.
+	* config/i386/cet.c: New file.
+	* config/i386/cetintrin.h: Likewise.
+	* config/i386/t-cet: Likewise.
+	* config/i386/cpuid.h (bit_SHSTK): New.
+	(bit_IBT): Likewise.
+	* config/i386/driver-i386.c (host_detect_local_cpu): Detect and
+	pass IBT and SHSTK bits.
+	* config/i386/i386-builtin-types.def
+	(VOID_FTYPE_UNSIGNED_PVOID): New.
+	(VOID_FTYPE_UINT64_PVOID): Likewise.
+	* config/i386/i386-builtin.def: Add CET intrinsics.
+	* config/i386/i386-c.c (ix86_target_macros_internal): Add
+	OPTION_MASK_ISA_IBT, OPTION_MASK_ISA_SHSTK handling.
+	* config/i386/i386-passes.def: Add pass_insert_endbranch pass.
+	* config/i386/i386-protos.h (make_pass_insert_endbranch): New
+	prototype.
+	* config/i386/i386.c (rest_of_insert_endbranch): New.
+	(pass_data_insert_endbranch): Likewise.
+	(pass_insert_endbranch): Likewise.
+	(make_pass_insert_endbranch): Likewise.
+	(ix86_notrack_prefixed_insn_p): Likewise.
+	(ix86_target_string): Add -mibt, -mshstk flags.
+	(ix86_option_override_internal): Add flag_cf_protection
+	processing.
+	(ix86_valid_target_attribute_inner_p): Set OPT_mibt, OPT_mshstk.
+	(ix86_print_operand): Add 'notrack' prefix output.
+	(ix86_init_mmx_sse_builtins): Add CET intrinsics.
+	(ix86_expand_builtin): Expand CET intrinsics.
+	(x86_output_mi_thunk): Add 'endbranch' instruction.
+	* config/i386/i386.h (TARGET_IBT): New.
+	(TARGET_IBT_P): Likewise.
+	(TARGET_SHSTK): Likewise.
+	(TARGET_SHSTK_P): Likewise.
+	* config/i386/i386.md (unspecv): Add UNSPECV_NOP_RDSSP,
+	UNSPECV_INCSSP, UNSPECV_SAVEPREVSSP, UNSPECV_RSTORSSP,
+	UNSPECV_WRSS, UNSPECV_WRUSS, UNSPECV_SETSSBSY, UNSPECV_CLRSSBSY.
+	(builtin_setjmp_setup): New pattern.
+	(builtin_longjmp): Likewise.
+	(rdssp<mode>): Likewise.
+	(incssp<mode>): Likewise.
+	(saveprevssp): Likewise.
+	(rstorssp): Likewise.
+	(wrss<mode>): Likewise.
+	(wruss<mode>): Likewise.
+	(setssbsy): Likewise.
+	(clrssbsy): Likewise.
+	(nop_endbr): Likewise.
+	* config/i386/i386.opt: Add -mcet, -mibt, -mshstk and -mcet-switch
+	options.
+	* config/i386/immintrin.h: Include <cetintrin.h>.
+	* config/i386/linux-common.h
+	(file_end_indicate_exec_stack_and_cet): New prototype.
+	(TARGET_ASM_FILE_END): New.
+
+2017-10-20  Jan Hubicka  <hubicka@ucw.cz>
+
+	* i386.c (ix86_builtin_vectorization_cost): Use existing rtx_cost
+	latencies instead of having separate table; make difference between
+	integer and float costs.
+	* i386.h (processor_costs): Remove scalar_stmt_cost,
+	scalar_load_cost, scalar_store_cost, vec_stmt_cost, vec_to_scalar_cost,
+	scalar_to_vec_cost, vec_align_load_cost, vec_unalign_load_cost,
+	vec_store_cost.
+	* x86-tune-costs.h: Remove entries which has been removed in
+	procesor_costs from all tables; make cond_taken_branch_cost
+	and cond_not_taken_branch_cost COST_N_INSNS based.
+
+2017-10-20  Jan Hubicka  <hubicka@ucw.cz>
+
+	* x86-tune-costs.h (intel_cost, generic_cost): Fix move costs.
+
+2017-10-20  Jakub Jelinek  <jakub@redhat.com>
+
+	* config/i386/i386.md (isa): Remove fma_avx512f.
+	* config/i386/sse.md (<avx512>_fmadd_<mode>_mask<round_name>,
+	<avx512>_fmadd_<mode>_mask3<round_name>,
+	<avx512>_fmsub_<mode>_mask<round_name>,
+	<avx512>_fmsub_<mode>_mask3<round_name>,
+	<avx512>_fnmadd_<mode>_mask<round_name>,
+	<avx512>_fnmadd_<mode>_mask3<round_name>,
+	<avx512>_fnmsub_<mode>_mask<round_name>,
+	<avx512>_fnmsub_<mode>_mask3<round_name>,
+	<avx512>_fmaddsub_<mode>_mask<round_name>,
+	<avx512>_fmaddsub_<mode>_mask3<round_name>,
+	<avx512>_fmsubadd_<mode>_mask<round_name>,
+	<avx512>_fmsubadd_<mode>_mask3<round_name>): Remove isa attribute.
+	(*vec_widen_umult_even_v16si<mask_name>,
+	*vec_widen_smult_even_v16si<mask_name>): Likewise.
+	(<mask_codefor>avx512bw_dbpsadbw<mode><mask_name>): Likewise.
+
+2017-10-20  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+	* extend.texi: Add 'nocf_check' documentation.
+	* gimple.texi: Add second parameter to
+	gimple_build_call_from_tree.
+	* invoke.texi: Add -fcf-protection documentation.
+	* rtl.texi: Add REG_CALL_NOTRACK documenation.
+
+2017-10-20  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82473
+	* tree-vect-loop.c (vectorizable_reduction): Properly get at
+	the largest input type.
+
+2017-10-20  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+	* c-attribs.c (handle_nocf_check_attribute): New function.
+	(c_common_attribute_table): Add 'nocf_check' handling.
+	* gimple-parser.c: Add second argument NULL to
+	gimple_build_call_from_tree.
+	* attrib.c (comp_type_attributes): Check nocf_check attribute.
+	* cfgexpand.c (expand_call_stmt): Set REG_CALL_NOCF_CHECK for
+	call insn.
+	* combine.c (distribute_notes): Add REG_CALL_NOCF_CHECK handling.
+	* common.opt: Add fcf-protection flag.
+	* emit-rtl.c (try_split): Add REG_CALL_NOCF_CHECK handling.
+	* flag-types.h: Add enum cf_protection_level.
+	* gimple.c (gimple_build_call_from_tree): Add second parameter.
+	Add 'nocf_check' attribute propagation to gimple call.
+	* gimple.h (gf_mask): Add GF_CALL_NOCF_CHECK.
+	(gimple_build_call_from_tree): Update prototype.
+	(gimple_call_nocf_check_p): New function.
+	(gimple_call_set_nocf_check): Likewise.
+	* gimplify.c: Add second argument to gimple_build_call_from_tree.
+	* ipa-icf.c: Add nocf_check attribute in statement hash.
+	* recog.c (peep2_attempt): Add REG_CALL_NOCF_CHECK handling.
+	* reg-notes.def: Add REG_NOTE (CALL_NOCF_CHECK).
+	* toplev.c (process_options): Add flag_cf_protection handling.
+
+2017-10-19  Jan Hubicka  <hubicka@ucw.cz>
+
+	* x86-tune-costs.h (core_cost): Fix div, move and sqrt latencies.
+
+2017-10-20  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82603
+	* tree-if-conv.c (predicate_mem_writes): Make sure to only
+	remove false predicated stores.
+
+2017-10-20  Richard Biener  <rguenther@suse.de>
+
+	* graphite-isl-ast-to-gimple.c
+	(translate_isl_ast_to_gimple::graphite_copy_stmts_from_block):
+	Remove return value and simplify, dump copied stmt after lhs
+	adjustment.
+	(translate_isl_ast_to_gimple::translate_isl_ast_node_user):
+	Reduce dump verbosity.
+	(gsi_insert_earliest): Likewise.
+	(translate_isl_ast_to_gimple::copy_bb_and_scalar_dependences): Adjust.
+	* graphite.c (print_global_statistics): Adjust dumping.
+	(print_graphite_scop_statistics): Likewise.
+	(print_graphite_statistics): Do not dump loops here.
+	(graphite_transform_loops): But here.
+
+2017-10-20  Nicolas Roche  <roche@adacore.com>
+
+	* configure.ac (ACX_PROG_GNAT): Append "libgnat" to include search dir.
+	* configure: Regenerate.
+
+2017-10-20  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82158
+	* tree-cfg.c (pass_warn_function_return::execute): In noreturn
+	functions when optimizing replace GIMPLE_RETURN stmts with
+	calls to __builtin_unreachable ().
+
+	PR sanitizer/82595
+	* config/gnu-user.h (LIBTSAN_EARLY_SPEC): Add libtsan_preinit.o
+	for -fsanitize=thread link of executables.
+	(LIBLSAN_EARLY_SPEC): Add liblsan_preinit.o for -fsanitize=leak
+	link of executables.
+
+	PR target/82370
+	* config/i386/sse.md (VI248_AVX2, VI248_AVX512BW, VI248_AVX512BW_2):
+	New mode iterators.
+	(<shift_insn><mode>3<mask_name>): Change the last of the 3
+	define_insns for logical vector shifts to use VI248_AVX512BW
+	iterator instead of VI48_AVX512, remove <mask_mode512bit_condition>
+	condition, useless isa and prefix attributes.  Change the first
+	2 of these define_insns to ...
+	(<mask_codefor><shift_insn><mode>3<mask_name>): ... this, new
+	define_insn for avx512vl.
+	(<shift_insn><mode>3): ... and this, new define_insn without
+	masking for non-avx512vl.
+
+	PR target/82370
+	* config/i386/sse.md (*andnot<mode>3,
+	<mask_codefor><code><mode>3<mask_name>, *<code><mode>3): Split
+	(=v,v,vm) alternative into (=x,x,xm) and (=v,v,vm), for 128-bit
+	and 256-bit vectors, the (=x,x,xm) alternative and when mask is
+	not applied use empty suffix even for TARGET_AVX512VL.
+	* config/i386/subst.md (mask_prefix3, mask_prefix4): When mask
+	is applied, supply evex,evex or evex,evex,evex instead of just
+	evex.
+
+2017-10-20  Julia Koval  <julia.koval@intel.com>
+
+	* common/config/i386/i386-common.c (OPTION_MASK_ISA_GFNI_SET,
+	(OPTION_MASK_ISA_GFNI_UNSET): New.
+	(ix86_handle_option): Handle OPT_mgfni.
+	* config/i386/cpuid.h (bit_GFNI): New.
+	* config/i386/driver-i386.c (host_detect_local_cpu): Detect gfni.
+	* config/i386/i386-c.c (ix86_target_macros_internal): Define __GFNI__.
+	* config/i386/i386.c (ix86_target_string): Add -mgfni.
+	(ix86_valid_target_attribute_inner_p): Add OPT_mgfni.
+	* config/i386/i386.h (TARGET_GFNI, TARGET_GFNI_P): New.
+	* config/i386/i386.opt: Add mgfni.
+
+2017-10-20  Orlando Arias  <oarias@knights.ucf.edu>
+
+	* config/msp430/msp430.c (msp430_option_override): Disable
+	-fdelete-null-pointer-checks.
+	* doc/invoke.text (-fdelete-null-pointer-checks): Document that.
+
+2017-10-19  Jan Hubicka  <hubicka@ucw.cz>
+
+	* x86-tune-costs.h (generic_cost, core_cost): Correct costs
+	of x87 and SSE instructions.
+
+2017-10-19  Jan Hubicka  <hubicka@ucw.cz>
+
+	* asan.c (create_cond_insert_point): Do not update edge count.
+	* auto-profile.c (afdo_propagate_edge): Update for edge count removal.
+	(afdo_propagate_circuit): Likewise.
+	(afdo_calculate_branch_prob): Likewise.
+	(afdo_annotate_cfg): Likewise.
+	* basic-block.h (struct edge_def): Remove count.
+	(edge_def::count): New accessor.
+	* bb-reorder.c (rotate_loop): Update.
+	(find_traces_1_round): Update.
+	(connect_traces): Update.
+	(sanitize_hot_paths): Update.
+	* cfg.c (unchecked_make_edge): Update.
+	(make_single_succ_edge): Update.
+	(check_bb_profile): Update.
+	(dump_edge_info): Update.
+	(update_bb_profile_for_threading): Update.
+	(scale_bbs_frequencies_int): Update.
+	(scale_bbs_frequencies_gcov_type): Update.
+	(scale_bbs_frequencies_profile_count): Update.
+	(scale_bbs_frequencies): Update.
+	* cfganal.c (connect_infinite_loops_to_exit): Update.
+	* cfgbuild.c (compute_outgoing_frequencies): Update.
+	(find_many_sub_basic_blocks): Update.
+	* cfgcleanup.c (try_forward_edges): Update.
+	(try_crossjump_to_edge): Update
+	* cfgexpand.c (expand_gimple_cond): Update
+	(expand_gimple_tailcall): Update
+	(construct_exit_block): Update
+	* cfghooks.c (verify_flow_info): Update
+	(redirect_edge_succ_nodup): Update
+	(split_edge): Update
+	(make_forwarder_block): Update
+	(duplicate_block): Update
+	(account_profile_record): Update
+	* cfgloop.c (find_subloop_latch_edge_by_profile): Update.
+	* cfgloopanal.c (expected_loop_iterations_unbounded): Update.
+	* cfgloopmanip.c (scale_loop_profile): Update.
+	(loopify): Update.
+	(lv_adjust_loop_entry_edge): Update.
+	* cfgrtl.c (try_redirect_by_replacing_jump): Update.
+	(force_nonfallthru_and_redirect): Update.
+	(purge_dead_edges): Update.
+	(rtl_flow_call_edges_add): Update.
+	* cgraphunit.c (init_lowered_empty_function): Update.
+	(cgraph_node::expand_thunk): Update.
+	* gimple-pretty-print.c (dump_probability): Update.
+	(dump_edge_probability): Update.
+	* gimple-ssa-isolate-paths.c (isolate_path): Update.
+	* haifa-sched.c (sched_create_recovery_edges): Update.
+	* hsa-gen.c (convert_switch_statements): Update.
+	* ifcvt.c (dead_or_predicable): Update.
+	* ipa-inline-transform.c (inline_transform): Update.
+	* ipa-split.c (split_function): Update.
+	* ipa-utils.c (ipa_merge_profiles): Update.
+	* loop-doloop.c (add_test): Update.
+	* loop-unroll.c (unroll_loop_runtime_iterations): Update.
+	* lto-streamer-in.c (input_cfg): Update.
+	(input_function): Update.
+	* lto-streamer-out.c (output_cfg): Update.
+	* modulo-sched.c (sms_schedule): Update.
+	* postreload-gcse.c (eliminate_partially_redundant_load): Update.
+	* predict.c (maybe_hot_edge_p): Update.
+	(unlikely_executed_edge_p): Update.
+	(probably_never_executed_edge_p): Update.
+	(dump_prediction): Update.
+	(drop_profile): Update.
+	(propagate_unlikely_bbs_forward): Update.
+	(determine_unlikely_bbs): Update.
+	(force_edge_cold): Update.
+	* profile.c (compute_branch_probabilities): Update.
+	* reg-stack.c (better_edge): Update.
+	* shrink-wrap.c (handle_simple_exit): Update.
+	* tracer.c (better_p): Update.
+	* trans-mem.c (expand_transaction): Update.
+	(split_bb_make_tm_edge): Update.
+	* tree-call-cdce.c: Update.
+	* tree-cfg.c (gimple_find_sub_bbs): Update.
+	(gimple_split_edge): Update.
+	(gimple_duplicate_sese_region): Update.
+	(gimple_duplicate_sese_tail): Update.
+	(gimple_flow_call_edges_add): Update.
+	(insert_cond_bb): Update.
+	(execute_fixup_cfg): Update.
+	* tree-cfgcleanup.c (cleanup_control_expr_graph): Update.
+	* tree-complex.c (expand_complex_div_wide): Update.
+	* tree-eh.c (lower_resx): Update.
+	(unsplit_eh): Update.
+	(cleanup_empty_eh_move_lp): Update.
+	* tree-inline.c (copy_edges_for_bb): Update.
+	(freqs_to_counts): Update.
+	(copy_cfg_body): Update.
+	* tree-ssa-dce.c (remove_dead_stmt): Update.
+	* tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update.
+	* tree-ssa-loop-im.c (execute_sm_if_changed): Update.
+	* tree-ssa-loop-ivcanon.c (remove_exits_and_undefined_stmts): Update.
+	(unloop_loops): Update.
+	* tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update.
+	* tree-ssa-loop-split.c (connect_loops): Update.
+	(split_loop): Update.
+	* tree-ssa-loop-unswitch.c (hoist_guard): Update.
+	* tree-ssa-phionlycprop.c (propagate_rhs_into_lhs): Update.
+	* tree-ssa-phiopt.c (replace_phi_edge_with_variable): Update.
+	* tree-ssa-reassoc.c (branch_fixup): Update.
+	* tree-ssa-tail-merge.c (replace_block_by): Update.
+	* tree-ssa-threadupdate.c (remove_ctrl_stmt_and_useless_edges): Update.
+	(compute_path_counts): Update.
+	(update_profile): Update.
+	(recompute_probabilities): Update.
+	(update_joiner_offpath_counts): Update.
+	(estimated_freqs_path): Update.
+	(freqs_to_counts_path): Update.
+	(clear_counts_path): Update.
+	(ssa_fix_duplicate_block_edges): Update.
+	(duplicate_thread_path): Update.
+	* tree-switch-conversion.c (hoist_edge_and_branch_if_true): Update.
+	(case_bit_test_cmp): Update.
+	(collect_switch_conv_info): Update.
+	(gen_inbound_check): Update.
+	(do_jump_if_equal): Update.
+	(emit_cmp_and_jump_insns): Update.
+	* tree-tailcall.c (decrease_profile): Update.
+	(eliminate_tail_call): Update.
+	* tree-vect-loop-manip.c (slpeel_add_loop_guard): Update.
+	(vect_do_peeling): Update.
+	* tree-vect-loop.c (scale_profile_for_vect_loop): Update.
+	* ubsan.c (ubsan_expand_null_ifn): Update.
+	(ubsan_expand_ptr_ifn): Update.
+	* value-prof.c (gimple_divmod_fixed_value): Update.
+	(gimple_mod_pow2): Update.
+	(gimple_mod_subtract): Update.
+	(gimple_ic): Update.
+	(gimple_stringop_fixed_value): Update.
+
+2017-10-19  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82618
+	* config/i386/i386.md (sub to cmp): New peephole2 pattern.
+
+2017-10-19  Alexander Monakov  <amonakov@ispras.ru>
+
+	PR rtl-optimization/82395
+	* ira-color.c (allocno_priority_compare_func): Fix comparison step
+	based on non_spilled_static_chain_regno_p.
+
+2017-10-19  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386.c (output_387_binary_op): Rewrite SSE part.
+	(ix86_emit_mode_set): Rewrite insn mnemonic construction.
+	(ix86_prepare_fp_compare_args): Redefine is_sse as bool.
+
+2017-10-19  Martin Sebor  <msebor@redhat.com>
+
+	PR tree-optimization/82596
+	* tree.c (array_at_struct_end_p): Handle STRING_CST.
+
+2017-10-19  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* asan.c (handle_builtin_alloca): Deal with all alloca variants.
+	(get_mem_refs_of_builtin_call): Likewise.
+	* builtins.c (expand_builtin_apply): Adjust call to
+	allocate_dynamic_stack_space.
+	(expand_builtin_alloca): For __builtin_alloca_with_align_and_max, pass
+	the third argument to allocate_dynamic_stack_space, otherwise -1.
+	(expand_builtin): Deal with all alloca variants.
+	(is_inexpensive_builtin): Likewise.
+	* builtins.def (BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX): New.
+	* calls.c (special_function_p): Deal with all alloca variants.
+	(initialize_argument_information): Adjust call to
+	allocate_dynamic_stack_space.
+	(expand_call): Likewise.
+	* cfgexpand.c (expand_call_stmt): Deal with all alloca variants.
+	* doc/extend.texi (Built-ins): Add __builtin_alloca_with_align_and_max
+	* explow.c (allocate_dynamic_stack_space): Add MAX_SIZE parameter and
+	use it for the stack usage computation.
+	* explow.h (allocate_dynamic_stack_space): Adjust prototype.
+	* function.c (gimplify_parameters): Call build_alloca_call_expr.
+	* gimple-ssa-warn-alloca.c (alloca_call_type): Simplify control flow.
+	Take into account 3rd argument of __builtin_alloca_with_align_and_max.
+	(in_loop_p): Remove first argument and useless check.
+	(pass_walloca::execute): Remove useless test and adjust call to above.
+	* gimple.c (gimple_build_call_from_tree): Deal with all alloc variants
+	* gimplify.c (gimplify_vla_decl): Call build_alloca_call_expr.
+	(gimplify_call_expr): Deal with all alloca variants.
+	* hsa-gen.c (gen_hsa_alloca): Likewise.
+	(gen_hsa_insns_for_call): Likewise.
+	* ipa-pure-const.c (special_builtin_state): Likewise.
+	* tree-chkp.c (chkp_build_returned_bound): Likewise.
+	* tree-object-size.c (alloc_object_size): Likewise.
+	* tree-ssa-alias.c (ref_maybe_used_by_call_p_1): Likewise.
+	(call_may_clobber_ref_p_1): Likewise.
+	* tree-ssa-ccp.c (evaluate_stmt): Likewise.
+	(ccp_fold_stmt): Likewise.
+	(optimize_stack_restore): Likewise.
+	* tree-ssa-dce.c (mark_stmt_if_obviously_necessary): Likewise.
+	(mark_all_reaching_defs_necessary_1): Likewise.
+	(propagate_necessity): Likewise.
+	(eliminate_unnecessary_stmts): Likewise.
+	* tree.c (build_common_builtin_nodes): Build
+	BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX.
+	(build_alloca_call_expr): New function.
+	* tree.h (ALLOCA_FUNCTION_CODE_P): New macro.
+	(CASE_BUILT_IN_ALLOCA): Likewise.
+	(build_alloca_call_expr): Declare.
+	* varasm.c (incorporeal_function_p): Deal with all alloca variants.
+
+2017-10-19  Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR debug/82509
+	* dwarf2out.c (new_die_raw): New static inline function.
+	(new_die): Use it to create the DIE.
+	(add_AT_external_die_ref): Likewise.
+	(clone_die): Likewise.
+	(clone_as_declaration): Likewise.
+	(dwarf2out_vms_debug_main_pointer): Likewise.
+	(base_type_die): Likewise.  Remove early return for corner cases.
+	Do not call add_pubtype on the DIE here.
+	(is_base_type): Remove ERROR_MARK and return 0 for VOID_TYPE.
+	(modified_type_die): Adjust the lookup for reverse order DIEs.  Skip
+	typedefs for base types with DW_AT_endianity.  Make sure a DIE with
+	native order exists for base types, attach the DIE manually and call
+	add_pubtype on it.  Do not equate a reverse order DIE to the type.
+
+2017-10-19  Richard Earnshaw  <rearnsha@arm.com>
+
+	* config/arm/arm.c (align_ok_ldrd_strd): New function.
+	(mem_ok_for_ldrd_strd): New parameter align.  Extract the alignment of
+	the mem into it.
+	(gen_operands_ldrd_strd): Validate the alignment of the accesses.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	* flag-types.h (enum sanitize_code): Add SANITIZE_BUILTIN.  Or
+	SANITIZE_BUILTIN into SANITIZE_UNDEFINED.
+	* sanitizer.def (BUILT_IN_UBSAN_HANDLE_INVALID_BUILTIN,
+	BUILT_IN_UBSAN_HANDLE_INVALID_BUILTIN_ABORT): New builtins.
+	* opts.c (sanitizer_opts): Add builtin.
+	* ubsan.c (instrument_builtin): New function.
+	(pass_ubsan::execute): Call it.
+	(pass_ubsan::gate): Enable even for SANITIZE_BUILTIN.
+	* doc/invoke.texi: Document -fsanitize=builtin.
+
+	* ubsan.c (ubsan_expand_null_ifn): Use _v1 suffixed type mismatch
+	builtins, store max (log2 (align), 0) into uchar field instead of
+	align into uptr field.
+	(ubsan_expand_objsize_ifn): Use _v1 suffixed type mismatch builtins,
+	store uchar 0 field instead of uptr 0 field.
+	(instrument_nonnull_return): Use _v1 suffixed nonnull return builtin,
+	instead of passing one address of struct with 2 locations pass
+	two addresses of structs with 1 location each.
+	* sanitizer.def (BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH,
+	BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_ABORT,
+	BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN,
+	BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_ABORT): Removed.
+	(BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1,
+	BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1_ABORT,
+	BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_V1,
+	BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_V1_ABORT): New builtins.
+
+2017-10-19  Martin Liska  <mliska@suse.cz>
+
+	PR driver/81829
+	* file-find.c (remove_prefix): Remove.
+	* file-find.h (remove_prefix): Likewise.
+	* gcc-ar.c: Remove smartness of lookup.
+
+2017-10-19  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	* config/rs6000/rs6000.md (*call_indirect_aix<mode>,
+	*call_value_indirect_aix<mode>, *call_indirect_elfv2<mode>,
+	*call_value_indirect_elfv2<mode>): Add correct mode to the unspec.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82580
+	* config/i386/i386.md (setcc + movzbl to xor + setcc): New peephole2.
+	(setcc + and to xor + setcc): New peephole2.
+
+2017-10-19  Tom de Vries  <tom@codesourcery.com>
+
+	* doc/sourcebuild.texi (Test Directives, Variants of
+	dg-require-support): Add dg-require-stack-size.
+
+2017-10-19  Martin Liska  <mliska@suse.cz>
+
+	PR sanitizer/82517
+	* gimplify.c (gimplify_decl_expr): Do not instrument variables
+	that have a large alignment.
+	(gimplify_target_expr): Likewise.
+
+2017-10-18  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	PR rtl-optimization/82602
+	* ira.c (rtx_moveable_p): Return false for volatile asm.
+
+2017-10-18  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82580
+	* config/i386/i386-modes.def (CCGZ): New CC mode.
+	* config/i386/i386.md (sub<mode>3_carry_ccgz): New insn pattern.
+	* config/i386/predicates.md (ix86_comparison_operator):
+	Handle CCGZmode.
+	* config/i386/i386.c (ix86_expand_branch) <case E_TImode>:
+	Emulate LE, LEU, GT, GTU, LT, LTU, GE and GEU double-word comparisons
+	with double-word subtraction.
+	(put_condition_code): Handle CCGZmode.
+
+2017-10-18  Aldy Hernandez  <aldyh@redhat.com>
+
+	* wide-int.cc (debug (const wide_int &)): New.
+	(debug (const wide_int *)): New.
+	(debug (const widest_int &)): New.
+	(debug (const widest_int *)): New.
+
+2017-10-18  Vladimir Makarov  <vmakarov@redhat.com>
+
+	PR middle-end/82556
+	* lra-constraints.c (curr_insn_transform): Use non-input operand
+	instead of output one for matched reload.
+
+2017-10-18  Bin Cheng  <bin.cheng@arm.com>
+
+	* tree-loop-distribution.c (INCLUDE_ALGORITHM): New header file.
+	(tree-ssa-loop-ivopts.h): New header file.
+	(struct builtin_info): New fields.
+	(classify_builtin_1): Compute and record base and offset parts for
+	memset builtin partition by calling strip_offset.
+	(offset_cmp, fuse_memset_builtins): New functions.
+	(finalize_partitions): Fuse adjacent memset partitions by calling
+	above function.
+	* tree-ssa-loop-ivopts.c (strip_offset): Delete static declaration.
+	Expose the interface.
+	* tree-ssa-loop-ivopts.h (strip_offset): New declaration.
+
+2017-10-18  Bin Cheng  <bin.cheng@arm.com>
+
+	PR tree-optimization/82574
+	* tree-loop-distribution.c (find_single_drs): New parameter.  Check
+	that data reference must be executed exactly once per iteration
+	against the outermost loop in nest.
+	(classify_partition): Update call to above function.
+
+2017-10-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82591
+	* graphite.c (graphite_transform_loops): Move code gen message
+	printing ...
+	* graphite-isl-ast-to-gimple.c (graphite_regenerate_ast_isl):
+	Here.  Handle scop_to_isl_ast failing.
+	(scop_to_isl_ast): Limit the number of ISL operations.
+
+2017-10-18  Richard Biener  <rguenther@suse.de>
+
+	* graphite-isl-ast-to-gimple.c
+	(translate_isl_ast_to_gimple::set_rename): Simplify.
+	(translate_isl_ast_to_gimple::set_rename_for_each_def): Inline...
+	(graphite_copy_stmts_from_block): ... here.
+	(copy_bb_and_scalar_dependences): Simplify.
+	(add_parameters_to_ivs_params): Canonicalize.
+	(generate_entry_out_of_ssa_copies): Simplify.
+	* graphite-sese-to-poly.c (extract_affine_name): Simplify
+	by passing in ISL dimension.
+	(parameter_index_in_region_1): Rename to ...
+	(parameter_index_in_region): ... this.
+	(extract_affine): Adjust assert, pass down parameter index.
+	(add_param_constraints): Use range-info when available.
+	(build_scop_context): Adjust.
+	* sese.c (new_sese_info): Adjust.
+	(free_sese_info): Likewise.
+	* sese.h (bb_map_t, rename_map_t, phi_rename, init_back_edge_pair_t):
+	Remove unused typedefs.
+	(struct sese_info_t): Simplify rename_map, remove incomplete_phis.
+
+2017-10-18  Martin Liska  <mliska@suse.cz>
+
+	* combine.c (simplify_compare_const): Add gcc_fallthrough.
+
+2017-10-18  Robin Dapp  <rdapp@linux.vnet.ibm.com>
+
+	* config/s390/s390.c (s390_bb_fallthru_entry_likely): New function.
+	(s390_sched_init): Do not reset s390_sched_state if we entered the
+	current basic block via a fallthru edge and all others are unlikely.
+
+2017-10-18  Robin Dapp  <rdapp@linux.vnet.ibm.com>
+
+	* config/s390/s390.c (NUM_SIDES): New variable.
+	(LONGRUNNING_THRESHOLD): New variable.
+	(LATENCY_FACTOR): New variable.
+	(s390_sched_score): Decrease score for long-running instructions on
+	wrong side.
+	(s390_sched_variable_issue): Perform bookkeeping for long-running
+	instructions.
+
+2017-10-18  Richard Biener  <rguenther@suse.de>
+
+	* graphite-isl-ast-to-gimple.c (gcc_expression_from_isl_ast_expr_id):
+	Simplify with removal of the parameter rename map.
+	(set_rename): Likewise.
+	(should_copy_to_new_region): Likewise.
+	(graphite_copy_stmts_from_block): Likewise.
+	(copy_bb_and_scalar_dependences): Remove initialization of
+	unused copied_bb_map.
+	(copy_def): Remove.
+	(copy_internal_parameters): Likewise.
+	(graphite_regenerate_ast_isl): Do not call copy_internal_parameters.
+	* graphite-scop-detection.c (scop_detection::stmt_simple_for_scop_p):
+	Use INTEGRAL_TYPE_P.
+	(parameter_index_in_region_1): Rename to ...
+	(assign_parameter_index_in_region): ... this.  Assert we have
+	a parameter we handle.
+	(scan_tree_for_params): Adjust.
+	* sese.h (parameter_rename_map_t): Remove.
+	(struct sese_info_t): Remove unused parameter_rename_map and
+	copied_bb_map members.
+	* sese.c (new_sese_info): Adjust.
+	(free_sese_info): Likewise.
+
+2017-10-18  Martin Liska  <mliska@suse.cz>
+
+	PR sanitizer/82545
+	* asan.c (asan_expand_poison_ifn): Do not put gimple stmt
+	on an abnormal edge.
+
+2017-10-18  Sebastian Huber  <sebastian.huber@embedded-brains.de>
+
+	* doc/invoke.texi (ffunction-sections and fdata-sections):
+	Update.
+
+2017-10-17  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* tree-ssa-loop-ivopts.c (add_autoinc_candidates): Bail out only if
+	the use statement can throw internally.
+
+2017-10-17  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* config/visium/visium.c (visium_select_cc_mode): Return CCmode for
+	any RTX present on the RHS of a SET.
+	* compare-elim.c (try_eliminate_compare): Restore comment.
+
+2017-10-17  Jakub Jelinek  <jakub@redhat.com>
+
+	* langhooks.h (struct lang_hooks): Document that tree_size langhook
+	may be also called on tcc_type nodes.
+	* langhooks.c (lhd_tree_size): Likewise.
+
+2017-10-17  David Malcolm  <dmalcolm@redhat.com>
+
+	* gimple-ssa-sprintf.c (fmtwarn): Update for changed signature of
+	format_warning_at_substring.
+	(maybe_warn): Convert source_range * param to a location_t.  Pass
+	UNKNOWN_LOCATION rather than NULL to fmtwarn.
+	(format_directive): Remove code to extract source_ranges and
+	source_range * in favor of just a location_t.
+	(parse_directive): Pass UNKNOWN_LOCATION rather than NULL to
+	fmtwarn.
+	* substring-locations.c (format_warning_va): Convert
+	source_range * param to a location_t.
+	(format_warning_at_substring): Likewise.
+	* substring-locations.h (format_warning_va): Likewise.
+	(format_warning_at_substring): Likewise.
+
+2017-10-17  Jan Hubicka  <hubicka@ucw.cz>
+
+	* target.h (enum vect_cost_for_stmt): Add vec_gather_load and
+	vec_scatter_store
+	* tree-vect-stmts.c (record_stmt_cost): Make difference between normal
+	and scatter/gather ops.
+
+	* aarch64/aarch64.c (aarch64_builtin_vectorization_cost): Add
+	vec_gather_load and vec_scatter_store.
+	* arm/arm.c (arm_builtin_vectorization_cost): Likewise.
+	* powerpcspe/powerpcspe.c (rs6000_builtin_vectorization_cost): Likewise.
+	* rs6000/rs6000.c (rs6000_builtin_vectorization_cost): Likewise.
+	* s390/s390.c (s390_builtin_vectorization_cost): Likewise.
+	* spu/spu.c (spu_builtin_vectorization_cost): Likewise.
+	* i386/i386.c (x86_builtin_vectorization_cost): Likewise.
+
+2017-10-17  Uros Bizjak  <ubizjak@gmail.com>
+
+	* reg-stack.c (compare_for_stack_reg): Add bool argument.
+	Detect FTST instruction and handle its register pops.  Only pop
+	second operand if can_pop_second_op is true.
+	(subst_stack_regs_pat) <case COMPARE>: Detect FCOMI instruction to
+	set can_pop_second_op to false in the compare_for_stack_reg call.
+
+	* config/i386/i386.md (*cmpi<FPCMP:unord><MODEF:mode>): Only call
+	output_fp_compare for stack register operands.
+	* config/i386/i386.c (output_fp_compare): Do not output SSE compare
+	instructions here.  Do not emit stack register pops here.  Assert
+	that FCOMPP pops next to top stack register.  Rewrite function.
+
+2017-10-17  Nathan Sidwell  <nathan@acm.org>
+
+	PR middle-end/82577
+	* alias.c (compare_base_decls): Check HAS_DECL_ASSEMBLER_NAME_P,
+	use DECL_ASSEMBLER_NAME_RAW.
+
+	PR middle-end/82546
+	* tree.c (tree_code_size): Reformat.  Punt to lang hook for unknown
+	TYPE nodes.
+
+2017-10-17  Qing Zhao <qing.zhao@oracle.com>
+	    Wilco Dijkstra <wilco.dijkstra@arm.com>
+
+	* builtins.c (expand_builtin_update_setjmp_buf): Add a
+	converstion to Pmode from the buf_addr.
+
+2017-10-17  Richard Biener  <rguenther@suse.de>
+
+	* graphite-dependences.c (scop_get_reads_and_writes): Change
+	output parameters to references.
+
+2017-10-17  Jackson Woodruff  <jackson.woodruff@arm.com>
+
+	PR 71026/tree-optimization
+	* fold-const.c (distribute_real_division): Removed.
+	(fold_binary_loc): Remove calls to distribute_real_divison.
+
+2017-10-17  Richard Biener  <rguenther@suse.de>
+
+	* graphite-scop-detection.c
+	(scop_detection::stmt_has_simple_data_refs_p): Always use
+	the full nest as region.
+	(try_generate_gimple_bb): Likewise.
+	* sese.c (scalar_evolution_in_region): Simplify now that
+	SCEV can handle instantiation in regions.
+	* tree-scalar-evolution.c (instantiate_scev_name): Also instantiate
+	in the non-loop part of a function if requested.
+
+2017-10-17  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82563
+	* graphite-isl-ast-to-gimple.c (generate_entry_out_of_ssa_copies):
+	New function.
+	(graphite_regenerate_ast_isl): Call it.
+	* graphite-scop-detection.c (build_scops): Remove entry edge split.
+
+2017-10-17  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/82549
+	* fold-const.c (optimize_bit_field_compare, fold_truth_andor_1):
+	Formatting fixes.  Instead of calling make_bit_field_ref with negative
+	bitpos return 0.
+
+2017-10-17  Olga Makhotina  <olga.makhotina@intel.com>
+
+	* config/i386/avx512dqintrin.h (_mm_mask_reduce_sd,
+	_mm_maskz_reduce_sd, _mm_mask_reduce_ss,=20
+	_mm_maskz_reduce_ss): New.
+	* config/i386/i386-builtin.def (__builtin_ia32_reducesd_mask,
+	__builtin_ia32_reducess_mask): Ditto..
+	(__builtin_ia32_reducesd, __builtin_ia32_reducess): Remove.
+	* config/i386/sse.md (reduces<mode>): Renamed to ...
+	(reduces<mode><mask_scalar_name>): ... this.
+	(vreduce<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}):
+	Changed to ...
+	(vreduce<ssescalarmodesuffix>\t{%3, %2, %1, %0<mask_scalar_operand4>|
+	%0<mask_scalar_operand4>, %1, %2, %3}): ... this.
+
+2017-10-16  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Add unique-ptr-tests.o.
+	* selftest-run-tests.c (selftest::run_tests): Call
+	selftest::unique_ptr_tests_cc_tests.
+	* selftest.h (selftest::unique_ptr_tests_cc_tests): New decl.
+	* unique-ptr-tests.cc: New file.
+
+2017-10-16  Vladimir Makarov  <vmakarov@redhat.com>
+
+	PR sanitizer/82353
+	* lra.c (collect_non_operand_hard_regs): Don't ignore operator
+	locations.
+	* lra-lives.c (bb_killed_pseudos, bb_gen_pseudos): Move up.
+	(make_hard_regno_born, make_hard_regno_dead): Update
+	bb_killed_pseudos and bb_gen_pseudos for fixed regs.
+
+2017-10-16  Jeff Law  <law@redhat.com>
+
+	* tree-ssa-dse.c (live_bytes_read): Fix thinko.
+
+2017-10-16  Jan Hubicka  <hubicka@ucw.cz>
+
+	* x86-tune-costs.h (znver1_cost): Fix move cost tables.
+
+2017-10-16  Olivier Hainque  <hainque@adacore.com>
+
+	* gcc/config.gcc (powerpc*-*-*spe*): Pick 8548 as the default
+	with_cpu if we were configured for an e500v2 target cpu name.
+
+2017-10-16  Thomas Preud'homme  <thomas.preudhomme@arm.com>
+
+	* config/arm/arm-cpus.in (cortex-m33): Add nodsp option.
+	* doc/invoke.texi: Document +nodsp as a valid extension for
+	-mcpu=cortex-m33.
+
+2017-10-16  Martin Liska  <mliska@suse.cz>
+
+	* sbitmap.c (bitmap_bit_in_range_p_checking): New function.
+	(test_set_range): Likewise.
+	(test_range_functions): Rename to ...
+	(test_bit_in_range): ... this.
+	(sbitmap_c_tests): Add new test.
+
+2017-10-16  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/aarch64/arm_neon.h (vdot_u32, vdotq_u32, vdot_s32, vdotq_s32):
+	New.
+	(vdot_lane_u32, vdot_laneq_u32, vdotq_lane_u32, vdotq_laneq_u32): New.
+	(vdot_lane_s32, vdot_laneq_s32, vdotq_lane_s32, vdotq_laneq_s32): New.
+
+2017-10-16  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/aarch64/aarch64-builtins.c
+	(aarch64_types_quadopu_lane_qualifiers): New.
+	(TYPES_QUADOPU_LANE): New.
+	* config/aarch64/aarch64-simd.md (aarch64_<sur>dot<vsi2qi>): New.
+	(<sur>dot_prod<vsi2qi>, aarch64_<sur>dot_lane<vsi2qi>): New.
+	(aarch64_<sur>dot_laneq<vsi2qi>): New.
+	* config/aarch64/aarch64-simd-builtins.def (sdot, udot): New.
+	(sdot_lane, udot_lane, sdot_laneq, udot_laneq): New.
+	* config/aarch64/iterators.md (sur): Add UNSPEC_SDOT, UNSPEC_UDOT.
+	(Vdottype, DOTPROD): New.
+	(sur): Add SDOT and UDOT.
+
+2017-10-16  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/aarch64/aarch64.h (AARCH64_FL_DOTPROD): New.
+	(AARCH64_ISA_DOTPROD, TARGET_DOTPROD): New.
+	* config/aarch64/aarch64-c.c (aarch64_update_cpp_builtins):
+	Add TARGET_DOTPROD.
+	* config/aarch64/aarch64-option-extensions.def (dotprod): New.
+	* config/aarch64/aarch64-cores.def (cortex-a55, cortex-a75):
+	Enable TARGET_DOTPROD.
+	(cortex-a75.cortex-a55): Likewise.
+	* doc/invoke.texi (aarch64-feature-modifiers): Document dotprod.
+
+2017-10-16  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/arm/arm-builtins.c (arm_unsigned_uternop_qualifiers): New.
+	(UTERNOP_QUALIFIERS, arm_umac_lane_qualifiers, UMAC_LANE_QUALIFIERS):
+	New.
+	* config/arm/arm_neon_builtins.def (sdot, udot, sdot_lane, udot_lane):
+	New.
+	* config/arm/iterators.md (DOTPROD, VSI2QI, vsi2qi): New.
+	(UNSPEC_DOT_S, UNSPEC_DOT_U, opsuffix): New.
+	* config/arm/neon.md (neon_<sup>dot<vsi2qi>): New.
+	(neon_<sup>dot_lane<vsi2qi>, <sup>dot_prod<vsi2qi>): New.
+	* config/arm/types.md (neon_dot, neon_dot_q): New.
+	* config/arm/unspecs.md (sup): Add UNSPEC_DOT_S, UNSPEC_DOT_U.
+
+2017-10-16  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/arm/arm.h (TARGET_DOTPROD): New.
+	* config/arm/arm.c (arm_arch_dotprod): New.
+	(arm_option_reconfigure_globals): Add arm_arch_dotprod.
+	* config/arm/arm-c.c (__ARM_FEATURE_DOTPROD): New.
+	* config/arm/arm-cpus.in (armv8.2-a): Enabled +dotprod.
+	(feature dotprod, group dotprod, ALL_SIMD_INTERNAL): New.
+	(ALL_FPU_INTERNAL): Use ALL_SIMD_INTERNAL.
+	* config/arm/t-multilib (v8_2_a_simd_variants): Add dotprod.
+	* doc/invoke.texi (armv8.2-a): Document dotprod
+
+2017-10-14  Jan Hubicka  <hubicka@ucw.cz>
+
+	* i386.c (ix86_vec_cost): New function.
+	(ix86_rtx_costs): Handle vector operations better.
+	* i386.h (struct processor_costs): Add sse_op, fmasd, fmass.
+	* x86-tune-costs.h: Add new costs to all tables.
+
+2017-10-14  Jan Hubicka  <hubicka@ucw.cz>
+
+	* i386.c (ix86_rtx_costs): Make difference between x87 and SSE
+	operations.
+	* i386.h (struct processor_costs): Add addss, mulss, mulsd, divss,
+	divsd, sqrtss and sqrtsd
+	* x86-tune-costs.h: Add new entries to all costs.
+	(znver1_cost): Fix to match real instruction latencies.
+
+2017-10-14  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>
+	    Michael Collison <michael.collison@arm.com>
+
+	* compare-elim.c: Include emit-rtl.h.
+	(can_merge_compare_into_arith): New function.
+	(try_validate_parallel): Likewise.
+	(try_merge_compare): Likewise.
+	(try_eliminate_compare): Call the above when no previous clobber
+	is available.
+	(execute_compare_elim_after_reload): Add DF_UD_CHAIN and DF_DU_CHAIN
+	dataflow problems.
+
+2017-10-14  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/62263
+	PR middle-end/82498
+	* tree-ssa-phiopt.c (value_replacement): Comment fix.  Handle
+	up to 2 preparation statements for ASSIGN in MIDDLE_BB.
+
+	PR middle-end/62263
+	PR middle-end/82498
+	* tree-ssa-forwprop.c (simplify_rotate): Allow def_arg1[N]
+	to be any operand_equal_p operands.  For & (B - 1) require
+	B to be power of 2.  Recognize
+	(X << (Y & (B - 1))) | (X >> ((-Y) & (B - 1))) and similar patterns.
+
+2017-10-14  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR bootstrap/82553
+	* optabs.c (expand_memory_blockage): Fix call of
+	targetm.have_memory_blockage.
+
+2017-10-14  Jakub Jelinek  <jakub@redhat.com>
+
+	PR bootstrap/82548
+	* config.gcc (*-*-solaris2*, i[34567]86-*-cygwin*,
+	x86_64-*-cygwin*, i[34567]86-*-mingw* | x86_64-*-mingw*): Append
+	objects to extra_objs instead of overwriting it.
+
+2017-10-14  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/sync.md (FILD_ATOMIC/FIST_ATOMIC FP load peephole2):
+	Use any_fp_register_operand as operand[3] predicate.  Simplify
+	equality test for operands[2] and operands[4] memory location.
+	(LDX_ATOMIC/STX_ATOMIC FP load peephole2): Ditto.
+	(FILD_ATOMIC/FIST_ATOMIC FP load peephole2 with mem blockage): New.
+	(LDX_ATOMIC/LDX_ATOMIC FP load peephole2 with mem blockage): Ditto.
+	(FILD_ATOMIC/FIST_ATOMIC FP store peephole2): Use
+	any_fp_register_operand as operand[1] predicate.  Simplify
+	equality test for operands[0] and operands[3] memory location.
+	(LDX_ATOMIC/STX_ATOMIC FP store peephole2): Ditto.
+	(FILD_ATOMIC/FIST_ATOMIC FP store peephole2 with mem blockage): New.
+	(LDX_ATOMIC/LDX_ATOMIC FP storepeephole2 with mem blockage): Ditto.
+
+2017-10-14  Uros Bizjak  <ubizjak@gmail.com>
+
+	* target-insns.def: Add memory_blockage.
+	* optabs.c (expand_memory_blockage): New function.
+	(expand_asm_memory_barrier): Rename ...
+	(expand_asm_memory_blockage): ... to this.
+	(expand_mem_thread_fence): Call expand_memory_blockage
+	instead of expand_asm_memory_barrier.
+	(expand_mem_singnal_fence): Ditto.
+	(expand_atomic_load): Ditto.
+	(expand_atomic_store): Ditto.
+	* doc/md.texi (Standard Pattern Names For Generation):
+	Document memory_blockage instruction pattern.
+
+2017-10-13  Sebastian Perta  <sebastian.perta@renesas.com>
+
+	* config/rl78/rl78.c (rl78_emit_libcall): New function.
+	* config/rl78/rl78-protos.h (rl78_emit_libcall): New function.
+	* config/rl78/rl78.md: New define_expand "adddi3".
+
+2017-10-13  Jan Hubicka  <hubicka@ucw.cz>
+
+	* cfghooks.c (verify_flow_info): Disable check that all probabilities
+	are set correctly.
+
+2017-10-13  Jeff Law  <law@redhat.com>
+
+	* tree-ssa-reassoc.c (reassociate_bb): Clarify code slighly.
+
+2017-10-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82274
+	* internal-fn.c (expand_mul_overflow): If both operands have
+	the same highpart of -1 or 0 and the topmost bit of lowpart
+	is different, overflow is if res <= 0 rather than res < 0.
+
+2017-10-13  Pat Haugen  <pthaugen@us.ibm.com>
+
+	* config/rs6000/rs6000.c (rs6000_builtin_vectorization_cost): Remove
+	TARGET_P9_VECTOR code for unaligned_load case.
+
+2017-10-13  Jan Hubicka  <hubicka@ucw.cz>
+
+	* cfghooks.c (verify_flow_info): Check that edge probabilities are set.
+
+2017-10-13  Nathan Sidwell  <nathan@acm.org>
+
+	* tree-core.h (tree_contains_struct): Make bool.
+	* tree.c (tree_contains_struct): Likewise.
+	* tree.h (MARK_TS_BASE): Remove do ... while (0) idiom.
+	(MARK_TS_TYPED, MARK_TS_COMMON, MARK_TS_TYPE_COMMON,
+	MARK_TS_TYPE_WITH_LANG_SPECIFIC, MARK_TS_DECL_MINIMAL,
+	MARK_TS_DECL_COMMON, MARK_TS_DECL_WRTL, MARK_TS_DECL_WITH_VIS,
+	MARK_TS_DECL_NON_COMMON): Likewise, use comma operator.
+
+2017-10-13  Richard Biener  <rguenther@suse.de>
+
+	* graphite-isl-ast-to-gimple.c
+	(translate_isl_ast_to_gimple::get_rename_from_scev): Remove unused
+	parameters and dominance check.
+	(translate_isl_ast_to_gimple::graphite_copy_stmts_from_block): Adjust.
+	(translate_isl_ast_to_gimple::copy_bb_and_scalar_dependences): Likewise.
+	(translate_isl_ast_to_gimple::graphite_regenerate_ast_isl):
+	Do not update SSA form here or do intermediate IL verification.
+	* graphite.c: Include tree-ssa.h and tree-into-ssa.h.
+	(graphite_initialize): Remove check on the number of loops in
+	the function and inline into graphite_transform_loops.
+	(graphite_finalize): Inline into graphite_transform_loops.
+	(graphite_transform_loops): Perform SSA update and IL verification
+	here.
+	* params.def (PARAM_GRAPHITE_MIN_LOOPS_PER_FUNCTION): Remove.
+
+2017-10-13  Richard Biener  <rguenther@suse.de>
+
+	* graphite-isl-ast-to-gimple.c (max_mode_int_precision,
+	graphite_expression_type_precision): Avoid global constructor
+	by moving ...
+	(translate_isl_ast_to_gimple::translate_isl_ast_to_gimple): Here.
+	(translate_isl_ast_to_gimple::graphite_expr_type): Add type member.
+	(translate_isl_ast_to_gimple::translate_isl_ast_node_for): Use it.
+	(translate_isl_ast_to_gimple::build_iv_mapping): Likewise.
+	(translate_isl_ast_to_gimple::graphite_create_new_guard): Likewise.
+	* graphite-sese-to-poly.c (build_original_schedule): Return nothing.
+
+2017-10-13  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/82499
+	* config/i386/i386.h (ix86_red_zone_size): New.
+	* config/i386/i386.md (push peephole2s): Replace
+	"!ix86_using_red_zone ()" with "ix86_red_zone_size == 0".
+
+2017-10-13  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* combine.c (can_change_dest_mode): Reject changes in
+	REGMODE_NATURAL_SIZE.
+
+2017-10-13  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* cfgexpand.c (expand_debug_expr): Use GET_MODE_UNIT_BITSIZE.
+	(expand_debug_source_expr): Likewise.
+	* combine.c (combine_simplify_rtx): Likewise.
+	* cse.c (fold_rtx): Likewise.
+	* fwprop.c (canonicalize_address): Likewise.
+	* targhooks.c (default_shift_truncation_mask): Likewise.
+
+2017-10-13  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* optabs.c (add_equal_note): Use GET_MODE_UNIT_SIZE.
+	(widened_mode): Likewise.
+	(expand_unop): Likewise.
+	* ree.c (transform_ifelse): Likewise.
+	(merge_def_and_ext): Likewise.
+	(combine_reaching_defs): Likewise.
+	* simplify-rtx.c (simplify_unary_operation_1): Likewise.
+
+2017-10-13  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* caller-save.c (replace_reg_with_saved_mem): Use byte_lowpart_offset.
+	* combine.c (gen_lowpart_for_combine): Likewise.
+	* dwarf2out.c (rtl_for_decl_location): Likewise.
+	* final.c (alter_subreg): Likewise.
+	* rtlhooks.c (gen_lowpart_general): Likewise.
+	(gen_lowpart_if_possible): Likewise.
+
+2017-10-13  Richard Sandiford  <richard.sandiford@linaro.org>
+	    Alan Hayward  <alan.hayward@arm.com>
+	    David Sherwood  <david.sherwood@arm.com>
+
+	* calls.c (expand_call): Use subreg_lowpart_offset.
+	* cse.c (cse_insn): Likewise.
+	* regcprop.c (copy_value): Likewise.
+	(copyprop_hardreg_forward_1): Likewise.
+
 2017-10-13  Jakub Jelinek  <jakub@redhat.com>
 
 	PR target/82524
@@ -185,7 +3546,7 @@
 
 2017-10-12  Jan Hubicka  <hubicka@ucw.cz>
 
-	* x86-tune-sched.c (ix86_adjust_cost): Fix Zen support.
+	* config/i386/x86-tune-sched.c (ix86_adjust_cost): Fix Zen support.
 
 2017-10-12  Uros Bizjak  <ubizjak@gmail.com>
 
@@ -385,7 +3746,7 @@
 2017-10-11  Jan Hubicka  <hubicka@ucw.cz>
 
 	* config.gcc (i386, x86_64): Add extra objects.
-	* i386/i386-protos.h (ix86_rip_relative_addr_p): Declare.
+	* config/i386/i386-protos.h (ix86_rip_relative_addr_p): Declare.
 	(ix86_min_insn_size): Declare.
 	(ix86_issue_rate): Declare.
 	(ix86_adjust_cost): Declare.
@@ -396,7 +3757,7 @@
 	(ix86_bd_do_dispatch): Declare.
 	(ix86_core2i7_init_hooks): Declare.
 	(ix86_atom_sched_reorder): Declare.
-	* i386/i386.c Move all CPU cost tables to x86-tune-costs.h.
+	* config/i386/i386.c Move all CPU cost tables to x86-tune-costs.h.
 	(COSTS_N_BYTES): Move to x86-tune-costs.h.
 	(DUMMY_STRINGOP_ALGS):Move to x86-tune-costs.h.
 	(rip_relative_addr_p): Rename to ...
@@ -467,12 +3828,12 @@
 	(debug_ready_dispatch): Move to ix86-tune-sched-bd.c.
 	(do_dispatch): Move to ix86-tune-sched-bd.c.
 	(has_dispatch): Move to ix86-tune-sched-bd.c.
-	* i386/t-i386: Add new object files.
-	* i386/x86-tune-costs.h: New file.
-	* i386/x86-tune-sched-atom.c: New file.
-	* i386/x86-tune-sched-bd.c: New file.
-	* i386/x86-tune-sched-core.c: New file.
-	* i386/x86-tune-sched.c: New file.
+	* config/i386/t-i386: Add new object files.
+	* config/i386/x86-tune-costs.h: New file.
+	* config/i386/x86-tune-sched-atom.c: New file.
+	* config/i386/x86-tune-sched-bd.c: New file.
+	* config/i386/x86-tune-sched-core.c: New file.
+	* config/i386/x86-tune-sched.c: New file.
 
 2017-10-11  Liu Hao  <lh_mouse@126.com>
 
@@ -973,12 +4334,12 @@
 
 2017-10-08  Jan Hubicka  <hubicka@ucw.cz>
 
-	* i386.c (ix86_expand_set_or_movmem): Disable 512bit loops for targets
-	that preffer 128bit.
+	* config/i386/i386.c (ix86_expand_set_or_movmem): Disable 512bit loops
+	for targets that preffer 128bit.
 
 2017-10-08  Jan Hubicka  <hubicka@ucw.cz>
 
-	* i386.c (has_dispatch): Disable for Ryzen.
+	* config/i386/i386.c (has_dispatch): Disable for Ryzen.
 
 2017-10-08  Olivier Hainque  <hainque@adacore.com>
 
@@ -1175,8 +4536,8 @@
 
 2017-10-05  Jan Hubicka <hubicka@ucw.cz>
 
-	* i386.c (ia32_multipass_dfa_lookahead): Default to issue rate
-	for post-reload scheduling.
+	* config/i386/i386.c (ia32_multipass_dfa_lookahead): Default to issue
+	rate for post-reload scheduling.
 
 2017-10-05  Tamar Christina  <tamar.christina@arm.com>
 
@@ -1184,13 +4545,13 @@
 
 2017-10-05  Jan Hubicka <hubicka@ucw.cz>
 
-	* i386.c (znver1_cost): Set branch_cost to 3 (instead of 2)
+	* config/i386/i386.c (znver1_cost): Set branch_cost to 3 (instead of 2)
 	to improve monte carlo in scimark.
 
 2017-10-05  Jan Hubicka <hubicka@ucw.cz>
 
-	* i386.c (ix86_size_cost, i386_cost, i486_cost, pentium_cost,
-	lakemont_cost, pentiumpro_cost, geode_cost, k6_cost,
+	* config/i386/i386.c (ix86_size_cost, i386_cost, i486_cost,
+	pentium_cost, lakemont_cost, pentiumpro_cost, geode_cost, k6_cost,
 	athlon_cost, k8_cost, amdfam10_cost, btver1_cost, btver2_cost,
 	pentium4_cost, nocona_cost): Set reassociation width to 1.
 	(bdver1_cost, bdver2_cost, bdver3_cost, bdver4_cost): Set reassociation
@@ -1205,7 +4566,7 @@
 	(ix86_reassociation_width): Rewrite using cost table; special case
 	plus/minus on Zen; honor X86_TUNE_SSE_SPLIT_REGS
 	and TARGET_AVX128_OPTIMAL.
-	* i386.h (processor_costs): Add
+	* config/i386/i386.h (processor_costs): Add
 	reassoc_int, reassoc_fp, reassoc_vec_int, reassoc_vec_fp.
 	(TARGET_VECTOR_PARALLEL_EXECUTION, TARGET_REASSOC_INT_TO_PARALLEL,
 	TARGET_REASSOC_FP_TO_PARALLEL): Remove.
@@ -25475,11 +28836,6 @@
 	* doc/invoke.texi: Replace inequality signs with square brackets
 	for -Wnormalized.
 
-2017-02-22  Bill Schmidt  <wschmidt@linux.vnet.ibm.com>
-
-	PR tree-optimization/68644
-	* gcc.dg/tree-ssa/ivopts-lt-2.c: Skip for powerpc*-*-*.
-
 2017-02-22  Matthew Fortune  <matthew.fortune@imgtec.com>
 
 	PR target/78660
@@ -27052,8 +30408,6 @@
 	* tree-vrp.c (process_assert_insertions): Properly adjust common
 	when removing a duplicate.
 
-	* gcc.dg/torture/pr79276.c: New testcase.
-
 2017-01-30  Richard Biener  <rguenther@suse.de>
 
 	PR tree-optimization/79256
diff --git a/gcc/DATESTAMP b/gcc/DATESTAMP
index 96354492ed7..2c700d42332 100644
--- a/gcc/DATESTAMP
+++ b/gcc/DATESTAMP
@@ -1 +1 @@
-20171013
+20171104
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index 0f7110d227a..7e23a230793 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1161,6 +1161,7 @@ FLAGS_TO_PASS = \
 	"libexecsubdir=$(libexecsubdir)" \
 	"datarootdir=$(datarootdir)" \
 	"datadir=$(datadir)" \
+	"libsubdir=$(libsubdir)" \
 	"localedir=$(localedir)"
 #
 # Lists of files for various purposes.
@@ -1447,7 +1448,6 @@ OBJS = \
 	sched-deps.o \
 	sched-ebb.o \
 	sched-rgn.o \
-	sdbout.o \
 	sel-sched-ir.o \
 	sel-sched-dump.o \
 	sel-sched.o \
@@ -1569,6 +1569,7 @@ OBJS = \
 	tree-vrp.o \
 	tree.o \
 	typed-splay-tree.o \
+	unique-ptr-tests.o \
 	valtrack.o \
 	value-prof.o \
 	var-tracking.o \
@@ -1591,7 +1592,7 @@ OBJS-libcommon = diagnostic.o diagnostic-color.o diagnostic-show-locus.o \
 	pretty-print.o intl.o \
 	sbitmap.o \
 	vec.o input.o version.o hash-table.o ggc-none.o memory-block.o \
-	selftest.o
+	selftest.o selftest-diagnostic.o
 
 # Objects in libcommon-target.a, used by drivers and by the core
 # compiler and containing target-dependent code.
@@ -2525,7 +2526,7 @@ GTFILES = $(CPP_ID_DATA_H) $(srcdir)/input.h $(srcdir)/coretypes.h \
   $(srcdir)/lists.c $(srcdir)/optabs-libfuncs.c \
   $(srcdir)/profile.c $(srcdir)/mcf.c \
   $(srcdir)/reg-stack.c $(srcdir)/cfgrtl.c \
-  $(srcdir)/sdbout.c $(srcdir)/stor-layout.c \
+  $(srcdir)/stor-layout.c \
   $(srcdir)/stringpool.c $(srcdir)/tree.c $(srcdir)/varasm.c \
   $(srcdir)/gimple.h \
   $(srcdir)/gimple-ssa.h \
diff --git a/gcc/acinclude.m4 b/gcc/acinclude.m4
index dbc0ba7e003..da4ddfd39ed 100644
--- a/gcc/acinclude.m4
+++ b/gcc/acinclude.m4
@@ -277,8 +277,7 @@ fi
 fi])
 
 AC_DEFUN([gcc_AC_INITFINI_ARRAY],
-[AC_REQUIRE([gcc_SUN_LD_VERSION])dnl
-AC_ARG_ENABLE(initfini-array,
+[AC_ARG_ENABLE(initfini-array,
 	[  --enable-initfini-array	use .init_array/.fini_array sections],
 	[], [
 AC_CACHE_CHECK(for .preinit_array/.init_array/.fini_array support,
@@ -556,43 +555,6 @@ if test $[$2] != yes; then
   $8
 fi])])
 
-dnl gcc_SUN_LD_VERSION
-dnl
-dnl Determines Sun linker version numbers, setting gcc_cv_sun_ld_vers to
-dnl the complete version number and gcc_cv_sun_ld_vers_{major, minor} to
-dnl the corresponding fields.
-dnl
-dnl ld and ld.so.1 are guaranteed to be updated in lockstep, so ld version
-dnl numbers can be used in ld.so.1 feature checks even if a different
-dnl linker is configured.
-dnl
-AC_DEFUN([gcc_SUN_LD_VERSION],
-[changequote(,)dnl
-if test "x${build}" = "x${target}" && test "x${build}" = "x${host}"; then
-  case "${target}" in
-    *-*-solaris2*)
-      #
-      # Solaris 2 ld -V output looks like this for a regular version:
-      #
-      # ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1699
-      #
-      # but test versions add stuff at the end:
-      #
-      # ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1701:onnv-ab196087-6931056-03/25/10
-      #
-      gcc_cv_sun_ld_ver=`/usr/ccs/bin/ld -V 2>&1`
-      if echo "$gcc_cv_sun_ld_ver" | grep 'Solaris Link Editors' > /dev/null; then
-	gcc_cv_sun_ld_vers=`echo $gcc_cv_sun_ld_ver | sed -n \
-	  -e 's,^.*: 5\.[0-9][0-9]*-\([0-9]\.[0-9][0-9]*\).*$,\1,p'`
-	gcc_cv_sun_ld_vers_major=`expr "$gcc_cv_sun_ld_vers" : '\([0-9]*\)'`
-	gcc_cv_sun_ld_vers_minor=`expr "$gcc_cv_sun_ld_vers" : '[0-9]*\.\([0-9]*\)'`
-      fi
-      ;;
-  esac
-fi
-changequote([,])dnl
-])
-
 dnl GCC_TARGET_TEMPLATE(KEY)
 dnl ------------------------
 dnl Define KEY as a valid configure key on the target machine.
diff --git a/gcc/ada/ChangeLog b/gcc/ada/ChangeLog
index 3e1f53762c0..6e2a7ffd099 100644
--- a/gcc/ada/ChangeLog
+++ b/gcc/ada/ChangeLog
@@ -1,3 +1,501 @@
+2017-10-31  Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR ada/82785
+	* gcc-interface/Makefile.in (m68k/Linux): Fix typo.
+
+2017-10-21  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/Makefile.in: Remove bogus settings for VxWorks.
+
+2017-10-21  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/utils.c (pad_type_hash): Use hashval_t for hash value.
+	(convert): Do not use an unchecked conversion for converting from a
+	type to another type padding it.
+
+2017-10-20  Doug Rupp  <rupp@adacore.com>
+
+	* libgnarl/s-osinte__linux.ads (Relative_Timed_Wait): Add variable
+	needed for using monotonic clock.
+	* libgnarl/s-taprop__linux.adb: Revert previous monotonic clock
+	changes.
+	* libgnarl/s-taprop__linux.adb, s-taprop__posix.adb: Unify and factor
+	out monotonic clock related functions body.
+	(Timed_Sleep, Timed_Delay, Montonic_Clock, RT_Resolution,
+	Compute_Deadline): Move to...
+	* libgnarl/s-tpopmo.adb: ... here. New separate package body.
+
+2017-10-20  Ed Schonberg  <schonberg@adacore.com>
+
+	* sem_util.adb (Is_Controlling_Limited_Procedure): Handle properly the
+	case where the controlling formal is an anonymous access to interface
+	type.
+	* exp_ch9.adb (Extract_Dispatching_Call): If controlling actual is an
+	access type, handle properly the the constructed dereference that
+	designates the object used in the rewritten synchronized call.
+	(Parameter_Block_Pack): If the type of the actual is by-copy, its
+	generated declaration in the parameter block does not need an
+	initialization even if the type is a null-excluding access type,
+	because it will be initialized with the value of the actual later on.
+	(Parameter_Block_Pack): Do not add controlling actual to parameter
+	block when its type is by-copy.
+
+2017-10-20  Justin Squirek  <squirek@adacore.com>
+
+	* sem_ch8.adb (Update_Use_Clause_Chain): Add sanity check to verify
+	scope stack traversal into the context clause.
+
+2017-10-20  Bob Duff  <duff@adacore.com>
+
+	* sinfo.ads: Fix a comment typo.
+
+2017-10-20  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* doc/gnat_ugn/building_executable_programs_with_gnat.rst (-flto): Add
+	warning against usage in conjunction with -gnatn.
+	(-fdump-xref): Delete entry.
+	* doc/gnat_ugn/gnat_utility_programs.rst (--ext): Remove mention of
+	-fdump-xref switch.
+	* gnat_ugn.texi: Regenerate.
+
+2017-10-20  Hristian Kirtchev  <kirtchev@adacore.com>
+
+	* sem_type.adb, exp_util.adb, sem_util.adb, sem_dim.adb, sem_elab.adb:
+	Minor reformatting.
+
+2017-10-20  Yannick Moy  <moy@adacore.com>
+
+	* sem_dim.adb (Analyze_Dimension_Binary_Op): Accept with a warning to
+	compare a dimensioned expression with a literal.
+	(Dim_Warning_For_Numeric_Literal): Do not issue a warning for the
+	special value zero.
+	* doc/gnat_ugn/gnat_and_program_execution.rst: Update description of
+	dimensionality system in GNAT.
+	* gnat_ugn.texi: Regenerate.
+
+2017-10-20  Yannick Moy  <moy@adacore.com>
+
+	* sem_ch6.adb (Analyze_Expression_Function.Freeze_Expr_Types): Remove
+	inadequate silencing of errors.
+	* sem_util.adb (Check_Part_Of_Reference): Do not issue an error when
+	checking the subprogram body generated from an expression function,
+	when this is done as part of the preanalysis done on expression
+	functions, as the subprogram body may not yet be attached in the AST.
+	The error if any will be issued later during the analysis of the body.
+	(Is_Aliased_View): Trivial rewrite with Is_Formal_Object.
+
+2017-10-20  Arnaud Charlet  <charlet@adacore.com>
+
+	* sem_ch8.adb (Update_Chain_In_Scope): Add missing [-gnatwu] marker for
+	warning on ineffective use clause.
+
+2017-10-20  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_ch11.ads (Warn_If_No_Local_Raise): Declare.
+	* exp_ch11.adb (Expand_Exception_Handlers): Use Warn_If_No_Local_Raise
+	to issue the warning on the absence of local raise.
+	(Possible_Local_Raise): Do not issue the warning for Call_Markers.
+	(Warn_If_No_Local_Raise): New procedure to issue the warning on the
+	absence of local raise.
+	* sem_elab.adb: Add with and use clauses for Exp_Ch11.
+	(Record_Elaboration_Scenario): Call Possible_Local_Raise in the cases
+	where a scenario could give rise to raising Program_Error.
+	* sem_elab.adb: Typo fixes.
+	* fe.h (Warn_If_No_Local_Raise): Declare.
+	* gcc-interface/gigi.h (get_exception_label): Change return type.
+	* gcc-interface/trans.c (gnu_constraint_error_label_stack): Change to
+	simple vector of Entity_Id.
+	(gnu_storage_error_label_stack): Likewise.
+	(gnu_program_error_label_stack): Likewise.
+	(gigi): Adjust to above changes.
+	(Raise_Error_to_gnu): Likewise.
+	(gnat_to_gnu) <N_Goto_Statement>: Set TREE_USED on the label.
+	(N_Push_Constraint_Error_Label): Push the label onto the stack.
+	(N_Push_Storage_Error_Label): Likewise.
+	(N_Push_Program_Error_Label): Likewise.
+	(N_Pop_Constraint_Error_Label): Pop the label from the stack and issue
+	a warning on the absence of local raise.
+	(N_Pop_Storage_Error_Label): Likewise.
+	(N_Pop_Program_Error_Label): Likewise.
+	(push_exception_label_stack): Delete.
+	(get_exception_label): Change return type to Entity_Id and adjust.
+	* gcc-interface/utils2.c (build_goto_raise): Change type of first
+	parameter to Entity_Id and adjust.  Set TREE_USED on the label.
+	(build_call_raise): Adjust calls to get_exception_label and also
+	build_goto_raise.
+	(build_call_raise_column): Likewise.
+	(build_call_raise_range): Likewise.
+	* doc/gnat_ugn/building_executable_programs_with_gnat.rst (-gnatw.x):
+	Document actual default behavior.
+
+2017-10-20  Piotr Trojanek  <trojanek@adacore.com>
+
+	* einfo.ads: Minor consistent punctuation in comment.  All numbered
+	items in the comment of Is_Internal are now terminated with a period.
+
+2017-10-20  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_util.adb (Build_Temporary): Mark created temporary entity as
+	internal.
+
+2017-10-20  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_type.adb (In_Generic_Actual): Simplified.
+
+2017-10-20  Justin Squirek  <squirek@adacore.com>
+
+	* sem_ch12.adb (Check_Formal_Package_Instance): Add sanity check to
+	verify a renaming exists for a generic formal before comparing it to
+	the actual as defaulted formals will not have a renamed_object.
+
+2017-10-20  Javier Miranda  <miranda@adacore.com>
+
+	* exp_ch6.adb (Replace_Returns): Fix wrong management of
+	N_Block_Statement nodes.
+
+2017-10-20  Bob Duff  <duff@adacore.com>
+
+	* exp_aggr.adb (Initialize_Array_Component): Avoid adjusting a
+	component of an array aggregate if it is initialized by a
+	build-in-place function call.
+	* exp_ch6.adb (Is_Build_In_Place_Result_Type): Use -gnatd.9 to disable
+	bip for nonlimited types.
+	* debug.adb: Document -gnatd.9.
+
+2017-10-20  Bob Duff  <duff@adacore.com>
+
+	* sem_ch12.adb: Remove redundant setting of Parent.
+
+2017-10-20  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sem_ch4.adb (Find_Concatenation_Types): Filter out operators if one
+	of the operands is a string literal.
+
+2017-10-20  Bob Duff  <duff@adacore.com>
+
+	* einfo.ads: Comment fix.
+
+2017-10-20  Clement Fumex  <fumex@adacore.com>
+
+	* switch-c.adb: Remove -gnatwm from the switches triggered by -gnateC.
+
+2017-10-20  Ed Schonberg  <schonberg@adacore.com>
+
+	* sem_dim.adb (Extract_Power): Accept dimension values that are not
+	non-negative integers when the dimensioned base type is an Integer
+	type.
+
+2017-10-20  Bob Duff  <duff@adacore.com>
+
+	* sinfo.ads, sinfo.adb (Alloc_For_BIP_Return): New flag to indicate
+	that an allocator came from a b-i-p return statement.
+	* exp_ch4.adb (Expand_Allocator_Expression): Avoid adjusting the return
+	object of a nonlimited build-in-place function call.
+	* exp_ch6.adb (Expand_N_Extended_Return_Statement): Set the
+	Alloc_For_BIP_Return flag on generated allocators.
+	* sem_ch5.adb (Analyze_Assignment): Move Assert to where it can't fail.
+	If the N_Assignment_Statement has been transformed into something else,
+	then Should_Transform_BIP_Assignment won't work.
+	* exp_ch3.adb (Expand_N_Object_Declaration): A previous revision said,
+	"Remove Adjust if we're building the return object of an extended
+	return statement in place." Back out that part of the change, because
+	the Alloc_For_BIP_Return flag is now used for that.
+
+2017-10-19  Bob Duff  <duff@adacore.com>
+
+	* exp_ch6.adb (Is_Build_In_Place_Result_Type): Fix silly bug -- "Typ"
+	should be "T".  Handle case of a subtype of a class-wide type.
+
+2017-10-19  Bob Duff  <duff@adacore.com>
+
+	* exp_util.adb: (Process_Statements_For_Controlled_Objects): Clarify
+	which node kinds can legitimately be ignored, and raise Program_Error
+	for others.
+
+2017-10-19  Hristian Kirtchev  <kirtchev@adacore.com>
+
+	* sem_elab.adb (Compilation_Unit): Handle the case of a subprogram
+	instantiation that acts as a compilation unit.
+	(Find_Code_Unit): Reimplemented.
+	(Find_Top_Unit): Reimplemented.
+	(Find_Unit_Entity): New routine.
+	(Process_Instantiation_SPARK): Correct the elaboration requirement a
+	package instantiation imposes on a unit.
+
+2017-10-19  Bob Duff  <duff@adacore.com>
+
+	* exp_ch6.adb (Is_Build_In_Place_Result_Type): Enable build-in-place
+	for a narrow set of controlled types.
+
+2017-10-19  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sinput.ads (Line_Start): Add pragma Inline.
+	* widechar.ads (Is_Start_Of_Wide_Char): Likewise.
+
+2017-10-19  Bob Duff  <duff@adacore.com>
+
+	* exp_attr.adb (Expand_N_Attribute_Reference): Disable
+	Make_Build_In_Place_Call_... for F(...)'Old, where F(...) is a
+	build-in-place function call so that the temp is declared in the right
+	place.
+
+2017-10-18  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/misc.c (gnat_tree_size): Move around.
+
+	* gcc-interface/utils.c (max_size): Deal with SSA names.
+
+2017-10-17  Jakub Jelinek  <jakub@redhat.com>
+
+	* gcc-interface/misc.c (gnat_tree_size): New function.
+	(LANG_HOOKS_TREE_SIZE): Redefine.
+
+2017-10-14  Hristian Kirtchev  <kirtchev@adacore.com>
+
+	* sem_elab.adb (In_Preelaborated_Context): A generic package subject to
+	Remote_Call_Interface is not a suitable preelaboratd context when the
+	call appears in the package body.
+
+2017-10-14  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* layout.ads (Set_Elem_Alignment): Add Align parameter defaulted to 0.
+	* layout.adb (Set_Elem_Alignment): Likewise.  Use M name as maximum
+	alignment for consistency.  If Align is non-zero, use the minimum of
+	Align and M for the alignment.
+	* cstand.adb (Build_Float_Type): Use Set_Elem_Alignment instead of
+	setting the alignment directly.
+
+2017-10-14  Ed Schonberg  <schonberg@adacore.com>
+
+	* sem_prag.adb (Analyze_Pragma, case Check): Defer evaluation of the
+	optional string in an Assert pragma until the expansion of the pragma
+	has rewritten it as a conditional statement, so that the string
+	argument is only evaluaed if the assertion fails. This is mandated by
+	RM 11.4.2.
+
+2017-10-14  Hristian Kirtchev  <kirtchev@adacore.com>
+
+	* debug.adb: Switch -gnatd.v and associated flag are now used to
+	enforce the SPARK rules for elaboration in SPARK code.
+	* sem_elab.adb: Describe switch -gnatd.v.
+	(Process_Call): Verify the SPARK rules only when -gnatd.v is in effect.
+	(Process_Instantiation): Verify the SPARK rules only when -gnatd.v is
+	in effect.
+	(Process_Variable_Assignment): Clarify why variable assignments are
+	processed reglardless of whether -gnatd.v is in effect.
+	* doc/gnat_ugn/elaboration_order_handling_in_gnat.rst: Update the
+	sections on elaboration code and compilation switches.
+	* gnat_ugn.texi: Regenerate.
+
+2017-10-14  Gary Dismukes  <dismukes@adacore.com>
+
+	* exp_util.adb, freeze.adb, sem_aggr.adb, sem_util.ads, sem_util.adb,
+	sem_warn.adb: Minor reformattings.
+
+2017-10-14  Ed Schonberg  <schonberg@adacore.com>
+
+	* doc/gnat_rm/implementation_defined_aspects.rst: Add documentation
+	for reverse iteration over formal containers.
+	* gnat_rm.texi: Regenerate.
+
+2017-10-14  Hristian Kirtchev  <kirtchev@adacore.com>
+
+	* sem_elab.adb (Ensure_Dynamic_Prior_Elaboration): Renamed to
+	Ensure_Prior_Elaboration_Dynamic for consistency reasons.
+	(Ensure_Static_Prior_Elaboration): Renamed to
+	Ensure_Prior_Elaboration_Static for consistency reasons.
+	(Info_Variable_Reference): Renamed to Info_Variable_Read in order to
+	reflect its new purpose.
+	(Is_Initialized): New routine.
+	(Is_Suitable_Variable_Reference): Renamed to Is_Suitable_Variable_Read
+	in order to reflect its new purpose.
+	(Is_Variable_Read): New routine.
+	(Output_Variable_Reference): Renamed to Output_Variable_Read in order
+	to reflect its new purpose.
+	(Process_Variable_Assignment): This routine now acts as a top level
+	dispatcher for variable assignments.
+	(Process_Variable_Assignment_Ada): New routine.
+	(Process_Variable_Assignment_SPARK): New routine.
+	(Process_Variable_Reference): Renamed to Process_Variable_Read in order
+	to reflects its new purpose. A reference to a variable is now suitable
+	for ABE processing only when it is a read. The logic in the routine now
+	reflects the latest SPARK elaboration rules.
+
+2017-10-14  Justin Squirek  <squirek@adacore.com>
+
+	* sem_ch8.adb (Analyze_Subprogram_Renaming): Modify condition that
+	triggers marking on formal subprograms.
+
+2017-10-14  Javier Miranda  <miranda@adacore.com>
+
+	* checks.adb (Ensure_Valid): Do not skip adding the validity check on
+	renamings of objects that come from the sources.
+
+2017-10-14  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* cstand.adb (Build_Float_Type): Move down Siz parameter, add Align
+	parameter and set the alignment of the type to Align.
+	(Copy_Float_Type): Adjust call to Build_Float_Type.
+	(Register_Float_Type): Add pragma Unreferenced for Precision.  Adjust
+	call to Build_Float_Type and do not set RM_Size and Alignment.
+
+2017-10-14  Patrick Bernardi  <bernardi@adacore.com>
+
+	* Makefile.rtl (GNATRTL_NONTASKING_OBJ): Add s-soliin to
+	GNATRTL_NONTASKING_OBJ.
+
+2017-10-14  Bob Duff  <duff@adacore.com>
+
+	* exp_ch6.adb (Is_Build_In_Place_Result_Type): Include code for
+	enabling b-i-p for nonlimited controlled types (but disabled).
+
+2017-10-14  Justin Squirek  <squirek@adacore.com>
+
+	* sem_elab.adb (Is_Suitable_Variable_Assignment): Replace call to
+	Has_Warnings_Off with Warnings_Off.
+
+2017-10-14  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sinfo.ads (Generic_Parent): Remove wrong (possibly obsolete) comment.
+
+2017-10-14  Hristian Kirtchev  <kirtchev@adacore.com>
+
+	* sem_ch3.adb (Analyze_Declarations): Analyze the contract of an
+	enclosing package at the end of the visible declarations.
+	* sem_prag.adb (Analyze_Initialization_Item): Suppress the analysis of
+	an initialization item which is undefined due to some illegality.
+
+2017-10-14  Patrick Bernardi  <bernardi@adacore.com>
+
+	* ali.adb: Add new ALI line 'T' to read the number of tasks contain
+	within each unit that require a default-sized primary and secondary
+	stack to be generated by the binder.
+	(Scan_ALI): Scan new 'T' lines.
+	* ali.ads: Add Primary_Stack_Count and Sec_Stack_Count to Unit_Record.
+	* bindgen.adb (Gen_Output_File): Count the number of default-sized
+	stacks within the closure that are to be created by the binder.
+	(Gen_Adainit, Gen_Output_File_Ada): Generate default-sized secondary
+	stacks and record these in System.Secodnary_Stack.
+	(Resolve_Binder_Options): Check if System.Secondary_Stack is in the
+	closure of the program being bound.
+	* bindusg.adb (Display): Add "-Q" switch. Remove rouge "--RTS" comment.
+	* exp_ch3.adb (Count_Default_Sized_Task_Stacks): New routine.
+	(Expand_N_Object_Declaration): Count the number of default-sized stacks
+	used by task objects contained within the object whose declaration is
+	being expanded.  Only performed when either the restrictions
+	No_Implicit_Heap_Allocations or No_Implicit_Task_Allocations are in
+	effect.
+	* exp_ch9.adb (Create_Secondary_Stack_For_Task): New routine.
+	(Expand_N_Task_Type_Declaration): Create a secondary stack as part of
+	the expansion of a task type if the size of the stack is known at
+	run-time and the restrictions No_Implicit_Heap_Allocations or
+	No_Implicit_Task_Allocations are in effect.
+	(Make_Task_Create_Call): If using a restricted profile provide
+	secondary stack parameter: either the statically created stack or null.
+	* lib-load.adb (Create_Dummy_Package_Unit, Load_Unit,
+	Load_Main_Source): Include Primary_Stack_Count and Sec_Stack_Count in
+	Unit_Record initialization expressions.
+	* lib-writ.adb (Add_Preprocessing_Dependency,
+	Ensure_System_Dependency): Include Primary_Stack_Count and
+	Sec_Stack_Count in Unit_Record initialization expression.
+	(Write_ALI): Write T lines.
+	(Write_Unit_Information): Do not output 'T' lines if there are no
+	stacks for the binder to generate.
+	* lib-writ.ads: Updated library information documentation to include
+	new T line entry.
+	* lib.adb (Increment_Primary_Stack_Count): New routine.
+	(Increment_Sec_Stack_Count): New routine.
+	(Primary_Stack_Count): New routine.
+	(Sec_Stack_Count): New routine.
+	* lib.ads: Add Primary_Stack_Count and Sec_Stack_Count components to
+	Unit_Record and updated documentation.
+	(Increment_Primary_Stack_Count): New routine along with pragma Inline.
+	(Increment_Sec_Stack_Count): New routine along with pragma Inline.
+	(Primary_Stack_Count): New routine along with pragma Inline.
+	(Sec_Stack_Count): New routine along with pragma Inline.
+	* opt.ads: New constant No_Stack_Size.	Flag Default_Stack_Size
+	redefined.  New flag Default_Sec_Stack_Size and
+	Quantity_Of_Default_Size_Sec_Stacks.
+	* rtfinal.c Fixed erroneous comment.
+	* rtsfind.ads: Moved RE_Default_Secondary_Stack_Size from
+	System.Secondary_Stack to System.Parameters.  Add RE_SS_Stack.
+	* sem_util.adb (Number_Of_Elements_In_Array): New routine.
+	* sem_util.ads (Number_Of_Elements_In_Array): New routine.
+	* switch-b.adb (Scan_Binder_Switches): Scan "-Q" switch.
+	* libgnarl/s-solita.adb (Get_Sec_Stack_Addr): Removed routine.
+	(Set_Sec_Stack_Addr): Removed routine.
+	(Get_Sec_Stack): New routine.
+	(Set_Sec_Stack): New routine.
+	(Init_Tasking_Soft_Links): Update System.Soft_Links reference to
+	reflect new procedure and global names.
+	* libgnarl/s-taprop__linux.adb, libgnarl/s-taprop__mingw.adb,
+	libgnarl/s-taprop__posix.adb, libgnarl/s-taprop__solaris.adb,
+	libgnarl/s-taprop__vxworks.adb (Register_Foreign_Thread): Update
+	parameter profile to allow the secondary stack size to be specified.
+	* libgnarl/s-tarest.adb (Create_Restricted_Task): Update the parameter
+	profile to include Sec_Stack_Address.  Update Tasking.Initialize_ATCB
+	call to remove Secondary_Stack_Size reference.  Add secondary stack
+	address and size to SSL.Create_TSD call.
+	(Task_Wrapper): Remove secondary stack creation.
+	* libgnarl/s-tarest.ads (Create_Restricted_Task,
+	Create_Restricted_Task_Sequential): Update parameter profile to include
+	Sec_Stack_Address and clarify the Size parameter.
+	* libgnarl/s-taskin.adb (Initialize_ATCB): Remove Secondary_Stack_Size
+	from profile and body.
+	(Initialize): Remove Secondary_Stack_Size from Initialize_ATCB call.
+	* libgnarl/s-taskin.ads: Removed component Secondary_Stack_Size from
+	Common_ATCB.
+	(Initialize_ATCB): Update the parameter profile to remove
+	Secondary_Stack_Size.
+	* libgnarl/s-tassta.adb (Create_Task): Updated parameter profile and
+	call to Initialize_ATCB.  Add secondary stack address and size to
+	SSL.Create_TSD call, and catch any storage exception from the call.
+	(Finalize_Global_Tasks): Update System.Soft_Links references to reflect
+	new subprogram and component names.
+	(Task_Wrapper): Remove secondary stack creation.
+	(Vulnerable_Complete_Master): Update to reflect TSD changes.
+	* libgnarl/s-tassta.ads: Reformat comments.
+	(Create_Task): Update parameter profile.
+	* libgnarl/s-tporft.adb (Register_Foreign_Thread): Update parameter
+	profile to include secondary stack size. Remove secondary size
+	parameter from Initialize_ATCB call and add it to Create_TSD call.
+	* libgnat/s-parame.adb, libgnat/s-parame__rtems.adb,
+	libgnat/s-parame__vxworks.adb (Default_Sec_Stack_Size): New routine.
+	* libgnat/s-parame.ads, libgnat/s-parame__ae653.ads,
+	libgnat/s-parame__hpux.ads, libgnat/s-parame__vxworks.ads: Remove type
+	Percentage.  Remove constants Dynamic, Sec_Stack_Percentage and
+	Sec_Stack_Dynamic.  Add constant Runtime_Default_Sec_Stack_Size and
+	Sec_Stack_Dynamic.
+	(Default_Sec_Stack_Size): New routine.
+	* libgnat/s-secsta.adb, libgnat/s-secsta.ads: New implementation. Is
+	now Preelaborate.
+	* libgnat/s-soflin.adb: Removed unused with-clauses.  With
+	System.Soft_Links.Initialize to initialize non-tasking TSD.
+	(Create_TSD): Update parameter profile. Initialize the TSD and
+	unconditionally call SS_Init.
+	(Destroy_TSD): Update SST.SS_Free call.
+	(Get_Sec_Stack_Addr_NT, Get_Sec_Stack_Addr_Soft, Set_Sec_Stack_Addr_NT,
+	Set_Sec_Stack_Addr_Soft): Remove routines.
+	(Get_Sec_Stack_NT, Get_Sec_Stack_Soft, Set_Sec_Stack_NT,
+	Set_Sec_Stack_Soft): Add routines.
+	(NT_TSD): Move to private part of package specification.
+	* libgnat/s-soflin.ads: New types Get_Stack_Call and Set_Stack_Call
+	with suppressed access checks.  Renamed *_Sec_Stack_Addr_* routines and
+	objects to *_Sec_Stack_*.  TSD: removed warning suppression and
+	component intialization. Changed Sec_Stack_Addr to Sec_Stack_Ptr.
+	(Create_TSD): Update parameter profile.
+	(NT_TSD): Move to private section from body.
+	* libgnat/s-soliin.adb, libgnat/s-soliin.ads: New files.
+	* libgnat/s-thread.ads (Thread_Body_Enter): Update parameter profile.
+	* libgnat/s-thread__ae653.adb (Get_Sec_Stack_Addr, Set_Sec_Stack_Addr):
+	Remove routine.
+	(Get_Sec_Stack, Set_Sec_Stack): Add routine.
+	(Thread_Body_Enter): Update parameter profile and body to adapt to new
+	System.Secondary_Stack.
+	(Init_RTS): Update body for new System.Soft_Links names.
+	* gcc-interface/Make-lang.in (GNAT_ADA_OBJS, GNATBIND_OBJS): Add
+	s-soliin.o.
+
 2017-10-10  Richard Sandiford  <richard.sandiford@linaro.org>
 
 	* gcc-interface/decl.c (annotate_value): Use wi::to_wide when
diff --git a/gcc/ada/Makefile.rtl b/gcc/ada/Makefile.rtl
index 021da824c0d..ed43ae5273c 100644
--- a/gcc/ada/Makefile.rtl
+++ b/gcc/ada/Makefile.rtl
@@ -659,6 +659,7 @@ GNATRTL_NONTASKING_OBJS= \
   s-sequio$(objext) \
   s-shasto$(objext) \
   s-soflin$(objext) \
+  s-soliin$(objext) \
   s-spsufi$(objext) \
   s-stache$(objext) \
   s-stalib$(objext) \
diff --git a/gcc/ada/ali.adb b/gcc/ada/ali.adb
index 2b1d472baba..959b3058728 100644
--- a/gcc/ada/ali.adb
+++ b/gcc/ada/ali.adb
@@ -58,6 +58,7 @@ package body ALI is
       'Z'    => True,   -- implicit with from instantiation
       'C'    => True,   -- SCO information
       'F'    => True,   -- SPARK cross-reference information
+      'T'    => True,   -- task stack information
       others => False);
 
    --------------------
@@ -842,7 +843,7 @@ package body ALI is
 
       if Read_Xref then
          Ignore :=
-           ('U' | 'W' | 'Y' | 'Z' | 'D' | 'X' => False, others => True);
+           ('T' | 'U' | 'W' | 'Y' | 'Z' | 'D' | 'X' => False, others => True);
 
       --  Read_Lines parameter given
 
@@ -1744,6 +1745,8 @@ package body ALI is
             UL.Elaborate_Body_Desirable := False;
             UL.Optimize_Alignment       := 'O';
             UL.Has_Finalizer            := False;
+            UL.Primary_Stack_Count      := 0;
+            UL.Sec_Stack_Count          := 0;
 
             if Debug_Flag_U then
                Write_Str (" ----> reading unit ");
@@ -2096,6 +2099,28 @@ package body ALI is
          Units.Table (Units.Last).Last_With := Withs.Last;
          Units.Table (Units.Last).Last_Arg  := Args.Last;
 
+         --  Scan out task stack information for the unit if present
+
+         Check_Unknown_Line;
+
+         if C = 'T' then
+            if Ignore ('T') then
+               Skip_Line;
+
+            else
+               Checkc (' ');
+               Skip_Space;
+
+               Units.Table (Units.Last).Primary_Stack_Count := Get_Nat;
+               Skip_Space;
+               Units.Table (Units.Last).Sec_Stack_Count := Get_Nat;
+               Skip_Space;
+               Skip_Eol;
+            end if;
+
+            C := Getc;
+         end if;
+
          --  If there are linker options lines present, scan them
 
          Name_Len := 0;
diff --git a/gcc/ada/ali.ads b/gcc/ada/ali.ads
index e15a1c455bd..3fa4d99fb09 100644
--- a/gcc/ada/ali.ads
+++ b/gcc/ada/ali.ads
@@ -388,11 +388,19 @@ package ALI is
       --  together as possible.
 
       Optimize_Alignment : Character;
-      --  Optimize_Alignment setting. Set to L/S/T/O for OL/OS/OT/OO present
+      --  Optimize_Alignment setting. Set to L/S/T/O for OL/OS/OT/OO present.
 
       Has_Finalizer : Boolean;
       --  Indicates whether a package body or a spec has a library-level
       --  finalization routine.
+
+      Primary_Stack_Count : Int;
+      --  Indicates the number of task objects declared in this unit that have
+      --  default sized primary stacks.
+
+      Sec_Stack_Count : Int;
+      --  Indicates the number of task objects declared in this unit that have
+      --  default sized secondary stacks.
    end record;
 
    package Units is new Table.Table (
diff --git a/gcc/ada/bindgen.adb b/gcc/ada/bindgen.adb
index a9ea20ebd9b..e3d875bc8cc 100644
--- a/gcc/ada/bindgen.adb
+++ b/gcc/ada/bindgen.adb
@@ -59,6 +59,14 @@ package body Bindgen is
    Num_Elab_Calls : Nat := 0;
    --  Number of generated calls to elaboration routines
 
+   Num_Primary_Stacks : Int := 0;
+   --  Number of default-sized primary stacks the binder needs to allocate for
+   --  task objects declared in the program.
+
+   Num_Sec_Stacks : Int := 0;
+   --  Number of default-sized primary stacks the binder needs to allocate for
+   --  task objects declared in the program.
+
    System_Restrictions_Used : Boolean := False;
    --  Flag indicating whether the unit System.Restrictions is in the closure
    --  of the partition. This is set by Resolve_Binder_Options, and is used
@@ -74,6 +82,12 @@ package body Bindgen is
    --  domains just before calling the main procedure from the environment
    --  task.
 
+   System_Secondary_Stack_Used : Boolean := False;
+   --  Flag indicating whether the unit System.Secondary_Stack is in the
+   --  closure of the partition. This is set by Resolve_Binder_Options, and
+   --  is used to initialize the package in cases where the run-time brings
+   --  in package but the secondary stack is not used.
+
    System_Tasking_Restricted_Stages_Used : Boolean := False;
    --  Flag indicating whether the unit System.Tasking.Restricted.Stages is in
    --  the closure of the partition. This is set by Resolve_Binder_Options,
@@ -179,8 +193,11 @@ package body Bindgen is
    --     Exception_Tracebacks_Symbolic : Integer;
    --     Detect_Blocking               : Integer;
    --     Default_Stack_Size            : Integer;
+   --     Default_Secondary_Stack_Size  : System.Parameters.Size_Type;
    --     Leap_Seconds_Support          : Integer;
    --     Main_CPU                      : Integer;
+   --     Default_Sized_SS_Pool         : System.Address;
+   --     Binder_Sec_Stacks_Count       : Natural;
 
    --  Main_Priority is the priority value set by pragma Priority in the main
    --  program. If no such pragma is present, the value is -1.
@@ -261,6 +278,9 @@ package body Bindgen is
    --  Default_Stack_Size is the default stack size used when creating an Ada
    --  task with no explicit Storage_Size clause.
 
+   --  Default_Secondary_Stack_Size is the default secondary stack size used
+   --  when creating an Ada task with no explicit Secondary_Stack_Size clause.
+
    --  Leap_Seconds_Support denotes whether leap seconds have been enabled or
    --  disabled. A value of zero indicates that leap seconds are turned "off",
    --  while a value of one signifies "on" status.
@@ -268,6 +288,14 @@ package body Bindgen is
    --  Main_CPU is the processor set by pragma CPU in the main program. If no
    --  such pragma is present, the value is -1.
 
+   --  Default_Sized_SS_Pool is set to the address of the default-sized
+   --  secondary stacks array generated by the binder. This pool of stacks is
+   --  generated when either the restriction No_Implicit_Heap_Allocations
+   --  or No_Implicit_Task_Allocations is active.
+
+   --  Binder_Sec_Stacks_Count is the number of generated secondary stacks in
+   --  the Default_Sized_SS_Pool.
+
    procedure WBI (Info : String) renames Osint.B.Write_Binder_Info;
    --  Convenient shorthand used throughout
 
@@ -554,6 +582,32 @@ package body Bindgen is
             WBI ("      procedure Start_Slave_CPUs;");
             WBI ("      pragma Import (C, Start_Slave_CPUs," &
                  " ""__gnat_start_slave_cpus"");");
+            WBI ("");
+         end if;
+
+         --  A restricted run-time may attempt to initialize the main task's
+         --  secondary stack even if the stack is not used. Consequently,
+         --  the binder needs to initialize Binder_Sec_Stacks_Count anytime
+         --  System.Secondary_Stack is in the enclosure of the partition.
+
+         if System_Secondary_Stack_Used then
+            WBI ("      Binder_Sec_Stacks_Count : Natural;");
+            WBI ("      pragma Import (Ada, Binder_Sec_Stacks_Count, " &
+                 """__gnat_binder_ss_count"");");
+            WBI ("");
+         end if;
+
+         if Sec_Stack_Used then
+            WBI ("      Default_Secondary_Stack_Size : " &
+                 "System.Parameters.Size_Type;");
+            WBI ("      pragma Import (C, Default_Secondary_Stack_Size, " &
+                 """__gnat_default_ss_size"");");
+
+            WBI ("      Default_Sized_SS_Pool : System.Address;");
+            WBI ("      pragma Import (Ada, Default_Sized_SS_Pool, " &
+                 """__gnat_default_ss_pool"");");
+
+            WBI ("");
          end if;
 
          WBI ("   begin");
@@ -588,6 +642,50 @@ package body Bindgen is
             WBI ("      null;");
          end if;
 
+         --  Generate default-sized secondary stack pool and set secondary
+         --  stack globals.
+
+         if Sec_Stack_Used then
+
+            --  Elaborate the body of the binder to initialize the default-
+            --  sized secondary stack pool.
+
+            WBI ("");
+            WBI ("      " & Get_Ada_Main_Name & "'Elab_Body;");
+
+            --  Generate the default-sized secondary stack pool and set the
+            --  related secondary stack globals.
+
+            Set_String ("      Default_Secondary_Stack_Size := ");
+
+            if Opt.Default_Sec_Stack_Size /= Opt.No_Stack_Size then
+               Set_Int (Opt.Default_Sec_Stack_Size);
+            else
+               Set_String ("System.Parameters.Runtime_Default_Sec_Stack_Size");
+            end if;
+
+            Set_Char (';');
+            Write_Statement_Buffer;
+
+            Set_String ("      Binder_Sec_Stacks_Count := ");
+            Set_Int (Num_Sec_Stacks);
+            Set_Char (';');
+            Write_Statement_Buffer;
+
+            WBI ("      Default_Sized_SS_Pool := " &
+                   "Sec_Default_Sized_Stacks'Address;");
+            WBI ("");
+
+         --  When a restricted run-time initializes the main task's secondary
+         --  stack but the program does not use it, no secondary stack is
+         --  generated. Binder_Sec_Stacks_Count is set to zero so the run-time
+         --  is aware that the lack of pre-allocated secondary stack is
+         --  expected.
+
+         elsif System_Secondary_Stack_Used then
+            WBI ("      Binder_Sec_Stacks_Count := 0;");
+         end if;
+
       --  Normal case (standard library not suppressed). Set all global values
       --  used by the run time.
 
@@ -647,6 +745,10 @@ package body Bindgen is
          WBI ("      Default_Stack_Size : Integer;");
          WBI ("      pragma Import (C, Default_Stack_Size, " &
               """__gl_default_stack_size"");");
+         WBI ("      Default_Secondary_Stack_Size : " &
+              "System.Parameters.Size_Type;");
+         WBI ("      pragma Import (C, Default_Secondary_Stack_Size, " &
+              """__gnat_default_ss_size"");");
          WBI ("      Leap_Seconds_Support : Integer;");
          WBI ("      pragma Import (C, Leap_Seconds_Support, " &
               """__gl_leap_seconds_support"");");
@@ -730,6 +832,18 @@ package body Bindgen is
                  & """__gnat_freeze_dispatching_domains"");");
          end if;
 
+         --  Secondary stack global variables
+
+         WBI ("      Binder_Sec_Stacks_Count : Natural;");
+         WBI ("      pragma Import (Ada, Binder_Sec_Stacks_Count, " &
+              """__gnat_binder_ss_count"");");
+
+         WBI ("      Default_Sized_SS_Pool : System.Address;");
+         WBI ("      pragma Import (Ada, Default_Sized_SS_Pool, " &
+              """__gnat_default_ss_pool"");");
+
+         WBI ("");
+
          --  Start of processing for Adainit
 
          WBI ("   begin");
@@ -870,9 +984,51 @@ package body Bindgen is
             WBI ("      Bind_Env_Addr := Bind_Env'Address;");
          end if;
 
-         --  Generate call to Install_Handler
-
          WBI ("");
+
+         --  Generate default-sized secondary stack pool and set secondary
+         --  stack globals.
+
+         if Sec_Stack_Used then
+
+            --  Elaborate the body of the binder to initialize the default-
+            --  sized secondary stack pool.
+
+            WBI ("      " & Get_Ada_Main_Name & "'Elab_Body;");
+
+            --  Generate the default-sized secondary stack pool and set the
+            --  related secondary stack globals.
+
+            Set_String ("      Default_Secondary_Stack_Size := ");
+
+            if Opt.Default_Sec_Stack_Size /= Opt.No_Stack_Size then
+               Set_Int (Opt.Default_Sec_Stack_Size);
+            else
+               Set_String ("System.Parameters.Runtime_Default_Sec_Stack_Size");
+            end if;
+
+            Set_Char (';');
+            Write_Statement_Buffer;
+
+            Set_String ("      Binder_Sec_Stacks_Count := ");
+            Set_Int (Num_Sec_Stacks);
+            Set_Char (';');
+            Write_Statement_Buffer;
+
+            Set_String ("      Default_Sized_SS_Pool := ");
+
+            if Num_Sec_Stacks > 0 then
+               Set_String ("Sec_Default_Sized_Stacks'Address;");
+            else
+               Set_String ("System.Null_Address;");
+            end if;
+
+            Write_Statement_Buffer;
+            WBI ("");
+         end if;
+
+         --  Generate call to Runtime_Initialize
+
          WBI ("      Runtime_Initialize (1);");
       end if;
 
@@ -888,17 +1044,6 @@ package body Bindgen is
          Write_Statement_Buffer;
       end if;
 
-      --  Generate assignment of default secondary stack size if set
-
-      if Sec_Stack_Used and then Default_Sec_Stack_Size /= -1 then
-         WBI ("");
-         Set_String ("      System.Secondary_Stack.");
-         Set_String ("Default_Secondary_Stack_Size := ");
-         Set_Int (Opt.Default_Sec_Stack_Size);
-         Set_Char (';');
-         Write_Statement_Buffer;
-      end if;
-
       --  Initialize stack limit variable of the environment task if the stack
       --  check method is stack limit and stack check is enabled.
 
@@ -2044,6 +2189,26 @@ package body Bindgen is
          end if;
       end loop;
 
+      --  Count the number of statically allocated stacks to be generated by
+      --  the binder. If the user has specified the number of default-sized
+      --  secondary stacks, use that number. Otherwise start the count at one
+      --  as the binder is responsible for creating a secondary stack for the
+      --  main task.
+
+      if Opt.Quantity_Of_Default_Size_Sec_Stacks /= -1 then
+         Num_Sec_Stacks := Quantity_Of_Default_Size_Sec_Stacks;
+      elsif Sec_Stack_Used then
+         Num_Sec_Stacks := 1;
+      end if;
+
+      for J in Units.First .. Units.Last loop
+         Num_Primary_Stacks :=
+           Num_Primary_Stacks + Units.Table (J).Primary_Stack_Count;
+
+         Num_Sec_Stacks :=
+           Num_Sec_Stacks + Units.Table (J).Sec_Stack_Count;
+      end loop;
+
       --  Generate output file in appropriate language
 
       Gen_Output_File_Ada (Filename, Elab_Order);
@@ -2114,9 +2279,11 @@ package body Bindgen is
          WBI ("with System.Scalar_Values;");
       end if;
 
-      --  Generate with of System.Secondary_Stack if active
+      --  Generate withs of System.Secondary_Stack and System.Parameters to
+      --  allow the generation of the default-sized secondary stack pool.
 
-      if Sec_Stack_Used and then Default_Sec_Stack_Size /= -1 then
+      if Sec_Stack_Used then
+         WBI ("with System.Parameters;");
          WBI ("with System.Secondary_Stack;");
       end if;
 
@@ -2156,10 +2323,10 @@ package body Bindgen is
             end if;
          end if;
 
-         --  Define exit status. Again in normal mode, this is in the
-         --  run-time library, and is initialized there, but in the
-         --  configurable runtime case, the variable is declared and
-         --  initialized in this file.
+         --  Define exit status. Again in normal mode, this is in the run-time
+         --  library, and is initialized there, but in the configurable
+         --  run-time case, the variable is declared and initialized in this
+         --  file.
 
          WBI ("");
 
@@ -2358,6 +2525,29 @@ package body Bindgen is
 
       Gen_Elab_Externals (Elab_Order);
 
+      --  Generate default-sized secondary stacks pool. At least one stack is
+      --  created and assigned to the environment task if secondary stacks are
+      --  used by the program.
+
+      if Sec_Stack_Used then
+         Set_String ("   Sec_Default_Sized_Stacks");
+         Set_String (" : array (1 .. ");
+         Set_Int (Num_Sec_Stacks);
+         Set_String (") of aliased System.Secondary_Stack.SS_Stack (");
+
+         if Opt.Default_Sec_Stack_Size /= No_Stack_Size then
+            Set_Int (Opt.Default_Sec_Stack_Size);
+         else
+            Set_String ("System.Parameters.Runtime_Default_Sec_Stack_Size");
+         end if;
+
+         Set_String (");");
+         Write_Statement_Buffer;
+         WBI ("");
+      end if;
+
+      --  Generate reference
+
       if not CodePeer_Mode then
          if not Suppress_Standard_Library_On_Target then
 
@@ -2389,8 +2579,8 @@ package body Bindgen is
 
          if not Suppress_Standard_Library_On_Target then
 
-            --  The B.1(39) implementation advice says that the adainit
-            --  and adafinal routines should be idempotent. Generate a flag to
+            --  The B.1(39) implementation advice says that the adainit and
+            --  adafinal routines should be idempotent. Generate a flag to
             --  ensure that. This is not needed if we are suppressing the
             --  standard library since it would never be referenced.
 
@@ -2873,6 +3063,11 @@ package body Bindgen is
 
          Check_Package (System_Restrictions_Used, "system.restrictions%s");
 
+         --  Ditto for the use of System.Secondary_Stack
+
+         Check_Package
+           (System_Secondary_Stack_Used, "system.secondary_stack%s");
+
          --  Ditto for use of an SMP bareboard runtime
 
          Check_Package (System_BB_CPU_Primitives_Multiprocessors_Used,
diff --git a/gcc/ada/bindusg.adb b/gcc/ada/bindusg.adb
index 6cf7710219e..7c17f939514 100644
--- a/gcc/ada/bindusg.adb
+++ b/gcc/ada/bindusg.adb
@@ -210,6 +210,11 @@ package body Bindusg is
       Write_Line
         ("  -P        Generate binder file suitable for CodePeer");
 
+      --  Line for Q switch
+
+      Write_Line
+        ("  -Qnnn     Generate nnn default-sized secondary stacks");
+
       --  Line for -r switch
 
       Write_Line
@@ -309,8 +314,6 @@ package body Bindusg is
       Write_Line
         ("  -z        No main subprogram (zero main)");
 
-      --  Line for --RTS
-
       --  Line for -Z switch
 
       Write_Line
diff --git a/gcc/ada/checks.adb b/gcc/ada/checks.adb
index a99da08c733..b2c26ca4981 100644
--- a/gcc/ada/checks.adb
+++ b/gcc/ada/checks.adb
@@ -5940,6 +5940,10 @@ package body Checks is
       --  In addition, we force a check if Force_Validity_Checks is set
 
       elsif not Comes_From_Source (Expr)
+        and then not
+          (Nkind (Expr) = N_Identifier
+            and then Present (Renamed_Object (Entity (Expr)))
+            and then Comes_From_Source (Renamed_Object (Entity (Expr))))
         and then not Force_Validity_Checks
         and then (Nkind (Expr) /= N_Unchecked_Type_Conversion
                     or else Kill_Range_Check (Expr))
diff --git a/gcc/ada/cstand.adb b/gcc/ada/cstand.adb
index fe480beb426..e45c0542f26 100644
--- a/gcc/ada/cstand.adb
+++ b/gcc/ada/cstand.adb
@@ -62,15 +62,22 @@ package body CStand is
    -----------------------
 
    procedure Build_Float_Type
-     (E    : Entity_Id;
-      Siz  : Int;
-      Rep  : Float_Rep_Kind;
-      Digs : Int);
+     (E     : Entity_Id;
+      Digs  : Int;
+      Rep   : Float_Rep_Kind;
+      Siz   : Int;
+      Align : Int);
    --  Procedure to build standard predefined float base type. The first
-   --  parameter is the entity for the type, and the second parameter is the
-   --  size in bits. The third parameter indicates the kind of representation
-   --  to be used. The fourth parameter is the digits value. Each type
+   --  parameter is the entity for the type. The second parameter is the
+   --  digits value. The third parameter indicates the representation to
+   --  be used for the type. The fourth parameter is the size in bits.
+   --  The fifth parameter is the alignment in storage units. Each type
    --  is added to the list of predefined floating point types.
+   --
+   --  Note that both RM_Size and Esize are set to the specified size, i.e.
+   --  we do not set the RM_Size to the precision passed by the back end.
+   --  This is consistent with the semantics of 'Size specified in the RM
+   --  because we cannot pack components of the type tighter than this size.
 
    procedure Build_Signed_Integer_Type (E : Entity_Id; Siz : Nat);
    --  Procedure to build standard predefined signed integer subtype. The
@@ -189,10 +196,11 @@ package body CStand is
    ----------------------
 
    procedure Build_Float_Type
-     (E    : Entity_Id;
-      Siz  : Int;
-      Rep  : Float_Rep_Kind;
-      Digs : Int)
+     (E     : Entity_Id;
+      Digs  : Int;
+      Rep   : Float_Rep_Kind;
+      Siz   : Int;
+      Align : Int)
    is
    begin
       Set_Type_Definition (Parent (E),
@@ -201,10 +209,10 @@ package body CStand is
 
       Set_Ekind                      (E, E_Floating_Point_Type);
       Set_Etype                      (E, E);
-      Set_Float_Rep (E, Rep);
-      Init_Size                      (E, Siz);
-      Set_Elem_Alignment             (E);
       Init_Digits_Value              (E, Digs);
+      Set_Float_Rep                  (E, Rep);
+      Init_Size                      (E, Siz);
+      Set_Elem_Alignment             (E, Align);
       Set_Float_Bounds               (E);
       Set_Is_Frozen                  (E);
       Set_Is_Public                  (E);
@@ -295,8 +303,9 @@ package body CStand is
 
    procedure Copy_Float_Type (To : Entity_Id; From : Entity_Id) is
    begin
-      Build_Float_Type (To, UI_To_Int (Esize (From)), Float_Rep (From),
-                        UI_To_Int (Digits_Value (From)));
+      Build_Float_Type
+        (To, UI_To_Int (Digits_Value (From)), Float_Rep (From),
+         UI_To_Int (Esize (From)), UI_To_Int (Alignment (From)));
    end Copy_Float_Type;
 
    ----------------------
@@ -2065,15 +2074,17 @@ package body CStand is
       Size      : Positive;
       Alignment : Natural)
    is
+      pragma Unreferenced (Precision);
+      --  See Build_Float_Type for the rationale
+
       Ent : constant Entity_Id := New_Standard_Entity;
 
    begin
       Set_Defining_Identifier (New_Node (N_Full_Type_Declaration, Stloc), Ent);
       Make_Name (Ent, Name);
       Set_Scope (Ent, Standard_Standard);
-      Build_Float_Type (Ent, Int (Size), Float_Rep, Pos (Digs));
-      Set_RM_Size (Ent, UI_From_Int (Int (Precision)));
-      Set_Alignment (Ent, UI_From_Int (Int (Alignment / 8)));
+      Build_Float_Type
+        (Ent, Pos (Digs), Float_Rep, Int (Size), Int (Alignment / 8));
 
       if No (Back_End_Float_Types) then
          Back_End_Float_Types := New_Elmt_List;
diff --git a/gcc/ada/debug.adb b/gcc/ada/debug.adb
index 4e747203394..442ce0873e5 100644
--- a/gcc/ada/debug.adb
+++ b/gcc/ada/debug.adb
@@ -112,7 +112,7 @@ package body Debug is
    --  d.s  Strict secondary stack management
    --  d.t  Disable static allocation of library level dispatch tables
    --  d.u  Enable Modify_Tree_For_C (update tree for c)
-   --  d.v
+   --  d.v  Enforce SPARK elaboration rules in SPARK code
    --  d.w  Do not check for infinite loops
    --  d.x  No exception handlers
    --  d.y  Disable implicit pragma Elaborate_All on task bodies
@@ -163,7 +163,7 @@ package body Debug is
    --  d.6  Do not avoid declaring unreferenced types in C code
    --  d.7
    --  d.8
-   --  d.9  Enable build-in-place for nonlimited types
+   --  d.9  Disable build-in-place for nonlimited types
 
    --  Debug flags for binder (GNATBIND)
 
@@ -600,6 +600,13 @@ package body Debug is
    --  d.u  Sets Modify_Tree_For_C mode in which tree is modified to make it
    --       easier to generate code using a C compiler.
 
+   --  d.v  This flag enforces the elaboration rules defined in the SPARK
+   --       Reference Manual, chapter 7.7, to all SPARK code within a unit. As
+   --       a result, constructs which violate the rules in chapter 7.7 are no
+   --       longer accepted, even if the implementation is able to statically
+   --       ensure that accepting these constructs does not introduce the
+   --       possibility of failing an elaboration check.
+
    --  d.w  This flag turns off the scanning of loops to detect possible
    --       infinite loops.
 
diff --git a/gcc/ada/doc/gnat_rm/implementation_defined_aspects.rst b/gcc/ada/doc/gnat_rm/implementation_defined_aspects.rst
index be7338f7436..c6018227b06 100644
--- a/gcc/ada/doc/gnat_rm/implementation_defined_aspects.rst
+++ b/gcc/ada/doc/gnat_rm/implementation_defined_aspects.rst
@@ -302,11 +302,15 @@ Aspect Iterable
 This aspect provides a light-weight mechanism for loops and quantified
 expressions over container types, without the overhead imposed by the tampering
 checks of standard Ada 2012 iterators. The value of the aspect is an aggregate
-with four named components: ``First``, ``Next``, ``Has_Element``, and ``Element`` (the
-last one being optional). When only 3 components are specified, only the
-``for .. in`` form of iteration over cursors is available. When all 4 components
-are specified, both this form and the ``for .. of`` form of iteration over
-elements are available. The following is a typical example of use:
+with six named components, or which the last three are optional: ``First``,
+ ``Next``, ``Has_Element``,``Element``, ``Last``, and ``Previous``.
+When only the first three components are specified, only the
+``for .. in`` form of iteration over cursors is available. When ``Element``
+is specified, both this form and the ``for .. of`` form of iteration over
+elements are available. If the last two components are specified, reverse
+iterations over the container can be specified (analogous to what can be done
+over predefined containers that support the Reverse_Iterator interface).
+The following is a typical example of use:
 
 .. code-block:: ada
 
diff --git a/gcc/ada/doc/gnat_ugn/building_executable_programs_with_gnat.rst b/gcc/ada/doc/gnat_ugn/building_executable_programs_with_gnat.rst
index 046fe35a825..b6447d05dd6 100644
--- a/gcc/ada/doc/gnat_ugn/building_executable_programs_with_gnat.rst
+++ b/gcc/ada/doc/gnat_ugn/building_executable_programs_with_gnat.rst
@@ -1243,21 +1243,13 @@ Alphabetical List of All Switches
   :file:`scos.adb`.
 
 
-.. index:: -fdump-xref  (gcc)
-
-:switch:`-fdump-xref`
-  Generates cross reference information in GLI files for C and C++ sources.
-  The GLI files have the same syntax as the ALI files for Ada, and can be used
-  for source navigation in IDEs and on the command line using e.g. gnatxref
-  and the :switch:`--ext=gli` switch.
-
-
 .. index:: -flto  (gcc)
 
 :switch:`-flto[={n}]`
   Enables Link Time Optimization. This switch must be used in conjunction
-  with the traditional :switch:`-Ox` switches and instructs the compiler to
-  defer most optimizations until the link stage. The advantage of this
+  with the :switch:`-Ox` switches (but not with the :switch:`-gnatn` switch
+  since it is a full replacement for the latter) and instructs the compiler
+  to defer most optimizations until the link stage. The advantage of this
   approach is that the compiler can do a whole-program analysis and choose
   the best interprocedural optimization strategy based on a complete view
   of the program, instead of a fragmentary view with the usual approach.
@@ -3898,8 +3890,8 @@ of the pragma in the :title:`GNAT_Reference_manual`).
   This switch activates warnings for exception usage when pragma Restrictions
   (No_Exception_Propagation) is in effect. Warnings are given for implicit or
   explicit exception raises which are not covered by a local handler, and for
-  exception handlers which do not cover a local raise. The default is that these
-  warnings are not given.
+  exception handlers which do not cover a local raise. The default is that
+  these warnings are given for units that contain exception handlers.
 
 
 :switch:`-gnatw.X`
diff --git a/gcc/ada/doc/gnat_ugn/elaboration_order_handling_in_gnat.rst b/gcc/ada/doc/gnat_ugn/elaboration_order_handling_in_gnat.rst
index d943c716d3f..c45d3fcdbee 100644
--- a/gcc/ada/doc/gnat_ugn/elaboration_order_handling_in_gnat.rst
+++ b/gcc/ada/doc/gnat_ugn/elaboration_order_handling_in_gnat.rst
@@ -133,8 +133,43 @@ Elaboration Order
 =================
 
 The sequence by which the elaboration code of all units within a partition is
-executed is referred to as **elaboration order**. The elaboration order depends
-on the following factors:
+executed is referred to as **elaboration order**.
+
+Within a single unit, elaboration code is executed in sequential order.
+
+::
+
+   package body Client is
+      Result : ... := Server.Func;
+
+      procedure Proc is
+         package Inst is new Server.Gen;
+      begin
+         Inst.Eval (Result);
+      end Proc;
+   begin
+      Proc;
+   end Client;
+
+In the example above, the elaboration order within package body ``Client`` is
+as follows:
+
+1. The object declaration of ``Result`` is elaborated.
+
+   * Function ``Server.Func`` is invoked.
+
+2. The subprogram body of ``Proc`` is elaborated.
+
+3. Procedure ``Proc`` is invoked.
+
+   * Generic unit ``Server.Gen`` is instantiated as ``Inst``.
+
+   * Instance ``Inst`` is elaborated.
+
+   * Procedure ``Inst.Eval`` is invoked.
+
+The elaboration order of all units within a partition depends on the following
+factors:
 
 * |withed| units
 
@@ -571,7 +606,7 @@ elaboration order and to diagnose elaboration problems.
   a partition is elaboration code. GNAT performs very few diagnostics and
   generates run-time checks to verify the elaboration order of a program. This
   behavior is identical to that specified by the Ada Reference Manual. The
-  dynamic model is enabled with compilation switch :switch:`-gnatE`.
+  dynamic model is enabled with compiler switch :switch:`-gnatE`.
 
 .. index:: Static elaboration model
 
@@ -860,7 +895,7 @@ SPARK Elaboration Model in GNAT
 The SPARK model is identical to the static model in its handling of internal
 targets. The SPARK model, however, requires explicit ``Elaborate`` or
 ``Elaborate_All`` pragmas to be present in the program when a target is
-external, and emits hard errors instead of warnings:
+external, and compiler switch :switch:`-gnatd.v` is in effect.
 
 ::
 
@@ -987,7 +1022,7 @@ available.
 * *Switch to more permissive elaboration model*
 
   If the compilation was performed using the static model, enable the dynamic
-  model with compilation switch :switch:`-gnatE`. GNAT will no longer generate
+  model with compiler switch :switch:`-gnatE`. GNAT will no longer generate
   implicit ``Elaborate`` and ``Elaborate_All`` pragmas, resulting in a behavior
   identical to that specified by the Ada Reference Manual. The binder will
   generate an executable program that may or may not raise ``Program_Error``,
@@ -1504,6 +1539,17 @@ the elaboration order chosen by the binder.
   When this switch is in effect, GNAT will ignore ``'Access`` of an entry,
   operator, or subprogram when the static model is in effect.
 
+.. index:: -gnatd.v  (gnat)
+
+:switch:`-gnatd.v`
+  Enforce SPARK elaboration rules in SPARK code
+
+  When this switch is in effect, GNAT will enforce the SPARK rules of
+  elaboration as defined in the SPARK Reference Manual, section 7.7. As a
+  result, constructs which violate the SPARK elaboration rules are no longer
+  accepted, even if GNAT is able to statically ensure that these constructs
+  will not lead to ABE problems.
+
 .. index:: -gnatd.y  (gnat)
 
 :switch:`-gnatd.y`
@@ -1558,7 +1604,7 @@ the elaboration order chosen by the binder.
   - *SPARK model*
 
     GNAT will indicate how an elaboration requirement is met by the context of
-    a unit.
+    a unit. This diagnostic requires compiler switch :switch:`-gnatd.v`.
 
     ::
 
@@ -1612,8 +1658,8 @@ none of the binder or compiler switches. If the binder succeeds in finding an
 elaboration order, then apart from possible cases involing dispatching calls
 and access-to-subprogram types, the program is free of elaboration errors.
 If it is important for the program to be portable to compilers other than GNAT,
-then the programmer should use compilation switch :switch:`-gnatel` and
-consider the messages about missing or implicitly created ``Elaborate`` and
+then the programmer should use compiler switch :switch:`-gnatel` and consider
+the messages about missing or implicitly created ``Elaborate`` and
 ``Elaborate_All`` pragmas.
 
 If the binder reports an elaboration circularity, the programmer has several
diff --git a/gcc/ada/doc/gnat_ugn/gnat_and_program_execution.rst b/gcc/ada/doc/gnat_ugn/gnat_and_program_execution.rst
index ac45cee3305..8f9f37cc0d8 100644
--- a/gcc/ada/doc/gnat_ugn/gnat_and_program_execution.rst
+++ b/gcc/ada/doc/gnat_ugn/gnat_and_program_execution.rst
@@ -3611,20 +3611,26 @@ combine a dimensioned and dimensionless value.  Thus an expression such as
 ``Acceleration``.
 
 The dimensionality checks for relationals use the same rules as
-for "+" and "-"; thus
+for "+" and "-", except when comparing to a literal; thus
 
   .. code-block:: ada
 
-        acc > 10.0
+        acc > len
 
 is equivalent to
 
   .. code-block:: ada
 
-       acc-10.0 > 0.0
+       acc-len > 0.0
+
+and is thus illegal, but
+
+  .. code-block:: ada
+
+        acc > 10.0
 
-and is thus illegal. Analogously a conditional expression
-requires the same dimension vector for each branch.
+is accepted with a warning. Analogously a conditional expression requires the
+same dimension vector for each branch (with no exception for literals).
 
 The dimension vector of a type conversion :samp:`T({expr})` is defined
 as follows, based on the nature of ``T``:
diff --git a/gcc/ada/doc/gnat_ugn/gnat_utility_programs.rst b/gcc/ada/doc/gnat_ugn/gnat_utility_programs.rst
index 3f5f2d64c6b..855bb8f3d4d 100644
--- a/gcc/ada/doc/gnat_ugn/gnat_utility_programs.rst
+++ b/gcc/ada/doc/gnat_ugn/gnat_utility_programs.rst
@@ -586,9 +586,9 @@ The following switches are available for ``gnatxref``:
 
 :switch:`--ext={extension}`
   Specify an alternate ali file extension. The default is ``ali`` and other
-  extensions (e.g. ``gli`` for C/C++ sources when using :switch:`-fdump-xref`)
-  may be specified via this switch. Note that if this switch overrides the
-  default, which means that only the new extension will be considered.
+  extensions (e.g. ``gli`` for C/C++ sources) may be specified via this switch.
+  Note that if this switch overrides the default, which means that only the
+  new extension will be considered.
 
 
 .. index:: --RTS (gnatxref)
diff --git a/gcc/ada/einfo.ads b/gcc/ada/einfo.ads
index d20440bcbf2..bfe14fcae7c 100644
--- a/gcc/ada/einfo.ads
+++ b/gcc/ada/einfo.ads
@@ -1312,9 +1312,9 @@ package Einfo is
 --       that represents an activation record pointer is an extra formal.
 
 --    Extra_Formals (Node28)
---       Applies to subprograms and subprogram types, and also to entries
---       and entry families. Returns first extra formal of the subprogram
---       or entry. Returns Empty if there are no extra formals.
+--       Applies to subprograms, subprogram types, entries, and entry
+--       families. Returns first extra formal of the subprogram or entry.
+--       Returns Empty if there are no extra formals.
 
 --    Finalization_Master (Node23) [root type only]
 --       Defined in access-to-controlled or access-to-class-wide types. The
@@ -2756,7 +2756,7 @@ package Einfo is
 --         1) Internal entities (such as temporaries generated for the result
 --         of an inlined function call or dummy variables generated for the
 --         debugger). Set to indicate that they need not be initialized, even
---         when scalars are initialized or normalized;
+--         when scalars are initialized or normalized.
 --
 --         2) Predefined primitives of tagged types. Set to mark that they
 --         have specific properties: first they are primitives even if they
diff --git a/gcc/ada/exp_aggr.adb b/gcc/ada/exp_aggr.adb
index 9faed933b9f..86621a4a06a 100644
--- a/gcc/ada/exp_aggr.adb
+++ b/gcc/ada/exp_aggr.adb
@@ -1251,6 +1251,7 @@ package body Exp_Aggr is
 
             if Finalization_OK
               and then not Is_Limited_Type (Comp_Typ)
+              and then not Is_Build_In_Place_Function_Call (Init_Expr)
               and then not
                 (Is_Array_Type (Comp_Typ)
                   and then Is_Controlled (Component_Type (Comp_Typ))
diff --git a/gcc/ada/exp_attr.adb b/gcc/ada/exp_attr.adb
index 719699566e4..70d39b7a916 100644
--- a/gcc/ada/exp_attr.adb
+++ b/gcc/ada/exp_attr.adb
@@ -1756,7 +1756,18 @@ package body Exp_Attr is
       --  and access to it must be passed to the function.
 
       if Is_Build_In_Place_Function_Call (Pref) then
-         Make_Build_In_Place_Call_In_Anonymous_Context (Pref);
+
+         --  If attribute is 'Old, the context is a postcondition, and
+         --  the temporary must go in the corresponding subprogram, not
+         --  the postcondition function or any created blocks, as when
+         --  the attribute appears in a quantified expression. This is
+         --  handled below in the expansion of the attribute.
+
+         if Attribute_Name (Parent (Pref)) = Name_Old then
+            null;
+         else
+            Make_Build_In_Place_Call_In_Anonymous_Context (Pref);
+         end if;
 
       --  Ada 2005 (AI-318-02): Specialization of the previous case for prefix
       --  containing build-in-place function calls whose returned object covers
diff --git a/gcc/ada/exp_ch11.adb b/gcc/ada/exp_ch11.adb
index 8711c89d0eb..7941cbd2ca6 100644
--- a/gcc/ada/exp_ch11.adb
+++ b/gcc/ada/exp_ch11.adb
@@ -6,7 +6,7 @@
 --                                                                          --
 --                                 B o d y                                  --
 --                                                                          --
---          Copyright (C) 1992-2016, Free Software Foundation, Inc.         --
+--          Copyright (C) 1992-2017, Free Software Foundation, Inc.         --
 --                                                                          --
 -- GNAT is free software;  you can  redistribute it  and/or modify it under --
 -- terms of the  GNU General Public License as published  by the Free Soft- --
@@ -64,7 +64,7 @@ package body Exp_Ch11 is
 
    procedure Warn_If_No_Propagation (N : Node_Id);
    --  Called for an exception raise that is not a local raise (and thus can
-   --  not be optimized to a goto. Issues warning if No_Exception_Propagation
+   --  not be optimized to a goto). Issues warning if No_Exception_Propagation
    --  restriction is set. N is the node for the raise or equivalent call.
 
    ---------------------------
@@ -998,15 +998,10 @@ package body Exp_Ch11 is
          --  if a source generated handler was not the target of a local raise.
 
          else
-            if Restriction_Active (No_Exception_Propagation)
-              and then not Has_Local_Raise (Handler)
+            if not Has_Local_Raise (Handler)
               and then Comes_From_Source (Handler)
-              and then Warn_On_Non_Local_Exception
             then
-               Warn_No_Exception_Propagation_Active (Handler);
-               Error_Msg_N
-                 ("\?X?this handler can never be entered, "
-                  & "and has been removed", Handler);
+               Warn_If_No_Local_Raise (Handler);
             end if;
 
             if No_Exception_Propagation_Active then
@@ -1859,8 +1854,12 @@ package body Exp_Ch11 is
          --  Otherwise, if the No_Exception_Propagation restriction is active
          --  and the warning is enabled, generate the appropriate warnings.
 
+         --  ??? Do not do it for the Call_Marker nodes inserted by the ABE
+         --  mechanism because this generates too many false positives.
+
          elsif Warn_On_Non_Local_Exception
            and then Restriction_Active (No_Exception_Propagation)
+           and then Nkind (N) /= N_Call_Marker
          then
             Warn_No_Exception_Propagation_Active (N);
 
@@ -2155,6 +2154,22 @@ package body Exp_Ch11 is
    end Get_RT_Exception_Name;
 
    ----------------------------
+   -- Warn_If_No_Local_Raise --
+   ----------------------------
+
+   procedure Warn_If_No_Local_Raise (N : Node_Id) is
+   begin
+      if Restriction_Active (No_Exception_Propagation)
+        and then Warn_On_Non_Local_Exception
+      then
+         Warn_No_Exception_Propagation_Active (N);
+
+         Error_Msg_N
+           ("\?X?this handler can never be entered, and has been removed", N);
+      end if;
+   end Warn_If_No_Local_Raise;
+
+   ----------------------------
    -- Warn_If_No_Propagation --
    ----------------------------
 
diff --git a/gcc/ada/exp_ch11.ads b/gcc/ada/exp_ch11.ads
index cdd53de626e..99efdeb2305 100644
--- a/gcc/ada/exp_ch11.ads
+++ b/gcc/ada/exp_ch11.ads
@@ -6,7 +6,7 @@
 --                                                                          --
 --                                 S p e c                                  --
 --                                                                          --
---          Copyright (C) 1992-2015, Free Software Foundation, Inc.         --
+--          Copyright (C) 1992-2017, Free Software Foundation, Inc.         --
 --                                                                          --
 -- GNAT is free software;  you can  redistribute it  and/or modify it under --
 -- terms of the  GNU General Public License as published  by the Free Soft- --
@@ -90,4 +90,9 @@ package Exp_Ch11 is
    --  is a local handler marking that it has a local raise. E is the entity
    --  of the corresponding exception.
 
+   procedure Warn_If_No_Local_Raise (N : Node_Id);
+   --  Called for an exception handler that is not the target of a local raise.
+   --  Issues warning if No_Exception_Propagation restriction is set. N is the
+   --  node for the handler.
+
 end Exp_Ch11;
diff --git a/gcc/ada/exp_ch3.adb b/gcc/ada/exp_ch3.adb
index 29e79dcead9..043a02c64ba 100644
--- a/gcc/ada/exp_ch3.adb
+++ b/gcc/ada/exp_ch3.adb
@@ -43,6 +43,7 @@ with Exp_Tss;  use Exp_Tss;
 with Exp_Util; use Exp_Util;
 with Freeze;   use Freeze;
 with Ghost;    use Ghost;
+with Lib;      use Lib;
 with Namet;    use Namet;
 with Nlists;   use Nlists;
 with Nmake;    use Nmake;
@@ -5580,6 +5581,15 @@ package body Exp_Ch3 is
       --  arithmetic might yield a meaningless value for the length of the
       --  array, or its corresponding attribute.
 
+      procedure Count_Default_Sized_Task_Stacks
+        (Typ         : Entity_Id;
+         Pri_Stacks  : out Int;
+         Sec_Stacks  : out Int);
+      --  Count the number of default-sized primary and secondary task stacks
+      --  required for task objects contained within type Typ. If the number of
+      --  task objects contained within the type is not known at compile time
+      --  the procedure will return the stack counts of zero.
+
       procedure Default_Initialize_Object (After : Node_Id);
       --  Generate all default initialization actions for object Def_Id. Any
       --  new code is inserted after node After.
@@ -5772,6 +5782,119 @@ package body Exp_Ch3 is
          end if;
       end Check_Large_Modular_Array;
 
+      -------------------------------------
+      -- Count_Default_Sized_Task_Stacks --
+      -------------------------------------
+
+      procedure Count_Default_Sized_Task_Stacks
+        (Typ         : Entity_Id;
+         Pri_Stacks  : out Int;
+         Sec_Stacks  : out Int)
+      is
+         Component : Entity_Id;
+
+      begin
+         --  To calculate the number of default-sized task stacks required for
+         --  an object of Typ, a depth-first recursive traversal of the AST
+         --  from the Typ entity node is undertaken. Only type nodes containing
+         --  task objects are visited.
+
+         Pri_Stacks := 0;
+         Sec_Stacks := 0;
+
+         if not Has_Task (Typ) then
+            return;
+         end if;
+
+         case Ekind (Typ) is
+            when E_Task_Subtype
+               | E_Task_Type
+            =>
+               --  A task type is found marking the bottom of the descent. If
+               --  the type has no representation aspect for the corresponding
+               --  stack then that stack is using the default size.
+
+               if Present (Get_Rep_Item (Typ, Name_Storage_Size)) then
+                  Pri_Stacks := 0;
+               else
+                  Pri_Stacks := 1;
+               end if;
+
+               if Present (Get_Rep_Item (Typ, Name_Secondary_Stack_Size)) then
+                  Sec_Stacks := 0;
+               else
+                  Sec_Stacks := 1;
+               end if;
+
+            when E_Array_Subtype
+               | E_Array_Type
+            =>
+               --  First find the number of default stacks contained within an
+               --  array component.
+
+               Count_Default_Sized_Task_Stacks
+                 (Component_Type (Typ),
+                  Pri_Stacks,
+                  Sec_Stacks);
+
+               --  Then multiply the result by the size of the array
+
+               declare
+                  Quantity : constant Int := Number_Of_Elements_In_Array (Typ);
+                  --  Number_Of_Elements_In_Array is non-trival, consequently
+                  --  its result is captured as an optimization.
+
+               begin
+                  Pri_Stacks := Pri_Stacks * Quantity;
+                  Sec_Stacks := Sec_Stacks * Quantity;
+               end;
+
+            when E_Protected_Subtype
+               | E_Protected_Type
+               | E_Record_Subtype
+               | E_Record_Type
+            =>
+               Component := First_Component_Or_Discriminant (Typ);
+
+               --  Recursively descend each component of the composite type
+               --  looking for tasks, but only if the component is marked as
+               --  having a task.
+
+               while Present (Component) loop
+                  if Has_Task (Etype (Component)) then
+                     declare
+                        P : Int;
+                        S : Int;
+
+                     begin
+                        Count_Default_Sized_Task_Stacks
+                          (Etype (Component), P, S);
+                        Pri_Stacks := Pri_Stacks + P;
+                        Sec_Stacks := Sec_Stacks + S;
+                     end;
+                  end if;
+
+                  Next_Component_Or_Discriminant (Component);
+               end loop;
+
+            when E_Limited_Private_Subtype
+               | E_Limited_Private_Type
+               | E_Record_Subtype_With_Private
+               | E_Record_Type_With_Private
+            =>
+               --  Switch to the full view of the private type to continue
+               --  search.
+
+               Count_Default_Sized_Task_Stacks
+                 (Full_View (Typ), Pri_Stacks, Sec_Stacks);
+
+            --  Other types should not contain tasks
+
+            when others =>
+               raise Program_Error;
+         end case;
+      end Count_Default_Sized_Task_Stacks;
+
       -------------------------------
       -- Default_Initialize_Object --
       -------------------------------
@@ -6198,6 +6321,37 @@ package body Exp_Ch3 is
 
       Check_Large_Modular_Array;
 
+      --  If No_Implicit_Heap_Allocations or No_Implicit_Task_Allocations
+      --  restrictions are active then default-sized secondary stacks are
+      --  generated by the binder and allocated by SS_Init. To provide the
+      --  binder the number of stacks to generate, the number of default-sized
+      --  stacks required for task objects contained within the object
+      --  declaration N is calculated here as it is at this point where
+      --  unconstrained types become constrained. The result is stored in the
+      --  enclosing unit's Unit_Record.
+
+      --  Note if N is an array object declaration that has an initialization
+      --  expression, a second object declaration for the initialization
+      --  expression is created by the compiler. To prevent double counting
+      --  of the stacks in this scenario, the stacks of the first array are
+      --  not counted.
+
+      if Has_Task (Typ)
+        and then not Restriction_Active (No_Secondary_Stack)
+        and then (Restriction_Active (No_Implicit_Heap_Allocations)
+          or else Restriction_Active (No_Implicit_Task_Allocations))
+        and then not (Ekind_In (Ekind (Typ), E_Array_Type, E_Array_Subtype)
+                      and then (Has_Init_Expression (N)))
+      then
+         declare
+            PS_Count, SS_Count : Int := 0;
+         begin
+            Count_Default_Sized_Task_Stacks (Typ, PS_Count, SS_Count);
+            Increment_Primary_Stack_Count (PS_Count);
+            Increment_Sec_Stack_Count (SS_Count);
+         end;
+      end if;
+
       --  Default initialization required, and no expression present
 
       if No (Expr) then
@@ -6649,15 +6803,7 @@ package body Exp_Ch3 is
             --  adjustment is required if we are going to rewrite the object
             --  declaration into a renaming declaration.
 
-            if Is_Build_In_Place_Result_Type (Typ)
-              and then Nkind (Parent (N)) = N_Extended_Return_Statement
-              and then
-                not Is_Definite_Subtype (Etype (Return_Applies_To
-                      (Return_Statement_Entity (Parent (N)))))
-            then
-               null;
-
-            elsif Needs_Finalization (Typ)
+            if Needs_Finalization (Typ)
               and then not Is_Limited_View (Typ)
               and then not Rewrite_As_Renaming
             then
diff --git a/gcc/ada/exp_ch4.adb b/gcc/ada/exp_ch4.adb
index 770341ce9eb..abf6d635451 100644
--- a/gcc/ada/exp_ch4.adb
+++ b/gcc/ada/exp_ch4.adb
@@ -1069,12 +1069,15 @@ package body Exp_Ch4 is
          --  object can be limited but not inherently limited if this allocator
          --  came from a return statement (we're allocating the result on the
          --  secondary stack). In that case, the object will be moved, so we do
-         --  want to Adjust.
+         --  want to Adjust. However, if it's a nonlimited build-in-place
+         --  function call, Adjust is not wanted.
 
          if Needs_Finalization (DesigT)
            and then Needs_Finalization (T)
            and then not Aggr_In_Place
            and then not Is_Limited_View (T)
+           and then not Alloc_For_BIP_Return (N)
+           and then not Is_Build_In_Place_Function_Call (Expression (N))
          then
             --  An unchecked conversion is needed in the classwide case because
             --  the designated type can be an ancestor of the subtype mark of
@@ -5561,6 +5564,7 @@ package body Exp_Ch4 is
          declare
             Cnn     : constant Entity_Id := Make_Temporary (Loc, 'C', N);
             Ptr_Typ : constant Entity_Id := Make_Temporary (Loc, 'A');
+
          begin
             --  Generate:
             --    type Ann is access all Typ;
@@ -5638,6 +5642,7 @@ package body Exp_Ch4 is
       then
          declare
             Cnn : constant Node_Id := Make_Temporary (Loc, 'C', N);
+
          begin
             Insert_Action (N,
               Make_Object_Declaration (Loc,
@@ -5678,6 +5683,7 @@ package body Exp_Ch4 is
 
             declare
                Cnn : constant Node_Id := Make_Temporary (Loc, 'C', N);
+
             begin
                Decl :=
                  Make_Object_Declaration (Loc,
diff --git a/gcc/ada/exp_ch6.adb b/gcc/ada/exp_ch6.adb
index 6c27741d37c..bca7e5deae4 100644
--- a/gcc/ada/exp_ch6.adb
+++ b/gcc/ada/exp_ch6.adb
@@ -712,7 +712,8 @@ package body Exp_Ch6 is
          Stmt := First (Stmts);
          while Present (Stmt) loop
             if Nkind (Stmt) = N_Block_Statement then
-               Replace_Returns (Param_Id, Statements (Stmt));
+               Replace_Returns (Param_Id,
+                 Statements (Handled_Statement_Sequence (Stmt)));
 
             elsif Nkind (Stmt) = N_Case_Statement then
                declare
@@ -5145,11 +5146,19 @@ package body Exp_Ch6 is
                         Set_No_Initialization (Heap_Allocator);
                      end if;
 
+                     --  Set the flag indicating that the allocator came from
+                     --  a build-in-place return statement, so we can avoid
+                     --  adjusting the allocated object. Note that this flag
+                     --  will be inherited by the copies made below.
+
+                     Set_Alloc_For_BIP_Return (Heap_Allocator);
+
                      --  The Pool_Allocator is just like the Heap_Allocator,
                      --  except we set Storage_Pool and Procedure_To_Call so
                      --  it will use the user-defined storage pool.
 
                      Pool_Allocator := New_Copy_Tree (Heap_Allocator);
+                     pragma Assert (Alloc_For_BIP_Return (Pool_Allocator));
 
                      --  Do not generate the renaming of the build-in-place
                      --  pool parameter on ZFP because the parameter is not
@@ -5191,6 +5200,7 @@ package body Exp_Ch6 is
 
                      else
                         SS_Allocator := New_Copy_Tree (Heap_Allocator);
+                        pragma Assert (Alloc_For_BIP_Return (SS_Allocator));
 
                         --  The heap and pool allocators are marked as
                         --  Comes_From_Source since they correspond to an
@@ -7239,8 +7249,68 @@ package body Exp_Ch6 is
 
       if Is_Limited_View (Typ) then
          return Ada_Version >= Ada_2005 and then not Debug_Flag_Dot_L;
+
       else
-         return Debug_Flag_Dot_9;
+         if Debug_Flag_Dot_9 then
+            return False;
+         end if;
+
+         if Has_Interfaces (Typ) then
+            return False;
+         end if;
+
+         declare
+            T : Entity_Id := Typ;
+         begin
+            --  For T'Class, return True if it's True for T. This is necessary
+            --  because a class-wide function might say "return F (...)", where
+            --  F returns the corresponding specific type. We need a loop in
+            --  case T is a subtype of a class-wide type.
+
+            while Is_Class_Wide_Type (T) loop
+               T := Etype (T);
+            end loop;
+
+            --  If this is a generic formal type in an instance, return True if
+            --  it's True for the generic actual type.
+
+            if Nkind (Parent (T)) = N_Subtype_Declaration
+              and then Present (Generic_Parent_Type (Parent (T)))
+            then
+               T := Entity (Subtype_Indication (Parent (T)));
+
+               if Present (Full_View (T)) then
+                  T := Full_View (T);
+               end if;
+            end if;
+
+            if Present (Underlying_Type (T)) then
+               T := Underlying_Type (T);
+            end if;
+
+            declare
+               Result : Boolean;
+               --  So we can stop here in the debugger
+            begin
+               --  ???For now, enable build-in-place for a very narrow set of
+               --  controlled types. Change "if True" to "if False" to
+               --  experiment more controlled types. Eventually, we would
+               --  like to enable build-in-place for all tagged types, all
+               --  types that need finalization, and all caller-unknown-size
+               --  types.
+
+               if True then
+                  Result := Is_Controlled (T)
+                    and then Present (Enclosing_Subprogram (T))
+                    and then not Is_Compilation_Unit (Enclosing_Subprogram (T))
+                    and then Ekind (Enclosing_Subprogram (T)) = E_Procedure;
+               else
+                  Result := Is_Controlled (T);
+               end if;
+
+               return Result;
+            end;
+         end;
       end if;
    end Is_Build_In_Place_Result_Type;
 
@@ -7326,7 +7396,12 @@ package body Exp_Ch6 is
          raise Program_Error;
       end if;
 
-      return Is_Build_In_Place_Function (Function_Id);
+      declare
+         Result : constant Boolean := Is_Build_In_Place_Function (Function_Id);
+         --  So we can stop here in the debugger
+      begin
+         return Result;
+      end;
    end Is_Build_In_Place_Function_Call;
 
    -----------------------
diff --git a/gcc/ada/exp_ch9.adb b/gcc/ada/exp_ch9.adb
index aca0c18e3b6..063b812f9bc 100644
--- a/gcc/ada/exp_ch9.adb
+++ b/gcc/ada/exp_ch9.adb
@@ -339,6 +339,14 @@ package body Exp_Ch9 is
    --  same parameter names and the same resolved types, but with new entities
    --  for the formals.
 
+   function Create_Secondary_Stack_For_Task (T : Node_Id) return Boolean;
+   --  Return whether a secondary stack for the task T should be created by the
+   --  expander. The secondary stack for a task will be created by the expander
+   --  if the size of the stack has been specified by the Secondary_Stack_Size
+   --  representation aspect and either the No_Implicit_Heap_Allocations or
+   --  No_Implicit_Task_Allocations restrictions are in effect and the
+   --  No_Secondary_Stack restriction is not.
+
    procedure Debug_Private_Data_Declarations (Decls : List_Id);
    --  Decls is a list which may contain the declarations created by Install_
    --  Private_Data_Declarations. All generated entities are marked as needing
@@ -5415,6 +5423,20 @@ package body Exp_Ch9 is
    end Convert_Concurrent;
 
    -------------------------------------
+   -- Create_Secondary_Stack_For_Task --
+   -------------------------------------
+
+   function Create_Secondary_Stack_For_Task (T : Node_Id) return Boolean is
+   begin
+      return
+        (Restriction_Active (No_Implicit_Heap_Allocations)
+          or else Restriction_Active (No_Implicit_Task_Allocations))
+        and then not Restriction_Active (No_Secondary_Stack)
+        and then Has_Rep_Item
+                   (T, Name_Secondary_Stack_Size, Check_Parents => False);
+   end Create_Secondary_Stack_For_Task;
+
+   -------------------------------------
    -- Debug_Private_Data_Declarations --
    -------------------------------------
 
@@ -11712,6 +11734,7 @@ package body Exp_Ch9 is
       Body_Decl  : Node_Id;
       Cdecls     : List_Id;
       Decl_Stack : Node_Id;
+      Decl_SS    : Node_Id;
       Elab_Decl  : Node_Id;
       Ent_Stack  : Entity_Id;
       Proc_Spec  : Node_Id;
@@ -11939,6 +11962,57 @@ package body Exp_Ch9 is
 
       end if;
 
+      --  Declare a static secondary stack if the conditions for a statically
+      --  generated stack are met.
+
+      if Create_Secondary_Stack_For_Task (TaskId) then
+         declare
+            Ritem     : Node_Id;
+            Size_Expr : Node_Id;
+
+         begin
+            --  First extract the secondary stack size from the task type's
+            --  representation aspect.
+
+            Ritem :=
+              Get_Rep_Item
+                (TaskId, Name_Secondary_Stack_Size, Check_Parents => False);
+
+            --  Get Secondary_Stack_Size expression. Can be a pragma or aspect.
+
+            if Nkind (Ritem) = N_Pragma then
+               Size_Expr :=
+                 Expression
+                   (First (Pragma_Argument_Associations (Ritem)));
+            else
+               Size_Expr := Expression (Ritem);
+            end if;
+
+            pragma Assert (Compile_Time_Known_Value (Size_Expr));
+
+            --  Create the secondary stack for the task
+
+            Decl_SS :=
+              Make_Component_Declaration (Loc,
+                Defining_Identifier  =>
+                  Make_Defining_Identifier (Loc, Name_uSecondary_Stack),
+                Component_Definition =>
+                  Make_Component_Definition (Loc,
+                    Aliased_Present     => True,
+                    Subtype_Indication  =>
+                      Make_Subtype_Indication (Loc,
+                        Subtype_Mark =>
+                          New_Occurrence_Of (RTE (RE_SS_Stack), Loc),
+                        Constraint   =>
+                          Make_Index_Or_Discriminant_Constraint (Loc,
+                            Constraints  => New_List (
+                              Make_Integer_Literal (Loc,
+                                Expr_Value (Size_Expr)))))));
+
+            Append_To (Cdecls, Decl_SS);
+         end;
+      end if;
+
       --  Add components for entry families
 
       Collect_Entry_Families (Loc, Cdecls, Size_Decl, Tasktyp);
@@ -12835,11 +12909,14 @@ package body Exp_Ch9 is
       end if;
 
       --  If the type of the dispatching object is an access type then return
-      --  an explicit dereference.
+      --  an explicit dereference  of a copy of the object, and note that
+      --  this is the controlling actual of the call.
 
       if Is_Access_Type (Etype (Object)) then
-         Object := Make_Explicit_Dereference (Sloc (N), Object);
+         Object :=
+           Make_Explicit_Dereference (Sloc (N), New_Copy_Tree (Object));
          Analyze (Object);
+         Set_Is_Controlling_Actual (Object);
       end if;
    end Extract_Dispatching_Call;
 
@@ -14136,11 +14213,33 @@ package body Exp_Ch9 is
            New_Occurrence_Of (Storage_Size_Variable (Ttyp), Loc));
       end if;
 
-      --  Secondary_Stack_Size parameter. Set Default_Secondary_Stack_Size
-      --  unless there is a Secondary_Stack_Size rep item, in which case we
-      --  take the value from the rep item. If the restriction
-      --  No_Secondary_Stack is active then a size of 0 is passed regardless
-      --  to prevent the allocation of the unused stack.
+      --  Secondary_Stack parameter used for restricted profiles
+
+      if Restricted_Profile then
+
+         --  If the secondary stack has been allocated by the expander then
+         --  pass its access pointer. Otherwise, pass null.
+
+         if Create_Secondary_Stack_For_Task (Ttyp) then
+            Append_To (Args,
+              Make_Attribute_Reference (Loc,
+                Prefix         =>
+                  Make_Selected_Component (Loc,
+                    Prefix        => Make_Identifier (Loc, Name_uInit),
+                    Selector_Name =>
+                      Make_Identifier (Loc, Name_uSecondary_Stack)),
+                Attribute_Name => Name_Unrestricted_Access));
+
+         else
+            Append_To (Args, Make_Null (Loc));
+         end if;
+      end if;
+
+      --  Secondary_Stack_Size parameter. Set RE_Unspecified_Size unless there
+      --  is a Secondary_Stack_Size rep item, in which case take the value from
+      --  the rep item. If the restriction No_Secondary_Stack is active then a
+      --  size of 0 is passed regardless to prevent the allocation of the
+      --  unused stack.
 
       if Restriction_Active (No_Secondary_Stack) then
          Append_To (Args, Make_Integer_Literal (Loc, 0));
@@ -14465,6 +14564,12 @@ package body Exp_Ch9 is
                 Object_Definition   =>
                   New_Occurrence_Of (Etype (Formal), Loc)));
 
+            --  The object is initialized with an explicit assignment
+            --  later. Indicate that it does not need an initialization
+            --  to prevent spurious warnings if the type excludes null.
+
+            Set_No_Initialization (Last (Decls));
+
             if Ekind (Formal) /= E_Out_Parameter then
 
                --  Generate:
@@ -14481,15 +14586,22 @@ package body Exp_Ch9 is
                    Expression => New_Copy_Tree (Actual)));
             end if;
 
-            --  Generate:
+            --  If the actual is not controlling, generate:
+
             --    Jnn'unchecked_access
 
-            Append_To (Params,
-              Make_Attribute_Reference (Loc,
-                Attribute_Name => Name_Unchecked_Access,
-                Prefix         => New_Occurrence_Of (Temp_Nam, Loc)));
+            --  and add it to aggegate for access to formals. Note that
+            --  the actual may be by-copy but still be a controlling actual
+            --  if it is an access to class-wide interface.
 
-            Has_Param := True;
+            if not Is_Controlling_Actual (Actual) then
+               Append_To (Params,
+                 Make_Attribute_Reference (Loc,
+                   Attribute_Name => Name_Unchecked_Access,
+                   Prefix         => New_Occurrence_Of (Temp_Nam, Loc)));
+
+               Has_Param := True;
+            end if;
 
          --  The controlling parameter is omitted
 
diff --git a/gcc/ada/exp_util.adb b/gcc/ada/exp_util.adb
index b1ab606f055..8fdd8aa8200 100644
--- a/gcc/ada/exp_util.adb
+++ b/gcc/ada/exp_util.adb
@@ -10817,8 +10817,17 @@ package body Exp_Util is
                Analyze (Block);
             end if;
 
-         when others =>
+         --  Could be e.g. a loop that was transformed into a block or null
+         --  statement. Do nothing for terminate alternatives.
+
+         when N_Block_Statement
+            | N_Null_Statement
+            | N_Terminate_Alternative
+         =>
             null;
+
+         when others =>
+            raise Program_Error;
       end case;
    end Process_Statements_For_Controlled_Objects;
 
@@ -10969,7 +10978,8 @@ package body Exp_Util is
          Related_Nod : Node_Id := Empty) return Entity_Id;
       --  Create an external symbol of the form xxx_FIRST/_LAST if Related_Nod
       --  is present (xxx is taken from the Chars field of Related_Nod),
-      --  otherwise it generates an internal temporary.
+      --  otherwise it generates an internal temporary. The created temporary
+      --  entity is marked as internal.
 
       ---------------------
       -- Build_Temporary --
@@ -10980,6 +10990,7 @@ package body Exp_Util is
          Id          : Character;
          Related_Nod : Node_Id := Empty) return Entity_Id
       is
+         Temp_Id  : Entity_Id;
          Temp_Nam : Name_Id;
 
       begin
@@ -10992,13 +11003,17 @@ package body Exp_Util is
                Temp_Nam := New_External_Name (Chars (Related_Id), "_LAST");
             end if;
 
-            return Make_Defining_Identifier (Loc, Temp_Nam);
+            Temp_Id := Make_Defining_Identifier (Loc, Temp_Nam);
 
          --  Otherwise generate an internal temporary
 
          else
-            return Make_Temporary (Loc, Id, Related_Nod);
+            Temp_Id := Make_Temporary (Loc, Id, Related_Nod);
          end if;
+
+         Set_Is_Internal (Temp_Id);
+
+         return Temp_Id;
       end Build_Temporary;
 
       --  Local variables
@@ -11249,7 +11264,7 @@ package body Exp_Util is
          --  Exp_Ch2.Expand_Renaming). Otherwise the temporary must be
          --  elaborated by gigi, and is of course not to be replaced in-line
          --  by the expression it renames, which would defeat the purpose of
-         --  removing the side-effect.
+         --  removing the side effect.
 
          if Nkind_In (Exp, N_Selected_Component, N_Indexed_Component)
            and then Has_Non_Standard_Rep (Etype (Prefix (Exp)))
@@ -12650,7 +12665,7 @@ package body Exp_Util is
            and then Variable_Ref
          then
             --  Exception is a prefix that is the result of a previous removal
-            --  of side-effects.
+            --  of side effects.
 
             return Is_Entity_Name (Prefix (N))
               and then not Comes_From_Source (Prefix (N))
diff --git a/gcc/ada/fe.h b/gcc/ada/fe.h
index 513cfa97daa..6b6d524bcd7 100644
--- a/gcc/ada/fe.h
+++ b/gcc/ada/fe.h
@@ -109,10 +109,12 @@ extern Nat       Serious_Errors_Detected;
 #define Get_Local_Raise_Call_Entity exp_ch11__get_local_raise_call_entity
 #define Get_RT_Exception_Entity exp_ch11__get_rt_exception_entity
 #define Get_RT_Exception_Name exp_ch11__get_rt_exception_name
+#define Warn_If_No_Local_Raise exp_ch11__warn_if_no_local_raise
 
 extern Entity_Id Get_Local_Raise_Call_Entity (void);
 extern Entity_Id Get_RT_Exception_Entity (int);
 extern void Get_RT_Exception_Name (int);
+extern void Warn_If_No_Local_Raise (int);
 
 /* exp_code:  */
 
diff --git a/gcc/ada/freeze.adb b/gcc/ada/freeze.adb
index 794fdf3d095..a106d68ae86 100644
--- a/gcc/ada/freeze.adb
+++ b/gcc/ada/freeze.adb
@@ -8450,7 +8450,7 @@ package body Freeze is
             --  The analysis of the expression may generate insert actions,
             --  which of course must not be executed. We wrap those actions
             --  in a procedure that is not called, and later on eliminated.
-            --  The following cases have no side-effects, and are analyzed
+            --  The following cases have no side effects, and are analyzed
             --  directly.
 
             if Nkind (Dcopy) = N_Identifier
diff --git a/gcc/ada/gcc-interface/Make-lang.in b/gcc/ada/gcc-interface/Make-lang.in
index 113c84f390b..9c7b6e1496f 100644
--- a/gcc/ada/gcc-interface/Make-lang.in
+++ b/gcc/ada/gcc-interface/Make-lang.in
@@ -390,6 +390,7 @@ GNAT_ADA_OBJS =	\
  ada/libgnat/s-restri.o	\
  ada/libgnat/s-secsta.o	\
  ada/libgnat/s-soflin.o	\
+ ada/libgnat/s-soliin.o	\
  ada/libgnat/s-sopco3.o	\
  ada/libgnat/s-sopco4.o	\
  ada/libgnat/s-sopco5.o	\
@@ -579,6 +580,7 @@ GNATBIND_OBJS = \
  ada/libgnat/s-restri.o   \
  ada/libgnat/s-secsta.o   \
  ada/libgnat/s-soflin.o   \
+ ada/libgnat/s-soliin.o   \
  ada/libgnat/s-sopco3.o   \
  ada/libgnat/s-sopco4.o   \
  ada/libgnat/s-sopco5.o   \
diff --git a/gcc/ada/gcc-interface/Makefile.in b/gcc/ada/gcc-interface/Makefile.in
index 2fa47caa547..b1621d11b11 100644
--- a/gcc/ada/gcc-interface/Makefile.in
+++ b/gcc/ada/gcc-interface/Makefile.in
@@ -627,10 +627,10 @@ ifeq ($(strip $(filter-out %86 x86_64 wrs vxworks vxworks7,$(target_cpu) $(targe
 
   ifeq ($(strip $(filter-out x86_64, $(target_cpu))),)
      X86CPU=x86_64
-     LIBGNAT_TARGET_PAIRS=s-atocou.adb<libgnat/s-atocou__builtin.adb
+     LIBGNAT_TARGET_PAIRS=$(X86_64_TARGET_PAIRS)
   else
      X86CPU=x86
-     LIBGNAT_TARGET_PAIRS=s-atocou.adb<libgnat/s-atocou__x86.adb
+     LIBGNAT_TARGET_PAIRS=$(X86_TARGET_PAIRS)
   endif
 
   LIBGNAT_TARGET_PAIRS+= \
@@ -653,10 +653,7 @@ ifeq ($(strip $(filter-out %86 x86_64 wrs vxworks vxworks7,$(target_cpu) $(targe
   g-socthi.ads<libgnat/g-socthi__vxworks.ads \
   g-socthi.adb<libgnat/g-socthi__vxworks.adb \
   g-stsifd.adb<libgnat/g-stsifd__sockets.adb \
-  $(ATOMICS_TARGET_PAIRS) \
-  $(CERTMATH_TARGET_PAIRS) \
-  $(CERTMATH_TARGET_PAIRS_SQRT_FPU) \
-  $(CERTMATH_TARGET_PAIRS_X86TRA)
+  $(ATOMICS_TARGET_PAIRS)
 
   TOOLS_TARGET_PAIRS=indepsw.adb<indepsw-gnu.adb
 
@@ -745,8 +742,7 @@ ifeq ($(strip $(filter-out %86 x86_64 wrs vxworks vxworks7,$(target_cpu) $(targe
     endif
   endif
 
-  EXTRA_GNATRTL_NONTASKING_OBJS += s-stchop.o \
-    $(CERTMATH_GNATRTL_OBJS) $(CERTMATH_GNATRTL_X86TRA_OBJS)
+  EXTRA_GNATRTL_NONTASKING_OBJS += s-stchop.o
   EXTRA_GNATRTL_TASKING_OBJS += i-vxinco.o s-vxwork.o s-vxwext.o
 
   EXTRA_LIBGNAT_OBJS+=vx_stack_info.o
@@ -845,7 +841,7 @@ ifeq ($(strip $(filter-out arm% coff wrs vx%,$(target_cpu) $(target_vendor) $(ta
     endif
   endif
 
-  EXTRA_GNATRTL_NONTASKING_OBJS=i-vxwork.o i-vxwoio.o $(CERTMATH_GNATRTL_OBJS) \
+  EXTRA_GNATRTL_NONTASKING_OBJS=i-vxwork.o i-vxwoio.o \
     s-stchop.o
   EXTRA_GNATRTL_TASKING_OBJS=i-vxinco.o s-vxwork.o s-vxwext.o
 
@@ -1633,7 +1629,7 @@ ifeq ($(strip $(filter-out m68k% linux%,$(target_cpu) $(target_os))),)
   a-intnam.ads<libgnarl/a-intnam__linux.ads \
   s-inmaop.adb<libgnarl/s-inmaop__posix.adb \
   s-intman.adb<libgnarl/s-intman__posix.adb \
-  s-linux.ads<libgnat/s-linux.ads \
+  s-linux.ads<libgnarl/s-linux.ads \
   s-osinte.adb<libgnarl/s-osinte__posix.adb \
   s-osinte.ads<libgnarl/s-osinte__linux.ads \
   s-osprim.adb<libgnat/s-osprim__posix.adb \
diff --git a/gcc/ada/gcc-interface/gigi.h b/gcc/ada/gcc-interface/gigi.h
index 4ddd0f0a8d2..a957de5e589 100644
--- a/gcc/ada/gcc-interface/gigi.h
+++ b/gcc/ada/gcc-interface/gigi.h
@@ -312,9 +312,9 @@ extern void post_error_ne_tree (const char *msg, Node_Id node, Entity_Id ent,
 extern void post_error_ne_tree_2 (const char *msg, Node_Id node, Entity_Id ent,
                                   tree t, int num);
 
-/* Return a label to branch to for the exception type in KIND or NULL_TREE
+/* Return a label to branch to for the exception type in KIND or Empty
    if none.  */
-extern tree get_exception_label (char kind);
+extern Entity_Id get_exception_label (char kind);
 
 /* If nonzero, pretend we are allocating at global level.  */
 extern int force_global;
diff --git a/gcc/ada/gcc-interface/misc.c b/gcc/ada/gcc-interface/misc.c
index 7e4b2e30286..4d7f432bff2 100644
--- a/gcc/ada/gcc-interface/misc.c
+++ b/gcc/ada/gcc-interface/misc.c
@@ -1373,6 +1373,23 @@ gnat_init_ts (void)
   MARK_TS_TYPED (EXIT_STMT);
 }
 
+/* Return the size of a tree with CODE, which is a language-specific tree code
+   in category tcc_constant, tcc_exceptional or tcc_type.  The default expects
+   never to be called.  */
+
+static size_t
+gnat_tree_size (enum tree_code code)
+{
+  gcc_checking_assert (code >= NUM_TREE_CODES);
+  switch (code)
+    {
+    case UNCONSTRAINED_ARRAY_TYPE:
+      return sizeof (tree_type_non_common);
+    default:
+      gcc_unreachable ();
+    }
+}
+
 /* Return the lang specific structure attached to NODE.  Allocate it (cleared)
    if needed.  */
 
@@ -1390,6 +1407,8 @@ get_lang_specific (tree node)
 #define LANG_HOOKS_NAME			"GNU Ada"
 #undef  LANG_HOOKS_IDENTIFIER_SIZE
 #define LANG_HOOKS_IDENTIFIER_SIZE	sizeof (struct tree_identifier)
+#undef  LANG_HOOKS_TREE_SIZE
+#define LANG_HOOKS_TREE_SIZE		gnat_tree_size
 #undef  LANG_HOOKS_INIT
 #define LANG_HOOKS_INIT			gnat_init
 #undef  LANG_HOOKS_OPTION_LANG_MASK
diff --git a/gcc/ada/gcc-interface/trans.c b/gcc/ada/gcc-interface/trans.c
index 8b094733806..d22d82ad610 100644
--- a/gcc/ada/gcc-interface/trans.c
+++ b/gcc/ada/gcc-interface/trans.c
@@ -211,9 +211,9 @@ typedef struct loop_info_d *loop_info;
 static GTY(()) vec<loop_info, va_gc> *gnu_loop_stack;
 
 /* The stacks for N_{Push,Pop}_*_Label.  */
-static GTY(()) vec<tree, va_gc> *gnu_constraint_error_label_stack;
-static GTY(()) vec<tree, va_gc> *gnu_storage_error_label_stack;
-static GTY(()) vec<tree, va_gc> *gnu_program_error_label_stack;
+static vec<Entity_Id> gnu_constraint_error_label_stack;
+static vec<Entity_Id> gnu_storage_error_label_stack;
+static vec<Entity_Id> gnu_program_error_label_stack;
 
 /* Map GNAT tree codes to GCC tree codes for simple expressions.  */
 static enum tree_code gnu_codes[Number_Node_Kinds];
@@ -226,7 +226,6 @@ static void record_code_position (Node_Id);
 static void insert_code_for (Node_Id);
 static void add_cleanup (tree, Node_Id);
 static void add_stmt_list (List_Id);
-static void push_exception_label_stack (vec<tree, va_gc> **, Entity_Id);
 static tree build_stmt_group (List_Id, bool);
 static inline bool stmt_group_may_fallthru (void);
 static enum gimplify_status gnat_gimplify_stmt (tree *);
@@ -647,9 +646,10 @@ gigi (Node_Id gnat_root,
   gnat_install_builtins ();
 
   vec_safe_push (gnu_except_ptr_stack, NULL_TREE);
-  vec_safe_push (gnu_constraint_error_label_stack, NULL_TREE);
-  vec_safe_push (gnu_storage_error_label_stack, NULL_TREE);
-  vec_safe_push (gnu_program_error_label_stack, NULL_TREE);
+
+  gnu_constraint_error_label_stack.safe_push (Empty);
+  gnu_storage_error_label_stack.safe_push (Empty);
+  gnu_program_error_label_stack.safe_push (Empty);
 
   /* Process any Pragma Ident for the main unit.  */
   if (Present (Ident_String (Main_Unit)))
@@ -5614,7 +5614,7 @@ Raise_Error_to_gnu (Node_Id gnat_node, tree *gnu_result_type_p)
   const bool with_extra_info
     = Exception_Extra_Info
       && !No_Exception_Handlers_Set ()
-      && !get_exception_label (kind);
+      && No (get_exception_label (kind));
   tree gnu_result = NULL_TREE, gnu_cond = NULL_TREE;
 
   /* The following processing is not required for correctness.  Its purpose is
@@ -7271,8 +7271,9 @@ gnat_to_gnu (Node_Id gnat_node)
       break;
 
     case N_Goto_Statement:
-      gnu_result
-	= build1 (GOTO_EXPR, void_type_node, gnat_to_gnu (Name (gnat_node)));
+      gnu_expr = gnat_to_gnu (Name (gnat_node));
+      gnu_result = build1 (GOTO_EXPR, void_type_node, gnu_expr);
+      TREE_USED (gnu_expr) = 1;
       break;
 
     /***************************/
@@ -7492,30 +7493,36 @@ gnat_to_gnu (Node_Id gnat_node)
       break;
 
     case N_Push_Constraint_Error_Label:
-      push_exception_label_stack (&gnu_constraint_error_label_stack,
-				  Exception_Label (gnat_node));
+      gnu_constraint_error_label_stack.safe_push (Exception_Label (gnat_node));
       break;
 
     case N_Push_Storage_Error_Label:
-      push_exception_label_stack (&gnu_storage_error_label_stack,
-				  Exception_Label (gnat_node));
+      gnu_storage_error_label_stack.safe_push (Exception_Label (gnat_node));
       break;
 
     case N_Push_Program_Error_Label:
-      push_exception_label_stack (&gnu_program_error_label_stack,
-				  Exception_Label (gnat_node));
+      gnu_program_error_label_stack.safe_push (Exception_Label (gnat_node));
       break;
 
     case N_Pop_Constraint_Error_Label:
-      gnu_constraint_error_label_stack->pop ();
+      gnat_temp = gnu_constraint_error_label_stack.pop ();
+      if (Present (gnat_temp)
+	  && !TREE_USED (gnat_to_gnu_entity (gnat_temp, NULL_TREE, false)))
+	Warn_If_No_Local_Raise (gnat_temp);
       break;
 
     case N_Pop_Storage_Error_Label:
-      gnu_storage_error_label_stack->pop ();
+      gnat_temp = gnu_storage_error_label_stack.pop ();
+      if (Present (gnat_temp)
+	  && !TREE_USED (gnat_to_gnu_entity (gnat_temp, NULL_TREE, false)))
+	Warn_If_No_Local_Raise (gnat_temp);
       break;
 
     case N_Pop_Program_Error_Label:
-      gnu_program_error_label_stack->pop ();
+      gnat_temp = gnu_program_error_label_stack.pop ();
+      if (Present (gnat_temp)
+	  && !TREE_USED (gnat_to_gnu_entity (gnat_temp, NULL_TREE, false)))
+	Warn_If_No_Local_Raise (gnat_temp);
       break;
 
     /******************************/
@@ -8029,20 +8036,6 @@ gnat_to_gnu_external (Node_Id gnat_node)
   return gnu_result;
 }
 
-/* Subroutine of above to push the exception label stack.  GNU_STACK is
-   a pointer to the stack to update and GNAT_LABEL, if present, is the
-   label to push onto the stack.  */
-
-static void
-push_exception_label_stack (vec<tree, va_gc> **gnu_stack, Entity_Id gnat_label)
-{
-  tree gnu_label = (Present (gnat_label)
-		    ? gnat_to_gnu_entity (gnat_label, NULL_TREE, false)
-		    : NULL_TREE);
-
-  vec_safe_push (*gnu_stack, gnu_label);
-}
-
 /* Return true if the statement list STMT_LIST is empty.  */
 
 static bool
@@ -10226,28 +10219,28 @@ post_error_ne_tree_2 (const char *msg, Node_Id node, Entity_Id ent, tree t,
   post_error_ne_tree (msg, node, ent, t);
 }
 
-/* Return a label to branch to for the exception type in KIND or NULL_TREE
+/* Return a label to branch to for the exception type in KIND or Empty
    if none.  */
 
-tree
+Entity_Id
 get_exception_label (char kind)
 {
   switch (kind)
     {
     case N_Raise_Constraint_Error:
-      return gnu_constraint_error_label_stack->last ();
+      return gnu_constraint_error_label_stack.last ();
 
     case N_Raise_Storage_Error:
-      return gnu_storage_error_label_stack->last ();
+      return gnu_storage_error_label_stack.last ();
 
     case N_Raise_Program_Error:
-      return gnu_program_error_label_stack->last ();
+      return gnu_program_error_label_stack.last ();
 
     default:
-      break;
+      return Empty;
     }
 
-  return NULL_TREE;
+  gcc_unreachable ();
 }
 
 /* Return the decl for the current elaboration procedure.  */
diff --git a/gcc/ada/gcc-interface/utils.c b/gcc/ada/gcc-interface/utils.c
index 6718da45a9a..bad5aeade13 100644
--- a/gcc/ada/gcc-interface/utils.c
+++ b/gcc/ada/gcc-interface/utils.c
@@ -101,7 +101,7 @@ static tree handle_vector_type_attribute (tree *, tree, tree, int, bool *);
 
 /* Fake handler for attributes we don't properly support, typically because
    they'd require dragging a lot of the common-c front-end circuitry.  */
-static tree fake_attribute_handler      (tree *, tree, tree, int, bool *);
+static tree fake_attribute_handler (tree *, tree, tree, int, bool *);
 
 /* Table of machine-independent internal attributes for Ada.  We support
    this minimal set of attributes to accommodate the needs of builtins.  */
@@ -222,8 +222,9 @@ static GTY((deletable)) tree free_block_chain;
 /* A hash table of padded types.  It is modelled on the generic type
    hash table in tree.c, which must thus be used as a reference.  */
 
-struct GTY((for_user)) pad_type_hash {
-  unsigned long hash;
+struct GTY((for_user)) pad_type_hash
+{
+  hashval_t hash;
   tree type;
 };
 
@@ -3595,6 +3596,10 @@ max_size (tree exp, bool max_p)
     case tcc_constant:
       return exp;
 
+    case tcc_exceptional:
+      gcc_assert (code == SSA_NAME);
+      return exp;
+
     case tcc_vl_exp:
       if (code == CALL_EXPR)
 	{
@@ -4245,10 +4250,13 @@ convert (tree type, tree expr)
 	return convert (type, TREE_OPERAND (expr, 0));
 
       /* If the inner type is of self-referential size and the expression type
-	 is a record, do this as an unchecked conversion.  But first pad the
-	 expression if possible to have the same size on both sides.  */
+	 is a record, do this as an unchecked conversion unless both types are
+	 essentially the same.  But first pad the expression if possible to
+	 have the same size on both sides.  */
       if (ecode == RECORD_TYPE
-	  && CONTAINS_PLACEHOLDER_P (DECL_SIZE (TYPE_FIELDS (type))))
+	  && CONTAINS_PLACEHOLDER_P (DECL_SIZE (TYPE_FIELDS (type)))
+	  && TYPE_MAIN_VARIANT (etype)
+	     != TYPE_MAIN_VARIANT (TREE_TYPE (TYPE_FIELDS (type))))
 	{
 	  if (TREE_CODE (TYPE_SIZE (etype)) == INTEGER_CST)
 	    expr = convert (maybe_pad_type (etype, TYPE_SIZE (type), 0, Empty,
diff --git a/gcc/ada/gcc-interface/utils2.c b/gcc/ada/gcc-interface/utils2.c
index 321bdbfbe47..7f3a3d3ff1a 100644
--- a/gcc/ada/gcc-interface/utils2.c
+++ b/gcc/ada/gcc-interface/utils2.c
@@ -1788,9 +1788,10 @@ build_call_n_expr (tree fndecl, int n, ...)
    MSG gives the exception's identity for the call to Local_Raise, if any.  */
 
 static tree
-build_goto_raise (tree label, int msg)
+build_goto_raise (Entity_Id gnat_label, int msg)
 {
-  tree gnu_result = build1 (GOTO_EXPR, void_type_node, label);
+  tree gnu_label = gnat_to_gnu_entity (gnat_label, NULL_TREE, false);
+  tree gnu_result = build1 (GOTO_EXPR, void_type_node, gnu_label);
   Entity_Id local_raise = Get_Local_Raise_Call_Entity ();
 
   /* If Local_Raise is present, build Local_Raise (Exception'Identity).  */
@@ -1808,6 +1809,7 @@ build_goto_raise (tree label, int msg)
 	= build2 (COMPOUND_EXPR, void_type_node, gnu_call, gnu_result);
     }
 
+  TREE_USED (gnu_label) = 1;
   return gnu_result;
 }
 
@@ -1860,13 +1862,13 @@ expand_sloc (Node_Id gnat_node, tree *filename, tree *line, tree *col)
 tree
 build_call_raise (int msg, Node_Id gnat_node, char kind)
 {
+  Entity_Id gnat_label = get_exception_label (kind);
   tree fndecl = gnat_raise_decls[msg];
-  tree label = get_exception_label (kind);
   tree filename, line;
 
   /* If this is to be done as a goto, handle that case.  */
-  if (label)
-    return build_goto_raise (label, msg);
+  if (Present (gnat_label))
+    return build_goto_raise (gnat_label, msg);
 
   expand_sloc (gnat_node, &filename, &line, NULL);
 
@@ -1884,13 +1886,13 @@ build_call_raise (int msg, Node_Id gnat_node, char kind)
 tree
 build_call_raise_column (int msg, Node_Id gnat_node, char kind)
 {
+  Entity_Id gnat_label = get_exception_label (kind);
   tree fndecl = gnat_raise_decls_ext[msg];
-  tree label = get_exception_label (kind);
   tree filename, line, col;
 
   /* If this is to be done as a goto, handle that case.  */
-  if (label)
-    return build_goto_raise (label, msg);
+  if (Present (gnat_label))
+    return build_goto_raise (gnat_label, msg);
 
   expand_sloc (gnat_node, &filename, &line, &col);
 
@@ -1909,13 +1911,13 @@ tree
 build_call_raise_range (int msg, Node_Id gnat_node, char kind,
 			tree index, tree first, tree last)
 {
+  Entity_Id gnat_label = get_exception_label (kind);
   tree fndecl = gnat_raise_decls_ext[msg];
-  tree label = get_exception_label (kind);
   tree filename, line, col;
 
   /* If this is to be done as a goto, handle that case.  */
-  if (label)
-    return build_goto_raise (label, msg);
+  if (Present (gnat_label))
+    return build_goto_raise (gnat_label, msg);
 
   expand_sloc (gnat_node, &filename, &line, &col);
 
diff --git a/gcc/ada/gnat_rm.texi b/gcc/ada/gnat_rm.texi
index 8ed58c4fc7f..b042e2be3e1 100644
--- a/gcc/ada/gnat_rm.texi
+++ b/gcc/ada/gnat_rm.texi
@@ -21,7 +21,7 @@
 
 @copying
 @quotation
-GNAT Reference Manual , Sep 29, 2017
+GNAT Reference Manual , Oct 14, 2017
 
 AdaCore
 
@@ -9413,11 +9413,20 @@ that it is separately controllable using pragma @code{Assertion_Policy}.
 This aspect provides a light-weight mechanism for loops and quantified
 expressions over container types, without the overhead imposed by the tampering
 checks of standard Ada 2012 iterators. The value of the aspect is an aggregate
-with four named components: @code{First}, @code{Next}, @code{Has_Element}, and @code{Element} (the
-last one being optional). When only 3 components are specified, only the
-@code{for .. in} form of iteration over cursors is available. When all 4 components
-are specified, both this form and the @code{for .. of} form of iteration over
-elements are available. The following is a typical example of use:
+with six named components, or which the last three are optional: @code{First},
+
+@quotation
+
+@code{Next}, @code{Has_Element},`@w{`}Element`@w{`}, @code{Last}, and @code{Previous}.
+@end quotation
+
+When only the first three components are specified, only the
+@code{for .. in} form of iteration over cursors is available. When @code{Element}
+is specified, both this form and the @code{for .. of} form of iteration over
+elements are available. If the last two components are specified, reverse
+iterations over the container can be specified (analogous to what can be done
+over predefined containers that support the Reverse_Iterator interface).
+The following is a typical example of use:
 
 @example
 type List is private with
diff --git a/gcc/ada/gnat_ugn.texi b/gcc/ada/gnat_ugn.texi
index a39c2572be0..947506799a5 100644
--- a/gcc/ada/gnat_ugn.texi
+++ b/gcc/ada/gnat_ugn.texi
@@ -21,7 +21,7 @@
 
 @copying
 @quotation
-GNAT User's Guide for Native Platforms , Oct 09, 2017
+GNAT User's Guide for Native Platforms , Oct 20, 2017
 
 AdaCore
 
@@ -8809,19 +8809,6 @@ in the compiler sources for details in files @code{scos.ads} and
 @code{scos.adb}.
 @end table
 
-@geindex -fdump-xref (gcc)
-
-
-@table @asis
-
-@item @code{-fdump-xref}
-
-Generates cross reference information in GLI files for C and C++ sources.
-The GLI files have the same syntax as the ALI files for Ada, and can be used
-for source navigation in IDEs and on the command line using e.g. gnatxref
-and the @code{--ext=gli} switch.
-@end table
-
 @geindex -flto (gcc)
 
 
@@ -8830,8 +8817,9 @@ and the @code{--ext=gli} switch.
 @item @code{-flto[=@emph{n}]}
 
 Enables Link Time Optimization. This switch must be used in conjunction
-with the traditional @code{-Ox} switches and instructs the compiler to
-defer most optimizations until the link stage. The advantage of this
+with the @code{-Ox} switches (but not with the @code{-gnatn} switch
+since it is a full replacement for the latter) and instructs the compiler
+to defer most optimizations until the link stage. The advantage of this
 approach is that the compiler can do a whole-program analysis and choose
 the best interprocedural optimization strategy based on a complete view
 of the program, instead of a fragmentary view with the usual approach.
@@ -12474,8 +12462,8 @@ should not complain at you.
 This switch activates warnings for exception usage when pragma Restrictions
 (No_Exception_Propagation) is in effect. Warnings are given for implicit or
 explicit exception raises which are not covered by a local handler, and for
-exception handlers which do not cover a local raise. The default is that these
-warnings are not given.
+exception handlers which do not cover a local raise. The default is that
+these warnings are given for units that contain exception handlers.
 
 @item @code{-gnatw.X}
 
@@ -17949,9 +17937,9 @@ Do not look for library files in the system default directory.
 @item @code{--ext=@emph{extension}}
 
 Specify an alternate ali file extension. The default is @code{ali} and other
-extensions (e.g. @code{gli} for C/C++ sources when using @code{-fdump-xref})
-may be specified via this switch. Note that if this switch overrides the
-default, which means that only the new extension will be considered.
+extensions (e.g. @code{gli} for C/C++ sources) may be specified via this switch.
+Note that if this switch overrides the default, which means that only the
+new extension will be considered.
 @end table
 
 @geindex --RTS (gnatxref)
@@ -22901,12 +22889,12 @@ combine a dimensioned and dimensionless value.  Thus an expression such as
 @code{Acceleration}.
 
 The dimensionality checks for relationals use the same rules as
-for "+" and "-"; thus
+for "+" and "-", except when comparing to a literal; thus
 
 @quotation
 
 @example
-acc > 10.0
+acc > len
 @end example
 @end quotation
 
@@ -22915,12 +22903,21 @@ is equivalent to
 @quotation
 
 @example
-acc-10.0 > 0.0
+acc-len > 0.0
+@end example
+@end quotation
+
+and is thus illegal, but
+
+@quotation
+
+@example
+acc > 10.0
 @end example
 @end quotation
 
-and is thus illegal. Analogously a conditional expression
-requires the same dimension vector for each branch.
+is accepted with a warning. Analogously a conditional expression requires the
+same dimension vector for each branch (with no exception for literals).
 
 The dimension vector of a type conversion @code{T(@emph{expr})} is defined
 as follows, based on the nature of @code{T}:
@@ -27187,8 +27184,62 @@ elaborated.
 
 
 The sequence by which the elaboration code of all units within a partition is
-executed is referred to as @strong{elaboration order}. The elaboration order depends
-on the following factors:
+executed is referred to as @strong{elaboration order}.
+
+Within a single unit, elaboration code is executed in sequential order.
+
+@example
+package body Client is
+   Result : ... := Server.Func;
+
+   procedure Proc is
+      package Inst is new Server.Gen;
+   begin
+      Inst.Eval (Result);
+   end Proc;
+begin
+   Proc;
+end Client;
+@end example
+
+In the example above, the elaboration order within package body @code{Client} is
+as follows:
+
+
+@enumerate 
+
+@item 
+The object declaration of @code{Result} is elaborated.
+
+
+@itemize *
+
+@item 
+Function @code{Server.Func} is invoked.
+@end itemize
+
+@item 
+The subprogram body of @code{Proc} is elaborated.
+
+@item 
+Procedure @code{Proc} is invoked.
+
+
+@itemize *
+
+@item 
+Generic unit @code{Server.Gen} is instantiated as @code{Inst}.
+
+@item 
+Instance @code{Inst} is elaborated.
+
+@item 
+Procedure @code{Inst.Eval} is invoked.
+@end itemize
+@end enumerate
+
+The elaboration order of all units within a partition depends on the following
+factors:
 
 
 @itemize *
@@ -27689,7 +27740,7 @@ dynamic model is in effect, GNAT assumes that all code within all units in
 a partition is elaboration code. GNAT performs very few diagnostics and
 generates run-time checks to verify the elaboration order of a program. This
 behavior is identical to that specified by the Ada Reference Manual. The
-dynamic model is enabled with compilation switch @code{-gnatE}.
+dynamic model is enabled with compiler switch @code{-gnatE}.
 @end itemize
 
 @geindex Static elaboration model
@@ -28001,7 +28052,7 @@ elaborated prior to the body of @code{Static_Model}.
 The SPARK model is identical to the static model in its handling of internal
 targets. The SPARK model, however, requires explicit @code{Elaborate} or
 @code{Elaborate_All} pragmas to be present in the program when a target is
-external, and emits hard errors instead of warnings:
+external, and compiler switch @code{-gnatd.v} is in effect.
 
 @example
 1. with Server;
@@ -28146,7 +28197,7 @@ code.
 @emph{Switch to more permissive elaboration model}
 
 If the compilation was performed using the static model, enable the dynamic
-model with compilation switch @code{-gnatE}. GNAT will no longer generate
+model with compiler switch @code{-gnatE}. GNAT will no longer generate
 implicit @code{Elaborate} and @code{Elaborate_All} pragmas, resulting in a behavior
 identical to that specified by the Ada Reference Manual. The binder will
 generate an executable program that may or may not raise @code{Program_Error},
@@ -28711,6 +28762,22 @@ When this switch is in effect, GNAT will ignore @code{'Access} of an entry,
 operator, or subprogram when the static model is in effect.
 @end table
 
+@geindex -gnatd.v (gnat)
+
+
+@table @asis
+
+@item @code{-gnatd.v}
+
+Enforce SPARK elaboration rules in SPARK code
+
+When this switch is in effect, GNAT will enforce the SPARK rules of
+elaboration as defined in the SPARK Reference Manual, section 7.7. As a
+result, constructs which violate the SPARK elaboration rules are no longer
+accepted, even if GNAT is able to statically ensure that these constructs
+will not lead to ABE problems.
+@end table
+
 @geindex -gnatd.y (gnat)
 
 
@@ -28785,7 +28852,7 @@ it will provide detailed traceback when an implicit @code{Elaborate} or
 @emph{SPARK model}
 
 GNAT will indicate how an elaboration requirement is met by the context of
-a unit.
+a unit. This diagnostic requires compiler switch @code{-gnatd.v}.
 
 @example
 1. with Server; pragma Elaborate_All (Server);
@@ -28846,8 +28913,8 @@ none of the binder or compiler switches. If the binder succeeds in finding an
 elaboration order, then apart from possible cases involing dispatching calls
 and access-to-subprogram types, the program is free of elaboration errors.
 If it is important for the program to be portable to compilers other than GNAT,
-then the programmer should use compilation switch @code{-gnatel} and
-consider the messages about missing or implicitly created @code{Elaborate} and
+then the programmer should use compiler switch @code{-gnatel} and consider
+the messages about missing or implicitly created @code{Elaborate} and
 @code{Elaborate_All} pragmas.
 
 If the binder reports an elaboration circularity, the programmer has several
diff --git a/gcc/ada/layout.adb b/gcc/ada/layout.adb
index 34c5b5d0f9a..52e84526ca4 100644
--- a/gcc/ada/layout.adb
+++ b/gcc/ada/layout.adb
@@ -843,7 +843,7 @@ package body Layout is
    -- Set_Elem_Alignment --
    ------------------------
 
-   procedure Set_Elem_Alignment (E : Entity_Id) is
+   procedure Set_Elem_Alignment (E : Entity_Id; Align : Nat := 0) is
    begin
       --  Do not set alignment for packed array types, this is handled in the
       --  backend.
@@ -869,15 +869,12 @@ package body Layout is
          return;
       end if;
 
-      --  Here we calculate the alignment as the largest power of two multiple
-      --  of System.Storage_Unit that does not exceed either the object size of
-      --  the type, or the maximum allowed alignment.
+      --  We attempt to set the alignment in all the other cases
 
       declare
          S : Int;
          A : Nat;
-
-         Max_Alignment : Nat;
+         M : Nat;
 
       begin
          --  The given Esize may be larger that int'last because of a previous
@@ -908,7 +905,7 @@ package body Layout is
            and then S = 8
            and then Is_Floating_Point_Type (E)
          then
-            Max_Alignment := Ttypes.Target_Double_Float_Alignment;
+            M := Ttypes.Target_Double_Float_Alignment;
 
          --  If the default alignment of "double" or larger scalar types is
          --  specifically capped, enforce the cap.
@@ -917,18 +914,27 @@ package body Layout is
            and then S >= 8
            and then Is_Scalar_Type (E)
          then
-            Max_Alignment := Ttypes.Target_Double_Scalar_Alignment;
+            M := Ttypes.Target_Double_Scalar_Alignment;
 
          --  Otherwise enforce the overall alignment cap
 
          else
-            Max_Alignment := Ttypes.Maximum_Alignment;
+            M := Ttypes.Maximum_Alignment;
          end if;
 
-         A := 1;
-         while 2 * A <= Max_Alignment and then 2 * A <= S loop
-            A := 2 * A;
-         end loop;
+         --  We calculate the alignment as the largest power-of-two multiple
+         --  of System.Storage_Unit that does not exceed the object size of
+         --  the type and the maximum allowed alignment, if none was specified.
+         --  Otherwise we only cap it to the maximum allowed alignment.
+
+         if Align = 0 then
+            A := 1;
+            while 2 * A <= S and then 2 * A <= M loop
+               A := 2 * A;
+            end loop;
+         else
+            A := Nat'Min (Align, M);
+         end if;
 
          --  If alignment is currently not set, then we can safely set it to
          --  this new calculated value.
diff --git a/gcc/ada/layout.ads b/gcc/ada/layout.ads
index 57aa93e4f5a..246970fd8fd 100644
--- a/gcc/ada/layout.ads
+++ b/gcc/ada/layout.ads
@@ -74,10 +74,11 @@ package Layout is
    --  types, the RM_Size is simply set to zero. This routine also sets
    --  the Is_Constrained flag in Def_Id.
 
-   procedure Set_Elem_Alignment (E : Entity_Id);
+   procedure Set_Elem_Alignment (E : Entity_Id; Align : Nat := 0);
    --  The front end always sets alignments for elementary types by calling
    --  this procedure. Note that we have to do this for discrete types (since
    --  the Alignment attribute is static), so we might as well do it for all
-   --  elementary types, since the processing is the same.
+   --  elementary types, as the processing is the same. If Align is nonzero,
+   --  it is an external alignment setting that we must respect.
 
 end Layout;
diff --git a/gcc/ada/lib-load.adb b/gcc/ada/lib-load.adb
index 977567d4983..0b0ea7f5057 100644
--- a/gcc/ada/lib-load.adb
+++ b/gcc/ada/lib-load.adb
@@ -214,34 +214,36 @@ package body Lib.Load is
       Unum := Units.Last;
 
       Units.Table (Unum) :=
-        (Cunit             => Cunit,
-         Cunit_Entity      => Cunit_Entity,
-         Dependency_Num    => 0,
-         Dynamic_Elab      => False,
-         Error_Location    => Sloc (With_Node),
-         Expected_Unit     => Spec_Name,
-         Fatal_Error       => Error_Detected,
-         Generate_Code     => False,
-         Has_RACW          => False,
-         Filler            => False,
-         Ident_String      => Empty,
+        (Cunit                  => Cunit,
+         Cunit_Entity           => Cunit_Entity,
+         Dependency_Num         => 0,
+         Dynamic_Elab           => False,
+         Error_Location         => Sloc (With_Node),
+         Expected_Unit          => Spec_Name,
+         Fatal_Error            => Error_Detected,
+         Generate_Code          => False,
+         Has_RACW               => False,
+         Filler                 => False,
+         Ident_String           => Empty,
 
          Is_Predefined_Renaming => Ren_Name,
          Is_Predefined_Unit     => Pre_Name or Ren_Name,
          Is_Internal_Unit       => Pre_Name or Ren_Name or GNAT_Name,
          Filler2                => False,
 
-         Loading           => False,
-         Main_Priority     => Default_Main_Priority,
-         Main_CPU          => Default_Main_CPU,
-         Munit_Index       => 0,
-         No_Elab_Code_All  => False,
-         Serial_Number     => 0,
-         Source_Index      => No_Source_File,
-         Unit_File_Name    => Fname,
-         Unit_Name         => Spec_Name,
-         Version           => 0,
-         OA_Setting        => 'O');
+         Loading                => False,
+         Main_Priority          => Default_Main_Priority,
+         Main_CPU               => Default_Main_CPU,
+         Primary_Stack_Count    => 0,
+         Sec_Stack_Count        => 0,
+         Munit_Index            => 0,
+         No_Elab_Code_All       => False,
+         Serial_Number          => 0,
+         Source_Index           => No_Source_File,
+         Unit_File_Name         => Fname,
+         Unit_Name              => Spec_Name,
+         Version                => 0,
+         OA_Setting             => 'O');
 
       Set_Comes_From_Source_Default (Save_CS);
       Set_Error_Posted (Cunit_Entity);
@@ -350,34 +352,37 @@ package body Lib.Load is
          end if;
 
          Units.Table (Main_Unit) :=
-           (Cunit             => Empty,
-            Cunit_Entity      => Empty,
-            Dependency_Num    => 0,
-            Dynamic_Elab      => False,
-            Error_Location    => No_Location,
-            Expected_Unit     => No_Unit_Name,
-            Fatal_Error       => None,
-            Generate_Code     => False,
-            Has_RACW          => False,
-            Filler            => False,
-            Ident_String      => Empty,
+           (Cunit                  => Empty,
+            Cunit_Entity           => Empty,
+            Dependency_Num         => 0,
+            Dynamic_Elab           => False,
+            Error_Location         => No_Location,
+            Expected_Unit          => No_Unit_Name,
+            Fatal_Error            => None,
+            Generate_Code          => False,
+            Has_RACW               => False,
+            Filler                 => False,
+            Ident_String           => Empty,
 
             Is_Predefined_Renaming => Ren_Name,
             Is_Predefined_Unit     => Pre_Name or Ren_Name,
             Is_Internal_Unit       => Pre_Name or Ren_Name or GNAT_Name,
             Filler2                => False,
 
-            Loading           => True,
-            Main_Priority     => Default_Main_Priority,
-            Main_CPU          => Default_Main_CPU,
-            Munit_Index       => 0,
-            No_Elab_Code_All  => False,
-            Serial_Number     => 0,
-            Source_Index      => Main_Source_File,
-            Unit_File_Name    => Fname,
-            Unit_Name         => No_Unit_Name,
-            Version           => Version,
-            OA_Setting        => 'O');
+            Loading                => True,
+            Main_Priority          => Default_Main_Priority,
+            Main_CPU               => Default_Main_CPU,
+            Primary_Stack_Count    => 0,
+            Sec_Stack_Count        => 0,
+
+            Munit_Index            => 0,
+            No_Elab_Code_All       => False,
+            Serial_Number          => 0,
+            Source_Index           => Main_Source_File,
+            Unit_File_Name         => Fname,
+            Unit_Name              => No_Unit_Name,
+            Version                => Version,
+            OA_Setting             => 'O');
       end if;
    end Load_Main_Source;
 
@@ -728,34 +733,36 @@ package body Lib.Load is
 
          if Src_Ind > No_Source_File then
             Units.Table (Unum) :=
-              (Cunit             => Empty,
-               Cunit_Entity      => Empty,
-               Dependency_Num    => 0,
-               Dynamic_Elab      => False,
-               Error_Location    => Sloc (Error_Node),
-               Expected_Unit     => Uname_Actual,
-               Fatal_Error       => None,
-               Generate_Code     => False,
-               Has_RACW          => False,
-               Filler            => False,
-               Ident_String      => Empty,
+              (Cunit                  => Empty,
+               Cunit_Entity           => Empty,
+               Dependency_Num         => 0,
+               Dynamic_Elab           => False,
+               Error_Location         => Sloc (Error_Node),
+               Expected_Unit          => Uname_Actual,
+               Fatal_Error            => None,
+               Generate_Code          => False,
+               Has_RACW               => False,
+               Filler                 => False,
+               Ident_String           => Empty,
 
                Is_Predefined_Renaming => Ren_Name,
                Is_Predefined_Unit     => Pre_Name or Ren_Name,
                Is_Internal_Unit       => Pre_Name or Ren_Name or GNAT_Name,
                Filler2                => False,
 
-               Loading           => True,
-               Main_Priority     => Default_Main_Priority,
-               Main_CPU          => Default_Main_CPU,
-               Munit_Index       => 0,
-               No_Elab_Code_All  => False,
-               Serial_Number     => 0,
-               Source_Index      => Src_Ind,
-               Unit_File_Name    => Fname,
-               Unit_Name         => Uname_Actual,
-               Version           => Source_Checksum (Src_Ind),
-               OA_Setting        => 'O');
+               Loading                => True,
+               Main_Priority          => Default_Main_Priority,
+               Main_CPU               => Default_Main_CPU,
+               Primary_Stack_Count    => 0,
+               Sec_Stack_Count        => 0,
+               Munit_Index            => 0,
+               No_Elab_Code_All       => False,
+               Serial_Number          => 0,
+               Source_Index           => Src_Ind,
+               Unit_File_Name         => Fname,
+               Unit_Name              => Uname_Actual,
+               Version                => Source_Checksum (Src_Ind),
+               OA_Setting             => 'O');
 
             --  Parse the new unit
 
diff --git a/gcc/ada/lib-writ.adb b/gcc/ada/lib-writ.adb
index d263b05dc1c..47109b4e3f9 100644
--- a/gcc/ada/lib-writ.adb
+++ b/gcc/ada/lib-writ.adb
@@ -96,6 +96,8 @@ package body Lib.Writ is
          Main_CPU               => -1,
          Munit_Index            => 0,
          No_Elab_Code_All       => False,
+         Primary_Stack_Count    => 0,
+         Sec_Stack_Count        => 0,
          Serial_Number          => 0,
          Version                => 0,
          Error_Location         => No_Location,
@@ -157,6 +159,8 @@ package body Lib.Writ is
          Main_CPU               => -1,
          Munit_Index            => 0,
          No_Elab_Code_All       => False,
+         Primary_Stack_Count    => 0,
+         Sec_Stack_Count        => 0,
          Serial_Number          => 0,
          Version                => 0,
          Error_Location         => No_Location,
@@ -616,6 +620,19 @@ package body Lib.Writ is
 
          Write_With_Lines;
 
+         --  Generate task stack lines
+
+         if Primary_Stack_Count (Unit_Num) > 0
+           or else Sec_Stack_Count (Unit_Num) > 0
+         then
+            Write_Info_Initiate ('T');
+            Write_Info_Char (' ');
+            Write_Info_Int (Primary_Stack_Count (Unit_Num));
+            Write_Info_Char (' ');
+            Write_Info_Int (Sec_Stack_Count (Unit_Num));
+            Write_Info_EOL;
+         end if;
+
          --  Generate the linker option lines
 
          for J in 1 .. Linker_Option_Lines.Last loop
diff --git a/gcc/ada/lib-writ.ads b/gcc/ada/lib-writ.ads
index f113b0a5993..a959e94e2fc 100644
--- a/gcc/ada/lib-writ.ads
+++ b/gcc/ada/lib-writ.ads
@@ -6,7 +6,7 @@
 --                                                                          --
 --                                 S p e c                                  --
 --                                                                          --
---          Copyright (C) 1992-2016, Free Software Foundation, Inc.         --
+--          Copyright (C) 1992-2017, Free Software Foundation, Inc.         --
 --                                                                          --
 -- GNAT is free software;  you can  redistribute it  and/or modify it under --
 -- terms of the  GNU General Public License as published  by the Free Soft- --
@@ -670,14 +670,33 @@ package Lib.Writ is
    --      binder do the consistency check, but not include the unit in the
    --      partition closure (unless it is properly With'ed somewhere).
 
+   --  --------------------
+   --  -- T  Task Stacks --
+   --  --------------------
+
+   --  Following the W lines (if any, or the U line if not), is an optional
+   --  line that identifies the number of default-sized primary and secondary
+   --  stacks that the binder needs to create for the tasks declared within the
+   --  unit. For each compilation unit, a line is present in the form:
+
+   --    T primary-stack-quantity secondary-stack-quantity
+
+   --     The first parameter of T defines the number of task objects declared
+   --     in the unit that have no Storage_Size specified. The second parameter
+   --     defines the number of task objects declared in the unit that have no
+   --     Secondary_Stack_Size specified. These values are non-zero only if
+   --     the restrictions No_Implicit_Heap_Allocations or
+   --     No_Implicit_Task_Allocations are active.
+
    --  -----------------------
    --  -- L  Linker_Options --
    --  -----------------------
 
-   --  Following the W lines (if any, or the U line if not), are an optional
-   --  series of lines that indicates the usage of the pragma Linker_Options in
-   --  the associated unit. For each appearance of a pragma Linker_Options (or
-   --  Link_With) in the unit, a line is present with the form:
+   --  Following the T and W lines (if any, or the U line if not), are
+   --  an optional series of lines that indicates the usage of the pragma
+   --  Linker_Options in the associated unit. For each appearance of a pragma
+   --  Linker_Options (or Link_With) in the unit, a line is present with the
+   --  form:
 
    --    L "string"
 
diff --git a/gcc/ada/lib.adb b/gcc/ada/lib.adb
index 8de6f355d0c..02eb1987d8e 100644
--- a/gcc/ada/lib.adb
+++ b/gcc/ada/lib.adb
@@ -178,6 +178,16 @@ package body Lib is
       return Units.Table (U).OA_Setting;
    end OA_Setting;
 
+   function Primary_Stack_Count (U : Unit_Number_Type) return Int is
+   begin
+      return Units.Table (U).Primary_Stack_Count;
+   end Primary_Stack_Count;
+
+   function Sec_Stack_Count  (U : Unit_Number_Type) return Int is
+   begin
+      return Units.Table (U).Sec_Stack_Count;
+   end Sec_Stack_Count;
+
    function Source_Index (U : Unit_Number_Type) return Source_File_Index is
    begin
       return Units.Table (U).Source_Index;
@@ -1027,6 +1037,26 @@ package body Lib is
       return Get_Source_Unit (N1) = Get_Source_Unit (N2);
    end In_Same_Source_Unit;
 
+   -----------------------------------
+   -- Increment_Primary_Stack_Count --
+   -----------------------------------
+
+   procedure Increment_Primary_Stack_Count (Increment : Int) is
+      PSC : Int renames Units.Table (Current_Sem_Unit).Primary_Stack_Count;
+   begin
+      PSC := PSC + Increment;
+   end Increment_Primary_Stack_Count;
+
+   -------------------------------
+   -- Increment_Sec_Stack_Count --
+   -------------------------------
+
+   procedure Increment_Sec_Stack_Count (Increment : Int) is
+      SSC : Int renames Units.Table (Current_Sem_Unit).Sec_Stack_Count;
+   begin
+      SSC := SSC + Increment;
+   end Increment_Sec_Stack_Count;
+
    -----------------------------
    -- Increment_Serial_Number --
    -----------------------------
diff --git a/gcc/ada/lib.ads b/gcc/ada/lib.ads
index be6864a3e83..c9686992f5a 100644
--- a/gcc/ada/lib.ads
+++ b/gcc/ada/lib.ads
@@ -370,6 +370,20 @@ package Lib is
    --      This is a character field containing L if Optimize_Alignment mode
    --      was set locally, and O/T/S for Off/Time/Space default if not.
 
+   --    Primary_Stack_Count
+   --      The number of primary stacks belonging to tasks defined within the
+   --      unit that have no Storage_Size specified when the either restriction
+   --      No_Implicit_Heap_Allocations or No_Implicit_Task_Allocations is
+   --      active. Only used by the binder to generate stacks for these tasks
+   --      at bind time.
+
+   --    Sec_Stack_Count
+   --      The number of secondary stacks belonging to tasks defined within the
+   --      unit that have no Secondary_Stack_Size specified when the either
+   --      the No_Implicit_Heap_Allocations or No_Implicit_Task_Allocations
+   --      restrictions are active. Only used by the binder to generate stacks
+   --      for these tasks at bind time.
+
    --    Serial_Number
    --      This field holds a serial number used by New_Internal_Name to
    --      generate unique temporary numbers on a unit by unit basis. The
@@ -441,15 +455,20 @@ package Lib is
    function Generate_Code    (U : Unit_Number_Type) return Boolean;
    function Ident_String     (U : Unit_Number_Type) return Node_Id;
    function Has_RACW         (U : Unit_Number_Type) return Boolean;
-   function Is_Predefined_Renaming (U : Unit_Number_Type) return Boolean;
-   function Is_Internal_Unit       (U : Unit_Number_Type) return Boolean;
-   function Is_Predefined_Unit     (U : Unit_Number_Type) return Boolean;
+   function Is_Predefined_Renaming
+                             (U : Unit_Number_Type) return Boolean;
+   function Is_Internal_Unit (U : Unit_Number_Type) return Boolean;
+   function Is_Predefined_Unit
+                             (U : Unit_Number_Type) return Boolean;
    function Loading          (U : Unit_Number_Type) return Boolean;
    function Main_CPU         (U : Unit_Number_Type) return Int;
    function Main_Priority    (U : Unit_Number_Type) return Int;
    function Munit_Index      (U : Unit_Number_Type) return Nat;
    function No_Elab_Code_All (U : Unit_Number_Type) return Boolean;
    function OA_Setting       (U : Unit_Number_Type) return Character;
+   function Primary_Stack_Count
+                             (U : Unit_Number_Type) return Int;
+   function Sec_Stack_Count  (U : Unit_Number_Type) return Int;
    function Source_Index     (U : Unit_Number_Type) return Source_File_Index;
    function Unit_File_Name   (U : Unit_Number_Type) return File_Name_Type;
    function Unit_Name        (U : Unit_Number_Type) return Unit_Name_Type;
@@ -662,6 +681,13 @@ package Lib is
    --  source unit, the criterion being that Get_Source_Unit yields the
    --  same value for each argument.
 
+   procedure Increment_Primary_Stack_Count (Increment : Int);
+   --  Increment the Primary_Stack_Count field for the current unit by
+   --  Increment.
+
+   procedure Increment_Sec_Stack_Count (Increment : Int);
+   --  Increment the Sec_Stack_Count field for the current unit by Increment
+
    function Increment_Serial_Number return Nat;
    --  Increment Serial_Number field for current unit, and return the
    --  incremented value.
@@ -794,6 +820,8 @@ private
    pragma Inline (Fatal_Error);
    pragma Inline (Generate_Code);
    pragma Inline (Has_RACW);
+   pragma Inline (Increment_Primary_Stack_Count);
+   pragma Inline (Increment_Sec_Stack_Count);
    pragma Inline (Increment_Serial_Number);
    pragma Inline (Loading);
    pragma Inline (Main_CPU);
@@ -809,6 +837,8 @@ private
    pragma Inline (Is_Predefined_Renaming);
    pragma Inline (Is_Internal_Unit);
    pragma Inline (Is_Predefined_Unit);
+   pragma Inline (Primary_Stack_Count);
+   pragma Inline (Sec_Stack_Count);
    pragma Inline (Set_Loading);
    pragma Inline (Set_Main_CPU);
    pragma Inline (Set_Main_Priority);
@@ -822,28 +852,30 @@ private
    --  The Units Table
 
    type Unit_Record is record
-      Unit_File_Name    : File_Name_Type;
-      Unit_Name         : Unit_Name_Type;
-      Munit_Index       : Nat;
-      Expected_Unit     : Unit_Name_Type;
-      Source_Index      : Source_File_Index;
-      Cunit             : Node_Id;
-      Cunit_Entity      : Entity_Id;
-      Dependency_Num    : Int;
-      Ident_String      : Node_Id;
-      Main_Priority     : Int;
-      Main_CPU          : Int;
-      Serial_Number     : Nat;
-      Version           : Word;
-      Error_Location    : Source_Ptr;
-      Fatal_Error       : Fatal_Type;
-      Generate_Code     : Boolean;
-      Has_RACW          : Boolean;
-      Dynamic_Elab      : Boolean;
-      No_Elab_Code_All  : Boolean;
-      Filler            : Boolean;
-      Loading           : Boolean;
-      OA_Setting        : Character;
+      Unit_File_Name         : File_Name_Type;
+      Unit_Name              : Unit_Name_Type;
+      Munit_Index            : Nat;
+      Expected_Unit          : Unit_Name_Type;
+      Source_Index           : Source_File_Index;
+      Cunit                  : Node_Id;
+      Cunit_Entity           : Entity_Id;
+      Dependency_Num         : Int;
+      Ident_String           : Node_Id;
+      Main_Priority          : Int;
+      Main_CPU               : Int;
+      Primary_Stack_Count    : Int;
+      Sec_Stack_Count        : Int;
+      Serial_Number          : Nat;
+      Version                : Word;
+      Error_Location         : Source_Ptr;
+      Fatal_Error            : Fatal_Type;
+      Generate_Code          : Boolean;
+      Has_RACW               : Boolean;
+      Dynamic_Elab           : Boolean;
+      No_Elab_Code_All       : Boolean;
+      Filler                 : Boolean;
+      Loading                : Boolean;
+      OA_Setting             : Character;
 
       Is_Predefined_Renaming : Boolean;
       Is_Internal_Unit       : Boolean;
@@ -856,36 +888,38 @@ private
    --  written by Tree_Gen, we do not write uninitialized values to the file.
 
    for Unit_Record use record
-      Unit_File_Name    at  0 range 0 .. 31;
-      Unit_Name         at  4 range 0 .. 31;
-      Munit_Index       at  8 range 0 .. 31;
-      Expected_Unit     at 12 range 0 .. 31;
-      Source_Index      at 16 range 0 .. 31;
-      Cunit             at 20 range 0 .. 31;
-      Cunit_Entity      at 24 range 0 .. 31;
-      Dependency_Num    at 28 range 0 .. 31;
-      Ident_String      at 32 range 0 .. 31;
-      Main_Priority     at 36 range 0 .. 31;
-      Main_CPU          at 40 range 0 .. 31;
-      Serial_Number     at 44 range 0 .. 31;
-      Version           at 48 range 0 .. 31;
-      Error_Location    at 52 range 0 .. 31;
-      Fatal_Error       at 56 range 0 ..  7;
-      Generate_Code     at 57 range 0 ..  7;
-      Has_RACW          at 58 range 0 ..  7;
-      Dynamic_Elab      at 59 range 0 ..  7;
-      No_Elab_Code_All  at 60 range 0 ..  7;
-      Filler            at 61 range 0 ..  7;
-      OA_Setting        at 62 range 0 ..  7;
-      Loading           at 63 range 0 ..  7;
-
-      Is_Predefined_Renaming at 64 range 0 .. 7;
-      Is_Internal_Unit       at 65 range 0 .. 7;
-      Is_Predefined_Unit     at 66 range 0 .. 7;
-      Filler2                at 67 range 0 .. 7;
+      Unit_File_Name         at  0 range 0 .. 31;
+      Unit_Name              at  4 range 0 .. 31;
+      Munit_Index            at  8 range 0 .. 31;
+      Expected_Unit          at 12 range 0 .. 31;
+      Source_Index           at 16 range 0 .. 31;
+      Cunit                  at 20 range 0 .. 31;
+      Cunit_Entity           at 24 range 0 .. 31;
+      Dependency_Num         at 28 range 0 .. 31;
+      Ident_String           at 32 range 0 .. 31;
+      Main_Priority          at 36 range 0 .. 31;
+      Main_CPU               at 40 range 0 .. 31;
+      Primary_Stack_Count    at 44 range 0 .. 31;
+      Sec_Stack_Count        at 48 range 0 .. 31;
+      Serial_Number          at 52 range 0 .. 31;
+      Version                at 56 range 0 .. 31;
+      Error_Location         at 60 range 0 .. 31;
+      Fatal_Error            at 64 range 0 ..  7;
+      Generate_Code          at 65 range 0 ..  7;
+      Has_RACW               at 66 range 0 ..  7;
+      Dynamic_Elab           at 67 range 0 ..  7;
+      No_Elab_Code_All       at 68 range 0 ..  7;
+      Filler                 at 69 range 0 ..  7;
+      OA_Setting             at 70 range 0 ..  7;
+      Loading                at 71 range 0 ..  7;
+
+      Is_Predefined_Renaming at 72 range 0 .. 7;
+      Is_Internal_Unit       at 73 range 0 .. 7;
+      Is_Predefined_Unit     at 74 range 0 .. 7;
+      Filler2                at 75 range 0 .. 7;
    end record;
 
-   for Unit_Record'Size use 68 * 8;
+   for Unit_Record'Size use 76 * 8;
    --  This ensures that we did not leave out any fields
 
    package Units is new Table.Table (
diff --git a/gcc/ada/libgnarl/s-osinte__linux.ads b/gcc/ada/libgnarl/s-osinte__linux.ads
index 87da7ff01a5..a2ba537fb37 100644
--- a/gcc/ada/libgnarl/s-osinte__linux.ads
+++ b/gcc/ada/libgnarl/s-osinte__linux.ads
@@ -448,6 +448,9 @@ package System.OS_Interface is
       abstime : access timespec) return int;
    pragma Import (C, pthread_cond_timedwait, "pthread_cond_timedwait");
 
+   Relative_Timed_Wait : constant Boolean := False;
+   --  pthread_cond_timedwait requires an absolute delay time
+
    --------------------------
    -- POSIX.1c  Section 13 --
    --------------------------
diff --git a/gcc/ada/libgnarl/s-solita.adb b/gcc/ada/libgnarl/s-solita.adb
index bb38578b06f..a5485aa268d 100644
--- a/gcc/ada/libgnarl/s-solita.adb
+++ b/gcc/ada/libgnarl/s-solita.adb
@@ -44,6 +44,7 @@ with Ada.Exceptions.Is_Null_Occurrence;
 with System.Task_Primitives.Operations;
 with System.Tasking;
 with System.Stack_Checking;
+with System.Secondary_Stack;
 
 package body System.Soft_Links.Tasking is
 
@@ -52,6 +53,8 @@ package body System.Soft_Links.Tasking is
 
    use Ada.Exceptions;
 
+   use type System.Secondary_Stack.SS_Stack_Ptr;
+
    use type System.Tasking.Task_Id;
    use type System.Tasking.Termination_Handler;
 
@@ -71,8 +74,8 @@ package body System.Soft_Links.Tasking is
    procedure Set_Jmpbuf_Address (Addr : Address);
    --  Get/Set Jmpbuf_Address for current task
 
-   function  Get_Sec_Stack_Addr return  Address;
-   procedure Set_Sec_Stack_Addr (Addr : Address);
+   function  Get_Sec_Stack return SST.SS_Stack_Ptr;
+   procedure Set_Sec_Stack (Stack : SST.SS_Stack_Ptr);
    --  Get/Set location of current task's secondary stack
 
    procedure Timed_Delay_T (Time : Duration; Mode : Integer);
@@ -93,14 +96,14 @@ package body System.Soft_Links.Tasking is
       return STPO.Self.Common.Compiler_Data.Jmpbuf_Address;
    end Get_Jmpbuf_Address;
 
-   function Get_Sec_Stack_Addr return  Address is
+   function Get_Sec_Stack return SST.SS_Stack_Ptr is
    begin
-      return Result : constant Address :=
-        STPO.Self.Common.Compiler_Data.Sec_Stack_Addr
+      return Result : constant SST.SS_Stack_Ptr :=
+        STPO.Self.Common.Compiler_Data.Sec_Stack_Ptr
       do
-         pragma Assert (Result /= Null_Address);
+         pragma Assert (Result /= null);
       end return;
-   end Get_Sec_Stack_Addr;
+   end Get_Sec_Stack;
 
    function Get_Stack_Info return Stack_Checking.Stack_Access is
    begin
@@ -116,10 +119,10 @@ package body System.Soft_Links.Tasking is
       STPO.Self.Common.Compiler_Data.Jmpbuf_Address := Addr;
    end Set_Jmpbuf_Address;
 
-   procedure Set_Sec_Stack_Addr (Addr : Address) is
+   procedure Set_Sec_Stack (Stack : SST.SS_Stack_Ptr) is
    begin
-      STPO.Self.Common.Compiler_Data.Sec_Stack_Addr := Addr;
-   end Set_Sec_Stack_Addr;
+      STPO.Self.Common.Compiler_Data.Sec_Stack_Ptr := Stack;
+   end Set_Sec_Stack;
 
    -------------------
    -- Timed_Delay_T --
@@ -213,20 +216,20 @@ package body System.Soft_Links.Tasking is
 
          SSL.Get_Jmpbuf_Address       := Get_Jmpbuf_Address'Access;
          SSL.Set_Jmpbuf_Address       := Set_Jmpbuf_Address'Access;
-         SSL.Get_Sec_Stack_Addr       := Get_Sec_Stack_Addr'Access;
+         SSL.Get_Sec_Stack            := Get_Sec_Stack'Access;
          SSL.Get_Stack_Info           := Get_Stack_Info'Access;
-         SSL.Set_Sec_Stack_Addr       := Set_Sec_Stack_Addr'Access;
+         SSL.Set_Sec_Stack            := Set_Sec_Stack'Access;
          SSL.Timed_Delay              := Timed_Delay_T'Access;
          SSL.Task_Termination_Handler := Task_Termination_Handler_T'Access;
 
          --  No need to create a new secondary stack, since we will use the
          --  default one created in s-secsta.adb.
 
-         SSL.Set_Sec_Stack_Addr     (SSL.Get_Sec_Stack_Addr_NT);
+         SSL.Set_Sec_Stack          (SSL.Get_Sec_Stack_NT);
          SSL.Set_Jmpbuf_Address     (SSL.Get_Jmpbuf_Address_NT);
       end if;
 
-      pragma Assert (Get_Sec_Stack_Addr /= Null_Address);
+      pragma Assert (Get_Sec_Stack /= null);
    end Init_Tasking_Soft_Links;
 
 end System.Soft_Links.Tasking;
diff --git a/gcc/ada/libgnarl/s-taprop__linux.adb b/gcc/ada/libgnarl/s-taprop__linux.adb
index 1dfcf39dd81..5da10824a15 100644
--- a/gcc/ada/libgnarl/s-taprop__linux.adb
+++ b/gcc/ada/libgnarl/s-taprop__linux.adb
@@ -38,9 +38,7 @@ pragma Polling (Off);
 --  Turn off polling, we do not want ATC polling to take place during tasking
 --  operations. It causes infinite loops and other problems.
 
-with Interfaces.C; use Interfaces;
-use type Interfaces.C.int;
-use type Interfaces.C.long;
+with Interfaces.C; use Interfaces; use type Interfaces.C.int;
 
 with System.Task_Info;
 with System.Tasking.Debug;
@@ -112,8 +110,6 @@ package body System.Task_Primitives.Operations is
    --  Constant to indicate that the thread identifier has not yet been
    --  initialized.
 
-   Base_Monotonic_Clock : Duration := 0.0;
-
    --------------------
    -- Local Packages --
    --------------------
@@ -141,6 +137,38 @@ package body System.Task_Primitives.Operations is
    package body Specific is separate;
    --  The body of this package is target specific
 
+   package Monotonic is
+
+      function Monotonic_Clock return Duration;
+      pragma Inline (Monotonic_Clock);
+      --  Returns "absolute" time, represented as an offset relative to "the
+      --  Epoch", which is Jan 1, 1970. This clock implementation is immune to
+      --  the system's clock changes.
+
+      function RT_Resolution return Duration;
+      pragma Inline (RT_Resolution);
+      --  Returns resolution of the underlying clock used to implement RT_Clock
+
+      procedure Timed_Sleep
+        (Self_ID  : ST.Task_Id;
+         Time     : Duration;
+         Mode     : ST.Delay_Modes;
+         Reason   : System.Tasking.Task_States;
+         Timedout : out Boolean;
+         Yielded  : out Boolean);
+      --  Combination of Sleep (above) and Timed_Delay
+
+      procedure Timed_Delay
+        (Self_ID : ST.Task_Id;
+         Time    : Duration;
+         Mode    : ST.Delay_Modes);
+      --  Implement the semantics of the delay statement.
+      --  The caller should be abort-deferred and should not hold any locks.
+
+   end Monotonic;
+
+   package body Monotonic is separate;
+
    ----------------------------------
    -- ATCB allocation/deallocation --
    ----------------------------------
@@ -152,11 +180,16 @@ package body System.Task_Primitives.Operations is
    -- Support for foreign threads --
    ---------------------------------
 
-   function Register_Foreign_Thread (Thread : Thread_Id) return Task_Id;
-   --  Allocate and Initialize a new ATCB for the current Thread
+   function Register_Foreign_Thread
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size) return Task_Id;
+   --  Allocate and initialize a new ATCB for the current Thread. The size of
+   --  the secondary stack can be optionally specified.
 
    function Register_Foreign_Thread
-     (Thread : Thread_Id) return Task_Id is separate;
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size)
+     return Task_Id is separate;
 
    -----------------------
    -- Local Subprograms --
@@ -164,11 +197,6 @@ package body System.Task_Primitives.Operations is
 
    procedure Abort_Handler (signo : Signal);
 
-   function Compute_Base_Monotonic_Clock return Duration;
-   --  The monotonic clock epoch is set to some undetermined time in the past
-   --  (typically system boot time). In order to use the monotonic clock for
-   --  absolute time, the offset from a known epoch is needed.
-
    function GNAT_pthread_condattr_setup
      (attr : access pthread_condattr_t) return C.int;
    pragma Import
@@ -270,100 +298,6 @@ package body System.Task_Primitives.Operations is
       end if;
    end Abort_Handler;
 
-   ----------------------------------
-   -- Compute_Base_Monotonic_Clock --
-   ----------------------------------
-
-   function Compute_Base_Monotonic_Clock return Duration is
-      Aft     : Duration;
-      Bef     : Duration;
-      Mon     : Duration;
-      Res_A   : Interfaces.C.int;
-      Res_B   : Interfaces.C.int;
-      Res_M   : Interfaces.C.int;
-      TS_Aft  : aliased timespec;
-      TS_Aft0 : aliased timespec;
-      TS_Bef  : aliased timespec;
-      TS_Bef0 : aliased timespec;
-      TS_Mon  : aliased timespec;
-      TS_Mon0 : aliased timespec;
-
-   begin
-      Res_B :=
-        clock_gettime
-          (clock_id => OSC.CLOCK_REALTIME,
-           tp       => TS_Bef0'Unchecked_Access);
-      pragma Assert (Res_B = 0);
-
-      Res_M :=
-        clock_gettime
-          (clock_id => OSC.CLOCK_RT_Ada,
-           tp       => TS_Mon0'Unchecked_Access);
-      pragma Assert (Res_M = 0);
-
-      Res_A :=
-        clock_gettime
-          (clock_id => OSC.CLOCK_REALTIME,
-           tp       => TS_Aft0'Unchecked_Access);
-      pragma Assert (Res_A = 0);
-
-      for I in 1 .. 10 loop
-
-         --  Guard against a leap second that will cause CLOCK_REALTIME to jump
-         --  backwards. In the extrenmely unlikely event we call clock_gettime
-         --  before and after the jump the epoch, the result will be off
-         --  slightly.
-         --  Use only results where the tv_sec values match, for the sake of
-         --  convenience.
-         --  Also try to calculate the most accurate epoch by taking the
-         --  minimum difference of 10 tries.
-
-         Res_B :=
-           clock_gettime
-             (clock_id => OSC.CLOCK_REALTIME,
-              tp       => TS_Bef'Unchecked_Access);
-         pragma Assert (Res_B = 0);
-
-         Res_M :=
-           clock_gettime
-             (clock_id => OSC.CLOCK_RT_Ada,
-              tp       => TS_Mon'Unchecked_Access);
-         pragma Assert (Res_M = 0);
-
-         Res_A :=
-           clock_gettime
-             (clock_id => OSC.CLOCK_REALTIME,
-              tp       => TS_Aft'Unchecked_Access);
-         pragma Assert (Res_A = 0);
-
-         --  The calls to clock_gettime before the loop were no good
-
-         if (TS_Bef0.tv_sec /= TS_Aft0.tv_sec
-               and then TS_Bef.tv_sec  = TS_Aft.tv_sec)
-
-           --  The most recent calls to clock_gettime were better
-
-           or else
-             (TS_Bef0.tv_sec = TS_Aft0.tv_sec
-                and then TS_Bef.tv_sec = TS_Aft.tv_sec
-                and then (TS_Aft.tv_nsec - TS_Bef.tv_nsec
-                            < TS_Aft0.tv_nsec - TS_Bef0.tv_nsec))
-         then
-            TS_Bef0 := TS_Bef;
-            TS_Aft0 := TS_Aft;
-            TS_Mon0 := TS_Mon;
-         end if;
-      end loop;
-
-      Bef := To_Duration (TS_Bef0);
-      Mon := To_Duration (TS_Mon0);
-      Aft := To_Duration (TS_Aft0);
-
-      --  Distribute the division, to avoid potential type overflow someday
-
-      return Bef / 2 + Aft / 2 - Mon;
-   end Compute_Base_Monotonic_Clock;
-
    --------------
    -- Lock_RTS --
    --------------
@@ -685,56 +619,7 @@ package body System.Task_Primitives.Operations is
       Mode     : ST.Delay_Modes;
       Reason   : System.Tasking.Task_States;
       Timedout : out Boolean;
-      Yielded  : out Boolean)
-   is
-      pragma Unreferenced (Reason);
-
-      Base_Time  : constant Duration := Monotonic_Clock;
-      Check_Time : Duration := Base_Time - Base_Monotonic_Clock;
-      Abs_Time   : Duration;
-      Request    : aliased timespec;
-      Result     : C.int;
-
-   begin
-      Timedout := True;
-      Yielded := False;
-
-      Abs_Time :=
-        (if Mode = Relative
-         then Duration'Min (Time, Max_Sensible_Delay) + Check_Time
-         else Duration'Min (Check_Time + Max_Sensible_Delay,
-                            Time - Base_Monotonic_Clock));
-
-      if Abs_Time > Check_Time then
-         Request := To_Timespec (Abs_Time);
-
-         loop
-            exit when Self_ID.Pending_ATC_Level < Self_ID.ATC_Nesting_Level;
-
-            Result :=
-              pthread_cond_timedwait
-                (cond    => Self_ID.Common.LL.CV'Access,
-                 mutex   => (if Single_Lock
-                             then Single_RTS_Lock'Access
-                             else Self_ID.Common.LL.L'Access),
-                 abstime => Request'Access);
-
-            Check_Time := Monotonic_Clock;
-            exit when Abs_Time + Base_Monotonic_Clock <= Check_Time
-                      or else Check_Time < Base_Time;
-
-            if Result in 0 | EINTR then
-
-               --  Somebody may have called Wakeup for us
-
-               Timedout := False;
-               exit;
-            end if;
-
-            pragma Assert (Result = ETIMEDOUT);
-         end loop;
-      end if;
-   end Timed_Sleep;
+      Yielded  : out Boolean) renames Monotonic.Timed_Sleep;
 
    -----------------
    -- Timed_Delay --
@@ -746,92 +631,19 @@ package body System.Task_Primitives.Operations is
    procedure Timed_Delay
      (Self_ID : Task_Id;
       Time    : Duration;
-      Mode    : ST.Delay_Modes)
-   is
-      Base_Time  : constant Duration := Monotonic_Clock;
-      Check_Time : Duration := Base_Time - Base_Monotonic_Clock;
-      Abs_Time   : Duration;
-      Request    : aliased timespec;
-
-      Result : C.int;
-      pragma Warnings (Off, Result);
-
-   begin
-      if Single_Lock then
-         Lock_RTS;
-      end if;
-
-      Write_Lock (Self_ID);
-
-      Abs_Time :=
-        (if Mode = Relative
-         then Time + Check_Time
-         else Duration'Min (Check_Time + Max_Sensible_Delay,
-                            Time - Base_Monotonic_Clock));
-
-      if Abs_Time > Check_Time then
-         Request := To_Timespec (Abs_Time);
-         Self_ID.Common.State := Delay_Sleep;
-
-         loop
-            exit when Self_ID.Pending_ATC_Level < Self_ID.ATC_Nesting_Level;
-
-            Result :=
-              pthread_cond_timedwait
-                (cond    => Self_ID.Common.LL.CV'Access,
-                 mutex   => (if Single_Lock
-                             then Single_RTS_Lock'Access
-                             else Self_ID.Common.LL.L'Access),
-                 abstime => Request'Access);
-
-            Check_Time := Monotonic_Clock;
-            exit when Abs_Time + Base_Monotonic_Clock <= Check_Time
-                      or else Check_Time < Base_Time;
-
-            pragma Assert (Result in 0 | ETIMEDOUT | EINTR);
-         end loop;
-
-         Self_ID.Common.State := Runnable;
-      end if;
-
-      Unlock (Self_ID);
-
-      if Single_Lock then
-         Unlock_RTS;
-      end if;
-
-      Result := sched_yield;
-   end Timed_Delay;
+      Mode    : ST.Delay_Modes) renames Monotonic.Timed_Delay;
 
    ---------------------
    -- Monotonic_Clock --
    ---------------------
 
-   function Monotonic_Clock return Duration is
-      TS     : aliased timespec;
-      Result : Interfaces.C.int;
-   begin
-      Result := clock_gettime
-        (clock_id => OSC.CLOCK_RT_Ada, tp => TS'Unchecked_Access);
-      pragma Assert (Result = 0);
-
-      return Base_Monotonic_Clock + To_Duration (TS);
-   end Monotonic_Clock;
+   function Monotonic_Clock return Duration renames Monotonic.Monotonic_Clock;
 
    -------------------
    -- RT_Resolution --
    -------------------
 
-   function RT_Resolution return Duration is
-      TS     : aliased timespec;
-      Result : C.int;
-
-   begin
-      Result := clock_getres (OSC.CLOCK_REALTIME, TS'Unchecked_Access);
-      pragma Assert (Result = 0);
-
-      return To_Duration (TS);
-   end RT_Resolution;
+   function RT_Resolution return Duration renames Monotonic.RT_Resolution;
 
    ------------
    -- Wakeup --
@@ -1607,8 +1419,6 @@ package body System.Task_Primitives.Operations is
 
       Interrupt_Management.Initialize;
 
-      Base_Monotonic_Clock := Compute_Base_Monotonic_Clock;
-
       --  Prepare the set of signals that should be unblocked in all tasks
 
       Result := sigemptyset (Unblocked_Signal_Mask'Access);
diff --git a/gcc/ada/libgnarl/s-taprop__mingw.adb b/gcc/ada/libgnarl/s-taprop__mingw.adb
index fa966514568..b14444ad185 100644
--- a/gcc/ada/libgnarl/s-taprop__mingw.adb
+++ b/gcc/ada/libgnarl/s-taprop__mingw.adb
@@ -190,11 +190,16 @@ package body System.Task_Primitives.Operations is
    -- Support for foreign threads --
    ---------------------------------
 
-   function Register_Foreign_Thread (Thread : Thread_Id) return Task_Id;
-   --  Allocate and Initialize a new ATCB for the current Thread
+   function Register_Foreign_Thread
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size) return Task_Id;
+   --  Allocate and initialize a new ATCB for the current Thread. The size of
+   --  the secondary stack can be optionally specified.
 
    function Register_Foreign_Thread
-     (Thread : Thread_Id) return Task_Id is separate;
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size)
+     return Task_Id is separate;
 
    ----------------------------------
    -- Condition Variable Functions --
diff --git a/gcc/ada/libgnarl/s-taprop__posix.adb b/gcc/ada/libgnarl/s-taprop__posix.adb
index 3efc1e0de1a..d9ee078b364 100644
--- a/gcc/ada/libgnarl/s-taprop__posix.adb
+++ b/gcc/ada/libgnarl/s-taprop__posix.adb
@@ -145,6 +145,38 @@ package body System.Task_Primitives.Operations is
    package body Specific is separate;
    --  The body of this package is target specific
 
+   package Monotonic is
+
+      function Monotonic_Clock return Duration;
+      pragma Inline (Monotonic_Clock);
+      --  Returns "absolute" time, represented as an offset relative to "the
+      --  Epoch", which is Jan 1, 1970. This clock implementation is immune to
+      --  the system's clock changes.
+
+      function RT_Resolution return Duration;
+      pragma Inline (RT_Resolution);
+      --  Returns resolution of the underlying clock used to implement RT_Clock
+
+      procedure Timed_Sleep
+        (Self_ID  : ST.Task_Id;
+         Time     : Duration;
+         Mode     : ST.Delay_Modes;
+         Reason   : System.Tasking.Task_States;
+         Timedout : out Boolean;
+         Yielded  : out Boolean);
+      --  Combination of Sleep (above) and Timed_Delay
+
+      procedure Timed_Delay
+        (Self_ID : ST.Task_Id;
+         Time    : Duration;
+         Mode    : ST.Delay_Modes);
+      --  Implement the semantics of the delay statement.
+      --  The caller should be abort-deferred and should not hold any locks.
+
+   end Monotonic;
+
+   package body Monotonic is separate;
+
    ----------------------------------
    -- ATCB allocation/deallocation --
    ----------------------------------
@@ -156,11 +188,16 @@ package body System.Task_Primitives.Operations is
    -- Support for foreign threads --
    ---------------------------------
 
-   function Register_Foreign_Thread (Thread : Thread_Id) return Task_Id;
-   --  Allocate and Initialize a new ATCB for the current Thread
+   function Register_Foreign_Thread
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size) return Task_Id;
+   --  Allocate and initialize a new ATCB for the current Thread. The size of
+   --  the secondary stack can be optionally specified.
 
    function Register_Foreign_Thread
-     (Thread : Thread_Id) return Task_Id is separate;
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size)
+     return Task_Id is separate;
 
    -----------------------
    -- Local Subprograms --
@@ -178,18 +215,6 @@ package body System.Task_Primitives.Operations is
    pragma Import (C,
      GNAT_pthread_condattr_setup, "__gnat_pthread_condattr_setup");
 
-   procedure Compute_Deadline
-     (Time       : Duration;
-      Mode       : ST.Delay_Modes;
-      Check_Time : out Duration;
-      Abs_Time   : out Duration;
-      Rel_Time   : out Duration);
-   --  Helper for Timed_Sleep and Timed_Delay: given a deadline specified by
-   --  Time and Mode, compute the current clock reading (Check_Time), and the
-   --  target absolute and relative clock readings (Abs_Time, Rel_Time). The
-   --  epoch for Time depends on Mode; the epoch for Check_Time and Abs_Time
-   --  is always that of CLOCK_RT_Ada.
-
    -------------------
    -- Abort_Handler --
    -------------------
@@ -248,67 +273,6 @@ package body System.Task_Primitives.Operations is
       end if;
    end Abort_Handler;
 
-   ----------------------
-   -- Compute_Deadline --
-   ----------------------
-
-   procedure Compute_Deadline
-     (Time       : Duration;
-      Mode       : ST.Delay_Modes;
-      Check_Time : out Duration;
-      Abs_Time   : out Duration;
-      Rel_Time   : out Duration)
-   is
-   begin
-      Check_Time := Monotonic_Clock;
-
-      --  Relative deadline
-
-      if Mode = Relative then
-         Abs_Time := Duration'Min (Time, Max_Sensible_Delay) + Check_Time;
-
-         if Relative_Timed_Wait then
-            Rel_Time := Duration'Min (Max_Sensible_Delay, Time);
-         end if;
-
-         pragma Warnings (Off);
-         --  Comparison "OSC.CLOCK_RT_Ada = OSC.CLOCK_REALTIME" is compile
-         --  time known.
-
-      --  Absolute deadline specified using the tasking clock (CLOCK_RT_Ada)
-
-      elsif Mode = Absolute_RT
-        or else OSC.CLOCK_RT_Ada = OSC.CLOCK_REALTIME
-      then
-         pragma Warnings (On);
-         Abs_Time := Duration'Min (Check_Time + Max_Sensible_Delay, Time);
-
-         if Relative_Timed_Wait then
-            Rel_Time := Duration'Min (Max_Sensible_Delay, Time - Check_Time);
-         end if;
-
-      --  Absolute deadline specified using the calendar clock, in the
-      --  case where it is not the same as the tasking clock: compensate for
-      --  difference between clock epochs (Base_Time - Base_Cal_Time).
-
-      else
-         declare
-            Cal_Check_Time : constant Duration := OS_Primitives.Clock;
-            RT_Time        : constant Duration :=
-                               Time + Check_Time - Cal_Check_Time;
-
-         begin
-            Abs_Time :=
-              Duration'Min (Check_Time + Max_Sensible_Delay, RT_Time);
-
-            if Relative_Timed_Wait then
-               Rel_Time :=
-                 Duration'Min (Max_Sensible_Delay, RT_Time - Check_Time);
-            end if;
-         end;
-      end if;
-   end Compute_Deadline;
-
    -----------------
    -- Stack_Guard --
    -----------------
@@ -595,60 +559,7 @@ package body System.Task_Primitives.Operations is
       Mode     : ST.Delay_Modes;
       Reason   : Task_States;
       Timedout : out Boolean;
-      Yielded  : out Boolean)
-   is
-      pragma Unreferenced (Reason);
-
-      Base_Time  : Duration;
-      Check_Time : Duration;
-      Abs_Time   : Duration;
-      Rel_Time   : Duration;
-
-      Request    : aliased timespec;
-      Result     : Interfaces.C.int;
-
-   begin
-      Timedout := True;
-      Yielded := False;
-
-      Compute_Deadline
-        (Time       => Time,
-         Mode       => Mode,
-         Check_Time => Check_Time,
-         Abs_Time   => Abs_Time,
-         Rel_Time   => Rel_Time);
-      Base_Time := Check_Time;
-
-      if Abs_Time > Check_Time then
-         Request :=
-           To_Timespec (if Relative_Timed_Wait then Rel_Time else Abs_Time);
-
-         loop
-            exit when Self_ID.Pending_ATC_Level < Self_ID.ATC_Nesting_Level;
-
-            Result :=
-              pthread_cond_timedwait
-                (cond    => Self_ID.Common.LL.CV'Access,
-                 mutex   => (if Single_Lock
-                             then Single_RTS_Lock'Access
-                             else Self_ID.Common.LL.L'Access),
-                 abstime => Request'Access);
-
-            Check_Time := Monotonic_Clock;
-            exit when Abs_Time <= Check_Time or else Check_Time < Base_Time;
-
-            if Result = 0 or Result = EINTR then
-
-               --  Somebody may have called Wakeup for us
-
-               Timedout := False;
-               exit;
-            end if;
-
-            pragma Assert (Result = ETIMEDOUT);
-         end loop;
-      end if;
-   end Timed_Sleep;
+      Yielded  : out Boolean) renames Monotonic.Timed_Sleep;
 
    -----------------
    -- Timed_Delay --
@@ -660,95 +571,19 @@ package body System.Task_Primitives.Operations is
    procedure Timed_Delay
      (Self_ID : Task_Id;
       Time    : Duration;
-      Mode    : ST.Delay_Modes)
-   is
-      Base_Time  : Duration;
-      Check_Time : Duration;
-      Abs_Time   : Duration;
-      Rel_Time   : Duration;
-      Request    : aliased timespec;
-
-      Result : Interfaces.C.int;
-      pragma Warnings (Off, Result);
-
-   begin
-      if Single_Lock then
-         Lock_RTS;
-      end if;
-
-      Write_Lock (Self_ID);
-
-      Compute_Deadline
-        (Time       => Time,
-         Mode       => Mode,
-         Check_Time => Check_Time,
-         Abs_Time   => Abs_Time,
-         Rel_Time   => Rel_Time);
-      Base_Time := Check_Time;
-
-      if Abs_Time > Check_Time then
-         Request :=
-           To_Timespec (if Relative_Timed_Wait then Rel_Time else Abs_Time);
-         Self_ID.Common.State := Delay_Sleep;
-
-         loop
-            exit when Self_ID.Pending_ATC_Level < Self_ID.ATC_Nesting_Level;
-
-            Result :=
-              pthread_cond_timedwait
-                (cond    => Self_ID.Common.LL.CV'Access,
-                 mutex   => (if Single_Lock
-                             then Single_RTS_Lock'Access
-                             else Self_ID.Common.LL.L'Access),
-                 abstime => Request'Access);
-
-            Check_Time := Monotonic_Clock;
-            exit when Abs_Time <= Check_Time or else Check_Time < Base_Time;
-
-            pragma Assert (Result = 0
-                             or else Result = ETIMEDOUT
-                             or else Result = EINTR);
-         end loop;
-
-         Self_ID.Common.State := Runnable;
-      end if;
-
-      Unlock (Self_ID);
-
-      if Single_Lock then
-         Unlock_RTS;
-      end if;
-
-      Result := sched_yield;
-   end Timed_Delay;
+      Mode    : ST.Delay_Modes) renames Monotonic.Timed_Delay;
 
    ---------------------
    -- Monotonic_Clock --
    ---------------------
 
-   function Monotonic_Clock return Duration is
-      TS     : aliased timespec;
-      Result : Interfaces.C.int;
-   begin
-      Result := clock_gettime
-        (clock_id => OSC.CLOCK_RT_Ada, tp => TS'Unchecked_Access);
-      pragma Assert (Result = 0);
-      return To_Duration (TS);
-   end Monotonic_Clock;
+   function Monotonic_Clock return Duration renames Monotonic.Monotonic_Clock;
 
    -------------------
    -- RT_Resolution --
    -------------------
 
-   function RT_Resolution return Duration is
-      TS     : aliased timespec;
-      Result : Interfaces.C.int;
-   begin
-      Result := clock_getres (OSC.CLOCK_REALTIME, TS'Unchecked_Access);
-      pragma Assert (Result = 0);
-
-      return To_Duration (TS);
-   end RT_Resolution;
+   function RT_Resolution return Duration renames Monotonic.RT_Resolution;
 
    ------------
    -- Wakeup --
diff --git a/gcc/ada/libgnarl/s-taprop__solaris.adb b/gcc/ada/libgnarl/s-taprop__solaris.adb
index e97662c12b1..26d83e584d6 100644
--- a/gcc/ada/libgnarl/s-taprop__solaris.adb
+++ b/gcc/ada/libgnarl/s-taprop__solaris.adb
@@ -237,11 +237,16 @@ package body System.Task_Primitives.Operations is
    -- Support for foreign threads --
    ---------------------------------
 
-   function Register_Foreign_Thread (Thread : Thread_Id) return Task_Id;
-   --  Allocate and Initialize a new ATCB for the current Thread
+   function Register_Foreign_Thread
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size) return Task_Id;
+   --  Allocate and initialize a new ATCB for the current Thread. The size of
+   --  the secondary stack can be optionally specified.
 
    function Register_Foreign_Thread
-     (Thread : Thread_Id) return Task_Id is separate;
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size)
+     return Task_Id is separate;
 
    ------------
    -- Checks --
diff --git a/gcc/ada/libgnarl/s-taprop__vxworks.adb b/gcc/ada/libgnarl/s-taprop__vxworks.adb
index b77fb106b37..83ebc22312e 100644
--- a/gcc/ada/libgnarl/s-taprop__vxworks.adb
+++ b/gcc/ada/libgnarl/s-taprop__vxworks.adb
@@ -149,11 +149,16 @@ package body System.Task_Primitives.Operations is
    -- Support for foreign threads --
    ---------------------------------
 
-   function Register_Foreign_Thread (Thread : Thread_Id) return Task_Id;
-   --  Allocate and Initialize a new ATCB for the current Thread
+   function Register_Foreign_Thread
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size) return Task_Id;
+   --  Allocate and initialize a new ATCB for the current Thread. The size of
+   --  the secondary stack can be optionally specified.
 
    function Register_Foreign_Thread
-     (Thread : Thread_Id) return Task_Id is separate;
+     (Thread         : Thread_Id;
+      Sec_Stack_Size : Size_Type := Unspecified_Size)
+     return Task_Id is separate;
 
    -----------------------
    -- Local Subprograms --
diff --git a/gcc/ada/libgnarl/s-tarest.adb b/gcc/ada/libgnarl/s-tarest.adb
index daff5c1c3ae..7b9f260927e 100644
--- a/gcc/ada/libgnarl/s-tarest.adb
+++ b/gcc/ada/libgnarl/s-tarest.adb
@@ -47,12 +47,6 @@ with Ada.Exceptions;
 
 with System.Task_Primitives.Operations;
 with System.Soft_Links.Tasking;
-with System.Storage_Elements;
-
-with System.Secondary_Stack;
-pragma Elaborate_All (System.Secondary_Stack);
---  Make sure the body of Secondary_Stack is elaborated before calling
---  Init_Tasking_Soft_Links. See comments for this routine for explanation.
 
 with System.Soft_Links;
 --  Used for the non-tasking routines (*_NT) that refer to global data. They
@@ -65,8 +59,6 @@ package body System.Tasking.Restricted.Stages is
 
    package STPO renames System.Task_Primitives.Operations;
    package SSL  renames System.Soft_Links;
-   package SSE  renames System.Storage_Elements;
-   package SST  renames System.Secondary_Stack;
 
    use Ada.Exceptions;
 
@@ -115,17 +107,18 @@ package body System.Tasking.Restricted.Stages is
    --  This should only be called by the Task_Wrapper procedure.
 
    procedure Create_Restricted_Task
-     (Priority             : Integer;
-      Stack_Address        : System.Address;
-      Size                 : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
-      Task_Info            : System.Task_Info.Task_Info_Type;
-      CPU                  : Integer;
-      State                : Task_Procedure_Access;
-      Discriminants        : System.Address;
-      Elaborated           : Access_Boolean;
-      Task_Image           : String;
-      Created_Task         : Task_Id);
+     (Priority          : Integer;
+      Stack_Address     : System.Address;
+      Stack_Size        : System.Parameters.Size_Type;
+      Sec_Stack_Address : System.Secondary_Stack.SS_Stack_Ptr;
+      Sec_Stack_Size    : System.Parameters.Size_Type;
+      Task_Info         : System.Task_Info.Task_Info_Type;
+      CPU               : Integer;
+      State             : Task_Procedure_Access;
+      Discriminants     : System.Address;
+      Elaborated        : Access_Boolean;
+      Task_Image        : String;
+      Created_Task      : Task_Id);
    --  Code shared between Create_Restricted_Task (the concurrent version) and
    --  Create_Restricted_Task_Sequential. See comment of the former in the
    --  specification of this package.
@@ -205,54 +198,6 @@ package body System.Tasking.Restricted.Stages is
       --
       --  DO NOT delete ID. As noted, it is needed on some targets.
 
-      function Secondary_Stack_Size return Storage_Elements.Storage_Offset;
-      --  Returns the size of the secondary stack for the task. For fixed
-      --  secondary stacks, the function will return the ATCB field
-      --  Secondary_Stack_Size if it is not set to Unspecified_Size,
-      --  otherwise a percentage of the stack is reserved using the
-      --  System.Parameters.Sec_Stack_Percentage property.
-
-      --  Dynamic secondary stacks are allocated in System.Soft_Links.
-      --  Create_TSD and thus the function returns 0 to suppress the
-      --  creation of the fixed secondary stack in the primary stack.
-
-      --------------------------
-      -- Secondary_Stack_Size --
-      --------------------------
-
-      function Secondary_Stack_Size return Storage_Elements.Storage_Offset is
-         use System.Storage_Elements;
-         use System.Secondary_Stack;
-
-      begin
-         if Parameters.Sec_Stack_Dynamic then
-            return 0;
-
-         elsif Self_ID.Common.Secondary_Stack_Size = Unspecified_Size then
-            return (Self_ID.Common.Compiler_Data.Pri_Stack_Info.Size
-                       * SSE.Storage_Offset (Sec_Stack_Percentage) / 100);
-         else
-            --  Use the size specified by aspect Secondary_Stack_Size padded
-            --  by the amount of space used by the stack data structure.
-
-            return Storage_Offset (Self_ID.Common.Secondary_Stack_Size) +
-                     Storage_Offset (Minimum_Secondary_Stack_Size);
-         end if;
-      end Secondary_Stack_Size;
-
-      Secondary_Stack : aliased Storage_Elements.Storage_Array
-                          (1 .. Secondary_Stack_Size);
-      for Secondary_Stack'Alignment use Standard'Maximum_Alignment;
-      --  This is the secondary stack data. Note that it is critical that this
-      --  have maximum alignment, since any kind of data can be allocated here.
-
-      pragma Warnings (Off);
-      Secondary_Stack_Address : System.Address := Secondary_Stack'Address;
-      pragma Warnings (On);
-      --  Address of secondary stack. In the fixed secondary stack case, this
-      --  value is not modified, causing a warning, hence the bracketing with
-      --  Warnings (Off/On).
-
       Cause : Cause_Of_Termination := Normal;
       --  Indicates the reason why this task terminates. Normal corresponds to
       --  a task terminating due to completing the last statement of its body.
@@ -266,15 +211,7 @@ package body System.Tasking.Restricted.Stages is
       --  execution of its task body, then EO will contain the associated
       --  exception occurrence. Otherwise, it will contain Null_Occurrence.
 
-   --  Start of processing for Task_Wrapper
-
    begin
-      if not Parameters.Sec_Stack_Dynamic then
-         Self_ID.Common.Compiler_Data.Sec_Stack_Addr :=
-           Secondary_Stack'Address;
-         SST.SS_Init (Secondary_Stack_Address, Integer (Secondary_Stack'Last));
-      end if;
-
       --  Initialize low-level TCB components, that cannot be initialized by
       --  the creator.
 
@@ -539,17 +476,18 @@ package body System.Tasking.Restricted.Stages is
    ----------------------------
 
    procedure Create_Restricted_Task
-     (Priority             : Integer;
-      Stack_Address        : System.Address;
-      Size                 : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
-      Task_Info            : System.Task_Info.Task_Info_Type;
-      CPU                  : Integer;
-      State                : Task_Procedure_Access;
-      Discriminants        : System.Address;
-      Elaborated           : Access_Boolean;
-      Task_Image           : String;
-      Created_Task         : Task_Id)
+     (Priority          : Integer;
+      Stack_Address     : System.Address;
+      Stack_Size        : System.Parameters.Size_Type;
+      Sec_Stack_Address : System.Secondary_Stack.SS_Stack_Ptr;
+      Sec_Stack_Size    : System.Parameters.Size_Type;
+      Task_Info         : System.Task_Info.Task_Info_Type;
+      CPU               : Integer;
+      State             : Task_Procedure_Access;
+      Discriminants     : System.Address;
+      Elaborated        : Access_Boolean;
+      Task_Image        : String;
+      Created_Task      : Task_Id)
    is
       Self_ID       : constant Task_Id := STPO.Self;
       Base_Priority : System.Any_Priority;
@@ -608,8 +546,7 @@ package body System.Tasking.Restricted.Stages is
 
       Initialize_ATCB
         (Self_ID, State, Discriminants, Self_ID, Elaborated, Base_Priority,
-         Base_CPU, null, Task_Info, Size, Secondary_Stack_Size,
-         Created_Task, Success);
+         Base_CPU, null, Task_Info, Stack_Size, Created_Task, Success);
 
       --  If we do our job right then there should never be any failures, which
       --  was probably said about the Titanic; so just to be safe, let's retain
@@ -639,25 +576,31 @@ package body System.Tasking.Restricted.Stages is
          Unlock_RTS;
       end if;
 
-      --  Create TSD as early as possible in the creation of a task, since it
-      --  may be used by the operation of Ada code within the task.
+      --  Create TSD as early as possible in the creation of a task, since
+      --  it may be used by the operation of Ada code within the task. If the
+      --  compiler has not allocated a secondary stack, a stack will be
+      --  allocated fromt the binder generated pool.
 
-      SSL.Create_TSD (Created_Task.Common.Compiler_Data);
+      SSL.Create_TSD
+        (Created_Task.Common.Compiler_Data,
+         Sec_Stack_Address,
+         Sec_Stack_Size);
    end Create_Restricted_Task;
 
    procedure Create_Restricted_Task
-     (Priority             : Integer;
-      Stack_Address        : System.Address;
-      Size                 : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
-      Task_Info            : System.Task_Info.Task_Info_Type;
-      CPU                  : Integer;
-      State                : Task_Procedure_Access;
-      Discriminants        : System.Address;
-      Elaborated           : Access_Boolean;
-      Chain                : in out Activation_Chain;
-      Task_Image           : String;
-      Created_Task         : Task_Id)
+     (Priority          : Integer;
+      Stack_Address     : System.Address;
+      Stack_Size        : System.Parameters.Size_Type;
+      Sec_Stack_Address : System.Secondary_Stack.SS_Stack_Ptr;
+      Sec_Stack_Size    : System.Parameters.Size_Type;
+      Task_Info         : System.Task_Info.Task_Info_Type;
+      CPU               : Integer;
+      State             : Task_Procedure_Access;
+      Discriminants     : System.Address;
+      Elaborated        : Access_Boolean;
+      Chain             : in out Activation_Chain;
+      Task_Image        : String;
+      Created_Task      : Task_Id)
    is
    begin
       if Partition_Elaboration_Policy = 'S' then
@@ -668,14 +611,14 @@ package body System.Tasking.Restricted.Stages is
          --  sequential, activation must be deferred.
 
          Create_Restricted_Task_Sequential
-           (Priority, Stack_Address, Size, Secondary_Stack_Size,
-            Task_Info, CPU, State, Discriminants, Elaborated,
+           (Priority, Stack_Address, Stack_Size, Sec_Stack_Address,
+            Sec_Stack_Size, Task_Info, CPU, State, Discriminants, Elaborated,
             Task_Image, Created_Task);
 
       else
          Create_Restricted_Task
-           (Priority, Stack_Address, Size, Secondary_Stack_Size,
-            Task_Info, CPU, State, Discriminants, Elaborated,
+           (Priority, Stack_Address, Stack_Size, Sec_Stack_Address,
+            Sec_Stack_Size, Task_Info, CPU, State, Discriminants, Elaborated,
             Task_Image, Created_Task);
 
          --  Append this task to the activation chain
@@ -690,22 +633,24 @@ package body System.Tasking.Restricted.Stages is
    ---------------------------------------
 
    procedure Create_Restricted_Task_Sequential
-     (Priority             : Integer;
-      Stack_Address        : System.Address;
-      Size                 : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
-      Task_Info            : System.Task_Info.Task_Info_Type;
-      CPU                  : Integer;
-      State                : Task_Procedure_Access;
-      Discriminants        : System.Address;
-      Elaborated           : Access_Boolean;
-      Task_Image           : String;
-      Created_Task         : Task_Id) is
+     (Priority          : Integer;
+      Stack_Address     : System.Address;
+      Stack_Size        : System.Parameters.Size_Type;
+      Sec_Stack_Address : System.Secondary_Stack.SS_Stack_Ptr;
+      Sec_Stack_Size    : System.Parameters.Size_Type;
+      Task_Info         : System.Task_Info.Task_Info_Type;
+      CPU               : Integer;
+      State             : Task_Procedure_Access;
+      Discriminants     : System.Address;
+      Elaborated        : Access_Boolean;
+      Task_Image        : String;
+      Created_Task      : Task_Id)
+   is
    begin
-      Create_Restricted_Task (Priority, Stack_Address, Size,
-                              Secondary_Stack_Size, Task_Info,
-                              CPU, State, Discriminants, Elaborated,
-                              Task_Image, Created_Task);
+      Create_Restricted_Task
+        (Priority, Stack_Address, Stack_Size, Sec_Stack_Address,
+         Sec_Stack_Size, Task_Info, CPU, State, Discriminants, Elaborated,
+         Task_Image, Created_Task);
 
       --  Append this task to the activation chain
 
diff --git a/gcc/ada/libgnarl/s-tarest.ads b/gcc/ada/libgnarl/s-tarest.ads
index ccc5683bd31..e51fa58ca61 100644
--- a/gcc/ada/libgnarl/s-tarest.ads
+++ b/gcc/ada/libgnarl/s-tarest.ads
@@ -43,8 +43,9 @@
 --  The restricted GNARLI is also composed of System.Protected_Objects and
 --  System.Protected_Objects.Single_Entry
 
-with System.Task_Info;
 with System.Parameters;
+with System.Secondary_Stack;
+with System.Task_Info;
 
 package System.Tasking.Restricted.Stages is
    pragma Elaborate_Body;
@@ -128,33 +129,38 @@ package System.Tasking.Restricted.Stages is
    --  by the binder generated code, before calling elaboration code.
 
    procedure Create_Restricted_Task
-     (Priority             : Integer;
-      Stack_Address        : System.Address;
-      Size                 : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
-      Task_Info            : System.Task_Info.Task_Info_Type;
-      CPU                  : Integer;
-      State                : Task_Procedure_Access;
-      Discriminants        : System.Address;
-      Elaborated           : Access_Boolean;
-      Chain                : in out Activation_Chain;
-      Task_Image           : String;
-      Created_Task         : Task_Id);
+     (Priority          : Integer;
+      Stack_Address     : System.Address;
+      Stack_Size        : System.Parameters.Size_Type;
+      Sec_Stack_Address : System.Secondary_Stack.SS_Stack_Ptr;
+      Sec_Stack_Size    : System.Parameters.Size_Type;
+      Task_Info         : System.Task_Info.Task_Info_Type;
+      CPU               : Integer;
+      State             : Task_Procedure_Access;
+      Discriminants     : System.Address;
+      Elaborated        : Access_Boolean;
+      Chain             : in out Activation_Chain;
+      Task_Image        : String;
+      Created_Task      : Task_Id);
    --  Compiler interface only. Do not call from within the RTS.
    --  This must be called to create a new task, when the partition
    --  elaboration policy is not specified (or is concurrent).
    --
    --  Priority is the task's priority (assumed to be in the
-   --  System.Any_Priority'Range)
+   --  System.Any_Priority'Range).
    --
    --  Stack_Address is the start address of the stack associated to the task,
    --  in case it has been preallocated by the compiler; it is equal to
    --  Null_Address when the stack needs to be allocated by the underlying
    --  operating system.
    --
-   --  Size is the stack size of the task to create
+   --  Stack_Size is the stack size of the task to create.
+   --
+   --  Sec_Stack_Address is the pointer to the secondary stack created by the
+   --  compiler. If null, the secondary stack is either allocated by the binder
+   --  or the run-time.
    --
-   --  Secondary_Stack_Size is the secondary stack size of the task to create
+   --  Secondary_Stack_Size is the secondary stack size of the task to create.
    --
    --  Task_Info is the task info associated with the created task, or
    --  Unspecified_Task_Info if none.
@@ -164,7 +170,7 @@ package System.Tasking.Restricted.Stages is
    --   checks are performed when analyzing the pragma, and dynamic ones are
    --   performed before setting the affinity at run time.
    --
-   --  State is the compiler generated task's procedure body
+   --  State is the compiler generated task's procedure body.
    --
    --  Discriminants is a pointer to a limited record whose discriminants are
    --  those of the task to create. This parameter should be passed as the
@@ -182,20 +188,21 @@ package System.Tasking.Restricted.Stages is
    --
    --  Created_Task is the resulting task.
    --
-   --  This procedure can raise Storage_Error if the task creation fails
+   --  This procedure can raise Storage_Error if the task creation fails.
 
    procedure Create_Restricted_Task_Sequential
-     (Priority             : Integer;
-      Stack_Address        : System.Address;
-      Size                 : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
-      Task_Info            : System.Task_Info.Task_Info_Type;
-      CPU                  : Integer;
-      State                : Task_Procedure_Access;
-      Discriminants        : System.Address;
-      Elaborated           : Access_Boolean;
-      Task_Image           : String;
-      Created_Task         : Task_Id);
+     (Priority          : Integer;
+      Stack_Address     : System.Address;
+      Stack_Size        : System.Parameters.Size_Type;
+      Sec_Stack_Address : System.Secondary_Stack.SS_Stack_Ptr;
+      Sec_Stack_Size    : System.Parameters.Size_Type;
+      Task_Info         : System.Task_Info.Task_Info_Type;
+      CPU               : Integer;
+      State             : Task_Procedure_Access;
+      Discriminants     : System.Address;
+      Elaborated        : Access_Boolean;
+      Task_Image        : String;
+      Created_Task      : Task_Id);
    --  Compiler interface only. Do not call from within the RTS.
    --  This must be called to create a new task, when the sequential partition
    --  elaboration policy is used.
diff --git a/gcc/ada/libgnarl/s-taskin.adb b/gcc/ada/libgnarl/s-taskin.adb
index 462e229645c..d9fc6e3213b 100644
--- a/gcc/ada/libgnarl/s-taskin.adb
+++ b/gcc/ada/libgnarl/s-taskin.adb
@@ -96,7 +96,6 @@ package body System.Tasking is
       Domain               : Dispatching_Domain_Access;
       Task_Info            : System.Task_Info.Task_Info_Type;
       Stack_Size           : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
       T                    : Task_Id;
       Success              : out Boolean)
    is
@@ -147,7 +146,6 @@ package body System.Tasking is
       T.Common.Specific_Handler         := null;
       T.Common.Debug_Events             := (others => False);
       T.Common.Task_Image_Len           := 0;
-      T.Common.Secondary_Stack_Size     := Secondary_Stack_Size;
 
       if T.Common.Parent = null then
 
@@ -244,7 +242,6 @@ package body System.Tasking is
          Domain               => System_Domain,
          Task_Info            => Task_Info.Unspecified_Task_Info,
          Stack_Size           => 0,
-         Secondary_Stack_Size => Parameters.Unspecified_Size,
          T                    => T,
          Success              => Success);
       pragma Assert (Success);
diff --git a/gcc/ada/libgnarl/s-taskin.ads b/gcc/ada/libgnarl/s-taskin.ads
index cd53cf93471..7c8b44b952c 100644
--- a/gcc/ada/libgnarl/s-taskin.ads
+++ b/gcc/ada/libgnarl/s-taskin.ads
@@ -37,12 +37,12 @@
 with Ada.Exceptions;
 with Ada.Unchecked_Conversion;
 
+with System.Multiprocessors;
 with System.Parameters;
-with System.Task_Info;
 with System.Soft_Links;
-with System.Task_Primitives;
 with System.Stack_Usage;
-with System.Multiprocessors;
+with System.Task_Info;
+with System.Task_Primitives;
 
 package System.Tasking is
    pragma Preelaborate;
@@ -702,13 +702,6 @@ package System.Tasking is
       --  need to do different things depending on the situation.
       --
       --  Protection: Self.L
-
-      Secondary_Stack_Size : System.Parameters.Size_Type;
-      --  Secondary_Stack_Size is the size of the secondary stack for the
-      --  task. Defined here since it is the responsibility of the task to
-      --  creates its own secondary stack.
-      --
-      --  Protected: Only accessed by Self
    end record;
 
    ---------------------------------------
@@ -1173,7 +1166,6 @@ package System.Tasking is
       Domain               : Dispatching_Domain_Access;
       Task_Info            : System.Task_Info.Task_Info_Type;
       Stack_Size           : System.Parameters.Size_Type;
-      Secondary_Stack_Size : System.Parameters.Size_Type;
       T                    : Task_Id;
       Success              : out Boolean);
    --  Initialize fields of the TCB for task T, and link into global TCB
diff --git a/gcc/ada/libgnarl/s-tassta.adb b/gcc/ada/libgnarl/s-tassta.adb
index 44c054fec3e..518a02c8b48 100644
--- a/gcc/ada/libgnarl/s-tassta.adb
+++ b/gcc/ada/libgnarl/s-tassta.adb
@@ -71,11 +71,11 @@ package body System.Tasking.Stages is
    package STPO renames System.Task_Primitives.Operations;
    package SSL  renames System.Soft_Links;
    package SSE  renames System.Storage_Elements;
-   package SST  renames System.Secondary_Stack;
 
    use Ada.Exceptions;
 
    use Parameters;
+   use Secondary_Stack;
    use Task_Primitives;
    use Task_Primitives.Operations;
 
@@ -465,7 +465,7 @@ package body System.Tasking.Stages is
 
    procedure Create_Task
      (Priority             : Integer;
-      Size                 : System.Parameters.Size_Type;
+      Stack_Size           : System.Parameters.Size_Type;
       Secondary_Stack_Size : System.Parameters.Size_Type;
       Task_Info            : System.Task_Info.Task_Info_Type;
       CPU                  : Integer;
@@ -604,8 +604,7 @@ package body System.Tasking.Stages is
       end if;
 
       Initialize_ATCB (Self_ID, State, Discriminants, P, Elaborated,
-        Base_Priority, Base_CPU, Domain, Task_Info, Size,
-        Secondary_Stack_Size, T, Success);
+        Base_Priority, Base_CPU, Domain, Task_Info, Stack_Size, T, Success);
 
       if not Success then
          Free (T);
@@ -692,10 +691,18 @@ package body System.Tasking.Stages is
            Dispatching_Domain_Tasks (Base_CPU) + 1;
       end if;
 
-      --  Create TSD as early as possible in the creation of a task, since it
-      --  may be used by the operation of Ada code within the task.
+      --  Create the secondary stack for the task as early as possible during
+      --  in the creation of a task, since it may be used by the operation of
+      --  Ada code within the task.
+
+      begin
+         SSL.Create_TSD (T.Common.Compiler_Data, null, Secondary_Stack_Size);
+      exception
+         when others =>
+            Initialization.Undefer_Abort_Nestable (Self_ID);
+            raise Storage_Error with "Secondary stack could not be allocated";
+      end;
 
-      SSL.Create_TSD (T.Common.Compiler_Data);
       T.Common.Activation_Link := Chain.T_ID;
       Chain.T_ID := T;
       Created_Task := T;
@@ -914,8 +921,8 @@ package body System.Tasking.Stages is
       SSL.Unlock_Task        := SSL.Task_Unlock_NT'Access;
       SSL.Get_Jmpbuf_Address := SSL.Get_Jmpbuf_Address_NT'Access;
       SSL.Set_Jmpbuf_Address := SSL.Set_Jmpbuf_Address_NT'Access;
-      SSL.Get_Sec_Stack_Addr := SSL.Get_Sec_Stack_Addr_NT'Access;
-      SSL.Set_Sec_Stack_Addr := SSL.Set_Sec_Stack_Addr_NT'Access;
+      SSL.Get_Sec_Stack      := SSL.Get_Sec_Stack_NT'Access;
+      SSL.Set_Sec_Stack      := SSL.Set_Sec_Stack_NT'Access;
       SSL.Check_Abort_Status := SSL.Check_Abort_Status_NT'Access;
       SSL.Get_Stack_Info     := SSL.Get_Stack_Info_NT'Access;
 
@@ -1014,7 +1021,6 @@ package body System.Tasking.Stages is
    --  at-end handler that the compiler generates.
 
    procedure Task_Wrapper (Self_ID : Task_Id) is
-      use type SSE.Storage_Offset;
       use System.Standard_Library;
       use System.Stack_Usage;
 
@@ -1027,52 +1033,6 @@ package body System.Tasking.Stages is
       Use_Alternate_Stack : constant Boolean := Alternate_Stack_Size /= 0;
       --  Whether to use above alternate signal stack for stack overflows
 
-      function Secondary_Stack_Size return Storage_Elements.Storage_Offset;
-      --  Returns the size of the secondary stack for the task. For fixed
-      --  secondary stacks, the function will return the ATCB field
-      --  Secondary_Stack_Size if it is not set to Unspecified_Size,
-      --  otherwise a percentage of the stack is reserved using the
-      --  System.Parameters.Sec_Stack_Percentage property.
-
-      --  Dynamic secondary stacks are allocated in System.Soft_Links.
-      --  Create_TSD and thus the function returns 0 to suppress the
-      --  creation of the fixed secondary stack in the primary stack.
-
-      --------------------------
-      -- Secondary_Stack_Size --
-      --------------------------
-
-      function Secondary_Stack_Size return Storage_Elements.Storage_Offset is
-         use System.Storage_Elements;
-
-      begin
-         if Parameters.Sec_Stack_Dynamic then
-            return 0;
-
-         elsif Self_ID.Common.Secondary_Stack_Size = Unspecified_Size then
-            return (Self_ID.Common.Compiler_Data.Pri_Stack_Info.Size
-                    * SSE.Storage_Offset (Sec_Stack_Percentage) / 100);
-         else
-            --  Use the size specified by aspect Secondary_Stack_Size padded
-            --  by the amount of space used by the stack data structure.
-
-            return Storage_Offset (Self_ID.Common.Secondary_Stack_Size) +
-                     Storage_Offset (SST.Minimum_Secondary_Stack_Size);
-         end if;
-      end Secondary_Stack_Size;
-
-      Secondary_Stack : aliased Storage_Elements.Storage_Array
-                          (1 .. Secondary_Stack_Size);
-      for Secondary_Stack'Alignment use Standard'Maximum_Alignment;
-      --  Actual area allocated for secondary stack. Note that it is critical
-      --  that this have maximum alignment, since any kind of data can be
-      --  allocated here.
-
-      Secondary_Stack_Address : System.Address := Secondary_Stack'Address;
-      --  Address of secondary stack. In the fixed secondary stack case, this
-      --  value is not modified, causing a warning, hence the bracketing with
-      --  Warnings (Off/On). But why is so much *more* bracketed???
-
       SEH_Table : aliased SSE.Storage_Array (1 .. 8);
       --  Structured Exception Registration table (2 words)
 
@@ -1136,14 +1096,6 @@ package body System.Tasking.Stages is
       Debug.Master_Hook
         (Self_ID, Self_ID.Common.Parent, Self_ID.Master_of_Task);
 
-      --  Assume a size of the stack taken at this stage
-
-      if not Parameters.Sec_Stack_Dynamic then
-         Self_ID.Common.Compiler_Data.Sec_Stack_Addr :=
-           Secondary_Stack'Address;
-         SST.SS_Init (Secondary_Stack_Address, Integer (Secondary_Stack'Last));
-      end if;
-
       if Use_Alternate_Stack then
          Self_ID.Common.Task_Alternate_Stack := Task_Alternate_Stack'Address;
       end if;
@@ -1197,15 +1149,6 @@ package body System.Tasking.Stages is
 
                Stack_Base := Bottom_Of_Stack'Address;
 
-               --  Also reduce the size of the stack to take into account the
-               --  secondary stack array declared in this frame. This is for
-               --  sure very conservative.
-
-               if not Parameters.Sec_Stack_Dynamic then
-                  Pattern_Size :=
-                    Pattern_Size - Natural (Secondary_Stack_Size);
-               end if;
-
                --  Adjustments for inner frames
 
                Pattern_Size := Pattern_Size -
@@ -1973,10 +1916,10 @@ package body System.Tasking.Stages is
          then
             Initialization.Task_Lock (Self_ID);
 
-            --  If Sec_Stack_Addr is not null, it means that Destroy_TSD
+            --  If Sec_Stack_Ptr is not null, it means that Destroy_TSD
             --  has not been called yet (case of an unactivated task).
 
-            if T.Common.Compiler_Data.Sec_Stack_Addr /= Null_Address then
+            if T.Common.Compiler_Data.Sec_Stack_Ptr /= null then
                SSL.Destroy_TSD (T.Common.Compiler_Data);
             end if;
 
diff --git a/gcc/ada/libgnarl/s-tassta.ads b/gcc/ada/libgnarl/s-tassta.ads
index bc837fc9af8..a1129a1085a 100644
--- a/gcc/ada/libgnarl/s-tassta.ads
+++ b/gcc/ada/libgnarl/s-tassta.ads
@@ -70,7 +70,7 @@ package System.Tasking.Stages is
    --   tE : aliased boolean := false;
    --   tZ : size_type := unspecified_size;
    --   type tV (discr : integer) is limited record
-   --      _task_id : task_id;
+   --      _task_id         : task_id;
    --   end record;
    --   procedure tB (_task : access tV);
    --   freeze tV [
@@ -168,7 +168,7 @@ package System.Tasking.Stages is
 
    procedure Create_Task
      (Priority             : Integer;
-      Size                 : System.Parameters.Size_Type;
+      Stack_Size           : System.Parameters.Size_Type;
       Secondary_Stack_Size : System.Parameters.Size_Type;
       Task_Info            : System.Task_Info.Task_Info_Type;
       CPU                  : Integer;
@@ -187,31 +187,44 @@ package System.Tasking.Stages is
    --
    --  Priority is the task's priority (assumed to be in range of type
    --   System.Any_Priority)
-   --  Size is the stack size of the task to create
-   --  Secondary_Stack_Size is the secondary stack size of the task to create
+   --
+   --  Stack_Size is the stack size of the task to create
+   --
+   --  Secondary_Stack_Size is the size of the secondary stack to be used by
+   --  the task.
+   --
    --  Task_Info is the task info associated with the created task, or
    --   Unspecified_Task_Info if none.
+   --
    --  CPU is the task affinity. Passed as an Integer because the undefined
    --   value is not in the range of CPU_Range. Static range checks are
    --   performed when analyzing the pragma, and dynamic ones are performed
    --   before setting the affinity at run time.
+   --
    --  Relative_Deadline is the relative deadline associated with the created
    --   task by means of a pragma Relative_Deadline, or 0.0 if none.
+   --
    --  Domain is the dispatching domain associated with the created task by
    --   means of a Dispatching_Domain pragma or aspect, or null if none.
+   --
    --  State is the compiler generated task's procedure body
+   --
    --  Discriminants is a pointer to a limited record whose discriminants
    --   are those of the task to create. This parameter should be passed as
    --   the single argument to State.
+   --
    --  Elaborated is a pointer to a Boolean that must be set to true on exit
    --   if the task could be successfully elaborated.
+   --
    --  Chain is a linked list of task that needs to be created. On exit,
    --   Created_Task.Activation_Link will be Chain.T_ID, and Chain.T_ID
    --   will be Created_Task (e.g the created task will be linked at the front
    --   of Chain).
+   --
    --  Task_Image is a string created by the compiler that the
    --   run time can store to ease the debugging and the
    --   Ada.Task_Identification facility.
+   --
    --  Created_Task is the resulting task.
    --
    --  This procedure can raise Storage_Error if the task creation failed.
diff --git a/gcc/ada/libgnarl/s-tpopmo.adb b/gcc/ada/libgnarl/s-tpopmo.adb
new file mode 100644
index 00000000000..b6164aa19ed
--- /dev/null
+++ b/gcc/ada/libgnarl/s-tpopmo.adb
@@ -0,0 +1,283 @@
+------------------------------------------------------------------------------
+--                                                                          --
+--                 GNAT RUN-TIME LIBRARY (GNARL) COMPONENTS                 --
+--                                                                          --
+--               SYSTEM.TASK_PRIMITIVES.OPERATIONS.MONOTONIC                --
+--                                                                          --
+--                                 B o d y                                  --
+--                                                                          --
+--         Copyright (C) 1992-2017, Free Software Foundation, Inc.          --
+--                                                                          --
+-- GNARL is free software; you can  redistribute it  and/or modify it under --
+-- terms of the  GNU General Public License as published  by the Free Soft- --
+-- ware  Foundation;  either version 3,  or (at your option) any later ver- --
+-- sion.  GNAT is distributed in the hope that it will be useful, but WITH- --
+-- OUT ANY WARRANTY;  without even the  implied warranty of MERCHANTABILITY --
+-- or FITNESS FOR A PARTICULAR PURPOSE.                                     --
+--                                                                          --
+-- As a special exception under Section 7 of GPL version 3, you are granted --
+-- additional permissions described in the GCC Runtime Library Exception,   --
+-- version 3.1, as published by the Free Software Foundation.               --
+--                                                                          --
+-- You should have received a copy of the GNU General Public License and    --
+-- a copy of the GCC Runtime Library Exception along with this program;     --
+-- see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see    --
+-- <http://www.gnu.org/licenses/>.                                          --
+--                                                                          --
+-- GNARL was developed by the GNARL team at Florida State University.       --
+-- Extensive contributions were provided by Ada Core Technologies, Inc.     --
+--                                                                          --
+------------------------------------------------------------------------------
+
+--  This is the Monotonic version of this package for Posix and Linux targets.
+
+separate (System.Task_Primitives.Operations)
+package body Monotonic is
+
+   -----------------------
+   -- Local Subprograms --
+   -----------------------
+
+   procedure Compute_Deadline
+     (Time       : Duration;
+      Mode       : ST.Delay_Modes;
+      Check_Time : out Duration;
+      Abs_Time   : out Duration;
+      Rel_Time   : out Duration);
+   --  Helper for Timed_Sleep and Timed_Delay: given a deadline specified by
+   --  Time and Mode, compute the current clock reading (Check_Time), and the
+   --  target absolute and relative clock readings (Abs_Time, Rel_Time). The
+   --  epoch for Time depends on Mode; the epoch for Check_Time and Abs_Time
+   --  is always that of CLOCK_RT_Ada.
+
+   ---------------------
+   -- Monotonic_Clock --
+   ---------------------
+
+   function Monotonic_Clock return Duration is
+      TS     : aliased timespec;
+      Result : Interfaces.C.int;
+   begin
+      Result := clock_gettime
+        (clock_id => OSC.CLOCK_RT_Ada, tp => TS'Unchecked_Access);
+      pragma Assert (Result = 0);
+
+      return To_Duration (TS);
+   end Monotonic_Clock;
+
+   -------------------
+   -- RT_Resolution --
+   -------------------
+
+   function RT_Resolution return Duration is
+      TS     : aliased timespec;
+      Result : Interfaces.C.int;
+
+   begin
+      Result := clock_getres (OSC.CLOCK_REALTIME, TS'Unchecked_Access);
+      pragma Assert (Result = 0);
+
+      return To_Duration (TS);
+   end RT_Resolution;
+
+   ----------------------
+   -- Compute_Deadline --
+   ----------------------
+
+   procedure Compute_Deadline
+     (Time       : Duration;
+      Mode       : ST.Delay_Modes;
+      Check_Time : out Duration;
+      Abs_Time   : out Duration;
+      Rel_Time   : out Duration)
+   is
+   begin
+      Check_Time := Monotonic_Clock;
+
+      --  Relative deadline
+
+      if Mode = Relative then
+         Abs_Time := Duration'Min (Time, Max_Sensible_Delay) + Check_Time;
+
+         if Relative_Timed_Wait then
+            Rel_Time := Duration'Min (Max_Sensible_Delay, Time);
+         end if;
+
+         pragma Warnings (Off);
+         --  Comparison "OSC.CLOCK_RT_Ada = OSC.CLOCK_REALTIME" is compile
+         --  time known.
+
+      --  Absolute deadline specified using the tasking clock (CLOCK_RT_Ada)
+
+      elsif Mode = Absolute_RT
+        or else OSC.CLOCK_RT_Ada = OSC.CLOCK_REALTIME
+      then
+         pragma Warnings (On);
+         Abs_Time := Duration'Min (Check_Time + Max_Sensible_Delay, Time);
+
+         if Relative_Timed_Wait then
+            Rel_Time := Duration'Min (Max_Sensible_Delay, Time - Check_Time);
+         end if;
+
+      --  Absolute deadline specified using the calendar clock, in the
+      --  case where it is not the same as the tasking clock: compensate for
+      --  difference between clock epochs (Base_Time - Base_Cal_Time).
+
+      else
+         declare
+            Cal_Check_Time : constant Duration := OS_Primitives.Clock;
+            RT_Time        : constant Duration :=
+                               Time + Check_Time - Cal_Check_Time;
+
+         begin
+            Abs_Time :=
+              Duration'Min (Check_Time + Max_Sensible_Delay, RT_Time);
+
+            if Relative_Timed_Wait then
+               Rel_Time :=
+                 Duration'Min (Max_Sensible_Delay, RT_Time - Check_Time);
+            end if;
+         end;
+      end if;
+   end Compute_Deadline;
+
+   -----------------
+   -- Timed_Sleep --
+   -----------------
+
+   --  This is for use within the run-time system, so abort is
+   --  assumed to be already deferred, and the caller should be
+   --  holding its own ATCB lock.
+
+   procedure Timed_Sleep
+     (Self_ID  : ST.Task_Id;
+      Time     : Duration;
+      Mode     : ST.Delay_Modes;
+      Reason   : System.Tasking.Task_States;
+      Timedout : out Boolean;
+      Yielded  : out Boolean)
+   is
+      pragma Unreferenced (Reason);
+
+      Base_Time  : Duration;
+      Check_Time : Duration;
+      Abs_Time   : Duration;
+      Rel_Time   : Duration;
+
+      Request    : aliased timespec;
+      Result     : Interfaces.C.int;
+
+   begin
+      Timedout := True;
+      Yielded := False;
+
+      Compute_Deadline
+        (Time       => Time,
+         Mode       => Mode,
+         Check_Time => Check_Time,
+         Abs_Time   => Abs_Time,
+         Rel_Time   => Rel_Time);
+      Base_Time := Check_Time;
+
+      if Abs_Time > Check_Time then
+         Request :=
+           To_Timespec (if Relative_Timed_Wait then Rel_Time else Abs_Time);
+
+         loop
+            exit when Self_ID.Pending_ATC_Level < Self_ID.ATC_Nesting_Level;
+
+            Result :=
+              pthread_cond_timedwait
+                (cond    => Self_ID.Common.LL.CV'Access,
+                 mutex   => (if Single_Lock
+                             then Single_RTS_Lock'Access
+                             else Self_ID.Common.LL.L'Access),
+                 abstime => Request'Access);
+
+            Check_Time := Monotonic_Clock;
+            exit when Abs_Time <= Check_Time or else Check_Time < Base_Time;
+
+            if Result in 0 | EINTR then
+
+               --  Somebody may have called Wakeup for us
+
+               Timedout := False;
+               exit;
+            end if;
+
+            pragma Assert (Result = ETIMEDOUT);
+         end loop;
+      end if;
+   end Timed_Sleep;
+
+   -----------------
+   -- Timed_Delay --
+   -----------------
+
+   --  This is for use in implementing delay statements, so we assume the
+   --  caller is abort-deferred but is holding no locks.
+
+   procedure Timed_Delay
+     (Self_ID : ST.Task_Id;
+      Time    : Duration;
+      Mode    : ST.Delay_Modes)
+   is
+      Base_Time  : Duration;
+      Check_Time : Duration;
+      Abs_Time   : Duration;
+      Rel_Time   : Duration;
+      Request    : aliased timespec;
+
+      Result : Interfaces.C.int;
+      pragma Warnings (Off, Result);
+
+   begin
+      if Single_Lock then
+         Lock_RTS;
+      end if;
+
+      Write_Lock (Self_ID);
+
+      Compute_Deadline
+        (Time       => Time,
+         Mode       => Mode,
+         Check_Time => Check_Time,
+         Abs_Time   => Abs_Time,
+         Rel_Time   => Rel_Time);
+      Base_Time := Check_Time;
+
+      if Abs_Time > Check_Time then
+         Request :=
+           To_Timespec (if Relative_Timed_Wait then Rel_Time else Abs_Time);
+         Self_ID.Common.State := Delay_Sleep;
+
+         loop
+            exit when Self_ID.Pending_ATC_Level < Self_ID.ATC_Nesting_Level;
+
+            Result :=
+              pthread_cond_timedwait
+                (cond    => Self_ID.Common.LL.CV'Access,
+                 mutex   => (if Single_Lock
+                             then Single_RTS_Lock'Access
+                             else Self_ID.Common.LL.L'Access),
+                 abstime => Request'Access);
+
+            Check_Time := Monotonic_Clock;
+            exit when Abs_Time <= Check_Time or else Check_Time < Base_Time;
+
+            pragma Assert (Result in 0 | ETIMEDOUT | EINTR);
+         end loop;
+
+         Self_ID.Common.State := Runnable;
+      end if;
+
+      Unlock (Self_ID);
+
+      if Single_Lock then
+         Unlock_RTS;
+      end if;
+
+      Result := sched_yield;
+   end Timed_Delay;
+
+end Monotonic;
diff --git a/gcc/ada/libgnarl/s-tporft.adb b/gcc/ada/libgnarl/s-tporft.adb
index 7b8a59276f8..56eda26e6a1 100644
--- a/gcc/ada/libgnarl/s-tporft.adb
+++ b/gcc/ada/libgnarl/s-tporft.adb
@@ -29,16 +29,16 @@
 --                                                                          --
 ------------------------------------------------------------------------------
 
-with System.Task_Info;
---  Use for Unspecified_Task_Info
-
-with System.Soft_Links;
---  used to initialize TSD for a C thread, in function Self
-
 with System.Multiprocessors;
+with System.Soft_Links;
+with System.Task_Info;
 
 separate (System.Task_Primitives.Operations)
-function Register_Foreign_Thread (Thread : Thread_Id) return Task_Id is
+function Register_Foreign_Thread
+  (Thread         : Thread_Id;
+   Sec_Stack_Size : Size_Type := Unspecified_Size)
+   return Task_Id
+is
    Local_ATCB : aliased Ada_Task_Control_Block (0);
    Self_Id    : Task_Id;
    Succeeded  : Boolean;
@@ -66,7 +66,7 @@ begin
      (Self_Id, null, Null_Address, Null_Task,
       Foreign_Task_Elaborated'Access,
       System.Priority'First, System.Multiprocessors.Not_A_Specific_CPU, null,
-      Task_Info.Unspecified_Task_Info, 0, 0, Self_Id, Succeeded);
+      Task_Info.Unspecified_Task_Info, 0, Self_Id, Succeeded);
    Unlock_RTS;
    pragma Assert (Succeeded);
 
@@ -92,7 +92,10 @@ begin
 
    Self_Id.Common.Task_Alternate_Stack := Null_Address;
 
-   System.Soft_Links.Create_TSD (Self_Id.Common.Compiler_Data);
+   --  Create the TSD for the task
+
+   System.Soft_Links.Create_TSD
+     (Self_Id.Common.Compiler_Data, null, Sec_Stack_Size);
 
    Enter_Task (Self_Id);
 
diff --git a/gcc/ada/libgnat/s-parame.adb b/gcc/ada/libgnat/s-parame.adb
index 0f4d45f2da8..359edacb95e 100644
--- a/gcc/ada/libgnat/s-parame.adb
+++ b/gcc/ada/libgnat/s-parame.adb
@@ -50,6 +50,34 @@ package body System.Parameters is
       end if;
    end Adjust_Storage_Size;
 
+   ----------------------------
+   -- Default_Sec_Stack_Size --
+   ----------------------------
+
+   function Default_Sec_Stack_Size return Size_Type is
+      Default_SS_Size : Integer;
+      pragma Import (C, Default_SS_Size,
+                     "__gnat_default_ss_size");
+   begin
+      --  There are two situations where the default secondary stack size is
+      --  set to zero:
+      --
+      --    * The user sets it to zero erroneously thinking it will disable
+      --      the secondary stack.
+      --
+      --    * Or more likely, we are building with an old compiler and
+      --      Default_SS_Size is never set.
+      --
+      --  In both case set the default secondary stack size to the run-time
+      --  default.
+
+      if Default_SS_Size > 0 then
+         return Size_Type (Default_SS_Size);
+      else
+         return Runtime_Default_Sec_Stack_Size;
+      end if;
+   end Default_Sec_Stack_Size;
+
    ------------------------
    -- Default_Stack_Size --
    ------------------------
diff --git a/gcc/ada/libgnat/s-parame.ads b/gcc/ada/libgnat/s-parame.ads
index f48c7e0973f..60a5e997021 100644
--- a/gcc/ada/libgnat/s-parame.ads
+++ b/gcc/ada/libgnat/s-parame.ads
@@ -64,20 +64,6 @@ package System.Parameters is
    Unspecified_Size : constant Size_Type := Size_Type'First;
    --  Value used to indicate that no size type is set
 
-   subtype Percentage is Size_Type range -1 .. 100;
-   Dynamic : constant Size_Type := -1;
-   --  The secondary stack ratio is a constant between 0 and 100 which
-   --  determines the percentage of the allocated task stack that is
-   --  used by the secondary stack (the rest being the primary stack).
-   --  The special value of minus one indicates that the secondary
-   --  stack is to be allocated from the heap instead.
-
-   Sec_Stack_Percentage : constant Percentage := Dynamic;
-   --  This constant defines the handling of the secondary stack
-
-   Sec_Stack_Dynamic : constant Boolean := Sec_Stack_Percentage = Dynamic;
-   --  Convenient Boolean for testing for dynamic secondary stack
-
    function Default_Stack_Size return Size_Type;
    --  Default task stack size used if none is specified
 
@@ -94,15 +80,27 @@ package System.Parameters is
    --    otherwise return given Size
 
    Default_Env_Stack_Size : constant Size_Type := 8_192_000;
-   --  Assumed size of the environment task, if no other information
-   --  is available. This value is used when stack checking is
-   --  enabled and no GNAT_STACK_LIMIT environment variable is set.
+   --  Assumed size of the environment task, if no other information is
+   --  available. This value is used when stack checking is enabled and
+   --  no GNAT_STACK_LIMIT environment variable is set.
 
    Stack_Grows_Down  : constant Boolean := True;
    --  This constant indicates whether the stack grows up (False) or
    --  down (True) in memory as functions are called. It is used for
    --  proper implementation of the stack overflow check.
 
+   Runtime_Default_Sec_Stack_Size : constant Size_Type := 10 * 1024;
+   --  The run-time chosen default size for secondary stacks that may be
+   --  overriden by the user with the use of binder -D switch.
+
+   function Default_Sec_Stack_Size return Size_Type;
+   --  The default initial size for secondary stacks that reflects any user
+   --  specified default via the binder -D switch.
+
+   Sec_Stack_Dynamic : constant Boolean := True;
+   --  Indicates if secondary stacks can grow and shrink at run-time. If False,
+   --  the size of a secondary stack is fixed at the point of its creation.
+
    ----------------------------------------------
    -- Characteristics of types in Interfaces.C --
    ----------------------------------------------
diff --git a/gcc/ada/libgnat/s-parame__ae653.ads b/gcc/ada/libgnat/s-parame__ae653.ads
index 8a787f007bc..42d438e72ea 100644
--- a/gcc/ada/libgnat/s-parame__ae653.ads
+++ b/gcc/ada/libgnat/s-parame__ae653.ads
@@ -62,20 +62,6 @@ package System.Parameters is
    Unspecified_Size : constant Size_Type := Size_Type'First;
    --  Value used to indicate that no size type is set
 
-   subtype Percentage is Size_Type range -1 .. 100;
-   Dynamic : constant Size_Type := -1;
-   --  The secondary stack ratio is a constant between 0 and 100 which
-   --  determines the percentage of the allocated task stack that is
-   --  used by the secondary stack (the rest being the primary stack).
-   --  The special value of minus one indicates that the secondary
-   --  stack is to be allocated from the heap instead.
-
-   Sec_Stack_Percentage : constant Percentage := 25;
-   --  This constant defines the handling of the secondary stack
-
-   Sec_Stack_Dynamic : constant Boolean := Sec_Stack_Percentage = Dynamic;
-   --  Convenient Boolean for testing for dynamic secondary stack
-
    function Default_Stack_Size return Size_Type;
    --  Default task stack size used if none is specified
 
@@ -103,6 +89,18 @@ package System.Parameters is
    --  down (True) in memory as functions are called. It is used for
    --  proper implementation of the stack overflow check.
 
+   Runtime_Default_Sec_Stack_Size : constant Size_Type := 10 * 1024;
+   --  The run-time chosen default size for secondary stacks that may be
+   --  overriden by the user with the use of binder -D switch.
+
+   function Default_Sec_Stack_Size return Size_Type;
+   --  The default size for secondary stacks that reflects any user specified
+   --  default via the binder -D switch.
+
+   Sec_Stack_Dynamic : constant Boolean := False;
+   --  Indicates if secondary stacks can grow and shrink at run-time. If False,
+   --  the size of a secondary stack is fixed at the point of its creation.
+
    ----------------------------------------------
    -- Characteristics of types in Interfaces.C --
    ----------------------------------------------
diff --git a/gcc/ada/libgnat/s-parame__hpux.ads b/gcc/ada/libgnat/s-parame__hpux.ads
index f20cfbebe7e..846b165561e 100644
--- a/gcc/ada/libgnat/s-parame__hpux.ads
+++ b/gcc/ada/libgnat/s-parame__hpux.ads
@@ -62,20 +62,6 @@ package System.Parameters is
    Unspecified_Size : constant Size_Type := Size_Type'First;
    --  Value used to indicate that no size type is set
 
-   subtype Percentage is Size_Type range -1 .. 100;
-   Dynamic : constant Size_Type := -1;
-   --  The secondary stack ratio is a constant between 0 and 100 which
-   --  determines the percentage of the allocated task stack that is
-   --  used by the secondary stack (the rest being the primary stack).
-   --  The special value of minus one indicates that the secondary
-   --  stack is to be allocated from the heap instead.
-
-   Sec_Stack_Percentage : constant Percentage := Dynamic;
-   --  This constant defines the handling of the secondary stack
-
-   Sec_Stack_Dynamic : constant Boolean := Sec_Stack_Percentage = Dynamic;
-   --  Convenient Boolean for testing for dynamic secondary stack
-
    function Default_Stack_Size return Size_Type;
    --  Default task stack size used if none is specified
 
@@ -101,6 +87,18 @@ package System.Parameters is
    --  down (True) in memory as functions are called. It is used for
    --  proper implementation of the stack overflow check.
 
+   Runtime_Default_Sec_Stack_Size : constant Size_Type := 10 * 1024;
+   --  The run-time chosen default size for secondary stacks that may be
+   --  overriden by the user with the use of binder -D switch.
+
+   function Default_Sec_Stack_Size return Size_Type;
+   --  The default initial size for secondary stacks that reflects any user
+   --  specified default via the binder -D switch.
+
+   Sec_Stack_Dynamic : constant Boolean := True;
+   --  Indicates if secondary stacks can grow and shrink at run-time. If False,
+   --  the size of a secondary stack is fixed at the point of its creation.
+
    ----------------------------------------------
    -- Characteristics of Types in Interfaces.C --
    ----------------------------------------------
diff --git a/gcc/ada/libgnat/s-parame__rtems.adb b/gcc/ada/libgnat/s-parame__rtems.adb
index aa131147eb6..5a19c4396da 100644
--- a/gcc/ada/libgnat/s-parame__rtems.adb
+++ b/gcc/ada/libgnat/s-parame__rtems.adb
@@ -6,7 +6,7 @@
 --                                                                          --
 --                                 B o d y                                  --
 --                                                                          --
---          Copyright (C) 1997-2009 Free Software Foundation, Inc.          --
+--          Copyright (C) 1997-2017, Free Software Foundation, Inc.         --
 --                                                                          --
 -- GNAT is free software;  you can  redistribute it  and/or modify it under --
 -- terms of the  GNU General Public License as published  by the Free Soft- --
@@ -39,6 +39,35 @@ package body System.Parameters is
    pragma Import (C, ada_pthread_minimum_stack_size,
      "_ada_pthread_minimum_stack_size");
 
+   -------------------------
+   -- Adjust_Storage_Size --
+   -------------------------
+
+   function Adjust_Storage_Size (Size : Size_Type) return Size_Type is
+   begin
+      if Size = Unspecified_Size then
+         return Default_Stack_Size;
+
+      elsif Size < Minimum_Stack_Size then
+         return Minimum_Stack_Size;
+
+      else
+         return Size;
+      end if;
+   end Adjust_Storage_Size;
+
+   ----------------------------
+   -- Default_Sec_Stack_Size --
+   ----------------------------
+
+   function Default_Sec_Stack_Size return Size_Type is
+      Default_SS_Size : Integer;
+      pragma Import (C, Default_SS_Size,
+                     "__gnat_default_ss_size");
+   begin
+      return Size_Type (Default_SS_Size);
+   end Default_Sec_Stack_Size;
+
    ------------------------
    -- Default_Stack_Size --
    ------------------------
@@ -58,21 +87,4 @@ package body System.Parameters is
       return Size_Type (ada_pthread_minimum_stack_size);
    end Minimum_Stack_Size;
 
-   -------------------------
-   -- Adjust_Storage_Size --
-   -------------------------
-
-   function Adjust_Storage_Size (Size : Size_Type) return Size_Type is
-   begin
-      if Size = Unspecified_Size then
-         return Default_Stack_Size;
-
-      elsif Size < Minimum_Stack_Size then
-         return Minimum_Stack_Size;
-
-      else
-         return Size;
-      end if;
-   end Adjust_Storage_Size;
-
 end System.Parameters;
diff --git a/gcc/ada/libgnat/s-parame__vxworks.adb b/gcc/ada/libgnat/s-parame__vxworks.adb
index 325aa2e4f08..97d74b6932e 100644
--- a/gcc/ada/libgnat/s-parame__vxworks.adb
+++ b/gcc/ada/libgnat/s-parame__vxworks.adb
@@ -48,6 +48,18 @@ package body System.Parameters is
       end if;
    end Adjust_Storage_Size;
 
+   ----------------------------
+   -- Default_Sec_Stack_Size --
+   ----------------------------
+
+   function Default_Sec_Stack_Size return Size_Type is
+      Default_SS_Size : Integer;
+      pragma Import (C, Default_SS_Size,
+                     "__gnat_default_ss_size");
+   begin
+      return Size_Type (Default_SS_Size);
+   end Default_Sec_Stack_Size;
+
    ------------------------
    -- Default_Stack_Size --
    ------------------------
diff --git a/gcc/ada/libgnat/s-parame__vxworks.ads b/gcc/ada/libgnat/s-parame__vxworks.ads
index 919361ad10d..e395e017b05 100644
--- a/gcc/ada/libgnat/s-parame__vxworks.ads
+++ b/gcc/ada/libgnat/s-parame__vxworks.ads
@@ -62,20 +62,6 @@ package System.Parameters is
    Unspecified_Size : constant Size_Type := Size_Type'First;
    --  Value used to indicate that no size type is set
 
-   subtype Percentage is Size_Type range -1 .. 100;
-   Dynamic : constant Size_Type := -1;
-   --  The secondary stack ratio is a constant between 0 and 100 which
-   --  determines the percentage of the allocated task stack that is
-   --  used by the secondary stack (the rest being the primary stack).
-   --  The special value of minus one indicates that the secondary
-   --  stack is to be allocated from the heap instead.
-
-   Sec_Stack_Percentage : constant Percentage := Dynamic;
-   --  This constant defines the handling of the secondary stack
-
-   Sec_Stack_Dynamic : constant Boolean := Sec_Stack_Percentage = Dynamic;
-   --  Convenient Boolean for testing for dynamic secondary stack
-
    function Default_Stack_Size return Size_Type;
    --  Default task stack size used if none is specified
 
@@ -103,6 +89,18 @@ package System.Parameters is
    --  down (True) in memory as functions are called. It is used for
    --  proper implementation of the stack overflow check.
 
+   Runtime_Default_Sec_Stack_Size : constant Size_Type := 10 * 1024;
+   --  The run-time chosen default size for secondary stacks that may be
+   --  overriden by the user with the use of binder -D switch.
+
+   function Default_Sec_Stack_Size return Size_Type;
+   --  The default initial size for secondary stacks that reflects any user
+   --  specified default via the binder -D switch.
+
+   Sec_Stack_Dynamic : constant Boolean := True;
+   --  Indicates if secondary stacks can grow and shrink at run-time. If False,
+   --  the size of a secondary stack is fixed at the point of its creation.
+
    ----------------------------------------------
    -- Characteristics of types in Interfaces.C --
    ----------------------------------------------
diff --git a/gcc/ada/libgnat/s-secsta.adb b/gcc/ada/libgnat/s-secsta.adb
index 0449ee4dbcd..b39cf0dc33d 100644
--- a/gcc/ada/libgnat/s-secsta.adb
+++ b/gcc/ada/libgnat/s-secsta.adb
@@ -31,203 +31,65 @@
 
 pragma Compiler_Unit_Warning;
 
-with System.Soft_Links;
-with System.Parameters;
-
 with Ada.Unchecked_Conversion;
 with Ada.Unchecked_Deallocation;
+with System.Soft_Links;
 
 package body System.Secondary_Stack is
 
    package SSL renames System.Soft_Links;
 
-   use type SSE.Storage_Offset;
    use type System.Parameters.Size_Type;
 
-   SS_Ratio_Dynamic : constant Boolean :=
-                        Parameters.Sec_Stack_Percentage = Parameters.Dynamic;
-   --  There are two entirely different implementations of the secondary
-   --  stack mechanism in this unit, and this Boolean is used to select
-   --  between them (at compile time, so the generated code will contain
-   --  only the code for the desired variant). If SS_Ratio_Dynamic is
-   --  True, then the secondary stack is dynamically allocated from the
-   --  heap in a linked list of chunks. If SS_Ration_Dynamic is False,
-   --  then the secondary stack is allocated statically by grabbing a
-   --  section of the primary stack and using it for this purpose.
-
-   type Memory is array (SS_Ptr range <>) of SSE.Storage_Element;
-   for Memory'Alignment use Standard'Maximum_Alignment;
-   --  This is the type used for actual allocation of secondary stack
-   --  areas. We require maximum alignment for all such allocations.
-
-   ---------------------------------------------------------------
-   -- Data Structures for Dynamically Allocated Secondary Stack --
-   ---------------------------------------------------------------
-
-   --  The following is a diagram of the data structures used for the
-   --  case of a dynamically allocated secondary stack, where the stack
-   --  is allocated as a linked list of chunks allocated from the heap.
-
-   --                                      +------------------+
-   --                                      |       Next       |
-   --                                      +------------------+
-   --                                      |                  | Last (200)
-   --                                      |                  |
-   --                                      |                  |
-   --                                      |                  |
-   --                                      |                  |
-   --                                      |                  |
-   --                                      |                  | First (101)
-   --                                      +------------------+
-   --                         +----------> |          |       |
-   --                         |            +--------- | ------+
-   --                         |                    ^  |
-   --                         |                    |  |
-   --                         |                    |  V
-   --                         |            +------ | ---------+
-   --                         |            |       |          |
-   --                         |            +------------------+
-   --                         |            |                  | Last (100)
-   --                         |            |         C        |
-   --                         |            |         H        |
-   --    +-----------------+  |   +------->|         U        |
-   --    |  Current_Chunk ----+   |        |         N        |
-   --    +-----------------+      |        |         K        |
-   --    |       Top      --------+        |                  | First (1)
-   --    +-----------------+               +------------------+
-   --    | Default_Size    |               |       Prev       |
-   --    +-----------------+               +------------------+
-   --
-
-   type Chunk_Id (First, Last : SS_Ptr);
-   type Chunk_Ptr is access all Chunk_Id;
-
-   type Chunk_Id (First, Last : SS_Ptr) is record
-      Prev, Next : Chunk_Ptr;
-      Mem        : Memory (First .. Last);
-   end record;
-
-   type Stack_Id is record
-      Top           : SS_Ptr;
-      Default_Size  : SSE.Storage_Count;
-      Current_Chunk : Chunk_Ptr;
-   end record;
-
-   type Stack_Ptr is access Stack_Id;
-   --  Pointer to record used to represent a dynamically allocated secondary
-   --  stack descriptor for a secondary stack chunk.
-
    procedure Free is new Ada.Unchecked_Deallocation (Chunk_Id, Chunk_Ptr);
    --  Free a dynamically allocated chunk
 
-   function To_Stack_Ptr is new
-     Ada.Unchecked_Conversion (Address, Stack_Ptr);
-   function To_Addr is new
-     Ada.Unchecked_Conversion (Stack_Ptr, Address);
-   --  Convert to and from address stored in task data structures
-
-   --------------------------------------------------------------
-   -- Data Structures for Statically Allocated Secondary Stack --
-   --------------------------------------------------------------
-
-   --  For the static case, the secondary stack is a single contiguous
-   --  chunk of storage, carved out of the primary stack, and represented
-   --  by the following data structure
-
-   type Fixed_Stack_Id is record
-      Top : SS_Ptr;
-      --  Index of next available location in Mem. This is initialized to
-      --  0, and then incremented on Allocate, and Decremented on Release.
-
-      Last : SS_Ptr;
-      --  Length of usable Mem array, which is thus the index past the
-      --  last available location in Mem. Mem (Last-1) can be used. This
-      --  is used to check that the stack does not overflow.
-
-      Max : SS_Ptr;
-      --  Maximum value of Top. Initialized to 0, and then may be incremented
-      --  on Allocate, but is never Decremented. The last used location will
-      --  be Mem (Max - 1), so Max is the maximum count of used stack space.
-
-      Mem : Memory (0 .. 0);
-      --  This is the area that is actually used for the secondary stack.
-      --  Note that the upper bound is a dummy value properly defined by
-      --  the value of Last. We never actually allocate objects of type
-      --  Fixed_Stack_Id, so the bounds declared here do not matter.
-   end record;
-
-   Dummy_Fixed_Stack : Fixed_Stack_Id;
-   pragma Warnings (Off, Dummy_Fixed_Stack);
-   --  Well it is not quite true that we never allocate an object of the
-   --  type. This dummy object is allocated for the purpose of getting the
-   --  offset of the Mem field via the 'Position attribute (such a nuisance
-   --  that we cannot apply this to a field of a type).
-
-   type Fixed_Stack_Ptr is access Fixed_Stack_Id;
-   --  Pointer to record used to describe statically allocated sec stack
-
-   function To_Fixed_Stack_Ptr is new
-     Ada.Unchecked_Conversion (Address, Fixed_Stack_Ptr);
-   --  Convert from address stored in task data structures
-
-   ----------------------------------
-   -- Minimum_Secondary_Stack_Size --
-   ----------------------------------
-
-   function Minimum_Secondary_Stack_Size return Natural is
-   begin
-      return Dummy_Fixed_Stack.Mem'Position;
-   end Minimum_Secondary_Stack_Size;
-
-   --------------
-   -- Allocate --
-   --------------
+   -----------------
+   -- SS_Allocate --
+   -----------------
 
    procedure SS_Allocate
      (Addr         : out Address;
       Storage_Size : SSE.Storage_Count)
    is
-      Max_Align : constant SS_Ptr := SS_Ptr (Standard'Maximum_Alignment);
-      Max_Size  : constant SS_Ptr :=
-                    ((SS_Ptr (Storage_Size) + Max_Align - 1) / Max_Align) *
-                      Max_Align;
-
+      Max_Align   : constant SS_Ptr := SS_Ptr (Standard'Maximum_Alignment);
+      Mem_Request : constant SS_Ptr :=
+                      ((SS_Ptr (Storage_Size) + Max_Align - 1) / Max_Align) *
+                        Max_Align;
+      --  Round up Storage_Size to the nearest multiple of the max alignment
+      --  value for the target. This ensures efficient stack access.
+
+      Stack : constant SS_Stack_Ptr := SSL.Get_Sec_Stack.all;
    begin
-      --  Case of fixed allocation secondary stack
-
-      if not SS_Ratio_Dynamic then
-         declare
-            Fixed_Stack : constant Fixed_Stack_Ptr :=
-                            To_Fixed_Stack_Ptr (SSL.Get_Sec_Stack_Addr.all);
+      --  Case of fixed secondary stack
 
-         begin
-            --  Check if max stack usage is increasing
+      if not SP.Sec_Stack_Dynamic then
+         --  Check if max stack usage is increasing
 
-            if Fixed_Stack.Top + Max_Size > Fixed_Stack.Max then
+         if Stack.Top + Mem_Request > Stack.Max then
 
-               --  If so, check if max size is exceeded
+            --  If so, check if the stack is exceeded, noting Stack.Top points
+            --  to the first free byte (so the value of Stack.Top on a fully
+            --  allocated stack will be Stack.Size + 1).
 
-               if Fixed_Stack.Top + Max_Size > Fixed_Stack.Last then
-                  raise Storage_Error;
-               end if;
+            if Stack.Top + Mem_Request > Stack.Size + 1 then
+               raise Storage_Error;
+            end if;
 
-               --  Record new max usage
+            --  Record new max usage
 
-               Fixed_Stack.Max := Fixed_Stack.Top + Max_Size;
-            end if;
+            Stack.Max := Stack.Top + Mem_Request;
+         end if;
 
-            --  Set resulting address and update top of stack pointer
+         --  Set resulting address and update top of stack pointer
 
-            Addr := Fixed_Stack.Mem (Fixed_Stack.Top)'Address;
-            Fixed_Stack.Top := Fixed_Stack.Top + Max_Size;
-         end;
+         Addr := Stack.Internal_Chunk.Mem (Stack.Top)'Address;
+         Stack.Top := Stack.Top + Mem_Request;
 
-      --  Case of dynamically allocated secondary stack
+      --  Case of dynamic secondary stack
 
       else
          declare
-            Stack : constant Stack_Ptr :=
-                      To_Stack_Ptr (SSL.Get_Sec_Stack_Addr.all);
             Chunk : Chunk_Ptr;
 
             To_Be_Released_Chunk : Chunk_Ptr;
@@ -235,7 +97,7 @@ package body System.Secondary_Stack is
          begin
             Chunk := Stack.Current_Chunk;
 
-            --  The Current_Chunk may not be the good one if a lot of release
+            --  The Current_Chunk may not be the best one if a lot of release
             --  operations have taken place. Go down the stack if necessary.
 
             while Chunk.First > Stack.Top loop
@@ -246,7 +108,7 @@ package body System.Secondary_Stack is
             --  sufficient, if not, go to the next one and eventually create
             --  the necessary room.
 
-            while Chunk.Last - Stack.Top + 1 < Max_Size loop
+            while Chunk.Last - Stack.Top + 1 < Mem_Request loop
                if Chunk.Next /= null then
 
                   --  Release unused non-first empty chunk
@@ -262,11 +124,11 @@ package body System.Secondary_Stack is
                --  Create new chunk of default size unless it is not sufficient
                --  to satisfy the current request.
 
-               elsif SSE.Storage_Count (Max_Size) <= Stack.Default_Size then
+               elsif Mem_Request <= Stack.Size then
                   Chunk.Next :=
                     new Chunk_Id
                       (First => Chunk.Last + 1,
-                       Last  => Chunk.Last + SS_Ptr (Stack.Default_Size));
+                       Last  => Chunk.Last + SS_Ptr (Stack.Size));
 
                   Chunk.Next.Prev := Chunk;
 
@@ -276,7 +138,7 @@ package body System.Secondary_Stack is
                   Chunk.Next :=
                     new Chunk_Id
                       (First => Chunk.Last + 1,
-                       Last  => Chunk.Last + Max_Size);
+                       Last  => Chunk.Last + Mem_Request);
 
                   Chunk.Next.Prev := Chunk;
                end if;
@@ -288,8 +150,15 @@ package body System.Secondary_Stack is
             --  Resulting address is the address pointed by Stack.Top
 
             Addr                := Chunk.Mem (Stack.Top)'Address;
-            Stack.Top           := Stack.Top + Max_Size;
+            Stack.Top           := Stack.Top + Mem_Request;
             Stack.Current_Chunk := Chunk;
+
+            --  Record new max usage
+
+            if Stack.Top > Stack.Max then
+               Stack.Max := Stack.Top;
+            end if;
+
          end;
       end if;
    end SS_Allocate;
@@ -298,40 +167,39 @@ package body System.Secondary_Stack is
    -- SS_Free --
    -------------
 
-   procedure SS_Free (Stk : in out Address) is
+   procedure SS_Free (Stack : in out SS_Stack_Ptr) is
+      procedure Free is
+         new Ada.Unchecked_Deallocation (SS_Stack, SS_Stack_Ptr);
    begin
-      --  Case of statically allocated secondary stack, nothing to free
-
-      if not SS_Ratio_Dynamic then
-         return;
+      --  If using dynamic secondary stack, free any external chunks
 
-      --  Case of dynamically allocated secondary stack
-
-      else
+      if SP.Sec_Stack_Dynamic then
          declare
-            Stack : Stack_Ptr := To_Stack_Ptr (Stk);
             Chunk : Chunk_Ptr;
 
             procedure Free is
-              new Ada.Unchecked_Deallocation (Stack_Id, Stack_Ptr);
+              new Ada.Unchecked_Deallocation (Chunk_Id, Chunk_Ptr);
 
          begin
             Chunk := Stack.Current_Chunk;
 
-            while Chunk.Prev /= null loop
-               Chunk := Chunk.Prev;
-            end loop;
+            --  Go to top of linked list and free backwards. Do not free the
+            --  internal chunk as it is part of SS_Stack.
 
             while Chunk.Next /= null loop
                Chunk := Chunk.Next;
-               Free (Chunk.Prev);
             end loop;
 
-            Free (Chunk);
-            Free (Stack);
-            Stk := Null_Address;
+            while Chunk.Prev /= null loop
+               Chunk := Chunk.Prev;
+               Free (Chunk.Next);
+            end loop;
          end;
       end if;
+
+      if Stack.Freeable then
+         Free (Stack);
+      end if;
    end SS_Free;
 
    ----------------
@@ -339,17 +207,13 @@ package body System.Secondary_Stack is
    ----------------
 
    function SS_Get_Max return Long_Long_Integer is
+      Stack : constant SS_Stack_Ptr := SSL.Get_Sec_Stack.all;
    begin
-      if SS_Ratio_Dynamic then
-         return -1;
-      else
-         declare
-            Fixed_Stack : constant Fixed_Stack_Ptr :=
-                            To_Fixed_Stack_Ptr (SSL.Get_Sec_Stack_Addr.all);
-         begin
-            return Long_Long_Integer (Fixed_Stack.Max);
-         end;
-      end if;
+      --  Stack.Max points to the first untouched byte in the stack, thus the
+      --  maximum number of bytes that have been allocated on the stack is one
+      --  less the value of Stack.Max.
+
+      return Long_Long_Integer (Stack.Max - 1);
    end SS_Get_Max;
 
    -------------
@@ -357,32 +221,25 @@ package body System.Secondary_Stack is
    -------------
 
    procedure SS_Info is
+      Stack : constant SS_Stack_Ptr := SSL.Get_Sec_Stack.all;
    begin
       Put_Line ("Secondary Stack information:");
 
       --  Case of fixed secondary stack
 
-      if not SS_Ratio_Dynamic then
-         declare
-            Fixed_Stack : constant Fixed_Stack_Ptr :=
-                            To_Fixed_Stack_Ptr (SSL.Get_Sec_Stack_Addr.all);
-
-         begin
-            Put_Line ("  Total size              : "
-                      & SS_Ptr'Image (Fixed_Stack.Last)
-                      & " bytes");
+      if not SP.Sec_Stack_Dynamic then
+         Put_Line ("  Total size              : "
+                   & SS_Ptr'Image (Stack.Size)
+                   & " bytes");
 
-            Put_Line ("  Current allocated space : "
-                      & SS_Ptr'Image (Fixed_Stack.Top)
-                      & " bytes");
-         end;
+         Put_Line ("  Current allocated space : "
+                   & SS_Ptr'Image (Stack.Top - 1)
+                   & " bytes");
 
-      --  Case of dynamically allocated secondary stack
+      --  Case of dynamic secondary stack
 
       else
          declare
-            Stack     : constant Stack_Ptr :=
-                          To_Stack_Ptr (SSL.Get_Sec_Stack_Addr.all);
             Nb_Chunks : Integer   := 1;
             Chunk     : Chunk_Ptr := Stack.Current_Chunk;
 
@@ -414,7 +271,7 @@ package body System.Secondary_Stack is
                       & Integer'Image (Nb_Chunks));
 
             Put_Line ("  Default size of Chunks : "
-                      & SSE.Storage_Count'Image (Stack.Default_Size));
+                      & SP.Size_Type'Image (Stack.Size));
          end;
       end if;
    end SS_Info;
@@ -424,42 +281,86 @@ package body System.Secondary_Stack is
    -------------
 
    procedure SS_Init
-     (Stk  : in out Address;
-      Size : Natural := Default_Secondary_Stack_Size)
+     (Stack : in out SS_Stack_Ptr;
+      Size  : SP.Size_Type := SP.Unspecified_Size)
    is
-   begin
-      --  Case of fixed size secondary stack
-
-      if not SS_Ratio_Dynamic then
-         declare
-            Fixed_Stack : constant Fixed_Stack_Ptr :=
-                            To_Fixed_Stack_Ptr (Stk);
-
-         begin
-            Fixed_Stack.Top  := 0;
-            Fixed_Stack.Max  := 0;
-
-            if Size <= Dummy_Fixed_Stack.Mem'Position then
-               Fixed_Stack.Last := 0;
-            else
-               Fixed_Stack.Last :=
-                 SS_Ptr (Size) - Dummy_Fixed_Stack.Mem'Position;
-            end if;
-         end;
-
-      --  Case of dynamically allocated secondary stack
+      use Parameters;
 
-      else
-         declare
-            Stack : Stack_Ptr;
-         begin
-            Stack               := new Stack_Id;
-            Stack.Current_Chunk := new Chunk_Id (1, SS_Ptr (Size));
-            Stack.Top           := 1;
-            Stack.Default_Size  := SSE.Storage_Count (Size);
-            Stk := To_Addr (Stack);
-         end;
+      Stack_Size : Size_Type;
+   begin
+      --  If Stack is not null then the stack has been allocated outside the
+      --  package (by the compiler or the user) and all that is left to do is
+      --  initialize the stack. Otherwise, SS_Init will allocate a secondary
+      --  stack from either the heap or the default-sized secondary stack pool
+      --  generated by the binder. In the later case, this pool is generated
+      --  only when the either No_Implicit_Heap_Allocations
+      --  or No_Implicit_Task_Allocations are active, and SS_Init will allocate
+      --  all requests for a secondary stack of Unspecified_Size from this
+      --  pool.
+
+      if Stack = null then
+         if Size = Unspecified_Size then
+            Stack_Size := Default_Sec_Stack_Size;
+         else
+            Stack_Size := Size;
+         end if;
+
+         if Size = Unspecified_Size
+           and then Binder_SS_Count > 0
+           and then Num_Of_Assigned_Stacks < Binder_SS_Count
+         then
+            --  The default-sized secondary stack pool is passed from the
+            --  binder to this package as an Address since it is not possible
+            --  to have a pointer to an array of unconstrained objects. A
+            --  pointer to the pool is obtainable via an unchecked conversion
+            --  to a constrained array of SS_Stacks that mirrors the one used
+            --  by the binder.
+
+            --  However, Ada understandably does not allow a local pointer to
+            --  a stack in the pool to be stored in a pointer outside of this
+            --  scope. While the conversion is safe in this case, since a view
+            --  of a global object is being used, using Unchecked_Access
+            --  would prevent users from specifying the restriction
+            --  No_Unchecked_Access whenever the secondary stack is used. As
+            --  a workaround, the local stack pointer is converted to a global
+            --  pointer via System.Address.
+
+            declare
+               type Stk_Pool_Array is array (1 .. Binder_SS_Count) of
+                 aliased SS_Stack (Default_SS_Size);
+               type Stk_Pool_Access is access Stk_Pool_Array;
+
+               function To_Stack_Pool is new
+                 Ada.Unchecked_Conversion (Address, Stk_Pool_Access);
+
+               pragma Warnings (Off);
+               function To_Global_Ptr is new
+                 Ada.Unchecked_Conversion (Address, SS_Stack_Ptr);
+               pragma Warnings (On);
+               --  Suppress aliasing warning since the pointer we return will
+               --  be the only access to the stack.
+
+               Local_Stk_Address : System.Address;
+
+            begin
+               Num_Of_Assigned_Stacks := Num_Of_Assigned_Stacks + 1;
+
+               Local_Stk_Address :=
+                 To_Stack_Pool
+                   (Default_Sized_SS_Pool) (Num_Of_Assigned_Stacks)'Address;
+               Stack := To_Global_Ptr (Local_Stk_Address);
+            end;
+
+            Stack.Freeable := False;
+         else
+            Stack := new SS_Stack (Stack_Size);
+            Stack.Freeable := True;
+         end if;
       end if;
+
+      Stack.Top := 1;
+      Stack.Max := 1;
+      Stack.Current_Chunk := Stack.Internal_Chunk'Access;
    end SS_Init;
 
    -------------
@@ -467,13 +368,9 @@ package body System.Secondary_Stack is
    -------------
 
    function SS_Mark return Mark_Id is
-      Sstk : constant System.Address := SSL.Get_Sec_Stack_Addr.all;
+      Stack : constant SS_Stack_Ptr := SSL.Get_Sec_Stack.all;
    begin
-      if SS_Ratio_Dynamic then
-         return (Sstk => Sstk, Sptr => To_Stack_Ptr (Sstk).Top);
-      else
-         return (Sstk => Sstk, Sptr => To_Fixed_Stack_Ptr (Sstk).Top);
-      end if;
+      return (Sec_Stack => Stack, Sptr => Stack.Top);
    end SS_Mark;
 
    ----------------
@@ -482,66 +379,7 @@ package body System.Secondary_Stack is
 
    procedure SS_Release (M : Mark_Id) is
    begin
-      if SS_Ratio_Dynamic then
-         To_Stack_Ptr (M.Sstk).Top := M.Sptr;
-      else
-         To_Fixed_Stack_Ptr (M.Sstk).Top := M.Sptr;
-      end if;
+      M.Sec_Stack.Top := M.Sptr;
    end SS_Release;
 
-   -------------------------
-   -- Package Elaboration --
-   -------------------------
-
-   --  Allocate a secondary stack for the main program to use
-
-   --  We make sure that the stack has maximum alignment. Some systems require
-   --  this (e.g. Sparc), and in any case it is a good idea for efficiency.
-
-   Stack : aliased Stack_Id;
-   for Stack'Alignment use Standard'Maximum_Alignment;
-
-   Static_Secondary_Stack_Size : constant := 10 * 1024;
-   --  Static_Secondary_Stack_Size must be static so that Chunk is allocated
-   --  statically, and not via dynamic memory allocation.
-
-   Chunk : aliased Chunk_Id (1, Static_Secondary_Stack_Size);
-   for Chunk'Alignment use Standard'Maximum_Alignment;
-   --  Default chunk used, unless gnatbind -D is specified with a value greater
-   --  than Static_Secondary_Stack_Size.
-
-begin
-   declare
-      Chunk_Address : Address;
-      Chunk_Access  : Chunk_Ptr;
-
-   begin
-      if Default_Secondary_Stack_Size <= Static_Secondary_Stack_Size then
-
-         --  Normally we allocate the secondary stack for the main program
-         --  statically, using the default secondary stack size.
-
-         Chunk_Access := Chunk'Access;
-
-      else
-         --  Default_Secondary_Stack_Size was increased via gnatbind -D, so we
-         --  need to allocate a chunk dynamically.
-
-         Chunk_Access :=
-           new Chunk_Id (1, SS_Ptr (Default_Secondary_Stack_Size));
-      end if;
-
-      if SS_Ratio_Dynamic then
-         Stack.Top           := 1;
-         Stack.Current_Chunk := Chunk_Access;
-         Stack.Default_Size  :=
-           SSE.Storage_Offset (Default_Secondary_Stack_Size);
-         System.Soft_Links.Set_Sec_Stack_Addr_NT (Stack'Address);
-
-      else
-         Chunk_Address := Chunk_Access.all'Address;
-         SS_Init (Chunk_Address, Default_Secondary_Stack_Size);
-         System.Soft_Links.Set_Sec_Stack_Addr_NT (Chunk_Address);
-      end if;
-   end;
 end System.Secondary_Stack;
diff --git a/gcc/ada/libgnat/s-secsta.ads b/gcc/ada/libgnat/s-secsta.ads
index 534708d1a6f..ae5ec888453 100644
--- a/gcc/ada/libgnat/s-secsta.ads
+++ b/gcc/ada/libgnat/s-secsta.ads
@@ -31,41 +31,27 @@
 
 pragma Compiler_Unit_Warning;
 
+with System.Parameters;
 with System.Storage_Elements;
 
 package System.Secondary_Stack is
+   pragma Preelaborate;
 
+   package SP renames System.Parameters;
    package SSE renames System.Storage_Elements;
 
-   Default_Secondary_Stack_Size : Natural := 10 * 1024;
-   --  Default size of a secondary stack. May be modified by binder -D switch
-   --  which causes the binder to generate an appropriate assignment in the
-   --  binder generated file.
+   type SS_Stack (Size : SP.Size_Type) is private;
+   --  Data structure for secondary stacks
 
-   function Minimum_Secondary_Stack_Size return Natural;
-   --  The minimum size of the secondary stack so that the internal
-   --  requirements of the stack are met.
+   type SS_Stack_Ptr is access all SS_Stack;
+   --  Pointer to secondary stack objects
 
    procedure SS_Init
-     (Stk  : in out Address;
-      Size : Natural := Default_Secondary_Stack_Size);
-   --  Initialize the secondary stack with a main stack of the given Size.
-   --
-   --  If System.Parameters.Sec_Stack_Percentage equals Dynamic, Stk is really
-   --  an OUT parameter that will be allocated on the heap. Then all further
-   --  allocations which do not overflow the main stack will not generate
-   --  dynamic (de)allocation calls. If the main Stack overflows, a new
-   --  chuck of at least the same size will be allocated and linked to the
-   --  previous chunk.
-   --
-   --  Otherwise (Sec_Stack_Percentage between 0 and 100), Stk is an IN
-   --  parameter that is already pointing to a Stack_Id. The secondary stack
-   --  in this case is fixed, and any attempt to allocate more than the initial
-   --  size will result in a Storage_Error being raised.
-   --
-   --  Note: the reason that Stk is passed is that SS_Init is called before
-   --  the proper interface is established to obtain the address of the
-   --  stack using System.Soft_Links.Get_Sec_Stack_Addr.
+     (Stack : in out SS_Stack_Ptr;
+      Size  : SP.Size_Type := SP.Unspecified_Size);
+   --  Initialize the secondary stack Stack. If Stack is null allocate a stack
+   --  from the heap or from the default-sized secondary stack pool if the
+   --  pool exists and the requested size is Unspecified_Size.
 
    procedure SS_Allocate
      (Addr         : out Address;
@@ -73,10 +59,9 @@ package System.Secondary_Stack is
    --  Allocate enough space for a 'Storage_Size' bytes object with Maximum
    --  alignment. The address of the allocated space is returned in Addr.
 
-   procedure SS_Free (Stk : in out Address);
-   --  Release the memory allocated for the Secondary Stack. That is
-   --  to say, all the allocated chunks. Upon return, Stk will be set
-   --  to System.Null_Address.
+   procedure SS_Free (Stack : in out SS_Stack_Ptr);
+   --  Release the memory allocated for the Stack. If the stack was statically
+   --  allocated the SS_Stack record is not freed.
 
    type Mark_Id is private;
    --  Type used to mark the stack for mark/release processing
@@ -85,17 +70,11 @@ package System.Secondary_Stack is
    --  Return the Mark corresponding to the current state of the stack
 
    procedure SS_Release (M : Mark_Id);
-   --  Restore the state of the stack corresponding to the mark M. If an
-   --  additional chunk have been allocated, it will never be freed during a
-   --  ??? missing comment here
+   --  Restore the state of the stack corresponding to the mark M
 
    function SS_Get_Max return Long_Long_Integer;
-   --  Return maximum used space in storage units for the current secondary
-   --  stack. For a dynamically allocated secondary stack, the returned
-   --  result is always -1. For a statically allocated secondary stack,
-   --  the returned value shows the largest amount of space allocated so
-   --  far during execution of the program to the current secondary stack,
-   --  i.e. the secondary stack for the current task.
+   --  Return the high water mark of the secondary stack for the current
+   --  secondary stack in bytes.
 
    generic
       with procedure Put_Line (S : String);
@@ -109,15 +88,142 @@ private
    --  Unused entity that is just present to ease the sharing of the pool
    --  mechanism for specific allocation/deallocation in the compiler
 
-   type SS_Ptr is new SSE.Integer_Address;
-   --  Stack pointer value for secondary stack
+   -------------------------------------
+   -- Secondary Stack Data Structures --
+   -------------------------------------
+
+   --  This package provides fixed and dynamically sized secondary stack
+   --  implementations centered around a common data structure SS_Stack. This
+   --  record contains an initial secondary stack allocation of the requested
+   --  size, and markers for the current top of the stack and the high-water
+   --  mark of the stack. A SS_Stack can be either pre-allocated outside the
+   --  package or SS_Init can allocate a stack from the heap or the
+   --  default-sized secondary stack from a pool generated by the binder.
+
+   --  For dynamically allocated secondary stacks, the stack can grow via a
+   --  linked list of stack chunks allocated from the heap. New chunks are
+   --  allocated once the initial static allocation and any existing chunks are
+   --  exhausted. The following diagram illustrated the data structures used
+   --  for a dynamically allocated secondary stack:
+   --
+   --                                       +------------------+
+   --                                       |       Next       |
+   --                                       +------------------+
+   --                                       |                  | Last (300)
+   --                                       |                  |
+   --                                       |                  |
+   --                                       |                  |
+   --                                       |                  |
+   --                                       |                  |
+   --                                       |                  | First (201)
+   --                                       +------------------+
+   --    +-----------------+       +------> |          |       |
+   --    |                 | (100) |        +--------- | ------+
+   --    |                 |       |                ^  |
+   --    |                 |       |                |  |
+   --    |                 |       |                |  V
+   --    |                 |       |        +------ | ---------+
+   --    |                 |       |        |       |          |
+   --    |                 |       |        +------------------+
+   --    |                 |       |        |                  | Last (200)
+   --    |                 |       |        |         C        |
+   --    |                 | (1)   |        |         H        |
+   --    +-----------------+       |  +---->|         U        |
+   --    |  Current_Chunk ---------+  |     |         N        |
+   --    +-----------------+          |     |         K        |
+   --    |       Top      ------------+     |                  | First (101)
+   --    +-----------------+                +------------------+
+   --    |       Size      |                |       Prev       |
+   --    +-----------------+                +------------------+
+   --
+   --  The implementation used by the runtime is controlled via the constant
+   --  System.Parameter.Sec_Stack_Dynamic. If True, the implementation is
+   --  permitted to grow the secondary stack at runtime. The implementation is
+   --  designed for the compiler to include only code to support the desired
+   --  secondary stack behavior.
+
+   subtype SS_Ptr is SP.Size_Type;
+   --  Stack pointer value for the current position within the secondary stack.
+   --  Size_Type is used as the base type since the Size discriminate of
+   --  SS_Stack forms the bounds of the internal memory array.
+
+   type Memory is array (SS_Ptr range <>) of SSE.Storage_Element;
+   for Memory'Alignment use Standard'Maximum_Alignment;
+   --  The region of memory that holds the stack itself. Requires maximum
+   --  alignment for efficient stack operations.
+
+   --  Chunk_Id
+
+   --  Chunk_Id is a contiguous block of dynamically allocated stack. First
+   --  and Last indicate the range of secondary stack addresses present in the
+   --  chunk. Chunk_Ptr points to a Chunk_Id block.
+
+   type Chunk_Id (First, Last : SS_Ptr);
+   type Chunk_Ptr is access all Chunk_Id;
+
+   type Chunk_Id (First, Last : SS_Ptr) is record
+      Prev, Next : Chunk_Ptr;
+      Mem        : Memory (First .. Last);
+   end record;
+
+   --  Secondary stack data structure
+
+   type SS_Stack (Size : SP.Size_Type) is record
+      Top : SS_Ptr;
+      --  Index of next available location in the stack. Initialized to 1 and
+      --  then incremented on Allocate and decremented on Release.
+
+      Max : SS_Ptr;
+      --  Contains the high-water mark of Top. Initialized to 1 and then
+      --  may be incremented on Allocate but never decremented. Since
+      --  Top = Size + 1 represents a fully used stack, Max - 1 indicates
+      --  the size of the stack used in bytes.
+
+      Current_Chunk : Chunk_Ptr;
+      --  A link to the chunk containing the highest range of the stack
+
+      Freeable : Boolean;
+      --  Indicates if an object of this type can be freed
+
+      Internal_Chunk : aliased Chunk_Id (1, Size);
+      --  Initial memory allocation of the secondary stack
+   end record;
 
    type Mark_Id is record
-      Sstk : System.Address;
-      Sptr : SS_Ptr;
+      Sec_Stack : SS_Stack_Ptr;
+      Sptr      : SS_Ptr;
    end record;
-   --  A mark value contains the address of the secondary stack structure,
-   --  as returned by System.Soft_Links.Get_Sec_Stack_Addr, and a stack
-   --  pointer value corresponding to the point of the mark call.
+   --  Contains the pointer to the secondary stack object and the stack pointer
+   --  value corresponding to the top of the stack at the time of the mark
+   --  call.
+
+   ------------------------------------
+   -- Binder Allocated Stack Support --
+   ------------------------------------
+
+   --  When the No_Implicit_Heap_Allocations or No_Implicit_Task_Allocations
+   --  restrictions are in effect the binder statically generates secondary
+   --  stacks for tasks who are using default-sized secondary stack. Assignment
+   --  of these stacks to tasks is handled by SS_Init. The following variables
+   --  assist SS_Init and are defined here so the runtime does not depend on
+   --  the binder.
+
+   Binder_SS_Count : Natural;
+   pragma Export (Ada, Binder_SS_Count, "__gnat_binder_ss_count");
+   --  The number of default sized secondary stacks allocated by the binder
+
+   Default_SS_Size : SP.Size_Type;
+   pragma Export (Ada, Default_SS_Size, "__gnat_default_ss_size");
+   --  The default size for secondary stacks. Defined here and not in init.c/
+   --  System.Init because these locations are not present on ZFP or
+   --  Ravenscar-SFP run-times.
+
+   Default_Sized_SS_Pool : System.Address;
+   pragma Export (Ada, Default_Sized_SS_Pool, "__gnat_default_ss_pool");
+   --  Address to the secondary stack pool generated by the binder that
+   --  contains default sized stacks.
+
+   Num_Of_Assigned_Stacks : Natural := 0;
+   --  The number of currently allocated secondary stacks
 
 end System.Secondary_Stack;
diff --git a/gcc/ada/libgnat/s-soflin.adb b/gcc/ada/libgnat/s-soflin.adb
index f604f4df3be..94ead0306fa 100644
--- a/gcc/ada/libgnat/s-soflin.adb
+++ b/gcc/ada/libgnat/s-soflin.adb
@@ -35,25 +35,19 @@ pragma Polling (Off);
 --  We must turn polling off for this unit, because otherwise we get an
 --  infinite loop from the code within the Poll routine itself.
 
-with System.Parameters;
-
 pragma Warnings (Off);
---  Disable warnings since System.Secondary_Stack is currently not Preelaborate
-with System.Secondary_Stack;
+--  Disable warnings as System.Soft_Links.Initialize is not Preelaborate. It is
+--  safe to with this unit as its elaboration routine will only be initializing
+--  NT_TSD, which is part of this package spec.
+with System.Soft_Links.Initialize;
 pragma Warnings (On);
 
 package body System.Soft_Links is
 
-   package SST renames System.Secondary_Stack;
-
-   NT_TSD : TSD;
-   --  Note: we rely on the default initialization of NT_TSD
-
-   --  Needed for Vx6Cert (Vx653mc) GOS cert and ravenscar-cert runtimes,
-   --  VxMILS cert, ravenscar-cert and full runtimes, Vx 5 default runtime
    Stack_Limit : aliased System.Address := System.Null_Address;
-
    pragma Export (C, Stack_Limit, "__gnat_stack_limit");
+   --  Needed for Vx6Cert (Vx653mc) GOS cert and ravenscar-cert runtimes,
+   --  VxMILS cert, ravenscar-cert and full runtimes, Vx 5 default runtime
 
    --------------------
    -- Abort_Defer_NT --
@@ -125,14 +119,16 @@ package body System.Soft_Links is
    -- Create_TSD --
    ----------------
 
-   procedure Create_TSD (New_TSD : in out TSD) is
-      use Parameters;
-      SS_Ratio_Dynamic : constant Boolean := Sec_Stack_Percentage = Dynamic;
+   procedure Create_TSD
+     (New_TSD        : in out TSD;
+      Sec_Stack      : SST.SS_Stack_Ptr;
+      Sec_Stack_Size : System.Parameters.Size_Type)
+   is
    begin
-      if SS_Ratio_Dynamic then
-         SST.SS_Init
-           (New_TSD.Sec_Stack_Addr, SST.Default_Secondary_Stack_Size);
-      end if;
+      New_TSD.Jmpbuf_Address := Null_Address;
+
+      New_TSD.Sec_Stack_Ptr := Sec_Stack;
+      SST.SS_Init (New_TSD.Sec_Stack_Ptr, Sec_Stack_Size);
    end Create_TSD;
 
    -----------------------
@@ -150,7 +146,7 @@ package body System.Soft_Links is
 
    procedure Destroy_TSD (Old_TSD : in out TSD) is
    begin
-      SST.SS_Free (Old_TSD.Sec_Stack_Addr);
+      SST.SS_Free (Old_TSD.Sec_Stack_Ptr);
    end Destroy_TSD;
 
    ---------------------
@@ -198,23 +194,23 @@ package body System.Soft_Links is
       return Get_Jmpbuf_Address.all;
    end Get_Jmpbuf_Address_Soft;
 
-   ---------------------------
-   -- Get_Sec_Stack_Addr_NT --
-   ---------------------------
+   ----------------------
+   -- Get_Sec_Stack_NT --
+   ----------------------
 
-   function Get_Sec_Stack_Addr_NT return  Address is
+   function Get_Sec_Stack_NT return SST.SS_Stack_Ptr is
    begin
-      return NT_TSD.Sec_Stack_Addr;
-   end Get_Sec_Stack_Addr_NT;
+      return NT_TSD.Sec_Stack_Ptr;
+   end Get_Sec_Stack_NT;
 
    -----------------------------
-   -- Get_Sec_Stack_Addr_Soft --
+   -- Get_Sec_Stack_Soft --
    -----------------------------
 
-   function Get_Sec_Stack_Addr_Soft return  Address is
+   function Get_Sec_Stack_Soft return SST.SS_Stack_Ptr is
    begin
-      return Get_Sec_Stack_Addr.all;
-   end Get_Sec_Stack_Addr_Soft;
+      return Get_Sec_Stack.all;
+   end Get_Sec_Stack_Soft;
 
    -----------------------
    -- Get_Stack_Info_NT --
@@ -254,23 +250,23 @@ package body System.Soft_Links is
       Set_Jmpbuf_Address (Addr);
    end Set_Jmpbuf_Address_Soft;
 
-   ---------------------------
-   -- Set_Sec_Stack_Addr_NT --
-   ---------------------------
+   ----------------------
+   -- Set_Sec_Stack_NT --
+   ----------------------
 
-   procedure Set_Sec_Stack_Addr_NT (Addr : Address) is
+   procedure Set_Sec_Stack_NT (Stack : SST.SS_Stack_Ptr) is
    begin
-      NT_TSD.Sec_Stack_Addr := Addr;
-   end Set_Sec_Stack_Addr_NT;
+      NT_TSD.Sec_Stack_Ptr := Stack;
+   end Set_Sec_Stack_NT;
 
-   -----------------------------
-   -- Set_Sec_Stack_Addr_Soft --
-   -----------------------------
+   ------------------------
+   -- Set_Sec_Stack_Soft --
+   ------------------------
 
-   procedure Set_Sec_Stack_Addr_Soft (Addr : Address) is
+   procedure Set_Sec_Stack_Soft (Stack : SST.SS_Stack_Ptr) is
    begin
-      Set_Sec_Stack_Addr (Addr);
-   end Set_Sec_Stack_Addr_Soft;
+      Set_Sec_Stack (Stack);
+   end Set_Sec_Stack_Soft;
 
    ------------------
    -- Task_Lock_NT --
@@ -308,5 +304,4 @@ package body System.Soft_Links is
    begin
       null;
    end Task_Unlock_NT;
-
 end System.Soft_Links;
diff --git a/gcc/ada/libgnat/s-soflin.ads b/gcc/ada/libgnat/s-soflin.ads
index 402ea84818b..4242fcee7ee 100644
--- a/gcc/ada/libgnat/s-soflin.ads
+++ b/gcc/ada/libgnat/s-soflin.ads
@@ -40,11 +40,15 @@
 pragma Compiler_Unit_Warning;
 
 with Ada.Exceptions;
+with System.Parameters;
+with System.Secondary_Stack;
 with System.Stack_Checking;
 
 package System.Soft_Links is
    pragma Preelaborate;
 
+   package SST renames System.Secondary_Stack;
+
    subtype EOA is Ada.Exceptions.Exception_Occurrence_Access;
    subtype EO is Ada.Exceptions.Exception_Occurrence;
 
@@ -89,6 +93,11 @@ package System.Soft_Links is
    type Set_EO_Call       is access procedure (Excep : EO);
    pragma Favor_Top_Level (Set_EO_Call);
 
+   type Get_Stack_Call    is access function return SST.SS_Stack_Ptr;
+   pragma Favor_Top_Level (Get_Stack_Call);
+   type Set_Stack_Call    is access procedure (Stack : SST.SS_Stack_Ptr);
+   pragma Favor_Top_Level (Set_Stack_Call);
+
    type Special_EO_Call   is access
      procedure (Excep : EO := Current_Target_Exception);
    pragma Favor_Top_Level (Special_EO_Call);
@@ -118,6 +127,8 @@ package System.Soft_Links is
    pragma Suppress (Access_Check, Set_Integer_Call);
    pragma Suppress (Access_Check, Get_EOA_Call);
    pragma Suppress (Access_Check, Set_EOA_Call);
+   pragma Suppress (Access_Check, Get_Stack_Call);
+   pragma Suppress (Access_Check, Set_Stack_Call);
    pragma Suppress (Access_Check, Timed_Delay_Call);
    pragma Suppress (Access_Check, Get_Stack_Access_Call);
    pragma Suppress (Access_Check, Task_Name_Call);
@@ -228,11 +239,11 @@ package System.Soft_Links is
    Get_Jmpbuf_Address : Get_Address_Call := Get_Jmpbuf_Address_NT'Access;
    Set_Jmpbuf_Address : Set_Address_Call := Set_Jmpbuf_Address_NT'Access;
 
-   function  Get_Sec_Stack_Addr_NT return  Address;
-   procedure Set_Sec_Stack_Addr_NT (Addr : Address);
+   function  Get_Sec_Stack_NT return  SST.SS_Stack_Ptr;
+   procedure Set_Sec_Stack_NT (Stack : SST.SS_Stack_Ptr);
 
-   Get_Sec_Stack_Addr : Get_Address_Call := Get_Sec_Stack_Addr_NT'Access;
-   Set_Sec_Stack_Addr : Set_Address_Call := Set_Sec_Stack_Addr_NT'Access;
+   Get_Sec_Stack : Get_Stack_Call := Get_Sec_Stack_NT'Access;
+   Set_Sec_Stack : Set_Stack_Call := Set_Sec_Stack_NT'Access;
 
    function Get_Current_Excep_NT return EOA;
 
@@ -320,19 +331,14 @@ package System.Soft_Links is
       --  must be initialized to the tasks requested stack size before the task
       --  can do its first stack check.
 
-      pragma Warnings (Off);
-      --  Needed because we are giving a non-static default to an object in
-      --  a preelaborated unit, which is formally not permitted, but OK here.
-
-      Jmpbuf_Address : System.Address := System.Null_Address;
+      Jmpbuf_Address : System.Address;
       --  Address of jump buffer used to store the address of the current
       --  longjmp/setjmp buffer for exception management. These buffers are
       --  threaded into a stack, and the address here is the top of the stack.
       --  A null address means that no exception handler is currently active.
 
-      Sec_Stack_Addr : System.Address := System.Null_Address;
-      pragma Warnings (On);
-      --  Address of currently allocated secondary stack
+      Sec_Stack_Ptr : SST.SS_Stack_Ptr;
+      --  Pointer of the allocated secondary stack
 
       Current_Excep : aliased EO;
       --  Exception occurrence that contains the information for the current
@@ -344,7 +350,10 @@ package System.Soft_Links is
       --  exception mechanism, organized as a stack with the most recent first.
    end record;
 
-   procedure Create_TSD (New_TSD : in out TSD);
+   procedure Create_TSD
+     (New_TSD        : in out TSD;
+      Sec_Stack      : SST.SS_Stack_Ptr;
+      Sec_Stack_Size : System.Parameters.Size_Type);
    pragma Inline (Create_TSD);
    --  Called from s-tassta when a new thread is created to perform
    --  any required initialization of the TSD.
@@ -370,10 +379,10 @@ package System.Soft_Links is
    pragma Inline (Get_Jmpbuf_Address_Soft);
    pragma Inline (Set_Jmpbuf_Address_Soft);
 
-   function  Get_Sec_Stack_Addr_Soft return  Address;
-   procedure Set_Sec_Stack_Addr_Soft (Addr : Address);
-   pragma Inline (Get_Sec_Stack_Addr_Soft);
-   pragma Inline (Set_Sec_Stack_Addr_Soft);
+   function  Get_Sec_Stack_Soft return  SST.SS_Stack_Ptr;
+   procedure Set_Sec_Stack_Soft (Stack : SST.SS_Stack_Ptr);
+   pragma Inline (Get_Sec_Stack_Soft);
+   pragma Inline (Set_Sec_Stack_Soft);
 
    --  The following is a dummy record designed to mimic Communication_Block as
    --  defined in s-tpobop.ads:
@@ -396,4 +405,11 @@ package System.Soft_Links is
       Comp_3 : Boolean;
    end record;
 
+private
+   NT_TSD : TSD;
+   --  The task specific data for the main task when the Ada tasking run-time
+   --  is not used. It relies on the default initialization of NT_TSD. It is
+   --  placed here and not the body to ensure the default initialization does
+   --  not clobber the secondary stack initialization that occurs as part of
+   --  System.Soft_Links.Initialization.
 end System.Soft_Links;
diff --git a/gcc/ada/libgnat/s-soliin.adb b/gcc/ada/libgnat/s-soliin.adb
new file mode 100644
index 00000000000..5364e46f6f4
--- /dev/null
+++ b/gcc/ada/libgnat/s-soliin.adb
@@ -0,0 +1,47 @@
+------------------------------------------------------------------------------
+--                                                                          --
+--                         GNAT COMPILER COMPONENTS                         --
+--                                                                          --
+--          S Y S T E M . S O F T _ L I N K S . I N I T I A L I Z E         --
+--                                                                          --
+--                                 B o d y                                  --
+--                                                                          --
+--            Copyright (C) 2017, Free Software Foundation, Inc.            --
+--                                                                          --
+-- GNAT is free software;  you can  redistribute it  and/or modify it under --
+-- terms of the  GNU General Public License as published  by the Free Soft- --
+-- ware  Foundation;  either version 3,  or (at your option) any later ver- --
+-- sion.  GNAT is distributed in the hope that it will be useful, but WITH- --
+-- OUT ANY WARRANTY;  without even the  implied warranty of MERCHANTABILITY --
+-- or FITNESS FOR A PARTICULAR PURPOSE.                                     --
+--                                                                          --
+-- As a special exception under Section 7 of GPL version 3, you are granted --
+-- additional permissions described in the GCC Runtime Library Exception,   --
+-- version 3.1, as published by the Free Software Foundation.               --
+--                                                                          --
+-- You should have received a copy of the GNU General Public License and    --
+-- a copy of the GCC Runtime Library Exception along with this program;     --
+-- see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see    --
+-- <http://www.gnu.org/licenses/>.                                          --
+--                                                                          --
+-- GNAT was originally developed  by the GNAT team at  New York University. --
+-- Extensive contributions were provided by Ada Core Technologies Inc.      --
+--                                                                          --
+------------------------------------------------------------------------------
+
+with System.Secondary_Stack;
+
+package body System.Soft_Links.Initialize is
+
+   package SSS renames System.Secondary_Stack;
+
+begin
+   --  Initialize the TSD of the main task
+
+   NT_TSD.Jmpbuf_Address := System.Null_Address;
+
+   --  Allocate and initialize the secondary stack for the main task
+
+   NT_TSD.Sec_Stack_Ptr := null;
+   SSS.SS_Init (NT_TSD.Sec_Stack_Ptr);
+end System.Soft_Links.Initialize;
diff --git a/gcc/ada/libgnat/s-soliin.ads b/gcc/ada/libgnat/s-soliin.ads
new file mode 100644
index 00000000000..ba9cf745f48
--- /dev/null
+++ b/gcc/ada/libgnat/s-soliin.ads
@@ -0,0 +1,48 @@
+------------------------------------------------------------------------------
+--                                                                          --
+--                         GNAT COMPILER COMPONENTS                         --
+--                                                                          --
+--          S Y S T E M . S O F T _ L I N K S . I N I T I A L I Z E         --
+--                                                                          --
+--                                 S p e c                                  --
+--                                                                          --
+--            Copyright (C) 2017, Free Software Foundation, Inc.            --
+--                                                                          --
+-- GNAT is free software;  you can  redistribute it  and/or modify it under --
+-- terms of the  GNU General Public License as published  by the Free Soft- --
+-- ware  Foundation;  either version 3,  or (at your option) any later ver- --
+-- sion.  GNAT is distributed in the hope that it will be useful, but WITH- --
+-- OUT ANY WARRANTY;  without even the  implied warranty of MERCHANTABILITY --
+-- or FITNESS FOR A PARTICULAR PURPOSE.                                     --
+--                                                                          --
+-- As a special exception under Section 7 of GPL version 3, you are granted --
+-- additional permissions described in the GCC Runtime Library Exception,   --
+-- version 3.1, as published by the Free Software Foundation.               --
+--                                                                          --
+-- You should have received a copy of the GNU General Public License and    --
+-- a copy of the GCC Runtime Library Exception along with this program;     --
+-- see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see    --
+-- <http://www.gnu.org/licenses/>.                                          --
+--                                                                          --
+-- GNAT was originally developed  by the GNAT team at  New York University. --
+-- Extensive contributions were provided by Ada Core Technologies Inc.      --
+--                                                                          --
+------------------------------------------------------------------------------
+
+--  This package exists to initialize the TSD record of the main task and in
+--  the process, allocate and initialize the secondary stack for the main task.
+--  The initialization routine is contained within its own package because
+--  System.Soft_Links and System.Secondary_Stack are both Preelaborate packages
+--  that are the parents to other Preelaborate System packages.
+
+--  Ideally, the secondary stack would be set up via __gnat_runtime_initialize
+--  to have the secondary stack active as early as possible and to remove the
+--  awkwardness of System.Soft_Links depending on a non-Preelaborate package.
+--  However, as this procedure only exists from 2014, for bootstrapping
+--  purposes the elaboration mechanism is used instead to perform these
+--  functions.
+
+package System.Soft_Links.Initialize is
+   pragma Elaborate_Body;
+   --  Allow this package to have a body
+end System.Soft_Links.Initialize;
diff --git a/gcc/ada/libgnat/s-thread.ads b/gcc/ada/libgnat/s-thread.ads
index cd4faaec1ed..185141b1f1b 100644
--- a/gcc/ada/libgnat/s-thread.ads
+++ b/gcc/ada/libgnat/s-thread.ads
@@ -42,10 +42,13 @@ with Ada.Unchecked_Conversion;
 
 with Interfaces.C;
 
+with System.Secondary_Stack;
 with System.Soft_Links;
 
 package System.Threads is
 
+   package SST renames System.Secondary_Stack;
+
    type ATSD is limited private;
    --  Type of the Ada thread specific data. It contains datas needed
    --  by the GNAT runtime.
@@ -71,8 +74,7 @@ package System.Threads is
    --  wrapper in the APEX process registration package.
 
    procedure Thread_Body_Enter
-     (Sec_Stack_Address    : System.Address;
-      Sec_Stack_Size       : Natural;
+     (Sec_Stack_Ptr        : SST.SS_Stack_Ptr;
       Process_ATSD_Address : System.Address);
    --  Enter thread body, see above for details
 
diff --git a/gcc/ada/libgnat/s-thread__ae653.adb b/gcc/ada/libgnat/s-thread__ae653.adb
index ca871286fce..9e8b2abb946 100644
--- a/gcc/ada/libgnat/s-thread__ae653.adb
+++ b/gcc/ada/libgnat/s-thread__ae653.adb
@@ -37,15 +37,11 @@ pragma Restrictions (No_Tasking);
 --  will be checked by the binder.
 
 with System.OS_Versions; use System.OS_Versions;
-with System.Secondary_Stack;
-pragma Elaborate_All (System.Secondary_Stack);
 
 package body System.Threads is
 
    use Interfaces.C;
 
-   package SSS renames System.Secondary_Stack;
-
    package SSL renames System.Soft_Links;
 
    Current_ATSD : aliased System.Address := System.Null_Address;
@@ -94,17 +90,16 @@ package body System.Threads is
    procedure Install_Handler;
    pragma Import (C, Install_Handler, "__gnat_install_handler");
 
-   function  Get_Sec_Stack_Addr return  Address;
+   function  Get_Sec_Stack return SST.SS_Stack_Ptr;
 
-   procedure Set_Sec_Stack_Addr (Addr : Address);
+   procedure Set_Sec_Stack (Stack : SST.SS_Stack_Ptr);
 
    -----------------------
    -- Thread_Body_Enter --
    -----------------------
 
    procedure Thread_Body_Enter
-     (Sec_Stack_Address    : System.Address;
-      Sec_Stack_Size       : Natural;
+     (Sec_Stack_Ptr        : SST.SS_Stack_Ptr;
       Process_ATSD_Address : System.Address)
    is
       --  Current_ATSD must already be a taskVar of taskIdSelf.
@@ -115,8 +110,8 @@ package body System.Threads is
 
    begin
 
-      TSD.Sec_Stack_Addr := Sec_Stack_Address;
-      SSS.SS_Init (TSD.Sec_Stack_Addr, Sec_Stack_Size);
+      TSD.Sec_Stack_Ptr := Sec_Stack_Ptr;
+      SST.SS_Init (TSD.Sec_Stack_Ptr);
       Current_ATSD := Process_ATSD_Address;
 
       Install_Handler;
@@ -166,23 +161,23 @@ package body System.Threads is
       pragma Assert (Result /= ERROR);
 
    begin
-      Main_ATSD.Sec_Stack_Addr := SSL.Get_Sec_Stack_Addr_NT;
+      Main_ATSD.Sec_Stack_Ptr := SSL.Get_Sec_Stack_NT;
       Current_ATSD := Main_ATSD'Address;
       Install_Handler;
-      SSL.Get_Sec_Stack_Addr := Get_Sec_Stack_Addr'Access;
-      SSL.Set_Sec_Stack_Addr := Set_Sec_Stack_Addr'Access;
+      SSL.Get_Sec_Stack := Get_Sec_Stack'Access;
+      SSL.Set_Sec_Stack := Set_Sec_Stack'Access;
    end Init_RTS;
 
-   ------------------------
-   -- Get_Sec_Stack_Addr --
-   ------------------------
+   -------------------
+   -- Get_Sec_Stack --
+   -------------------
 
-   function  Get_Sec_Stack_Addr return  Address is
+   function  Get_Sec_Stack return SST.SS_Stack_Ptr is
       CTSD : constant ATSD_Access := From_Address (Current_ATSD);
    begin
       pragma Assert (CTSD /= null);
-      return CTSD.Sec_Stack_Addr;
-   end Get_Sec_Stack_Addr;
+      return CTSD.Sec_Stack_Ptr;
+   end Get_Sec_Stack;
 
    --------------
    -- Register --
@@ -229,16 +224,16 @@ package body System.Threads is
       return Result;
    end Register;
 
-   ------------------------
-   -- Set_Sec_Stack_Addr --
-   ------------------------
+   -------------------
+   -- Set_Sec_Stack --
+   -------------------
 
-   procedure Set_Sec_Stack_Addr (Addr : Address) is
+   procedure Set_Sec_Stack (Stack : SST.SS_Stack_Ptr) is
       CTSD : constant ATSD_Access := From_Address (Current_ATSD);
    begin
       pragma Assert (CTSD /= null);
-      CTSD.Sec_Stack_Addr := Addr;
-   end Set_Sec_Stack_Addr;
+      CTSD.Sec_Stack_Ptr := Stack;
+   end Set_Sec_Stack;
 
 begin
    --  Initialize run-time library
diff --git a/gcc/ada/opt.ads b/gcc/ada/opt.ads
index 687d1eb75b9..96e2f3e2f92 100644
--- a/gcc/ada/opt.ads
+++ b/gcc/ada/opt.ads
@@ -462,18 +462,21 @@ package Opt is
    --    otherwise:   "pragma Default_Storage_Pool (X);" applies, and
    --                 this points to the name X.
    --  Push_Scope and Pop_Scope in Sem_Ch8 save and restore this value.
-   Default_Stack_Size : Int := -1;
+
+   No_Stack_Size : constant := -1;
+
+   Default_Stack_Size : Int := No_Stack_Size;
    --  GNATBIND
-   --  Set to default primary stack size in units of bytes. Set by
-   --  the -dnnn switch for the binder. A value of -1 indicates that no
-   --  default was set by the binder.
+   --  Set to default primary stack size in units of bytes. Set by the -dnnn
+   --  switch for the binder. A value of No_Stack_Size indicates that
+   --  no default was set by the binder.
 
-   Default_Sec_Stack_Size : Int := -1;
+   Default_Sec_Stack_Size : Int := No_Stack_Size;
    --  GNATBIND
-   --  Set to default secondary stack size in units of bytes. Set by
-   --  the -Dnnn switch for the binder. A value of -1 indicates that no
-   --  default was set by the binder, and that the default should be the
-   --  initial value of System.Secondary_Stack.Default_Secondary_Stack_Size.
+   --  Set to default secondary stack size in units of bytes. Set by the -Dnnn
+   --  switch for the binder. A value of No_Stack_Size indicates that no
+   --  default was set by the binder and the run-time value should be used
+   --  instead.
 
    Default_SSO : Character := ' ';
    --  GNAT
@@ -1313,6 +1316,13 @@ package Opt is
    --  Indicates if a project file is used or not. Set to In_Use by the first
    --  SFNP pragma.
 
+   Quantity_Of_Default_Size_Sec_Stacks : Int := -1;
+   --  GNATBIND
+   --  The number of default sized secondary stacks that the binder should
+   --  generate. Allows ZFP users to have the binder generate extra stacks if
+   --  needed to support multithreaded applications. A value of -1 indicates
+   --  that no size was set by the binder.
+
    Queuing_Policy : Character := ' ';
    --  GNAT, GNATBIND
    --  Set to ' ' for the default case (no queuing policy specified). Reset to
diff --git a/gcc/ada/rtfinal.c b/gcc/ada/rtfinal.c
index 8f7e163cded..9398af393ba 100644
--- a/gcc/ada/rtfinal.c
+++ b/gcc/ada/rtfinal.c
@@ -6,7 +6,7 @@
  *                                                                          *
  *                          C Implementation File                           *
  *                                                                          *
- *             Copyright (C) 2014, Free Software Foundation, Inc.           *
+ *            Copyright (C) 2014-2017, Free Software Foundation, Inc.       *
  *                                                                          *
  * GNAT is free software;  you can  redistribute it  and/or modify it under *
  * terms of the  GNU General Public License as published  by the Free Soft- *
@@ -40,7 +40,7 @@ extern void __gnat_runtime_finalize (void);
    at all, the intention is that this be replaced by system specific code
    where finalization is required.
 
-   Note that __gnat_runtime_initialize() is called in adafinal()   */
+   Note that __gnat_runtime_finalize() is called in adafinal()   */
 
 extern int __gnat_rt_init_count;
 /*  see initialize.c  */
diff --git a/gcc/ada/rtsfind.ads b/gcc/ada/rtsfind.ads
index bdad2520fd4..c4d7d3c80c6 100644
--- a/gcc/ada/rtsfind.ads
+++ b/gcc/ada/rtsfind.ads
@@ -1249,6 +1249,7 @@ package Rtsfind is
      RE_Set_63,                          -- System.Pack_63
 
      RE_Adjust_Storage_Size,             -- System.Parameters
+     RE_Default_Secondary_Stack_Size,    -- System.Parameters
      RE_Default_Stack_Size,              -- System.Parameters
      RE_Garbage_Collected,               -- System.Parameters
      RE_Size_Type,                       -- System.Parameters
@@ -1424,12 +1425,12 @@ package Rtsfind is
      RE_IS_Ilf,                          -- System.Scalar_Values
      RE_IS_Ill,                          -- System.Scalar_Values
 
-     RE_Default_Secondary_Stack_Size,    -- System.Secondary_Stack
      RE_Mark_Id,                         -- System.Secondary_Stack
      RE_SS_Allocate,                     -- System.Secondary_Stack
      RE_SS_Pool,                         -- System.Secondary_Stack
      RE_SS_Mark,                         -- System.Secondary_Stack
      RE_SS_Release,                      -- System.Secondary_Stack
+     RE_SS_Stack,                        -- System.Secondary_Stack
 
      RE_Shared_Var_Lock,                 -- System.Shared_Storage
      RE_Shared_Var_Unlock,               -- System.Shared_Storage
@@ -2487,6 +2488,7 @@ package Rtsfind is
      RE_Set_63                           => System_Pack_63,
 
      RE_Adjust_Storage_Size              => System_Parameters,
+     RE_Default_Secondary_Stack_Size     => System_Parameters,
      RE_Default_Stack_Size               => System_Parameters,
      RE_Garbage_Collected                => System_Parameters,
      RE_Size_Type                        => System_Parameters,
@@ -2662,12 +2664,12 @@ package Rtsfind is
      RE_IS_Ilf                           => System_Scalar_Values,
      RE_IS_Ill                           => System_Scalar_Values,
 
-     RE_Default_Secondary_Stack_Size     => System_Secondary_Stack,
      RE_Mark_Id                          => System_Secondary_Stack,
      RE_SS_Allocate                      => System_Secondary_Stack,
      RE_SS_Mark                          => System_Secondary_Stack,
      RE_SS_Pool                          => System_Secondary_Stack,
      RE_SS_Release                       => System_Secondary_Stack,
+     RE_SS_Stack                         => System_Secondary_Stack,
 
      RE_Shared_Var_Lock                  => System_Shared_Storage,
      RE_Shared_Var_Unlock                => System_Shared_Storage,
diff --git a/gcc/ada/sem_aggr.adb b/gcc/ada/sem_aggr.adb
index 677d59999dd..6c29b38b93a 100644
--- a/gcc/ada/sem_aggr.adb
+++ b/gcc/ada/sem_aggr.adb
@@ -1594,7 +1594,7 @@ package body Sem_Aggr is
             --  unless the expression covers a single component, or the
             --  expander is inactive.
 
-            --  In SPARK mode, expressions that can perform side-effects will
+            --  In SPARK mode, expressions that can perform side effects will
             --  be recognized by the gnat2why back-end, and the whole
             --  subprogram will be ignored. So semantic analysis can be
             --  performed safely.
@@ -3605,7 +3605,7 @@ package body Sem_Aggr is
                      --  This is redundant if the others_choice covers only
                      --  one component (small optimization possible???), but
                      --  indispensable otherwise, because each one must be
-                     --  expanded individually to preserve side-effects.
+                     --  expanded individually to preserve side effects.
 
                      --  Ada 2005 (AI-287): In case of default initialization
                      --  of components, we duplicate the corresponding default
@@ -3881,7 +3881,7 @@ package body Sem_Aggr is
          --  expansion is delayed until the enclosing aggregate is expanded
          --  into assignments. In that case, do not generate checks on the
          --  expression, because they will be generated later, and will other-
-         --  wise force a copy (to remove side-effects) that would leave a
+         --  wise force a copy (to remove side effects) that would leave a
          --  dynamic-sized aggregate in the code, something that gigi cannot
          --  handle.
 
diff --git a/gcc/ada/sem_ch12.adb b/gcc/ada/sem_ch12.adb
index 223703d2a43..ac5035fd1bc 100644
--- a/gcc/ada/sem_ch12.adb
+++ b/gcc/ada/sem_ch12.adb
@@ -5305,8 +5305,7 @@ package body Sem_Ch12 is
             Valid_Operator_Definition (Act_Decl_Id);
          end if;
 
-         Set_Alias  (Act_Decl_Id, Anon_Id);
-         Set_Parent (Act_Decl_Id, Parent (Anon_Id));
+         Set_Alias (Act_Decl_Id, Anon_Id);
          Set_Has_Completion (Act_Decl_Id);
          Set_Related_Instance (Pack_Id, Act_Decl_Id);
 
@@ -6460,10 +6459,11 @@ package body Sem_Ch12 is
          elsif Ekind (E1) = E_Package then
             Check_Mismatch
               (Ekind (E1) /= Ekind (E2)
-                or else Renamed_Object (E1) /= Renamed_Object (E2));
+                or else (Present (Renamed_Object (E2))
+                          and then Renamed_Object (E1) /=
+                                     Renamed_Object (E2)));
 
          elsif Is_Overloadable (E1) then
-
             --  Verify that the actual subprograms match. Note that actuals
             --  that are attributes are rewritten as subprograms. If the
             --  subprogram in the formal package is defaulted, no check is
diff --git a/gcc/ada/sem_ch3.adb b/gcc/ada/sem_ch3.adb
index c163aab8e78..1e3b78ccf2f 100644
--- a/gcc/ada/sem_ch3.adb
+++ b/gcc/ada/sem_ch3.adb
@@ -2820,24 +2820,10 @@ package body Sem_Ch3 is
 
          --  Analyze the contracts of packages and their bodies
 
-         if Nkind (Context) = N_Package_Specification then
-
-            --  When a package has private declarations, its contract must be
-            --  analyzed at the end of the said declarations. This way both the
-            --  analysis and freeze actions are properly synchronized in case
-            --  of private type use within the contract.
-
-            if L = Private_Declarations (Context) then
-               Analyze_Package_Contract (Defining_Entity (Context));
-
-            --  Otherwise the contract is analyzed at the end of the visible
-            --  declarations.
-
-            elsif L = Visible_Declarations (Context)
-              and then No (Private_Declarations (Context))
-            then
-               Analyze_Package_Contract (Defining_Entity (Context));
-            end if;
+         if Nkind (Context) = N_Package_Specification
+           and then L = Visible_Declarations (Context)
+         then
+            Analyze_Package_Contract (Defining_Entity (Context));
 
          elsif Nkind (Context) = N_Package_Body then
             Analyze_Package_Body_Contract (Defining_Entity (Context));
diff --git a/gcc/ada/sem_ch4.adb b/gcc/ada/sem_ch4.adb
index fad52ebd106..538023524e3 100644
--- a/gcc/ada/sem_ch4.adb
+++ b/gcc/ada/sem_ch4.adb
@@ -6431,10 +6431,24 @@ package body Sem_Ch4 is
       Op_Id : Entity_Id;
       N     : Node_Id)
    is
-      Op_Type : constant Entity_Id := Etype (Op_Id);
+      Is_String : constant Boolean := Nkind (L) = N_String_Literal
+                                        or else
+                                      Nkind (R) = N_String_Literal;
+      Op_Type   : constant Entity_Id := Etype (Op_Id);
 
    begin
       if Is_Array_Type (Op_Type)
+
+        --  Small but very effective optimization: if at least one operand is a
+        --  string literal, then the type of the operator must be either array
+        --  of characters or array of strings.
+
+        and then (not Is_String
+                    or else
+                  Is_Character_Type (Component_Type (Op_Type))
+                    or else
+                  Is_String_Type (Component_Type (Op_Type)))
+
         and then not Is_Limited_Type (Op_Type)
 
         and then (Has_Compatible_Type (L, Op_Type)
diff --git a/gcc/ada/sem_ch5.adb b/gcc/ada/sem_ch5.adb
index 8c92669876c..10002ea08c2 100644
--- a/gcc/ada/sem_ch5.adb
+++ b/gcc/ada/sem_ch5.adb
@@ -1090,12 +1090,14 @@ package body Sem_Ch5 is
       --  the context of the assignment statement. Restore the expander mode
       --  now so that assignment statement can be properly expanded.
 
-      if Nkind (N) = N_Assignment_Statement and then Has_Target_Names (N) then
-         Expander_Mode_Restore;
-         Full_Analysis := Save_Full_Analysis;
-      end if;
+      if Nkind (N) = N_Assignment_Statement then
+         if Has_Target_Names (N) then
+            Expander_Mode_Restore;
+            Full_Analysis := Save_Full_Analysis;
+         end if;
 
-      pragma Assert (not Should_Transform_BIP_Assignment (Typ => T1));
+         pragma Assert (not Should_Transform_BIP_Assignment (Typ => T1));
+      end if;
    end Analyze_Assignment;
 
    -----------------------------
diff --git a/gcc/ada/sem_ch6.adb b/gcc/ada/sem_ch6.adb
index a85ca60cd5f..4f719e9b81c 100644
--- a/gcc/ada/sem_ch6.adb
+++ b/gcc/ada/sem_ch6.adb
@@ -442,18 +442,12 @@ package body Sem_Ch6 is
       begin
          --  Preanalyze a duplicate of the expression to have available the
          --  minimum decoration needed to locate referenced unfrozen types
-         --  without adding any decoration to the function expression. This
-         --  preanalysis is performed with errors disabled to avoid reporting
-         --  spurious errors on Ghost entities (since the expression is not
-         --  fully analyzed).
+         --  without adding any decoration to the function expression.
 
          Push_Scope (Def_Id);
          Install_Formals (Def_Id);
-         Ignore_Errors_Enable := Ignore_Errors_Enable + 1;
 
          Preanalyze_Spec_Expression (Dup_Expr, Etype (Def_Id));
-
-         Ignore_Errors_Enable := Ignore_Errors_Enable - 1;
          End_Scope;
 
          --  Restore certain attributes of Def_Id since the preanalysis may
diff --git a/gcc/ada/sem_ch8.adb b/gcc/ada/sem_ch8.adb
index aa53045498b..bdc8aba1e1f 100644
--- a/gcc/ada/sem_ch8.adb
+++ b/gcc/ada/sem_ch8.adb
@@ -3644,19 +3644,16 @@ package body Sem_Ch8 is
       --  and mark any use_package_clauses that affect the visibility of the
       --  implicit generic actual.
 
-      if From_Default (N)
-        and then Is_Generic_Actual_Subprogram (New_S)
-        and then Present (Alias (New_S))
+      if Is_Generic_Actual_Subprogram (New_S)
+        and then (Is_Intrinsic_Subprogram (New_S) or else From_Default (N))
       then
-         Mark_Use_Clauses (Alias (New_S));
+         Mark_Use_Clauses (New_S);
 
-      --  Check intrinsic operators used as generic actuals since they may
-      --  make a use_type_clause effective.
+         --  Handle overloaded subprograms
 
-      elsif Is_Generic_Actual_Subprogram (New_S)
-        and then Is_Intrinsic_Subprogram (New_S)
-      then
-         Mark_Use_Clauses (New_S);
+         if Present (Alias (New_S)) then
+            Mark_Use_Clauses (Alias (New_S));
+         end if;
       end if;
    end Analyze_Subprogram_Renaming;
 
@@ -9078,7 +9075,7 @@ package body Sem_Ch8 is
                   then
                      Error_Msg_Node_1 := Entity (N);
                      Error_Msg_NE
-                       ("use clause for package &? has no effect",
+                       ("use clause for package & has no effect?u?",
                         Curr, Entity (N));
                   end if;
 
@@ -9087,7 +9084,7 @@ package body Sem_Ch8 is
                else
                   Error_Msg_Node_1 := Etype (N);
                   Error_Msg_NE
-                    ("use clause for }? has no effect", Curr, Etype (N));
+                    ("use clause for } has no effect?u?", Curr, Etype (N));
                end if;
             end if;
 
@@ -9111,10 +9108,10 @@ package body Sem_Ch8 is
       --  Deal with use clauses within the context area if the current
       --  scope is a compilation unit.
 
-      if Is_Compilation_Unit (Current_Scope) then
-
-         pragma Assert (Scope_Stack.Last /= Scope_Stack.First);
-
+      if Is_Compilation_Unit (Current_Scope)
+        and then Sloc (Scope_Stack.Table
+                        (Scope_Stack.Last - 1).Entity) = Standard_Location
+      then
          Update_Chain_In_Scope (Scope_Stack.Last - 1);
       end if;
    end Update_Use_Clause_Chain;
diff --git a/gcc/ada/sem_dim.adb b/gcc/ada/sem_dim.adb
index 6330703e071..a271ca55960 100644
--- a/gcc/ada/sem_dim.adb
+++ b/gcc/ada/sem_dim.adb
@@ -518,25 +518,17 @@ package body Sem_Dim is
          Position : Dimension_Position)
       is
       begin
-         --  Integer case
-
-         if Is_Integer_Type (Def_Id) then
-
-            --  Dimension value must be an integer literal
-
-            if Nkind (Expr) = N_Integer_Literal then
-               Dimensions (Position) := +Whole (UI_To_Int (Intval (Expr)));
-            else
-               Error_Msg_N ("integer literal expected", Expr);
-            end if;
+         Dimensions (Position) := Create_Rational_From (Expr, True);
+         Processed (Position) := True;
 
-         --  Float case
+         --  If the dimensioned root type is an integer type, it is not
+         --  particularly useful, and fractional dimensions do not make
+         --  much sense for such types, so previously we used to reject
+         --  dimensions of integer types that were not integer literals.
+         --  However, the manipulation of dimensions does not depend on
+         --  the kind of root type, so we can accept this usage for rare
+         --  cases where dimensions are specified for integer values.
 
-         else
-            Dimensions (Position) := Create_Rational_From (Expr, True);
-         end if;
-
-         Processed (Position) := True;
       end Extract_Power;
 
       ------------------------
@@ -1585,6 +1577,20 @@ package body Sem_Dim is
                   then
                      null;
 
+                  --  Numeric literal case. Issue a warning to indicate the
+                  --  literal is treated as if its dimension matches the type
+                  --  dimension.
+
+                  elsif Nkind_In (Original_Node (L), N_Integer_Literal,
+                                                     N_Real_Literal)
+                  then
+                     Dim_Warning_For_Numeric_Literal (L, Etype (R));
+
+                  elsif Nkind_In (Original_Node (R), N_Integer_Literal,
+                                                     N_Real_Literal)
+                  then
+                     Dim_Warning_For_Numeric_Literal (R, Etype (L));
+
                   else
                      Error_Dim_Msg_For_Binary_Op (N, L, R);
                   end if;
@@ -2732,6 +2738,24 @@ package body Sem_Dim is
 
    procedure Dim_Warning_For_Numeric_Literal (N : Node_Id; Typ : Entity_Id) is
    begin
+      --  Consider the literal zero (integer 0 or real 0.0) to be of any
+      --  dimension.
+
+      case Nkind (Original_Node (N)) is
+         when N_Real_Literal =>
+            if Expr_Value_R (N) = Ureal_0 then
+               return;
+            end if;
+
+         when N_Integer_Literal =>
+            if Expr_Value (N) = Uint_0 then
+               return;
+            end if;
+
+         when others =>
+            null;
+      end case;
+
       --  Initialize name buffer
 
       Name_Len := 0;
diff --git a/gcc/ada/sem_elab.adb b/gcc/ada/sem_elab.adb
index 5ba6938cf97..8dec4280eb3 100644
--- a/gcc/ada/sem_elab.adb
+++ b/gcc/ada/sem_elab.adb
@@ -27,6 +27,7 @@ with Atree;    use Atree;
 with Debug;    use Debug;
 with Einfo;    use Einfo;
 with Errout;   use Errout;
+with Exp_Ch11; use Exp_Ch11;
 with Exp_Tss;  use Exp_Tss;
 with Exp_Util; use Exp_Util;
 with Lib;      use Lib;
@@ -159,7 +160,7 @@ package body Sem_Elab is
    --
    --      -  Instantiations
    --
-   --      -  References to variables
+   --      -  Reads of variables
    --
    --      -  Task activation
    --
@@ -175,7 +176,7 @@ package body Sem_Elab is
    --
    --      - For instantiations, the target is the generic template
    --
-   --      - For references to variables, the target is the variable
+   --      - For reads of variables, the target is the variable
    --
    --      - For task activation, the target is the task body
    --
@@ -292,7 +293,7 @@ package body Sem_Elab is
    --  |       |                                                            |
    --  |       +--> Process_Variable_Assignment                             |
    --  |       |                                                            |
-   --  |       +--> Process_Variable_Reference                              |
+   --  |       +--> Process_Variable_Read                                   |
    --  |                                                                    |
    --  +------------------------- Processing phase -------------------------+
 
@@ -348,7 +349,7 @@ package body Sem_Elab is
    --           ABE mechanism effectively ignores all calls which cause the
    --           elaboration flow to "leave" the instance.
    --
-   --  -gnatd.o conservarive elaboration order for indirect calls
+   --  -gnatd.o conservative elaboration order for indirect calls
    --
    --           The ABE mechanism treats '[Unrestricted_]Access of an entry,
    --           operator, or subprogram as an immediate invocation of the
@@ -361,6 +362,13 @@ package body Sem_Elab is
    --           entries, operators, and subprograms. As a result, the scenarios
    --           are not recorder or processed.
    --
+   --  -gnatd.v enforce SPARK elaboration rules in SPARK code
+   --
+   --           The ABE mechanism applies some of the SPARK elaboration rules
+   --           defined in the SPARK reference manual, chapter 7.7. Note that
+   --           certain rules are always enforced, regardless of whether the
+   --           switch is active.
+   --
    --  -gnatd.y disable implicit pragma Elaborate_All on task bodies
    --
    --           The ABE mechanism does not generate implicit Elaborate_All when
@@ -776,14 +784,6 @@ package body Sem_Elab is
    --  message, otherwise it emits an error. If flag In_SPARK is set, then
    --  string " in SPARK" is added to the end of the message.
 
-   procedure Ensure_Dynamic_Prior_Elaboration
-     (N        : Node_Id;
-      Unit_Id  : Entity_Id;
-      Prag_Nam : Name_Id);
-   --  Guarantee the elaboration of unit Unit_Id with respect to the main unit
-   --  by suggesting the use of Elaborate[_All] with name Prag_Nam. N denotes
-   --  the related scenario.
-
    procedure Ensure_Prior_Elaboration
      (N            : Node_Id;
       Unit_Id      : Entity_Id;
@@ -792,7 +792,15 @@ package body Sem_Elab is
    --  N denotes the related scenario. Flag In_Task_Body should be set when the
    --  need for elaboration is initiated from a task body.
 
-   procedure Ensure_Static_Prior_Elaboration
+   procedure Ensure_Prior_Elaboration_Dynamic
+     (N        : Node_Id;
+      Unit_Id  : Entity_Id;
+      Prag_Nam : Name_Id);
+   --  Guarantee the elaboration of unit Unit_Id with respect to the main unit
+   --  by suggesting the use of Elaborate[_All] with name Prag_Nam. N denotes
+   --  the related scenario.
+
+   procedure Ensure_Prior_Elaboration_Static
      (N        : Node_Id;
       Unit_Id  : Entity_Id;
       Prag_Nam : Name_Id);
@@ -808,6 +816,7 @@ package body Sem_Elab is
      (Call      : Node_Id;
       Target_Id : out Entity_Id;
       Attrs     : out Call_Attributes);
+   pragma Inline (Extract_Call_Attributes);
    --  Obtain attributes Attrs associated with call Call. Target_Id is the
    --  entity of the call target.
 
@@ -828,6 +837,7 @@ package body Sem_Elab is
       Inst_Id  : out Entity_Id;
       Gen_Id   : out Entity_Id;
       Attrs    : out Instantiation_Attributes);
+   pragma Inline (Extract_Instantiation_Attributes);
    --  Obtain attributes Attrs associated with expanded instantiation Exp_Inst.
    --  Inst is the instantiation. Inst_Id is the entity of the instance. Gen_Id
    --  is the entity of the generic unit being instantiated.
@@ -841,13 +851,15 @@ package body Sem_Elab is
    procedure Extract_Task_Attributes
      (Typ   : Entity_Id;
       Attrs : out Task_Attributes);
+   pragma Inline (Extract_Task_Attributes);
    --  Obtain attributes Attrs associated with task type Typ
 
    procedure Extract_Variable_Reference_Attributes
      (Ref    : Node_Id;
       Var_Id : out Entity_Id;
       Attrs  : out Variable_Attributes);
-   --  Obtain attributes Attrs associated with reference Ref which mentions
+   pragma Inline (Extract_Variable_Reference_Attributes);
+   --  Obtain attributes Attrs associated with reference Ref that mentions
    --  variable Var_Id.
 
    function Find_Code_Unit (N : Node_Or_Entity_Id) return Entity_Id;
@@ -872,6 +884,10 @@ package body Sem_Elab is
    --  is obtained by logically unwinding instantiations and subunits when N
    --  resides within one.
 
+   function Find_Unit_Entity (N : Node_Id) return Entity_Id;
+   pragma Inline (Find_Unit_Entity);
+   --  Return the entity of unit N
+
    function First_Formal_Type (Subp_Id : Entity_Id) return Entity_Id;
    pragma Inline (First_Formal_Type);
    --  Return the type of subprogram Subp_Id's first formal parameter. If the
@@ -908,6 +924,7 @@ package body Sem_Elab is
    function In_External_Instance
      (N           : Node_Id;
       Target_Decl : Node_Id) return Boolean;
+   pragma Inline (In_External_Instance);
    --  Determine whether a target desctibed by its declaration Target_Decl
    --  resides in a package instance which is external to scenario N.
 
@@ -931,28 +948,30 @@ package body Sem_Elab is
       In_SPARK  : Boolean);
    --  Output information concerning call Call which invokes target Target_Id.
    --  If flag Info_Msg is set, the routine emits an information message,
-   --  otherwise it emits an error. If flag In_SPARK is set, then string " in
-   --  SPARK" is added to the end of the message.
+   --  otherwise it emits an error. If flag In_SPARK is set, then the string
+   --  " in SPARK" is added to the end of the message.
 
    procedure Info_Instantiation
      (Inst     : Node_Id;
       Gen_Id   : Entity_Id;
       Info_Msg : Boolean;
       In_SPARK : Boolean);
+   pragma Inline (Info_Instantiation);
    --  Output information concerning instantiation Inst which instantiates
    --  generic unit Gen_Id. If flag Info_Msg is set, the routine emits an
    --  information message, otherwise it emits an error. If flag In_SPARK
    --  is set, then string " in SPARK" is added to the end of the message.
 
-   procedure Info_Variable_Reference
+   procedure Info_Variable_Read
      (Ref      : Node_Id;
       Var_Id   : Entity_Id;
       Info_Msg : Boolean;
       In_SPARK : Boolean);
-   --  Output information concerning reference Ref which mentions variable
-   --  Var_Id. If flag Info_Msg is set, the routine emits an information
-   --  message, otherwise it emits an error. If flag In_SPARK is set, then
-   --  string " in SPARK" is added to the end of the message.
+   pragma Inline (Info_Variable_Read);
+   --  Output information concerning reference Ref which reads variable Var_Id.
+   --  If flag Info_Msg is set, the routine emits an information message,
+   --  otherwise it emits an error. If flag In_SPARK is set, then string " in
+   --  SPARK" is added to the end of the message.
 
    function Insertion_Node (N : Node_Id; Ins_Nod : Node_Id) return Node_Id;
    pragma Inline (Insertion_Node);
@@ -1026,6 +1045,7 @@ package body Sem_Elab is
      (N           : Node_Id;
       Target_Decl : Node_Id;
       Target_Body : Node_Id) return Boolean;
+   pragma Inline (Is_Guaranteed_ABE);
    --  Determine whether scenario N with a target described by its initial
    --  declaration Target_Decl and body Target_Decl results in a guaranteed
    --  ABE.
@@ -1035,6 +1055,10 @@ package body Sem_Elab is
    --  Determine whether arbitrary entity Id denotes internally generated
    --  routine Initial_Condition.
 
+   function Is_Initialized (Obj_Decl : Node_Id) return Boolean;
+   pragma Inline (Is_Initialized);
+   --  Determine whether object declaration Obj_Decl is initialized
+
    function Is_Invariant_Proc (Id : Entity_Id) return Boolean;
    pragma Inline (Is_Invariant_Proc);
    --  Determine whether arbitrary entity Id denotes an invariant procedure
@@ -1139,10 +1163,10 @@ package body Sem_Elab is
    --  Determine whether arbitrary node N denotes a suitable assignment for ABE
    --  processing.
 
-   function Is_Suitable_Variable_Reference (N : Node_Id) return Boolean;
-   pragma Inline (Is_Suitable_Variable_Reference);
-   --  Determine whether arbitrary node N is a suitable reference to a variable
-   --  for ABE processing.
+   function Is_Suitable_Variable_Read (N : Node_Id) return Boolean;
+   pragma Inline (Is_Suitable_Variable_Read);
+   --  Determine whether arbitrary node N is a suitable variable read for ABE
+   --  processing.
 
    function Is_Task_Entry (Id : Entity_Id) return Boolean;
    pragma Inline (Is_Task_Entry);
@@ -1234,7 +1258,7 @@ package body Sem_Elab is
       Call_Attrs   : Call_Attributes;
       Target_Id    : Entity_Id;
       In_Task_Body : Boolean);
-   --  Top level dispatcher for processing of calls. Perform ABE checks and
+   --  Top-level dispatcher for processing of calls. Perform ABE checks and
    --  diagnostics for call Call which invokes target Target_Id. Call_Attrs
    --  are the attributes of the call. Flag In_Task_Body should be set when
    --  the processing is initiated from a task body.
@@ -1334,10 +1358,24 @@ package body Sem_Elab is
    --  should be set when the processing is initiated from a task body.
 
    procedure Process_Variable_Assignment (Asmt : Node_Id);
-   --  Perform ABE checks and diagnostics for assignment statement Asmt
-
-   procedure Process_Variable_Reference (Ref : Node_Id);
-   --  Perform ABE checks and diagnostics for variable reference Ref
+   --  Top level dispatcher for processing of variable assignments. Perform ABE
+   --  checks and diagnostics for assignment statement Asmt.
+
+   procedure Process_Variable_Assignment_Ada
+     (Asmt   : Node_Id;
+      Var_Id : Entity_Id);
+   --  Perform ABE checks and diagnostics for assignment statement Asmt that
+   --  updates the value of variable Var_Id using the Ada rules.
+
+   procedure Process_Variable_Assignment_SPARK
+     (Asmt   : Node_Id;
+      Var_Id : Entity_Id);
+   --  Perform ABE checks and diagnostics for assignment statement Asmt that
+   --  updates the value of variable Var_Id using the SPARK rules.
+
+   procedure Process_Variable_Read (Ref : Node_Id);
+   --  Perform ABE checks and diagnostics for reference Ref that reads a
+   --  variable.
 
    procedure Push_Active_Scenario (N : Node_Id);
    pragma Inline (Push_Active_Scenario);
@@ -1359,6 +1397,7 @@ package body Sem_Elab is
    --  should be set when the traversal is initiated from a task body.
 
    procedure Update_Elaboration_Scenario (New_N : Node_Id; Old_N : Node_Id);
+   pragma Inline (Update_Elaboration_Scenario);
    --  Update all relevant internal data structures when scenario Old_N is
    --  transformed into scenario New_N by Atree.Rewrite.
 
@@ -1774,7 +1813,7 @@ package body Sem_Elab is
          --  be on another machine.
 
          if Ekind (Body_Id) = E_Package_Body
-           and then Ekind (Spec_Id) = E_Package
+           and then Ekind_In (Spec_Id, E_Generic_Package, E_Package)
            and then (Is_Remote_Call_Interface (Spec_Id)
                       or else Is_Remote_Types (Spec_Id))
          then
@@ -1870,7 +1909,20 @@ package body Sem_Elab is
          Comp_Unit := Parent (Unit_Declaration_Node (Unit_Id));
       end if;
 
-      if Nkind (Comp_Unit) = N_Subunit then
+      --  Handle the case where a subprogram instantiation which acts as a
+      --  compilation unit is expanded into an anonymous package that wraps
+      --  the instantiated subprogram.
+
+      if Nkind (Comp_Unit) = N_Package_Specification
+        and then Nkind_In (Original_Node (Parent (Comp_Unit)),
+                           N_Function_Instantiation,
+                           N_Procedure_Instantiation)
+      then
+         Comp_Unit := Parent (Parent (Comp_Unit));
+
+      --  Handle the case where the compilation unit is a subunit
+
+      elsif Nkind (Comp_Unit) = N_Subunit then
          Comp_Unit := Parent (Comp_Unit);
       end if;
 
@@ -1939,97 +1991,6 @@ package body Sem_Elab is
       return Elaboration_Context_Index (Key mod Elaboration_Context_Max);
    end Elaboration_Context_Hash;
 
-   --------------------------------------
-   -- Ensure_Dynamic_Prior_Elaboration --
-   --------------------------------------
-
-   procedure Ensure_Dynamic_Prior_Elaboration
-     (N        : Node_Id;
-      Unit_Id  : Entity_Id;
-      Prag_Nam : Name_Id)
-   is
-      procedure Info_Missing_Pragma;
-      pragma Inline (Info_Missing_Pragma);
-      --  Output information concerning missing Elaborate or Elaborate_All
-      --  pragma with name Prag_Nam for scenario N which ensures the prior
-      --  elaboration of Unit_Id.
-
-      -------------------------
-      -- Info_Missing_Pragma --
-      -------------------------
-
-      procedure Info_Missing_Pragma is
-      begin
-         --  Internal units are ignored as they cause unnecessary noise
-
-         if not In_Internal_Unit (Unit_Id) then
-
-            --  The name of the unit subjected to the elaboration pragma is
-            --  fully qualified to improve the clarity of the info message.
-
-            Error_Msg_Name_1     := Prag_Nam;
-            Error_Msg_Qual_Level := Nat'Last;
-
-            Error_Msg_NE ("info: missing pragma % for unit &", N, Unit_Id);
-            Error_Msg_Qual_Level := 0;
-         end if;
-      end Info_Missing_Pragma;
-
-      --  Local variables
-
-      Elab_Attrs : Elaboration_Attributes;
-      Level      : Enclosing_Level_Kind;
-
-   --  Start of processing for Ensure_Dynamic_Prior_Elaboration
-
-   begin
-      Elab_Attrs := Elaboration_Context.Get (Unit_Id);
-
-      --  Nothing to do when the unit is guaranteed prior elaboration by means
-      --  of a source Elaborate[_All] pragma.
-
-      if Present (Elab_Attrs.Source_Pragma) then
-         return;
-      end if;
-
-      --  Output extra information on a missing Elaborate[_All] pragma when
-      --  switch -gnatel (info messages on implicit Elaborate[_All] pragmas
-      --  is in effect.
-
-      if Elab_Info_Messages then
-
-         --  Performance note: parent traversal
-
-         Level := Find_Enclosing_Level (N);
-
-         --  Declaration level scenario
-
-         if (Is_Suitable_Call (N) or else Is_Suitable_Instantiation (N))
-           and then Level = Declaration_Level
-         then
-            null;
-
-         --  Library level scenario
-
-         elsif Level in Library_Level then
-            null;
-
-         --  Instantiation library level scenario
-
-         elsif Level = Instantiation then
-            null;
-
-         --  Otherwise the scenario does not appear at the proper level and
-         --  cannot possibly act as a top level scenario.
-
-         else
-            return;
-         end if;
-
-         Info_Missing_Pragma;
-      end if;
-   end Ensure_Dynamic_Prior_Elaboration;
-
    ------------------------------
    -- Ensure_Prior_Elaboration --
    ------------------------------
@@ -2147,7 +2108,7 @@ package body Sem_Elab is
       --  effect.
 
       elsif Dynamic_Elaboration_Checks then
-         Ensure_Dynamic_Prior_Elaboration
+         Ensure_Prior_Elaboration_Dynamic
            (N        => N,
             Unit_Id  => Unit_Id,
             Prag_Nam => Prag_Nam);
@@ -2158,18 +2119,109 @@ package body Sem_Elab is
       else
          pragma Assert (Static_Elaboration_Checks);
 
-         Ensure_Static_Prior_Elaboration
+         Ensure_Prior_Elaboration_Static
            (N        => N,
             Unit_Id  => Unit_Id,
             Prag_Nam => Prag_Nam);
       end if;
    end Ensure_Prior_Elaboration;
 
+   --------------------------------------
+   -- Ensure_Prior_Elaboration_Dynamic --
+   --------------------------------------
+
+   procedure Ensure_Prior_Elaboration_Dynamic
+     (N        : Node_Id;
+      Unit_Id  : Entity_Id;
+      Prag_Nam : Name_Id)
+   is
+      procedure Info_Missing_Pragma;
+      pragma Inline (Info_Missing_Pragma);
+      --  Output information concerning missing Elaborate or Elaborate_All
+      --  pragma with name Prag_Nam for scenario N, which would ensure the
+      --  prior elaboration of Unit_Id.
+
+      -------------------------
+      -- Info_Missing_Pragma --
+      -------------------------
+
+      procedure Info_Missing_Pragma is
+      begin
+         --  Internal units are ignored as they cause unnecessary noise
+
+         if not In_Internal_Unit (Unit_Id) then
+
+            --  The name of the unit subjected to the elaboration pragma is
+            --  fully qualified to improve the clarity of the info message.
+
+            Error_Msg_Name_1     := Prag_Nam;
+            Error_Msg_Qual_Level := Nat'Last;
+
+            Error_Msg_NE ("info: missing pragma % for unit &", N, Unit_Id);
+            Error_Msg_Qual_Level := 0;
+         end if;
+      end Info_Missing_Pragma;
+
+      --  Local variables
+
+      Elab_Attrs : Elaboration_Attributes;
+      Level      : Enclosing_Level_Kind;
+
+   --  Start of processing for Ensure_Prior_Elaboration_Dynamic
+
+   begin
+      Elab_Attrs := Elaboration_Context.Get (Unit_Id);
+
+      --  Nothing to do when the unit is guaranteed prior elaboration by means
+      --  of a source Elaborate[_All] pragma.
+
+      if Present (Elab_Attrs.Source_Pragma) then
+         return;
+      end if;
+
+      --  Output extra information on a missing Elaborate[_All] pragma when
+      --  switch -gnatel (info messages on implicit Elaborate[_All] pragmas
+      --  is in effect.
+
+      if Elab_Info_Messages then
+
+         --  Performance note: parent traversal
+
+         Level := Find_Enclosing_Level (N);
+
+         --  Declaration-level scenario
+
+         if (Is_Suitable_Call (N) or else Is_Suitable_Instantiation (N))
+           and then Level = Declaration_Level
+         then
+            null;
+
+         --  Library-level scenario
+
+         elsif Level in Library_Level then
+            null;
+
+         --  Instantiation library-level scenario
+
+         elsif Level = Instantiation then
+            null;
+
+         --  Otherwise the scenario does not appear at the proper level and
+         --  cannot possibly act as a top-level scenario.
+
+         else
+            return;
+         end if;
+
+         Info_Missing_Pragma;
+      end if;
+   end Ensure_Prior_Elaboration_Dynamic;
+
    -------------------------------------
-   -- Ensure_Static_Prior_Elaboration --
+   -- Ensure_Prior_Elaboration_Static --
    -------------------------------------
 
-   procedure Ensure_Static_Prior_Elaboration
+   procedure Ensure_Prior_Elaboration_Static
      (N        : Node_Id;
       Unit_Id  : Entity_Id;
       Prag_Nam : Name_Id)
@@ -2177,8 +2229,9 @@ package body Sem_Elab is
       function Find_With_Clause
         (Items     : List_Id;
          Withed_Id : Entity_Id) return Node_Id;
-      --  Find a non-limited with clause in the list of context items Items
-      --  which withs unit Withed_Id. Return Empty if no such clause is found.
+      pragma Inline (Find_With_Clause);
+      --  Find a nonlimited with clause in the list of context items Items
+      --  that withs unit Withed_Id. Return Empty if no such clause is found.
 
       procedure Info_Implicit_Pragma;
       pragma Inline (Info_Implicit_Pragma);
@@ -2253,7 +2306,7 @@ package body Sem_Elab is
       Elab_Attrs : Elaboration_Attributes;
       Items      : List_Id;
 
-   --  Start of processing for Ensure_Static_Prior_Elaboration
+   --  Start of processing for Ensure_Prior_Elaboration_Static
 
    begin
       Elab_Attrs := Elaboration_Context.Get (Unit_Id);
@@ -2347,7 +2400,7 @@ package body Sem_Elab is
       if Elab_Info_Messages then
          Info_Implicit_Pragma;
       end if;
-   end Ensure_Static_Prior_Elaboration;
+   end Ensure_Prior_Elaboration_Static;
 
    -----------------------------
    -- Extract_Assignment_Name --
@@ -2898,10 +2951,8 @@ package body Sem_Elab is
    --------------------
 
    function Find_Code_Unit (N : Node_Or_Entity_Id) return Entity_Id is
-      N_Unit : constant Node_Id := Unit (Cunit (Get_Code_Unit (N)));
-
    begin
-      return Defining_Entity (N_Unit, Concurrent_Subunit => True);
+      return Find_Unit_Entity (Unit (Cunit (Get_Code_Unit (N))));
    end Find_Code_Unit;
 
    ---------------------------
@@ -2921,7 +2972,7 @@ package body Sem_Elab is
          Full_Context : Boolean);
       --  Add unit Unit_Id to the elaboration context. Prag denotes the pragma
       --  which prompted the inclusion of the unit to the elaboration context.
-      --  If flag Full_Context is set, examine the non-limited clauses of unit
+      --  If flag Full_Context is set, examine the nonlimited clauses of unit
       --  Unit_Id and add each withed unit to the context.
 
       procedure Find_Elaboration_Context (Comp_Unit : Node_Id);
@@ -3018,7 +3069,7 @@ package body Sem_Elab is
 
          if Full_Context then
 
-            --  Process all non-limited with clauses found in the context of
+            --  Process all nonlimited with clauses found in the context of
             --  the current unit. Note that limited clauses do not impose an
             --  elaboration order.
 
@@ -3370,12 +3421,47 @@ package body Sem_Elab is
    -------------------
 
    function Find_Top_Unit (N : Node_Or_Entity_Id) return Entity_Id is
-      N_Unit : constant Node_Id := Unit (Cunit (Get_Top_Level_Code_Unit (N)));
-
    begin
-      return Defining_Entity (N_Unit, Concurrent_Subunit => True);
+      return Find_Unit_Entity (Unit (Cunit (Get_Top_Level_Code_Unit (N))));
    end Find_Top_Unit;
 
+   ----------------------
+   -- Find_Unit_Entity --
+   ----------------------
+
+   function Find_Unit_Entity (N : Node_Id) return Entity_Id is
+      Context : constant Node_Id := Parent (N);
+      Orig_N  : constant Node_Id := Original_Node (N);
+
+   begin
+      --  The unit denotes a package body of an instantiation which acts as
+      --  a compilation unit. The proper entity is that of the package spec.
+
+      if Nkind (N) = N_Package_Body
+        and then Nkind (Orig_N) = N_Package_Instantiation
+        and then Nkind (Context) = N_Compilation_Unit
+      then
+         return Corresponding_Spec (N);
+
+      --  The unit denotes an anonymous package created to wrap a subprogram
+      --  instantiation which acts as a compilation unit. The proper entity is
+      --  that of the "related instance".
+
+      elsif Nkind (N) = N_Package_Declaration
+        and then Nkind_In (Orig_N, N_Function_Instantiation,
+                                   N_Procedure_Instantiation)
+        and then Nkind (Context) = N_Compilation_Unit
+      then
+         return
+           Related_Instance (Defining_Entity (N, Concurrent_Subunit => True));
+
+      --  Otherwise the proper entity is the defining entity
+
+      else
+         return Defining_Entity (N, Concurrent_Subunit => True);
+      end if;
+   end Find_Unit_Entity;
+
    -----------------------
    -- First_Formal_Type --
    -----------------------
@@ -4140,11 +4226,11 @@ package body Sem_Elab is
          In_SPARK => In_SPARK);
    end Info_Instantiation;
 
-   -----------------------------
-   -- Info_Variable_Reference --
-   -----------------------------
+   ------------------------
+   -- Info_Variable_Read --
+   ------------------------
 
-   procedure Info_Variable_Reference
+   procedure Info_Variable_Read
      (Ref      : Node_Id;
       Var_Id   : Entity_Id;
       Info_Msg : Boolean;
@@ -4152,12 +4238,12 @@ package body Sem_Elab is
    is
    begin
       Elab_Msg_NE
-        (Msg      => "reference to variable & during elaboration",
+        (Msg      => "read of variable & during elaboration",
          N        => Ref,
          Id       => Var_Id,
          Info_Msg => Info_Msg,
          In_SPARK => In_SPARK);
-   end Info_Variable_Reference;
+   end Info_Variable_Read;
 
    --------------------
    -- Insertion_Node --
@@ -4642,6 +4728,18 @@ package body Sem_Elab is
         Ekind (Id) = E_Procedure and then Is_Initial_Condition_Procedure (Id);
    end Is_Initial_Condition_Proc;
 
+   --------------------
+   -- Is_Initialized --
+   --------------------
+
+   function Is_Initialized (Obj_Decl : Node_Id) return Boolean is
+   begin
+      --  To qualify, the object declaration must have an expression
+
+      return
+        Present (Expression (Obj_Decl)) or else Has_Init_Expression (Obj_Decl);
+   end Is_Initialized;
+
    -----------------------
    -- Is_Invariant_Proc --
    -----------------------
@@ -5102,7 +5200,7 @@ package body Sem_Elab is
           or else Is_Suitable_Call (N)
           or else Is_Suitable_Instantiation (N)
           or else Is_Suitable_Variable_Assignment (N)
-          or else Is_Suitable_Variable_Reference (N);
+          or else Is_Suitable_Variable_Read (N);
    end Is_Suitable_Scenario;
 
    -------------------------------------
@@ -5182,11 +5280,7 @@ package body Sem_Elab is
       --  To qualify, the assignment must meet the following prerequisites:
 
       return
-
-        --  The variable must be a source entity and susceptible to warnings
-
         Comes_From_Source (Var_Id)
-          and then not Has_Warnings_Off (Var_Id)
 
           --  The variable must be declared in the spec of compilation unit U
 
@@ -5196,29 +5290,23 @@ package body Sem_Elab is
 
           and then Find_Enclosing_Level (Var_Decl) = Package_Spec
 
-          --  The variable must lack initialization
-
-          and then not Has_Init_Expression (Var_Decl)
-          and then No (Expression (Var_Decl))
-
           --  The assignment must occur in the body of compilation unit U
 
           and then Nkind (N_Unit) = N_Package_Body
           and then Present (Corresponding_Body (Var_Unit))
-          and then Corresponding_Body (Var_Unit) = N_Unit_Id
-
-          --  The package spec must lack pragma Elaborate_Body
-
-          and then not Has_Pragma_Elaborate_Body (Var_Unit_Id);
+          and then Corresponding_Body (Var_Unit) = N_Unit_Id;
    end Is_Suitable_Variable_Assignment;
 
-   ------------------------------------
-   -- Is_Suitable_Variable_Reference --
-   ------------------------------------
+   -------------------------------
+   -- Is_Suitable_Variable_Read --
+   -------------------------------
 
-   function Is_Suitable_Variable_Reference (N : Node_Id) return Boolean is
+   function Is_Suitable_Variable_Read (N : Node_Id) return Boolean is
       function In_Pragma (Nod : Node_Id) return Boolean;
-      --  Determine whether arbitrary node N appears within a pragma
+      --  Determine whether arbitrary node Nod appears within a pragma
+
+      function Is_Variable_Read (Ref : Node_Id) return Boolean;
+      --  Determine whether variable reference Ref constitutes a read
 
       ---------------
       -- In_Pragma --
@@ -5245,12 +5333,88 @@ package body Sem_Elab is
          return False;
       end In_Pragma;
 
+      ----------------------
+      -- Is_Variable_Read --
+      ----------------------
+
+      function Is_Variable_Read (Ref : Node_Id) return Boolean is
+         function Is_Out_Actual (Call : Node_Id) return Boolean;
+         --  Determine whether the corresponding formal of actual Ref which
+         --  appears in call Call has mode OUT.
+
+         -------------------
+         -- Is_Out_Actual --
+         -------------------
+
+         function Is_Out_Actual (Call : Node_Id) return Boolean is
+            Actual     : Node_Id;
+            Call_Attrs : Call_Attributes;
+            Formal     : Entity_Id;
+            Target_Id  : Entity_Id;
+
+         begin
+            Extract_Call_Attributes
+              (Call      => Call,
+               Target_Id => Target_Id,
+               Attrs     => Call_Attrs);
+
+            --  Inspect the actual and formal parameters, trying to find the
+            --  corresponding formal for Ref.
+
+            Actual := First_Actual (Call);
+            Formal := First_Formal (Target_Id);
+            while Present (Actual) and then Present (Formal) loop
+               if Actual = Ref then
+                  return Ekind (Formal) = E_Out_Parameter;
+               end if;
+
+               Next_Actual (Actual);
+               Next_Formal (Formal);
+            end loop;
+
+            return False;
+         end Is_Out_Actual;
+
+         --  Local variables
+
+         Context : constant Node_Id := Parent (Ref);
+
+      --  Start of processing for Is_Variable_Read
+
+      begin
+         --  The majority of variable references are reads, and they can appear
+         --  in a great number of contexts. To determine whether a reference is
+         --  a read, it is more practical to find out whether it is a write.
+
+         --  A reference is a write when it appears immediately on the left-
+         --  hand side of an assignment.
+
+         if Nkind (Context) = N_Assignment_Statement
+           and then Name (Context) = Ref
+         then
+            return False;
+
+         --  A reference is a write when it acts as an actual in a subprogram
+         --  call and the corresponding formal has mode OUT.
+
+         elsif Nkind_In (Context, N_Function_Call,
+                                  N_Procedure_Call_Statement)
+           and then Is_Out_Actual (Context)
+         then
+            return False;
+         end if;
+
+         --  Any other reference is a read
+
+         return True;
+      end Is_Variable_Read;
+
       --  Local variables
 
       Prag   : Node_Id;
       Var_Id : Entity_Id;
 
-   --  Start of processing for Is_Suitable_Variable_Reference
+   --  Start of processing for Is_Suitable_Variable_Read
 
    begin
       --  This scenario is relevant only when the static model is in effect
@@ -5262,8 +5426,7 @@ package body Sem_Elab is
          return False;
 
       --  Attributes and operator sumbols are not considered to be suitable
-      --  references to variables even though they are part of predicate
-      --  Is_Entity_Name.
+      --  references even though they are part of predicate Is_Entity_Name.
 
       elsif not Nkind_In (N, N_Expanded_Name, N_Identifier) then
          return False;
@@ -5303,6 +5466,10 @@ package body Sem_Elab is
           and then Get_SPARK_Mode_From_Annotation (Prag) = On
           and then Is_SPARK_Mode_On_Node (N)
 
+          --  The reference must denote a variable read
+
+          and then Is_Variable_Read (N)
+
           --  The reference must not be considered when it appears in a pragma.
           --  If the pragma has run-time semantics, then the reference will be
           --  reconsidered once the pragma is expanded.
@@ -5310,7 +5477,7 @@ package body Sem_Elab is
           --  Performance note: parent traversal
 
           and then not In_Pragma (N);
-   end Is_Suitable_Variable_Reference;
+   end Is_Suitable_Variable_Read;
 
    -------------------
    -- Is_Task_Entry --
@@ -5485,8 +5652,8 @@ package body Sem_Elab is
                Info_Msg => False,
                In_SPARK => True);
 
-         elsif Is_Suitable_Variable_Reference (N) then
-            Info_Variable_Reference
+         elsif Is_Suitable_Variable_Read (N) then
+            Info_Variable_Read
               (Ref      => N,
                Var_Id   => Target_Id,
                Info_Msg => False,
@@ -5650,8 +5817,9 @@ package body Sem_Elab is
       procedure Output_Variable_Assignment (N : Node_Id);
       --  Emit a specific diagnostic message for assignment statement N
 
-      procedure Output_Variable_Reference (N : Node_Id);
-      --  Emit a specific diagnostic message for variable reference N
+      procedure Output_Variable_Read (N : Node_Id);
+      --  Emit a specific diagnostic message for reference N which reads a
+      --  variable.
 
       -------------------
       -- Output_Access --
@@ -5980,11 +6148,11 @@ package body Sem_Elab is
          Error_Msg_NE ("\\  variable & assigned #", Error_Nod, Var_Id);
       end Output_Variable_Assignment;
 
-      -------------------------------
-      -- Output_Variable_Reference --
-      -------------------------------
+      --------------------------
+      -- Output_Variable_Read --
+      --------------------------
 
-      procedure Output_Variable_Reference (N : Node_Id) is
+      procedure Output_Variable_Read (N : Node_Id) is
          Dummy  : Variable_Attributes;
          Var_Id : Entity_Id;
 
@@ -5995,8 +6163,8 @@ package body Sem_Elab is
             Attrs  => Dummy);
 
          Error_Msg_Sloc := Sloc (N);
-         Error_Msg_NE ("\\  variable & referenced #", Error_Nod, Var_Id);
-      end Output_Variable_Reference;
+         Error_Msg_NE ("\\  variable & read #", Error_Nod, Var_Id);
+      end Output_Variable_Read;
 
       --  Local variables
 
@@ -6057,10 +6225,10 @@ package body Sem_Elab is
          elsif Nkind (N) = N_Assignment_Statement then
             Output_Variable_Assignment (N);
 
-         --  Variable references
+         --  Variable read
 
-         elsif Is_Suitable_Variable_Reference (N) then
-            Output_Variable_Reference (N);
+         elsif Is_Suitable_Variable_Read (N) then
+            Output_Variable_Read (N);
 
          else
             pragma Assert (False);
@@ -6166,7 +6334,7 @@ package body Sem_Elab is
       end if;
 
       --  Treat the attribute as an immediate invocation of the target when
-      --  switch -gnatd.o (conservarive elaboration order for indirect calls)
+      --  switch -gnatd.o (conservative elaboration order for indirect calls)
       --  is in effect. Note that the prior elaboration of the unit containing
       --  the target is ensured processing the corresponding call marker.
 
@@ -6781,16 +6949,18 @@ package body Sem_Elab is
       elsif Is_Up_Level_Target (Target_Attrs.Spec_Decl) then
          return;
 
-      --  The SPARK rules are in effect
+      --  The SPARK rules are verified only when -gnatd.v (enforce SPARK
+      --  elaboration rules in SPARK code) is in effect.
 
-      elsif SPARK_Rules_On then
+      elsif SPARK_Rules_On and Debug_Flag_Dot_V then
          Process_Call_SPARK
            (Call         => Call,
             Call_Attrs   => Call_Attrs,
             Target_Id    => Target_Id,
             Target_Attrs => Target_Attrs);
 
-      --  Otherwise the Ada rules are in effect
+      --  Otherwise the Ada rules are in effect, or SPARK code is allowed to
+      --  violate the SPARK rules.
 
       else
          Process_Call_Ada
@@ -7349,9 +7519,10 @@ package body Sem_Elab is
       elsif Is_Up_Level_Target (Gen_Attrs.Spec_Decl) then
          return;
 
-      --  The SPARK rules are in effect
+      --  The SPARK rules are verified only when -gnatd.v (enforce SPARK
+      --  elaboration rules in SPARK code) is in effect.
 
-      elsif SPARK_Rules_On then
+      elsif SPARK_Rules_On and Debug_Flag_Dot_V then
          Process_Instantiation_SPARK
            (Exp_Inst   => Exp_Inst,
             Inst       => Inst,
@@ -7359,7 +7530,8 @@ package body Sem_Elab is
             Gen_Id     => Gen_Id,
             Gen_Attrs  => Gen_Attrs);
 
-      --  Otherwise the Ada rules are in effect
+      --  Otherwise the Ada rules are in effect, or SPARK code is allowed to
+      --  violate the SPARK rules.
 
       else
          Process_Instantiation_Ada
@@ -7675,9 +7847,9 @@ package body Sem_Elab is
       --  ABE ramifications of the instantiation.
 
       if Nkind (Inst) = N_Package_Instantiation then
-         Req_Nam := Name_Elaborate;
-      else
          Req_Nam := Name_Elaborate_All;
+      else
+         Req_Nam := Name_Elaborate;
       end if;
 
       Meet_Elaboration_Requirement
@@ -7732,31 +7904,76 @@ package body Sem_Elab is
    ---------------------------------
 
    procedure Process_Variable_Assignment (Asmt : Node_Id) is
-      Var_Id  : constant Entity_Id := Entity (Extract_Assignment_Name (Asmt));
-      Spec_Id : Entity_Id;
+      Var_Id : constant Entity_Id := Entity (Extract_Assignment_Name (Asmt));
+      Prag   : constant Node_Id   := SPARK_Pragma (Var_Id);
+
+      SPARK_Rules_On : Boolean;
+      --  This flag is set when the SPARK rules are in effect
 
    begin
+      --  The SPARK rules are in effect when both the assignment and the
+      --  variable are subject to SPARK_Mode On.
+
+      SPARK_Rules_On :=
+        Present (Prag)
+          and then Get_SPARK_Mode_From_Annotation (Prag) = On
+          and then Is_SPARK_Mode_On_Node (Asmt);
+
       --  Output relevant information when switch -gnatel (info messages on
       --  implicit Elaborate[_All] pragmas) is in effect.
 
       if Elab_Info_Messages then
-         Error_Msg_NE
-           ("info: assignment to & during elaboration", Asmt, Var_Id);
+         Elab_Msg_NE
+           (Msg      => "assignment to & during elaboration",
+            N        => Asmt,
+            Id       => Var_Id,
+            Info_Msg => True,
+            In_SPARK => SPARK_Rules_On);
       end if;
 
-      Spec_Id := Find_Top_Unit (Var_Id);
+      --  The SPARK rules are in effect. These rules are applied regardless of
+      --  whether -gnatd.v (enforce SPARK elaboration rules in SPARK code) is
+      --  in effect because the static model cannot ensure safe assignment of
+      --  variables.
 
-      --  Generate an implicit Elaborate_Body in the spec
+      if SPARK_Rules_On then
+         Process_Variable_Assignment_SPARK
+           (Asmt   => Asmt,
+            Var_Id => Var_Id);
 
-      Set_Elaborate_Body_Desirable (Spec_Id);
+      --  Otherwise the Ada rules are in effect
 
-      --  No warning is emitted for internal uses. This behaviour parallels
-      --  that of the old ABE mechanism.
+      else
+         Process_Variable_Assignment_Ada
+           (Asmt   => Asmt,
+            Var_Id => Var_Id);
+      end if;
+   end Process_Variable_Assignment;
 
-      if GNAT_Mode then
-         null;
+   -------------------------------------
+   -- Process_Variable_Assignment_Ada --
+   -------------------------------------
+
+   procedure Process_Variable_Assignment_Ada
+     (Asmt   : Node_Id;
+      Var_Id : Entity_Id)
+   is
+      Var_Decl : constant Node_Id   := Declaration_Node (Var_Id);
+      Spec_Id  : constant Entity_Id := Find_Top_Unit (Var_Decl);
+
+   begin
+      --  Emit a warning when an uninitialized variable declared in a package
+      --  spec without a pragma Elaborate_Body is initialized by elaboration
+      --  code within the corresponding body.
+
+      if not Warnings_Off (Var_Id)
+        and then not Is_Initialized (Var_Decl)
+        and then not Has_Pragma_Elaborate_Body (Spec_Id)
+      then
+         --  Generate an implicit Elaborate_Body in the spec
+
+         Set_Elaborate_Body_Desirable (Spec_Id);
 
-      else
          Error_Msg_NE
            ("??variable & can be accessed by clients before this "
             & "initialization", Asmt, Var_Id);
@@ -7767,13 +7984,44 @@ package body Sem_Elab is
 
          Output_Active_Scenarios (Asmt);
       end if;
-   end Process_Variable_Assignment;
+   end Process_Variable_Assignment_Ada;
 
-   --------------------------------
-   -- Process_Variable_Reference --
-   --------------------------------
+   ---------------------------------------
+   -- Process_Variable_Assignment_SPARK --
+   ---------------------------------------
+
+   procedure Process_Variable_Assignment_SPARK
+     (Asmt   : Node_Id;
+      Var_Id : Entity_Id)
+   is
+      Var_Decl : constant Node_Id   := Declaration_Node (Var_Id);
+      Spec_Id  : constant Entity_Id := Find_Top_Unit (Var_Decl);
+
+   begin
+      --  Emit an error when an initialized variable declared in a package spec
+      --  without pragma Elaborate_Body is further modified by elaboration code
+      --  within the corresponding body.
+
+      if Is_Initialized (Var_Decl)
+        and then not Has_Pragma_Elaborate_Body (Spec_Id)
+      then
+         Error_Msg_NE
+           ("variable & modified by elaboration code in package body",
+            Asmt, Var_Id);
 
-   procedure Process_Variable_Reference (Ref : Node_Id) is
+         Error_Msg_NE
+           ("\add pragma ""Elaborate_Body"" to spec & to ensure full "
+            & "initialization", Asmt, Spec_Id);
+
+         Output_Active_Scenarios (Asmt);
+      end if;
+   end Process_Variable_Assignment_SPARK;
+
+   ---------------------------
+   -- Process_Variable_Read --
+   ---------------------------
+
+   procedure Process_Variable_Read (Ref : Node_Id) is
       Var_Attrs : Variable_Attributes;
       Var_Id    : Entity_Id;
 
@@ -7788,22 +8036,42 @@ package body Sem_Elab is
 
       if Elab_Info_Messages then
          Elab_Msg_NE
-           (Msg      => "reference to variable & during elaboration",
+           (Msg      => "read of variable & during elaboration",
             N        => Ref,
             Id       => Var_Id,
             Info_Msg => True,
             In_SPARK => True);
       end if;
 
-      --  A source variable reference imposes an Elaborate_All requirement on
-      --  the context of the main unit. Determine whethe the context has a
-      --  pragma strong enough to meet the requirement.
+      --  Nothing to do when the variable appears within the main unit because
+      --  diagnostics on reads are relevant only for external variables.
 
-      Meet_Elaboration_Requirement
-        (N         => Ref,
-         Target_Id => Var_Id,
-         Req_Nam   => Name_Elaborate_All);
-   end Process_Variable_Reference;
+      if Is_Same_Unit (Var_Attrs.Unit_Id, Cunit_Entity (Main_Unit)) then
+         null;
+
+      --  Nothing to do when the variable is already initialized. Note that the
+      --  variable may be further modified by the external unit.
+
+      elsif Is_Initialized (Declaration_Node (Var_Id)) then
+         null;
+
+      --  Nothing to do when the external unit guarantees the initialization of
+      --  the variable by means of pragma Elaborate_Body.
+
+      elsif Has_Pragma_Elaborate_Body (Var_Attrs.Unit_Id) then
+         null;
+
+      --  A variable read imposes an Elaborate requirement on the context of
+      --  the main unit. Determine whether the context has a pragma strong
+      --  enough to meet the requirement.
+
+      else
+         Meet_Elaboration_Requirement
+           (N         => Ref,
+            Target_Id => Var_Id,
+            Req_Nam   => Name_Elaborate);
+      end if;
+   end Process_Variable_Read;
 
    --------------------------
    -- Push_Active_Scenario --
@@ -7874,10 +8142,10 @@ package body Sem_Elab is
       elsif Is_Suitable_Variable_Assignment (N) then
          Process_Variable_Assignment (N);
 
-      --  Variable references
+      --  Variable read
 
-      elsif Is_Suitable_Variable_Reference (N) then
-         Process_Variable_Reference (N);
+      elsif Is_Suitable_Variable_Read (N) then
+         Process_Variable_Read (N);
       end if;
 
       --  Remove the current scenario from the stack of active scenarios once
@@ -7938,20 +8206,40 @@ package body Sem_Elab is
       --  listed below are not considered. The categories are:
 
       --   'Access for entries, operators, and subprograms
+      --    Assignments to variables
       --    Calls (includes task activation)
       --    Instantiations
-      --    Variable assignments
-      --    Variable references
+      --    Reads of variables
 
-      elsif Is_Suitable_Access (N)
-        or else Is_Suitable_Variable_Assignment (N)
-        or else Is_Suitable_Variable_Reference (N)
-      then
-         null;
+      elsif Is_Suitable_Access (N) then
+
+         --  Signal any enclosing local exception handlers that the 'Access may
+         --  raise Program_Error due to a failed ABE check when switch -gnatd.o
+         --  (conservative elaboration order for indirect calls) is in effect.
+         --  Marking the exception handlers ensures proper expansion by both
+         --  the front and back end restriction when No_Exception_Propagation
+         --  is in effect.
+
+         if Debug_Flag_Dot_O then
+            Possible_Local_Raise (N, Standard_Program_Error);
+         end if;
 
       elsif Is_Suitable_Call (N) or else Is_Suitable_Instantiation (N) then
          Declaration_Level_OK := True;
 
+         --  Signal any enclosing local exception handlers that the call or
+         --  instantiation may raise Program_Error due to a failed ABE check.
+         --  Marking the exception handlers ensures proper expansion by both
+         --  the front and back end restriction when No_Exception_Propagation
+         --  is in effect.
+
+         Possible_Local_Raise (N, Standard_Program_Error);
+
+      elsif Is_Suitable_Variable_Assignment (N)
+        or else Is_Suitable_Variable_Read (N)
+      then
+         null;
+
       --  Otherwise the input does not denote a suitable scenario
 
       else
@@ -8004,7 +8292,7 @@ package body Sem_Elab is
 
       --  Mark a scenario which may produce run-time conditional ABE checks or
       --  guaranteed ABE failures as recorded. The flag ensures that scenario
-      --  rewritting performed by Atree.Rewrite will be properly reflected in
+      --  rewriting performed by Atree.Rewrite will be properly reflected in
       --  all relevant internal data structures.
 
       if Is_Check_Emitting_Scenario (N) then
diff --git a/gcc/ada/sem_prag.adb b/gcc/ada/sem_prag.adb
index 0456101092a..eae149805fa 100644
--- a/gcc/ada/sem_prag.adb
+++ b/gcc/ada/sem_prag.adb
@@ -2818,10 +2818,16 @@ package body Sem_Prag is
                                              E_Constant,
                                              E_Variable)
                then
+                  --  When the initialization item is undefined, it appears as
+                  --  Any_Id. Do not continue with the analysis of the item.
+
+                  if Item_Id = Any_Id then
+                     null;
+
                   --  The state or variable must be declared in the visible
                   --  declarations of the package (SPARK RM 7.1.5(7)).
 
-                  if not Contains (States_And_Objs, Item_Id) then
+                  elsif not Contains (States_And_Objs, Item_Id) then
                      Error_Msg_Name_1 := Chars (Pack_Id);
                      SPARK_Msg_NE
                        ("initialization item & must appear in the visible "
@@ -13236,23 +13242,21 @@ package body Sem_Prag is
                Set_SCO_Pragma_Enabled (Loc);
             end if;
 
-            --  Deal with analyzing the string argument
+            --  Deal with analyzing the string argument. If checks are not
+            --  on we don't want any expansion (since such expansion would
+            --  not get properly deleted) but we do want to analyze (to get
+            --  proper references). The Preanalyze_And_Resolve routine does
+            --  just what we want. Ditto if pragma is active, because it will
+            --  be rewritten as an if-statement whose analysis will complete
+            --  analysis and expansion of the string message. This makes a
+            --  difference in the unusual case where the expression for the
+            --  string may have a side effect, such as raising an exception.
+            --  This is mandated by RM 11.4.2, which specifies that the string
+            --  expression is only evaluated if the check fails and
+            --  Assertion_Error is to be raised.
 
             if Arg_Count = 3 then
-
-               --  If checks are not on we don't want any expansion (since
-               --  such expansion would not get properly deleted) but
-               --  we do want to analyze (to get proper references).
-               --  The Preanalyze_And_Resolve routine does just what we want
-
-               if Is_Ignored (N) then
-                  Preanalyze_And_Resolve (Str, Standard_String);
-
-                  --  Otherwise we need a proper analysis and expansion
-
-               else
-                  Analyze_And_Resolve (Str, Standard_String);
-               end if;
+               Preanalyze_And_Resolve (Str, Standard_String);
             end if;
 
             --  Now you might think we could just do the same with the Boolean
diff --git a/gcc/ada/sem_res.adb b/gcc/ada/sem_res.adb
index 68c1a0892a6..f5c5f9e96dc 100644
--- a/gcc/ada/sem_res.adb
+++ b/gcc/ada/sem_res.adb
@@ -4843,9 +4843,8 @@ package body Sem_Res is
              (Comes_From_Source (Parent (N))
                or else
                  (Ekind (Current_Scope) = E_Function
-                   and then Nkind
-                     (Original_Node (Unit_Declaration_Node (Current_Scope)))
-                       = N_Expression_Function))
+                   and then Nkind (Original_Node (Unit_Declaration_Node
+                              (Current_Scope))) = N_Expression_Function))
            and then not In_Instance_Body
          then
             if not OK_For_Limited_Init (Etype (E), Expression (E)) then
diff --git a/gcc/ada/sem_type.adb b/gcc/ada/sem_type.adb
index 05315852511..e2b3afdf898 100644
--- a/gcc/ada/sem_type.adb
+++ b/gcc/ada/sem_type.adb
@@ -2838,11 +2838,9 @@ package body Sem_Type is
          return False;
 
       elsif Nkind (Par) in N_Declaration then
-         if Nkind (Par) = N_Object_Declaration then
-            return Present (Corresponding_Generic_Association (Par));
-         else
-            return False;
-         end if;
+         return
+           Nkind (Par) = N_Object_Declaration
+             and then Present (Corresponding_Generic_Association (Par));
 
       elsif Nkind (Par) = N_Object_Renaming_Declaration then
          return Present (Corresponding_Generic_Association (Par));
diff --git a/gcc/ada/sem_util.adb b/gcc/ada/sem_util.adb
index f003ef5a8ac..3698bbf16bd 100644
--- a/gcc/ada/sem_util.adb
+++ b/gcc/ada/sem_util.adb
@@ -3354,10 +3354,13 @@ package body Sem_Util is
            and then not Comes_From_Source (Par)
          then
             --  Continue to examine the context if the reference appears in a
-            --  subprogram body which was previously an expression function.
+            --  subprogram body which was previously an expression function,
+            --  unless this is during preanalysis (when In_Spec_Expression is
+            --  True), as the body may not yet be inserted in the tree.
 
             if Nkind (Par) = N_Subprogram_Body
               and then Was_Expression_Function (Par)
+              and then not In_Spec_Expression
             then
                null;
 
@@ -12545,10 +12548,8 @@ package body Sem_Util is
                  or else (Present (Renamed_Object (E))
                            and then Is_Aliased_View (Renamed_Object (E)))))
 
-           or else ((Is_Formal (E)
-                      or else Ekind_In (E, E_Generic_In_Out_Parameter,
-                                           E_Generic_In_Parameter))
-                    and then Is_Tagged_Type (Etype (E)))
+           or else ((Is_Formal (E) or else Is_Formal_Object (E))
+                      and then Is_Tagged_Type (Etype (E)))
 
            or else (Is_Concurrent_Type (E) and then In_Open_Scopes (E))
 
@@ -13185,17 +13186,29 @@ package body Sem_Util is
    function Is_Controlling_Limited_Procedure
      (Proc_Nam : Entity_Id) return Boolean
    is
+      Param     : Node_Id;
       Param_Typ : Entity_Id := Empty;
 
    begin
       if Ekind (Proc_Nam) = E_Procedure
         and then Present (Parameter_Specifications (Parent (Proc_Nam)))
       then
-         Param_Typ := Etype (Parameter_Type (First (
-                        Parameter_Specifications (Parent (Proc_Nam)))));
+         Param := Parameter_Type (First (
+                    Parameter_Specifications (Parent (Proc_Nam))));
 
-      --  In this case where an Itype was created, the procedure call has been
-      --  rewritten.
+         --  The formal may be an anonymous access type.
+
+         if Nkind (Param) = N_Access_Definition then
+            Param_Typ := Entity (Subtype_Mark (Param));
+
+         else
+            Param_Typ := Etype (Param);
+         end if;
+
+      --  In the case where an Itype was created for a dispatchin call, the
+      --  procedure call has been rewritten. The actual may be an access to
+      --  interface type in which case it is the designated type that is the
+      --  controlling type.
 
       elsif Present (Associated_Node_For_Itype (Proc_Nam))
         and then Present (Original_Node (Associated_Node_For_Itype (Proc_Nam)))
@@ -13206,6 +13219,10 @@ package body Sem_Util is
          Param_Typ :=
            Etype (First (Parameter_Associations
                           (Associated_Node_For_Itype (Proc_Nam))));
+
+         if Ekind (Param_Typ) = E_Anonymous_Access_Type then
+            Param_Typ := Directly_Designated_Type (Param_Typ);
+         end if;
       end if;
 
       if Present (Param_Typ) then
@@ -13387,7 +13404,7 @@ package body Sem_Util is
                end if;
 
             --  A discriminant check on a selected component may be expanded
-            --  into a dereference when removing side-effects. Recover the
+            --  into a dereference when removing side effects. Recover the
             --  original node and its type, which may be unconstrained.
 
             elsif Nkind (P) = N_Explicit_Dereference
@@ -20584,6 +20601,51 @@ package body Sem_Util is
       return False;
    end Null_To_Null_Address_Convert_OK;
 
+   ---------------------------------
+   -- Number_Of_Elements_In_Array --
+   ---------------------------------
+
+   function Number_Of_Elements_In_Array (T : Entity_Id) return Int is
+      Indx : Node_Id;
+      Typ  : Entity_Id;
+      Low  : Node_Id;
+      High : Node_Id;
+      Num  : Int := 1;
+
+   begin
+      pragma Assert (Is_Array_Type (T));
+
+      Indx := First_Index (T);
+      while Present (Indx) loop
+         Typ := Underlying_Type (Etype (Indx));
+
+         --  Never look at junk bounds of a generic type
+
+         if Is_Generic_Type (Typ) then
+            return 0;
+         end if;
+
+         --  Check the array bounds are known at compile time and return zero
+         --  if they are not.
+
+         Low  := Type_Low_Bound (Typ);
+         High := Type_High_Bound (Typ);
+
+         if not Compile_Time_Known_Value (Low) then
+            return 0;
+         elsif not Compile_Time_Known_Value (High) then
+            return 0;
+         else
+            Num :=
+              Num * UI_To_Int ((Expr_Value (High) - Expr_Value (Low) + 1));
+         end if;
+
+         Next_Index (Indx);
+      end loop;
+
+      return Num;
+   end Number_Of_Elements_In_Array;
+
    -------------------------
    -- Object_Access_Level --
    -------------------------
@@ -20603,7 +20665,7 @@ package body Sem_Util is
       --  This construct appears in the context of dispatching calls.
 
       function Reference_To (Obj : Node_Id) return Node_Id;
-      --  An explicit dereference is created when removing side-effects from
+      --  An explicit dereference is created when removing side effects from
       --  expressions for constraint checking purposes. In this case a local
       --  access type is created for it. The correct access level is that of
       --  the original source node. We detect this case by noting that the
diff --git a/gcc/ada/sem_util.ads b/gcc/ada/sem_util.ads
index 2ebd54f3989..c6958cb1aaa 100644
--- a/gcc/ada/sem_util.ads
+++ b/gcc/ada/sem_util.ads
@@ -2275,6 +2275,11 @@ package Sem_Util is
    --   2) N is a comparison operator, one of the operands is null, and the
    --      type of the other operand is a descendant of System.Address.
 
+   function Number_Of_Elements_In_Array (T : Entity_Id) return Int;
+   --  Returns the number of elements in the array T if the index bounds of T
+   --  is known at compile time. If the bounds are not known at compile time,
+   --  the function returns the value zero.
+
    function Object_Access_Level (Obj : Node_Id) return Uint;
    --  Return the accessibility level of the view of the object Obj. For
    --  convenience, qualified expressions applied to object names are also
diff --git a/gcc/ada/sem_warn.adb b/gcc/ada/sem_warn.adb
index 91f430a29f5..0e498d3e6cb 100644
--- a/gcc/ada/sem_warn.adb
+++ b/gcc/ada/sem_warn.adb
@@ -509,7 +509,7 @@ package body Sem_Warn is
             end if;
 
             --  If the condition contains a function call, we consider it may
-            --  be modified by side-effects from a procedure call. Otherwise,
+            --  be modified by side effects from a procedure call. Otherwise,
             --  we consider the condition may not be modified, although that
             --  might happen if Variable is itself a by-reference parameter,
             --  and the procedure called modifies the global object referred to
diff --git a/gcc/ada/sinfo.adb b/gcc/ada/sinfo.adb
index e4f8608eb73..dc4e8fb2c1a 100644
--- a/gcc/ada/sinfo.adb
+++ b/gcc/ada/sinfo.adb
@@ -203,6 +203,14 @@ package body Sinfo is
       return Flag4 (N);
    end Aliased_Present;
 
+   function Alloc_For_BIP_Return
+      (N : Node_Id) return Boolean is
+   begin
+      pragma Assert (False
+        or else NT (N).Nkind = N_Allocator);
+      return Flag1 (N);
+   end Alloc_For_BIP_Return;
+
    function All_Others
       (N : Node_Id) return Boolean is
    begin
@@ -3626,6 +3634,14 @@ package body Sinfo is
       Set_Flag4 (N, Val);
    end Set_Aliased_Present;
 
+   procedure Set_Alloc_For_BIP_Return
+      (N : Node_Id; Val : Boolean := True) is
+   begin
+      pragma Assert (False
+        or else NT (N).Nkind = N_Allocator);
+      Set_Flag1 (N, Val);
+   end Set_Alloc_For_BIP_Return;
+
    procedure Set_All_Others
       (N : Node_Id; Val : Boolean := True) is
    begin
diff --git a/gcc/ada/sinfo.ads b/gcc/ada/sinfo.ads
index 247d127982d..cf220e4e563 100644
--- a/gcc/ada/sinfo.ads
+++ b/gcc/ada/sinfo.ads
@@ -770,7 +770,7 @@ package Sinfo is
    --  The following flag fields appear in all nodes:
 
    --  Analyzed
-   --    This flag is used to indicate that a node (and all its children have
+   --    This flag is used to indicate that a node (and all its children) have
    --    been analyzed. It is used to avoid reanalysis of a node that has
    --    already been analyzed, both for efficiency and functional correctness
    --    reasons.
@@ -903,6 +903,10 @@ package Sinfo is
    --    known at compile time, this field points to an N_Range node with those
    --    bounds. Otherwise Empty.
 
+   --  Alloc_For_BIP_Return (Flag1-Sem)
+   --    Present in N_Allocator nodes. True if the allocator is one of those
+   --    generated for a build-in-place return statement.
+
    --  All_Others (Flag11-Sem)
    --    Present in an N_Others_Choice node. This flag is set for an others
    --    exception where all exceptions are to be caught, even those that are
@@ -1472,10 +1476,7 @@ package Sinfo is
    --  Generic_Parent (Node5-Sem)
    --    Generic_Parent is defined on declaration nodes that are instances. The
    --    value of Generic_Parent is the generic entity from which the instance
-   --    is obtained. Generic_Parent is also defined for the renaming
-   --    declarations and object declarations created for the actuals in an
-   --    instantiation. The generic parent of such a declaration is the
-   --    corresponding generic association in the Instantiation node.
+   --    is obtained.
 
    --  Generic_Parent_Type (Node4-Sem)
    --    Generic_Parent_Type is defined on Subtype_Declaration nodes for the
@@ -4776,6 +4777,7 @@ package Sinfo is
       --  Subpool_Handle_Name (Node4) (set to Empty if not present)
       --  Storage_Pool (Node1-Sem)
       --  Procedure_To_Call (Node2-Sem)
+      --  Alloc_For_BIP_Return (Flag1-Sem)
       --  Null_Exclusion_Present (Flag11)
       --  No_Initialization (Flag13-Sem)
       --  Is_Static_Coextension (Flag14-Sem)
@@ -7840,7 +7842,7 @@ package Sinfo is
 
       --  The required semantics is that the set of actions is executed in
       --  the order in which it appears, as though they appeared by themselves
-      --  in the enclosing list of declarations of statements. Unlike what
+      --  in the enclosing list of declarations or statements. Unlike what
       --  happens when using an N_Block_Statement, no new scope is introduced.
 
       --  Note: for the time being, this is used only as a transient
@@ -9128,6 +9130,9 @@ package Sinfo is
    function Aliased_Present
      (N : Node_Id) return Boolean;    -- Flag4
 
+   function Alloc_For_BIP_Return
+     (N : Node_Id) return Boolean;    -- Flag1
+
    function All_Others
      (N : Node_Id) return Boolean;    -- Flag11
 
@@ -10217,6 +10222,9 @@ package Sinfo is
    procedure Set_Aliased_Present
      (N : Node_Id; Val : Boolean := True);    -- Flag4
 
+   procedure Set_Alloc_For_BIP_Return
+     (N : Node_Id; Val : Boolean := True);    -- Flag1
+
    procedure Set_All_Others
      (N : Node_Id; Val : Boolean := True);    -- Flag11
 
@@ -13066,6 +13074,7 @@ package Sinfo is
    pragma Inline (Address_Warning_Posted);
    pragma Inline (Aggregate_Bounds);
    pragma Inline (Aliased_Present);
+   pragma Inline (Alloc_For_BIP_Return);
    pragma Inline (All_Others);
    pragma Inline (All_Present);
    pragma Inline (Alternatives);
@@ -13426,6 +13435,7 @@ package Sinfo is
    pragma Inline (Set_Address_Warning_Posted);
    pragma Inline (Set_Aggregate_Bounds);
    pragma Inline (Set_Aliased_Present);
+   pragma Inline (Set_Alloc_For_BIP_Return);
    pragma Inline (Set_All_Others);
    pragma Inline (Set_All_Present);
    pragma Inline (Set_Alternatives);
diff --git a/gcc/ada/sinput.ads b/gcc/ada/sinput.ads
index bde59b131dd..ecbe83cdd88 100644
--- a/gcc/ada/sinput.ads
+++ b/gcc/ada/sinput.ads
@@ -755,6 +755,8 @@ private
    pragma Inline (Num_Source_Files);
    pragma Inline (Num_Source_Lines);
 
+   pragma Inline (Line_Start);
+
    No_Instance_Id : constant Instance_Id := 0;
 
    -------------------------
diff --git a/gcc/ada/switch-b.adb b/gcc/ada/switch-b.adb
index 52a72e4de40..61fe4404b7d 100644
--- a/gcc/ada/switch-b.adb
+++ b/gcc/ada/switch-b.adb
@@ -391,6 +391,18 @@ package body Switch.B is
             Ptr := Ptr + 1;
             Quiet_Output := True;
 
+         --  Processing for Q switch
+
+         when 'Q' =>
+            if Ptr = Max then
+               Bad_Switch (Switch_Chars);
+            end if;
+
+            Ptr := Ptr + 1;
+            Scan_Pos
+              (Switch_Chars, Max, Ptr,
+               Quantity_Of_Default_Size_Sec_Stacks, C);
+
          --  Processing for r switch
 
          when 'r' =>
diff --git a/gcc/ada/switch-c.adb b/gcc/ada/switch-c.adb
index cd6b2006e22..5ad10e348a5 100644
--- a/gcc/ada/switch-c.adb
+++ b/gcc/ada/switch-c.adb
@@ -548,7 +548,6 @@ package body Switch.C is
                         Warn_On_Bad_Fixed_Value          := True; -- -gnatwb
                         Warn_On_Biased_Representation    := True; -- -gnatw.b
                         Warn_On_Export_Import            := True; -- -gnatwx
-                        Warn_On_Modified_Unread          := True; -- -gnatwm
                         Warn_On_No_Value_Assigned        := True; -- -gnatwv
                         Warn_On_Object_Renames_Function  := True; -- -gnatw.r
                         Warn_On_Overlap                  := True; -- -gnatw.i
diff --git a/gcc/ada/widechar.ads b/gcc/ada/widechar.ads
index a6e8293ae5d..3d2f9170976 100644
--- a/gcc/ada/widechar.ads
+++ b/gcc/ada/widechar.ads
@@ -6,7 +6,7 @@
 --                                                                          --
 --                                 S p e c                                  --
 --                                                                          --
---          Copyright (C) 1992-2014, Free Software Foundation, Inc.         --
+--          Copyright (C) 1992-2017, Free Software Foundation, Inc.         --
 --                                                                          --
 -- GNAT is free software;  you can  redistribute it  and/or modify it under --
 -- terms of the  GNU General Public License as published  by the Free Soft- --
@@ -95,4 +95,7 @@ package Widechar is
       P : Source_Ptr) return Boolean;
    --  Determines if S (P) is the start of a wide character sequence
 
+private
+   pragma Inline (Is_Start_Of_Wide_Char);
+
 end Widechar;
diff --git a/gcc/alias.c b/gcc/alias.c
index f288299ec32..c69ef410eda 100644
--- a/gcc/alias.c
+++ b/gcc/alias.c
@@ -2054,13 +2054,15 @@ compare_base_decls (tree base1, tree base2)
     return 1;
 
   /* If we have two register decls with register specification we
-     cannot decide unless their assembler name is the same.  */
+     cannot decide unless their assembler names are the same.  */
   if (DECL_REGISTER (base1)
       && DECL_REGISTER (base2)
+      && HAS_DECL_ASSEMBLER_NAME_P (base1)
+      && HAS_DECL_ASSEMBLER_NAME_P (base2)
       && DECL_ASSEMBLER_NAME_SET_P (base1)
       && DECL_ASSEMBLER_NAME_SET_P (base2))
     {
-      if (DECL_ASSEMBLER_NAME (base1) == DECL_ASSEMBLER_NAME (base2))
+      if (DECL_ASSEMBLER_NAME_RAW (base1) == DECL_ASSEMBLER_NAME_RAW (base2))
 	return 1;
       return -1;
     }
@@ -2331,7 +2333,7 @@ addr_side_effect_eval (rtx addr, poly_int64 size, int n_refs)
 static inline bool
 offset_overlap_p (poly_int64 c, poly_int64 xsize, poly_int64 ysize)
 {
-  if (known_zero (xsize) || known_zero (ysize))
+  if (must_eq (xsize, 0) || must_eq (ysize, 0))
     return true;
 
   if (may_ge (c, 0))
@@ -2561,7 +2563,7 @@ memrefs_conflict_p (poly_int64 xsize, rtx x, poly_int64 ysize, rtx y,
 	{
 	  if (may_gt (xsize, 0))
 	    xsize = -xsize;
-	  if (maybe_nonzero (xsize))
+	  if (may_ne (xsize, 0))
 	    xsize += sc + 1;
 	  c -= sc + 1;
 	  return memrefs_conflict_p (xsize, canon_rtx (XEXP (x, 0)),
@@ -2576,7 +2578,7 @@ memrefs_conflict_p (poly_int64 xsize, rtx x, poly_int64 ysize, rtx y,
 	{
 	  if (may_gt (ysize, 0))
 	    ysize = -ysize;
-	  if (maybe_nonzero (ysize))
+	  if (may_ne (ysize, 0))
 	    ysize += sc + 1;
 	  c += sc + 1;
 	  return memrefs_conflict_p (xsize, x,
diff --git a/gcc/asan.c b/gcc/asan.c
index 779aa78976d..d00089d04dc 100644
--- a/gcc/asan.c
+++ b/gcc/asan.c
@@ -628,10 +628,9 @@ handle_builtin_alloca (gcall *call, gimple_stmt_iterator *iter)
   tree ptr_type = gimple_call_lhs (call) ? TREE_TYPE (gimple_call_lhs (call))
 					 : ptr_type_node;
   tree partial_size = NULL_TREE;
-  bool alloca_with_align
-    = DECL_FUNCTION_CODE (callee) == BUILT_IN_ALLOCA_WITH_ALIGN;
   unsigned int align
-    = alloca_with_align ? tree_to_uhwi (gimple_call_arg (call, 1)) : 0;
+    = DECL_FUNCTION_CODE (callee) == BUILT_IN_ALLOCA
+      ? 0 : tree_to_uhwi (gimple_call_arg (call, 1));
 
   /* If ALIGN > ASAN_RED_ZONE_SIZE, we embed left redzone into first ALIGN
      bytes of allocated space.  Otherwise, align alloca to ASAN_RED_ZONE_SIZE
@@ -793,8 +792,7 @@ get_mem_refs_of_builtin_call (gcall *call,
       handle_builtin_stack_restore (call, iter);
       break;
 
-    case BUILT_IN_ALLOCA_WITH_ALIGN:
-    case BUILT_IN_ALLOCA:
+    CASE_BUILT_IN_ALLOCA:
       handle_builtin_alloca (call, iter);
       break;
     /* And now the __atomic* and __sync builtins.
@@ -1804,13 +1802,13 @@ create_cond_insert_point (gimple_stmt_iterator *iter,
     ? profile_probability::very_unlikely ()
     : profile_probability::very_likely ();
   e->probability = fallthrough_probability.invert ();
+  then_bb->count = e->count ();
   if (create_then_fallthru_edge)
     make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU);
 
   /* Set up the fallthrough basic block.  */
   e = find_edge (cond_bb, fallthru_bb);
   e->flags = EDGE_FALSE_VALUE;
-  e->count = cond_bb->count;
   e->probability = fallthrough_probability;
 
   /* Update dominance info for the newly created then_bb; note that
@@ -2946,6 +2944,9 @@ asan_finish_file (void)
       TREE_CONSTANT (ctor) = 1;
       TREE_STATIC (ctor) = 1;
       DECL_INITIAL (var) = ctor;
+      SET_DECL_ALIGN (var, MAX (DECL_ALIGN (var),
+				ASAN_SHADOW_GRANULARITY * BITS_PER_UNIT));
+
       varpool_node::finalize_decl (var);
 
       tree fn = builtin_decl_implicit (BUILT_IN_ASAN_REGISTER_GLOBALS);
@@ -3401,6 +3402,10 @@ asan_expand_poison_ifn (gimple_stmt_iterator *iter,
 	      {
 		edge e = gimple_phi_arg_edge (phi, i);
 
+		/* Do not insert on an edge we can't split.  */
+		if (e->flags & EDGE_ABNORMAL)
+		  continue;
+
 		if (call_to_insert == NULL)
 		  call_to_insert = gimple_copy (call);
 
diff --git a/gcc/attribs.c b/gcc/attribs.c
index 4ef35b861f8..809f4c3a8d5 100644
--- a/gcc/attribs.c
+++ b/gcc/attribs.c
@@ -1125,9 +1125,9 @@ attribute_value_equal (const_tree attr1, const_tree attr2)
 				     TREE_VALUE (attr2)) == 1);
     }
 
-  if ((flag_openmp || flag_openmp_simd)
-      && TREE_VALUE (attr1) && TREE_VALUE (attr2)
+  if (TREE_VALUE (attr1)
       && TREE_CODE (TREE_VALUE (attr1)) == OMP_CLAUSE
+      && TREE_VALUE (attr2)
       && TREE_CODE (TREE_VALUE (attr2)) == OMP_CLAUSE)
     return omp_declare_simd_clauses_equal (TREE_VALUE (attr1),
 					   TREE_VALUE (attr2));
@@ -1182,6 +1182,9 @@ comp_type_attributes (const_tree type1, const_tree type2)
     }
   if (lookup_attribute ("transaction_safe", CONST_CAST_TREE (a)))
     return 0;
+  if ((lookup_attribute ("nocf_check", TYPE_ATTRIBUTES (type1)) != NULL)
+      ^ (lookup_attribute ("nocf_check", TYPE_ATTRIBUTES (type2)) != NULL))
+    return 0;
   /* As some type combinations - like default calling-convention - might
      be compatible, we have to call the target hook to get the final result.  */
   return targetm.comp_type_attributes (type1, type2);
@@ -1319,6 +1322,44 @@ merge_decl_attributes (tree olddecl, tree newdecl)
 			   DECL_ATTRIBUTES (newdecl));
 }
 
+/* Duplicate all attributes with name NAME in ATTR list to *ATTRS if
+   they are missing there.  */
+
+void
+duplicate_one_attribute (tree *attrs, tree attr, const char *name)
+{
+  attr = lookup_attribute (name, attr);
+  if (!attr)
+    return;
+  tree a = lookup_attribute (name, *attrs);
+  while (attr)
+    {
+      tree a2;
+      for (a2 = a; a2; a2 = lookup_attribute (name, TREE_CHAIN (a2)))
+	if (attribute_value_equal (attr, a2))
+	  break;
+      if (!a2)
+	{
+	  a2 = copy_node (attr);
+	  TREE_CHAIN (a2) = *attrs;
+	  *attrs = a2;
+	}
+      attr = lookup_attribute (name, TREE_CHAIN (attr));
+    }
+}
+
+/* Duplicate all attributes from user DECL to the corresponding
+   builtin that should be propagated.  */
+
+void
+copy_attributes_to_builtin (tree decl)
+{
+  tree b = builtin_decl_explicit (DECL_FUNCTION_CODE (decl));
+  if (b)
+    duplicate_one_attribute (&DECL_ATTRIBUTES (b),
+			     DECL_ATTRIBUTES (decl), "omp declare simd");
+}
+
 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
 
 /* Specialization of merge_decl_attributes for various Windows targets.
diff --git a/gcc/attribs.h b/gcc/attribs.h
index 65e002ce988..f4bfe03e467 100644
--- a/gcc/attribs.h
+++ b/gcc/attribs.h
@@ -77,6 +77,16 @@ extern tree remove_attribute (const char *, tree);
 
 extern tree merge_attributes (tree, tree);
 
+/* Duplicate all attributes with name NAME in ATTR list to *ATTRS if
+   they are missing there.  */
+
+extern void duplicate_one_attribute (tree *, tree, const char *);
+
+/* Duplicate all attributes from user DECL to the corresponding
+   builtin that should be propagated.  */
+
+extern void copy_attributes_to_builtin (tree);
+
 /* Given two Windows decl attributes lists, possibly including
    dllimport, return a list of their union .  */
 extern tree merge_dllimport_decl_attributes (tree, tree);
diff --git a/gcc/auto-profile.c b/gcc/auto-profile.c
index 9226e202d50..130d8df5b1e 100644
--- a/gcc/auto-profile.c
+++ b/gcc/auto-profile.c
@@ -852,7 +852,7 @@ autofdo_source_profile::read ()
 {
   if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION)
     {
-      inform (0, "Not expected TAG.");
+      inform (UNKNOWN_LOCATION, "Not expected TAG.");
       return false;
     }
 
@@ -1234,7 +1234,7 @@ afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
       if (!is_edge_annotated (e, *annotated_edge))
 	num_unknown_edge++, unknown_edge = e;
       else
-	total_known_count += e->count;
+	total_known_count += e->count ();
 
     if (num_unknown_edge == 0)
       {
@@ -1251,7 +1251,8 @@ afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
       }
     else if (num_unknown_edge == 1 && is_bb_annotated (bb, *annotated_bb))
       {
-        unknown_edge->count = bb->count - total_known_count;
+        unknown_edge->probability
+	  = total_known_count.probability_in (bb->count);
         set_edge_annotated (unknown_edge, annotated_edge);
         changed = true;
       }
@@ -1349,15 +1350,13 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
           if (!e->probability.initialized_p ()
 	      && !is_edge_annotated (ep, *annotated_edge))
             {
-              ep->probability = profile_probability::never ();
-              ep->count = profile_count::zero ().afdo ();
+              ep->probability = profile_probability::never ().afdo ();
               set_edge_annotated (ep, annotated_edge);
             }
         }
       if (total == 1 && !is_edge_annotated (only_one, *annotated_edge))
         {
           only_one->probability = e->probability;
-          only_one->count = e->count;
           set_edge_annotated (only_one, annotated_edge);
         }
     }
@@ -1433,23 +1432,16 @@ afdo_calculate_branch_prob (bb_set *annotated_bb, edge_set *annotated_edge)
       if (!is_edge_annotated (e, *annotated_edge))
         num_unknown_succ++;
       else
-        total_count += e->count;
+        total_count += e->count ();
     }
     if (num_unknown_succ == 0 && total_count > profile_count::zero ())
       {
         FOR_EACH_EDGE (e, ei, bb->succs)
-        e->probability = e->count.probability_in (total_count);
+          e->probability = e->count ().probability_in (total_count);
       }
   }
   FOR_ALL_BB_FN (bb, cfun)
-  {
-    edge e;
-    edge_iterator ei;
-
-    FOR_EACH_EDGE (e, ei, bb->succs)
-      e->count = bb->count.apply_probability (e->probability);
     bb->aux = NULL;
-  }
 
   loop_optimizer_finalize ();
   free_dominance_info (CDI_DOMINATORS);
@@ -1551,7 +1543,7 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts)
        counters are zero when not seen by autoFDO.  */
     bb->count = profile_count::zero ().afdo ();
     FOR_EACH_EDGE (e, ei, bb->succs)
-      e->count = profile_count::zero ().afdo ();
+      e->probability = profile_probability::uninitialized ();
 
     if (afdo_set_bb_count (bb, promoted_stmts))
       set_bb_annotated (bb, &annotated_bb);
diff --git a/gcc/basic-block.h b/gcc/basic-block.h
index c0c47784c02..5a5ddbfcb6d 100644
--- a/gcc/basic-block.h
+++ b/gcc/basic-block.h
@@ -46,8 +46,9 @@ struct GTY((user)) edge_def {
 
   int flags;			/* see cfg-flags.def */
   profile_probability probability;
-  profile_count count;		/* Expected number of executions calculated
-				   in profile.c  */
+
+  /* Return count of edge E.  */
+  inline profile_count count () const;
 };
 
 /* Masks for edge.flags.  */
@@ -147,9 +148,6 @@ struct GTY((chain_next ("%h.next_bb"), chain_prev ("%h.prev_bb"))) basic_block_d
   /* Expected number of executions: calculated in profile.c.  */
   profile_count count;
 
-  /* Expected frequency.  Normalized to be in range 0 to BB_FREQ_MAX.  */
-  int frequency;
-
   /* The discriminator for this block.  The discriminator distinguishes
      among several basic blocks that share a common locus, allowing for
      more accurate sample-based profiling.  */
@@ -300,7 +298,7 @@ enum cfg_bb_flags
 					 ? EDGE_SUCC ((bb), 1) : EDGE_SUCC ((bb), 0))
 
 /* Return expected execution frequency of the edge E.  */
-#define EDGE_FREQUENCY(e)		e->probability.apply (e->src->frequency)
+#define EDGE_FREQUENCY(e)		e->count ().to_frequency (cfun)
 
 /* Compute a scale factor (or probability) suitable for scaling of
    gcov_type values via apply_probability() and apply_scale().  */
@@ -639,4 +637,10 @@ has_abnormal_call_or_eh_pred_edge_p (basic_block bb)
   return false;
 }
 
+/* Return count of edge E.  */
+inline profile_count edge_def::count () const
+{
+  return src->count.apply_probability (probability);
+}
+
 #endif /* GCC_BASIC_BLOCK_H */
diff --git a/gcc/bb-reorder.c b/gcc/bb-reorder.c
index 4dad298fe59..f7c1f4c971e 100644
--- a/gcc/bb-reorder.c
+++ b/gcc/bb-reorder.c
@@ -256,8 +256,8 @@ push_to_next_round_p (const_basic_block bb, int round, int number_of_rounds,
 
   there_exists_another_round = round < number_of_rounds - 1;
 
-  block_not_hot_enough = (bb->frequency < exec_th
-			  || bb->count < count_th
+  block_not_hot_enough = (bb->count.to_frequency (cfun) < exec_th
+			  || bb->count.ipa () < count_th
 			  || probably_never_executed_bb_p (cfun, bb));
 
   if (there_exists_another_round
@@ -293,9 +293,9 @@ find_traces (int *n_traces, struct trace *traces)
     {
       bbd[e->dest->index].heap = heap;
       bbd[e->dest->index].node = heap->insert (bb_to_key (e->dest), e->dest);
-      if (e->dest->frequency > max_entry_frequency)
-	max_entry_frequency = e->dest->frequency;
-      if (e->dest->count.initialized_p () && e->dest->count > max_entry_count)
+      if (e->dest->count.to_frequency (cfun) > max_entry_frequency)
+	max_entry_frequency = e->dest->count.to_frequency (cfun);
+      if (e->dest->count.ipa_p () && e->dest->count > max_entry_count)
 	max_entry_count = e->dest->count;
     }
 
@@ -329,8 +329,10 @@ find_traces (int *n_traces, struct trace *traces)
 	  for (bb = traces[i].first;
 	       bb != traces[i].last;
 	       bb = (basic_block) bb->aux)
-	    fprintf (dump_file, "%d [%d] ", bb->index, bb->frequency);
-	  fprintf (dump_file, "%d [%d]\n", bb->index, bb->frequency);
+	    fprintf (dump_file, "%d [%d] ", bb->index,
+		     bb->count.to_frequency (cfun));
+	  fprintf (dump_file, "%d [%d]\n", bb->index,
+		   bb->count.to_frequency (cfun));
 	}
       fflush (dump_file);
     }
@@ -374,11 +376,11 @@ rotate_loop (edge back_edge, struct trace *trace, int trace_n)
 		{
 		  /* The current edge E is also preferred.  */
 		  int freq = EDGE_FREQUENCY (e);
-		  if (freq > best_freq || e->count > best_count)
+		  if (freq > best_freq || e->count () > best_count)
 		    {
 		      best_freq = freq;
-		      if (e->count.initialized_p ())
-		        best_count = e->count;
+		      if (e->count ().initialized_p ())
+		        best_count = e->count ();
 		      best_edge = e;
 		      best_bb = bb;
 		    }
@@ -392,17 +394,17 @@ rotate_loop (edge back_edge, struct trace *trace, int trace_n)
 		  /* The current edge E is preferred.  */
 		  is_preferred = true;
 		  best_freq = EDGE_FREQUENCY (e);
-		  best_count = e->count;
+		  best_count = e->count ();
 		  best_edge = e;
 		  best_bb = bb;
 		}
 	      else
 		{
 		  int freq = EDGE_FREQUENCY (e);
-		  if (!best_edge || freq > best_freq || e->count > best_count)
+		  if (!best_edge || freq > best_freq || e->count () > best_count)
 		    {
 		      best_freq = freq;
-		      best_count = e->count;
+		      best_count = e->count ();
 		      best_edge = e;
 		      best_bb = bb;
 		    }
@@ -529,7 +531,7 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
 
 	  if (dump_file)
 	    fprintf (dump_file, "Basic block %d was visited in trace %d\n",
-		     bb->index, *n_traces - 1);
+		     bb->index, *n_traces);
 
 	  ends_in_call = block_ends_with_call_p (bb);
 
@@ -545,11 +547,13 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
 		  && bb_visited_trace (e->dest) != *n_traces)
 		continue;
 
+	      /* If partitioning hot/cold basic blocks, don't consider edges
+		 that cross section boundaries.  */
 	      if (BB_PARTITION (e->dest) != BB_PARTITION (bb))
 		continue;
 
 	      prob = e->probability;
-	      freq = e->dest->frequency;
+	      freq = e->dest->count.to_frequency (cfun);
 
 	      /* The only sensible preference for a call instruction is the
 		 fallthru edge.  Don't bother selecting anything else.  */
@@ -571,12 +575,9 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
 		  || !prob.initialized_p ()
 		  || ((prob.to_reg_br_prob_base () < branch_th
 		       || EDGE_FREQUENCY (e) < exec_th
-		      || e->count < count_th) && (!for_size)))
+		      || e->count ().ipa () < count_th) && (!for_size)))
 		continue;
 
-	      /* If partitioning hot/cold basic blocks, don't consider edges
-		 that cross section boundaries.  */
-
 	      if (better_edge_p (bb, e, prob, freq, best_prob, best_freq,
 				 best_edge))
 		{
@@ -586,12 +587,28 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
 		}
 	    }
 
-	  /* If the best destination has multiple predecessors, and can be
-	     duplicated cheaper than a jump, don't allow it to be added
-	     to a trace.  We'll duplicate it when connecting traces.  */
-	  if (best_edge && EDGE_COUNT (best_edge->dest->preds) >= 2
+	  /* If the best destination has multiple predecessors and can be
+	     duplicated cheaper than a jump, don't allow it to be added to
+	     a trace; we'll duplicate it when connecting the traces later.
+	     However, we need to check that this duplication wouldn't leave
+	     the best destination with only crossing predecessors, because
+	     this would change its effective partition from hot to cold.  */
+	  if (best_edge
+	      && EDGE_COUNT (best_edge->dest->preds) >= 2
 	      && copy_bb_p (best_edge->dest, 0))
-	    best_edge = NULL;
+	    {
+	      bool only_crossing_preds = true;
+	      edge e;
+	      edge_iterator ei;
+	      FOR_EACH_EDGE (e, ei, best_edge->dest->preds)
+		if (e != best_edge && !(e->flags & EDGE_CROSSING))
+		  {
+		    only_crossing_preds = false;
+		    break;
+		  }
+	      if (!only_crossing_preds)
+		best_edge = NULL;
+	    }
 
 	  /* If the best destination has multiple successors or predecessors,
 	     don't allow it to be added when optimizing for size.  This makes
@@ -656,7 +673,7 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
 		      || !prob.initialized_p ()
 		      || prob.to_reg_br_prob_base () < branch_th
 		      || freq < exec_th
-		      || e->count < count_th)
+		      || e->count ().ipa () < count_th)
 		    {
 		      /* When partitioning hot/cold basic blocks, make sure
 			 the cold blocks (and only the cold blocks) all get
@@ -691,7 +708,7 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
 		  if (best_edge->dest != bb)
 		    {
 		      if (EDGE_FREQUENCY (best_edge)
-			  > 4 * best_edge->dest->frequency / 5)
+			  > 4 * best_edge->dest->count.to_frequency (cfun) / 5)
 			{
 			  /* The loop has at least 4 iterations.  If the loop
 			     header is not the first block of the function
@@ -768,8 +785,8 @@ find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
 			    & EDGE_CAN_FALLTHRU)
 			&& !(single_succ_edge (e->dest)->flags & EDGE_COMPLEX)
 			&& single_succ (e->dest) == best_edge->dest
-			&& (2 * e->dest->frequency >= EDGE_FREQUENCY (best_edge)
-			    || for_size))
+			&& (2 * e->dest->count.to_frequency (cfun)
+			    >= EDGE_FREQUENCY (best_edge) || for_size))
 		      {
 			best_edge = e;
 			if (dump_file)
@@ -930,9 +947,9 @@ bb_to_key (basic_block bb)
 
   if (priority)
     /* The block with priority should have significantly lower key.  */
-    return -(100 * BB_FREQ_MAX + 100 * priority + bb->frequency);
+    return -(100 * BB_FREQ_MAX + 100 * priority + bb->count.to_frequency (cfun));
 
-  return -bb->frequency;
+  return -bb->count.to_frequency (cfun);
 }
 
 /* Return true when the edge E from basic block BB is better than the temporary
@@ -988,16 +1005,6 @@ better_edge_p (const_basic_block bb, const_edge e, profile_probability prob,
   else
     is_better_edge = false;
 
-  /* If we are doing hot/cold partitioning, make sure that we always favor
-     non-crossing edges over crossing edges.  */
-
-  if (!is_better_edge
-      && flag_reorder_blocks_and_partition
-      && cur_best_edge
-      && (cur_best_edge->flags & EDGE_CROSSING)
-      && !(e->flags & EDGE_CROSSING))
-    is_better_edge = true;
-
   return is_better_edge;
 }
 
@@ -1285,7 +1292,7 @@ connect_traces (int n_traces, struct trace *traces)
 				&& !connected[bbd[di].start_of_trace]
 				&& BB_PARTITION (e2->dest) == current_partition
 				&& EDGE_FREQUENCY (e2) >= freq_threshold
-				&& e2->count >= count_threshold
+				&& e2->count ().ipa () >= count_threshold
 				&& (!best2
 				    || e2->probability > best2->probability
 				    || (e2->probability == best2->probability
@@ -1311,8 +1318,8 @@ connect_traces (int n_traces, struct trace *traces)
 		  && copy_bb_p (best->dest,
 				optimize_edge_for_speed_p (best)
 				&& EDGE_FREQUENCY (best) >= freq_threshold
-				&& (!best->count.initialized_p ()
-				    || best->count >= count_threshold)))
+				&& (!best->count ().initialized_p ()
+				    || best->count ().ipa () >= count_threshold)))
 		{
 		  basic_block new_bb;
 
@@ -1370,7 +1377,7 @@ copy_bb_p (const_basic_block bb, int code_may_grow)
   int max_size = uncond_jump_length;
   rtx_insn *insn;
 
-  if (!bb->frequency)
+  if (!bb->count.to_frequency (cfun))
     return false;
   if (EDGE_COUNT (bb->preds) < 2)
     return false;
@@ -1454,7 +1461,6 @@ fix_up_crossing_landing_pad (eh_landing_pad old_lp, basic_block old_bb)
   last_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
   new_bb = create_basic_block (new_label, jump, last_bb);
   new_bb->aux = last_bb->aux;
-  new_bb->frequency = post_bb->frequency;
   new_bb->count = post_bb->count;
   last_bb->aux = new_bb;
 
@@ -1512,7 +1518,6 @@ sanitize_hot_paths (bool walk_up, unsigned int cold_bb_count,
       edge_iterator ei;
       profile_probability highest_probability
 				 = profile_probability::uninitialized ();
-      int highest_freq = 0;
       profile_count highest_count = profile_count::uninitialized ();
       bool found = false;
 
@@ -1528,7 +1533,7 @@ sanitize_hot_paths (bool walk_up, unsigned int cold_bb_count,
 
 	  /* Do not expect profile insanities when profile was not adjusted.  */
 	  if (e->probability == profile_probability::never ()
-	      || e->count == profile_count::zero ())
+	      || e->count () == profile_count::zero ())
 	    continue;
 
           if (BB_PARTITION (reach_bb) != BB_COLD_PARTITION)
@@ -1539,11 +1544,8 @@ sanitize_hot_paths (bool walk_up, unsigned int cold_bb_count,
           /* The following loop will look for the hottest edge via
              the edge count, if it is non-zero, then fallback to the edge
              frequency and finally the edge probability.  */
-          if (!highest_count.initialized_p () || e->count > highest_count)
-            highest_count = e->count;
-          int edge_freq = EDGE_FREQUENCY (e);
-          if (edge_freq > highest_freq)
-            highest_freq = edge_freq;
+          if (!(e->count () > highest_count))
+            highest_count = e->count ();
           if (!highest_probability.initialized_p ()
 	      || e->probability > highest_probability)
             highest_probability = e->probability;
@@ -1563,22 +1565,17 @@ sanitize_hot_paths (bool walk_up, unsigned int cold_bb_count,
             continue;
 	  /* Do not expect profile insanities when profile was not adjusted.  */
 	  if (e->probability == profile_probability::never ()
-	      || e->count == profile_count::zero ())
+	      || e->count () == profile_count::zero ())
 	    continue;
           /* Select the hottest edge using the edge count, if it is non-zero,
              then fallback to the edge frequency and finally the edge
              probability.  */
-          if (highest_count > 0)
-            {
-              if (e->count < highest_count)
-                continue;
-            }
-          else if (highest_freq)
+          if (highest_count.initialized_p ())
             {
-              if (EDGE_FREQUENCY (e) < highest_freq)
+              if (!(e->count () >= highest_count))
                 continue;
             }
-          else if (e->probability < highest_probability)
+          else if (!(e->probability >= highest_probability))
             continue;
 
           basic_block reach_bb = walk_up ? e->src : e->dest;
diff --git a/gcc/brig/ChangeLog b/gcc/brig/ChangeLog
index fa7668486b2..dede3f405f5 100644
--- a/gcc/brig/ChangeLog
+++ b/gcc/brig/ChangeLog
@@ -1,3 +1,12 @@
+2017-10-31  Henry Linjamäki  <henry.linjamaki@parmance.com>
+
+	* brig-lang.c (brig_langhook_type_for_mode): Fix PR 82771.
+
+2017-10-23  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* brig-lang.c (brig_langhook_type_for_mode): Use scalar_int_mode
+	and scalar_float_mode.
+
 2017-10-09  Pekka Jääskeläinen  <pekka.jaaskelainen@parmance.com>
 
 	* brigfrontend/brig-to-generic.cc: Support BRIG_KIND_NONE
diff --git a/gcc/brig/brig-lang.c b/gcc/brig/brig-lang.c
index cff605541d0..f34d9587632 100644
--- a/gcc/brig/brig-lang.c
+++ b/gcc/brig/brig-lang.c
@@ -280,9 +280,9 @@ brig_langhook_type_for_mode (machine_mode mode, int unsignedp)
 
   scalar_int_mode imode;
   scalar_float_mode fmode;
-  if (is_int_mode (mode, &imode))
+  if (is_float_mode (mode, &fmode))
     {
-      switch (GET_MODE_BITSIZE (imode))
+      switch (GET_MODE_BITSIZE (fmode))
 	{
 	case 32:
 	  return float_type_node;
@@ -291,15 +291,15 @@ brig_langhook_type_for_mode (machine_mode mode, int unsignedp)
 	default:
 	  /* We have to check for long double in order to support
 	     i386 excess precision.  */
-	  if (imode == TYPE_MODE (long_double_type_node))
+	  if (fmode == TYPE_MODE (long_double_type_node))
 	    return long_double_type_node;
 
 	  gcc_unreachable ();
 	  return NULL_TREE;
 	}
     }
-  else if (is_float_mode (mode, &fmode))
-    return brig_langhook_type_for_size (GET_MODE_BITSIZE (fmode), unsignedp);
+  else if (is_int_mode (mode, &imode))
+    return brig_langhook_type_for_size (GET_MODE_BITSIZE (imode), unsignedp);
   else
     {
       /* E.g., build_common_builtin_nodes () asks for modes/builtins
diff --git a/gcc/bt-load.c b/gcc/bt-load.c
index 1da0ad62f1e..90922082e7b 100644
--- a/gcc/bt-load.c
+++ b/gcc/bt-load.c
@@ -185,7 +185,7 @@ static int first_btr, last_btr;
 static int
 basic_block_freq (const_basic_block bb)
 {
-  return bb->frequency;
+  return bb->count.to_frequency (cfun);
 }
 
 /* If the rtx at *XP references (sets or reads) any branch target
diff --git a/gcc/builtin-types.def b/gcc/builtin-types.def
index 18946033502..877f2aef424 100644
--- a/gcc/builtin-types.def
+++ b/gcc/builtin-types.def
@@ -544,6 +544,20 @@ DEF_FUNCTION_TYPE_3 (BT_FN_DOUBLE_DOUBLE_DOUBLE_DOUBLE,
 		     BT_DOUBLE, BT_DOUBLE, BT_DOUBLE, BT_DOUBLE)
 DEF_FUNCTION_TYPE_3 (BT_FN_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE,
 		     BT_LONGDOUBLE, BT_LONGDOUBLE, BT_LONGDOUBLE, BT_LONGDOUBLE)
+DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT16_FLOAT16_FLOAT16_FLOAT16,
+		     BT_FLOAT16, BT_FLOAT16, BT_FLOAT16, BT_FLOAT16)
+DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT32_FLOAT32_FLOAT32_FLOAT32,
+		     BT_FLOAT32, BT_FLOAT32, BT_FLOAT32, BT_FLOAT32)
+DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT64_FLOAT64_FLOAT64_FLOAT64,
+		     BT_FLOAT64, BT_FLOAT64, BT_FLOAT64, BT_FLOAT64)
+DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT128_FLOAT128_FLOAT128_FLOAT128,
+		     BT_FLOAT128, BT_FLOAT128, BT_FLOAT128, BT_FLOAT128)
+DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT32X_FLOAT32X_FLOAT32X_FLOAT32X,
+		     BT_FLOAT32X, BT_FLOAT32X, BT_FLOAT32X, BT_FLOAT32X)
+DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT64X_FLOAT64X_FLOAT64X_FLOAT64X,
+		     BT_FLOAT64X, BT_FLOAT64X, BT_FLOAT64X, BT_FLOAT64X)
+DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT128X_FLOAT128X_FLOAT128X_FLOAT128X,
+		     BT_FLOAT128X, BT_FLOAT128X, BT_FLOAT128X, BT_FLOAT128X)
 DEF_FUNCTION_TYPE_3 (BT_FN_FLOAT_FLOAT_FLOAT_INTPTR,
 		     BT_FLOAT, BT_FLOAT, BT_FLOAT, BT_INT_PTR)
 DEF_FUNCTION_TYPE_3 (BT_FN_DOUBLE_DOUBLE_DOUBLE_INTPTR,
diff --git a/gcc/builtins.c b/gcc/builtins.c
index c50d7f43f76..b0fe2a42980 100644
--- a/gcc/builtins.c
+++ b/gcc/builtins.c
@@ -1208,6 +1208,7 @@ void
 expand_builtin_update_setjmp_buf (rtx buf_addr)
 {
   machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
+  buf_addr = convert_memory_address (Pmode, buf_addr);
   rtx stack_save
     = gen_rtx_MEM (sa_mode,
 		   memory_address
@@ -1615,7 +1616,7 @@ expand_builtin_apply (rtx function, rtx arguments, rtx argsize)
      arguments to the outgoing arguments address.  We can pass TRUE
      as the 4th argument because we just saved the stack pointer
      and will restore it right after the call.  */
-  allocate_dynamic_stack_space (argsize, 0, BIGGEST_ALIGNMENT, true);
+  allocate_dynamic_stack_space (argsize, 0, BIGGEST_ALIGNMENT, -1, true);
 
   /* Set DRAP flag to true, even though allocate_dynamic_stack_space
      may have already set current_function_calls_alloca to true.
@@ -1822,14 +1823,26 @@ expand_builtin_classify_type (tree exp)
   return GEN_INT (no_type_class);
 }
 
-/* This helper macro, meant to be used in mathfn_built_in below,
-   determines which among a set of three builtin math functions is
-   appropriate for a given type mode.  The `F' and `L' cases are
-   automatically generated from the `double' case.  */
+/* This helper macro, meant to be used in mathfn_built_in below, determines
+   which among a set of builtin math functions is appropriate for a given type
+   mode.  The `F' (float) and `L' (long double) are automatically generated
+   from the 'double' case.  If a function supports the _Float<N> and _Float<N>X
+   types, there are additional types that are considered with 'F32', 'F64',
+   'F128', etc. suffixes.  */
 #define CASE_MATHFN(MATHFN) \
   CASE_CFN_##MATHFN: \
   fcode = BUILT_IN_##MATHFN; fcodef = BUILT_IN_##MATHFN##F ; \
   fcodel = BUILT_IN_##MATHFN##L ; break;
+/* Similar to the above, but also add support for the _Float<N> and _Float<N>X
+   types.  */
+#define CASE_MATHFN_FLOATN(MATHFN) \
+  CASE_CFN_##MATHFN: \
+  fcode = BUILT_IN_##MATHFN; fcodef = BUILT_IN_##MATHFN##F ; \
+  fcodel = BUILT_IN_##MATHFN##L ; fcodef16 = BUILT_IN_##MATHFN##F16 ; \
+  fcodef32 = BUILT_IN_##MATHFN##F32; fcodef64 = BUILT_IN_##MATHFN##F64 ; \
+  fcodef128 = BUILT_IN_##MATHFN##F128 ; fcodef32x = BUILT_IN_##MATHFN##F32X ; \
+  fcodef64x = BUILT_IN_##MATHFN##F64X ; fcodef128x = BUILT_IN_##MATHFN##F128X ;\
+  break;
 /* Similar to above, but appends _R after any F/L suffix.  */
 #define CASE_MATHFN_REENT(MATHFN) \
   case CFN_BUILT_IN_##MATHFN##_R: \
@@ -1846,7 +1859,15 @@ expand_builtin_classify_type (tree exp)
 static built_in_function
 mathfn_built_in_2 (tree type, combined_fn fn)
 {
+  tree mtype;
   built_in_function fcode, fcodef, fcodel;
+  built_in_function fcodef16 = END_BUILTINS;
+  built_in_function fcodef32 = END_BUILTINS;
+  built_in_function fcodef64 = END_BUILTINS;
+  built_in_function fcodef128 = END_BUILTINS;
+  built_in_function fcodef32x = END_BUILTINS;
+  built_in_function fcodef64x = END_BUILTINS;
+  built_in_function fcodef128x = END_BUILTINS;
 
   switch (fn)
     {
@@ -1860,7 +1881,7 @@ mathfn_built_in_2 (tree type, combined_fn fn)
     CASE_MATHFN (CBRT)
     CASE_MATHFN (CEIL)
     CASE_MATHFN (CEXPI)
-    CASE_MATHFN (COPYSIGN)
+    CASE_MATHFN_FLOATN (COPYSIGN)
     CASE_MATHFN (COS)
     CASE_MATHFN (COSH)
     CASE_MATHFN (DREM)
@@ -1873,9 +1894,9 @@ mathfn_built_in_2 (tree type, combined_fn fn)
     CASE_MATHFN (FABS)
     CASE_MATHFN (FDIM)
     CASE_MATHFN (FLOOR)
-    CASE_MATHFN (FMA)
-    CASE_MATHFN (FMAX)
-    CASE_MATHFN (FMIN)
+    CASE_MATHFN_FLOATN (FMA)
+    CASE_MATHFN_FLOATN (FMAX)
+    CASE_MATHFN_FLOATN (FMIN)
     CASE_MATHFN (FMOD)
     CASE_MATHFN (FREXP)
     CASE_MATHFN (GAMMA)
@@ -1929,7 +1950,7 @@ mathfn_built_in_2 (tree type, combined_fn fn)
     CASE_MATHFN (SIN)
     CASE_MATHFN (SINCOS)
     CASE_MATHFN (SINH)
-    CASE_MATHFN (SQRT)
+    CASE_MATHFN_FLOATN (SQRT)
     CASE_MATHFN (TAN)
     CASE_MATHFN (TANH)
     CASE_MATHFN (TGAMMA)
@@ -1942,12 +1963,27 @@ mathfn_built_in_2 (tree type, combined_fn fn)
       return END_BUILTINS;
     }
 
-  if (TYPE_MAIN_VARIANT (type) == double_type_node)
+  mtype = TYPE_MAIN_VARIANT (type);
+  if (mtype == double_type_node)
     return fcode;
-  else if (TYPE_MAIN_VARIANT (type) == float_type_node)
+  else if (mtype == float_type_node)
     return fcodef;
-  else if (TYPE_MAIN_VARIANT (type) == long_double_type_node)
+  else if (mtype == long_double_type_node)
     return fcodel;
+  else if (mtype == float16_type_node)
+    return fcodef16;
+  else if (mtype == float32_type_node)
+    return fcodef32;
+  else if (mtype == float64_type_node)
+    return fcodef64;
+  else if (mtype == float128_type_node)
+    return fcodef128;
+  else if (mtype == float32x_type_node)
+    return fcodef32x;
+  else if (mtype == float64x_type_node)
+    return fcodef64x;
+  else if (mtype == float128x_type_node)
+    return fcodef128x;
   else
     return END_BUILTINS;
 }
@@ -2001,6 +2037,9 @@ associated_internal_fn (tree fndecl)
     {
 #define DEF_INTERNAL_FLT_FN(NAME, FLAGS, OPTAB, TYPE) \
     CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME;
+#define DEF_INTERNAL_FLT_FLOATN_FN(NAME, FLAGS, OPTAB, TYPE) \
+    CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME; \
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_##NAME): return IFN_##NAME;
 #define DEF_INTERNAL_INT_FN(NAME, FLAGS, OPTAB, TYPE) \
     CASE_INT_FN (BUILT_IN_##NAME): return IFN_##NAME;
 #include "internal-fn.def"
@@ -2074,6 +2113,7 @@ expand_builtin_mathfn_ternary (tree exp, rtx target, rtx subtarget)
   switch (DECL_FUNCTION_CODE (fndecl))
     {
     CASE_FLT_FN (BUILT_IN_FMA):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
       builtin_optab = fma_optab; break;
     default:
       gcc_unreachable ();
@@ -4864,19 +4904,22 @@ expand_builtin_alloca (tree exp)
   rtx result;
   unsigned int align;
   tree fndecl = get_callee_fndecl (exp);
-  bool alloca_with_align = (DECL_FUNCTION_CODE (fndecl)
-			    == BUILT_IN_ALLOCA_WITH_ALIGN);
+  HOST_WIDE_INT max_size;
+  enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
   bool alloca_for_var = CALL_ALLOCA_FOR_VAR_P (exp);
   bool valid_arglist
-    = (alloca_with_align
-       ? validate_arglist (exp, INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE)
-       : validate_arglist (exp, INTEGER_TYPE, VOID_TYPE));
+    = (fcode == BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX
+       ? validate_arglist (exp, INTEGER_TYPE, INTEGER_TYPE, INTEGER_TYPE,
+			   VOID_TYPE)
+       : fcode == BUILT_IN_ALLOCA_WITH_ALIGN
+	 ? validate_arglist (exp, INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE)
+	 : validate_arglist (exp, INTEGER_TYPE, VOID_TYPE));
 
   if (!valid_arglist)
     return NULL_RTX;
 
-  if ((alloca_with_align && !warn_vla_limit)
-      || (!alloca_with_align && !warn_alloca_limit))
+  if ((alloca_for_var && !warn_vla_limit)
+      || (!alloca_for_var && !warn_alloca_limit))
     {
       /* -Walloca-larger-than and -Wvla-larger-than settings override
 	 the more general -Walloc-size-larger-than so unless either of
@@ -4891,13 +4934,19 @@ expand_builtin_alloca (tree exp)
   op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
 
   /* Compute the alignment.  */
-  align = (alloca_with_align
-	   ? TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 1))
-	   : BIGGEST_ALIGNMENT);
+  align = (fcode == BUILT_IN_ALLOCA
+	   ? BIGGEST_ALIGNMENT
+	   : TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 1)));
+
+  /* Compute the maximum size.  */
+  max_size = (fcode == BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX
+              ? TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 2))
+              : -1);
 
   /* Allocate the desired space.  If the allocation stems from the declaration
      of a variable-sized object, it cannot accumulate.  */
-  result = allocate_dynamic_stack_space (op0, 0, align, alloca_for_var);
+  result
+    = allocate_dynamic_stack_space (op0, 0, align, max_size, alloca_for_var);
   result = convert_memory_address (ptr_mode, result);
 
   return result;
@@ -6491,8 +6540,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, machine_mode mode,
       && fcode != BUILT_IN_EXECLE
       && fcode != BUILT_IN_EXECVP
       && fcode != BUILT_IN_EXECVE
-      && fcode != BUILT_IN_ALLOCA
-      && fcode != BUILT_IN_ALLOCA_WITH_ALIGN
+      && !ALLOCA_FUNCTION_CODE_P (fcode)
       && fcode != BUILT_IN_FREE
       && fcode != BUILT_IN_CHKP_SET_PTR_BOUNDS
       && fcode != BUILT_IN_CHKP_INIT_PTR_BOUNDS
@@ -6568,6 +6616,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, machine_mode mode,
       break;
 
     CASE_FLT_FN (BUILT_IN_FMA):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
       target = expand_builtin_mathfn_ternary (exp, target, subtarget);
       if (target)
 	return target;
@@ -6721,8 +6770,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, machine_mode mode,
       else
 	return XEXP (DECL_RTL (DECL_RESULT (current_function_decl)), 0);
 
-    case BUILT_IN_ALLOCA:
-    case BUILT_IN_ALLOCA_WITH_ALIGN:
+    CASE_BUILT_IN_ALLOCA:
       target = expand_builtin_alloca (exp);
       if (target)
 	return target;
@@ -10416,8 +10464,7 @@ is_inexpensive_builtin (tree decl)
     switch (DECL_FUNCTION_CODE (decl))
       {
       case BUILT_IN_ABS:
-      case BUILT_IN_ALLOCA:
-      case BUILT_IN_ALLOCA_WITH_ALIGN:
+      CASE_BUILT_IN_ALLOCA:
       case BUILT_IN_BSWAP16:
       case BUILT_IN_BSWAP32:
       case BUILT_IN_BSWAP64:
diff --git a/gcc/builtins.def b/gcc/builtins.def
index 1c1efceea21..26118f1766b 100644
--- a/gcc/builtins.def
+++ b/gcc/builtins.def
@@ -128,6 +128,29 @@ along with GCC; see the file COPYING3.  If not see
   DEF_BUILTIN_CHKP (ENUM, "__builtin_" NAME, BUILT_IN_NORMAL, TYPE,	\
 		    TYPE, true, true, true, ATTRS, false, true)
 
+/* A set of GCC builtins for _FloatN and _FloatNx types.  TYPE_MACRO is called
+   with an argument such as FLOAT32 to produce the enum value for the type.  If
+   we are compiling for the C language with GNU extensions, we enable the name
+   without the __builtin_ prefix as well as the name with the __builtin_
+   prefix.  C++ does not enable these names by default because they don't have
+   the _Float<N> and _Float<N>X keywords, and a class based library should use
+   the __builtin_ names.  */
+#undef DEF_FLOATN_BUILTIN
+#define DEF_FLOATN_BUILTIN(ENUM, NAME, TYPE, ATTRS)	\
+  DEF_BUILTIN (ENUM, "__builtin_" NAME, BUILT_IN_NORMAL, TYPE, TYPE,	\
+	       targetm.floatn_builtin_p ((int) ENUM), true, true, ATTRS, \
+	       false, true)
+#undef DEF_EXT_LIB_FLOATN_NX_BUILTINS
+#define DEF_EXT_LIB_FLOATN_NX_BUILTINS(ENUM, NAME, TYPE_MACRO, ATTRS)	\
+  DEF_FLOATN_BUILTIN (ENUM ## F16, NAME "f16", TYPE_MACRO (FLOAT16), ATTRS) \
+  DEF_FLOATN_BUILTIN (ENUM ## F32, NAME "f32", TYPE_MACRO (FLOAT32), ATTRS) \
+  DEF_FLOATN_BUILTIN (ENUM ## F64, NAME "f64", TYPE_MACRO (FLOAT64), ATTRS) \
+  DEF_FLOATN_BUILTIN (ENUM ## F128, NAME "f128", TYPE_MACRO (FLOAT128), ATTRS) \
+  DEF_FLOATN_BUILTIN (ENUM ## F32X, NAME "f32x", TYPE_MACRO (FLOAT32X), ATTRS) \
+  DEF_FLOATN_BUILTIN (ENUM ## F64X, NAME "f64x", TYPE_MACRO (FLOAT64X), ATTRS) \
+  DEF_FLOATN_BUILTIN (ENUM ## F128X, NAME "f128x", TYPE_MACRO (FLOAT128X), \
+		      ATTRS)
+
 /* Like DEF_LIB_BUILTIN, except that the function is only a part of
    the standard in C94 or above.  */
 #undef DEF_C94_BUILTIN
@@ -324,7 +347,7 @@ DEF_C99_BUILTIN        (BUILT_IN_COPYSIGN, "copysign", BT_FN_DOUBLE_DOUBLE_DOUBL
 DEF_C99_BUILTIN        (BUILT_IN_COPYSIGNF, "copysignf", BT_FN_FLOAT_FLOAT_FLOAT, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_C99_BUILTIN        (BUILT_IN_COPYSIGNL, "copysignl", BT_FN_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE, ATTR_CONST_NOTHROW_LEAF_LIST)
 #define COPYSIGN_TYPE(F) BT_FN_##F##_##F##_##F
-DEF_GCC_FLOATN_NX_BUILTINS (BUILT_IN_COPYSIGN, "copysign", COPYSIGN_TYPE, ATTR_CONST_NOTHROW_LEAF_LIST)
+DEF_EXT_LIB_FLOATN_NX_BUILTINS (BUILT_IN_COPYSIGN, "copysign", COPYSIGN_TYPE, ATTR_CONST_NOTHROW_LEAF_LIST)
 #undef COPYSIGN_TYPE
 DEF_LIB_BUILTIN        (BUILT_IN_COS, "cos", BT_FN_DOUBLE_DOUBLE, ATTR_MATHFN_FPROUNDING)
 DEF_C99_C90RES_BUILTIN (BUILT_IN_COSF, "cosf", BT_FN_FLOAT_FLOAT, ATTR_MATHFN_FPROUNDING)
@@ -357,7 +380,7 @@ DEF_LIB_BUILTIN        (BUILT_IN_FABS, "fabs", BT_FN_DOUBLE_DOUBLE, ATTR_CONST_N
 DEF_C99_C90RES_BUILTIN (BUILT_IN_FABSF, "fabsf", BT_FN_FLOAT_FLOAT, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_C99_C90RES_BUILTIN (BUILT_IN_FABSL, "fabsl", BT_FN_LONGDOUBLE_LONGDOUBLE, ATTR_CONST_NOTHROW_LEAF_LIST)
 #define FABS_TYPE(F) BT_FN_##F##_##F
-DEF_GCC_FLOATN_NX_BUILTINS (BUILT_IN_FABS, "fabs", FABS_TYPE, ATTR_CONST_NOTHROW_LEAF_LIST)
+DEF_EXT_LIB_FLOATN_NX_BUILTINS (BUILT_IN_FABS, "fabs", FABS_TYPE, ATTR_CONST_NOTHROW_LEAF_LIST)
 #undef FABS_TYPE
 DEF_GCC_BUILTIN        (BUILT_IN_FABSD32, "fabsd32", BT_FN_DFLOAT32_DFLOAT32, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_GCC_BUILTIN        (BUILT_IN_FABSD64, "fabsd64", BT_FN_DFLOAT64_DFLOAT64, ATTR_CONST_NOTHROW_LEAF_LIST)
@@ -382,12 +405,21 @@ DEF_C99_C90RES_BUILTIN (BUILT_IN_FLOORL, "floorl", BT_FN_LONGDOUBLE_LONGDOUBLE,
 DEF_C99_BUILTIN        (BUILT_IN_FMA, "fma", BT_FN_DOUBLE_DOUBLE_DOUBLE_DOUBLE, ATTR_MATHFN_FPROUNDING)
 DEF_C99_BUILTIN        (BUILT_IN_FMAF, "fmaf", BT_FN_FLOAT_FLOAT_FLOAT_FLOAT, ATTR_MATHFN_FPROUNDING)
 DEF_C99_BUILTIN        (BUILT_IN_FMAL, "fmal", BT_FN_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE, ATTR_MATHFN_FPROUNDING)
+#define FMA_TYPE(F) BT_FN_##F##_##F##_##F##_##F
+DEF_EXT_LIB_FLOATN_NX_BUILTINS (BUILT_IN_FMA, "fma", FMA_TYPE, ATTR_MATHFN_FPROUNDING)
+#undef FMA_TYPE
 DEF_C99_BUILTIN        (BUILT_IN_FMAX, "fmax", BT_FN_DOUBLE_DOUBLE_DOUBLE, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_C99_BUILTIN        (BUILT_IN_FMAXF, "fmaxf", BT_FN_FLOAT_FLOAT_FLOAT, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_C99_BUILTIN        (BUILT_IN_FMAXL, "fmaxl", BT_FN_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE, ATTR_CONST_NOTHROW_LEAF_LIST)
+#define FMAX_TYPE(F) BT_FN_##F##_##F##_##F
+DEF_EXT_LIB_FLOATN_NX_BUILTINS (BUILT_IN_FMAX, "fmax", FMAX_TYPE, ATTR_CONST_NOTHROW_LEAF_LIST)
+#undef FMAX_TYPE
 DEF_C99_BUILTIN        (BUILT_IN_FMIN, "fmin", BT_FN_DOUBLE_DOUBLE_DOUBLE, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_C99_BUILTIN        (BUILT_IN_FMINF, "fminf", BT_FN_FLOAT_FLOAT_FLOAT, ATTR_CONST_NOTHROW_LEAF_LIST)
 DEF_C99_BUILTIN        (BUILT_IN_FMINL, "fminl", BT_FN_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE, ATTR_CONST_NOTHROW_LEAF_LIST)
+#define FMIN_TYPE(F) BT_FN_##F##_##F##_##F
+DEF_EXT_LIB_FLOATN_NX_BUILTINS (BUILT_IN_FMIN, "fmin", FMIN_TYPE, ATTR_CONST_NOTHROW_LEAF_LIST)
+#undef FMIN_TYPE
 DEF_LIB_BUILTIN        (BUILT_IN_FMOD, "fmod", BT_FN_DOUBLE_DOUBLE_DOUBLE, ATTR_MATHFN_FPROUNDING_ERRNO)
 DEF_C99_C90RES_BUILTIN (BUILT_IN_FMODF, "fmodf", BT_FN_FLOAT_FLOAT_FLOAT, ATTR_MATHFN_FPROUNDING_ERRNO)
 DEF_C99_C90RES_BUILTIN (BUILT_IN_FMODL, "fmodl", BT_FN_LONGDOUBLE_LONGDOUBLE_LONGDOUBLE, ATTR_MATHFN_FPROUNDING_ERRNO)
@@ -495,7 +527,7 @@ DEF_C99_BUILTIN        (BUILT_IN_NAN, "nan", BT_FN_DOUBLE_CONST_STRING, ATTR_CON
 DEF_C99_BUILTIN        (BUILT_IN_NANF, "nanf", BT_FN_FLOAT_CONST_STRING, ATTR_CONST_NOTHROW_NONNULL)
 DEF_C99_BUILTIN        (BUILT_IN_NANL, "nanl", BT_FN_LONGDOUBLE_CONST_STRING, ATTR_CONST_NOTHROW_NONNULL)
 #define NAN_TYPE(F) BT_FN_##F##_CONST_STRING
-DEF_GCC_FLOATN_NX_BUILTINS (BUILT_IN_NAN, "nan", NAN_TYPE, ATTR_CONST_NOTHROW_NONNULL)
+DEF_EXT_LIB_FLOATN_NX_BUILTINS (BUILT_IN_NAN, "nan", NAN_TYPE, ATTR_CONST_NOTHROW_NONNULL)
 DEF_GCC_BUILTIN        (BUILT_IN_NAND32, "nand32", BT_FN_DFLOAT32_CONST_STRING, ATTR_CONST_NOTHROW_NONNULL)
 DEF_GCC_BUILTIN        (BUILT_IN_NAND64, "nand64", BT_FN_DFLOAT64_CONST_STRING, ATTR_CONST_NOTHROW_NONNULL)
 DEF_GCC_BUILTIN        (BUILT_IN_NAND128, "nand128", BT_FN_DFLOAT128_CONST_STRING, ATTR_CONST_NOTHROW_NONNULL)
@@ -564,6 +596,9 @@ DEF_C99_C90RES_BUILTIN (BUILT_IN_SINL, "sinl", BT_FN_LONGDOUBLE_LONGDOUBLE, ATTR
 DEF_LIB_BUILTIN        (BUILT_IN_SQRT, "sqrt", BT_FN_DOUBLE_DOUBLE, ATTR_MATHFN_FPROUNDING_ERRNO)
 DEF_C99_C90RES_BUILTIN (BUILT_IN_SQRTF, "sqrtf", BT_FN_FLOAT_FLOAT, ATTR_MATHFN_FPROUNDING_ERRNO)
 DEF_C99_C90RES_BUILTIN (BUILT_IN_SQRTL, "sqrtl", BT_FN_LONGDOUBLE_LONGDOUBLE, ATTR_MATHFN_FPROUNDING_ERRNO)
+#define SQRT_TYPE(F) BT_FN_##F##_##F
+DEF_EXT_LIB_FLOATN_NX_BUILTINS (BUILT_IN_SQRT, "sqrt", SQRT_TYPE, ATTR_MATHFN_FPROUNDING_ERRNO)
+#undef SQRT_TYPE
 DEF_LIB_BUILTIN        (BUILT_IN_TAN, "tan", BT_FN_DOUBLE_DOUBLE, ATTR_MATHFN_FPROUNDING)
 DEF_C99_C90RES_BUILTIN (BUILT_IN_TANF, "tanf", BT_FN_FLOAT_FLOAT, ATTR_MATHFN_FPROUNDING)
 DEF_LIB_BUILTIN        (BUILT_IN_TANH, "tanh", BT_FN_DOUBLE_DOUBLE, ATTR_MATHFN_FPROUNDING)
@@ -921,6 +956,7 @@ DEF_BUILTIN_STUB (BUILT_IN_SETJMP_RECEIVER, "__builtin_setjmp_receiver")
 DEF_BUILTIN_STUB (BUILT_IN_STACK_SAVE, "__builtin_stack_save")
 DEF_BUILTIN_STUB (BUILT_IN_STACK_RESTORE, "__builtin_stack_restore")
 DEF_BUILTIN_STUB (BUILT_IN_ALLOCA_WITH_ALIGN, "__builtin_alloca_with_align")
+DEF_BUILTIN_STUB (BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX, "__builtin_alloca_with_align_and_max")
 
 /* An internal version of memcmp, used when the result is only tested for
    equality with zero.  */
diff --git a/gcc/c-family/ChangeLog b/gcc/c-family/ChangeLog
index ee6fc87dd6f..e8476426fcb 100644
--- a/gcc/c-family/ChangeLog
+++ b/gcc/c-family/ChangeLog
@@ -1,3 +1,52 @@
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.c (binary_op_error): Update for renaming of
+	error_at_rich_loc.
+	(c_parse_error): Likewise.
+	* c-warn.c (warn_logical_not_parentheses): Likewise for
+	renaming of inform_at_rich_loc.
+	(warn_for_restrict): Likewise for renaming of
+	warning_at_rich_loc_n.
+
+2017-10-30  Joseph Myers  <joseph@codesourcery.com>
+
+	* c.opt (std=c17, std=gnu17, std=iso9899:2017): New options.
+	* c-opts.c (set_std_c17): New function.
+	(c_common_init_options): Use gnu17 as default C version.
+	(c_common_handle_option): Handle -std=c17 and -std=gnu17.
+
+2017-10-27  Michael Meissner  <meissner@linux.vnet.ibm.com>
+
+	* c-cppbuiltin.c (mode_has_fma): Add support for PowerPC KFmode.
+	(c_cpp_builtins): If a machine has a fast fma _Float<N> and
+	_Float<N>X variant, define __FP_FAST_FMA<N> and/or
+	__FP_FAST_FMA<N>X.
+
+2017-10-23  Marek Polacek  <polacek@redhat.com>
+
+	PR c/82681
+	* c-warn.c (warnings_for_convert_and_check): Fix typos.
+
+2017-10-19  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* c-common.c (check_builtin_function_arguments): Also check arguments
+	of __builtin_alloca_with_align_and_max.
+
+2017-10-17  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-format.c (format_warning_at_char): Pass UNKNOWN_LOCATION
+	rather than NULL to format_warning_va.
+	(check_format_types): Likewise when calling format_type_warning.
+	Remove code to extract source_ranges and source_range * in favor
+	of just a location_t.
+	(format_type_warning): Convert source_range * param to a
+	location_t.
+
+2017-10-13  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-gimplify.c (c_gimplify_expr): Handle [LR]ROTATE_EXPR like
+	[LR]SHIFT_EXPR.
+
 2017-10-12  David Malcolm  <dmalcolm@redhat.com>
 
 	* c-common.c (enum missing_token_insertion_kind): New enum.
diff --git a/gcc/c-family/c-attribs.c b/gcc/c-family/c-attribs.c
index bd8ca306c2d..bb75cba4c39 100644
--- a/gcc/c-family/c-attribs.c
+++ b/gcc/c-family/c-attribs.c
@@ -65,6 +65,7 @@ static tree handle_asan_odr_indicator_attribute (tree *, tree, tree, int,
 static tree handle_stack_protect_attribute (tree *, tree, tree, int, bool *);
 static tree handle_noinline_attribute (tree *, tree, tree, int, bool *);
 static tree handle_noclone_attribute (tree *, tree, tree, int, bool *);
+static tree handle_nocf_check_attribute (tree *, tree, tree, int, bool *);
 static tree handle_noicf_attribute (tree *, tree, tree, int, bool *);
 static tree handle_noipa_attribute (tree *, tree, tree, int, bool *);
 static tree handle_leaf_attribute (tree *, tree, tree, int, bool *);
@@ -367,6 +368,8 @@ const struct attribute_spec c_common_attribute_table[] =
   { "patchable_function_entry",	1, 2, true, false, false,
 			      handle_patchable_function_entry_attribute,
 			      false },
+  { "nocf_check",		      0, 0, false, true, true,
+			      handle_nocf_check_attribute, true },
   { NULL,                     0, 0, false, false, false, NULL, false }
 };
 
@@ -772,6 +775,30 @@ handle_noclone_attribute (tree *node, tree name,
   return NULL_TREE;
 }
 
+/* Handle a "nocf_check" attribute; arguments as in
+   struct attribute_spec.handler.  */
+
+static tree
+handle_nocf_check_attribute (tree *node, tree name,
+			  tree ARG_UNUSED (args),
+			  int ARG_UNUSED (flags), bool *no_add_attrs)
+{
+  if (TREE_CODE (*node) != FUNCTION_TYPE
+      && TREE_CODE (*node) != METHOD_TYPE)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
+  else if (!(flag_cf_protection & CF_BRANCH))
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored. Use "
+				"-fcf-protection option to enable it", name);
+      *no_add_attrs = true;
+    }
+
+  return NULL_TREE;
+}
+
 /* Handle a "no_icf" attribute; arguments as in
    struct attribute_spec.handler.  */
 
diff --git a/gcc/c-family/c-common.c b/gcc/c-family/c-common.c
index 1f2bf646e76..83c6aadda27 100644
--- a/gcc/c-family/c-common.c
+++ b/gcc/c-family/c-common.c
@@ -2704,9 +2704,9 @@ binary_op_error (rich_location *richloc, enum tree_code code,
     default:
       gcc_unreachable ();
     }
-  error_at_rich_loc (richloc,
-		     "invalid operands to binary %s (have %qT and %qT)",
-		     opname, type0, type1);
+  error_at (richloc,
+	    "invalid operands to binary %s (have %qT and %qT)",
+	    opname, type0, type1);
 }
 
 /* Given an expression as a tree, return its original type.  Do this
@@ -5705,6 +5705,16 @@ check_builtin_function_arguments (location_t loc, vec<location_t> arg_loc,
 
   switch (DECL_FUNCTION_CODE (fndecl))
     {
+    case BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX:
+      if (!tree_fits_uhwi_p (args[2]))
+	{
+	  error_at (ARG_LOCATION (2),
+		    "third argument to function %qE must be a constant integer",
+		    fndecl);
+	  return false;
+	}
+      /* fall through */
+
     case BUILT_IN_ALLOCA_WITH_ALIGN:
       {
 	/* Get the requested alignment (in bits) if it's a constant
@@ -5944,7 +5954,7 @@ c_parse_error (const char *gmsgid, enum cpp_ttype token_type,
       else
 	message = catenate_messages (gmsgid, " before %s'\\x%x'");
 
-      error_at_rich_loc (richloc, message, prefix, val);
+      error_at (richloc, message, prefix, val);
       free (message);
       message = NULL;
     }
@@ -5972,7 +5982,7 @@ c_parse_error (const char *gmsgid, enum cpp_ttype token_type,
   else if (token_type == CPP_NAME)
     {
       message = catenate_messages (gmsgid, " before %qE");
-      error_at_rich_loc (richloc, message, value);
+      error_at (richloc, message, value);
       free (message);
       message = NULL;
     }
@@ -5985,16 +5995,16 @@ c_parse_error (const char *gmsgid, enum cpp_ttype token_type,
   else if (token_type < N_TTYPES)
     {
       message = catenate_messages (gmsgid, " before %qs token");
-      error_at_rich_loc (richloc, message, cpp_type2name (token_type, token_flags));
+      error_at (richloc, message, cpp_type2name (token_type, token_flags));
       free (message);
       message = NULL;
     }
   else
-    error_at_rich_loc (richloc, gmsgid);
+    error_at (richloc, gmsgid);
 
   if (message)
     {
-      error_at_rich_loc (richloc, message);
+      error_at (richloc, message);
       free (message);
     }
 #undef catenate_messages
diff --git a/gcc/c-family/c-cppbuiltin.c b/gcc/c-family/c-cppbuiltin.c
index 4330c9102d9..2ac9616b72f 100644
--- a/gcc/c-family/c-cppbuiltin.c
+++ b/gcc/c-family/c-cppbuiltin.c
@@ -82,6 +82,11 @@ mode_has_fma (machine_mode mode)
       return !!HAVE_fmadf4;
 #endif
 
+#ifdef HAVE_fmakf4	/* PowerPC if long double != __float128.  */
+    case E_KFmode:
+      return !!HAVE_fmakf4;
+#endif
+
 #ifdef HAVE_fmaxf4
     case E_XFmode:
       return !!HAVE_fmaxf4;
@@ -1119,7 +1124,7 @@ c_cpp_builtins (cpp_reader *pfile)
 	       floatn_nx_types[i].extended ? "X" : "");
       sprintf (csuffix, "F%d%s", floatn_nx_types[i].n,
 	       floatn_nx_types[i].extended ? "x" : "");
-      builtin_define_float_constants (prefix, csuffix, "%s", NULL,
+      builtin_define_float_constants (prefix, csuffix, "%s", csuffix,
 				      FLOATN_NX_TYPE_NODE (i));
     }
 
diff --git a/gcc/c-family/c-format.c b/gcc/c-family/c-format.c
index 0dba9793311..164d0353967 100644
--- a/gcc/c-family/c-format.c
+++ b/gcc/c-family/c-format.c
@@ -97,7 +97,8 @@ format_warning_at_char (location_t fmt_string_loc, tree format_string_cst,
 
   substring_loc fmt_loc (fmt_string_loc, string_type, char_idx, char_idx,
 			 char_idx);
-  bool warned = format_warning_va (fmt_loc, NULL, NULL, opt, gmsgid, &ap);
+  bool warned = format_warning_va (fmt_loc, UNKNOWN_LOCATION, NULL, opt,
+				   gmsgid, &ap);
   va_end (ap);
 
   return warned;
@@ -1039,7 +1040,7 @@ static void check_format_types (const substring_loc &fmt_loc,
 				char conversion_char,
 				vec<location_t> *arglocs);
 static void format_type_warning (const substring_loc &fmt_loc,
-				 source_range *param_range,
+				 location_t param_loc,
 				 format_wanted_type *, tree,
 				 tree,
 				 const format_kind_info *fki,
@@ -3073,8 +3074,9 @@ check_format_types (const substring_loc &fmt_loc,
       cur_param = types->param;
       if (!cur_param)
         {
-	  format_type_warning (fmt_loc, NULL, types, wanted_type, NULL, fki,
-			       offset_to_type_start, conversion_char);
+	  format_type_warning (fmt_loc, UNKNOWN_LOCATION, types, wanted_type,
+			       NULL, fki, offset_to_type_start,
+			       conversion_char);
           continue;
         }
 
@@ -3084,23 +3086,15 @@ check_format_types (const substring_loc &fmt_loc,
       orig_cur_type = cur_type;
       char_type_flag = 0;
 
-      source_range param_range;
-      source_range *param_range_ptr;
+      location_t param_loc = UNKNOWN_LOCATION;
       if (EXPR_HAS_LOCATION (cur_param))
-	{
-	  param_range = EXPR_LOCATION_RANGE (cur_param);
-	  param_range_ptr = &param_range;
-	}
+	param_loc = EXPR_LOCATION (cur_param);
       else if (arglocs)
 	{
 	  /* arg_num is 1-based.  */
 	  gcc_assert (types->arg_num > 0);
-	  location_t param_loc = (*arglocs)[types->arg_num - 1];
-	  param_range = get_range_from_loc (line_table, param_loc);
-	  param_range_ptr = &param_range;
+	  param_loc = (*arglocs)[types->arg_num - 1];
 	}
-      else
-	param_range_ptr = NULL;
 
       STRIP_NOPS (cur_param);
 
@@ -3166,7 +3160,7 @@ check_format_types (const substring_loc &fmt_loc,
 	    }
 	  else
 	    {
-	      format_type_warning (fmt_loc, param_range_ptr,
+	      format_type_warning (fmt_loc, param_loc,
 				   types, wanted_type, orig_cur_type, fki,
 				   offset_to_type_start, conversion_char);
 	      break;
@@ -3236,7 +3230,7 @@ check_format_types (const substring_loc &fmt_loc,
 	  && TYPE_PRECISION (cur_type) == TYPE_PRECISION (wanted_type))
 	continue;
       /* Now we have a type mismatch.  */
-      format_type_warning (fmt_loc, param_range_ptr, types,
+      format_type_warning (fmt_loc, param_loc, types,
 			   wanted_type, orig_cur_type, fki,
 			   offset_to_type_start, conversion_char);
     }
@@ -3544,8 +3538,9 @@ get_corrected_substring (const substring_loc &fmt_loc,
 /* Give a warning about a format argument of different type from that expected.
    The range of the diagnostic is taken from WHOLE_FMT_LOC; the caret location
    is based on the location of the char at TYPE->offset_loc.
-   If non-NULL, PARAM_RANGE is the source range of the
-   relevant argument.  WANTED_TYPE is the type the argument should have,
+   PARAM_LOC is the location of the relevant argument, or UNKNOWN_LOCATION
+   if this is unavailable.
+   WANTED_TYPE is the type the argument should have,
    possibly stripped of pointer dereferences.  The description (such as "field
    precision"), the placement in the format string, a possibly more
    friendly name of WANTED_TYPE, and the number of pointer dereferences
@@ -3566,7 +3561,7 @@ get_corrected_substring (const substring_loc &fmt_loc,
                           V~~~~~~~~ : range of WHOLE_FMT_LOC, from cols 23-31
       sprintf (d, "before %-+*.*lld after", int_expr, int_expr, long_expr);
                                 ^ ^                             ^~~~~~~~~
-                                | ` CONVERSION_CHAR: 'd'        *PARAM_RANGE
+                                | ` CONVERSION_CHAR: 'd'        PARAM_LOC
                                 type starts here
 
    OFFSET_TO_TYPE_START is 13, the offset to the "lld" within the
@@ -3574,7 +3569,7 @@ get_corrected_substring (const substring_loc &fmt_loc,
 
 static void
 format_type_warning (const substring_loc &whole_fmt_loc,
-		     source_range *param_range,
+		     location_t param_loc,
 		     format_wanted_type *type,
 		     tree wanted_type, tree arg_type,
 		     const format_kind_info *fki,
@@ -3636,7 +3631,7 @@ format_type_warning (const substring_loc &whole_fmt_loc,
     {
       if (arg_type)
 	format_warning_at_substring
-	  (fmt_loc, param_range,
+	  (fmt_loc, param_loc,
 	   corrected_substring, OPT_Wformat_,
 	   "%s %<%s%.*s%> expects argument of type %<%s%s%>, "
 	   "but argument %d has type %qT",
@@ -3646,7 +3641,7 @@ format_type_warning (const substring_loc &whole_fmt_loc,
 	   wanted_type_name, p, arg_num, arg_type);
       else
 	format_warning_at_substring
-	  (fmt_loc, param_range,
+	  (fmt_loc, param_loc,
 	   corrected_substring, OPT_Wformat_,
 	   "%s %<%s%.*s%> expects a matching %<%s%s%> argument",
 	   gettext (kind_descriptions[kind]),
@@ -3657,7 +3652,7 @@ format_type_warning (const substring_loc &whole_fmt_loc,
     {
       if (arg_type)
 	format_warning_at_substring
-	  (fmt_loc, param_range,
+	  (fmt_loc, param_loc,
 	   corrected_substring, OPT_Wformat_,
 	   "%s %<%s%.*s%> expects argument of type %<%T%s%>, "
 	   "but argument %d has type %qT",
@@ -3667,7 +3662,7 @@ format_type_warning (const substring_loc &whole_fmt_loc,
 	   wanted_type, p, arg_num, arg_type);
       else
 	format_warning_at_substring
-	  (fmt_loc, param_range,
+	  (fmt_loc, param_loc,
 	   corrected_substring, OPT_Wformat_,
 	   "%s %<%s%.*s%> expects a matching %<%T%s%> argument",
 	   gettext (kind_descriptions[kind]),
diff --git a/gcc/c-family/c-gimplify.c b/gcc/c-family/c-gimplify.c
index 6a4b7c77a34..91f9bf9c7a3 100644
--- a/gcc/c-family/c-gimplify.c
+++ b/gcc/c-family/c-gimplify.c
@@ -229,6 +229,8 @@ c_gimplify_expr (tree *expr_p, gimple_seq *pre_p ATTRIBUTE_UNUSED,
     {
     case LSHIFT_EXPR:
     case RSHIFT_EXPR:
+    case LROTATE_EXPR:
+    case RROTATE_EXPR:
       {
 	/* We used to convert the right operand of a shift-expression
 	   to an integer_type_node in the FEs.  But it is unnecessary
diff --git a/gcc/c-family/c-opts.c b/gcc/c-family/c-opts.c
index 6bd535532d3..32120e636c2 100644
--- a/gcc/c-family/c-opts.c
+++ b/gcc/c-family/c-opts.c
@@ -115,6 +115,7 @@ static void set_std_cxx2a (int);
 static void set_std_c89 (int, int);
 static void set_std_c99 (int);
 static void set_std_c11 (int);
+static void set_std_c17 (int);
 static void check_deps_environment_vars (void);
 static void handle_deferred_opts (void);
 static void sanitize_cpp_opts (void);
@@ -236,8 +237,8 @@ c_common_init_options (unsigned int decoded_options_count,
 
   if (c_language == clk_c)
     {
-      /* The default for C is gnu11.  */
-      set_std_c11 (false /* ISO */);
+      /* The default for C is gnu17.  */
+      set_std_c17 (false /* ISO */);
 
       /* If preprocessing assembly language, accept any of the C-family
 	 front end options since the driver may pass them through.  */
@@ -675,6 +676,16 @@ c_common_handle_option (size_t scode, const char *arg, int value,
 	set_std_c11 (false /* ISO */);
       break;
 
+    case OPT_std_c17:
+      if (!preprocessing_asm_p)
+	set_std_c17 (true /* ISO */);
+      break;
+
+    case OPT_std_gnu17:
+      if (!preprocessing_asm_p)
+	set_std_c17 (false /* ISO */);
+      break;
+
     case OPT_trigraphs:
       cpp_opts->trigraphs = 1;
       break;
@@ -1559,6 +1570,21 @@ set_std_c11 (int iso)
   lang_hooks.name = "GNU C11";
 }
 
+/* Set the C 17 standard (without GNU extensions if ISO).  */
+static void
+set_std_c17 (int iso)
+{
+  cpp_set_lang (parse_in, iso ? CLK_STDC17: CLK_GNUC17);
+  flag_no_asm = iso;
+  flag_no_nonansi_builtin = iso;
+  flag_iso = iso;
+  flag_isoc11 = 1;
+  flag_isoc99 = 1;
+  flag_isoc94 = 1;
+  lang_hooks.name = "GNU C17";
+}
+
+
 /* Set the C++ 98 standard (without GNU extensions if ISO).  */
 static void
 set_std_cxx98 (int iso)
diff --git a/gcc/c-family/c-warn.c b/gcc/c-family/c-warn.c
index cb1db0327c3..09ef6856cf9 100644
--- a/gcc/c-family/c-warn.c
+++ b/gcc/c-family/c-warn.c
@@ -496,8 +496,8 @@ warn_logical_not_parentheses (location_t location, enum tree_code code,
       rich_location richloc (line_table, lhs_loc);
       richloc.add_fixit_insert_before (lhs_loc, "(");
       richloc.add_fixit_insert_after (lhs_loc, ")");
-      inform_at_rich_loc (&richloc, "add parentheses around left hand side "
-			  "expression to silence this warning");
+      inform (&richloc, "add parentheses around left hand side "
+	      "expression to silence this warning");
     }
 }
 
@@ -1215,12 +1215,12 @@ warnings_for_convert_and_check (location_t loc, tree type, tree expr,
       if (cst)
 	warning_at (loc, OPT_Woverflow,
 		    "overflow in conversion from %qT to %qT "
-		    "chages value from %qE to %qE",
+		    "changes value from %qE to %qE",
 		    exprtype, type, expr, result);
       else
 	warning_at (loc, OPT_Woverflow,
 		    "overflow in conversion from %qT to %qT "
-		    "chages the value of %qE",
+		    "changes the value of %qE",
 		    exprtype, type, expr);
     }
   else
@@ -2391,13 +2391,13 @@ warn_for_restrict (unsigned param_pos, tree *argarray, unsigned nargs)
 	richloc.add_range (EXPR_LOCATION (arg), false);
     }
 
-  warning_at_rich_loc_n (&richloc, OPT_Wrestrict, arg_positions.length (),
-			 "passing argument %i to restrict-qualified parameter"
-			 " aliases with argument %Z",
-			 "passing argument %i to restrict-qualified parameter"
-			 " aliases with arguments %Z",
-			 param_pos + 1, arg_positions.address (),
-			 arg_positions.length ());
+  warning_n (&richloc, OPT_Wrestrict, arg_positions.length (),
+	     "passing argument %i to restrict-qualified parameter"
+	     " aliases with argument %Z",
+	     "passing argument %i to restrict-qualified parameter"
+	     " aliases with arguments %Z",
+	     param_pos + 1, arg_positions.address (),
+	     arg_positions.length ());
 }
 
 /* Callback function to determine whether an expression TP or one of its
diff --git a/gcc/c-family/c.opt b/gcc/c-family/c.opt
index 13d2a59b8a5..dae124ac1c2 100644
--- a/gcc/c-family/c.opt
+++ b/gcc/c-family/c.opt
@@ -1944,6 +1944,10 @@ std=c1x
 C ObjC Alias(std=c11)
 Deprecated in favor of -std=c11.
 
+std=c17
+C ObjC
+Conform to the ISO 2017 C standard.
+
 std=c89
 C ObjC Alias(std=c90)
 Conform to the ISO 1990 C standard.
@@ -2006,6 +2010,10 @@ std=gnu1x
 C ObjC Alias(std=gnu11)
 Deprecated in favor of -std=gnu11.
 
+std=gnu17
+C ObjC
+Conform to the ISO 2017 C standard with GNU extensions.
+
 std=gnu89
 C ObjC Alias(std=gnu90)
 Conform to the ISO 1990 C standard with GNU extensions.
@@ -2042,6 +2050,10 @@ std=iso9899:2011
 C ObjC Alias(std=c11)
 Conform to the ISO 2011 C standard.
 
+std=iso9899:2017
+C ObjC Alias(std=c17)
+Conform to the ISO 2017 C standard.
+
 traditional
 Driver
 
diff --git a/gcc/c/ChangeLog b/gcc/c/ChangeLog
index 1f697f17f99..60feeea9022 100644
--- a/gcc/c/ChangeLog
+++ b/gcc/c/ChangeLog
@@ -1,3 +1,47 @@
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-decl.c (implicit_decl_warning): Update for renaming of
+	pedwarn_at_rich_loc and warning_at_rich_loc.
+	(implicitly_declare): Likewise for renaming of inform_at_rich_loc.
+	(undeclared_variable): Likewise for renaming of error_at_rich_loc.
+	* c-parser.c (c_parser_declaration_or_fndef): Likewise.
+	(c_parser_struct_or_union_specifier): Likewise for renaming of
+	pedwarn_at_rich_loc.
+	(c_parser_parameter_declaration): Likewise for renaming of
+	error_at_rich_loc.
+	* c-typeck.c (build_component_ref): Likewise.
+	(build_unary_op): Likewise for renaming of inform_at_rich_loc.
+	(pop_init_level): Likewise for renaming of warning_at_rich_loc.
+	(set_init_label): Likewise for renaming of error_at_rich_loc.
+
+2017-10-30  Richard Biener  <rguenther@suse.de>
+
+	* gimple-parser.c (c_parser_gimple_statement): Parse conditional
+	stmts.
+
+2017-10-27  Michael Meissner  <meissner@linux.vnet.ibm.com>
+
+	* c-decl.c (header_for_builtin_fn): Add support for copysign, fma,
+	fmax, fmin, and sqrt _Float<N> and _Float<N>X variants.
+
+2017-10-25  David Malcolm  <dmalcolm@redhat.com>
+
+	PR c/7356
+	* c-parser.c (c_parser_declaration_or_fndef): Detect missing
+	semicolons.
+
+2017-10-25  Jakub Jelinek  <jakub@redhat.com>
+
+	PR libstdc++/81706
+	* c-decl.c (merge_decls): Copy "omp declare simd" attributes from
+	newdecl to corresponding __builtin_ if any.
+
+2017-10-24  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82466
+	* c-decl.c (diagnose_mismatched_decls): Use
+	OPT_Wbuiltin_declaration_mismatch.
+
 2017-10-12  David Malcolm  <dmalcolm@redhat.com>
 
 	* c-parser.c (c_parser_require): Add "type_is_unique" param and
diff --git a/gcc/c/c-decl.c b/gcc/c/c-decl.c
index 26b34ab3e50..d95a2b6ea4f 100644
--- a/gcc/c/c-decl.c
+++ b/gcc/c/c-decl.c
@@ -1837,7 +1837,8 @@ diagnose_mismatched_decls (tree newdecl, tree olddecl,
 	  locate_old_decl (olddecl);
 	}
       else if (TREE_PUBLIC (newdecl))
-	warning (0, "built-in function %q+D declared as non-function",
+	warning (OPT_Wbuiltin_declaration_mismatch,
+		 "built-in function %q+D declared as non-function",
 		 newdecl);
       else
 	warning (OPT_Wshadow, "declaration of %q+D shadows "
@@ -2569,6 +2570,8 @@ merge_decls (tree newdecl, tree olddecl, tree newtype, tree oldtype)
 			set_builtin_decl_declared_p (fncode, true);
 		      break;
 		    }
+
+		  copy_attributes_to_builtin (newdecl);
 		}
 	    }
 	  else
@@ -3116,10 +3119,10 @@ implicit_decl_warning (location_t loc, tree id, tree olddecl)
 	{
 	  gcc_rich_location richloc (loc);
 	  richloc.add_fixit_replace (hint);
-	  warned = pedwarn_at_rich_loc
-	    (&richloc, OPT_Wimplicit_function_declaration,
-	     "implicit declaration of function %qE; did you mean %qs?",
-	     id, hint);
+	  warned = pedwarn (&richloc, OPT_Wimplicit_function_declaration,
+			    "implicit declaration of function %qE;"
+			    " did you mean %qs?",
+			    id, hint);
 	}
       else
 	warned = pedwarn (loc, OPT_Wimplicit_function_declaration,
@@ -3129,7 +3132,7 @@ implicit_decl_warning (location_t loc, tree id, tree olddecl)
     {
       gcc_rich_location richloc (loc);
       richloc.add_fixit_replace (hint);
-      warned = warning_at_rich_loc
+      warned = warning_at
 	(&richloc, OPT_Wimplicit_function_declaration,
 	 G_("implicit declaration of function %qE; did you mean %qs?"),
 	 id, hint);
@@ -3160,6 +3163,7 @@ header_for_builtin_fn (enum built_in_function fcode)
     CASE_FLT_FN (BUILT_IN_CBRT):
     CASE_FLT_FN (BUILT_IN_CEIL):
     CASE_FLT_FN (BUILT_IN_COPYSIGN):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
     CASE_FLT_FN (BUILT_IN_COS):
     CASE_FLT_FN (BUILT_IN_COSH):
     CASE_FLT_FN (BUILT_IN_ERF):
@@ -3168,11 +3172,15 @@ header_for_builtin_fn (enum built_in_function fcode)
     CASE_FLT_FN (BUILT_IN_EXP2):
     CASE_FLT_FN (BUILT_IN_EXPM1):
     CASE_FLT_FN (BUILT_IN_FABS):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_FABS):
     CASE_FLT_FN (BUILT_IN_FDIM):
     CASE_FLT_FN (BUILT_IN_FLOOR):
     CASE_FLT_FN (BUILT_IN_FMA):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
     CASE_FLT_FN (BUILT_IN_FMAX):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMAX):
     CASE_FLT_FN (BUILT_IN_FMIN):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMIN):
     CASE_FLT_FN (BUILT_IN_FMOD):
     CASE_FLT_FN (BUILT_IN_FREXP):
     CASE_FLT_FN (BUILT_IN_HYPOT):
@@ -3204,6 +3212,7 @@ header_for_builtin_fn (enum built_in_function fcode)
     CASE_FLT_FN (BUILT_IN_SINH):
     CASE_FLT_FN (BUILT_IN_SINCOS):
     CASE_FLT_FN (BUILT_IN_SQRT):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_SQRT):
     CASE_FLT_FN (BUILT_IN_TAN):
     CASE_FLT_FN (BUILT_IN_TANH):
     CASE_FLT_FN (BUILT_IN_TGAMMA):
@@ -3392,10 +3401,9 @@ implicitly_declare (location_t loc, tree functionid)
 		    {
 		      rich_location richloc (line_table, loc);
 		      maybe_add_include_fixit (&richloc, header);
-		      inform_at_rich_loc
-			(&richloc,
-			 "include %qs or provide a declaration of %qD",
-			 header, decl);
+		      inform (&richloc,
+			      "include %qs or provide a declaration of %qD",
+			      header, decl);
 		    }
 		  newtype = TREE_TYPE (decl);
 		}
@@ -3463,10 +3471,10 @@ undeclared_variable (location_t loc, tree id)
 	{
 	  gcc_rich_location richloc (loc);
 	  richloc.add_fixit_replace (guessed_id);
-	  error_at_rich_loc (&richloc,
-			     "%qE undeclared here (not in a function);"
-			     " did you mean %qs?",
-			     id, guessed_id);
+	  error_at (&richloc,
+		    "%qE undeclared here (not in a function);"
+		    " did you mean %qs?",
+		    id, guessed_id);
 	}
       else
 	error_at (loc, "%qE undeclared here (not in a function)", id);
@@ -3481,11 +3489,10 @@ undeclared_variable (location_t loc, tree id)
 	    {
 	      gcc_rich_location richloc (loc);
 	      richloc.add_fixit_replace (guessed_id);
-	      error_at_rich_loc
-		(&richloc,
-		 "%qE undeclared (first use in this function);"
-		 " did you mean %qs?",
-		 id, guessed_id);
+	      error_at (&richloc,
+			"%qE undeclared (first use in this function);"
+			" did you mean %qs?",
+			id, guessed_id);
 	    }
 	  else
 	    error_at (loc, "%qE undeclared (first use in this function)", id);
diff --git a/gcc/c/c-parser.c b/gcc/c/c-parser.c
index 6b843247911..7bca5f1a2a7 100644
--- a/gcc/c/c-parser.c
+++ b/gcc/c/c-parser.c
@@ -1785,26 +1785,26 @@ c_parser_declaration_or_fndef (c_parser *parser, bool fndef_ok,
 	{
 	  /* This is not C++ with its implicit typedef.  */
 	  richloc.add_fixit_insert_before ("struct ");
-	  error_at_rich_loc (&richloc,
-			     "unknown type name %qE;"
-			     " use %<struct%> keyword to refer to the type",
-			     name);
+	  error_at (&richloc,
+		    "unknown type name %qE;"
+		    " use %<struct%> keyword to refer to the type",
+		    name);
 	}
       else if (tag_exists_p (UNION_TYPE, name))
 	{
 	  richloc.add_fixit_insert_before ("union ");
-	  error_at_rich_loc (&richloc,
-			     "unknown type name %qE;"
-			     " use %<union%> keyword to refer to the type",
-			     name);
+	  error_at (&richloc,
+		    "unknown type name %qE;"
+		    " use %<union%> keyword to refer to the type",
+		    name);
 	}
       else if (tag_exists_p (ENUMERAL_TYPE, name))
 	{
 	  richloc.add_fixit_insert_before ("enum ");
-	  error_at_rich_loc (&richloc,
-			     "unknown type name %qE;"
-			     " use %<enum%> keyword to refer to the type",
-			     name);
+	  error_at (&richloc,
+		    "unknown type name %qE;"
+		    " use %<enum%> keyword to refer to the type",
+		    name);
 	}
       else
 	{
@@ -1812,9 +1812,9 @@ c_parser_declaration_or_fndef (c_parser *parser, bool fndef_ok,
 	  if (hint)
 	    {
 	      richloc.add_fixit_replace (hint);
-	      error_at_rich_loc (&richloc,
-				 "unknown type name %qE; did you mean %qs?",
-				 name, hint);
+	      error_at (&richloc,
+			"unknown type name %qE; did you mean %qs?",
+			name, hint);
 	    }
 	  else
 	    error_at (here, "unknown type name %qE", name);
@@ -2241,11 +2241,37 @@ c_parser_declaration_or_fndef (c_parser *parser, bool fndef_ok,
 	}
       if (!start_function (specs, declarator, all_prefix_attrs))
 	{
-	  /* This can appear in many cases looking nothing like a
-	     function definition, so we don't give a more specific
-	     error suggesting there was one.  */
-	  c_parser_error (parser, "expected %<=%>, %<,%>, %<;%>, %<asm%> "
-			  "or %<__attribute__%>");
+	  /* At this point we've consumed:
+	       declaration-specifiers declarator
+	     and the next token isn't CPP_EQ, CPP_COMMA, CPP_SEMICOLON,
+	     RID_ASM, RID_ATTRIBUTE, or RID_IN,
+	     but the
+	       declaration-specifiers declarator
+	     aren't grokkable as a function definition, so we have
+	     an error.  */
+	  gcc_assert (!c_parser_next_token_is (parser, CPP_SEMICOLON));
+	  if (c_parser_next_token_starts_declspecs (parser))
+	    {
+	      /* If we have
+		   declaration-specifiers declarator decl-specs
+		 then assume we have a missing semicolon, which would
+		 give us:
+		   declaration-specifiers declarator  decl-specs
+						    ^
+						    ;
+		   <~~~~~~~~~ declaration ~~~~~~~~~~>
+		 Use c_parser_require to get an error with a fix-it hint.  */
+	      c_parser_require (parser, CPP_SEMICOLON, "expected %<;%>");
+	      parser->error = false;
+	    }
+	  else
+	    {
+	      /* This can appear in many cases looking nothing like a
+		 function definition, so we don't give a more specific
+		 error suggesting there was one.  */
+	      c_parser_error (parser, "expected %<=%>, %<,%>, %<;%>, %<asm%> "
+			      "or %<__attribute__%>");
+	    }
 	  if (nested)
 	    c_pop_function_context ();
 	  break;
@@ -3142,9 +3168,8 @@ c_parser_struct_or_union_specifier (c_parser *parser)
 		= c_parser_peek_token (parser)->location;
 	      gcc_rich_location richloc (semicolon_loc);
 	      richloc.add_fixit_remove ();
-	      pedwarn_at_rich_loc
-		(&richloc, OPT_Wpedantic,
-		 "extra semicolon in struct or union specified");
+	      pedwarn (&richloc, OPT_Wpedantic,
+		       "extra semicolon in struct or union specified");
 	      c_parser_consume_token (parser);
 	      continue;
 	    }
@@ -4047,9 +4072,9 @@ c_parser_parameter_declaration (c_parser *parser, tree attrs)
 	    {
 	      gcc_rich_location richloc (token->location);
 	      richloc.add_fixit_replace (hint);
-	      error_at_rich_loc (&richloc,
-				 "unknown type name %qE; did you mean %qs?",
-				 token->value, hint);
+	      error_at (&richloc,
+			"unknown type name %qE; did you mean %qs?",
+			token->value, hint);
 	    }
 	  else
 	    error_at (token->location, "unknown type name %qE", token->value);
diff --git a/gcc/c/c-typeck.c b/gcc/c/c-typeck.c
index ad980548a74..e28dfc2884e 100644
--- a/gcc/c/c-typeck.c
+++ b/gcc/c/c-typeck.c
@@ -2406,10 +2406,9 @@ build_component_ref (location_t loc, tree datum, tree component,
 	      gcc_rich_location rich_loc (reported_loc);
 	      if (component_loc != UNKNOWN_LOCATION)
 		rich_loc.add_fixit_misspelled_id (component_loc, guessed_id);
-	      error_at_rich_loc
-		(&rich_loc,
-		 "%qT has no member named %qE; did you mean %qE?",
-		 type, component, guessed_id);
+	      error_at (&rich_loc,
+			"%qT has no member named %qE; did you mean %qE?",
+			type, component, guessed_id);
 	    }
 	  else
 	    error_at (loc, "%qT has no member named %qE", type, component);
@@ -2483,9 +2482,9 @@ build_component_ref (location_t loc, tree datum, tree component,
       rich_location richloc (line_table, loc);
       /* "loc" should be the "." token.  */
       richloc.add_fixit_replace ("->");
-      error_at_rich_loc (&richloc,
-			 "%qE is a pointer; did you mean to use %<->%>?",
-			 datum);
+      error_at (&richloc,
+		"%qE is a pointer; did you mean to use %<->%>?",
+		datum);
       return error_mark_node;
     }
   else if (code != ERROR_MARK)
@@ -4276,8 +4275,7 @@ build_unary_op (location_t location, enum tree_code code, tree xarg,
 	    {
 	      gcc_rich_location richloc (location);
 	      richloc.add_fixit_insert_before (location, "!");
-	      inform_at_rich_loc (&richloc, "did you mean to use logical "
-				  "not?");
+	      inform (&richloc, "did you mean to use logical not?");
 	    }
 	  if (!noconvert)
 	    arg = default_conversion (arg);
@@ -8256,9 +8254,9 @@ pop_init_level (location_t loc, int implicit,
       && !constructor_zeroinit)
     {
       gcc_assert (initializer_stack->missing_brace_richloc);
-      warning_at_rich_loc (initializer_stack->missing_brace_richloc,
-			   OPT_Wmissing_braces,
-			   "missing braces around initializer");
+      warning_at (initializer_stack->missing_brace_richloc,
+		  OPT_Wmissing_braces,
+		  "missing braces around initializer");
     }
 
   /* Warn when some struct elements are implicitly initialized to zero.  */
@@ -8580,10 +8578,9 @@ set_init_label (location_t loc, tree fieldname, location_t fieldname_loc,
 	{
 	  gcc_rich_location rich_loc (fieldname_loc);
 	  rich_loc.add_fixit_misspelled_id (fieldname_loc, guessed_id);
-	  error_at_rich_loc
-	    (&rich_loc,
-	     "%qT has no member named %qE; did you mean %qE?",
-	     constructor_type, fieldname, guessed_id);
+	  error_at (&rich_loc,
+		    "%qT has no member named %qE; did you mean %qE?",
+		    constructor_type, fieldname, guessed_id);
 	}
       else
 	error_at (fieldname_loc, "%qT has no member named %qE",
diff --git a/gcc/c/gimple-parser.c b/gcc/c/gimple-parser.c
index ab335b6e78c..aea675ffabb 100644
--- a/gcc/c/gimple-parser.c
+++ b/gcc/c/gimple-parser.c
@@ -276,7 +276,7 @@ c_parser_gimple_statement (c_parser *parser, gimple_seq *seq)
       && TREE_CODE (lhs.value) == CALL_EXPR)
     {
       gimple *call;
-      call = gimple_build_call_from_tree (lhs.value);
+      call = gimple_build_call_from_tree (lhs.value, NULL);
       gimple_seq_add_stmt (seq, call);
       gimple_set_location (call, loc);
       return;
@@ -407,7 +407,7 @@ c_parser_gimple_statement (c_parser *parser, gimple_seq *seq)
       rhs = c_parser_gimple_unary_expression (parser);
       if (rhs.value != error_mark_node)
 	{
-	  gimple *call = gimple_build_call_from_tree (rhs.value);
+	  gimple *call = gimple_build_call_from_tree (rhs.value, NULL);
 	  gimple_call_set_lhs (call, lhs.value);
 	  gimple_seq_add_stmt (seq, call);
 	  gimple_set_location (call, loc);
@@ -419,6 +419,23 @@ c_parser_gimple_statement (c_parser *parser, gimple_seq *seq)
   if (lhs.value != error_mark_node
       && rhs.value != error_mark_node)
     {
+      /* If we parsed a comparison and the next token is a '?' then
+         parse a conditional expression.  */
+      if (COMPARISON_CLASS_P (rhs.value)
+	  && c_parser_next_token_is (parser, CPP_QUERY))
+	{
+	  struct c_expr trueval, falseval;
+	  c_parser_consume_token (parser);
+	  trueval = c_parser_gimple_postfix_expression (parser);
+	  falseval.set_error ();
+	  if (c_parser_require (parser, CPP_COLON, "expected %<:%>"))
+	    falseval = c_parser_gimple_postfix_expression (parser);
+	  if (trueval.value == error_mark_node
+	      || falseval.value == error_mark_node)
+	    return;
+	  rhs.value = build3_loc (loc, COND_EXPR, TREE_TYPE (trueval.value),
+				  rhs.value, trueval.value, falseval.value);
+	}
       assign = gimple_build_assign (lhs.value, rhs.value);
       gimple_seq_add_stmt (seq, assign);
       gimple_set_location (assign, loc);
diff --git a/gcc/calls.c b/gcc/calls.c
index ff9724358c5..477bc369036 100644
--- a/gcc/calls.c
+++ b/gcc/calls.c
@@ -444,7 +444,7 @@ emit_call_1 (rtx funexp, tree fntree ATTRIBUTE_UNUSED, tree fndecl ATTRIBUTE_UNU
      if no arguments are actually popped.  If the target does not have
      "call" or "call_value" insns, then we must use the popping versions
      even if the call has no arguments to pop.  */
-  else if (maybe_nonzero (n_popped)
+  else if (may_ne (n_popped, 0)
 	   || !(valreg
 		? targetm.have_call_value ()
 		: targetm.have_call ()))
@@ -523,7 +523,7 @@ emit_call_1 (rtx funexp, tree fntree ATTRIBUTE_UNUSED, tree fndecl ATTRIBUTE_UNU
      if the context of the call as a whole permits.  */
   inhibit_defer_pop = old_inhibit_defer_pop;
 
-  if (maybe_nonzero (n_popped))
+  if (may_ne (n_popped, 0))
     {
       if (!already_popped)
 	CALL_INSN_FUNCTION_USAGE (call_insn)
@@ -555,7 +555,7 @@ emit_call_1 (rtx funexp, tree fntree ATTRIBUTE_UNUSED, tree fndecl ATTRIBUTE_UNU
 	 If returning from the subroutine does pop the args, indicate that the
 	 stack pointer will be changed.  */
 
-      if (maybe_nonzero (rounded_stack_size))
+      if (may_ne (rounded_stack_size, 0))
 	{
 	  if (ecf_flags & ECF_NORETURN)
 	    /* Just pretend we did the pop.  */
@@ -578,7 +578,7 @@ emit_call_1 (rtx funexp, tree fntree ATTRIBUTE_UNUSED, tree fndecl ATTRIBUTE_UNU
 
      ??? It will be worthwhile to enable combine_stack_adjustments even for
      such machines.  */
-  else if (maybe_nonzero (n_popped))
+  else if (may_ne (n_popped, 0))
     anti_adjust_stack (gen_int_mode (n_popped, Pmode));
 }
 
@@ -644,16 +644,9 @@ special_function_p (const_tree fndecl, int flags)
 	flags |= ECF_RETURNS_TWICE;
     }
 
-  if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
-    switch (DECL_FUNCTION_CODE (fndecl))
-      {
-      case BUILT_IN_ALLOCA:
-      case BUILT_IN_ALLOCA_WITH_ALIGN:
-	flags |= ECF_MAY_BE_ALLOCA;
-	break;
-      default:
-	break;
-      }
+  if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
+      && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (fndecl)))
+    flags |= ECF_MAY_BE_ALLOCA;
 
   return flags;
 }
@@ -735,8 +728,7 @@ gimple_alloca_call_p (const gimple *stmt)
   if (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
     switch (DECL_FUNCTION_CODE (fndecl))
       {
-      case BUILT_IN_ALLOCA:
-      case BUILT_IN_ALLOCA_WITH_ALIGN:
+      CASE_BUILT_IN_ALLOCA:
         return true;
       default:
 	break;
@@ -756,8 +748,7 @@ alloca_call_p (const_tree exp)
       && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
     switch (DECL_FUNCTION_CODE (fndecl))
       {
-      case BUILT_IN_ALLOCA:
-      case BUILT_IN_ALLOCA_WITH_ALIGN:
+      CASE_BUILT_IN_ALLOCA:
         return true;
       default:
 	break;
@@ -1857,6 +1848,8 @@ initialize_argument_information (int num_actuals ATTRIBUTE_UNUSED,
 		  copy = allocate_dynamic_stack_space (size_rtx,
 						       TYPE_ALIGN (type),
 						       TYPE_ALIGN (type),
+						       max_int_size_in_bytes
+						       (type),
 						       true);
 		  copy = gen_rtx_MEM (BLKmode, copy);
 		  set_mem_attributes (copy, type, 1);
@@ -2186,7 +2179,7 @@ finalize_must_preallocate (int must_preallocate, int num_actuals,
 	      += int_size_in_bytes (TREE_TYPE (args[i].tree_value));
 	}
 
-      if (maybe_nonzero (args_size->constant)
+      if (may_ne (args_size->constant, 0)
 	  && may_ge (copy_to_evaluate_size * 2, args_size->constant))
 	must_preallocate = 1;
     }
@@ -2465,7 +2458,7 @@ mem_might_overlap_already_clobbered_arg_p (rtx addr, poly_uint64 size)
   else if (!poly_int_rtx_p (val, &i))
     return true;
 
-  if (known_zero (size))
+  if (must_eq (size, 0U))
     return false;
 
   if (STACK_GROWS_DOWNWARD)
@@ -2839,7 +2832,7 @@ shift_return_value (machine_mode mode, bool left_p, rtx value)
   machine_mode value_mode = GET_MODE (value);
   poly_int64 shift = GET_MODE_BITSIZE (value_mode) - GET_MODE_BITSIZE (mode);
 
-  if (known_zero (shift))
+  if (must_eq (shift, 0))
     return false;
 
   /* Use ashr rather than lshr for right shifts.  This is for the benefit
@@ -3386,7 +3379,7 @@ expand_call (tree exp, rtx target, int ignore)
 			      structure_value_addr))
       && (args_size.var
 	  || (!ACCUMULATE_OUTGOING_ARGS
-	      && maybe_nonzero (args_size.constant))))
+	      && may_ne (args_size.constant, 0))))
     structure_value_addr = copy_to_reg (structure_value_addr);
 
   /* Tail calls can make things harder to debug, and we've traditionally
@@ -3503,9 +3496,9 @@ expand_call (tree exp, rtx target, int ignore)
 	 Also do the adjustments before a throwing call, otherwise
 	 exception handling can fail; PR 19225. */
       if (may_ge (pending_stack_adjust, 32)
-	  || (maybe_nonzero (pending_stack_adjust)
+	  || (may_ne (pending_stack_adjust, 0)
 	      && (flags & ECF_MAY_BE_ALLOCA))
-	  || (maybe_nonzero (pending_stack_adjust)
+	  || (may_ne (pending_stack_adjust, 0)
 	      && flag_exceptions && !(flags & ECF_NOTHROW))
 	  || pass == 0)
 	do_pending_stack_adjust ();
@@ -3686,7 +3679,7 @@ expand_call (tree exp, rtx target, int ignore)
 
 		  /* Special case this because overhead of `push_block' in
 		     this case is non-trivial.  */
-		  if (known_zero (needed))
+		  if (must_eq (needed, 0))
 		    argblock = virtual_outgoing_args_rtx;
 		  else
 		    {
@@ -3745,8 +3738,8 @@ expand_call (tree exp, rtx target, int ignore)
 	      /* We can pass TRUE as the 4th argument because we just
 		 saved the stack pointer and will restore it right after
 		 the call.  */
-	      allocate_dynamic_stack_space (push_size, 0,
-					    BIGGEST_ALIGNMENT, true);
+	      allocate_dynamic_stack_space (push_size, 0, BIGGEST_ALIGNMENT,
+					    -1, true);
 	    }
 
 	  /* If argument evaluation might modify the stack pointer,
@@ -3781,7 +3774,7 @@ expand_call (tree exp, rtx target, int ignore)
 	{
 	  /* When the stack adjustment is pending, we get better code
 	     by combining the adjustments.  */
-	  if (maybe_nonzero (pending_stack_adjust)
+	  if (may_ne (pending_stack_adjust, 0)
 	      && ! inhibit_defer_pop
 	      && (combine_pending_stack_adjustment_and_call
 		  (&pending_stack_adjust,
diff --git a/gcc/cfg.c b/gcc/cfg.c
index 01e68aeda51..062788afdc0 100644
--- a/gcc/cfg.c
+++ b/gcc/cfg.c
@@ -68,6 +68,7 @@ init_flow (struct function *the_fun)
   if (!the_fun->cfg)
     the_fun->cfg = ggc_cleared_alloc<control_flow_graph> ();
   n_edges_for_fn (the_fun) = 0;
+  the_fun->cfg->count_max = profile_count::uninitialized ();
   ENTRY_BLOCK_PTR_FOR_FN (the_fun)
     = alloc_block ();
   ENTRY_BLOCK_PTR_FOR_FN (the_fun)->index = ENTRY_BLOCK;
@@ -263,7 +264,6 @@ unchecked_make_edge (basic_block src, basic_block dst, int flags)
   e = ggc_cleared_alloc<edge_def> ();
   n_edges_for_fn (cfun)++;
 
-  e->count = profile_count::uninitialized ();
   e->probability = profile_probability::uninitialized ();
   e->src = src;
   e->dest = dst;
@@ -334,7 +334,6 @@ make_single_succ_edge (basic_block src, basic_block dest, int flags)
   edge e = make_edge (src, dest, flags);
 
   e->probability = profile_probability::always ();
-  e->count = src->count;
   return e;
 }
 
@@ -445,37 +444,18 @@ check_bb_profile (basic_block bb, FILE * file, int indent)
 		       ";; %sInvalid sum of outgoing probabilities %.1f%%\n",
 		       s_indent, isum * 100.0 / REG_BR_PROB_BASE);
 	    }
-	  profile_count lsum = profile_count::zero ();
-	  FOR_EACH_EDGE (e, ei, bb->succs)
-	    lsum += e->count;
-	  if (EDGE_COUNT (bb->succs) && lsum.differs_from_p (bb->count))
-	    {
-	      fprintf (file, ";; %sInvalid sum of outgoing counts ",
-		       s_indent);
-	      lsum.dump (file);
-	      fprintf (file, ", should be ");
-	      bb->count.dump (file);
-	      fprintf (file, "\n");
-	    }
 	}
     }
   if (bb != ENTRY_BLOCK_PTR_FOR_FN (fun))
     {
-      int sum = 0;
-      FOR_EACH_EDGE (e, ei, bb->preds)
-	sum += EDGE_FREQUENCY (e);
-      if (abs (sum - bb->frequency) > 100)
-	fprintf (file,
-		 ";; %sInvalid sum of incoming frequencies %i, should be %i\n",
-		 s_indent, sum, bb->frequency);
-      profile_count lsum = profile_count::zero ();
+      profile_count sum = profile_count::zero ();
       FOR_EACH_EDGE (e, ei, bb->preds)
-	lsum += e->count;
-      if (lsum.differs_from_p (bb->count))
+	sum += e->count ();
+      if (sum.differs_from_p (bb->count))
 	{
 	  fprintf (file, ";; %sInvalid sum of incoming counts ",
 		   s_indent);
-	  lsum.dump (file);
+	  sum.dump (file);
 	  fprintf (file, ", should be ");
 	  bb->count.dump (file);
 	  fprintf (file, "\n");
@@ -522,10 +502,10 @@ dump_edge_info (FILE *file, edge e, dump_flags_t flags, int do_succ)
       fprintf (file, "] ");
     }
 
-  if (e->count.initialized_p () && do_details)
+  if (e->count ().initialized_p () && do_details)
     {
       fputs (" count:", file);
-      e->count.dump (file);
+      e->count ().dump (file);
     }
 
   if (e->flags && do_details)
@@ -777,7 +757,6 @@ dump_bb_info (FILE *outf, basic_block bb, int indent, dump_flags_t flags,
 	      fputs (", count ", outf);
 	      bb->count.dump (outf);
 	    }
-	  fprintf (outf, ", freq %i", bb->frequency);
 	  if (maybe_hot_bb_p (fun, bb))
 	    fputs (", maybe hot", outf);
 	  if (probably_never_executed_bb_p (fun, bb))
@@ -869,15 +848,15 @@ brief_dump_cfg (FILE *file, dump_flags_t flags)
     }
 }
 
-/* An edge originally destinating BB of FREQUENCY and COUNT has been proved to
+/* An edge originally destinating BB of COUNT has been proved to
    leave the block by TAKEN_EDGE.  Update profile of BB such that edge E can be
    redirected to destination of TAKEN_EDGE.
 
    This function may leave the profile inconsistent in the case TAKEN_EDGE
-   frequency or count is believed to be lower than FREQUENCY or COUNT
+   frequency or count is believed to be lower than COUNT
    respectively.  */
 void
-update_bb_profile_for_threading (basic_block bb, int edge_frequency,
+update_bb_profile_for_threading (basic_block bb, 
 				 profile_count count, edge taken_edge)
 {
   edge c;
@@ -892,16 +871,10 @@ update_bb_profile_for_threading (basic_block bb, int edge_frequency,
     }
   bb->count -= count;
 
-  bb->frequency -= edge_frequency;
-  if (bb->frequency < 0)
-    bb->frequency = 0;
-
   /* Compute the probability of TAKEN_EDGE being reached via threaded edge.
      Watch for overflows.  */
-  if (bb->frequency)
-    /* FIXME: We should get edge frequency as count.  */
-    prob = profile_probability::probability_in_gcov_type
-		 (edge_frequency, bb->frequency);
+  if (bb->count.nonzero_p ())
+    prob = count.probability_in (bb->count);
   else
     prob = profile_probability::never ();
   if (prob > taken_edge->probability)
@@ -925,9 +898,9 @@ update_bb_profile_for_threading (basic_block bb, int edge_frequency,
   if (prob == profile_probability::never ())
     {
       if (dump_file)
-	fprintf (dump_file, "Edge frequencies of bb %i has been reset, "
-		 "frequency of block should end up being 0, it is %i\n",
-		 bb->index, bb->frequency);
+	fprintf (dump_file, "Edge probabilities of bb %i has been reset, "
+		 "count of block should end up being 0, it is non-zero\n",
+		 bb->index);
       EDGE_SUCC (bb, 0)->probability = profile_probability::guessed_always ();
       ei = ei_start (bb->succs);
       ei_next (&ei);
@@ -941,10 +914,6 @@ update_bb_profile_for_threading (basic_block bb, int edge_frequency,
     }
 
   gcc_assert (bb == taken_edge->src);
-  if (dump_file && taken_edge->count < count)
-    fprintf (dump_file, "edge %i->%i count became negative after threading",
-	     taken_edge->src->index, taken_edge->dest->index);
-  taken_edge->count -= count;
 }
 
 /* Multiply all frequencies of basic blocks in array BBS of length NBBS
@@ -953,7 +922,6 @@ void
 scale_bbs_frequencies_int (basic_block *bbs, int nbbs, int num, int den)
 {
   int i;
-  edge e;
   if (num < 0)
     num = 0;
 
@@ -973,21 +941,10 @@ scale_bbs_frequencies_int (basic_block *bbs, int nbbs, int num, int den)
 
   for (i = 0; i < nbbs; i++)
     {
-      edge_iterator ei;
-      bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
-      /* Make sure the frequencies do not grow over BB_FREQ_MAX.  */
-      if (bbs[i]->frequency > BB_FREQ_MAX)
-	bbs[i]->frequency = BB_FREQ_MAX;
       bbs[i]->count = bbs[i]->count.apply_scale (num, den);
-      FOR_EACH_EDGE (e, ei, bbs[i]->succs)
-	e->count = e->count.apply_scale (num, den);
     }
 }
 
-/* numbers smaller than this value are safe to multiply without getting
-   64bit overflow.  */
-#define MAX_SAFE_MULTIPLIER (1 << (sizeof (int64_t) * 4 - 1))
-
 /* Multiply all frequencies of basic blocks in array BBS of length NBBS
    by NUM/DEN, in gcov_type arithmetic.  More accurate than previous
    function but considerably slower.  */
@@ -996,38 +953,9 @@ scale_bbs_frequencies_gcov_type (basic_block *bbs, int nbbs, gcov_type num,
 				 gcov_type den)
 {
   int i;
-  edge e;
-  gcov_type fraction = RDIV (num * 65536, den);
-
-  gcc_assert (fraction >= 0);
 
-  if (num < MAX_SAFE_MULTIPLIER)
-    for (i = 0; i < nbbs; i++)
-      {
-	edge_iterator ei;
-	bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
-	if (bbs[i]->count <= MAX_SAFE_MULTIPLIER)
-	  bbs[i]->count = bbs[i]->count.apply_scale (num, den);
-	else
-	  bbs[i]->count = bbs[i]->count.apply_scale (fraction, 65536);
-	FOR_EACH_EDGE (e, ei, bbs[i]->succs)
-	  if (bbs[i]->count <= MAX_SAFE_MULTIPLIER)
-	    e->count =  e->count.apply_scale (num, den);
-	  else
-	    e->count = e->count.apply_scale (fraction, 65536);
-      }
-   else
-    for (i = 0; i < nbbs; i++)
-      {
-	edge_iterator ei;
-	if (sizeof (gcov_type) > sizeof (int))
-	  bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
-	else
-	  bbs[i]->frequency = RDIV (bbs[i]->frequency * fraction, 65536);
-	bbs[i]->count = bbs[i]->count.apply_scale (fraction, 65536);
-	FOR_EACH_EDGE (e, ei, bbs[i]->succs)
-	  e->count = e->count.apply_scale (fraction, 65536);
-      }
+  for (i = 0; i < nbbs; i++)
+    bbs[i]->count = bbs[i]->count.apply_scale (num, den);
 }
 
 /* Multiply all frequencies of basic blocks in array BBS of length NBBS
@@ -1038,17 +966,9 @@ scale_bbs_frequencies_profile_count (basic_block *bbs, int nbbs,
 				     profile_count num, profile_count den)
 {
   int i;
-  edge e;
-
-  for (i = 0; i < nbbs; i++)
-    {
-      edge_iterator ei;
-      bbs[i]->frequency = RDIV (bbs[i]->frequency * num.to_gcov_type (),
-				den.to_gcov_type ());
+  if (num == profile_count::zero () || den.nonzero_p ())
+    for (i = 0; i < nbbs; i++)
       bbs[i]->count = bbs[i]->count.apply_scale (num, den);
-      FOR_EACH_EDGE (e, ei, bbs[i]->succs)
-	e->count =  e->count.apply_scale (num, den);
-    }
 }
 
 /* Multiply all frequencies of basic blocks in array BBS of length NBBS
@@ -1059,16 +979,9 @@ scale_bbs_frequencies (basic_block *bbs, int nbbs,
 		       profile_probability p)
 {
   int i;
-  edge e;
 
   for (i = 0; i < nbbs; i++)
-    {
-      edge_iterator ei;
-      bbs[i]->frequency = p.apply (bbs[i]->frequency);
-      bbs[i]->count = bbs[i]->count.apply_probability (p);
-      FOR_EACH_EDGE (e, ei, bbs[i]->succs)
-	e->count =  e->count.apply_probability (p);
-    }
+    bbs[i]->count = bbs[i]->count.apply_probability (p);
 }
 
 /* Helper types for hash tables.  */
diff --git a/gcc/cfg.h b/gcc/cfg.h
index 81b243a1a9e..e8129ddb190 100644
--- a/gcc/cfg.h
+++ b/gcc/cfg.h
@@ -71,6 +71,9 @@ struct GTY(()) control_flow_graph {
   /* Maximal number of entities in the single jumptable.  Used to estimate
      final flowgraph size.  */
   int max_jumptable_ents;
+
+  /* Maximal count of BB in function.  */
+  profile_count count_max;
 };
 
 
@@ -103,7 +106,7 @@ extern void debug_bb (basic_block);
 extern basic_block debug_bb_n (int);
 extern void dump_bb_info (FILE *, basic_block, int, dump_flags_t, bool, bool);
 extern void brief_dump_cfg (FILE *, dump_flags_t);
-extern void update_bb_profile_for_threading (basic_block, int, profile_count, edge);
+extern void update_bb_profile_for_threading (basic_block, profile_count, edge);
 extern void scale_bbs_frequencies_int (basic_block *, int, int, int);
 extern void scale_bbs_frequencies_gcov_type (basic_block *, int, gcov_type,
 					     gcov_type);
diff --git a/gcc/cfganal.c b/gcc/cfganal.c
index 394d986c945..8bf8a53fa58 100644
--- a/gcc/cfganal.c
+++ b/gcc/cfganal.c
@@ -612,7 +612,6 @@ connect_infinite_loops_to_exit (void)
       basic_block deadend_block = dfs_find_deadend (unvisited_block);
       edge e = make_edge (deadend_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
 			  EDGE_FAKE);
-      e->count = profile_count::zero ();
       e->probability = profile_probability::never ();
       dfs.add_bb (deadend_block);
     }
@@ -1555,3 +1554,42 @@ single_pred_before_succ_order (void)
 #undef MARK_VISITED
 #undef VISITED_P
 }
+
+/* Ignoring loop backedges, if BB has precisely one incoming edge then
+   return that edge.  Otherwise return NULL.
+
+   When IGNORE_NOT_EXECUTABLE is true, also ignore edges that are not marked
+   as executable.  */
+
+edge
+single_pred_edge_ignoring_loop_edges (basic_block bb,
+				      bool ignore_not_executable)
+{
+  edge retval = NULL;
+  edge e;
+  edge_iterator ei;
+
+  FOR_EACH_EDGE (e, ei, bb->preds)
+    {
+      /* A loop back edge can be identified by the destination of
+	 the edge dominating the source of the edge.  */
+      if (dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
+	continue;
+
+      /* We can safely ignore edges that are not executable.  */
+      if (ignore_not_executable
+	  && (e->flags & EDGE_EXECUTABLE) == 0)
+	continue;
+
+      /* If we have already seen a non-loop edge, then we must have
+	 multiple incoming non-loop edges and thus we return NULL.  */
+      if (retval)
+	return NULL;
+
+      /* This is the first non-loop incoming edge we have found.  Record
+	 it.  */
+      retval = e;
+    }
+
+  return retval;
+}
diff --git a/gcc/cfganal.h b/gcc/cfganal.h
index 39bb5e547a5..c5cb51d9cf8 100644
--- a/gcc/cfganal.h
+++ b/gcc/cfganal.h
@@ -77,5 +77,8 @@ extern void bitmap_intersection_of_preds (sbitmap, sbitmap *, basic_block);
 extern void bitmap_union_of_succs (sbitmap, sbitmap *, basic_block);
 extern void bitmap_union_of_preds (sbitmap, sbitmap *, basic_block);
 extern basic_block * single_pred_before_succ_order (void);
+extern edge single_incoming_edge_ignoring_loop_edges (basic_block, bool);
+extern edge single_pred_edge_ignoring_loop_edges (basic_block, bool);
+
 
 #endif /* GCC_CFGANAL_H */
diff --git a/gcc/cfgbuild.c b/gcc/cfgbuild.c
index 62956b2a6a2..a0926752143 100644
--- a/gcc/cfgbuild.c
+++ b/gcc/cfgbuild.c
@@ -499,7 +499,6 @@ find_bb_boundaries (basic_block bb)
 	  remove_edge (fallthru);
 	  /* BB is unreachable at this point - we need to determine its profile
 	     once edges are built.  */
-	  bb->frequency = 0;
 	  bb->count = profile_count::uninitialized ();
 	  flow_transfer_insn = NULL;
 	  debug_insn = NULL;
@@ -576,10 +575,8 @@ compute_outgoing_frequencies (basic_block b)
 	  e = BRANCH_EDGE (b);
 	  e->probability
 		 = profile_probability::from_reg_br_prob_note (probability);
-	  e->count = b->count.apply_probability (e->probability);
 	  f = FALLTHRU_EDGE (b);
 	  f->probability = e->probability.invert ();
-	  f->count = b->count - e->count;
 	  return;
 	}
       else
@@ -591,7 +588,6 @@ compute_outgoing_frequencies (basic_block b)
     {
       e = single_succ_edge (b);
       e->probability = profile_probability::always ();
-      e->count = b->count;
       return;
     }
   else
@@ -610,10 +606,6 @@ compute_outgoing_frequencies (basic_block b)
       if (complex_edge)
         guess_outgoing_edge_probabilities (b);
     }
-
-  if (b->count.initialized_p ())
-    FOR_EACH_EDGE (e, ei, b->succs)
-      e->count = b->count.apply_probability (e->probability);
 }
 
 /* Assume that some pass has inserted labels or control flow
@@ -676,18 +668,15 @@ find_many_sub_basic_blocks (sbitmap blocks)
 	  {
 	    bool initialized_src = false, uninitialized_src = false;
 	    bb->count = profile_count::zero ();
-	    bb->frequency = 0;
 	    FOR_EACH_EDGE (e, ei, bb->preds)
 	      {
-		if (e->count.initialized_p ())
+		if (e->count ().initialized_p ())
 		  {
-		    bb->count += e->count;
+		    bb->count += e->count ();
 		    initialized_src = true;
 		  }
 		else
 		  uninitialized_src = true;
-		if (e->probability.initialized_p ())
-		  bb->frequency += EDGE_FREQUENCY (e);
 	      }
 	    /* When some edges are missing with read profile, this is
 	       most likely because RTL expansion introduced loop.
@@ -699,7 +688,7 @@ find_many_sub_basic_blocks (sbitmap blocks)
 	       precisely once.  */
 	    if (!initialized_src
 		|| (uninitialized_src
-		     && profile_status_for_fn (cfun) != PROFILE_READ))
+		     && profile_status_for_fn (cfun) < PROFILE_GUESSED))
 	      bb->count = profile_count::uninitialized ();
 	  }
  	/* If nothing changed, there is no need to create new BBs.  */
diff --git a/gcc/cfgcleanup.c b/gcc/cfgcleanup.c
index 25327d545ba..3a32938b05d 100644
--- a/gcc/cfgcleanup.c
+++ b/gcc/cfgcleanup.c
@@ -558,9 +558,7 @@ try_forward_edges (int mode, basic_block b)
       else
 	{
 	  /* Save the values now, as the edge may get removed.  */
-	  profile_count edge_count = e->count;
-	  profile_probability edge_probability = e->probability;
-	  int edge_frequency;
+	  profile_count edge_count = e->count ();
 	  int n = 0;
 
 	  e->goto_locus = goto_locus;
@@ -585,8 +583,6 @@ try_forward_edges (int mode, basic_block b)
 	  /* We successfully forwarded the edge.  Now update profile
 	     data: for each edge we traversed in the chain, remove
 	     the original edge's execution count.  */
-	  edge_frequency = edge_probability.apply (b->frequency);
-
 	  do
 	    {
 	      edge t;
@@ -596,16 +592,12 @@ try_forward_edges (int mode, basic_block b)
 		  gcc_assert (n < nthreaded_edges);
 		  t = threaded_edges [n++];
 		  gcc_assert (t->src == first);
-		  update_bb_profile_for_threading (first, edge_frequency,
-						   edge_count, t);
+		  update_bb_profile_for_threading (first, edge_count, t);
 		  update_br_prob_note (first);
 		}
 	      else
 		{
 		  first->count -= edge_count;
-		  first->frequency -= edge_frequency;
-		  if (first->frequency < 0)
-		    first->frequency = 0;
 		  /* It is possible that as the result of
 		     threading we've removed edge as it is
 		     threaded to the fallthru edge.  Avoid
@@ -616,7 +608,6 @@ try_forward_edges (int mode, basic_block b)
 		  t = single_succ_edge (first);
 		}
 
-	      t->count -= edge_count;
 	      first = t->dest;
 	    }
 	  while (first != target);
@@ -2111,7 +2102,7 @@ try_crossjump_to_edge (int mode, edge e1, edge e2,
   else
     redirect_edges_to = osrc2;
 
-  /* Recompute the frequencies and counts of outgoing edges.  */
+  /* Recompute the counts of destinations of outgoing edges.  */
   FOR_EACH_EDGE (s, ei, redirect_edges_to->succs)
     {
       edge s2;
@@ -2130,34 +2121,23 @@ try_crossjump_to_edge (int mode, edge e1, edge e2,
 	    break;
 	}
 
-      s->count += s2->count;
-
       /* Take care to update possible forwarder blocks.  We verified
 	 that there is no more than one in the chain, so we can't run
 	 into infinite loop.  */
       if (FORWARDER_BLOCK_P (s->dest))
-	{
-	  single_succ_edge (s->dest)->count += s2->count;
-	  s->dest->count += s2->count;
-	  s->dest->frequency += EDGE_FREQUENCY (s);
-	}
+	s->dest->count += s->count ();
 
       if (FORWARDER_BLOCK_P (s2->dest))
-	{
-	  single_succ_edge (s2->dest)->count -= s2->count;
-	  s2->dest->count -= s2->count;
-	  s2->dest->frequency -= EDGE_FREQUENCY (s);
-	  if (s2->dest->frequency < 0)
-	    s2->dest->frequency = 0;
-	}
+	s2->dest->count -= s->count ();
 
-      if (!redirect_edges_to->frequency && !src1->frequency)
+      /* FIXME: Is this correct? Should be rewritten to count API.  */
+      if (redirect_edges_to->count.nonzero_p () && src1->count.nonzero_p ())
 	s->probability = s->probability.combine_with_freq
-			   (redirect_edges_to->frequency,
-			    s2->probability, src1->frequency);
+			   (redirect_edges_to->count.to_frequency (cfun),
+			    s2->probability, src1->count.to_frequency (cfun));
     }
 
-  /* Adjust count and frequency for the block.  An earlier jump
+  /* Adjust count for the block.  An earlier jump
      threading pass may have left the profile in an inconsistent
      state (see update_bb_profile_for_threading) so we must be
      prepared for overflows.  */
@@ -2165,9 +2145,6 @@ try_crossjump_to_edge (int mode, edge e1, edge e2,
   do
     {
       tmp->count += src1->count;
-      tmp->frequency += src1->frequency;
-      if (tmp->frequency > BB_FREQ_MAX)
-        tmp->frequency = BB_FREQ_MAX;
       if (tmp == redirect_edges_to)
         break;
       tmp = find_fallthru_edge (tmp->succs)->dest;
diff --git a/gcc/cfgexpand.c b/gcc/cfgexpand.c
index 07b5c9df4c6..06a8af8a166 100644
--- a/gcc/cfgexpand.c
+++ b/gcc/cfgexpand.c
@@ -446,7 +446,7 @@ add_stack_var (tree decl)
   v->size = tree_to_poly_uint64 (size);
   /* Ensure that all variables have size, so that &a != &b for any two
      variables that are simultaneously live.  */
-  if (known_zero (v->size))
+  if (must_eq (v->size, 0U))
     v->size = 1;
   v->alignb = align_local_variable (decl);
   /* An alignment of zero can mightily confuse us later.  */
@@ -1183,7 +1183,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 
 	  /* If there were any variables requiring "large" alignment, allocate
 	     space.  */
-	  if (maybe_nonzero (large_size) && ! large_allocation_done)
+	  if (may_ne (large_size, 0U) && ! large_allocation_done)
 	    {
 	      poly_int64 loffset;
 	      rtx large_allocsize;
@@ -1397,10 +1397,18 @@ expand_one_ssa_partition (tree var)
     }
 
   machine_mode reg_mode = promote_ssa_mode (var, NULL);
-
   rtx x = gen_reg_rtx (reg_mode);
 
   set_rtl (var, x);
+
+  /* For a promoted variable, X will not be used directly but wrapped in a
+     SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
+     will assume that its upper bits can be inferred from its lower bits.
+     Therefore, if X isn't initialized on every path from the entry, then
+     we must do it manually in order to fulfill the above assumption.  */
+  if (reg_mode != TYPE_MODE (TREE_TYPE (var))
+      && bitmap_bit_p (SA.partitions_for_undefined_values, part))
+    emit_move_insn (x, CONST0_RTX (reg_mode));
 }
 
 /* Record the association between the RTL generated for partition PART
@@ -2521,8 +2529,7 @@ expand_gimple_cond (basic_block bb, gcond *stmt)
   dest = false_edge->dest;
   redirect_edge_succ (false_edge, new_bb);
   false_edge->flags |= EDGE_FALLTHRU;
-  new_bb->count = false_edge->count;
-  new_bb->frequency = EDGE_FREQUENCY (false_edge);
+  new_bb->count = false_edge->count ();
   loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
   add_bb_to_loop (new_bb, loop);
   if (loop->latch == bb
@@ -2648,8 +2655,7 @@ expand_call_stmt (gcall *stmt)
   CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (stmt);
   if (decl
       && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL
-      && (DECL_FUNCTION_CODE (decl) == BUILT_IN_ALLOCA
-	  || DECL_FUNCTION_CODE (decl) == BUILT_IN_ALLOCA_WITH_ALIGN))
+      && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
     CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (stmt);
   else
     CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (stmt);
@@ -2673,12 +2679,28 @@ expand_call_stmt (gcall *stmt)
 	  }
     }
 
+  rtx_insn *before_call = get_last_insn ();
   lhs = gimple_call_lhs (stmt);
   if (lhs)
     expand_assignment (lhs, exp, false);
   else
     expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
 
+  /* If the gimple call is an indirect call and has 'nocf_check'
+     attribute find a generated CALL insn to mark it as no
+     control-flow verification is needed.  */
+  if (gimple_call_nocf_check_p (stmt)
+      && !gimple_call_fndecl (stmt))
+    {
+      rtx_insn *last = get_last_insn ();
+      while (!CALL_P (last)
+	     && last != before_call)
+	last = PREV_INSN (last);
+
+      if (last != before_call)
+	add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
+    }
+
   mark_transaction_restart_calls (stmt);
 }
 
@@ -3832,20 +3854,13 @@ expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru)
      the exit block.  */
 
   probability = profile_probability::never ();
-  profile_count count = profile_count::zero ();
 
   for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
     {
       if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
 	{
 	  if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
-	    {
-	      e->dest->count -= e->count;
-	      e->dest->frequency -= EDGE_FREQUENCY (e);
-	      if (e->dest->frequency < 0)
-		e->dest->frequency = 0;
-	    }
-	  count += e->count;
+	    e->dest->count -= e->count ();
 	  probability += e->probability;
 	  remove_edge (e);
 	}
@@ -3875,7 +3890,6 @@ expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru)
   e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
 		 | EDGE_SIBCALL);
   e->probability = probability;
-  e->count = count;
   BB_END (bb) = last;
   update_bb_for_insn (bb);
 
@@ -4472,7 +4486,7 @@ expand_debug_expr (tree exp)
 				 &unsignedp, &reversep, &volatilep);
 	rtx orig_op0;
 
-	if (known_zero (bitsize))
+	if (must_eq (bitsize, 0))
 	  return NULL;
 
 	orig_op0 = op0 = expand_debug_expr (tem);
@@ -4516,12 +4530,12 @@ expand_debug_expr (tree exp)
 	      /* Bitfield.  */
 	      mode1 = smallest_int_mode_for_size (bitsize);
 	    poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
-	    if (maybe_nonzero (bytepos))
+	    if (may_ne (bytepos, 0))
 	      {
 		op0 = adjust_address_nv (op0, mode1, bytepos);
 		bitpos = num_trailing_bits (bitpos);
 	      }
-	    else if (known_zero (bitpos)
+	    else if (must_eq (bitpos, 0)
 		     && must_eq (bitsize, GET_MODE_BITSIZE (mode)))
 	      op0 = adjust_address_nv (op0, mode, 0);
 	    else if (GET_MODE (op0) != mode1)
@@ -4533,7 +4547,7 @@ expand_debug_expr (tree exp)
 	    set_mem_attributes (op0, exp, 0);
 	  }
 
-	if (known_zero (bitpos) && mode == GET_MODE (op0))
+	if (must_eq (bitpos, 0) && mode == GET_MODE (op0))
 	  return op0;
 
 	if (may_lt (bitpos, 0))
@@ -5863,7 +5877,6 @@ construct_init_block (void)
   init_block = create_basic_block (NEXT_INSN (get_insns ()),
 				   get_last_insn (),
 				   ENTRY_BLOCK_PTR_FOR_FN (cfun));
-  init_block->frequency = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
   init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
   add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
   if (e)
@@ -5927,7 +5940,7 @@ construct_exit_block (void)
   while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
     head = NEXT_INSN (head);
   /* But make sure exit_block starts with RETURN_LABEL, otherwise the
-     bb frequency counting will be confused.  Any instructions before that
+     bb count counting will be confused.  Any instructions before that
      label are emitted for the case where PREV_BB falls through into the
      exit block, so append those instructions to prev_bb in that case.  */
   if (NEXT_INSN (head) != return_label)
@@ -5940,7 +5953,6 @@ construct_exit_block (void)
 	}
     }
   exit_block = create_basic_block (NEXT_INSN (head), end, prev_bb);
-  exit_block->frequency = EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency;
   exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
   add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
 
@@ -5959,12 +5971,8 @@ construct_exit_block (void)
   FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
     if (e2 != e)
       {
-	e->count -= e2->count;
-	exit_block->count -= e2->count;
-	exit_block->frequency -= EDGE_FREQUENCY (e2);
+	exit_block->count -= e2->count ();
       }
-  if (exit_block->frequency < 0)
-    exit_block->frequency = 0;
   update_bb_for_insn (exit_block);
 }
 
diff --git a/gcc/cfghooks.c b/gcc/cfghooks.c
index 18dc49a035e..4a224243e32 100644
--- a/gcc/cfghooks.c
+++ b/gcc/cfghooks.c
@@ -146,12 +146,15 @@ verify_flow_info (void)
 	  error ("verify_flow_info: Wrong count of block %i", bb->index);
 	  err = 1;
 	}
-      if (bb->frequency < 0)
+      /* FIXME: Graphite and SLJL and target code still tends to produce
+	 edges with no probablity.  */
+      if (profile_status_for_fn (cfun) >= PROFILE_GUESSED
+          && !bb->count.initialized_p () && !flag_graphite && 0)
 	{
-	  error ("verify_flow_info: Wrong frequency of block %i %i",
-		 bb->index, bb->frequency);
+	  error ("verify_flow_info: Missing count of block %i", bb->index);
 	  err = 1;
 	}
+
       FOR_EACH_EDGE (e, ei, bb->succs)
 	{
 	  if (last_visited [e->dest->index] == bb)
@@ -160,15 +163,18 @@ verify_flow_info (void)
 		     e->src->index, e->dest->index);
 	      err = 1;
 	    }
-	  if (!e->probability.verify ())
+	  /* FIXME: Graphite and SLJL and target code still tends to produce
+	     edges with no probablity.  */
+	  if (profile_status_for_fn (cfun) >= PROFILE_GUESSED
+	      && !e->probability.initialized_p () && !flag_graphite && 0)
 	    {
-	      error ("verify_flow_info: Wrong probability of edge %i->%i",
-		     e->src->index, e->dest->index);
+	      error ("Uninitialized probability of edge %i->%i", e->src->index,
+		     e->dest->index);
 	      err = 1;
 	    }
-	  if (!e->count.verify ())
+	  if (!e->probability.verify ())
 	    {
-	      error ("verify_flow_info: Wrong count of edge %i->%i",
+	      error ("verify_flow_info: Wrong probability of edge %i->%i",
 		     e->src->index, e->dest->index);
 	      err = 1;
 	    }
@@ -311,7 +317,6 @@ dump_bb_for_graph (pretty_printer *pp, basic_block bb)
   /* TODO: Add pretty printer for counter.  */
   if (bb->count.initialized_p ())
     pp_printf (pp, "COUNT:" "%" PRId64, bb->count.to_gcov_type ());
-  pp_printf (pp, " FREQ:%i |", bb->frequency);
   pp_write_text_to_stream (pp);
   if (!(dump_flags & TDF_SLIM))
     cfg_hooks->dump_bb_for_graph (pp, bb);
@@ -443,7 +448,6 @@ redirect_edge_succ_nodup (edge e, basic_block new_succ)
     {
       s->flags |= e->flags;
       s->probability += e->probability;
-      s->count += e->count;
       /* FIXME: This should be called via a hook and only for IR_GIMPLE.  */
       redirect_edge_var_map_dup (s, e);
       remove_edge (e);
@@ -510,7 +514,6 @@ split_block_1 (basic_block bb, void *i)
     return NULL;
 
   new_bb->count = bb->count;
-  new_bb->frequency = bb->frequency;
   new_bb->discriminator = bb->discriminator;
 
   if (dom_info_available_p (CDI_DOMINATORS))
@@ -622,8 +625,7 @@ basic_block
 split_edge (edge e)
 {
   basic_block ret;
-  profile_count count = e->count;
-  int freq = EDGE_FREQUENCY (e);
+  profile_count count = e->count ();
   edge f;
   bool irr = (e->flags & EDGE_IRREDUCIBLE_LOOP) != 0;
   struct loop *loop;
@@ -637,9 +639,7 @@ split_edge (edge e)
 
   ret = cfg_hooks->split_edge (e);
   ret->count = count;
-  ret->frequency = freq;
   single_succ_edge (ret)->probability = profile_probability::always ();
-  single_succ_edge (ret)->count = count;
 
   if (irr)
     {
@@ -867,8 +867,6 @@ make_forwarder_block (basic_block bb, bool (*redirect_edge_p) (edge),
   fallthru = split_block_after_labels (bb);
   dummy = fallthru->src;
   dummy->count = profile_count::zero ();
-  dummy->frequency = 0;
-  fallthru->count = profile_count::zero ();
   bb = fallthru->dest;
 
   /* Redirect back edges we want to keep.  */
@@ -878,12 +876,7 @@ make_forwarder_block (basic_block bb, bool (*redirect_edge_p) (edge),
 
       if (redirect_edge_p (e))
 	{
-	  dummy->frequency += EDGE_FREQUENCY (e);
-	  if (dummy->frequency > BB_FREQ_MAX)
-	    dummy->frequency = BB_FREQ_MAX;
-
-	  dummy->count += e->count;
-	  fallthru->count += e->count;
+	  dummy->count += e->count ();
 	  ei_next (&ei);
 	  continue;
 	}
@@ -1069,7 +1062,7 @@ duplicate_block (basic_block bb, edge e, basic_block after)
 {
   edge s, n;
   basic_block new_bb;
-  profile_count new_count = e ? e->count : profile_count::uninitialized ();
+  profile_count new_count = e ? e->count (): profile_count::uninitialized ();
   edge_iterator ei;
 
   if (!cfg_hooks->duplicate_block)
@@ -1093,13 +1086,6 @@ duplicate_block (basic_block bb, edge e, basic_block after)
 	 is no need to actually check for duplicated edges.  */
       n = unchecked_make_edge (new_bb, s->dest, s->flags);
       n->probability = s->probability;
-      if (e && bb->count > profile_count::zero ())
-	{
-	  n->count = s->count.apply_scale (new_count, bb->count);
-	  s->count -= n->count;
-	}
-      else
-	n->count = s->count;
       n->aux = s->aux;
     }
 
@@ -1108,19 +1094,10 @@ duplicate_block (basic_block bb, edge e, basic_block after)
       new_bb->count = new_count;
       bb->count -= new_count;
 
-      new_bb->frequency = EDGE_FREQUENCY (e);
-      bb->frequency -= EDGE_FREQUENCY (e);
-
       redirect_edge_and_branch_force (e, new_bb);
-
-      if (bb->frequency < 0)
-	bb->frequency = 0;
     }
   else
-    {
-      new_bb->count = bb->count;
-      new_bb->frequency = bb->frequency;
-    }
+    new_bb->count = bb->count;
 
   set_bb_original (new_bb, bb);
   set_bb_copy (bb, new_bb);
@@ -1463,23 +1440,16 @@ account_profile_record (struct profile_record *record, int after_pass)
 	    record->num_mismatched_freq_out[after_pass]++;
 	  profile_count lsum = profile_count::zero ();
 	  FOR_EACH_EDGE (e, ei, bb->succs)
-	    lsum += e->count;
+	    lsum += e->count ();
 	  if (EDGE_COUNT (bb->succs) && (lsum.differs_from_p (bb->count)))
 	    record->num_mismatched_count_out[after_pass]++;
 	}
       if (bb != ENTRY_BLOCK_PTR_FOR_FN (cfun)
 	  && profile_status_for_fn (cfun) != PROFILE_ABSENT)
 	{
-	  int sum = 0;
-	  FOR_EACH_EDGE (e, ei, bb->preds)
-	    sum += EDGE_FREQUENCY (e);
-	  if (abs (sum - bb->frequency) > 100
-	      || (MAX (sum, bb->frequency) > 10
-		  && abs ((sum - bb->frequency) * 100 / (MAX (sum, bb->frequency) + 1)) > 10))
-	    record->num_mismatched_freq_in[after_pass]++;
 	  profile_count lsum = profile_count::zero ();
 	  FOR_EACH_EDGE (e, ei, bb->preds)
-	    lsum += e->count;
+	    lsum += e->count ();
 	  if (lsum.differs_from_p (bb->count))
 	    record->num_mismatched_count_in[after_pass]++;
 	}
diff --git a/gcc/cfgloop.c b/gcc/cfgloop.c
index c3bd9c05013..d82da97d7af 100644
--- a/gcc/cfgloop.c
+++ b/gcc/cfgloop.c
@@ -599,15 +599,15 @@ find_subloop_latch_edge_by_profile (vec<edge> latches)
 
   FOR_EACH_VEC_ELT (latches, i, e)
     {
-      if (e->count > mcount)
+      if (e->count ()> mcount)
 	{
 	  me = e;
-	  mcount = e->count;
+	  mcount = e->count();
 	}
-      tcount += e->count;
+      tcount += e->count();
     }
 
-  if (!tcount.initialized_p () || tcount < HEAVY_EDGE_MIN_SAMPLES
+  if (!tcount.initialized_p () || !(tcount.ipa () > HEAVY_EDGE_MIN_SAMPLES)
       || (tcount - mcount).apply_scale (HEAVY_EDGE_RATIO, 1) > tcount)
     return NULL;
 
diff --git a/gcc/cfgloopanal.c b/gcc/cfgloopanal.c
index 73710abac6e..78a3c9387aa 100644
--- a/gcc/cfgloopanal.c
+++ b/gcc/cfgloopanal.c
@@ -213,9 +213,10 @@ average_num_loop_insns (const struct loop *loop)
 	if (NONDEBUG_INSN_P (insn))
 	  binsns++;
 
-      ratio = loop->header->frequency == 0
+      ratio = loop->header->count.to_frequency (cfun) == 0
 	      ? BB_FREQ_MAX
-	      : (bb->frequency * BB_FREQ_MAX) / loop->header->frequency;
+	      : (bb->count.to_frequency (cfun) * BB_FREQ_MAX)
+		 / loop->header->count.to_frequency (cfun);
       ninsns += binsns * ratio;
     }
   free (bbs);
@@ -245,58 +246,38 @@ expected_loop_iterations_unbounded (const struct loop *loop,
   /* If we have no profile at all, use AVG_LOOP_NITER.  */
   if (profile_status_for_fn (cfun) == PROFILE_ABSENT)
     expected = PARAM_VALUE (PARAM_AVG_LOOP_NITER);
-  else if (loop->latch && (loop->latch->count.reliable_p ()
-			   || loop->header->count.reliable_p ()))
+  else if (loop->latch && (loop->latch->count.initialized_p ()
+			   || loop->header->count.initialized_p ()))
     {
       profile_count count_in = profile_count::zero (),
 		    count_latch = profile_count::zero ();
 
       FOR_EACH_EDGE (e, ei, loop->header->preds)
 	if (e->src == loop->latch)
-	  count_latch = e->count;
+	  count_latch = e->count ();
 	else
-	  count_in += e->count;
+	  count_in += e->count ();
 
       if (!count_latch.initialized_p ())
-	;
-      else if (!(count_in > profile_count::zero ()))
+	expected = PARAM_VALUE (PARAM_AVG_LOOP_NITER);
+      else if (!count_in.nonzero_p ())
 	expected = count_latch.to_gcov_type () * 2;
       else
 	{
 	  expected = (count_latch.to_gcov_type () + count_in.to_gcov_type ()
 		      - 1) / count_in.to_gcov_type ();
-	  if (read_profile_p)
+	  if (read_profile_p
+	      && count_latch.reliable_p () && count_in.reliable_p ())
 	    *read_profile_p = true;
 	}
     }
-  if (expected == -1)
-    {
-      int freq_in, freq_latch;
-
-      freq_in = 0;
-      freq_latch = 0;
-
-      FOR_EACH_EDGE (e, ei, loop->header->preds)
-	if (flow_bb_inside_loop_p (loop, e->src))
-	  freq_latch += EDGE_FREQUENCY (e);
-	else
-	  freq_in += EDGE_FREQUENCY (e);
-
-      if (freq_in == 0)
-	{
-	  /* If we have no profile at all, use AVG_LOOP_NITER iterations.  */
-	  if (!freq_latch)
-	    expected = PARAM_VALUE (PARAM_AVG_LOOP_NITER);
-	  else
-	    expected = freq_latch * 2;
-	}
-      else
-        expected = (freq_latch + freq_in - 1) / freq_in;
-    }
+  else
+    expected = PARAM_VALUE (PARAM_AVG_LOOP_NITER);
 
   HOST_WIDE_INT max = get_max_loop_iterations_int (loop);
   if (max != -1 && max < expected)
     return max;
+ 
   return expected;
 }
 
diff --git a/gcc/cfgloopmanip.c b/gcc/cfgloopmanip.c
index fd335c3fe1d..1f55137ed97 100644
--- a/gcc/cfgloopmanip.c
+++ b/gcc/cfgloopmanip.c
@@ -536,7 +536,6 @@ scale_loop_profile (struct loop *loop, profile_probability p,
       if (e)
 	{
 	  edge other_e;
-	  int freq_delta;
 	  profile_count count_delta;
 
           FOR_EACH_EDGE (other_e, ei, e->src->succs)
@@ -545,27 +544,18 @@ scale_loop_profile (struct loop *loop, profile_probability p,
 	      break;
 
 	  /* Probability of exit must be 1/iterations.  */
-	  freq_delta = EDGE_FREQUENCY (e);
+	  count_delta = e->count ();
 	  e->probability = profile_probability::always ()
 				.apply_scale (1, iteration_bound);
 	  other_e->probability = e->probability.invert ();
-	  freq_delta -= EDGE_FREQUENCY (e);
+	  count_delta -= e->count ();
 
-	  /* Adjust counts accordingly.  */
-	  count_delta = e->count;
-	  e->count = e->src->count.apply_probability (e->probability);
-	  other_e->count = e->src->count.apply_probability (other_e->probability);
-	  count_delta -= e->count;
-
-	  /* If latch exists, change its frequency and count, since we changed
+	  /* If latch exists, change its count, since we changed
 	     probability of exit.  Theoretically we should update everything from
 	     source of exit edge to latch, but for vectorizer this is enough.  */
 	  if (loop->latch
 	      && loop->latch != e->src)
 	    {
-	      loop->latch->frequency += freq_delta;
-	      if (loop->latch->frequency < 0)
-		loop->latch->frequency = 0;
 	      loop->latch->count += count_delta;
 	    }
 	}
@@ -575,34 +565,20 @@ scale_loop_profile (struct loop *loop, profile_probability p,
 	 we look at the actual profile, if it is available.  */
       p = p.apply_scale (iteration_bound, iterations);
 
-      bool determined = false;
       if (loop->header->count.initialized_p ())
 	{
 	  profile_count count_in = profile_count::zero ();
 
 	  FOR_EACH_EDGE (e, ei, loop->header->preds)
 	    if (e->src != loop->latch)
-	      count_in += e->count;
+	      count_in += e->count ();
 
 	  if (count_in > profile_count::zero () )
 	    {
 	      p = count_in.probability_in (loop->header->count.apply_scale
 						 (iteration_bound, 1));
-	      determined = true;
 	    }
 	}
-      if (!determined && loop->header->frequency)
-	{
-	  int freq_in = 0;
-
-	  FOR_EACH_EDGE (e, ei, loop->header->preds)
-	    if (e->src != loop->latch)
-	      freq_in += EDGE_FREQUENCY (e);
-
-	  if (freq_in != 0)
-	    p = profile_probability::probability_in_gcov_type
-			 (freq_in * iteration_bound, loop->header->frequency);
-	}
       if (!(p > profile_probability::never ()))
 	p = profile_probability::very_unlikely ();
     }
@@ -804,7 +780,7 @@ create_empty_loop_on_edge (edge entry_edge,
   loop->latch = loop_latch;
   add_loop (loop, outer);
 
-  /* TODO: Fix frequencies and counts.  */
+  /* TODO: Fix counts.  */
   scale_loop_frequencies (loop, profile_probability::even ());
 
   /* Update dominators.  */
@@ -870,16 +846,12 @@ loopify (edge latch_edge, edge header_edge,
   basic_block pred_bb = header_edge->src;
   struct loop *loop = alloc_loop ();
   struct loop *outer = loop_outer (succ_bb->loop_father);
-  int freq;
   profile_count cnt;
-  edge e;
-  edge_iterator ei;
 
   loop->header = header_edge->dest;
   loop->latch = latch_edge->src;
 
-  freq = EDGE_FREQUENCY (header_edge);
-  cnt = header_edge->count;
+  cnt = header_edge->count ();
 
   /* Redirect edges.  */
   loop_redirect_edge (latch_edge, loop->header);
@@ -907,15 +879,10 @@ loopify (edge latch_edge, edge header_edge,
     remove_bb_from_loops (switch_bb);
   add_bb_to_loop (switch_bb, outer);
 
-  /* Fix frequencies.  */
+  /* Fix counts.  */
   if (redirect_all_edges)
     {
-      switch_bb->frequency = freq;
       switch_bb->count = cnt;
-      FOR_EACH_EDGE (e, ei, switch_bb->succs)
-	{
-	  e->count = switch_bb->count.apply_probability (e->probability);
-	}
     }
   scale_loop_frequencies (loop, false_scale);
   scale_loop_frequencies (succ_bb->loop_father, true_scale);
@@ -1177,7 +1144,7 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
     {
       /* Calculate coefficients by that we have to scale frequencies
 	 of duplicated loop bodies.  */
-      freq_in = header->frequency;
+      freq_in = header->count.to_frequency (cfun);
       freq_le = EDGE_FREQUENCY (latch_edge);
       if (freq_in == 0)
 	freq_in = 1;
@@ -1650,8 +1617,6 @@ lv_adjust_loop_entry_edge (basic_block first_head, basic_block second_head,
 		  current_ir_type () == IR_GIMPLE ? EDGE_TRUE_VALUE : 0);
   e1->probability = then_prob;
   e->probability = else_prob;
-  e1->count = e->count.apply_probability (e1->probability);
-  e->count = e->count.apply_probability (e->probability);
 
   set_immediate_dominator (CDI_DOMINATORS, first_head, new_head);
   set_immediate_dominator (CDI_DOMINATORS, second_head, new_head);
diff --git a/gcc/cfgrtl.c b/gcc/cfgrtl.c
index 739d1bb9490..ae469088eec 100644
--- a/gcc/cfgrtl.c
+++ b/gcc/cfgrtl.c
@@ -1156,7 +1156,6 @@ try_redirect_by_replacing_jump (edge e, basic_block target, bool in_cfglayout)
     e->flags = 0;
 
   e->probability = profile_probability::always ();
-  e->count = src->count;
 
   if (e->dest != target)
     redirect_edge_succ (e, target);
@@ -1505,9 +1504,7 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label)
 	  int prob = XINT (note, 0);
 
 	  b->probability = profile_probability::from_reg_br_prob_note (prob);
-	  b->count = e->count.apply_probability (b->probability);
 	  e->probability -= e->probability;
-	  e->count -= b->count;
 	}
     }
 
@@ -1536,6 +1533,7 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label)
 
 	  basic_block bb = create_basic_block (BB_HEAD (e->dest), NULL,
 					       ENTRY_BLOCK_PTR_FOR_FN (cfun));
+	  bb->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
 
 	  /* Change the existing edge's source to be the new block, and add
 	     a new edge from the entry block to the new block.  */
@@ -1615,7 +1613,7 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label)
   if (EDGE_COUNT (e->src->succs) >= 2 || abnormal_edge_flags || asm_goto_edge)
     {
       rtx_insn *new_head;
-      profile_count count = e->count;
+      profile_count count = e->count ();
       profile_probability probability = e->probability;
       /* Create the new structures.  */
 
@@ -1631,7 +1629,6 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label)
 
       jump_block = create_basic_block (new_head, NULL, e->src);
       jump_block->count = count;
-      jump_block->frequency = EDGE_FREQUENCY (e);
 
       /* Make sure new block ends up in correct hot/cold section.  */
 
@@ -1640,7 +1637,6 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label)
       /* Wire edge in.  */
       new_edge = make_edge (e->src, jump_block, EDGE_FALLTHRU);
       new_edge->probability = probability;
-      new_edge->count = count;
 
       /* Redirect old edge.  */
       redirect_edge_pred (e, jump_block);
@@ -1655,13 +1651,10 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label)
       if (asm_goto_edge)
 	{
 	  new_edge->probability = new_edge->probability.apply_scale (1, 2);
-	  new_edge->count = new_edge->count.apply_scale (1, 2);
 	  jump_block->count = jump_block->count.apply_scale (1, 2);
-	  jump_block->frequency /= 2;
 	  edge new_edge2 = make_edge (new_edge->src, target,
 				      e->flags & ~EDGE_FALLTHRU);
 	  new_edge2->probability = probability - new_edge->probability;
-	  new_edge2->count = count - new_edge->count;
 	}
 
       new_bb = jump_block;
@@ -2251,9 +2244,23 @@ void
 update_br_prob_note (basic_block bb)
 {
   rtx note;
-  if (!JUMP_P (BB_END (bb)) || !BRANCH_EDGE (bb)->probability.initialized_p ())
-    return;
   note = find_reg_note (BB_END (bb), REG_BR_PROB, NULL_RTX);
+  if (!JUMP_P (BB_END (bb)) || !BRANCH_EDGE (bb)->probability.initialized_p ())
+    {
+      if (note)
+	{
+	  rtx *note_link, this_rtx;
+
+	  note_link = &REG_NOTES (BB_END (bb));
+	  for (this_rtx = *note_link; this_rtx; this_rtx = XEXP (this_rtx, 1))
+	    if (this_rtx == note)
+	      {
+		*note_link = XEXP (this_rtx, 1);
+		break;
+	      }
+	}
+      return;
+    }
   if (!note
       || XINT (note, 0) == BRANCH_EDGE (bb)->probability.to_reg_br_prob_note ())
     return;
@@ -3155,7 +3162,6 @@ purge_dead_edges (basic_block bb)
       if (single_succ_p (bb))
 	{
 	  single_succ_edge (bb)->probability = profile_probability::always ();
-	  single_succ_edge (bb)->count = bb->count;
 	}
       else
 	{
@@ -3168,8 +3174,6 @@ purge_dead_edges (basic_block bb)
 	  b->probability = profile_probability::from_reg_br_prob_note
 					 (XINT (note, 0));
 	  f->probability = b->probability.invert ();
-	  b->count = bb->count.apply_probability (b->probability);
-	  f->count = bb->count.apply_probability (f->probability);
 	}
 
       return purged;
@@ -3221,7 +3225,6 @@ purge_dead_edges (basic_block bb)
   gcc_assert (single_succ_p (bb));
 
   single_succ_edge (bb)->probability = profile_probability::always ();
-  single_succ_edge (bb)->count = bb->count;
 
   if (dump_file)
     fprintf (dump_file, "Purged non-fallthru edges from bb %i\n",
@@ -3633,7 +3636,6 @@ relink_block_chain (bool stay_in_cfglayout_mode)
 	    fprintf (dump_file, "compensation ");
 	  else
 	    fprintf (dump_file, "bb %i ", bb->index);
-	  fprintf (dump_file, " [%i]\n", bb->frequency);
 	}
     }
 
@@ -4906,7 +4908,6 @@ rtl_flow_call_edges_add (sbitmap blocks)
 
 	      edge ne = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE);
 	      ne->probability = profile_probability::guessed_never ();
-	      ne->count = profile_count::guessed_zero ();
 	    }
 
 	  if (insn == BB_HEAD (bb))
@@ -5045,7 +5046,7 @@ rtl_account_profile_record (basic_block bb, int after_pass,
 	    += insn_cost (insn, true) * bb->count.to_gcov_type ();
 	else if (profile_status_for_fn (cfun) == PROFILE_GUESSED)
 	  record->time[after_pass]
-	    += insn_cost (insn, true) * bb->frequency;
+	    += insn_cost (insn, true) * bb->count.to_frequency (cfun);
       }
 }
 
diff --git a/gcc/cgraph.c b/gcc/cgraph.c
index d8da3dd76cd..7c3507c6ece 100644
--- a/gcc/cgraph.c
+++ b/gcc/cgraph.c
@@ -862,7 +862,7 @@ symbol_table::create_edge (cgraph_node *caller, cgraph_node *callee,
   edge->next_callee = NULL;
   edge->lto_stmt_uid = 0;
 
-  edge->count = count;
+  edge->count = count.ipa ();
   edge->frequency = freq;
   gcc_checking_assert (freq >= 0);
   gcc_checking_assert (freq <= CGRAPH_FREQ_MAX);
@@ -1308,7 +1308,7 @@ cgraph_edge::redirect_call_stmt_to_callee (void)
 	  /* We are producing the final function body and will throw away the
 	     callgraph edges really soon.  Reset the counts/frequencies to
 	     keep verifier happy in the case of roundoff errors.  */
-	  e->count = gimple_bb (e->call_stmt)->count;
+	  e->count = gimple_bb (e->call_stmt)->count.ipa ();
 	  e->frequency = compute_call_stmt_bb_frequency
 			  (e->caller->decl, gimple_bb (e->call_stmt));
 	}
@@ -1338,7 +1338,7 @@ cgraph_edge::redirect_call_stmt_to_callee (void)
 	    prob = profile_probability::even ();
 	  new_stmt = gimple_ic (e->call_stmt,
 				dyn_cast<cgraph_node *> (ref->referred),
-				prob, e->count, e->count + e2->count);
+				prob);
 	  e->speculative = false;
 	  e->caller->set_call_stmt_including_clones (e->call_stmt, new_stmt,
 						     false);
@@ -1644,7 +1644,7 @@ cgraph_update_edges_for_call_stmt_node (cgraph_node *node,
 	  /* Otherwise remove edge and create new one; we can't simply redirect
 	     since function has changed, so inline plan and other information
 	     attached to edge is invalid.  */
-	  count = e->count;
+	  count = e->count.ipa ();
 	  frequency = e->frequency;
  	  if (e->indirect_unknown_callee || e->inline_failed)
 	    e->remove ();
@@ -1655,7 +1655,7 @@ cgraph_update_edges_for_call_stmt_node (cgraph_node *node,
 	{
 	  /* We are seeing new direct call; compute profile info based on BB.  */
 	  basic_block bb = gimple_bb (new_stmt);
-	  count = bb->count;
+	  count = bb->count.ipa ();
 	  frequency = compute_call_stmt_bb_frequency (current_function_decl,
 						      bb);
 	}
@@ -2530,6 +2530,53 @@ cgraph_node::set_nothrow_flag (bool nothrow)
   return changed;
 }
 
+/* Worker to set malloc flag.  */
+static void
+set_malloc_flag_1 (cgraph_node *node, bool malloc_p, bool *changed)
+{
+  if (malloc_p && !DECL_IS_MALLOC (node->decl))
+    {
+      DECL_IS_MALLOC (node->decl) = true;
+      *changed = true;
+    }
+
+  ipa_ref *ref;
+  FOR_EACH_ALIAS (node, ref)
+    {
+      cgraph_node *alias = dyn_cast<cgraph_node *> (ref->referring);
+      if (!malloc_p || alias->get_availability () > AVAIL_INTERPOSABLE)
+	set_malloc_flag_1 (alias, malloc_p, changed);
+    }
+
+  for (cgraph_edge *e = node->callers; e; e = e->next_caller)
+    if (e->caller->thunk.thunk_p
+	&& (!malloc_p || e->caller->get_availability () > AVAIL_INTERPOSABLE))
+      set_malloc_flag_1 (e->caller, malloc_p, changed);
+}
+
+/* Set DECL_IS_MALLOC on NODE's decl and on NODE's aliases if any.  */
+
+bool
+cgraph_node::set_malloc_flag (bool malloc_p)
+{
+  bool changed = false;
+
+  if (!malloc_p || get_availability () > AVAIL_INTERPOSABLE)
+    set_malloc_flag_1 (this, malloc_p, &changed);
+  else
+    {
+      ipa_ref *ref;
+
+      FOR_EACH_ALIAS (this, ref)
+	{
+	  cgraph_node *alias = dyn_cast<cgraph_node *> (ref->referring);
+	  if (!malloc_p || alias->get_availability () > AVAIL_INTERPOSABLE)
+	    set_malloc_flag_1 (alias, malloc_p, &changed);
+	}
+    }
+  return changed;
+}
+
 /* Worker to set_const_flag.  */
 
 static void
@@ -3035,9 +3082,14 @@ bool
 cgraph_edge::verify_count_and_frequency ()
 {
   bool error_found = false;
-  if (count < 0)
+  if (!count.verify ())
     {
-      error ("caller edge count is negative");
+      error ("caller edge count invalid");
+      error_found = true;
+    }
+  if (count.initialized_p () && !(count.ipa () == count))
+    {
+      error ("caller edge count is local");
       error_found = true;
     }
   if (frequency < 0)
@@ -3136,9 +3188,14 @@ cgraph_node::verify_node (void)
 	       identifier_to_locale (e->callee->name ()));
 	error_found = true;
       }
-  if (count < 0)
+  if (!count.verify ())
+    {
+      error ("cgraph count invalid");
+      error_found = true;
+    }
+  if (count.initialized_p () && !(count.ipa () == count))
     {
-      error ("execution count is negative");
+      error ("cgraph count is local");
       error_found = true;
     }
   if (global.inlined_to && same_comdat_group)
@@ -3222,7 +3279,9 @@ cgraph_node::verify_node (void)
     {
       if (e->verify_count_and_frequency ())
 	error_found = true;
+      /* FIXME: re-enable once cgraph is converted to counts.  */
       if (gimple_has_body_p (e->caller->decl)
+	  && 0
 	  && !e->caller->global.inlined_to
 	  && !e->speculative
 	  /* Optimized out calls are redirected to __builtin_unreachable.  */
@@ -3245,9 +3304,11 @@ cgraph_node::verify_node (void)
     {
       if (e->verify_count_and_frequency ())
 	error_found = true;
+      /* FIXME: re-enable once cgraph is converted to counts.  */
       if (gimple_has_body_p (e->caller->decl)
 	  && !e->caller->global.inlined_to
 	  && !e->speculative
+	  && 0
 	  && (e->frequency
 	      != compute_call_stmt_bb_frequency (e->caller->decl,
 						 gimple_bb (e->call_stmt))))
diff --git a/gcc/cgraph.h b/gcc/cgraph.h
index 1758e8b08c1..84824e9f814 100644
--- a/gcc/cgraph.h
+++ b/gcc/cgraph.h
@@ -1151,6 +1151,10 @@ public:
      if any to NOTHROW.  */
   bool set_nothrow_flag (bool nothrow);
 
+  /* SET DECL_IS_MALLOC on cgraph_node's decl and on aliases of the node
+     if any.  */
+  bool set_malloc_flag (bool malloc_p);
+
   /* If SET_CONST is true, mark function, aliases and thunks to be ECF_CONST.
     If SET_CONST if false, clear the flag.
 
diff --git a/gcc/cgraphbuild.c b/gcc/cgraphbuild.c
index d853acd883d..dd4bf9a7fa3 100644
--- a/gcc/cgraphbuild.c
+++ b/gcc/cgraphbuild.c
@@ -190,21 +190,8 @@ record_eh_tables (cgraph_node *node, function *fun)
 int
 compute_call_stmt_bb_frequency (tree decl, basic_block bb)
 {
-  int entry_freq = ENTRY_BLOCK_PTR_FOR_FN
-  		     (DECL_STRUCT_FUNCTION (decl))->frequency;
-  int freq = bb->frequency;
-
-  if (profile_status_for_fn (DECL_STRUCT_FUNCTION (decl)) == PROFILE_ABSENT)
-    return CGRAPH_FREQ_BASE;
-
-  if (!entry_freq)
-    entry_freq = 1, freq++;
-
-  freq = freq * CGRAPH_FREQ_BASE / entry_freq;
-  if (freq > CGRAPH_FREQ_MAX)
-    freq = CGRAPH_FREQ_MAX;
-
-  return freq;
+  return bb->count.to_cgraph_frequency
+      (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (decl))->count);
 }
 
 /* Mark address taken in STMT.  */
@@ -415,7 +402,7 @@ cgraph_edge::rebuild_edges (void)
   node->remove_callees ();
   node->remove_all_references ();
 
-  node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
+  node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ();
 
   FOR_EACH_BB_FN (bb, cfun)
     {
diff --git a/gcc/cgraphunit.c b/gcc/cgraphunit.c
index 9385dc825ab..c5183a02058 100644
--- a/gcc/cgraphunit.c
+++ b/gcc/cgraphunit.c
@@ -1601,17 +1601,12 @@ init_lowered_empty_function (tree decl, bool in_ssa, profile_count count)
 
   /* Create BB for body of the function and connect it properly.  */
   ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = count;
-  ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency = BB_FREQ_MAX;
   EXIT_BLOCK_PTR_FOR_FN (cfun)->count = count;
-  EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency = BB_FREQ_MAX;
   bb = create_basic_block (NULL, ENTRY_BLOCK_PTR_FOR_FN (cfun));
   bb->count = count;
-  bb->frequency = BB_FREQ_MAX;
   e = make_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), bb, EDGE_FALLTHRU);
-  e->count = count;
   e->probability = profile_probability::always ();
   e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
-  e->count = count;
   e->probability = profile_probability::always ();
   add_bb_to_loop (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
 
@@ -1854,8 +1849,12 @@ cgraph_node::expand_thunk (bool output_asm_thunks, bool force_gimple_thunk)
       else
 	resdecl = DECL_RESULT (thunk_fndecl);
 
+      profile_count cfg_count = count;
+      if (!cfg_count.initialized_p ())
+	cfg_count = profile_count::from_gcov_type (BB_FREQ_MAX).guessed_local ();
+
       bb = then_bb = else_bb = return_bb
-	= init_lowered_empty_function (thunk_fndecl, true, count);
+	= init_lowered_empty_function (thunk_fndecl, true, cfg_count);
 
       bsi = gsi_start_bb (bb);
 
@@ -1968,14 +1967,11 @@ cgraph_node::expand_thunk (bool output_asm_thunks, bool force_gimple_thunk)
 		     adjustment, because that's why we're emitting a
 		     thunk.  */
 		  then_bb = create_basic_block (NULL, bb);
-		  then_bb->count = count - count.apply_scale (1, 16);
-		  then_bb->frequency = BB_FREQ_MAX - BB_FREQ_MAX / 16;
+		  then_bb->count = cfg_count - cfg_count.apply_scale (1, 16);
 		  return_bb = create_basic_block (NULL, then_bb);
-		  return_bb->count = count;
-		  return_bb->frequency = BB_FREQ_MAX;
+		  return_bb->count = cfg_count;
 		  else_bb = create_basic_block (NULL, else_bb);
-		  then_bb->count = count.apply_scale (1, 16);
-		  then_bb->frequency = BB_FREQ_MAX / 16;
+		  else_bb->count = cfg_count.apply_scale (1, 16);
 		  add_bb_to_loop (then_bb, bb->loop_father);
 		  add_bb_to_loop (return_bb, bb->loop_father);
 		  add_bb_to_loop (else_bb, bb->loop_father);
@@ -1988,17 +1984,14 @@ cgraph_node::expand_thunk (bool output_asm_thunks, bool force_gimple_thunk)
 		  e = make_edge (bb, then_bb, EDGE_TRUE_VALUE);
 		  e->probability = profile_probability::guessed_always ()
 					.apply_scale (1, 16);
-		  e->count = count - count.apply_scale (1, 16);
 		  e = make_edge (bb, else_bb, EDGE_FALSE_VALUE);
 		  e->probability = profile_probability::guessed_always ()
 					.apply_scale (1, 16);
-		  e->count = count.apply_scale (1, 16);
 		  make_single_succ_edge (return_bb,
 					 EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
 		  make_single_succ_edge (then_bb, return_bb, EDGE_FALLTHRU);
 		  e = make_edge (else_bb, return_bb, EDGE_FALLTHRU);
 		  e->probability = profile_probability::always ();
-		  e->count = count.apply_scale (1, 16);
 		  bsi = gsi_last_bb (then_bb);
 		}
 
@@ -2033,8 +2026,10 @@ cgraph_node::expand_thunk (bool output_asm_thunks, bool force_gimple_thunk)
 	}
 
       cfun->gimple_df->in_ssa_p = true;
+      counts_to_freqs ();
       profile_status_for_fn (cfun)
-        = count.initialized_p () ? PROFILE_READ : PROFILE_GUESSED;
+        = cfg_count.initialized_p () && cfg_count.ipa_p ()
+	  ? PROFILE_READ : PROFILE_GUESSED;
       /* FIXME: C++ FE should stop setting TREE_ASM_WRITTEN on thunks.  */
       TREE_ASM_WRITTEN (thunk_fndecl) = false;
       delete_unreachable_blocks ();
diff --git a/gcc/color-macros.h b/gcc/color-macros.h
new file mode 100644
index 00000000000..37ed4d197cf
--- /dev/null
+++ b/gcc/color-macros.h
@@ -0,0 +1,108 @@
+/* Terminal color manipulation macros.
+   Copyright (C) 2005-2017 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_COLOR_MACROS_H
+#define GCC_COLOR_MACROS_H
+
+/* Select Graphic Rendition (SGR, "\33[...m") strings.  */
+/* Also Erase in Line (EL) to Right ("\33[K") by default.  */
+/*    Why have EL to Right after SGR?
+	 -- The behavior of line-wrapping when at the bottom of the
+	    terminal screen and at the end of the current line is often
+	    such that a new line is introduced, entirely cleared with
+	    the current background color which may be different from the
+	    default one (see the boolean back_color_erase terminfo(5)
+	    capability), thus scrolling the display by one line.
+	    The end of this new line will stay in this background color
+	    even after reverting to the default background color with
+	    "\33[m', unless it is explicitly cleared again with "\33[K"
+	    (which is the behavior the user would instinctively expect
+	    from the whole thing).  There may be some unavoidable
+	    background-color flicker at the end of this new line because
+	    of this (when timing with the monitor's redraw is just right).
+	 -- The behavior of HT (tab, "\t") is usually the same as that of
+	    Cursor Forward Tabulation (CHT) with a default parameter
+	    of 1 ("\33[I"), i.e., it performs pure movement to the next
+	    tab stop, without any clearing of either content or screen
+	    attributes (including background color); try
+	       printf 'asdfqwerzxcv\rASDF\tZXCV\n'
+	    in a bash(1) shell to demonstrate this.  This is not what the
+	    user would instinctively expect of HT (but is ok for CHT).
+	    The instinctive behavior would include clearing the terminal
+	    cells that are skipped over by HT with blank cells in the
+	    current screen attributes, including background color;
+	    the boolean dest_tabs_magic_smso terminfo(5) capability
+	    indicates this saner behavior for HT, but only some rare
+	    terminals have it (although it also indicates a special
+	    glitch with standout mode in the Teleray terminal for which
+	    it was initially introduced).  The remedy is to add "\33K"
+	    after each SGR sequence, be it START (to fix the behavior
+	    of any HT after that before another SGR) or END (to fix the
+	    behavior of an HT in default background color that would
+	    follow a line-wrapping at the bottom of the screen in another
+	    background color, and to complement doing it after START).
+	    Piping GCC's output through a pager such as less(1) avoids
+	    any HT problems since the pager performs tab expansion.
+
+      Generic disadvantages of this remedy are:
+	 -- Some very rare terminals might support SGR but not EL (nobody
+	    will use "gcc -fdiagnostics-color" on a terminal that does not
+	    support SGR in the first place).
+	 -- Having these extra control sequences might somewhat complicate
+	    the task of any program trying to parse "gcc -fdiagnostics-color"
+	    output in order to extract structuring information from it.
+      A specific disadvantage to doing it after SGR START is:
+	 -- Even more possible background color flicker (when timing
+	    with the monitor's redraw is just right), even when not at the
+	    bottom of the screen.
+      There are no additional disadvantages specific to doing it after
+      SGR END.
+
+      It would be impractical for GCC to become a full-fledged
+      terminal program linked against ncurses or the like, so it will
+      not detect terminfo(5) capabilities.  */
+
+#define COLOR_SEPARATOR		";"
+#define COLOR_NONE		"00"
+#define COLOR_BOLD		"01"
+#define COLOR_UNDERSCORE	"04"
+#define COLOR_BLINK		"05"
+#define COLOR_REVERSE		"07"
+#define COLOR_FG_BLACK		"30"
+#define COLOR_FG_RED		"31"
+#define COLOR_FG_GREEN		"32"
+#define COLOR_FG_YELLOW		"33"
+#define COLOR_FG_BLUE		"34"
+#define COLOR_FG_MAGENTA	"35"
+#define COLOR_FG_CYAN		"36"
+#define COLOR_FG_WHITE		"37"
+#define COLOR_BG_BLACK		"40"
+#define COLOR_BG_RED		"41"
+#define COLOR_BG_GREEN		"42"
+#define COLOR_BG_YELLOW		"43"
+#define COLOR_BG_BLUE		"44"
+#define COLOR_BG_MAGENTA	"45"
+#define COLOR_BG_CYAN		"46"
+#define COLOR_BG_WHITE		"47"
+#define SGR_START		"\33["
+#define SGR_END			"m\33[K"
+#define SGR_SEQ(str)		SGR_START str SGR_END
+#define SGR_RESET		SGR_SEQ("")
+
+#endif  /* GCC_COLOR_MACROS_H */
diff --git a/gcc/combine.c b/gcc/combine.c
index ff0cb2a7c62..99cc343192e 100644
--- a/gcc/combine.c
+++ b/gcc/combine.c
@@ -3028,6 +3028,13 @@ try_combine (rtx_insn *i3, rtx_insn *i2, rtx_insn *i1, rtx_insn *i0,
       else
 	fprintf (dump_file, "\nTrying %d -> %d:\n",
 		 INSN_UID (i2), INSN_UID (i3));
+
+      if (i0)
+	dump_insn_slim (dump_file, i0);
+      if (i1)
+	dump_insn_slim (dump_file, i1);
+      dump_insn_slim (dump_file, i2);
+      dump_insn_slim (dump_file, i3);
     }
 
   /* If multiple insns feed into one of I2 or I3, they can be in any
@@ -12112,6 +12119,7 @@ simplify_compare_const (enum rtx_code code, machine_mode mode,
 	  const_op -= 1;
 	  code = LEU;
 	  /* ... fall through ...  */
+	  gcc_fallthrough ();
 	}
       /* (unsigned) < 0x80000000 is equivalent to >= 0.  */
       else if (is_a <scalar_int_mode> (mode, &int_mode)
@@ -12149,6 +12157,7 @@ simplify_compare_const (enum rtx_code code, machine_mode mode,
 	  const_op -= 1;
 	  code = GTU;
 	  /* ... fall through ...  */
+	  gcc_fallthrough ();
 	}
 
       /* (unsigned) >= 0x80000000 is equivalent to < 0.  */
@@ -14504,6 +14513,7 @@ distribute_notes (rtx notes, rtx_insn *from_insn, rtx_insn *i3, rtx_insn *i2,
 	case REG_SETJMP:
 	case REG_TM:
 	case REG_CALL_DECL:
+	case REG_CALL_NOCF_CHECK:
 	  /* These notes must remain with the call.  It should not be
 	     possible for both I2 and I3 to be a call.  */
 	  if (CALL_P (i3))
@@ -14686,6 +14696,17 @@ distribute_notes (rtx notes, rtx_insn *from_insn, rtx_insn *i3, rtx_insn *i2,
 		  && CALL_P (from_insn)
 		  && find_reg_fusage (from_insn, USE, XEXP (note, 0)))
 		place = from_insn;
+	      else if (i2 && reg_set_p (XEXP (note, 0), PATTERN (i2)))
+		{
+		  /* If the new I2 sets the same register that is marked
+		     dead in the note, we do not in general know where to
+		     put the note.  One important case we _can_ handle is
+		     when the note comes from I3.  */
+		  if (from_insn == i3)
+		    place = i3;
+		  else
+		    break;
+		}
 	      else if (reg_referenced_p (XEXP (note, 0), PATTERN (i3)))
 		place = i3;
 	      else if (i2 != 0 && next_nonnote_nondebug_insn (i2) == i3
@@ -14699,11 +14720,6 @@ distribute_notes (rtx notes, rtx_insn *from_insn, rtx_insn *i3, rtx_insn *i2,
 		       || rtx_equal_p (XEXP (note, 0), elim_i0))
 		break;
 	      tem_insn = i3;
-	      /* If the new I2 sets the same register that is marked dead
-		 in the note, we do not know where to put the note.
-		 Give up.  */
-	      if (i2 != 0 && reg_set_p (XEXP (note, 0), PATTERN (i2)))
-		break;
 	    }
 
 	  if (place == 0)
diff --git a/gcc/common.opt b/gcc/common.opt
index c95da640174..f8f2ed3db8a 100644
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -741,6 +741,10 @@ Wsuggest-attribute=noreturn
 Common Var(warn_suggest_attribute_noreturn) Warning
 Warn about functions which might be candidates for __attribute__((noreturn)).
 
+Wsuggest-attribute=malloc
+Common Var(warn_suggest_attribute_malloc) Warning
+Warn about functions which might be candidates for __attribute__((malloc)).
+
 Wsuggest-final-types
 Common Var(warn_suggest_final_types) Warning
 Warn about C++ polymorphic types where adding final keyword would improve code quality.
@@ -1620,6 +1624,29 @@ finline-atomics
 Common Report Var(flag_inline_atomics) Init(1) Optimization
 Inline __atomic operations when a lock free instruction sequence is available.
 
+fcf-protection
+Common RejectNegative Alias(fcf-protection=,full)
+
+fcf-protection=
+Common Report Joined RejectNegative Enum(cf_protection_level) Var(flag_cf_protection) Init(CF_NONE)
+-fcf-protection=[full|branch|return|none]	Instrument functions with checks to verify jump/call/return control-flow transfer
+instructions have valid targets.
+
+Enum
+Name(cf_protection_level) Type(enum cf_protection_level) UnknownError(unknown Cotrol-Flow Protection Level %qs)
+
+EnumValue
+Enum(cf_protection_level) String(full) Value(CF_FULL)
+
+EnumValue
+Enum(cf_protection_level) String(branch) Value(CF_BRANCH)
+
+EnumValue
+Enum(cf_protection_level) String(return) Value(CF_RETURN)
+
+EnumValue
+Enum(cf_protection_level) String(none) Value(CF_NONE)
+
 finstrument-functions
 Common Report Var(flag_instrument_function_entry_exit)
 Instrument function entry and exit with profiling calls.
@@ -2846,11 +2873,23 @@ Common Driver RejectNegative JoinedOrMissing
 Generate debug information in default format.
 
 gcoff
-Common Driver JoinedOrMissing Negative(gdwarf)
-Generate debug information in COFF format.
+Common Driver Ignore Warn(switch %qs no longer supported)
+Does nothing.  Preserved for backward compatibility.
+
+gcoff1
+Common Driver Ignore Warn(switch %qs no longer supported)
+Does nothing.  Preserved for backward compatibility.
+
+gcoff2
+Common Driver Ignore Warn(switch %qs no longer supported)
+Does nothing.  Preserved for backward compatibility.
+
+gcoff3
+Common Driver Ignore Warn(switch %qs no longer supported)
+Does nothing.  Preserved for backward compatibility.
 
 gcolumn-info
-Common Driver Var(debug_column_info,1) Init(0)
+Common Driver Var(debug_column_info,1) Init(1)
 Record DW_AT_decl_column and DW_AT_call_column in DWARF.
 
 gdwarf
@@ -2914,7 +2953,7 @@ Common Driver JoinedOrMissing Negative(gxcoff+)
 Generate debug information in XCOFF format.
 
 gxcoff+
-Common Driver JoinedOrMissing Negative(gcoff)
+Common Driver JoinedOrMissing Negative(gdwarf)
 Generate debug information in extended XCOFF format.
 
 Enum
diff --git a/gcc/common/config/i386/i386-common.c b/gcc/common/config/i386/i386-common.c
index 4185176495a..ada918e6f2a 100644
--- a/gcc/common/config/i386/i386-common.c
+++ b/gcc/common/config/i386/i386-common.c
@@ -137,6 +137,9 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA_CLZERO_SET OPTION_MASK_ISA_CLZERO
 #define OPTION_MASK_ISA_PKU_SET OPTION_MASK_ISA_PKU
 #define OPTION_MASK_ISA_RDPID_SET OPTION_MASK_ISA_RDPID
+#define OPTION_MASK_ISA_GFNI_SET OPTION_MASK_ISA_GFNI
+#define OPTION_MASK_ISA_IBT_SET OPTION_MASK_ISA_IBT
+#define OPTION_MASK_ISA_SHSTK_SET OPTION_MASK_ISA_SHSTK
 
 /* Define a set of ISAs which aren't available when a given ISA is
    disabled.  MMX and SSE ISAs are handled separately.  */
@@ -202,6 +205,9 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA_CLZERO_UNSET OPTION_MASK_ISA_CLZERO
 #define OPTION_MASK_ISA_PKU_UNSET OPTION_MASK_ISA_PKU
 #define OPTION_MASK_ISA_RDPID_UNSET OPTION_MASK_ISA_RDPID
+#define OPTION_MASK_ISA_GFNI_UNSET OPTION_MASK_ISA_GFNI
+#define OPTION_MASK_ISA_IBT_UNSET OPTION_MASK_ISA_IBT
+#define OPTION_MASK_ISA_SHSTK_UNSET OPTION_MASK_ISA_SHSTK
 
 /* SSE4 includes both SSE4.1 and SSE4.2.  -mno-sse4 should the same
    as -mno-sse4.1. */
@@ -484,6 +490,48 @@ ix86_handle_option (struct gcc_options *opts,
 	}
       return true;
 
+    case OPT_mgfni:
+      if (value)
+	{
+	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_GFNI_SET;
+	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA_GFNI_SET;
+	}
+      else
+	{
+	  opts->x_ix86_isa_flags2 &= ~OPTION_MASK_ISA_GFNI_UNSET;
+	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA_GFNI_UNSET;
+	}
+      return true;
+
+    case OPT_mcet:
+    case OPT_mibt:
+      if (value)
+	{
+	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_IBT_SET;
+	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA_IBT_SET;
+	}
+      else
+	{
+	  opts->x_ix86_isa_flags2 &= ~OPTION_MASK_ISA_IBT_UNSET;
+	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA_IBT_UNSET;
+	}
+      if (code != OPT_mcet)
+	return true;
+      /* fall through.  */
+
+    case OPT_mshstk:
+      if (value)
+	{
+	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_SHSTK_SET;
+	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA_SHSTK_SET;
+	}
+      else
+	{
+	  opts->x_ix86_isa_flags2 &= ~OPTION_MASK_ISA_SHSTK_UNSET;
+	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA_SHSTK_UNSET;
+	}
+      return true;
+
     case OPT_mavx5124fmaps:
       if (value)
 	{
diff --git a/gcc/compare-elim.c b/gcc/compare-elim.c
index 7e557a245b5..17d08842d15 100644
--- a/gcc/compare-elim.c
+++ b/gcc/compare-elim.c
@@ -65,6 +65,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tm_p.h"
 #include "insn-config.h"
 #include "recog.h"
+#include "emit-rtl.h"
 #include "cfgrtl.h"
 #include "tree-pass.h"
 #include "domwalk.h"
@@ -96,6 +97,9 @@ struct comparison
   /* The insn prior to the comparison insn that clobbers the flags.  */
   rtx_insn *prev_clobber;
 
+  /* The insn prior to the comparison insn that sets in_a REG.  */
+  rtx_insn *in_a_setter;
+
   /* The two values being compared.  These will be either REGs or
      constants.  */
   rtx in_a, in_b;
@@ -308,26 +312,22 @@ can_eliminate_compare (rtx compare, rtx eh_note, struct comparison *cmp)
 edge
 find_comparison_dom_walker::before_dom_children (basic_block bb)
 {
-  struct comparison *last_cmp;
-  rtx_insn *insn, *next, *last_clobber;
-  bool last_cmp_valid;
+  rtx_insn *insn, *next;
   bool need_purge = false;
-  bitmap killed;
-
-  killed = BITMAP_ALLOC (NULL);
+  rtx_insn *last_setter[FIRST_PSEUDO_REGISTER];
 
   /* The last comparison that was made.  Will be reset to NULL
      once the flags are clobbered.  */
-  last_cmp = NULL;
+  struct comparison *last_cmp = NULL;
 
   /* True iff the last comparison has not been clobbered, nor
      have its inputs.  Used to eliminate duplicate compares.  */
-  last_cmp_valid = false;
+  bool last_cmp_valid = false;
 
   /* The last insn that clobbered the flags, if that insn is of
      a form that may be valid for eliminating a following compare.
      To be reset to NULL once the flags are set otherwise.  */
-  last_clobber = NULL;
+  rtx_insn *last_clobber = NULL;
 
   /* Propagate the last live comparison throughout the extended basic block. */
   if (single_pred_p (bb))
@@ -337,6 +337,7 @@ find_comparison_dom_walker::before_dom_children (basic_block bb)
 	last_cmp_valid = last_cmp->inputs_valid;
     }
 
+  memset (last_setter, 0, sizeof (last_setter));
   for (insn = BB_HEAD (bb); insn; insn = next)
     {
       rtx src;
@@ -345,10 +346,6 @@ find_comparison_dom_walker::before_dom_children (basic_block bb)
       if (!NONDEBUG_INSN_P (insn))
 	continue;
 
-      /* Compute the set of registers modified by this instruction.  */
-      bitmap_clear (killed);
-      df_simulate_find_defs (insn, killed);
-
       src = conforming_compare (insn);
       if (src)
 	{
@@ -372,6 +369,13 @@ find_comparison_dom_walker::before_dom_children (basic_block bb)
 	  last_cmp->in_b = XEXP (src, 1);
 	  last_cmp->eh_note = eh_note;
 	  last_cmp->orig_mode = GET_MODE (src);
+	  if (last_cmp->in_b == const0_rtx
+	      && last_setter[REGNO (last_cmp->in_a)])
+	    {
+	      rtx set = single_set (last_setter[REGNO (last_cmp->in_a)]);
+	      if (set && rtx_equal_p (SET_DEST (set), last_cmp->in_a))
+		last_cmp->in_a_setter = last_setter[REGNO (last_cmp->in_a)];
+	    }
 	  all_compares.safe_push (last_cmp);
 
 	  /* It's unusual, but be prepared for comparison patterns that
@@ -387,28 +391,36 @@ find_comparison_dom_walker::before_dom_children (basic_block bb)
 	    find_flags_uses_in_insn (last_cmp, insn);
 
 	  /* Notice if this instruction kills the flags register.  */
-	  if (bitmap_bit_p (killed, targetm.flags_regnum))
-	    {
-	      /* See if this insn could be the "clobber" that eliminates
-		 a future comparison.   */
-	      last_clobber = (arithmetic_flags_clobber_p (insn) ? insn : NULL);
-
-	      /* In either case, the previous compare is no longer valid.  */
-	      last_cmp = NULL;
-	      last_cmp_valid = false;
-	    }
+	  df_ref def;
+	  FOR_EACH_INSN_DEF (def, insn)
+	    if (DF_REF_REGNO (def) == targetm.flags_regnum)
+	      {
+		/* See if this insn could be the "clobber" that eliminates
+		   a future comparison.   */
+		last_clobber = (arithmetic_flags_clobber_p (insn)
+				? insn : NULL);
+
+		/* In either case, the previous compare is no longer valid.  */
+		last_cmp = NULL;
+		last_cmp_valid = false;
+		break;
+	      }
 	}
 
-      /* Notice if any of the inputs to the comparison have changed.  */
-      if (last_cmp_valid
-	  && (bitmap_bit_p (killed, REGNO (last_cmp->in_a))
-	      || (REG_P (last_cmp->in_b)
-		  && bitmap_bit_p (killed, REGNO (last_cmp->in_b)))))
-	last_cmp_valid = false;
+      /* Notice if any of the inputs to the comparison have changed
+	 and remember last insn that sets each register.  */
+      df_ref def;
+      FOR_EACH_INSN_DEF (def, insn)
+	{
+	  if (last_cmp_valid
+	      && (DF_REF_REGNO (def) == REGNO (last_cmp->in_a)
+		  || (REG_P (last_cmp->in_b)
+		      && DF_REF_REGNO (def) == REGNO (last_cmp->in_b))))
+	    last_cmp_valid = false;
+	  last_setter[DF_REF_REGNO (def)] = insn;
+	}
     }
 
-  BITMAP_FREE (killed);
-
   /* Remember the live comparison for subsequent members of
      the extended basic block.  */
   if (last_cmp)
@@ -579,6 +591,133 @@ equivalent_reg_at_start (rtx reg, rtx_insn *end, rtx_insn *start)
   return reg;
 }
 
+/* Return true if it is okay to merge the comparison CMP_INSN with
+   the instruction ARITH_INSN.  Both instructions are assumed to be in the
+   same basic block with ARITH_INSN appearing before CMP_INSN.  This checks
+   that there are no uses or defs of the condition flags or control flow
+   changes between the two instructions.  */
+
+static bool
+can_merge_compare_into_arith (rtx_insn *cmp_insn, rtx_insn *arith_insn)
+{
+  for (rtx_insn *insn = PREV_INSN (cmp_insn);
+       insn && insn != arith_insn;
+       insn = PREV_INSN (insn))
+    {
+      if (!NONDEBUG_INSN_P (insn))
+	continue;
+      /* Bail if there are jumps or calls in between.  */
+      if (!NONJUMP_INSN_P (insn))
+	return false;
+
+      /* Bail on old-style asm statements because they lack
+	 data flow information.  */
+      if (GET_CODE (PATTERN (insn)) == ASM_INPUT)
+	return false;
+
+      df_ref ref;
+      /* Find a USE of the flags register.  */
+      FOR_EACH_INSN_USE (ref, insn)
+	if (DF_REF_REGNO (ref) == targetm.flags_regnum)
+	  return false;
+
+      /* Find a DEF of the flags register.  */
+      FOR_EACH_INSN_DEF (ref, insn)
+	if (DF_REF_REGNO (ref) == targetm.flags_regnum)
+	  return false;
+    }
+  return true;
+}
+
+/* Given two SET expressions, SET_A and SET_B determine whether they form
+   a recognizable pattern when emitted in parallel.  Return that parallel
+   if so.  Otherwise return NULL.  */
+
+static rtx
+try_validate_parallel (rtx set_a, rtx set_b)
+{
+  rtx par = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, set_a, set_b));
+  rtx_insn *insn = make_insn_raw (par);
+
+  if (insn_invalid_p (insn, false))
+    {
+      crtl->emit.x_cur_insn_uid--;
+      return NULL_RTX;
+    }
+
+  SET_PREV_INSN (insn) = NULL_RTX;
+  SET_NEXT_INSN (insn) = NULL_RTX;
+  INSN_LOCATION (insn) = 0;
+  return insn;
+}
+
+/* For a comparison instruction described by CMP check if it compares a
+   register with zero i.e. it is of the form CC := CMP R1, 0.
+   If it is, find the instruction defining R1 (say I1) and try to create a
+   PARALLEL consisting of I1 and the comparison, representing a flag-setting
+   arithmetic instruction.  Example:
+   I1: R1 := R2 + R3
+   <instructions that don't read the condition register>
+   I2: CC := CMP R1 0
+   I2 can be merged with I1 into:
+   I1: { CC := CMP (R2 + R3) 0 ; R1 := R2 + R3 }
+   This catches cases where R1 is used between I1 and I2 and therefore
+   combine and other RTL optimisations will not try to propagate it into
+   I2.  Return true if we succeeded in merging CMP.  */
+
+static bool
+try_merge_compare (struct comparison *cmp)
+{
+  rtx_insn *cmp_insn = cmp->insn;
+
+  if (cmp->in_b != const0_rtx || cmp->in_a_setter == NULL)
+    return false;
+  rtx in_a = cmp->in_a;
+  df_ref use;
+
+  FOR_EACH_INSN_USE (use, cmp_insn)
+    if (DF_REF_REGNO (use) == REGNO (in_a))
+      break;
+  if (!use)
+    return false;
+
+  rtx_insn *def_insn = cmp->in_a_setter;
+  rtx set = single_set (def_insn);
+
+  if (!can_merge_compare_into_arith (cmp_insn, def_insn))
+    return false;
+
+  rtx src = SET_SRC (set);
+  rtx flags = maybe_select_cc_mode (cmp, src, CONST0_RTX (GET_MODE (src)));
+  if (!flags)
+    {
+    /* We may already have a change group going through maybe_select_cc_mode.
+       Discard it properly.  */
+      cancel_changes (0);
+      return false;
+    }
+
+  rtx flag_set
+    = gen_rtx_SET (flags, gen_rtx_COMPARE (GET_MODE (flags),
+					   copy_rtx (src),
+					   CONST0_RTX (GET_MODE (src))));
+  rtx arith_set = copy_rtx (PATTERN (def_insn));
+  rtx par = try_validate_parallel (flag_set, arith_set);
+  if (!par)
+    {
+      /* We may already have a change group going through maybe_select_cc_mode.
+	 Discard it properly.  */
+      cancel_changes (0);
+      return false;
+    }
+  if (!apply_change_group ())
+    return false;
+  emit_insn_after (par, def_insn);
+  delete_insn (def_insn);
+  delete_insn (cmp->insn);
+  return true;
+}
+
 /* Attempt to replace a comparison with a prior arithmetic insn that can
    compute the same flags value as the comparison itself.  Return true if
    successful, having made all rtl modifications necessary.  */
@@ -588,6 +727,9 @@ try_eliminate_compare (struct comparison *cmp)
 {
   rtx flags, in_a, in_b, cmp_src;
 
+  if (try_merge_compare (cmp))
+    return true;
+
   /* We must have found an interesting "clobber" preceding the compare.  */
   if (cmp->prev_clobber == NULL)
     return false;
diff --git a/gcc/config.gcc b/gcc/config.gcc
index 22702396a9f..3dace854c95 100644
--- a/gcc/config.gcc
+++ b/gcc/config.gcc
@@ -378,7 +378,8 @@ i[34567]86-*-*)
 		       avx512ifmaintrin.h avx512ifmavlintrin.h avx512vbmiintrin.h
 		       avx512vbmivlintrin.h avx5124fmapsintrin.h avx5124vnniwintrin.h
 		       avx512vpopcntdqintrin.h clwbintrin.h mwaitxintrin.h
-		       clzerointrin.h pkuintrin.h sgxintrin.h"
+		       clzerointrin.h pkuintrin.h sgxintrin.h cetintrin.h
+		       gfniintrin.h"
 	;;
 x86_64-*-*)
 	cpu_type=i386
@@ -402,7 +403,8 @@ x86_64-*-*)
 		       avx512ifmaintrin.h avx512ifmavlintrin.h avx512vbmiintrin.h
 		       avx512vbmivlintrin.h avx5124fmapsintrin.h avx5124vnniwintrin.h
 		       avx512vpopcntdqintrin.h clwbintrin.h mwaitxintrin.h
-		       clzerointrin.h pkuintrin.h sgxintrin.h"
+		       clzerointrin.h pkuintrin.h sgxintrin.h cetintrin.h
+		       gfniintrin.h"
 	;;
 ia64-*-*)
 	extra_headers=ia64intrin.h
@@ -459,7 +461,7 @@ powerpc*-*-*)
 	extra_objs="rs6000-string.o rs6000-p8swap.o"
 	extra_headers="ppc-asm.h altivec.h htmintrin.h htmxlintrin.h"
 	extra_headers="${extra_headers} bmi2intrin.h bmiintrin.h"
-	extra_headers="${extra_headers} xmmintrin.h mm_malloc.h"
+	extra_headers="${extra_headers} xmmintrin.h mm_malloc.h emmintrin.h"
 	extra_headers="${extra_headers} mmintrin.h x86intrin.h"
 	extra_headers="${extra_headers} ppu_intrinsics.h spu2vmx.h vec_types.h si2vmx.h"
 	extra_headers="${extra_headers} paired.h"
@@ -874,7 +876,7 @@ case ${target} in
   tmake_file="${tmake_file} t-sol2 t-slibgcc"
   c_target_objs="${c_target_objs} sol2-c.o"
   cxx_target_objs="${cxx_target_objs} sol2-c.o sol2-cxx.o"
-  extra_objs="sol2.o sol2-stubs.o"
+  extra_objs="${extra_objs} sol2.o sol2-stubs.o"
   extra_options="${extra_options} sol2.opt"
   case ${enable_threads}:${have_pthread_h}:${have_thread_h} in
     "":yes:* | yes:yes:* )
@@ -1692,7 +1694,7 @@ i[34567]86-*-cygwin*)
 	tmake_file="${tmake_file} i386/t-cygming t-slibgcc"
 	target_gtfiles="\$(srcdir)/config/i386/winnt.c"
 	extra_options="${extra_options} i386/cygming.opt i386/cygwin.opt"
-	extra_objs="winnt.o winnt-stubs.o"
+	extra_objs="${extra_objs} winnt.o winnt-stubs.o"
 	c_target_objs="${c_target_objs} msformat-c.o"
 	cxx_target_objs="${cxx_target_objs} winnt-cxx.o msformat-c.o"
 	if test x$enable_threads = xyes; then
@@ -1708,7 +1710,7 @@ x86_64-*-cygwin*)
 	tmake_file="${tmake_file} i386/t-cygming t-slibgcc i386/t-cygwin-w64"
 	target_gtfiles="\$(srcdir)/config/i386/winnt.c"
 	extra_options="${extra_options} i386/cygming.opt i386/cygwin.opt"
-	extra_objs="winnt.o winnt-stubs.o"
+	extra_objs="${extra_objs} winnt.o winnt-stubs.o"
 	c_target_objs="${c_target_objs} msformat-c.o"
 	cxx_target_objs="${cxx_target_objs} winnt-cxx.o msformat-c.o"
 	if test x$enable_threads = xyes; then
@@ -1783,7 +1785,7 @@ i[34567]86-*-mingw* | x86_64-*-mingw*)
 		*)
 			;;
 	esac
-	extra_objs="winnt.o winnt-stubs.o"
+	extra_objs="${extra_objs} winnt.o winnt-stubs.o"
 	c_target_objs="${c_target_objs} msformat-c.o"
 	cxx_target_objs="${cxx_target_objs} winnt-cxx.o msformat-c.o"
 	gas=yes
@@ -3437,11 +3439,18 @@ if test x$with_cpu = x ; then
       esac
       ;;
     powerpc*-*-*spe*)
+      # For SPE, start with 8540, then upgrade to 8548 if
+      # --enable-e500-double was requested explicitly or if we were
+      # configured for e500v2.
+      with_cpu=8540
       if test x$enable_e500_double = xyes; then
-         with_cpu=8548
-      else
-         with_cpu=8540
-      fi       
+        with_cpu=8548
+      fi
+      case ${target_noncanonical} in
+        e500v2*)
+          with_cpu=8548
+          ;;
+      esac
       ;;
     sparc*-*-*)
       case ${target} in
@@ -4544,7 +4553,8 @@ case ${target} in
 	i[34567]86-*-darwin* | x86_64-*-darwin*)
 		;;
 	i[34567]86-*-linux* | x86_64-*-linux*)
-		tmake_file="$tmake_file i386/t-linux"
+		extra_objs="${extra_objs} cet.o"
+		tmake_file="$tmake_file i386/t-linux i386/t-cet"
 		;;
 	i[34567]86-*-kfreebsd*-gnu | x86_64-*-kfreebsd*-gnu)
 		tmake_file="$tmake_file i386/t-kfreebsd"
diff --git a/gcc/config.in b/gcc/config.in
index 89d7108e8db..5651bcba431 100644
--- a/gcc/config.in
+++ b/gcc/config.in
@@ -717,6 +717,12 @@
 #endif
 
 
+/* Define if your assembler supports -xbrace_comment option. */
+#ifndef USED_FOR_TARGET
+#undef HAVE_AS_XBRACE_COMMENT_OPTION
+#endif
+
+
 /* Define to 1 if you have the `atoq' function. */
 #ifndef USED_FOR_TARGET
 #undef HAVE_ATOQ
diff --git a/gcc/config/aarch64/aarch64-builtins.c b/gcc/config/aarch64/aarch64-builtins.c
index 8ca4cfc299f..8479b6a1f2c 100644
--- a/gcc/config/aarch64/aarch64-builtins.c
+++ b/gcc/config/aarch64/aarch64-builtins.c
@@ -170,6 +170,11 @@ aarch64_types_quadop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_none, qualifier_none, qualifier_none,
       qualifier_none, qualifier_lane_index };
 #define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
+static enum aarch64_type_qualifiers
+aarch64_types_quadopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
+      qualifier_unsigned, qualifier_lane_index };
+#define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers)
 
 static enum aarch64_type_qualifiers
 aarch64_types_binop_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
@@ -1064,7 +1069,8 @@ aarch64_simd_expand_args (rtx target, int icode, int have_retval,
 		    = GET_MODE_NUNITS (builtin_mode).to_constant ();
 		  aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
 		  /* Keep to GCC-vector-extension lane indices in the RTL.  */
-		  op[opc] = endian_lane_rtx (builtin_mode, INTVAL (op[opc]));
+		  op[opc] = aarch64_endian_lane_rtx (builtin_mode,
+						     INTVAL (op[opc]));
 		}
 	      goto constant_arg;
 
@@ -1078,7 +1084,7 @@ aarch64_simd_expand_args (rtx target, int icode, int have_retval,
 		    = GET_MODE_NUNITS (vmode).to_constant ();
 		  aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
 		  /* Keep to GCC-vector-extension lane indices in the RTL.  */
-		  op[opc] = endian_lane_rtx (vmode, INTVAL (op[opc]));
+		  op[opc] = aarch64_endian_lane_rtx (vmode, INTVAL (op[opc]));
 		}
 	      /* Fall through - if the lane index isn't a constant then
 		 the next case will error.  */
diff --git a/gcc/config/aarch64/aarch64-c.c b/gcc/config/aarch64/aarch64-c.c
index d7f42b3d5ab..80fe1838eb6 100644
--- a/gcc/config/aarch64/aarch64-c.c
+++ b/gcc/config/aarch64/aarch64-c.c
@@ -108,6 +108,7 @@ aarch64_update_cpp_builtins (cpp_reader *pfile)
 
 
   aarch64_def_or_undef (TARGET_CRC32, "__ARM_FEATURE_CRC32", pfile);
+  aarch64_def_or_undef (TARGET_DOTPROD, "__ARM_FEATURE_DOTPROD", pfile);
 
   cpp_undef (pfile, "__AARCH64_CMODEL_TINY__");
   cpp_undef (pfile, "__AARCH64_CMODEL_SMALL__");
@@ -176,7 +177,7 @@ aarch64_pragma_target_parse (tree args, tree pop_target)
      information that it specifies.  */
   if (args)
     {
-      if (!aarch64_process_target_attr (args, "pragma"))
+      if (!aarch64_process_target_attr (args))
 	return false;
 
       aarch64_override_options_internal (&global_options);
diff --git a/gcc/config/aarch64/aarch64-cores.def b/gcc/config/aarch64/aarch64-cores.def
index 10893324d3f..cdf047c0fa2 100644
--- a/gcc/config/aarch64/aarch64-cores.def
+++ b/gcc/config/aarch64/aarch64-cores.def
@@ -83,8 +83,13 @@ AARCH64_CORE("thunderx2t99",  thunderx2t99,  thunderx2t99, 8_1A,  AARCH64_FL_FOR
 /* ARMv8.2-A Architecture Processors.  */
 
 /* ARM ('A') cores. */
-AARCH64_CORE("cortex-a55",  cortexa55, cortexa53, 8_2A,  AARCH64_FL_FOR_ARCH8_2 | AARCH64_FL_RCPC, cortexa53, 0x41, 0xd05, -1)
-AARCH64_CORE("cortex-a75",  cortexa75, cortexa57, 8_2A,  AARCH64_FL_FOR_ARCH8_2 | AARCH64_FL_RCPC, cortexa73, 0x41, 0xd0a, -1)
+AARCH64_CORE("cortex-a55",  cortexa55, cortexa53, 8_2A,  AARCH64_FL_FOR_ARCH8_2 | AARCH64_FL_RCPC | AARCH64_FL_DOTPROD, cortexa53, 0x41, 0xd05, -1)
+AARCH64_CORE("cortex-a75",  cortexa75, cortexa57, 8_2A,  AARCH64_FL_FOR_ARCH8_2 | AARCH64_FL_RCPC | AARCH64_FL_DOTPROD, cortexa73, 0x41, 0xd0a, -1)
+
+/* ARMv8.3-A Architecture Processors.  */
+
+/* Qualcomm ('Q') cores. */
+AARCH64_CORE("saphira",     saphira,    falkor,    8_3A,  AARCH64_FL_FOR_ARCH8_3 | AARCH64_FL_CRYPTO | AARCH64_FL_RCPC, saphira,   0x51, 0xC01, -1)
 
 /* ARMv8-A big.LITTLE implementations.  */
 
@@ -95,6 +100,6 @@ AARCH64_CORE("cortex-a73.cortex-a53",  cortexa73cortexa53, cortexa53, 8A,  AARCH
 
 /* ARM DynamIQ big.LITTLE configurations.  */
 
-AARCH64_CORE("cortex-a75.cortex-a55",  cortexa75cortexa55, cortexa53, 8_2A,  AARCH64_FL_FOR_ARCH8_2 | AARCH64_FL_RCPC, cortexa73, 0x41, AARCH64_BIG_LITTLE (0xd0a, 0xd05), -1)
+AARCH64_CORE("cortex-a75.cortex-a55",  cortexa75cortexa55, cortexa53, 8_2A,  AARCH64_FL_FOR_ARCH8_2 | AARCH64_FL_RCPC | AARCH64_FL_DOTPROD, cortexa73, 0x41, AARCH64_BIG_LITTLE (0xd0a, 0xd05), -1)
 
 #undef AARCH64_CORE
diff --git a/gcc/config/aarch64/aarch64-modes.def b/gcc/config/aarch64/aarch64-modes.def
index 6d519e08fa2..11bbdfcb55e 100644
--- a/gcc/config/aarch64/aarch64-modes.def
+++ b/gcc/config/aarch64/aarch64-modes.def
@@ -31,15 +31,10 @@ ADJUST_FLOAT_FORMAT (HF, &ieee_half_format);
 
 /* Vector modes.  */
 
-VECTOR_BOOL_MODE (32);
-VECTOR_BOOL_MODE (16);
-VECTOR_BOOL_MODE (8);
-VECTOR_BOOL_MODE (4);
-
-ADJUST_BYTESIZE (V32BI, aarch64_sve_vg);
-ADJUST_BYTESIZE (V16BI, aarch64_sve_vg);
-ADJUST_BYTESIZE (V8BI, aarch64_sve_vg);
-ADJUST_BYTESIZE (V4BI, aarch64_sve_vg);
+VECTOR_BOOL_MODE (32, 4);
+VECTOR_BOOL_MODE (16, 4);
+VECTOR_BOOL_MODE (8, 4);
+VECTOR_BOOL_MODE (4, 4);
 
 ADJUST_NUNITS (V32BI, aarch64_sve_vg * 8);
 ADJUST_NUNITS (V16BI, aarch64_sve_vg * 4);
@@ -65,6 +60,10 @@ INT_MODE (OI, 32);
 INT_MODE (CI, 48);
 INT_MODE (XI, 64);
 
+/* Define SVE modes for NVECS vectors.  VB, VH, VS and VD are the prefixes
+   for 8-bit, 16-bit, 32-bit and 64-bit elements respectively.  It isn't
+   strictly necessary to set the alignment here, since the default would
+   be clamped to BIGGEST_ALIGNMENT anyhow, but it seems clearer.  */
 #define SVE_MODES(NVECS, VB, VH, VS, VD) \
   VECTOR_MODES (INT, 32 * NVECS); \
   VECTOR_MODES (FLOAT, 32 * NVECS); \
@@ -73,9 +72,20 @@ INT_MODE (XI, 64);
   ADJUST_NUNITS (VH##HI, aarch64_sve_vg * NVECS * 4); \
   ADJUST_NUNITS (VS##SI, aarch64_sve_vg * NVECS * 2); \
   ADJUST_NUNITS (VD##DI, aarch64_sve_vg * NVECS); \
+  ADJUST_NUNITS (VH##HF, aarch64_sve_vg * NVECS * 4); \
   ADJUST_NUNITS (VS##SF, aarch64_sve_vg * NVECS * 2); \
-  ADJUST_NUNITS (VD##DF, aarch64_sve_vg * NVECS);
-
+  ADJUST_NUNITS (VD##DF, aarch64_sve_vg * NVECS); \
+  \
+  ADJUST_ALIGNMENT (VB##QI, 16); \
+  ADJUST_ALIGNMENT (VH##HI, 16); \
+  ADJUST_ALIGNMENT (VS##SI, 16); \
+  ADJUST_ALIGNMENT (VD##DI, 16); \
+  ADJUST_ALIGNMENT (VH##HF, 16); \
+  ADJUST_ALIGNMENT (VS##SF, 16); \
+  ADJUST_ALIGNMENT (VD##DF, 16);
+
+/* Give SVE vectors the names normally used for 256-bit vectors.
+   The actual number depends on command-line flags.  */
 SVE_MODES (1, V32, V16, V8, V4)
 SVE_MODES (2, V64, V32, V16, V8)
 SVE_MODES (3, V96, V48, V24, V12)
diff --git a/gcc/config/aarch64/aarch64-option-extensions.def b/gcc/config/aarch64/aarch64-option-extensions.def
index e348dcd3752..a0bc50ca576 100644
--- a/gcc/config/aarch64/aarch64-option-extensions.def
+++ b/gcc/config/aarch64/aarch64-option-extensions.def
@@ -43,8 +43,8 @@
 AARCH64_OPT_EXTENSION("fp", AARCH64_FL_FP, 0, AARCH64_FL_SIMD | AARCH64_FL_CRYPTO | AARCH64_FL_F16 | AARCH64_FL_SVE, "fp")
 
 /* Enabling "simd" also enables "fp".
-   Disabling "simd" also disables "crypto" and "sve".  */
-AARCH64_OPT_EXTENSION("simd", AARCH64_FL_SIMD, AARCH64_FL_FP, AARCH64_FL_CRYPTO | AARCH64_FL_SVE, "asimd")
+   Disabling "simd" also disables "crypto", "dotprod" and "sve".  */
+AARCH64_OPT_EXTENSION("simd", AARCH64_FL_SIMD, AARCH64_FL_FP, AARCH64_FL_CRYPTO | AARCH64_FL_DOTPROD | AARCH64_FL_SVE, "asimd")
 
 /* Enabling "crypto" also enables "fp", "simd".
    Disabling "crypto" just disables "crypto".  */
@@ -57,8 +57,8 @@ AARCH64_OPT_EXTENSION("crc", AARCH64_FL_CRC, 0, 0, "crc32")
 AARCH64_OPT_EXTENSION("lse", AARCH64_FL_LSE, 0, 0, "atomics")
 
 /* Enabling "fp16" also enables "fp".
-   Disabling "fp16" just disables "fp16".  */
-AARCH64_OPT_EXTENSION("fp16", AARCH64_FL_F16, AARCH64_FL_FP, 0, "fphp asimdhp")
+   Disabling "fp16" disables "sve" and "fp16".  */
+AARCH64_OPT_EXTENSION("fp16", AARCH64_FL_F16, AARCH64_FL_FP, AARCH64_FL_SVE, "fphp asimdhp")
 
 /* Enabling or disabling "rcpc" only changes "rcpc".  */
 AARCH64_OPT_EXTENSION("rcpc", AARCH64_FL_RCPC, 0, 0, "lrcpc")
@@ -67,7 +67,12 @@ AARCH64_OPT_EXTENSION("rcpc", AARCH64_FL_RCPC, 0, 0, "lrcpc")
    Disabling "rdma" just disables "rdma".  */
 AARCH64_OPT_EXTENSION("rdma", AARCH64_FL_RDMA, AARCH64_FL_FP | AARCH64_FL_SIMD, 0, "asimdrdm")
 
-/* Enabling "sve" also enables "fp" and "simd".
+/* Enabling "dotprod" also enables "simd".
+   Disabling "dotprod" only disables "dotprod".  */
+AARCH64_OPT_EXTENSION("dotprod", AARCH64_FL_DOTPROD, AARCH64_FL_SIMD, 0, "asimddp")
+
+/* Enabling "sve" also enables "fp16", "fp" and "simd".
    Disabling "sve" just disables "sve".  */
-AARCH64_OPT_EXTENSION("sve", AARCH64_FL_SVE, AARCH64_FL_FP | AARCH64_FL_SIMD, 0, "sve")
+AARCH64_OPT_EXTENSION("sve", AARCH64_FL_SVE, AARCH64_FL_FP | AARCH64_FL_SIMD | AARCH64_FL_F16, 0, "sve")
+
 #undef AARCH64_OPT_EXTENSION
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index 705675f6d91..39c5b0a7965 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -373,6 +373,7 @@ bool aarch64_legitimate_pic_operand_p (rtx);
 bool aarch64_mask_and_shift_for_ubfiz_p (scalar_int_mode, rtx, rtx);
 bool aarch64_zero_extend_const_eq (machine_mode, rtx, machine_mode, rtx);
 bool aarch64_move_imm (HOST_WIDE_INT, machine_mode);
+opt_machine_mode aarch64_sve_pred_mode (unsigned int);
 bool aarch64_sve_cnt_immediate_p (rtx);
 bool aarch64_sve_addvl_addpl_immediate_p (rtx);
 bool aarch64_sve_inc_dec_immediate_p (rtx);
@@ -445,8 +446,8 @@ const char * aarch64_output_probe_stack_range (rtx, rtx);
 void aarch64_err_no_fpadvsimd (machine_mode, const char *);
 void aarch64_expand_epilogue (bool);
 void aarch64_expand_mov_immediate (rtx, rtx, rtx (*) (rtx, rtx) = 0);
-void aarch64_expand_sve_mem_move (rtx, rtx, machine_mode,
-				  rtx (*) (rtx, rtx, rtx));
+void aarch64_emit_sve_pred_move (rtx, rtx, rtx);
+void aarch64_expand_sve_mem_move (rtx, rtx, machine_mode);
 void aarch64_expand_prologue (void);
 void aarch64_expand_vector_init (rtx, rtx);
 void aarch64_init_cumulative_args (CUMULATIVE_ARGS *, const_tree, rtx,
@@ -469,7 +470,7 @@ void aarch64_simd_emit_reg_reg_move (rtx *, machine_mode, unsigned int);
 rtx aarch64_simd_expand_builtin (int, tree, rtx);
 
 void aarch64_simd_lane_bounds (rtx, HOST_WIDE_INT, HOST_WIDE_INT, const_tree);
-rtx endian_lane_rtx (machine_mode, unsigned int);
+rtx aarch64_endian_lane_rtx (machine_mode, unsigned int);
 
 void aarch64_split_128bit_move (rtx, rtx);
 
@@ -508,7 +509,7 @@ void aarch64_expand_sve_vcond (machine_mode, machine_mode, rtx *);
 
 void aarch64_init_builtins (void);
 
-bool aarch64_process_target_attr (tree, const char*);
+bool aarch64_process_target_attr (tree);
 void aarch64_override_options_internal (struct gcc_options *);
 
 rtx aarch64_expand_builtin (tree exp,
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def b/gcc/config/aarch64/aarch64-simd-builtins.def
index d713d5d8b88..52d01342372 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -205,6 +205,14 @@
   BUILTIN_VSDQ_I_DI (BINOP, srshl, 0)
   BUILTIN_VSDQ_I_DI (BINOP_UUS, urshl, 0)
 
+  /* Implemented by aarch64_<sur><dotprod>{_lane}{q}<dot_mode>.  */
+  BUILTIN_VB (TERNOP, sdot, 0)
+  BUILTIN_VB (TERNOPU, udot, 0)
+  BUILTIN_VB (QUADOP_LANE, sdot_lane, 0)
+  BUILTIN_VB (QUADOPU_LANE, udot_lane, 0)
+  BUILTIN_VB (QUADOP_LANE, sdot_laneq, 0)
+  BUILTIN_VB (QUADOPU_LANE, udot_laneq, 0)
+
   BUILTIN_VDQ_I (SHIFTIMM, ashr, 3)
   VAR1 (SHIFTIMM, ashr_simd, 0, di)
   BUILTIN_VDQ_I (SHIFTIMM, lshr, 3)
diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index 60635aa418c..fcc49e3a2f8 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -80,7 +80,7 @@
           )))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
   }
   [(set_attr "type" "neon_dup<q>")]
@@ -95,13 +95,13 @@
           )))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
     return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
   }
   [(set_attr "type" "neon_dup<q>")]
 )
 
-(define_insn "*aarch64_simd_mov<mode>"
+(define_insn "*aarch64_simd_mov<VD:mode>"
   [(set (match_operand:VD 0 "nonimmediate_operand"
 		"=w, m,  m,  w, ?r, ?w, ?r, w")
 	(match_operand:VD 1 "general_operand"
@@ -124,12 +124,12 @@
      default: gcc_unreachable ();
      }
 }
-  [(set_attr "type" "neon_load1_1reg<q>, neon_stp, neon_store1_1reg<q>,\
+  [(set_attr "type" "neon_load1_1reg<q>, store_8, neon_store1_1reg<q>,\
 		     neon_logic<q>, neon_to_gp<q>, f_mcr,\
 		     mov_reg, neon_move<q>")]
 )
 
-(define_insn "*aarch64_simd_mov<mode>"
+(define_insn "*aarch64_simd_mov<VQ:mode>"
   [(set (match_operand:VQ 0 "nonimmediate_operand"
 		"=w, Umq,  m,  w, ?r, ?w, ?r, w")
 	(match_operand:VQ 1 "general_operand"
@@ -158,8 +158,8 @@
 	gcc_unreachable ();
     }
 }
-  [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
-		     neon_stp, neon_logic<q>, multiple, multiple,\
+  [(set_attr "type" "neon_load1_1reg<q>, store_16, neon_store1_1reg<q>,\
+		     neon_logic<q>, multiple, multiple,\
 		     multiple, neon_move<q>")
    (set_attr "length" "4,4,4,4,8,8,8,4")]
 )
@@ -391,6 +391,87 @@
 }
 )
 
+;; These instructions map to the __builtins for the Dot Product operations.
+(define_insn "aarch64_<sur>dot<vsi2qi>"
+  [(set (match_operand:VS 0 "register_operand" "=w")
+	(plus:VS (match_operand:VS 1 "register_operand" "0")
+		(unspec:VS [(match_operand:<VSI2QI> 2 "register_operand" "w")
+			    (match_operand:<VSI2QI> 3 "register_operand" "w")]
+		DOTPROD)))]
+  "TARGET_DOTPROD"
+  "<sur>dot\\t%0.<Vtype>, %2.<Vdottype>, %3.<Vdottype>"
+  [(set_attr "type" "neon_dot")]
+)
+
+;; These expands map to the Dot Product optab the vectorizer checks for.
+;; The auto-vectorizer expects a dot product builtin that also does an
+;; accumulation into the provided register.
+;; Given the following pattern
+;;
+;; for (i=0; i<len; i++) {
+;;     c = a[i] * b[i];
+;;     r += c;
+;; }
+;; return result;
+;;
+;; This can be auto-vectorized to
+;; r  = a[0]*b[0] + a[1]*b[1] + a[2]*b[2] + a[3]*b[3];
+;;
+;; given enough iterations.  However the vectorizer can keep unrolling the loop
+;; r += a[4]*b[4] + a[5]*b[5] + a[6]*b[6] + a[7]*b[7];
+;; r += a[8]*b[8] + a[9]*b[9] + a[10]*b[10] + a[11]*b[11];
+;; ...
+;;
+;; and so the vectorizer provides r, in which the result has to be accumulated.
+(define_expand "<sur>dot_prod<vsi2qi>"
+  [(set (match_operand:VS 0 "register_operand")
+	(plus:VS (unspec:VS [(match_operand:<VSI2QI> 1 "register_operand")
+			    (match_operand:<VSI2QI> 2 "register_operand")]
+		 DOTPROD)
+		(match_operand:VS 3 "register_operand")))]
+  "TARGET_DOTPROD"
+{
+  emit_insn (
+    gen_aarch64_<sur>dot<vsi2qi> (operands[3], operands[3], operands[1],
+				    operands[2]));
+  emit_insn (gen_rtx_SET (operands[0], operands[3]));
+  DONE;
+})
+
+;; These instructions map to the __builtins for the Dot Product
+;; indexed operations.
+(define_insn "aarch64_<sur>dot_lane<vsi2qi>"
+  [(set (match_operand:VS 0 "register_operand" "=w")
+	(plus:VS (match_operand:VS 1 "register_operand" "0")
+		(unspec:VS [(match_operand:<VSI2QI> 2 "register_operand" "w")
+			    (match_operand:V8QI 3 "register_operand" "<h_con>")
+			    (match_operand:SI 4 "immediate_operand" "i")]
+		DOTPROD)))]
+  "TARGET_DOTPROD"
+  {
+    operands[4]
+      = GEN_INT (ENDIAN_LANE_N (V8QImode, INTVAL (operands[4])));
+    return "<sur>dot\\t%0.<Vtype>, %2.<Vdottype>, %3.4b[%4]";
+  }
+  [(set_attr "type" "neon_dot")]
+)
+
+(define_insn "aarch64_<sur>dot_laneq<vsi2qi>"
+  [(set (match_operand:VS 0 "register_operand" "=w")
+	(plus:VS (match_operand:VS 1 "register_operand" "0")
+		(unspec:VS [(match_operand:<VSI2QI> 2 "register_operand" "w")
+			    (match_operand:V16QI 3 "register_operand" "<h_con>")
+			    (match_operand:SI 4 "immediate_operand" "i")]
+		DOTPROD)))]
+  "TARGET_DOTPROD"
+  {
+    operands[4]
+      = GEN_INT (ENDIAN_LANE_N (V16QImode, INTVAL (operands[4])));
+    return "<sur>dot\\t%0.<Vtype>, %2.<Vdottype>, %3.4b[%4]";
+  }
+  [(set_attr "type" "neon_dot")]
+)
+
 (define_expand "copysign<mode>3"
   [(match_operand:VHSDF 0 "register_operand")
    (match_operand:VHSDF 1 "register_operand")
@@ -419,7 +500,7 @@
       (match_operand:VMUL 3 "register_operand" "w")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
   }
   [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
@@ -435,7 +516,7 @@
       (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
     return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
   }
   [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
@@ -488,7 +569,7 @@
        (match_operand:DF 3 "register_operand" "w")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
     return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
   }
   [(set_attr "type" "neon_fp_mul_d_scalar_q")]
@@ -567,9 +648,8 @@
       case 0:
 	return "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>";
       case 1:
-	return (aarch64_output_simd_mov_immediate
-		(operands[2], GET_MODE_BITSIZE (<MODE>mode).to_constant (),
-		 AARCH64_CHECK_BIC));
+	return aarch64_output_simd_mov_immediate (operands[2], <bitsize>,
+						  AARCH64_CHECK_BIC);
       default:
 	gcc_unreachable ();
       }
@@ -589,9 +669,8 @@
       case 0:
 	return "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>";
       case 1:
-	return (aarch64_output_simd_mov_immediate
-		(operands[2], GET_MODE_BITSIZE (<MODE>mode).to_constant (),
-		 AARCH64_CHECK_ORR));
+	return aarch64_output_simd_mov_immediate (operands[2], <bitsize>,
+						  AARCH64_CHECK_ORR);
       default:
 	gcc_unreachable ();
       }
@@ -1073,7 +1152,7 @@
 	 (match_operand:VDQHS 4 "register_operand" "0")))]
  "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
@@ -1091,7 +1170,7 @@
 	 (match_operand:VDQHS 4 "register_operand" "0")))]
  "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
     return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
@@ -1131,7 +1210,7 @@
 	   (match_operand:VDQHS 3 "register_operand" "w"))))]
  "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
@@ -1149,7 +1228,7 @@
 	   (match_operand:VDQHS 3 "register_operand" "w"))))]
  "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
     return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
@@ -1719,7 +1798,7 @@
       (match_operand:VDQF 4 "register_operand" "0")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
@@ -1736,7 +1815,7 @@
       (match_operand:VDQSF 4 "register_operand" "0")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
     return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
@@ -1764,7 +1843,7 @@
       (match_operand:DF 4 "register_operand" "0")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
     return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
   }
   [(set_attr "type" "neon_fp_mla_d_scalar_q")]
@@ -1794,7 +1873,7 @@
       (match_operand:VDQF 4 "register_operand" "0")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
@@ -1812,7 +1891,7 @@
       (match_operand:VDQSF 4 "register_operand" "0")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
     return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
   }
   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
@@ -1842,7 +1921,7 @@
       (match_operand:DF 4 "register_operand" "0")))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
     return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
   }
   [(set_attr "type" "neon_fp_mla_d_scalar_q")]
@@ -2175,7 +2254,7 @@
 	       UNSPEC_ADDV)]
   "TARGET_SIMD"
   {
-    rtx elt = endian_lane_rtx (<MODE>mode, 0);
+    rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
     rtx scratch = gen_reg_rtx (<MODE>mode);
     emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
     emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
@@ -2226,7 +2305,7 @@
 		    UNSPEC_FADDV))]
  "TARGET_SIMD"
 {
-  rtx elt = endian_lane_rtx (V4SFmode, 0);
+  rtx elt = aarch64_endian_lane_rtx (V4SFmode, 0);
   rtx scratch = gen_reg_rtx (V4SFmode);
   emit_insn (gen_aarch64_faddpv4sf (scratch, operands[1], operands[1]));
   emit_insn (gen_aarch64_faddpv4sf (scratch, scratch, scratch));
@@ -2268,7 +2347,7 @@
 		  FMAXMINV)]
   "TARGET_SIMD"
   {
-    rtx elt = endian_lane_rtx (<MODE>mode, 0);
+    rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
     rtx scratch = gen_reg_rtx (<MODE>mode);
     emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
 							      operands[1]));
@@ -2284,7 +2363,7 @@
 		    MAXMINV)]
   "TARGET_SIMD"
   {
-    rtx elt = endian_lane_rtx (<MODE>mode, 0);
+    rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
     rtx scratch = gen_reg_rtx (<MODE>mode);
     emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
 							      operands[1]));
@@ -2809,7 +2888,7 @@
 	    (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
   }
   [(set_attr "type" "neon_to_gp<q>")]
@@ -2823,7 +2902,7 @@
 	    (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "umov\\t%w0, %1.<Vetype>[%2]";
   }
   [(set_attr "type" "neon_to_gp<q>")]
@@ -2839,7 +2918,7 @@
 	  (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     switch (which_alternative)
       {
 	case 0:
@@ -3215,7 +3294,7 @@
 	 UNSPEC_FMULX))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[3]));
     return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_fp_mul_<Vetype>_scalar<q>")]
@@ -3234,7 +3313,7 @@
 	 UNSPEC_FMULX))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
     return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
@@ -3268,7 +3347,7 @@
 	 UNSPEC_FMULX))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
     return "fmulx\t%<Vetype>0, %<Vetype>1, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "fmul<Vetype>")]
@@ -3354,7 +3433,7 @@
 	 VQDMULH))]
   "TARGET_SIMD"
   "*
-   operands[3] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
+   operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
 )
@@ -3369,7 +3448,7 @@
 	 VQDMULH))]
   "TARGET_SIMD"
   "*
-   operands[3] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
+   operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
 )
@@ -3384,7 +3463,7 @@
 	 VQDMULH))]
   "TARGET_SIMD"
   "*
-   operands[3] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
+   operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
    return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
 )
@@ -3399,7 +3478,7 @@
 	 VQDMULH))]
   "TARGET_SIMD"
   "*
-   operands[3] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
+   operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
    return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
 )
@@ -3431,7 +3510,7 @@
 	  SQRDMLH_AS))]
    "TARGET_SIMD_RDMA"
    {
-     operands[4] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
+     operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
      return
       "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
    }
@@ -3449,7 +3528,7 @@
 	  SQRDMLH_AS))]
    "TARGET_SIMD_RDMA"
    {
-     operands[4] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
+     operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
      return
       "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<Vetype>[%4]";
    }
@@ -3469,7 +3548,7 @@
 	  SQRDMLH_AS))]
    "TARGET_SIMD_RDMA"
    {
-     operands[4] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
+     operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
      return
       "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
    }
@@ -3487,7 +3566,7 @@
 	  SQRDMLH_AS))]
    "TARGET_SIMD_RDMA"
    {
-     operands[4] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
+     operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
      return
       "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<v>[%4]";
    }
@@ -3531,7 +3610,7 @@
 	    (const_int 1))))]
   "TARGET_SIMD"
   {
-    operands[4] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
+    operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
     return
       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
   }
@@ -3555,7 +3634,7 @@
 	    (const_int 1))))]
   "TARGET_SIMD"
   {
-    operands[4] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
+    operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
     return
       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
   }
@@ -3578,7 +3657,7 @@
 	    (const_int 1))))]
   "TARGET_SIMD"
   {
-    operands[4] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
+    operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
     return
       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
   }
@@ -3601,7 +3680,7 @@
 	    (const_int 1))))]
   "TARGET_SIMD"
   {
-    operands[4] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
+    operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
     return
       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
   }
@@ -3696,7 +3775,7 @@
 	      (const_int 1))))]
   "TARGET_SIMD"
   {
-    operands[4] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
+    operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
     return
      "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
   }
@@ -3722,7 +3801,7 @@
 	      (const_int 1))))]
   "TARGET_SIMD"
   {
-    operands[4] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
+    operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
     return
      "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
   }
@@ -3869,7 +3948,7 @@
 	     (const_int 1)))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
@@ -3890,7 +3969,7 @@
 	     (const_int 1)))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
@@ -3910,7 +3989,7 @@
 	     (const_int 1)))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
@@ -3930,7 +4009,7 @@
 	     (const_int 1)))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
@@ -4008,7 +4087,7 @@
 	     (const_int 1)))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
     return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
@@ -4031,7 +4110,7 @@
 	     (const_int 1)))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
     return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
   }
   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
@@ -4537,7 +4616,7 @@
 		   UNSPEC_LD2_LANE))]
   "TARGET_SIMD"
   {
-    operands[3] = endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
     return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
   }
   [(set_attr "type" "neon_load2_one_lane")]
@@ -4581,7 +4660,7 @@
 		   UNSPEC_ST2_LANE))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
   }
   [(set_attr "type" "neon_store2_one_lane<q>")]
@@ -4635,7 +4714,7 @@
 		   UNSPEC_LD3_LANE))]
   "TARGET_SIMD"
 {
-    operands[3] = endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
     return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
 }
   [(set_attr "type" "neon_load3_one_lane")]
@@ -4679,7 +4758,7 @@
 		    UNSPEC_ST3_LANE))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
   }
   [(set_attr "type" "neon_store3_one_lane<q>")]
@@ -4733,7 +4812,7 @@
 		   UNSPEC_LD4_LANE))]
   "TARGET_SIMD"
 {
-    operands[3] = endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
     return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
 }
   [(set_attr "type" "neon_load4_one_lane")]
@@ -4777,7 +4856,7 @@
 		    UNSPEC_ST4_LANE))]
   "TARGET_SIMD"
   {
-    operands[2] = endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
     return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
   }
   [(set_attr "type" "neon_store4_one_lane<q>")]
@@ -5280,6 +5359,9 @@
 [(set_attr "type" "multiple")]
 )
 
+;; This instruction's pattern is generated directly by
+;; aarch64_expand_vec_perm_const, so any changes to the pattern would
+;; need corresponding changes there.
 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
   [(set (match_operand:VALL_F16 0 "register_operand" "=w")
 	(unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
@@ -5290,7 +5372,10 @@
   [(set_attr "type" "neon_permute<q>")]
 )
 
-;; Note immediate (third) operand is lane index not byte index.
+;; This instruction's pattern is generated directly by
+;; aarch64_expand_vec_perm_const, so any changes to the pattern would
+;; need corresponding changes there.  Note that the immediate (third)
+;; operand is a lane index not a byte index.
 (define_insn "aarch64_ext<mode>"
   [(set (match_operand:VALL_F16 0 "register_operand" "=w")
         (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
@@ -5306,6 +5391,9 @@
   [(set_attr "type" "neon_ext<q>")]
 )
 
+;; This instruction's pattern is generated directly by
+;; aarch64_expand_vec_perm_const, so any changes to the pattern would
+;; need corresponding changes there.
 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
   [(set (match_operand:VALL_F16 0 "register_operand" "=w")
 	(unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")]
diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
index b6f839f5643..7052063bb23 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -1,4 +1,4 @@
-;; Machine description for AArch64 AdvSIMD architecture.
+;; Machine description for AArch64 SVE.
 ;; Copyright (C) 2009-2016 Free Software Foundation, Inc.
 ;; Contributed by ARM Ltd.
 ;;
@@ -18,17 +18,66 @@
 ;; along with GCC; see the file COPYING3.  If not see
 ;; <http://www.gnu.org/licenses/>.
 
+;; Note on the handling of big-endian SVE
+;; --------------------------------------
+;;
+;; On big-endian systems, Advanced SIMD mov<mode> patterns act in the
+;; same way as movdi or movti would: the first byte of memory goes
+;; into the most significant byte of the register and the last byte
+;; of memory goes into the least significant byte of the register.
+;; This is the most natural ordering for Advanced SIMD and matches
+;; the ABI layout for 64-bit and 128-bit vector types.
+;;
+;; As a result, the order of bytes within the register is what GCC
+;; expects for a big-endian target, and subreg offsets therefore work
+;; as expected, with the first element in memory having subreg offset 0
+;; and the last element in memory having the subreg offset associated
+;; with a big-endian lowpart.  However, this ordering also means that
+;; GCC's lane numbering does not match the architecture's numbering:
+;; GCC always treats the element at the lowest address in memory
+;; (subreg offset 0) as element 0, while the architecture treats
+;; the least significant end of the register as element 0.
+;;
+;; The situation for SVE is different.  We want the layout of the
+;; SVE register to be same for mov<mode> as it is for maskload<mode>:
+;; logically, a mov<mode> load should be indistinguishable from a
+;; maskload<mode> whose mask is all true.  We therefore need the
+;; register layout to match LD1 rather than LDR.  The ABI layout of
+;; SVE types also matches LD1 byte ordering rather than LDR byte ordering.
+;;
+;; As a result, the architecture lane numbering matches GCC's lane
+;; numbering, with element 0 always being the first in memory.
+;; However:
+;;
+;; - Applying a subreg offset to a register does not give the element
+;;   that GCC expects: the first element in memory has the subreg offset
+;;   associated with a big-endian lowpart while the last element in memory
+;;   has subreg offset 0.  We handle this via TARGET_CAN_CHANGE_MODE_CLASS.
+;;
+;; - We cannot use LDR and STR for spill slots that might be accessed
+;;   via subregs, since although the elements have the order GCC expects,
+;;   the order of the bytes within the elements is different.  We instead
+;;   access spill slots via LD1 and ST1, using secondary reloads to
+;;   reserve a predicate register.
+
+
+;; SVE data moves.
 (define_expand "mov<mode>"
   [(set (match_operand:SVE_ALL 0 "nonimmediate_operand")
 	(match_operand:SVE_ALL 1 "general_operand"))]
   "TARGET_SVE"
   {
-    if (MEM_P (operands[0]) || MEM_P (operands[1]))
+    /* Use the predicated load and store patterns where possible.
+       This is required for big-endian targets (see the comment at the
+       head of the file) and increases the addressing choices for
+       little-endian.  */
+    if ((MEM_P (operands[0]) || MEM_P (operands[1]))
+        && can_create_pseudo_p ())
       {
-	aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode,
-				     gen_pred_mov<mode>);
+	aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode);
 	DONE;
       }
+
     if (CONSTANT_P (operands[1]))
       {
 	aarch64_expand_mov_immediate (operands[0], operands[1],
@@ -38,26 +87,14 @@
   }
 )
 
-;; This will always load and store the elements in little-endian order.
-;: We therefore try to restrict its use to spill slots and make sure that
-;; all loads and stores to spill slots go through this pattern, so that
-;; everything agrees on the local endianness.  In particular:
-;;
-;; 1) The pattern doesn't accept moves involving memory operands before
-;;    register allocation.  The moves should use the richer pred_mov<mode>
-;;    pattern instead.
-;;
-;; 2) Big-endian targets use TARGET_CANNOT_SUBSTITUTE_MEM_EQUIV_P to disallow
-;;    memory equivalences during register allocation.  This should ensure
-;;    that any memory replacements of registers are fresh spill slots.
-;;
-;; ??? There is still the hole that this is effectively a REVB operation
-;; on big-endian targets, rather than the simple move that the RTL pattern
-;; claims.
-(define_insn "aarch64_sve_mov<mode>"
+;; Unpredicated moves (little-endian).  Only allow memory operations
+;; during and after RA; before RA we want the predicated load and
+;; store patterns to be used instead.
+(define_insn "*aarch64_sve_mov<mode>_le"
   [(set (match_operand:SVE_ALL 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w")
 	(match_operand:SVE_ALL 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))]
   "TARGET_SVE
+   && !BYTES_BIG_ENDIAN
    && ((lra_in_progress || reload_completed)
        || (register_operand (operands[0], <MODE>mode)
 	   && nonmemory_operand (operands[1], <MODE>mode)))"
@@ -68,12 +105,51 @@
    * return aarch64_output_sve_mov_immediate (operands[1]);"
 )
 
-(define_insn "pred_mov<mode>"
+;; Unpredicated moves (big-endian).  Memory accesses require secondary
+;; reloads.
+(define_insn "*aarch64_sve_mov<mode>_be"
+  [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w")
+	(match_operand:SVE_ALL 1 "aarch64_nonmemory_operand" "w, Dn"))]
+  "TARGET_SVE && BYTES_BIG_ENDIAN"
+  "@
+   mov\t%0.d, %1.d
+   * return aarch64_output_sve_mov_immediate (operands[1]);"
+)
+
+;; Handle big-endian memory reloads.  We use byte PTRUE for all modes
+;; to try to encourage reuse.
+(define_expand "aarch64_sve_reload_be"
+  [(parallel
+     [(set (match_operand 0)
+           (match_operand 1))
+      (clobber (match_operand:V32BI 2 "register_operand" "=Upl"))])]
+  "TARGET_SVE && BYTES_BIG_ENDIAN"
+  {
+    /* Create a PTRUE.  */
+    emit_move_insn (operands[2], CONSTM1_RTX (V32BImode));
+
+    /* Refer to the PTRUE in the appropriate mode for this move.  */
+    machine_mode mode = GET_MODE (operands[0]);
+    machine_mode pred_mode
+      = aarch64_sve_pred_mode (GET_MODE_UNIT_SIZE (mode)).require ();
+    rtx pred = gen_lowpart (pred_mode, operands[2]);
+
+    /* Emit a predicated load or store.  */
+    aarch64_emit_sve_pred_move (operands[0], pred, operands[1]);
+    DONE;
+  }
+)
+
+;; A predicated load or store for which the predicate is known to be
+;; all-true.  Note that this pattern is generated directly by
+;; aarch64_emit_sve_pred_move, so changes to this pattern will
+;; need changes there as well.
+(define_insn "*pred_mov<mode>"
   [(set (match_operand:SVE_ALL 0 "nonimmediate_operand" "=w, m")
 	(unspec:SVE_ALL
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
 	   (match_operand:SVE_ALL 2 "nonimmediate_operand" "m, w")]
-	  UNSPEC_PRED_MOVE))]
+	  UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE
    && (register_operand (operands[0], <MODE>mode)
        || register_operand (operands[2], <MODE>mode))"
@@ -82,6 +158,62 @@
    st1<Vesize>\t%2.<Vetype>, %1, %0"
 )
 
+(define_expand "movmisalign<mode>"
+  [(set (match_operand:SVE_ALL 0 "nonimmediate_operand")
+	(match_operand:SVE_ALL 1 "general_operand"))]
+  "TARGET_SVE"
+  {
+    /* Equivalent to a normal move for our purpooses.  */
+    emit_move_insn (operands[0], operands[1]);
+    DONE;
+  }
+)
+
+(define_insn "maskload<mode><vpred>"
+  [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
+	(unspec:SVE_ALL
+	  [(match_operand:<VPRED> 2 "register_operand" "Upl")
+	   (match_operand:SVE_ALL 1 "memory_operand" "m")]
+	  UNSPEC_LD1_SVE))]
+  "TARGET_SVE"
+  "ld1<Vesize>\t%0.<Vetype>, %2/z, %1"
+)
+
+(define_insn "maskstore<mode><vpred>"
+  [(set (match_operand:SVE_ALL 0 "memory_operand" "+m")
+	(unspec:SVE_ALL [(match_operand:<VPRED> 2 "register_operand" "Upl")
+			 (match_operand:SVE_ALL 1 "register_operand" "w")
+			 (match_dup 0)]
+			UNSPEC_ST1_SVE))]
+  "TARGET_SVE"
+  "st1<Vesize>\t%1.<Vetype>, %2, %0"
+)
+
+(define_expand "firstfault_load<mode>"
+  [(set (match_operand:SVE_ALL 0 "register_operand")
+	(unspec:SVE_ALL
+	  [(match_operand:SVE_ALL 1 "aarch64_sve_ldff1_operand")
+	   (match_dup 2)
+	   (reg:SI FFRT_REGNUM)]
+	  UNSPEC_LDFF1))]
+  "TARGET_SVE"
+  {
+    operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
+  }
+)
+
+(define_insn "*firstfault_load<mode>"
+  [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
+	(unspec:SVE_ALL
+	  [(match_operand:SVE_ALL 1 "aarch64_sve_ldff1_operand" "Utf")
+	   (match_operand:<VPRED> 2 "register_operand" "Upl")
+	   (reg:SI FFRT_REGNUM)]
+	  UNSPEC_LDFF1))]
+  "TARGET_SVE"
+  "ldff1<Vesize>\t%0.<Vetype>, %2/z, %j1";
+)
+
+;; SVE structure moves.
 (define_expand "mov<mode>"
   [(set (match_operand:SVE_STRUCT 0 "nonimmediate_operand")
 	(match_operand:SVE_STRUCT 1 "general_operand"))]
@@ -89,8 +221,7 @@
   {
     if (MEM_P (operands[0]) || MEM_P (operands[1]))
       {
-	aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode,
-				     gen_pred_mov<mode>);
+	aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode);
 	DONE;
       }
     if (CONSTANT_P (operands[1]))
@@ -101,17 +232,38 @@
   }
 )
 
-;; See the comments above the SVE_ALL aarch64_sve_mov<mode> for details
-;; of the memory handling.
-(define_insn_and_split "aarch64_sve_mov<mode>"
+;; Unpredicated structure moves (little-endian).  Only allow memory operations
+;; during and after RA; before RA we want the predicated load and store
+;; patterns to be used instead.
+(define_insn "*aarch64_sve_mov<mode>_le"
   [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w")
 	(match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))]
   "TARGET_SVE
+   && !BYTES_BIG_ENDIAN
    && ((lra_in_progress || reload_completed)
        || (register_operand (operands[0], <MODE>mode)
 	   && nonmemory_operand (operands[1], <MODE>mode)))"
   "#"
-  "&& reload_completed"
+  [(set_attr "length" "<insn_length>")]
+)
+
+;; Unpredicated structure moves (big-endian).  Memory accesses require
+;; secondary reloads.
+(define_insn "*aarch64_sve_mov<mode>_le"
+  [(set (match_operand:SVE_STRUCT 0 "register_operand" "=w, w")
+	(match_operand:SVE_STRUCT 1 "aarch64_nonmemory_operand" "w, Dn"))]
+  "TARGET_SVE && BYTES_BIG_ENDIAN"
+  "#"
+  [(set_attr "length" "<insn_length>")]
+)
+
+;; Split unpredicated structure moves into pieces.  This is the same
+;; for both big-endian and little-endian code, although it only needs
+;; to handle memory operands for little-endian code.
+(define_split
+  [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand")
+	(match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand"))]
+  "TARGET_SVE && reload_completed"
   [(const_int 0)]
   {
     rtx dest = operands[0];
@@ -125,11 +277,10 @@
 					     i * BYTES_PER_SVE_VECTOR);
 	  rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, src, <MODE>mode,
 					    i * BYTES_PER_SVE_VECTOR);
-	  emit_insn (gen_aarch64_sve_mov<mode> (subdest, subsrc));
+	  emit_insn (gen_rtx_SET (subdest, subsrc));
 	}
     DONE;
   }
-  [(set_attr "length" "<insn_length>")]
 )
 
 (define_insn_and_split "pred_mov<mode>"
@@ -137,7 +288,7 @@
 	(unspec:SVE_STRUCT
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
 	   (match_operand:SVE_STRUCT 2 "aarch64_sve_struct_nonimmediate_operand" "Utx, w")]
-	  UNSPEC_PRED_MOVE))]
+	  UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE
    && (register_operand (operands[0], <MODE>mode)
        || register_operand (operands[2], <MODE>mode))"
@@ -153,7 +304,7 @@
 	rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, operands[2],
 					  <MODE>mode,
 					  i * BYTES_PER_SVE_VECTOR);
-	emit_insn (gen_pred_mov<vsingle> (subdest, operands[1], subsrc));
+	aarch64_emit_sve_pred_move (subdest, operands[1], subsrc);
       }
     DONE;
   }
@@ -170,73 +321,147 @@
   }
 )
 
-(define_expand "movmisalign<mode>"
-  [(set (match_operand:SVE_ALL 0 "nonimmediate_operand")
-	(match_operand:SVE_ALL 1 "general_operand"))]
+(define_insn "*aarch64_sve_mov<mode>"
+  [(set (match_operand:PRED_ALL 0 "nonimmediate_operand" "=Upa, m, Upa, Upa, Upa")
+	(match_operand:PRED_ALL 1 "general_operand" "Upa, Upa, m, Dz, Dm"))]
+  "TARGET_SVE
+   && (register_operand (operands[0], <MODE>mode)
+       || register_operand (operands[1], <MODE>mode))"
+  "@
+   mov\t%0.b, %1.b
+   str\t%1, %0
+   ldr\t%0, %1
+   pfalse\t%0.b
+   * return aarch64_output_ptrue (<MODE>mode, '<Vetype>');"
+)
+
+;; Handle extractions from a predicate by converting to an integer vector
+;; and extracting from there.
+(define_expand "vec_extract<vpred><Vel>"
+  [(match_operand:<VEL> 0 "register_operand")
+   (match_operand:<VPRED> 1 "register_operand")
+   (match_operand:SI 2 "nonmemory_operand")
+   ;; Dummy operand to which we can attach the iterator.
+   (reg:SVE_I V0_REGNUM)]
   "TARGET_SVE"
   {
-    /* Equivalent to a normal move for our purpooses.  */
-    emit_move_insn (operands[0], operands[1]);
+    rtx tmp = gen_reg_rtx (<MODE>mode);
+    emit_insn (gen_aarch64_sve_dup<mode>_const (tmp, operands[1],
+						CONST1_RTX (<MODE>mode),
+						CONST0_RTX (<MODE>mode)));
+    emit_insn (gen_vec_extract<mode><Vel> (operands[0], tmp, operands[2]));
     DONE;
   }
 )
 
-(define_insn "maskload<mode><vpred>"
-  [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
-	(unspec:SVE_ALL
-	  [(match_operand:SVE_ALL 1 "memory_operand" "m")
-	   (match_operand:<VPRED> 2 "register_operand" "Upl")]
-	  UNSPEC_LD1_SVE))]
+(define_expand "vec_extract<mode><Vel>"
+  [(set (match_operand:<VEL> 0 "register_operand")
+	(vec_select:<VEL>
+	  (match_operand:SVE_ALL 1 "register_operand")
+	  (parallel [(match_operand:SI 2 "nonmemory_operand")])))]
   "TARGET_SVE"
-  "ld1<Vesize>\t%0.<Vetype>, %2/z, %1"
+  {
+    poly_int64 val;
+    if (poly_int_rtx_p (operands[2], &val)
+	&& must_eq (val, GET_MODE_NUNITS (<MODE>mode) - 1))
+      {
+	/* The last element can be extracted with a LASTB and a false
+	   predicate.  */
+	rtx sel = force_reg (<VPRED>mode, CONST0_RTX (<VPRED>mode));
+	emit_insn (gen_extract_last_<mode> (operands[0], operands[1], sel));
+	DONE;
+      }
+    if (!CONST_INT_P (operands[2]))
+      {
+	/* Create an index with operand[2] as the base and -1 as the step.
+	   It will then be zero for the element we care about.  */
+	rtx index = gen_lowpart (<VEL_INT>mode, operands[2]);
+	index = force_reg (<VEL_INT>mode, index);
+	rtx series = gen_reg_rtx (<V_INT_EQUIV>mode);
+	emit_insn (gen_vec_series<v_int_equiv> (series, index, constm1_rtx));
+
+	/* Get a predicate that is true for only that element.  */
+	rtx zero = CONST0_RTX (<V_INT_EQUIV>mode);
+	rtx cmp = gen_rtx_EQ (<V_INT_EQUIV>mode, series, zero);
+	rtx sel = gen_reg_rtx (<VPRED>mode);
+	emit_insn (gen_vec_cmp<v_int_equiv><vpred> (sel, cmp, series, zero));
+
+	/* Select the element using LASTB.  */
+	emit_insn (gen_extract_last_<mode> (operands[0], operands[1], sel));
+	DONE;
+      }
+  }
 )
 
-(define_insn "maskstore<mode><vpred>"
-  [(set (match_operand:SVE_ALL 0 "memory_operand" "+m")
-	(unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "w")
-			 (match_operand:<VPRED> 2 "register_operand" "Upl")
-			 (match_dup 0)]
-			UNSPEC_ST1_SVE))]
-  "TARGET_SVE"
-  "st1<Vesize>\t%1.<Vetype>, %2, %0"
+;; Extract an element from the Advanced SIMD portion of the register.
+;; We don't just reuse the aarch64-simd.md pattern because we don't
+;; want any chnage in lane number on big-endian targets.
+(define_insn "*vec_extract<mode><Vel>_v128"
+  [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
+	(vec_select:<VEL>
+	  (match_operand:SVE_ALL 1 "register_operand" "w, w, w")
+	  (parallel [(match_operand:SI 2 "const_int_operand")])))]
+  "TARGET_SVE
+   && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 0, 15)"
+  {
+    operands[1] = gen_lowpart (<V128>mode, operands[1]);
+    switch (which_alternative)
+      {
+	case 0:
+	  return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
+	case 1:
+	  return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
+	case 2:
+	  return "st1\\t{%1.<Vetype>}[%2], %0";
+	default:
+	  gcc_unreachable ();
+      }
+  }
+  [(set_attr "type" "neon_to_gp_q, neon_dup_q, neon_store1_one_lane_q")]
 )
 
-(define_expand "firstfault_load<mode>"
-  [(set (match_operand:SVE_ALL 0 "register_operand")
-	(unspec:SVE_ALL
-	  [(match_operand:SVE_ALL 1 "aarch64_sve_ldff1_operand")
-	   (match_dup 2)
-	   (reg:SI FFRT_REGNUM)]
-	  UNSPEC_LDFF1))]
-  "TARGET_SVE"
+;; Extract an element in the range of DUP.  This pattern allows the
+;; source and destination to be different.
+(define_insn "*vec_extract<mode><Vel>_dup"
+  [(set (match_operand:<VEL> 0 "register_operand" "=w")
+	(vec_select:<VEL>
+	  (match_operand:SVE_ALL 1 "register_operand" "w")
+	  (parallel [(match_operand:SI 2 "const_int_operand")])))]
+  "TARGET_SVE
+   && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 16, 63)"
   {
-    operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode));
+    operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
+    return "dup\t%0.<Vetype>, %1.<Vetype>[%2]";
   }
 )
 
-(define_insn "*firstfault_load<mode>"
-  [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
-	(unspec:SVE_ALL
-	  [(match_operand:SVE_ALL 1 "aarch64_sve_ldff1_operand" "Utf")
-	   (match_operand:<VPRED> 2 "register_operand" "Upl")
-	   (reg:SI FFRT_REGNUM)]
-	  UNSPEC_LDFF1))]
-  "TARGET_SVE"
-  "ldff1<Vesize>\t%0.<Vetype>, %2/z, %j1";
+;; Extract an element outside the range of DUP.  This pattern requires the
+;; source and destination to be the same.
+(define_insn "*vec_extract<mode><Vel>_ext"
+  [(set (match_operand:<VEL> 0 "register_operand" "=w")
+	(vec_select:<VEL>
+	  (match_operand:SVE_ALL 1 "register_operand" "0")
+	  (parallel [(match_operand:SI 2 "const_int_operand")])))]
+  "TARGET_SVE && INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode) >= 64"
+  {
+    operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
+    operands[2] = GEN_INT (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode));
+    return "ext\t%0.b, %0.b, %0.b, #%2";
+  }
 )
 
-(define_insn "*aarch64_sve_mov<mode>"
-  [(set (match_operand:PRED_ALL 0 "nonimmediate_operand" "=Upa, m, Upa, Upa, Upa")
-	(match_operand:PRED_ALL 1 "general_operand" "Upa, Upa, m, Dz, Dm"))]
-  "TARGET_SVE
-   && (register_operand (operands[0], <MODE>mode)
-       || register_operand (operands[1], <MODE>mode))"
+;; Extract the last active element of operand 1 into operand 0.
+;; If no elements are active, extract the last inactive element instead.
+(define_insn "extract_last_<mode>"
+  [(set (match_operand:<VEL> 0 "register_operand" "=r, w")
+	(unspec:<VEL>
+	  [(match_operand:SVE_ALL 1 "register_operand" "w, w")
+	   (match_operand:<VPRED> 2 "register_operand" "Upl, Upl")]
+	  UNSPEC_LASTB))]
+  "TARGET_SVE"
   "@
-   mov\t%0.b, %1.b
-   str\t%1, %0
-   ldr\t%0, %1
-   pfalse\t%0.b
-   * return aarch64_output_ptrue (<MODE>mode, '<Vetype>');"
+   lastb\t%<vwcore>0, %2, %1.<Vetype>
+   lastb\t%<Vetype>0, %2, %1.<Vetype>"
 )
 
 (define_expand "vec_duplicate<mode>"
@@ -318,13 +543,13 @@
 (define_insn "vec_series<mode>"
   [(set (match_operand:SVE_I 0 "register_operand" "=w, w, w")
 	(vec_series:SVE_I
-	  (match_operand:<VEL> 1 "aarch64_sve_index_operand" "r, Di, r")
-	  (match_operand:<VEL> 2 "aarch64_sve_index_operand" "r, r, Di")))]
+	  (match_operand:<VEL> 1 "aarch64_sve_index_operand" "Di, r, r")
+	  (match_operand:<VEL> 2 "aarch64_sve_index_operand" "r, Di, r")))]
   "TARGET_SVE"
   "@
-   index\t%0.<Vetype>, %<vw>1, %<vw>2
    index\t%0.<Vetype>, #%1, %<vw>2
-   index\t%0.<Vetype>, %<vw>1, #%2"
+   index\t%0.<Vetype>, %<vw>1, #%2
+   index\t%0.<Vetype>, %<vw>1, %<vw>2"
 )
 
 (define_expand "vec_gather_loads<mode>"
@@ -451,6 +676,8 @@
    st1<Vesize>\t%3.<Vetype>, %4, [%0, %1.<Vetype><gather_scaled_modu>]"
 )
 
+;; Optimize {x, x, x, x, ...} + {0, n, 2*n, 3*n, ...} if n is in range
+;; of an INDEX instruction.
 (define_insn "*vec_series<mode>_plus"
   [(set (match_operand:SVE_I 0 "register_operand" "=w")
 	(plus:SVE_I
@@ -626,15 +853,16 @@
    (match_operand:SVE_ALL 2 "register_operand")
    (match_operand:<V_INT_EQUIV> 3)]
   "TARGET_SIMD"
-{
-  unsigned int nunits;
-  if (GET_MODE_NUNITS (<MODE>mode).is_constant (&nunits)
-      && aarch64_expand_vec_perm_const (operands[0], operands[1],
-					operands[2], operands[3], nunits))
-    DONE;
-  else
-    FAIL;
-})
+  {
+    unsigned int nunits;
+    if (GET_MODE_NUNITS (<MODE>mode).is_constant (&nunits)
+	&& aarch64_expand_vec_perm_const (operands[0], operands[1],
+					  operands[2], operands[3], nunits))
+      DONE;
+    else
+      FAIL;
+  }
+)
 
 (define_expand "vec_perm<mode>"
   [(match_operand:SVE_ALL 0 "register_operand")
@@ -659,18 +887,18 @@
 (define_expand "<perm_optab>_<mode>"
   [(set (match_operand:SVE_ALL 0 "register_operand")
 	(unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand")
-			  (match_operand:SVE_ALL 2 "register_operand")]
-			 OPTAB_PERMUTE))]
+			 (match_operand:SVE_ALL 2 "register_operand")]
+			OPTAB_PERMUTE))]
   "TARGET_SVE")
 
 (define_insn "vec_reverse_<mode>"
   [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
 	(unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "w")]
-			 UNSPEC_REV))]
+			UNSPEC_REV))]
   "TARGET_SVE"
   "rev\t%0.<Vetype>, %1.<Vetype>")
 
-(define_insn "sve_tbl1<mode>"
+(define_insn "*aarch64_sve_tbl<mode>"
   [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
 	(unspec:SVE_ALL
 	  [(match_operand:SVE_ALL 1 "register_operand" "w")
@@ -680,7 +908,7 @@
   "tbl\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
 )
 
-(define_insn "sve_<perm_insn><perm_hilo><mode>"
+(define_insn "*aarch64_sve_<perm_insn><perm_hilo><mode>"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(unspec:PRED_ALL [(match_operand:PRED_ALL 1 "register_operand" "Upa")
 			  (match_operand:PRED_ALL 2 "register_operand" "Upa")]
@@ -689,7 +917,7 @@
   "<perm_insn><perm_hilo>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
 )
 
-(define_insn "sve_<perm_insn><perm_hilo><mode>"
+(define_insn "*aarch64_sve_<perm_insn><perm_hilo><mode>"
   [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
 	(unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "w")
 			 (match_operand:SVE_ALL 2 "register_operand" "w")]
@@ -698,7 +926,7 @@
   "<perm_insn><perm_hilo>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
 )
 
-(define_insn "sve_rev64<mode>"
+(define_insn "*aarch64_sve_rev64<mode>"
   [(set (match_operand:SVE_BHS 0 "register_operand" "=w")
 	(unspec:SVE_BHS
 	  [(match_operand:V4BI 1 "register_operand" "Upl")
@@ -709,7 +937,7 @@
   "rev<Vesize>\t%0.d, %1/m, %2.d"
 )
 
-(define_insn "sve_rev32<mode>"
+(define_insn "*aarch64_sve_rev32<mode>"
   [(set (match_operand:SVE_BH 0 "register_operand" "=w")
 	(unspec:SVE_BH
 	  [(match_operand:V8BI 1 "register_operand" "Upl")
@@ -720,7 +948,7 @@
   "rev<Vesize>\t%0.s, %1/m, %2.s"
 )
 
-(define_insn "sve_rev16v32qi"
+(define_insn "*aarch64_sve_rev16v32qi"
   [(set (match_operand:V32QI 0 "register_operand" "=w")
 	(unspec:V32QI
 	  [(match_operand:V16BI 1 "register_operand" "Upl")
@@ -731,12 +959,12 @@
   "revb\t%0.h, %1/m, %2.h"
 )
 
-(define_insn "sve_dup_lane<mode>"
+(define_insn "*aarch64_sve_dup_lane<mode>"
   [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
 	(vec_duplicate:SVE_ALL
 	  (vec_select:<VEL>
 	    (match_operand:SVE_ALL 1 "register_operand" "w")
-	    (parallel [(match_operand:SI 2 "const_int_operand" "i")]))))]
+	    (parallel [(match_operand:SI 2 "const_int_operand")]))))]
   "TARGET_SVE
    && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 0, 63)"
   "dup\t%0.<Vetype>, %1.<Vetype>[%2]"
@@ -744,17 +972,17 @@
 
 ;; Note that the immediate (third) operand is the lane index not
 ;; the byte index.
-(define_insn "aarch64_ext<mode>"
+(define_insn "*aarch64_sve_ext<mode>"
   [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
 	(unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "0")
 			 (match_operand:SVE_ALL 2 "register_operand" "w")
-			 (match_operand:SI 3 "const_int_operand" "i")]
+			 (match_operand:SI 3 "const_int_operand")]
 			UNSPEC_EXT))]
   "TARGET_SVE
    && IN_RANGE (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode), 0, 255)"
   {
     operands[3] = GEN_INT (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode));
-    return "ext\\t%0.b, %1.b, %2.b, #%3";
+    return "ext\\t%0.b, %0.b, %2.b, #%3";
   }
 )
 
@@ -765,7 +993,7 @@
 	  (match_operand:SVE_I 2 "aarch64_sve_add_operand" "vsa, vsn, vsi, w")))]
   "TARGET_SVE"
   "@
-   add\t%0.<Vetype>, %0.<Vetype>, #%2
+   add\t%0.<Vetype>, %0.<Vetype>, #%D2
    sub\t%0.<Vetype>, %0.<Vetype>, #%N2
    * return aarch64_output_sve_inc_dec_immediate (\"%0.<Vetype>\", operands[2]);
    add\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
@@ -779,9 +1007,10 @@
   "TARGET_SVE"
   "@
    sub\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>
-   subr\t%0.<Vetype>, %0.<Vetype>, #%1"
+   subr\t%0.<Vetype>, %0.<Vetype>, #%D1"
 )
 
+;; Unpredicated multiplication.
 (define_expand "mul<mode>3"
   [(set (match_operand:SVE_I 0 "register_operand")
 	(unspec:SVE_I
@@ -796,9 +1025,10 @@
   }
 )
 
-;; We don't actually need the predicate for the first operand, but using Upa
-;; or X isn't likely to gain much and would make the instruction seem less
-;; uniform to the register allocator.
+;; Multiplication predicated with a PTRUE.  We don't actually need the
+;; predicate for the first alternative, but using Upa or X isn't likely
+;; to gain much and would make the instruction seem less uniform to the
+;; register allocator.
 (define_insn "*mul<mode>3"
   [(set (match_operand:SVE_I 0 "register_operand" "=w, w")
 	(unspec:SVE_I
@@ -843,6 +1073,7 @@
    mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
 )
 
+;; Unpredicated NEG, NOT and POPCOUNT.
 (define_expand "<optab><mode>2"
   [(set (match_operand:SVE_I 0 "register_operand")
 	(unspec:SVE_I
@@ -855,6 +1086,7 @@
   }
 )
 
+;; NEG, NOT and POPCOUNT predicated with a PTRUE.
 (define_insn "*<optab><mode>2"
   [(set (match_operand:SVE_I 0 "register_operand" "=w")
 	(unspec:SVE_I
@@ -866,6 +1098,7 @@
   "<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
 )
 
+;; Vector AND, ORR and XOR.
 (define_insn "<optab><mode>3"
   [(set (match_operand:SVE_I 0 "register_operand" "=w, w")
 	(LOGICAL:SVE_I
@@ -877,6 +1110,9 @@
    <logical>\t%0.d, %1.d, %2.d"
 )
 
+;; Vector AND, ORR and XOR on floating-point modes.  We avoid subregs
+;; by providing this, but we need to use UNSPECs since rtx logical ops
+;; aren't defined for floating-point modes.
 (define_insn "*<optab><mode>3"
   [(set (match_operand:SVE_F 0 "register_operand" "=w")
 	(unspec:SVE_F [(match_operand:SVE_F 1 "register_operand" "w")
@@ -897,6 +1133,7 @@
   "bic\t%0.d, %2.d, %1.d"
 )
 
+;; Predicate AND.  We can reuse one of the inputs as the GP.
 (define_insn "and<mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(and:PRED_ALL (match_operand:PRED_ALL 1 "register_operand" "Upa")
@@ -905,6 +1142,7 @@
   "and\t%0.b, %1/z, %1.b, %2.b"
 )
 
+;; Unpredicated predicate ORR and XOR.
 (define_expand "<optab><mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand")
 	(and:PRED_ALL
@@ -918,6 +1156,7 @@
   }
 )
 
+;; Predicated predicate ORR and XOR.
 (define_insn "pred_<optab><mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(and:PRED_ALL
@@ -929,12 +1168,12 @@
   "<logical>\t%0.b, %1/z, %2.b, %3.b"
 )
 
-;; Perform a logical operation on the active bits of operands 2 and 3,
-;; using operand 1 as the GP.  Store the result in operand 0 and set the flags
-;; in the same way as for PTEST.  The (and ...) in the UNSPEC_PTEST_PTRUE is
-;; logically redundant, but means that the tested value is structurally
-;; equivalent to rhs of the second set.
-(define_insn "*pred_<optab><mode>3_cc"
+;; Perform a logical operation on operands 2 and 3, using operand 1 as
+;; the GP (which is known to be a PTRUE).  Store the result in operand 0
+;; and set the flags in the same way as for PTEST.  The (and ...) in the
+;; UNSPEC_PTEST_PTRUE is logically redundant, but means that the tested
+;; value is structurally equivalent to rhs of the second set.
+(define_insn "*<optab><mode>3_cc"
   [(set (reg:CC CC_REGNUM)
 	(compare:CC
 	  (unspec:SI [(match_operand:PRED_ALL 1 "register_operand" "Upa")
@@ -952,6 +1191,7 @@
   "<logical>s\t%0.b, %1/z, %2.b, %3.b"
 )
 
+;; Unpredicated predicate inverse.
 (define_expand "one_cmpl<mode>2"
   [(set (match_operand:PRED_ALL 0 "register_operand")
 	(and:PRED_ALL
@@ -963,6 +1203,7 @@
   }
 )
 
+;; Predicated predicate inverse.
 (define_insn "*one_cmpl<mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(and:PRED_ALL
@@ -972,6 +1213,7 @@
   "not\t%0.b, %1/z, %2.b"
 )
 
+;; Predicated predicate BIC and ORN.
 (define_insn "*<nlogical><mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(and:PRED_ALL
@@ -983,6 +1225,7 @@
   "<nlogical>\t%0.b, %1/z, %3.b, %2.b"
 )
 
+;; Predicated predicate NAND and NOR.
 (define_insn "*<logical_nn><mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(and:PRED_ALL
@@ -1004,6 +1247,7 @@
   "brka\t%0.b, %1/z, %2.b"
 )
 
+;; Unpredicated LSL, LSR and ASR by a vector.
 (define_expand "v<optab><mode>3"
   [(set (match_operand:SVE_I 0 "register_operand")
 	(unspec:SVE_I
@@ -1018,6 +1262,10 @@
   }
 )
 
+;; LSL, LSR and ASR by a vector, predicated with a PTRUE.  We don't
+;; actually need the predicate for the first alternative, but using Upa
+;; or X isn't likely to gain much and would make the instruction seem
+;; less uniform to the register allocator.
 (define_insn "*v<optab><mode>3"
   [(set (match_operand:SVE_I 0 "register_operand" "=w, w")
 	(unspec:SVE_I
@@ -1032,6 +1280,8 @@
    <shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
 )
 
+;; LSL, LSR and ASR by a scalar, which expands into one of the vector
+;; shifts above.
 (define_expand "<ASHIFT:optab><mode>3"
   [(set (match_operand:SVE_I 0 "register_operand")
 	(ASHIFT:SVE_I (match_operand:SVE_I 1 "register_operand")
@@ -1057,7 +1307,7 @@
   }
 )
 
-;; Test all bits of operand 1.  Operand 0 is a PTRUE GP.
+;; Test all bits of operand 1.  Operand 0 is a GP that is known to hold PTRUE.
 ;;
 ;; Using UNSPEC_PTEST_PTRUE allows combine patterns to assume that the GP
 ;; is a PTRUE even if the optimizers haven't yet been able to propagate
@@ -1074,6 +1324,8 @@
   "ptest\t%0, %1.b"
 )
 
+;; Set element I of the result if operand1 + J < operand2 for all J in [0, I].
+;; with the comparison being unsigned.
 (define_insn "while_ult<GPI:mode><PRED_ALL:mode>"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(unspec:PRED_ALL [(match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
@@ -1084,6 +1336,9 @@
   "whilelo\t%0.<PRED_ALL:Vetype>, %<w>1, %<w>2"
 )
 
+;; WHILELO sets the flags in the same way as a PTEST with a PTRUE GP.
+;; Handle the case in which both results are useful.  The GP operand
+;; to the PTEST isn't needed, so we allow it to be anything.
 (define_insn_and_split "while_ult<GPI:mode><PRED_ALL:mode>_cc"
   [(set (reg:CC CC_REGNUM)
 	(compare:CC
@@ -1112,39 +1367,39 @@
   }
 )
 
-;; Predicated comparison.
+;; Predicated integer comparison.
 (define_insn "*vec_cmp<cmp_op>_<mode>"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
 	(unspec:<VPRED>
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
 	   (match_operand:SVE_I 2 "register_operand" "w, w")
-	   (match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "w, <imm_con>")]
+	   (match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "<imm_con>, w")]
 	  SVE_COND_INT_CMP))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_SVE"
   "@
-   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>
-   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3"
+   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
+   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
 )
 
-;; Predicated comparison in which only the flags result is interesting.
+;; Predicated integer comparison in which only the flags result is interesting.
 (define_insn "*vec_cmp<cmp_op>_<mode>_ptest"
   [(set (reg:CC CC_REGNUM)
 	(compare:CC
 	  (unspec:SI
 	    [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
 	     (unspec:<VPRED>
-	       [(match_operand:SVE_I 2 "register_operand" "w, w")
-		(match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "w, <imm_con>")
-		(match_dup 3)]
+	       [(match_dup 1)
+	        (match_operand:SVE_I 2 "register_operand" "w, w")
+		(match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "<imm_con>, w")]
 	       SVE_COND_INT_CMP)]
 	    UNSPEC_PTEST_PTRUE)
 	  (const_int 0)))
    (clobber (match_scratch:<VPRED> 0 "=Upa, Upa"))]
   "TARGET_SVE"
   "@
-   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>
-   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3"
+   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
+   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
 )
 
 ;; Predicated comparison in which both the flag and predicate results
@@ -1157,7 +1412,7 @@
 	     (unspec:<VPRED>
 	       [(match_dup 1)
 		(match_operand:SVE_I 2 "register_operand" "w, w")
-		(match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "w, <imm_con>")]
+		(match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "<imm_con>, w")]
 	       SVE_COND_INT_CMP)]
 	    UNSPEC_PTEST_PTRUE)
 	  (const_int 0)))
@@ -1169,8 +1424,8 @@
 	  SVE_COND_INT_CMP))]
   "TARGET_SVE"
   "@
-   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>
-   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3"
+   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
+   cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
 )
 
 (define_insn "*fold_cond_<cmp_op><mode>"
@@ -1179,29 +1434,32 @@
 	  (unspec:<VPRED>
 	    [(match_operand:<VPRED> 1 "aarch64_simd_imm_minus_one")
 	     (match_operand:SVE_I 2 "register_operand" "w, w")
-	     (match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "w, <imm_con>")]
+	     (match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "<imm_con>, w")]
 	    SVE_COND_INT_CMP)
 	  (match_operand:<VPRED> 4 "register_operand" "Upl, Upl")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_SVE"
   "@
-   cmp<cmp_op>\t%0.<Vetype>, %4/z, %2.<Vetype>, %3.<Vetype>
-   cmp<cmp_op>\t%0.<Vetype>, %4/z, %2.<Vetype>, #%3"
+   cmp<cmp_op>\t%0.<Vetype>, %4/z, %2.<Vetype>, #%3
+   cmp<cmp_op>\t%0.<Vetype>, %4/z, %2.<Vetype>, %3.<Vetype>"
 )
 
+;; Predicated floating-point comparison (excluding FCMUO, which doesn't
+;; allow #0.0 as an operand).
 (define_insn "*vec_fcm<cmp_op><mode>"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
 	(unspec:<VPRED>
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
 	   (match_operand:SVE_F 2 "register_operand" "w, w")
-	   (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "w, Dz")]
+	   (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
 	  SVE_COND_FP_CMP))]
   "TARGET_SVE"
   "@
-   fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>
-   fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0"
+   fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0
+   fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
 )
 
+;; Predicated FCMUO.
 (define_insn "*vec_fcmuo<mode>"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
 	(unspec:<VPRED>
@@ -1256,7 +1514,8 @@
   "sel\t%0.<Vetype>, %3, %1.<Vetype>, %2.<Vetype>"
 )
 
-(define_insn "*sel_dup<mode>_const"
+;; Selects between a duplicated immediate and zero.
+(define_insn "aarch64_sve_dup<mode>_const"
   [(set (match_operand:SVE_I 0 "register_operand" "=w")
 	(unspec:SVE_I
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl")
@@ -1267,6 +1526,8 @@
   "mov\t%0.<Vetype>, %1/z, #%2"
 )
 
+;; Integer (signed) vcond.  Don't enforce an immediate range here, since it
+;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead.
 (define_expand "vcond<mode><v_int_equiv>"
   [(set (match_operand:SVE_ALL 0 "register_operand")
 	(if_then_else:SVE_ALL
@@ -1282,21 +1543,8 @@
   }
 )
 
-(define_expand "vcond<mode><v_fp_equiv>"
-  [(set (match_operand:SVE_SD 0 "register_operand")
-	(if_then_else:SVE_SD
-	  (match_operator 3 "comparison_operator"
-	    [(match_operand:<V_FP_EQUIV> 4 "register_operand")
-	     (match_operand:<V_FP_EQUIV> 5 "aarch64_simd_reg_or_zero")])
-	  (match_operand:SVE_SD 1 "register_operand")
-	  (match_operand:SVE_SD 2 "register_operand")))]
-  "TARGET_SVE"
-  {
-    aarch64_expand_sve_vcond (<MODE>mode, <V_FP_EQUIV>mode, operands);
-    DONE;
-  }
-)
-
+;; Integer vcondu.  Don't enforce an immediate range here, since it
+;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead.
 (define_expand "vcondu<mode><v_int_equiv>"
   [(set (match_operand:SVE_ALL 0 "register_operand")
 	(if_then_else:SVE_ALL
@@ -1312,6 +1560,27 @@
   }
 )
 
+;; Floating-point vcond.  All comparisons except FCMUO allow a zero
+;; operand; aarch64_expand_sve_vcond handles the case of an FCMUO
+;; with zero.
+(define_expand "vcond<mode><v_fp_equiv>"
+  [(set (match_operand:SVE_SD 0 "register_operand")
+	(if_then_else:SVE_SD
+	  (match_operator 3 "comparison_operator"
+	    [(match_operand:<V_FP_EQUIV> 4 "register_operand")
+	     (match_operand:<V_FP_EQUIV> 5 "aarch64_simd_reg_or_zero")])
+	  (match_operand:SVE_SD 1 "register_operand")
+	  (match_operand:SVE_SD 2 "register_operand")))]
+  "TARGET_SVE"
+  {
+    aarch64_expand_sve_vcond (<MODE>mode, <V_FP_EQUIV>mode, operands);
+    DONE;
+  }
+)
+
+;; Signed integer comparisons.  Don't enforce an immediate range here, since
+;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int
+;; instead.
 (define_expand "vec_cmp<mode><vpred>"
   [(parallel
     [(set (match_operand:<VPRED> 0 "register_operand")
@@ -1327,6 +1596,9 @@
   }
 )
 
+;; Unsigned integer comparisons.  Don't enforce an immediate range here, since
+;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int
+;; instead.
 (define_expand "vec_cmpu<mode><vpred>"
   [(parallel
     [(set (match_operand:<VPRED> 0 "register_operand")
@@ -1342,6 +1614,9 @@
   }
 )
 
+;; Floating-point comparisons.  All comparisons except FCMUO allow a zero
+;; operand; aarch64_expand_sve_vec_cmp_float handles the case of an FCMUO
+;; with zero.
 (define_expand "vec_cmp<mode><vpred>"
   [(set (match_operand:<VPRED> 0 "register_operand")
 	(match_operator:<VPRED> 1 "comparison_operator"
@@ -1355,6 +1630,7 @@
   }
 )
 
+;; Branch based on predicate equality or inequality.
 (define_expand "cbranch<mode>4"
   [(set (pc)
 	(if_then_else
@@ -1381,7 +1657,7 @@
   }
 )
 
-;; Max/Min operations.
+;; Unpredicated integer MIN/MAX.
 (define_expand "<su><maxmin><mode>3"
   [(set (match_operand:SVE_I 0 "register_operand")
 	(unspec:SVE_I
@@ -1395,6 +1671,7 @@
   }
 )
 
+;; Integer MIN/MAX predicated with a PTRUE.
 (define_insn "*<su><maxmin><mode>3"
   [(set (match_operand:SVE_I 0 "register_operand" "=w")
 	(unspec:SVE_I
@@ -1406,6 +1683,7 @@
   "<su><maxmin>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
 )
 
+;; Unpredicated floating-point MIN/MAX.
 (define_expand "<su><maxmin><mode>3"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1419,6 +1697,7 @@
   }
 )
 
+;; Floating-point MIN/MAX predicated with a PTRUE.
 (define_insn "*<su><maxmin><mode>3"
   [(set (match_operand:SVE_F 0 "register_operand" "=w")
 	(unspec:SVE_F
@@ -1430,6 +1709,7 @@
   "f<maxmin>nm\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>"
 )
 
+;; Unpredicated fmin/fmax.
 (define_expand "<maxmin_uns><mode>3"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1444,6 +1724,7 @@
   }
 )
 
+;; fmin/fmax predicated with a PTRUE.
 (define_insn "*<maxmin_uns><mode>3"
   [(set (match_operand:SVE_F 0 "register_operand" "=w")
 	(unspec:SVE_F
@@ -1505,6 +1786,7 @@
    clastb\t%<vw>0, %1, %<vw>0, %3.<Vetype>"
 )
 
+;; Unpredicated integer add reduction.
 (define_expand "reduc_plus_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand")
 	(unspec:<VEL> [(match_dup 2)
@@ -1516,6 +1798,7 @@
   }
 )
 
+;; Predicated integer add reduction.  The result is always 64-bits.
 (define_insn "*reduc_plus_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand" "=w")
 	(unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
@@ -1525,6 +1808,7 @@
   "uaddv\t%d0, %1, %2.<Vetype>"
 )
 
+;; Unpredicated floating-point add reduction.
 (define_expand "reduc_plus_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand")
 	(unspec:<VEL> [(match_dup 2)
@@ -1536,6 +1820,7 @@
   }
 )
 
+;; Predicated floating-point add reduction.
 (define_insn "*reduc_plus_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand" "=w")
 	(unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
@@ -1545,6 +1830,7 @@
   "faddv\t%<Vetype>0, %1, %2.<Vetype>"
 )
 
+;; Unpredicated integer MIN/MAX reduction.
 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand")
 	(unspec:<VEL> [(match_dup 2)
@@ -1556,6 +1842,7 @@
   }
 )
 
+;; Predicated integer MIN/MAX reduction.
 (define_insn "*reduc_<maxmin_uns>_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand" "=w")
 	(unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
@@ -1565,6 +1852,7 @@
   "<maxmin_uns_op>v\t%<Vetype>0, %1, %2.<Vetype>"
 )
 
+;; Unpredicated floating-point MIN/MAX reduction.
 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand")
 	(unspec:<VEL> [(match_dup 2)
@@ -1576,6 +1864,7 @@
   }
 )
 
+;; Predicated floating-point MIN/MAX reduction.
 (define_insn "*reduc_<maxmin_uns>_scal_<mode>"
   [(set (match_operand:<VEL> 0 "register_operand" "=w")
 	(unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl")
@@ -1644,6 +1933,7 @@
   "fadda\t%<Vetype>0, %2, %<Vetype>0, %3.<Vetype>"
 )
 
+;; Unpredicated floating-point addition.
 (define_expand "add<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1658,6 +1948,7 @@
   }
 )
 
+;; Floating-point addition predicated with a PTRUE.
 (define_insn "*add<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w, w")
 	(unspec:SVE_F
@@ -1673,6 +1964,7 @@
    fadd\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>"
 )
 
+;; Unpredicated floating-point subtraction.
 (define_expand "sub<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1687,24 +1979,26 @@
   }
 )
 
+;; Floating-point subtraction predicated with a PTRUE.
 (define_insn "*sub<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w, w, w")
 	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl")
 	   (minus:SVE_F
-	     (match_operand:SVE_F 2 "aarch64_sve_float_arith_operand" "w, 0, 0, vfa")
-	     (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand" "w, vfa, vfn, 0"))]
+	     (match_operand:SVE_F 2 "aarch64_sve_float_arith_operand" "0, 0, vfa, w")
+	     (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand" "vfa, vfn, 0, w"))]
 	  UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE
    && (register_operand (operands[2], <MODE>mode)
        || register_operand (operands[3], <MODE>mode))"
   "@
-   fsub\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>
    fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3
    fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3
-   fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2"
+   fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2
+   fsub\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>"
 )
 
+;; Unpredicated floating-point multiplication.
 (define_expand "mul<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1719,6 +2013,7 @@
   }
 )
 
+;; Floating-point multiplication predicated with a PTRUE.
 (define_insn "*mul<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w")
 	(unspec:SVE_F
@@ -1733,8 +2028,7 @@
    fmul\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>"
 )
 
-;; Note: fma is %0 = (%1 * %2) + %3
-
+;; Unpredicated fma (%0 = (%1 * %2) + %3).
 (define_expand "fma<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1749,6 +2043,7 @@
   }
 )
 
+;; fma predicated with a PTRUE.
 (define_insn "*fma<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w")
 	(unspec:SVE_F
@@ -1763,6 +2058,7 @@
    fmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
 )
 
+;; Unpredicated fnma (%0 = (-%1 * %2) + %3).
 (define_expand "fnma<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1778,6 +2074,7 @@
   }
 )
 
+;; fnma predicated with a PTRUE.
 (define_insn "*fnma<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w")
 	(unspec:SVE_F
@@ -1793,12 +2090,12 @@
    fmls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
 )
 
-(define_expand "fnms<mode>4"
+;; Unpredicated fms (%0 = (%1 * %2) - %3).
+(define_expand "fms<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
 	  [(match_dup 4)
-	   (fma:SVE_F (neg:SVE_F
-			(match_operand:SVE_F 1 "register_operand"))
+	   (fma:SVE_F (match_operand:SVE_F 1 "register_operand")
 		      (match_operand:SVE_F 2 "register_operand")
 		      (neg:SVE_F
 			(match_operand:SVE_F 3 "register_operand")))]
@@ -1809,27 +2106,29 @@
   }
 )
 
-(define_insn "*fnms<mode>4"
+;; fms predicated with a PTRUE.
+(define_insn "*fms<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w")
 	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
-	   (fma:SVE_F (neg:SVE_F
-			(match_operand:SVE_F 2 "register_operand" "%0, w"))
+	   (fma:SVE_F (match_operand:SVE_F 2 "register_operand" "%0, w")
 		      (match_operand:SVE_F 3 "register_operand" "w, w")
 		      (neg:SVE_F
 			(match_operand:SVE_F 4 "register_operand" "w, 0")))]
 	  UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE"
   "@
-   fnmad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
-   fnmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
+   fnmsb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
+   fnmls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
 )
 
-(define_expand "fms<mode>4"
+;; Unpredicated fnms (%0 = (-%1 * %2) - %3).
+(define_expand "fnms<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
 	  [(match_dup 4)
-	   (fma:SVE_F (match_operand:SVE_F 1 "register_operand")
+	   (fma:SVE_F (neg:SVE_F
+			(match_operand:SVE_F 1 "register_operand"))
 		      (match_operand:SVE_F 2 "register_operand")
 		      (neg:SVE_F
 			(match_operand:SVE_F 3 "register_operand")))]
@@ -1840,21 +2139,24 @@
   }
 )
 
-(define_insn "*fms<mode>4"
+;; fnms predicated with a PTRUE.
+(define_insn "*fnms<mode>4"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w")
 	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
-	   (fma:SVE_F (match_operand:SVE_F 2 "register_operand" "%0, w")
+	   (fma:SVE_F (neg:SVE_F
+			(match_operand:SVE_F 2 "register_operand" "%0, w"))
 		      (match_operand:SVE_F 3 "register_operand" "w, w")
 		      (neg:SVE_F
 			(match_operand:SVE_F 4 "register_operand" "w, 0")))]
 	  UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE"
   "@
-   fnmsb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
-   fnmls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
+   fnmad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
+   fnmla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>"
 )
 
+;; Unpredicated floating-point division.
 (define_expand "div<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1868,6 +2170,7 @@
   }
 )
 
+;; Floating-point division predicated with a PTRUE.
 (define_insn "*div<mode>3"
   [(set (match_operand:SVE_F 0 "register_operand" "=w, w")
 	(unspec:SVE_F
@@ -1881,6 +2184,7 @@
    fdivr\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
 )
 
+;; Unpredicated FNEG, FABS and FSQRT.
 (define_expand "<optab><mode>2"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1893,6 +2197,7 @@
   }
 )
 
+;; FNEG, FABS and FSQRT predicated with a PTRUE.
 (define_insn "*<optab><mode>2"
   [(set (match_operand:SVE_F 0 "register_operand" "=w")
 	(unspec:SVE_F
@@ -1903,6 +2208,7 @@
   "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>"
 )
 
+;; Unpredicated FRINTy.
 (define_expand "<frint_pattern><mode>2"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1916,6 +2222,7 @@
   }
 )
 
+;; FRINTy predicated with a PTRUE.
 (define_insn "*<frint_pattern><mode>2"
   [(set (match_operand:SVE_F 0 "register_operand" "=w")
 	(unspec:SVE_F
@@ -1927,7 +2234,8 @@
   "frint<frint_suffix>\t%0.<Vetype>, %1/m, %2.<Vetype>"
 )
 
-;; Convert float to integer of the same size (sf to si or df to di).
+;; Unpredicated conversion of floats to integers of the same size (HF to HI,
+;; SF to SI or DF to DI).
 (define_expand "<fix_trunc_optab><mode><v_int_equiv>2"
   [(set (match_operand:<V_INT_EQUIV> 0 "register_operand")
 	(unspec:<V_INT_EQUIV>
@@ -1941,6 +2249,19 @@
   }
 )
 
+;; Conversion of SF to DI, SI or HI, predicated with a PTRUE.
+(define_insn "*<fix_trunc_optab>v16hsf<mode>2"
+  [(set (match_operand:SVE_HSDI 0 "register_operand" "=w")
+	(unspec:SVE_HSDI
+	  [(match_operand:<VPRED> 1 "register_operand" "Upl")
+	   (FIXUORS:SVE_HSDI
+	     (match_operand:V16HF 2 "register_operand" "w"))]
+	  UNSPEC_MERGE_PTRUE))]
+  "TARGET_SVE"
+  "fcvtz<su>\t%0.<Vetype>, %1/m, %2.h"
+)
+
+;; Conversion of SF to DI or SI, predicated with a PTRUE.
 (define_insn "*<fix_trunc_optab>v8sf<mode>2"
   [(set (match_operand:SVE_SDI 0 "register_operand" "=w")
 	(unspec:SVE_SDI
@@ -1952,6 +2273,7 @@
   "fcvtz<su>\t%0.<Vetype>, %1/m, %2.s"
 )
 
+;; Conversion of DF to DI or SI, predicated with a PTRUE.
 (define_insn "*<fix_trunc_optab>v4df<mode>2"
   [(set (match_operand:SVE_SDI 0 "register_operand" "=w")
 	(unspec:SVE_SDI
@@ -1963,7 +2285,8 @@
   "fcvtz<su>\t%0.<Vetype>, %1/m, %2.d"
 )
 
-;; Convert integer to float of the same size (si to sf or di to df).
+;; Unpredicated conversion of integers to floats of the same size
+;; (HI to HF, SI to SF or DI to DF).
 (define_expand "<optab><v_int_equiv><mode>2"
   [(set (match_operand:SVE_F 0 "register_operand")
 	(unspec:SVE_F
@@ -1977,6 +2300,20 @@
   }
 )
 
+;; Conversion of DI, SI or HI to the same number of HFs, predicated
+;; with a PTRUE.
+(define_insn "*<optab><mode>v16hf2"
+  [(set (match_operand:V16HF 0 "register_operand" "=w")
+	(unspec:V16HF
+	  [(match_operand:<VPRED> 1 "register_operand" "Upl")
+	   (FLOATUORS:V16HF
+	     (match_operand:SVE_HSDI 2 "register_operand" "w"))]
+	  UNSPEC_MERGE_PTRUE))]
+  "TARGET_SVE"
+  "<su_optab>cvtf\t%0.h, %1/m, %2.<Vetype>"
+)
+
+;; Conversion of DI or SI to the same number of SFs, predicated with a PTRUE.
 (define_insn "*<optab><mode>v8sf2"
   [(set (match_operand:V8SF 0 "register_operand" "=w")
 	(unspec:V8SF
@@ -1988,6 +2325,7 @@
   "<su_optab>cvtf\t%0.s, %1/m, %2.<Vetype>"
 )
 
+;; Conversion of DI or SI to DF, predicated with a PTRUE.
 (define_insn "*<optab><mode>v4df2"
   [(set (match_operand:V4DF 0 "register_operand" "=w")
 	(unspec:V4DF
@@ -1999,30 +2337,35 @@
   "<su_optab>cvtf\t%0.d, %1/m, %2.<Vetype>"
 )
 
-(define_insn "*truncv4dfv8sf2"
-  [(set (match_operand:V8SF 0 "register_operand" "=w")
-	(unspec:V8SF
-	  [(match_operand:V4BI 1 "register_operand" "Upl")
-	   (unspec:V8SF
-	     [(match_operand:V4DF 2 "register_operand" "w")]
+;; Conversion of DFs to the same number of SFs, or SFs to the same number
+;; of HFs.
+(define_insn "*trunc<Vwide><mode>2"
+  [(set (match_operand:SVE_HSF 0 "register_operand" "=w")
+	(unspec:SVE_HSF
+	  [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl")
+	   (unspec:SVE_HSF
+	     [(match_operand:<VWIDE> 2 "register_operand" "w")]
 	     UNSPEC_FLOAT_CONVERT)]
 	  UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE"
-  "fcvt\t%0.s, %1/m, %2.d"
+  "fcvt\t%0.<Vetype>, %1/m, %2.<Vewtype>"
 )
 
-(define_insn "*extendv8sfv4df2"
-  [(set (match_operand:V4DF 0 "register_operand" "=w")
-	(unspec:V4DF
-	  [(match_operand:V4BI 1 "register_operand" "Upl")
-	   (unspec:V4DF
-	     [(match_operand:V8SF 2 "register_operand" "w")]
+;; Conversion of SFs to the same number of DFs, or HFs to the same number
+;; of SFs.
+(define_insn "*extend<mode><Vwide>2"
+  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
+	(unspec:<VWIDE>
+	  [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl")
+	   (unspec:<VWIDE>
+	     [(match_operand:SVE_HSF 2 "register_operand" "w")]
 	     UNSPEC_FLOAT_CONVERT)]
 	  UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE"
-  "fcvt\t%0.d, %1/m, %2.s"
+  "fcvt\t%0.<Vewtype>, %1/m, %2.<Vetype>"
 )
 
+;; PUNPKHI and PUNPKLO.
 (define_insn "vec_unpack<su>_<perm_hilo>_<mode>"
   [(set (match_operand:<VWIDE> 0 "register_operand" "=Upa")
 	(unspec:<VWIDE> [(match_operand:PRED_BHS 1 "register_operand" "Upa")]
@@ -2031,6 +2374,7 @@
   "punpk<perm_hilo>\t%0.h, %1.b"
 )
 
+;; SUNPKHI, UUNPKHI, SUNPKLO and UUNPKLO.
 (define_insn "vec_unpack<su>_<perm_hilo>_<SVE_BHSI:mode>"
   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
 	(unspec:<VWIDE> [(match_operand:SVE_BHSI 1 "register_operand" "w")]
@@ -2039,38 +2383,37 @@
   "<su>unpk<perm_hilo>\t%0.<Vewtype>, %1.<Vetype>"
 )
 
-;; Used by the vec_unpacks_<perm_hilo>_v8sf expander to unpack the bit
-;; representation of a V8SF without conversion.  The choice between signed
-;; and unsigned isn't significant.
-(define_insn "*vec_unpacku_<perm_hilo>_v8sf_no_convert"
-  [(set (match_operand:V8SF 0 "register_operand" "=w")
-	(unspec:V8SF [(match_operand:V8SF 1 "register_operand" "w")]
-		     UNPACK_UNSIGNED))]
+;; Used by the vec_unpacks_<perm_hilo>_<mode> expander to unpack the bit
+;; representation of a V8SF or V16HF without conversion.  The choice between
+;; signed and unsigned isn't significant.
+(define_insn "*vec_unpacku_<perm_hilo>_<mode>_no_convert"
+  [(set (match_operand:SVE_HSF 0 "register_operand" "=w")
+	(unspec:SVE_HSF [(match_operand:SVE_HSF 1 "register_operand" "w")]
+			UNPACK_UNSIGNED))]
   "TARGET_SVE"
-  "uunpk<perm_hilo>\t%0.d, %1.s"
+  "uunpk<perm_hilo>\t%0.<Vewtype>, %1.<Vetype>"
 )
 
-;; Float to double
-;; unpack from v8sf with no conversion
-;; then float convert the unpacked v8sf to v4df
-(define_expand "vec_unpacks_<perm_hilo>_v8sf"
+;; Unpack one half of a V8SF to V4DF, or one half of a V16HF to V8SF.
+;; First unpack the source without conversion, then float-convert the
+;; unpacked source.
+(define_expand "vec_unpacks_<perm_hilo>_<mode>"
   [(set (match_dup 2)
-	(unspec:V8SF [(match_operand:V8SF 1 "register_operand")]
-		     UNPACK_UNSIGNED))
-   (set (match_operand:V4DF 0 "register_operand")
-	(unspec:V4DF [(match_dup 3)
-		      (unspec:V4DF [(match_dup 2)] UNSPEC_FLOAT_CONVERT)]
-		     UNSPEC_MERGE_PTRUE))]
+	(unspec:SVE_HSF [(match_operand:SVE_HSF 1 "register_operand")]
+			UNPACK_UNSIGNED))
+   (set (match_operand:<VWIDE> 0 "register_operand")
+	(unspec:<VWIDE> [(match_dup 3)
+			 (unspec:<VWIDE> [(match_dup 2)] UNSPEC_FLOAT_CONVERT)]
+			UNSPEC_MERGE_PTRUE))]
   "TARGET_SVE"
   {
-    operands[2] = gen_reg_rtx (V8SFmode);
-    operands[3] = force_reg (V4BImode, CONSTM1_RTX (V4BImode));
+    operands[2] = gen_reg_rtx (<MODE>mode);
+    operands[3] = force_reg (<VWIDE_PRED>mode, CONSTM1_RTX (<VWIDE_PRED>mode));
   }
 )
 
-;; Int to double
-;; unpack from v8si to v4di
-;; then convert v8si to v4df
+;; Unpack one half of a V8SI to V4DF.  First unpack from V8SI to V4DI,
+;; reinterpret the V4DI as a V8SI, then convert the unpacked V8SI to V4DF.
 (define_expand "vec_unpack<su_optab>_float_<perm_hilo>_v8si"
   [(set (match_dup 2)
 	(unspec:V4DI [(match_operand:V8SI 1 "register_operand")]
@@ -2087,6 +2430,8 @@
   }
 )
 
+;; Predicate pack.  Use UZP1 on the narrower type, which discards
+;; the high part of each wide element.
 (define_insn "vec_pack_trunc_<Vwide>"
   [(set (match_operand:PRED_BHS 0 "register_operand" "=Upa")
 	(unspec:PRED_BHS
@@ -2097,6 +2442,8 @@
   "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
 )
 
+;; Integer pack.  Use UZP1 on the narrower type, which discards
+;; the high part of each wide element.
 (define_insn "vec_pack_trunc_<Vwide>"
   [(set (match_operand:SVE_BHSI 0 "register_operand" "=w")
 	(unspec:SVE_BHSI
@@ -2107,44 +2454,32 @@
   "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
 )
 
-(define_insn "*vec_pack_trunc_v4df_no_convert"
-  [(set (match_operand:V8SF 0 "register_operand" "=w")
-	(unspec:V8SF [(match_operand:V8SF 1 "register_operand" "w")
-		      (match_operand:V8SF 2 "register_operand" "w")]
-		     UNSPEC_PACK))]
-  "TARGET_SVE"
-  "uzp1\t%0.s, %1.s, %2.s"
-)
-
-;; Double to float
-;; float convert both inputs from v4df to v8sf
-;; then pack them together with no conversion
-(define_expand "vec_pack_trunc_v4df"
+;; Convert two vectors of DF to SF, or two vectors of SF to HF, and pack
+;; the results into a single vector.
+(define_expand "vec_pack_trunc_<Vwide>"
   [(set (match_dup 4)
-	(unspec:V8SF
+	(unspec:SVE_HSF
 	  [(match_dup 3)
-	   (unspec:V8SF [(match_operand:V4DF 1 "register_operand")]
-			UNSPEC_FLOAT_CONVERT)]
+	   (unspec:SVE_HSF [(match_operand:<VWIDE> 1 "register_operand")]
+			   UNSPEC_FLOAT_CONVERT)]
 	  UNSPEC_MERGE_PTRUE))
    (set (match_dup 5)
-	(unspec:V8SF
+	(unspec:SVE_HSF
 	  [(match_dup 3)
-	   (unspec:V8SF [(match_operand:V4DF 2 "register_operand")]
-			UNSPEC_FLOAT_CONVERT)]
+	   (unspec:SVE_HSF [(match_operand:<VWIDE> 2 "register_operand")]
+			   UNSPEC_FLOAT_CONVERT)]
 	  UNSPEC_MERGE_PTRUE))
-   (set (match_operand:V8SF 0 "register_operand")
-	(unspec:V8SF [(match_dup 4) (match_dup 5)] UNSPEC_PACK))]
+   (set (match_operand:SVE_HSF 0 "register_operand")
+	(unspec:SVE_HSF [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))]
   "TARGET_SVE"
   {
-    operands[3] = force_reg (V4BImode, CONSTM1_RTX (V4BImode));
-    operands[4] = gen_reg_rtx (V8SFmode);
-    operands[5] = gen_reg_rtx (V8SFmode);
+    operands[3] = force_reg (<VWIDE_PRED>mode, CONSTM1_RTX (<VWIDE_PRED>mode));
+    operands[4] = gen_reg_rtx (<MODE>mode);
+    operands[5] = gen_reg_rtx (<MODE>mode);
   }
 )
 
-;; Double to int
-;; float convert both inputs from v4df to v8si
-;; then pack v4di to v8si
+;; Convert two vectors of DF to SI and pack the results into a single vector.
 (define_expand "vec_pack_<su>fix_trunc_v4df"
   [(set (match_dup 4)
 	(unspec:V8SI
@@ -2157,14 +2492,12 @@
 	   (FIXUORS:V8SI (match_operand:V4DF 2 "register_operand"))]
 	  UNSPEC_MERGE_PTRUE))
    (set (match_operand:V8SI 0 "register_operand")
-	(unspec:V8SI [(match_dup 6) (match_dup 7)] UNSPEC_PACK))]
+	(unspec:V8SI [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))]
   "TARGET_SVE"
   {
     operands[3] = force_reg (V4BImode, CONSTM1_RTX (V4BImode));
     operands[4] = gen_reg_rtx (V8SImode);
     operands[5] = gen_reg_rtx (V8SImode);
-    operands[6] = gen_rtx_SUBREG (V4DImode, operands[4], 0);
-    operands[7] = gen_rtx_SUBREG (V4DImode, operands[5], 0);
   }
 )
 
@@ -2205,18 +2538,6 @@
   "<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>"
 )
 
-(define_insn "extract_last_<mode>"
-  [(set (match_operand:<VEL> 0 "register_operand" "=r, w")
-	(unspec:<VEL>
-	  [(match_operand:SVE_ALL 1 "register_operand" "w, w")
-	   (match_operand:<VPRED> 2 "register_operand" "Upl, Upl")]
-	  UNSPEC_LASTB))]
-  "TARGET_SVE"
-  "@
-   lastb\t%<vwcore>0, %2, %1.<Vetype>
-   lastb\t%<Vetype>0, %2, %1.<Vetype>"
-)
-
 (define_insn "read_nf<mode>"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(unspec:PRED_ALL [(reg:SI FFRT_REGNUM)] UNSPEC_READ_NF))
diff --git a/gcc/config/aarch64/aarch64-tune.md b/gcc/config/aarch64/aarch64-tune.md
index 7fcd6cb2c2e..7b3a7460561 100644
--- a/gcc/config/aarch64/aarch64-tune.md
+++ b/gcc/config/aarch64/aarch64-tune.md
@@ -1,5 +1,5 @@
 ;; -*- buffer-read-only: t -*-
 ;; Generated automatically by gentune.sh from aarch64-cores.def
 (define_attr "tune"
-	"cortexa35,cortexa53,cortexa57,cortexa72,cortexa73,thunderx,thunderxt88p1,thunderxt88,thunderxt81,thunderxt83,xgene1,falkor,qdf24xx,exynosm1,thunderx2t99p1,vulcan,thunderx2t99,cortexa55,cortexa75,cortexa57cortexa53,cortexa72cortexa53,cortexa73cortexa35,cortexa73cortexa53,cortexa75cortexa55"
+	"cortexa35,cortexa53,cortexa57,cortexa72,cortexa73,thunderx,thunderxt88p1,thunderxt88,thunderxt81,thunderxt83,xgene1,falkor,qdf24xx,exynosm1,thunderx2t99p1,vulcan,thunderx2t99,cortexa55,cortexa75,saphira,cortexa57cortexa53,cortexa72cortexa53,cortexa73cortexa35,cortexa73cortexa53,cortexa75cortexa55"
 	(const (symbol_ref "((enum attr_tune) aarch64_tune)")))
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index a1894a79dd7..b5a179784a6 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -880,6 +880,34 @@ static const struct tune_params qdf24xx_tunings =
   &qdf24xx_prefetch_tune
 };
 
+/* Tuning structure for the Qualcomm Saphira core.  Default to falkor values
+   for now.  */
+static const struct tune_params saphira_tunings =
+{
+  &generic_extra_costs,
+  &generic_addrcost_table,
+  &generic_regmove_cost,
+  &generic_vector_cost,
+  &generic_branch_cost,
+  &generic_approx_modes,
+  4, /* memmov_cost  */
+  4, /* issue_rate  */
+  (AARCH64_FUSE_MOV_MOVK | AARCH64_FUSE_ADRP_ADD
+   | AARCH64_FUSE_MOVK_MOVK), /* fuseable_ops  */
+  16,	/* function_align.  */
+  8,	/* jump_align.  */
+  16,	/* loop_align.  */
+  2,	/* int_reassoc_width.  */
+  4,	/* fp_reassoc_width.  */
+  1,	/* vec_reassoc_width.  */
+  2,	/* min_div_recip_mul_sf.  */
+  2,	/* min_div_recip_mul_df.  */
+  0,	/* max_case_values.  */
+  tune_params::AUTOPREFETCHER_WEAK,	/* autoprefetcher_model.  */
+  (AARCH64_EXTRA_TUNE_NONE),		/* tune_flags.  */
+  &generic_prefetch_tune
+};
+
 static const struct tune_params thunderx2t99_tunings =
 {
   &thunderx2t99_extra_costs,
@@ -1092,13 +1120,15 @@ aarch64_dbx_register_number (unsigned regno)
      return AARCH64_DWARF_V0 + regno - V0_REGNUM;
    else if (PR_REGNUM_P (regno))
      return AARCH64_DWARF_P0 + regno - P0_REGNUM;
+   else if (regno == VG_REGNUM)
+     return AARCH64_DWARF_VG;
 
    /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
       equivalent DWARF register.  */
    return DWARF_FRAME_REGISTERS;
 }
 
-/* Return true if MODE is any of the AdvSIMD structure modes.  */
+/* Return true if MODE is any of the Advanced SIMD structure modes.  */
 static bool
 aarch64_advsimd_struct_mode_p (machine_mode mode)
 {
@@ -1117,15 +1147,19 @@ aarch64_sve_pred_mode_p (machine_mode mode)
 	      || mode == V4BImode));
 }
 
-/* Return a set of flags describing the vector properties of mode MODE.
-   Ignore modes that are not supported by the current target.  */
+/* Three mutually-exclusive flags describing a vector or predicate type.  */
 const unsigned int VEC_ADVSIMD  = 1;
 const unsigned int VEC_SVE_DATA = 2;
 const unsigned int VEC_SVE_PRED = 4;
+/* Can be used in combination with VEC_ADVSIMD or VEC_SVE_DATA to indicate
+   a structure of 2, 3 or 4 vectors.  */
+const unsigned int VEC_STRUCT   = 8;
+/* Useful combinations of the above.  */
 const unsigned int VEC_ANY_SVE  = VEC_SVE_DATA | VEC_SVE_PRED;
 const unsigned int VEC_ANY_DATA = VEC_ADVSIMD | VEC_SVE_DATA;
-const unsigned int VEC_STRUCT   = 8;
 
+/* Return a set of flags describing the vector properties of mode MODE.
+   Ignore modes that are not supported by the current target.  */
 static unsigned int
 aarch64_classify_vector_mode (machine_mode mode)
 {
@@ -1145,7 +1179,7 @@ aarch64_classify_vector_mode (machine_mode mode)
 	  || inner == DImode
 	  || inner == DFmode))
     {
-      if (TARGET_SVE && inner != HFmode)
+      if (TARGET_SVE)
 	{
 	  if (must_eq (GET_MODE_BITSIZE (mode), BITS_PER_SVE_VECTOR))
 	    return VEC_SVE_DATA;
@@ -1207,14 +1241,14 @@ aarch64_array_mode_supported_p (machine_mode mode,
   return false;
 }
 
-/* Implement TARGET_VECTORIZE_GET_MASK_MODE.  */
+/* Return the SVE predicate mode to use for elements that have
+   ELEM_NBYTES bytes, if such a mode exists.  */
 
-static opt_machine_mode
-aarch64_get_mask_mode (poly_uint64 nunits, poly_uint64 nbytes)
+opt_machine_mode
+aarch64_sve_pred_mode (unsigned int elem_nbytes)
 {
-  if (TARGET_SVE && must_eq (nbytes, BYTES_PER_SVE_VECTOR))
+  if (TARGET_SVE)
     {
-      unsigned int elem_nbytes = vector_element_size (nbytes, nunits);
       if (elem_nbytes == 1)
 	return V32BImode;
       if (elem_nbytes == 2)
@@ -1224,6 +1258,21 @@ aarch64_get_mask_mode (poly_uint64 nunits, poly_uint64 nbytes)
       if (elem_nbytes == 8)
 	return V4BImode;
     }
+  return opt_machine_mode ();
+}
+
+/* Implement TARGET_VECTORIZE_GET_MASK_MODE.  */
+
+static opt_machine_mode
+aarch64_get_mask_mode (poly_uint64 nunits, poly_uint64 nbytes)
+{
+  if (TARGET_SVE && must_eq (nbytes, BYTES_PER_SVE_VECTOR))
+    {
+      unsigned int elem_nbytes = vector_element_size (nbytes, nunits);
+      machine_mode pred_mode;
+      if (aarch64_sve_pred_mode (elem_nbytes).exists (&pred_mode))
+	return pred_mode;
+    }
 
   return default_get_mask_mode (nunits, nbytes);
 }
@@ -1233,7 +1282,11 @@ aarch64_get_mask_mode (poly_uint64 nunits, poly_uint64 nbytes)
 static unsigned int
 aarch64_hard_regno_nregs (unsigned regno, machine_mode mode)
 {
-  HOST_WIDE_INT size;
+  /* ??? Logically we should only need to provide a value when
+     HARD_REGNO_MODE_OK says that the combination is valid,
+     but at the moment we need to handle all modes.  Just ignore
+     any runtime parts for registers that can't store them.  */
+  HOST_WIDE_INT lowest_size = constant_lower_bound (GET_MODE_SIZE (mode));
   switch (aarch64_regno_regclass (regno))
     {
     case FP_REGS:
@@ -1241,19 +1294,13 @@ aarch64_hard_regno_nregs (unsigned regno, machine_mode mode)
       if (aarch64_sve_data_mode_p (mode))
 	return exact_div (GET_MODE_SIZE (mode),
 			  BYTES_PER_SVE_VECTOR).to_constant ();
-      /* ??? In the rest of this function it would probably make
-	 sense to assert for invalid modes, but it's likely that there
-	 are still callers to this function that don't check
-	 HARD_REGNO_MODE_OK first.  */
-      size = constant_lower_bound (GET_MODE_SIZE (mode));
-      return CEIL (size, UNITS_PER_VREG);
+      return CEIL (lowest_size, UNITS_PER_VREG);
     case PR_REGS:
     case PR_LO_REGS:
     case PR_HI_REGS:
       return 1;
     default:
-      size = constant_lower_bound (GET_MODE_SIZE (mode));
-      return CEIL (size, UNITS_PER_WORD);
+      return CEIL (lowest_size, UNITS_PER_WORD);
     }
   gcc_unreachable ();
 }
@@ -1266,6 +1313,10 @@ aarch64_hard_regno_mode_ok (unsigned regno, machine_mode mode)
   if (GET_MODE_CLASS (mode) == MODE_CC)
     return regno == CC_REGNUM;
 
+  if (regno == VG_REGNUM)
+    /* This is expected to have the same size as _Unwind_Word.  */
+    return mode == DImode;
+
   unsigned int vec_flags = aarch64_classify_vector_mode (mode);
   if (vec_flags & VEC_SVE_PRED)
     return PR_REGNUM_P (regno);
@@ -1325,26 +1376,19 @@ aarch64_regmode_natural_size (machine_mode mode)
 
 /* Implement HARD_REGNO_CALLER_SAVE_MODE.  */
 machine_mode
-aarch64_hard_regno_caller_save_mode (unsigned regno, unsigned nregs,
+aarch64_hard_regno_caller_save_mode (unsigned regno, unsigned,
 				     machine_mode mode)
 {
-  /* Handle modes that fit within single registers.  */
-  if (GP_REGNUM_P (regno) || FP_REGNUM_P (regno))
-    {
-      if (must_ge (GET_MODE_SIZE (mode), 4))
-	return mode;
-      else
-	return SImode;
-    }
   /* The predicate mode determines which bits are significant and
      which are "don't care".  Decreasing the number of lanes would
      lose data while increasing the number of lanes would make bits
      unnecessarily significant.  */
-  else if (PR_REGNUM_P (regno))
+  if (PR_REGNUM_P (regno))
+    return mode;
+  if (must_ge (GET_MODE_SIZE (mode), 4))
     return mode;
-  /* Fall back to generic for multi-reg and very large modes.  */
   else
-    return choose_hard_reg_mode (regno, nregs, false);
+    return SImode;
 }
 
 /* Implement TARGET_CONSTANT_ALIGNMENT.  Make strings word-aligned so
@@ -2053,9 +2097,9 @@ aarch64_sve_cnt_immediate_p (rtx x)
    operand (a vector pattern followed by a multiplier in the range [1, 16]).
    PREFIX is the mnemonic without the size suffix and OPERANDS is the
    first part of the operands template (the part that comes before the
-   vector size itself).  FACTOR is the multiple of VQ that is needed.
-   NELTS_PER_VQ, if nonzero, is the number of elements in each vector
-   quadword.  If it is zero, we can use any element size.  */
+   vector size itself).  FACTOR is the number of quadwords.
+   NELTS_PER_VQ, if nonzero, is the number of elements in each quadword.
+   If it is zero, we can use any element size.  */
 
 static char *
 aarch64_output_sve_cnt_immediate (const char *prefix, const char *operands,
@@ -2070,7 +2114,7 @@ aarch64_output_sve_cnt_immediate (const char *prefix, const char *operands,
        multiplier is 1 whereever possible.  */
     nelts_per_vq = factor & -factor;
   int shift = std::min (exact_log2 (nelts_per_vq), 4);
-  gcc_assert (shift > 0);
+  gcc_assert (IN_RANGE (shift, 1, 4));
   char suffix = "dwhb"[shift - 1];
 
   factor >>= shift;
@@ -2133,15 +2177,20 @@ aarch64_sve_addvl_addpl_immediate_p (rtx x)
 char *
 aarch64_output_sve_addvl_addpl (rtx dest, rtx base, rtx offset)
 {
-  static char buffer[sizeof ("addpl\t%x0, %x1, #-32")];
+  static char buffer[sizeof ("addpl\t%x0, %x1, #-") + 3 * sizeof (int)];
   poly_int64 offset_value = rtx_to_poly_int64 (offset);
   gcc_assert (aarch64_sve_addvl_addpl_immediate_p (offset_value));
 
-  /* Use INC if possible.  */
-  if (rtx_equal_p (dest, base)
-      && GP_REGNUM_P (REGNO (dest))
-      && aarch64_sve_cnt_immediate_p (offset_value))
-    return aarch64_output_sve_cnt_immediate ("inc", "%x0", offset);
+  /* Use INC or DEC if possible.  */
+  if (rtx_equal_p (dest, base) && GP_REGNUM_P (REGNO (dest)))
+    {
+      if (aarch64_sve_cnt_immediate_p (offset_value))
+	return aarch64_output_sve_cnt_immediate ("inc", "%x0",
+						 offset_value.coeffs[1], 0);
+      if (aarch64_sve_cnt_immediate_p (-offset_value))
+	return aarch64_output_sve_cnt_immediate ("dec", "%x0",
+						 -offset_value.coeffs[1], 0);
+    }
 
   int factor = offset_value.coeffs[1];
   if ((factor & 15) == 0)
@@ -2340,39 +2389,30 @@ aarch64_internal_mov_immediate (rtx dest, rtx imm, bool generate,
   return num_insns;
 }
 
-/* Return the number of temporary registers that aarch64_add_constant
+/* Return the number of temporary registers that aarch64_add_offset_1
    would need to add OFFSET to a register.  */
 
 static unsigned int
-aarch64_add_constant_temporaries (HOST_WIDE_INT offset)
+aarch64_add_offset_1_temporaries (HOST_WIDE_INT offset)
 {
   return abs_hwi (offset) < 0x1000000 ? 0 : 1;
 }
 
-/* Set DEST to REG + DELTA.  MODE is the mode of the addition.
-   FRAME_RELATED_P should be true if the RTX_FRAME_RELATED flag should
-   be set and CFA adjustments added to the generated instructions.
-
-   TEMP1, if nonnull, is a register of mode MODE that can be used as a
-   temporary if register allocation is already complete.  This temporary
-   register may overlap DEST but must not overlap REG.  If TEMP1 is known
-   to hold abs (DELTA), EMIT_MOVE_IMM can be set to false to avoid emitting
-   the immediate again.
-
-   Since this function may be used to adjust the stack pointer, we must
-   ensure that it cannot cause transient stack deallocation (for example
-   by first incrementing SP and then decrementing when adjusting by a
-   large immediate).  */
+/* A subroutine of aarch64_add_offset that handles the case in which
+   OFFSET is known at compile time.  The arguments are otherwise the same.  */
 
 static void
-aarch64_add_constant_internal (scalar_int_mode mode, rtx dest, rtx temp1,
-			       rtx src, HOST_WIDE_INT delta,
-			       bool frame_related_p, bool emit_move_imm)
+aarch64_add_offset_1 (scalar_int_mode mode, rtx dest,
+		      rtx src, HOST_WIDE_INT offset, rtx temp1,
+		      bool frame_related_p, bool emit_move_imm)
 {
-  HOST_WIDE_INT mdelta = abs_hwi (delta);
+  gcc_assert (emit_move_imm || temp1 != NULL_RTX);
+  gcc_assert (temp1 == NULL_RTX || !reg_overlap_mentioned_p (temp1, src));
+
+  HOST_WIDE_INT moffset = abs_hwi (offset);
   rtx_insn *insn;
 
-  if (!mdelta)
+  if (!moffset)
     {
       if (!rtx_equal_p (dest, src))
 	{
@@ -2383,48 +2423,49 @@ aarch64_add_constant_internal (scalar_int_mode mode, rtx dest, rtx temp1,
     }
 
   /* Single instruction adjustment.  */
-  if (aarch64_uimm12_shift (mdelta))
+  if (aarch64_uimm12_shift (moffset))
     {
-      insn = emit_insn (gen_add3_insn (dest, src, GEN_INT (delta)));
+      insn = emit_insn (gen_add3_insn (dest, src, GEN_INT (offset)));
       RTX_FRAME_RELATED_P (insn) = frame_related_p;
       return;
     }
 
-  /* Emit 2 additions/subtractions if the adjustment is less than 24 bits.
-     Only do this if mdelta is not a 16-bit move as adjusting using a move
-     is better.  */
-  if (mdelta < 0x1000000 && (!temp1 || !aarch64_move_imm (mdelta, mode)))
+  /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
+     and either:
+
+     a) the offset cannot be loaded by a 16-bit move or
+     b) there is no spare register into which we can move it.  */
+  if (moffset < 0x1000000
+      && ((!temp1 && !can_create_pseudo_p ())
+	  || !aarch64_move_imm (moffset, mode)))
     {
-      HOST_WIDE_INT low_off = mdelta & 0xfff;
+      HOST_WIDE_INT low_off = moffset & 0xfff;
 
-      low_off = delta < 0 ? -low_off : low_off;
+      low_off = offset < 0 ? -low_off : low_off;
       insn = emit_insn (gen_add3_insn (dest, src, GEN_INT (low_off)));
       RTX_FRAME_RELATED_P (insn) = frame_related_p;
-      insn = emit_insn (gen_add2_insn (dest, GEN_INT (delta - low_off)));
+      insn = emit_insn (gen_add2_insn (dest, GEN_INT (offset - low_off)));
       RTX_FRAME_RELATED_P (insn) = frame_related_p;
       return;
     }
 
   /* Emit a move immediate if required and an addition/subtraction.  */
-  if (!temp1 || emit_move_imm)
-    temp1 = aarch64_force_temporary (mode, temp1, GEN_INT (mdelta));
-  insn = emit_insn (delta < 0 ? gen_sub3_insn (dest, src, temp1)
-			      : gen_add3_insn (dest, src, temp1));
+  if (emit_move_imm)
+    {
+      gcc_assert (temp1 != NULL_RTX || can_create_pseudo_p ());
+      temp1 = aarch64_force_temporary (mode, temp1, GEN_INT (moffset));
+    }
+  insn = emit_insn (offset < 0
+		    ? gen_sub3_insn (dest, src, temp1)
+		    : gen_add3_insn (dest, src, temp1));
   if (frame_related_p)
     {
       RTX_FRAME_RELATED_P (insn) = frame_related_p;
-      rtx adj = plus_constant (mode, src, delta);
+      rtx adj = plus_constant (mode, src, offset);
       add_reg_note (insn, REG_CFA_ADJUST_CFA, gen_rtx_SET (dest, adj));
     }
 }
 
-static inline void
-aarch64_add_constant (scalar_int_mode mode, rtx reg, rtx temp1,
-		      HOST_WIDE_INT delta)
-{
-  aarch64_add_constant_internal (mode, reg, temp1, reg, delta, false, true);
-}
-
 /* Return the number of temporary registers that aarch64_add_offset
    would need to move OFFSET into a register or add OFFSET to a register;
    ADD_P is true if we want the latter rather than the former.  */
@@ -2455,16 +2496,29 @@ aarch64_offset_temporaries (bool add_p, poly_int64 offset)
 	 be shifted).  */
       count += 1;
     }
-  return count + aarch64_add_constant_temporaries (constant);
+  return count + aarch64_add_offset_1_temporaries (constant);
 }
 
-/* Set DEST to REG + OFFSET.  MODE is the mode of the addition.
-   FRAME_RELATED_P should be true if the RTX_FRAME_RELATED flag should
+/* If X can be represented as a poly_int64, return the number
+   of temporaries that are required to add it to a register.
+   Return -1 otherwise.  */
+
+int
+aarch64_add_offset_temporaries (rtx x)
+{
+  poly_int64 offset;
+  if (!poly_int_rtx_p (x, &offset))
+    return -1;
+  return aarch64_offset_temporaries (true, offset);
+}
+
+/* Set DEST to SRC + OFFSET.  MODE is the mode of the addition.
+   FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
    be set and CFA adjustments added to the generated instructions.
 
    TEMP1, if nonnull, is a register of mode MODE that can be used as a
    temporary if register allocation is already complete.  This temporary
-   register may overlap DEST if !FRAME_RELATED_P but must not overlap REG.
+   register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
    If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
    false to avoid emitting the immediate again.
 
@@ -2477,20 +2531,22 @@ aarch64_offset_temporaries (bool add_p, poly_int64 offset)
    large immediate).  */
 
 static void
-aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx temp1, rtx temp2,
-		    rtx reg, poly_int64 offset, bool frame_related_p,
-		    bool emit_move_imm = true)
+aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx src,
+		    poly_int64 offset, rtx temp1, rtx temp2,
+		    bool frame_related_p, bool emit_move_imm = true)
 {
-  if (temp1 && frame_related_p)
-    gcc_checking_assert (!reg_overlap_mentioned_p (dest, temp1));
-  if (temp2)
-    gcc_checking_assert (!reg_overlap_mentioned_p (dest, temp2));
+  gcc_assert (emit_move_imm || temp1 != NULL_RTX);
+  gcc_assert (temp1 == NULL_RTX || !reg_overlap_mentioned_p (temp1, src));
+  gcc_assert (temp1 == NULL_RTX
+	      || !frame_related_p
+	      || !reg_overlap_mentioned_p (temp1, dest));
+  gcc_assert (temp2 == NULL_RTX || !reg_overlap_mentioned_p (dest, temp2));
 
   /* Try using ADDVL or ADDPL to add the whole value.  */
-  if (reg != const0_rtx && aarch64_sve_addvl_addpl_immediate_p (offset))
+  if (src != const0_rtx && aarch64_sve_addvl_addpl_immediate_p (offset))
     {
       rtx offset_rtx = gen_int_mode (offset, mode);
-      rtx_insn *insn = emit_insn (gen_add3_insn (dest, reg, offset_rtx));
+      rtx_insn *insn = emit_insn (gen_add3_insn (dest, src, offset_rtx));
       RTX_FRAME_RELATED_P (insn) = frame_related_p;
       return;
     }
@@ -2504,22 +2560,22 @@ aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx temp1, rtx temp2,
 
   /* Try using ADDVL or ADDPL to add the VG-based part.  */
   poly_int64 poly_offset (factor, factor);
-  if (reg != const0_rtx
+  if (src != const0_rtx
       && aarch64_sve_addvl_addpl_immediate_p (poly_offset))
     {
       rtx offset_rtx = gen_int_mode (poly_offset, mode);
-      rtx addr = gen_rtx_PLUS (mode, reg, offset_rtx);
       if (frame_related_p)
 	{
-	  rtx_insn *insn = emit_insn (gen_rtx_SET (dest, addr));
+	  rtx_insn *insn = emit_insn (gen_add3_insn (dest, src, offset_rtx));
 	  RTX_FRAME_RELATED_P (insn) = true;
-	  reg = dest;
+	  src = dest;
 	}
       else
 	{
-	  reg = aarch64_force_temporary (mode, temp1, addr);
+	  rtx addr = gen_rtx_PLUS (mode, src, offset_rtx);
+	  src = aarch64_force_temporary (mode, temp1, addr);
 	  temp1 = temp2;
-	  temp2 = 0;
+	  temp2 = NULL_RTX;
 	}
     }
   /* Otherwise use a CNT-based sequence.  */
@@ -2533,10 +2589,9 @@ aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx temp1, rtx temp2,
 	  code = MINUS;
 	}
 
-      /* Calculate CNTD * FACTOR / 2.  */
+      /* Calculate CNTD * FACTOR / 2.  First try to fold the division
+	 into the multiplication.  */
       rtx val;
-
-      /* First try to fold the division into the multiplication.  */
       int shift = 0;
       if (factor & 1)
 	/* Use a right shift by 1.  */
@@ -2548,9 +2603,10 @@ aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx temp1, rtx temp2,
 	{
 	  if (factor > 16 * 8)
 	    {
-	      /* Use "CNTB Xn, ALL, MUL #NEW_FACTOR", then shift the
-		 result into position.  */
-	      int extra_shift = exact_log2 (low_bit) - 3;
+	      /* "CNTB Xn, ALL, MUL #FACTOR" is out of range, so calculate
+		 the value with the minimum multiplier and shift it into
+		 position.  */
+	      int extra_shift = exact_log2 (low_bit);
 	      shift += extra_shift;
 	      factor >>= extra_shift;
 	    }
@@ -2564,7 +2620,7 @@ aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx temp1, rtx temp2,
 
 	  /* Go back to using a negative multiplication factor if we have
 	     no register from which to subtract.  */
-	  if (code == MINUS && reg == const0_rtx)
+	  if (code == MINUS && src == const0_rtx)
 	    {
 	      factor = -factor;
 	      code = PLUS;
@@ -2587,11 +2643,11 @@ aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx temp1, rtx temp2,
 	  val = gen_rtx_ASHIFTRT (mode, val, const1_rtx);
 	}
 
-      /* Calculate REG +/- VL*FACTOR.  */
-      if (reg != const0_rtx)
+      /* Calculate SRC +/- CNTD * FACTOR / 2.  */
+      if (src != const0_rtx)
 	{
 	  val = aarch64_force_temporary (mode, temp1, val);
-	  val = gen_rtx_fmt_ee (code, mode, reg, val);
+	  val = gen_rtx_fmt_ee (code, mode, src, val);
 	}
       else if (code == MINUS)
 	{
@@ -2606,62 +2662,58 @@ aarch64_add_offset (scalar_int_mode mode, rtx dest, rtx temp1, rtx temp2,
 	    {
 	      RTX_FRAME_RELATED_P (insn) = true;
 	      add_reg_note (insn, REG_CFA_ADJUST_CFA,
-			    gen_rtx_SET (dest, plus_constant (Pmode, reg,
+			    gen_rtx_SET (dest, plus_constant (Pmode, src,
 							      poly_offset)));
 	    }
-	  reg = dest;
+	  src = dest;
 	  if (constant == 0)
 	    return;
 	}
       else
 	{
-	  reg = aarch64_force_temporary (mode, temp1, val);
+	  src = aarch64_force_temporary (mode, temp1, val);
 	  temp1 = temp2;
-	  temp2 = 0;
+	  temp2 = NULL_RTX;
 	}
 
       emit_move_imm = true;
     }
 
-  aarch64_add_constant_internal (mode, dest, temp1, reg, constant,
-				 frame_related_p, emit_move_imm);
+  aarch64_add_offset_1 (mode, dest, src, constant, temp1,
+			frame_related_p, emit_move_imm);
 }
 
-static inline void
-aarch64_add_sp (rtx temp1, rtx temp2, poly_int64 delta, bool emit_move_imm)
-{
-  aarch64_add_offset (Pmode, stack_pointer_rtx, temp1, temp2,
-		      stack_pointer_rtx, delta, true, emit_move_imm);
-}
+/* Like aarch64_add_offset, but the offset is given as an rtx rather
+   than a poly_int64.  */
 
-static inline void
-aarch64_sub_sp (rtx temp1, rtx temp2, poly_int64 delta, bool frame_related_p)
+void
+aarch64_split_add_offset (scalar_int_mode mode, rtx dest, rtx src,
+			  rtx offset_rtx, rtx temp1, rtx temp2)
 {
-  aarch64_add_offset (Pmode, stack_pointer_rtx, temp1, temp2,
-		      stack_pointer_rtx, -delta, frame_related_p);
+  aarch64_add_offset (mode, dest, src, rtx_to_poly_int64 (offset_rtx),
+		      temp1, temp2, false);
 }
 
-/* If X is a polynomial constant, return the number of temporaries that
-   are required to add it to a register.  Return -1 otherwise.  */
+/* Add DELTA to the stack pointer, marking the instructions frame-related.
+   TEMP1 is available as a temporary if nonnull.  EMIT_MOVE_IMM is false
+   if TEMP1 already contains abs (DELTA).  */
 
-int
-aarch64_add_offset_temporaries (rtx x)
+static inline void
+aarch64_add_sp (rtx temp1, rtx temp2, poly_int64 delta, bool emit_move_imm)
 {
-  poly_int64 offset;
-  if (!poly_int_rtx_p (x, &offset))
-    return -1;
-  return aarch64_offset_temporaries (true, offset);
+  aarch64_add_offset (Pmode, stack_pointer_rtx, stack_pointer_rtx, delta,
+		      temp1, temp2, true, emit_move_imm);
 }
 
-/* Like aarch64_add_offset, but the offset is given as an rtx rather
-   than a poly_int64.  */
+/* Subtract DELTA from the stack pointer, marking the instructions
+   frame-related if FRAME_RELATED_P.  TEMP1 is available as a temporary
+   if nonnull.  */
 
-void
-aarch64_split_add_offset (scalar_int_mode mode, rtx dest, rtx temp1,
-			  rtx temp2, rtx reg, rtx offset_rtx)
+static inline void
+aarch64_sub_sp (rtx temp1, rtx temp2, poly_int64 delta, bool frame_related_p)
 {
-  aarch64_add_offset (mode, dest, temp1, temp2, reg,
-		      rtx_to_poly_int64 (offset_rtx), false);
+  aarch64_add_offset (Pmode, stack_pointer_rtx, stack_pointer_rtx, -delta,
+		      temp1, temp2, frame_related_p);
 }
 
 /* Set DEST to (vec_series BASE STEP).  */
@@ -2672,11 +2724,7 @@ aarch64_expand_vec_series (rtx dest, rtx base, rtx step)
   machine_mode mode = GET_MODE (dest);
   scalar_mode inner = GET_MODE_INNER (mode);
 
-  /* At this point we have to decide which variant of the index insn to use:
-       1. index imm, reg
-       2. index reg, imm
-       3. index reg, reg.  */
-
+  /* Each operand can be a register or an immediate in the range [-16, 15].  */
   if (!aarch64_sve_index_immediate_p (base))
     base = force_reg (inner, base);
   if (!aarch64_sve_index_immediate_p (step))
@@ -2724,32 +2772,25 @@ aarch64_expand_mov_immediate (rtx dest, rtx imm,
 		 than that.  */
 	      if (partial_subreg_p (int_mode, SImode))
 		{
+		  /* We shouldn't be doing symbol calculations in modes
+		     narrower than SImode.  */
+		  gcc_assert (base == const0_rtx);
 		  dest = gen_lowpart (SImode, dest);
 		  int_mode = SImode;
 		}
 	      if (base != const0_rtx)
 		{
 		  base = aarch64_force_temporary (int_mode, dest, base);
-		  aarch64_add_offset (int_mode, dest, NULL_RTX, NULL_RTX,
-				      base, offset, false);
+		  aarch64_add_offset (int_mode, dest, base, offset,
+				      NULL_RTX, NULL_RTX, false);
 		}
 	      else
-		aarch64_add_offset (int_mode, dest, dest, NULL_RTX,
-				    base, offset, false);
+		aarch64_add_offset (int_mode, dest, base, offset,
+				    dest, NULL_RTX, false);
 	    }
 	  return;
 	}
 
-      /* Cope with complex (const ...) expressions involving VL.  */
-      if (GET_CODE (base) != SYMBOL_REF
-	  && GET_CODE (base) != LABEL_REF)
-	{
-	  base = force_reg (int_mode, base);
-	  aarch64_add_offset (int_mode, dest, dest, NULL_RTX,
-			      base, offset, false);
-	  return;
-	}
-
       sty = aarch64_classify_symbol (base, const_offset);
       switch (sty)
 	{
@@ -2759,8 +2800,8 @@ aarch64_expand_mov_immediate (rtx dest, rtx imm,
 	    {
 	      gcc_assert (can_create_pseudo_p ());
 	      base = aarch64_force_temporary (int_mode, dest, base);
-	      aarch64_add_offset (int_mode, dest, NULL_RTX, NULL_RTX,
-				  base, const_offset, false);
+	      aarch64_add_offset (int_mode, dest, base, const_offset,
+				  NULL_RTX, NULL_RTX, false);
 	      return;
 	    }
 
@@ -2799,8 +2840,8 @@ aarch64_expand_mov_immediate (rtx dest, rtx imm,
 	    {
 	      gcc_assert(can_create_pseudo_p ());
 	      base = aarch64_force_temporary (int_mode, dest, base);
-	      aarch64_add_offset (int_mode, dest, NULL_RTX, NULL_RTX,
-				  base, const_offset, false);
+	      aarch64_add_offset (int_mode, dest, base, const_offset,
+				  NULL_RTX, NULL_RTX, false);
 	      return;
 	    }
 	  /* FALLTHRU */
@@ -2847,7 +2888,7 @@ aarch64_expand_mov_immediate (rtx dest, rtx imm,
 	{
 	  rtx mem = force_const_mem (mode, imm);
 	  gcc_assert (mem);
-	  emit_insn (gen_rtx_SET (dest, mem));
+	  emit_move_insn (dest, mem);
 	}
 
       return;
@@ -2857,31 +2898,39 @@ aarch64_expand_mov_immediate (rtx dest, rtx imm,
 				  as_a <scalar_int_mode> (mode));
 }
 
+/* Emit an SVE predicated move from SRC to DEST.  PRED is a predicate
+   that is known to contain PTRUE.  */
+
+void
+aarch64_emit_sve_pred_move (rtx dest, rtx pred, rtx src)
+{
+  emit_insn (gen_rtx_SET (dest, gen_rtx_UNSPEC (GET_MODE (dest),
+						gen_rtvec (2, pred, src),
+						UNSPEC_MERGE_PTRUE)));
+}
+
 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
    operand is in memory.  In this case we need to use the predicated LD1
    and ST1 instead of LDR and STR, both for correctness on big-endian
    targets and because LD1 and ST1 support a wider range of addressing modes.
-   PRED_MODE is the mode of the predicate and GEN_PRED_MOVE is the
-   generator for the predicated move pattern.  */
+   PRED_MODE is the mode of the predicate that should be used.  */
 
 void
-aarch64_expand_sve_mem_move (rtx dest, rtx src, machine_mode pred_mode,
-			     rtx (*gen_pred_move) (rtx, rtx, rtx))
+aarch64_expand_sve_mem_move (rtx dest, rtx src, machine_mode pred_mode)
 {
   machine_mode mode = GET_MODE (dest);
   rtx ptrue = force_reg (pred_mode, CONSTM1_RTX (pred_mode));
-  if (register_operand (src, mode)
-      || register_operand (dest, mode))
-    emit_insn (gen_pred_move (dest, ptrue, src));
-  else
+  if (!register_operand (src, mode)
+      && !register_operand (dest, mode))
     {
       rtx tmp = gen_reg_rtx (mode);
       if (MEM_P (src))
-	emit_insn (gen_pred_move (tmp, ptrue, src));
+	aarch64_emit_sve_pred_move (tmp, ptrue, src);
       else
 	emit_move_insn (tmp, src);
-      emit_insn (gen_pred_move (dest, ptrue, tmp));
+      src = tmp;
     }
+  aarch64_emit_sve_pred_move (dest, ptrue, src);
 }
 
 static bool
@@ -2908,7 +2957,8 @@ aarch64_pass_by_reference (cumulative_args_t pcum ATTRIBUTE_UNUSED,
   if (mode == BLKmode && type)
     size = int_size_in_bytes (type);
   else
-    /* We don't support passing and returning SVE types.  */
+    /* No frontends can create types with variable-sized modes, so we
+       shouldn't be asked to pass or return them.  */
     size = GET_MODE_SIZE (mode).to_constant ();
 
   /* Aggregates are passed by reference based on their size.  */
@@ -3137,7 +3187,8 @@ aarch64_layout_arg (cumulative_args_t pcum_v, machine_mode mode,
   if (type)
     size = int_size_in_bytes (type);
   else
-    /* We don't support passing and returning SVE types.  */
+    /* No frontends can create types with variable-sized modes, so we
+       shouldn't be asked to pass or return them.  */
     size = GET_MODE_SIZE (mode).to_constant ();
   size = ROUND_UP (size, UNITS_PER_WORD);
 
@@ -3422,7 +3473,8 @@ aarch64_pad_reg_upward (machine_mode mode, const_tree type,
       if (type)
 	size = int_size_in_bytes (type);
       else
-	/* We don't support passing and returning SVE types.  */
+	/* No frontends can create types with variable-sized modes, so we
+	   shouldn't be asked to pass or return them.  */
 	size = GET_MODE_SIZE (mode).to_constant ();
       if (size < 2 * UNITS_PER_WORD)
 	return true;
@@ -3452,12 +3504,19 @@ aarch64_libgcc_cmp_return_mode (void)
 #define PROBE_STACK_FIRST_REG  9
 #define PROBE_STACK_SECOND_REG 10
 
-/* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
+/* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
    inclusive.  These are offsets from the current stack pointer.  */
 
 static void
-aarch64_emit_probe_stack_range (HOST_WIDE_INT first, poly_int64 size)
+aarch64_emit_probe_stack_range (HOST_WIDE_INT first, poly_int64 poly_size)
 {
+  HOST_WIDE_INT size;
+  if (!poly_size.is_constant (&size))
+    {
+      sorry ("stack probes for SVE frames");
+      return;
+    }
+
   rtx reg1 = gen_rtx_REG (Pmode, PROBE_STACK_FIRST_REG);
 
   /* See the same assertion on PROBE_INTERVAL above.  */
@@ -3465,20 +3524,19 @@ aarch64_emit_probe_stack_range (HOST_WIDE_INT first, poly_int64 size)
 
   /* See if we have a constant small number of probes to generate.  If so,
      that's the easy case.  */
-  HOST_WIDE_INT const_size;
-  if (size.is_constant (&const_size) && const_size <= PROBE_INTERVAL)
+  if (size <= PROBE_INTERVAL)
     {
-      const HOST_WIDE_INT base = ROUND_UP (const_size, ARITH_FACTOR);
+      const HOST_WIDE_INT base = ROUND_UP (size, ARITH_FACTOR);
 
       emit_set_insn (reg1,
 		     plus_constant (Pmode,
 				    stack_pointer_rtx, -(first + base)));
-      emit_stack_probe (plus_constant (Pmode, reg1, base - const_size));
+      emit_stack_probe (plus_constant (Pmode, reg1, base - size));
     }
 
   /* The run-time loop is made up of 8 insns in the generic case while the
      compile-time loop is made up of 4+2*(n-2) insns for n # of intervals.  */
-  else if (size.is_constant (&const_size) && const_size <= 4 * PROBE_INTERVAL)
+  else if (size <= 4 * PROBE_INTERVAL)
     {
       HOST_WIDE_INT i, rem;
 
@@ -3491,14 +3549,14 @@ aarch64_emit_probe_stack_range (HOST_WIDE_INT first, poly_int64 size)
       /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
 	 it exceeds SIZE.  If only two probes are needed, this will not
 	 generate any code.  Then probe at FIRST + SIZE.  */
-      for (i = 2 * PROBE_INTERVAL; i < const_size; i += PROBE_INTERVAL)
+      for (i = 2 * PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
 	{
 	  emit_set_insn (reg1,
 			 plus_constant (Pmode, reg1, -PROBE_INTERVAL));
 	  emit_stack_probe (reg1);
 	}
 
-      rem = const_size - (i - PROBE_INTERVAL);
+      rem = size - (i - PROBE_INTERVAL);
       if (rem > 256)
 	{
 	  const HOST_WIDE_INT base = ROUND_UP (rem, ARITH_FACTOR);
@@ -3521,12 +3579,8 @@ aarch64_emit_probe_stack_range (HOST_WIDE_INT first, poly_int64 size)
 
       /* Step 1: round SIZE to the previous multiple of the interval.  */
 
-      rtx rounded_size;
-      if (size.is_constant (&const_size))
-	rounded_size = GEN_INT (const_size & -PROBE_INTERVAL);
-      else
-	/* Pending SVE support.  */
-	gcc_unreachable ();
+      HOST_WIDE_INT rounded_size = size & -PROBE_INTERVAL;
+
 
       /* Step 2: compute initial and final value of the loop counter.  */
 
@@ -3535,24 +3589,18 @@ aarch64_emit_probe_stack_range (HOST_WIDE_INT first, poly_int64 size)
 		     plus_constant (Pmode, stack_pointer_rtx, -first));
 
       /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE.  */
-      HOST_WIDE_INT adjustment = -first;
-      if (CONST_INT_P (rounded_size))
-	adjustment -= INTVAL (rounded_size);
-      if (!aarch64_uimm12_shift (adjustment))
+      HOST_WIDE_INT adjustment = - (first + rounded_size);
+      if (! aarch64_uimm12_shift (adjustment))
 	{
-	  aarch64_internal_mov_immediate (reg2,
-					  gen_int_mode (adjustment, Pmode),
+	  aarch64_internal_mov_immediate (reg2, GEN_INT (adjustment),
 					  true, Pmode);
 	  emit_set_insn (reg2, gen_rtx_PLUS (Pmode, stack_pointer_rtx, reg2));
 	}
       else
 	{
 	  emit_set_insn (reg2,
-			 plus_constant (Pmode, stack_pointer_rtx,
-					adjustment));
+			 plus_constant (Pmode, stack_pointer_rtx, adjustment));
 	}
-      if (!CONST_INT_P (rounded_size))
-	emit_set_insn (reg2, gen_rtx_MINUS (Pmode, reg2, rounded_size));
 	  	
       /* Step 3: the loop
 
@@ -3572,11 +3620,9 @@ aarch64_emit_probe_stack_range (HOST_WIDE_INT first, poly_int64 size)
       /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
 	 that SIZE is equal to ROUNDED_SIZE.  */
 
-      if (!size.is_constant (&const_size))
-	gcc_unreachable ();
-      else if (const_size != INTVAL (rounded_size))
+      if (size != rounded_size)
 	{
-	  HOST_WIDE_INT rem = const_size - INTVAL (rounded_size);
+	  HOST_WIDE_INT rem = size - rounded_size;
 
 	  if (rem > 256)
 	    {
@@ -3615,7 +3661,14 @@ aarch64_output_probe_stack_range (rtx reg1, rtx reg2)
   output_asm_insn ("sub\t%0, %0, %1", xops);
 
   /* Probe at TEST_ADDR.  */
-  output_asm_insn ("str\txzr, [%0]", xops);
+  if (flag_stack_clash_protection)
+    {
+      gcc_assert (xops[0] == stack_pointer_rtx);
+      xops[1] = GEN_INT (PROBE_INTERVAL - 8);
+      output_asm_insn ("str\txzr, [%0, %1]", xops);
+    }
+  else
+    output_asm_insn ("str\txzr, [%0]", xops);
 
   /* Test if TEST_ADDR == LAST_ADDR.  */
   xops[1] = reg2;
@@ -3632,16 +3685,13 @@ aarch64_output_probe_stack_range (rtx reg1, rtx reg2)
 static bool
 aarch64_frame_pointer_required (void)
 {
-  /* In aarch64_override_options_after_change
-     flag_omit_leaf_frame_pointer turns off the frame pointer by
-     default.  Turn it back on now if we've not got a leaf
-     function.  */
-  if (flag_omit_leaf_frame_pointer
-      && (!crtl->is_leaf || df_regs_ever_live_p (LR_REGNUM)))
-    return true;
-
-  /* Force a frame pointer for EH returns so the return address is at FP+8.  */
-  if (crtl->calls_eh_return)
+  /* Use the frame pointer if enabled and it is not a leaf function, unless
+     leaf frame pointer omission is disabled.  If the frame pointer is enabled,
+     force the frame pointer in leaf functions which use LR.  */
+  if (flag_omit_frame_pointer == 2
+      && !(flag_omit_leaf_frame_pointer
+	   && crtl->is_leaf
+	   && !df_regs_ever_live_p (LR_REGNUM)))
     return true;
 
   return false;
@@ -3659,6 +3709,10 @@ aarch64_layout_frame (void)
   if (reload_completed && cfun->machine->frame.laid_out)
     return;
 
+  /* Force a frame chain for EH returns so the return address is at FP+8.  */
+  cfun->machine->frame.emit_frame_chain
+    = frame_pointer_needed || crtl->calls_eh_return;
+
 #define SLOT_NOT_REQUIRED (-2)
 #define SLOT_REQUIRED     (-1)
 
@@ -3693,14 +3747,14 @@ aarch64_layout_frame (void)
 	last_fp_reg = regno;
       }
 
-  if (frame_pointer_needed)
+  if (cfun->machine->frame.emit_frame_chain)
     {
       /* FP and LR are placed in the linkage record.  */
       cfun->machine->frame.reg_offset[R29_REGNUM] = 0;
       cfun->machine->frame.wb_candidate1 = R29_REGNUM;
       cfun->machine->frame.reg_offset[R30_REGNUM] = UNITS_PER_WORD;
       cfun->machine->frame.wb_candidate2 = R30_REGNUM;
-      offset += 2 * UNITS_PER_WORD;
+      offset = 2 * UNITS_PER_WORD;
     }
 
   /* Now assign stack slots for them.  */
@@ -3752,6 +3806,8 @@ aarch64_layout_frame (void)
 			   STACK_BOUNDARY / BITS_PER_UNIT);
 
   /* Both these values are already aligned.  */
+  gcc_assert (multiple_p (crtl->outgoing_args_size,
+			  STACK_BOUNDARY / BITS_PER_UNIT));
   cfun->machine->frame.frame_size
     = (cfun->machine->frame.hard_fp_offset
        + crtl->outgoing_args_size);
@@ -3804,20 +3860,6 @@ aarch64_layout_frame (void)
       cfun->machine->frame.final_adjust
 	= cfun->machine->frame.frame_size - cfun->machine->frame.callee_adjust;
     }
-  else if (!frame_pointer_needed
-	   && varargs_and_saved_regs_size < max_push_offset)
-    {
-      /* Frame with large local area and outgoing arguments (this pushes the
-	 callee-saves first, followed by the locals and outgoing area):
-	 stp reg1, reg2, [sp, -varargs_and_saved_regs_size]!
-	 stp reg3, reg4, [sp, 16]
-	 sub sp, sp, frame_size - varargs_and_saved_regs_size  */
-      cfun->machine->frame.callee_adjust = varargs_and_saved_regs_size;
-      cfun->machine->frame.final_adjust
-	= cfun->machine->frame.frame_size - cfun->machine->frame.callee_adjust;
-      cfun->machine->frame.hard_fp_offset = cfun->machine->frame.callee_adjust;
-      cfun->machine->frame.locals_offset = cfun->machine->frame.hard_fp_offset;
-    }
   else
     {
       /* Frame with large local area and outgoing arguments using frame pointer:
@@ -4141,6 +4183,9 @@ aarch64_restore_callee_saves (machine_mode mode,
     }
 }
 
+/* Return true if OFFSET is a signed 4-bit value multiplied by the size
+   of MODE.  */
+
 static inline bool
 offset_4bit_signed_scaled_p (machine_mode mode, poly_int64 offset)
 {
@@ -4149,6 +4194,9 @@ offset_4bit_signed_scaled_p (machine_mode mode, poly_int64 offset)
 	  && IN_RANGE (multiple, -8, 7));
 }
 
+/* Return true if OFFSET is a unsigned 6-bit value multiplied by the size
+   of MODE.  */
+
 static inline bool
 offset_6bit_unsigned_scaled_p (machine_mode mode, poly_int64 offset)
 {
@@ -4157,6 +4205,9 @@ offset_6bit_unsigned_scaled_p (machine_mode mode, poly_int64 offset)
 	  && IN_RANGE (multiple, 0, 63));
 }
 
+/* Return true if OFFSET is a signed 7-bit value multiplied by the size
+   of MODE.  */
+
 bool
 aarch64_offset_7bit_signed_scaled_p (machine_mode mode, poly_int64 offset)
 {
@@ -4165,6 +4216,8 @@ aarch64_offset_7bit_signed_scaled_p (machine_mode mode, poly_int64 offset)
 	  && IN_RANGE (multiple, -64, 63));
 }
 
+/* Return true if OFFSET is a signed 9-bit value.  */
+
 static inline bool
 offset_9bit_signed_unscaled_p (machine_mode mode ATTRIBUTE_UNUSED,
 			       poly_int64 offset)
@@ -4174,6 +4227,9 @@ offset_9bit_signed_unscaled_p (machine_mode mode ATTRIBUTE_UNUSED,
 	  && IN_RANGE (const_offset, -256, 255));
 }
 
+/* Return true if OFFSET is a signed 9-bit value multiplied by the size
+   of MODE.  */
+
 static inline bool
 offset_9bit_signed_scaled_p (machine_mode mode, poly_int64 offset)
 {
@@ -4182,6 +4238,9 @@ offset_9bit_signed_scaled_p (machine_mode mode, poly_int64 offset)
 	  && IN_RANGE (multiple, -256, 255));
 }
 
+/* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
+   of MODE.  */
+
 static inline bool
 offset_12bit_unsigned_scaled_p (machine_mode mode, poly_int64 offset)
 {
@@ -4406,6 +4465,144 @@ aarch64_set_handled_components (sbitmap components)
       cfun->machine->reg_is_wrapped_separately[regno] = true;
 }
 
+/* Allocate POLY_SIZE bytes of stack space using TEMP1 and TEMP2 as scratch
+   registers.  */
+
+static void
+aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
+					poly_int64 poly_size)
+{
+  HOST_WIDE_INT size;
+  if (!poly_size.is_constant (&size))
+    {
+      sorry ("stack probes for SVE frames");
+      return;
+    }
+
+  HOST_WIDE_INT probe_interval
+    = 1 << PARAM_VALUE (PARAM_STACK_CLASH_PROTECTION_PROBE_INTERVAL);
+  HOST_WIDE_INT guard_size
+    = 1 << PARAM_VALUE (PARAM_STACK_CLASH_PROTECTION_GUARD_SIZE);
+  HOST_WIDE_INT guard_used_by_caller = 1024;
+
+  /* SIZE should be large enough to require probing here.  ie, it
+     must be larger than GUARD_SIZE - GUARD_USED_BY_CALLER.
+
+     We can allocate GUARD_SIZE - GUARD_USED_BY_CALLER as a single chunk
+     without any probing.  */
+  gcc_assert (size >= guard_size - guard_used_by_caller);
+  aarch64_sub_sp (temp1, temp2, guard_size - guard_used_by_caller, true);
+  HOST_WIDE_INT orig_size = size;
+  size -= (guard_size - guard_used_by_caller);
+
+  HOST_WIDE_INT rounded_size = size & -probe_interval;
+  HOST_WIDE_INT residual = size - rounded_size;
+
+  /* We can handle a small number of allocations/probes inline.  Otherwise
+     punt to a loop.  */
+  if (rounded_size && rounded_size <= 4 * probe_interval)
+    {
+      /* We don't use aarch64_sub_sp here because we don't want to
+	 repeatedly load TEMP1.  */
+      if (probe_interval > ARITH_FACTOR)
+	emit_move_insn (temp1, GEN_INT (-probe_interval));
+      else
+	temp1 = GEN_INT (-probe_interval);
+
+      for (HOST_WIDE_INT i = 0; i < rounded_size; i += probe_interval)
+	{
+	  rtx_insn *insn = emit_insn (gen_add2_insn (stack_pointer_rtx,
+						     temp1));
+          add_reg_note (insn, REG_STACK_CHECK, const0_rtx);
+
+	  if (probe_interval > ARITH_FACTOR)
+	    {
+	      RTX_FRAME_RELATED_P (insn) = 1;
+	      rtx adj = plus_constant (Pmode, stack_pointer_rtx, rounded_size);
+	      add_reg_note (insn, REG_CFA_ADJUST_CFA,
+			    gen_rtx_SET (stack_pointer_rtx, adj));
+	    }
+
+	  emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
+					   (probe_interval
+					    - GET_MODE_SIZE (word_mode))));
+	  emit_insn (gen_blockage ());
+	}
+      dump_stack_clash_frame_info (PROBE_INLINE, size != rounded_size);
+    }
+  else if (rounded_size)
+    {
+      /* Compute the ending address.  */
+      unsigned int scratchreg = REGNO (temp1);
+      emit_move_insn (temp1, GEN_INT (-rounded_size));
+      rtx_insn *insn
+	 = emit_insn (gen_add3_insn (temp1, stack_pointer_rtx, temp1));
+
+      /* For the initial allocation, we don't have a frame pointer
+	 set up, so we always need CFI notes.  If we're doing the
+	 final allocation, then we may have a frame pointer, in which
+	 case it is the CFA, otherwise we need CFI notes.
+
+	 We can determine which allocation we are doing by looking at
+	 the temporary register.  IP0 is the initial allocation, IP1
+	 is the final allocation.  */
+      if (scratchreg == IP0_REGNUM || !frame_pointer_needed)
+	{
+	  /* We want the CFA independent of the stack pointer for the
+	     duration of the loop.  */
+	  add_reg_note (insn, REG_CFA_DEF_CFA,
+			plus_constant (Pmode, temp1,
+				       (rounded_size + (orig_size - size))));
+	  RTX_FRAME_RELATED_P (insn) = 1;
+	}
+
+      /* This allocates and probes the stack.
+
+	 It also probes at a 4k interval regardless of the value of
+	 PARAM_STACK_CLASH_PROTECTION_PROBE_INTERVAL.  */
+      insn = emit_insn (gen_probe_stack_range (stack_pointer_rtx,
+					       stack_pointer_rtx, temp1));
+
+      /* Now reset the CFA register if needed.  */
+      if (scratchreg == IP0_REGNUM || !frame_pointer_needed)
+	{
+	  add_reg_note (insn, REG_CFA_DEF_CFA,
+			plus_constant (Pmode, stack_pointer_rtx,
+				       (rounded_size + (orig_size - size))));
+	  RTX_FRAME_RELATED_P (insn) = 1;
+	}
+
+      emit_insn (gen_blockage ());
+      dump_stack_clash_frame_info (PROBE_LOOP, size != rounded_size);
+    }
+  else
+    dump_stack_clash_frame_info (PROBE_INLINE, size != rounded_size);
+
+  /* Handle any residuals.
+     Note that any residual must be probed.  */
+  if (residual)
+    {
+      aarch64_sub_sp (temp1, temp2, residual, true);
+      add_reg_note (get_last_insn (), REG_STACK_CHECK, const0_rtx);
+      emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
+				       (residual - GET_MODE_SIZE (word_mode))));
+      emit_insn (gen_blockage ());
+    }
+  return;
+}
+
+/* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
+   is saved at BASE + OFFSET.  */
+
+static void
+aarch64_add_cfa_expression (rtx_insn *insn, unsigned int reg,
+			    rtx base, poly_int64 offset)
+{
+  rtx mem = gen_frame_mem (DImode, plus_constant (Pmode, base, offset));
+  add_reg_note (insn, REG_CFA_EXPRESSION,
+		gen_rtx_SET (mem, regno_reg_rtx[reg]));
+}
+
 /* AArch64 stack frames generated by this compiler look like:
 
 	+-------------------------------+
@@ -4460,6 +4657,7 @@ aarch64_expand_prologue (void)
   poly_int64 callee_offset = cfun->machine->frame.callee_offset;
   unsigned reg1 = cfun->machine->frame.wb_candidate1;
   unsigned reg2 = cfun->machine->frame.wb_candidate2;
+  bool emit_frame_chain = cfun->machine->frame.emit_frame_chain;
   rtx_insn *insn;
 
   /* Sign return address for functions.  */
@@ -4489,26 +4687,133 @@ aarch64_expand_prologue (void)
 
   rtx ip0_rtx = gen_rtx_REG (Pmode, IP0_REGNUM);
   rtx ip1_rtx = gen_rtx_REG (Pmode, IP1_REGNUM);
-  aarch64_sub_sp (ip0_rtx, ip1_rtx, initial_adjust, true);
+
+  /* We do not fully protect aarch64 against stack clash style attacks
+     as doing so would be prohibitively expensive with less utility over
+     time as newer compilers are deployed.
+
+     We assume the guard is at least 64k.  Furthermore, we assume that
+     the caller has not pushed the stack pointer more than 1k into
+     the guard.  A caller that pushes the stack pointer than 1k into
+     the guard is considered invalid.
+
+     Note that the caller's ability to push the stack pointer into the
+     guard is a function of the number and size of outgoing arguments and/or
+     dynamic stack allocations due to the mandatory save of the link register
+     in the caller's frame.
+
+     With those assumptions the callee can allocate up to 63k of stack
+     space without probing.
+
+     When probing is needed, we emit a probe at the start of the prologue
+     and every PARAM_STACK_CLASH_PROTECTION_PROBE_INTERVAL bytes thereafter.
+
+     We have to track how much space has been allocated, but we do not
+     track stores into the stack as implicit probes except for the
+     fp/lr store.  */
+  HOST_WIDE_INT guard_size
+    = 1 << PARAM_VALUE (PARAM_STACK_CLASH_PROTECTION_GUARD_SIZE);
+  HOST_WIDE_INT guard_used_by_caller = 1024;
+  if (flag_stack_clash_protection)
+    {
+      if (must_eq (frame_size, 0))
+	dump_stack_clash_frame_info (NO_PROBE_NO_FRAME, false);
+      else if (must_lt (initial_adjust, guard_size - guard_used_by_caller)
+	       && must_lt (final_adjust, guard_size - guard_used_by_caller))
+	dump_stack_clash_frame_info (NO_PROBE_SMALL_FRAME, true);
+    }
+
+  /* In theory we should never have both an initial adjustment
+     and a callee save adjustment.  Verify that is the case since the
+     code below does not handle it for -fstack-clash-protection.  */
+  gcc_assert (must_eq (initial_adjust, 0) || callee_adjust == 0);
+
+  /* Only probe if the initial adjustment is larger than the guard
+     less the amount of the guard reserved for use by the caller's
+     outgoing args.  */
+  if (flag_stack_clash_protection
+      && may_ge (initial_adjust, guard_size - guard_used_by_caller))
+    aarch64_allocate_and_probe_stack_space (ip0_rtx, ip1_rtx, initial_adjust);
+  else
+    aarch64_sub_sp (ip0_rtx, ip1_rtx, initial_adjust, true);
 
   if (callee_adjust != 0)
     aarch64_push_regs (reg1, reg2, callee_adjust);
 
-  if (frame_pointer_needed)
+  if (emit_frame_chain)
     {
+      poly_int64 reg_offset = callee_adjust;
       if (callee_adjust == 0)
-	aarch64_save_callee_saves (DImode, callee_offset, R29_REGNUM,
-				   R30_REGNUM, false);
-      aarch64_add_offset (Pmode, hard_frame_pointer_rtx, ip1_rtx, ip0_rtx,
-			  stack_pointer_rtx, callee_offset, true);
+	{
+	  reg1 = R29_REGNUM;
+	  reg2 = R30_REGNUM;
+	  reg_offset = callee_offset;
+	  aarch64_save_callee_saves (DImode, reg_offset, reg1, reg2, false);
+	}
+      aarch64_add_offset (Pmode, hard_frame_pointer_rtx,
+			  stack_pointer_rtx, callee_offset,
+			  ip1_rtx, ip0_rtx, frame_pointer_needed);
+      if (!frame_size.is_constant ())
+	{
+	  /* Variable-sized frames need to describe the save slot address
+	     using DW_CFA_expression rather than DW_CFA_offset.  This means
+	     that the locations of the registers that we've already saved
+	     do not automatically change as the CFA definition changes.
+	     We instead need to re-express the save slots with addresses
+	     based on the frame pointer rather than the stack pointer.  */
+	  rtx_insn *insn = get_last_insn ();
+	  gcc_assert (RTX_FRAME_RELATED_P (insn));
+
+	  /* Add an explicit CFA definition if this was previously
+	     implicit.  */
+	  if (!find_reg_note (insn, REG_CFA_ADJUST_CFA, NULL_RTX))
+	    {
+	      rtx src = plus_constant (Pmode, stack_pointer_rtx,
+				       callee_offset);
+	      add_reg_note (insn, REG_CFA_ADJUST_CFA,
+			    gen_rtx_SET (hard_frame_pointer_rtx, src));
+	    }
+
+	  /* Change the save slot expressions for the registers that
+	     we've already saved.  */
+	  reg_offset -= callee_offset;
+	  aarch64_add_cfa_expression (insn, reg2, hard_frame_pointer_rtx,
+				      reg_offset + UNITS_PER_WORD);
+	  aarch64_add_cfa_expression (insn, reg1, hard_frame_pointer_rtx,
+				      reg_offset);
+	}
       emit_insn (gen_stack_tie (stack_pointer_rtx, hard_frame_pointer_rtx));
     }
 
   aarch64_save_callee_saves (DImode, callee_offset, R0_REGNUM, R30_REGNUM,
-			     callee_adjust != 0 || frame_pointer_needed);
+			     callee_adjust != 0 || emit_frame_chain);
   aarch64_save_callee_saves (DFmode, callee_offset, V0_REGNUM, V31_REGNUM,
-			     callee_adjust != 0 || frame_pointer_needed);
-  aarch64_sub_sp (ip1_rtx, ip0_rtx, final_adjust, !frame_pointer_needed);
+			     callee_adjust != 0 || emit_frame_chain);
+
+  /* We may need to probe the final adjustment as well.  */
+  if (flag_stack_clash_protection && may_ne (final_adjust, 0))
+    {
+      /* First probe if the final adjustment is larger than the guard size
+	 less the amount of the guard reserved for use by the caller's
+	 outgoing args.  */
+      if (may_ge (final_adjust, guard_size - guard_used_by_caller))
+	aarch64_allocate_and_probe_stack_space (ip1_rtx, ip0_rtx,
+						final_adjust);
+      else
+	aarch64_sub_sp (ip1_rtx, ip0_rtx, final_adjust, !frame_pointer_needed);
+
+      /* We must also probe if the final adjustment is larger than the guard
+	 that is assumed used by the caller.  This may be sub-optimal.  */
+      if (may_ge (final_adjust, guard_used_by_caller))
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "Stack clash aarch64 large outgoing arg, probing\n");
+	  emit_stack_probe (stack_pointer_rtx);
+	}
+    }
+  else
+    aarch64_sub_sp (ip1_rtx, ip0_rtx, final_adjust, !frame_pointer_needed);
 }
 
 /* Return TRUE if we can use a simple_return insn.
@@ -4549,8 +4854,13 @@ aarch64_expand_epilogue (bool for_sibcall)
   unsigned reg2 = cfun->machine->frame.wb_candidate2;
   rtx cfi_ops = NULL;
   rtx_insn *insn;
+  /* A stack clash protection prologue may not have left IP0_REGNUM or
+     IP1_REGNUM in a usable state.  The same is true for allocations
+     with an SVE component, since we then need both temporary registers
+     for each allocation.  */
   bool can_inherit_p = (initial_adjust.is_constant ()
-			&& final_adjust.is_constant ());
+			&& final_adjust.is_constant ()
+			&& !flag_stack_clash_protection);
 
   /* We need to add memory barrier to prevent read from deallocated stack.  */
   bool need_barrier_p = may_ne (get_frame_size ()
@@ -4572,9 +4882,9 @@ aarch64_expand_epilogue (bool for_sibcall)
   if (frame_pointer_needed && (may_ne (final_adjust, 0) || cfun->calls_alloca))
     /* If writeback is used when restoring callee-saves, the CFA
        is restored on the instruction doing the writeback.  */
-    aarch64_add_offset (Pmode, stack_pointer_rtx, ip1_rtx, ip0_rtx,
+    aarch64_add_offset (Pmode, stack_pointer_rtx,
 			hard_frame_pointer_rtx, -callee_offset,
-			callee_adjust == 0);
+			ip1_rtx, ip0_rtx, callee_adjust == 0);
   else
     aarch64_add_sp (ip1_rtx, ip0_rtx, final_adjust,
 		    !can_inherit_p || df_regs_ever_live_p (IP1_REGNUM));
@@ -4711,7 +5021,7 @@ aarch64_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED,
   temp1 = gen_rtx_REG (Pmode, IP1_REGNUM);
 
   if (vcall_offset == 0)
-    aarch64_add_constant (Pmode, this_rtx, temp1, delta);
+    aarch64_add_offset (Pmode, this_rtx, this_rtx, delta, temp1, temp0, false);
   else
     {
       gcc_assert ((vcall_offset & (POINTER_BYTES - 1)) == 0);
@@ -4723,7 +5033,8 @@ aarch64_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED,
 	    addr = gen_rtx_PRE_MODIFY (Pmode, this_rtx,
 				       plus_constant (Pmode, this_rtx, delta));
 	  else
-	    aarch64_add_constant (Pmode, this_rtx, temp1, delta);
+	    aarch64_add_offset (Pmode, this_rtx, this_rtx, delta,
+				temp1, temp0, false);
 	}
 
       if (Pmode == ptr_mode)
@@ -5996,8 +6307,8 @@ aarch64_const_vec_all_same_int_p (rtx x, HOST_WIDE_INT val)
   return aarch64_const_vec_all_same_in_range_p (x, val, val);
 }
 
-/* Return true if VEC is a constant in which every element is in the
-   range [MINVAL, MAXVAL].  */
+/* Return true if VEC is a constant in which every element is in the range
+   [MINVAL, MAXVAL].  The elements do not need to have the same value.  */
 
 static bool
 aarch64_const_vec_all_in_range_p (rtx vec,
@@ -6125,6 +6436,10 @@ aarch64_print_vector_float_operand (FILE *f, rtx x, bool negate)
    The acceptable formatting commands given by CODE are:
      'c':		An integer or symbol address without a preceding #
 			sign.
+     'C':		Take the duplicated element in a vector constant
+			and print it in hex.
+     'D':		Take the duplicated element in a vector constant
+			and print it as an unsigned integer, in decimal.
      'e':		Print the sign/zero-extend size as a character 8->b,
 			16->h, 32->w.
      'p':		Prints N such that 2^N == X (X must be power of 2 and
@@ -6134,6 +6449,8 @@ aarch64_print_vector_float_operand (FILE *f, rtx x, bool negate)
 			of regs.
      'm':		Print a condition (eq, ne, etc).
      'M':		Same as 'm', but invert condition.
+     'N':		Take the duplicated element in a vector constant
+			and print the negative of it in decimal.
      'b/h/s/d/q':	Print a scalar FP/SIMD register name.
      'S/T/U/V':		Print a FP/SIMD register name for a register list.
 			The register printed is the FP/SIMD register name
@@ -6350,6 +6667,20 @@ aarch64_print_operand (FILE *f, rtx x, int code)
       }
       break;
 
+    case 'D':
+      {
+	/* Print a replicated constant in decimal, treating it as
+	   unsigned.  */
+	if (!const_vec_duplicate_p (x, &elt) || !CONST_INT_P (elt))
+	  {
+	    output_operand_lossage ("invalid operand for '%%%c'", code);
+	    return;
+	  }
+	scalar_mode inner_mode = GET_MODE_INNER (GET_MODE (x));
+	asm_fprintf (f, "%wd", UINTVAL (elt) & GET_MODE_MASK (inner_mode));
+      }
+      break;
+
     case 'w':
     case 'x':
       if (x == const0_rtx
@@ -6881,13 +7212,15 @@ aarch64_legitimize_address_displacement (rtx *offset1, rtx *offset2,
       /* Split an out-of-range address displacement into a base and
 	 offset.  Use 4KB range for 1- and 2-byte accesses and a 16KB
 	 range otherwise to increase opportunities for sharing the base
-	 address of different sizes.  For unaligned accesses and TI/TF
-	 mode use the signed 9-bit range.  */
-      second_offset = const_offset & (size < 4 ? 0xfff : 0x3ffc);
-      if (mode == TImode
-	  || mode == TFmode
-	  || (const_offset & (size - 1)) != 0)
-	second_offset = (const_offset + 0x100) & ~0x1ff;
+	 address of different sizes.  Unaligned accesses use the signed
+	 9-bit range, TImode/TFmode use the intersection of signed
+	 scaled 7-bit and signed 9-bit offset.  */
+      if (mode == TImode || mode == TFmode)
+	second_offset = ((const_offset + 0x100) & 0x1f8) - 0x100;
+      else if ((const_offset & (size - 1)) != 0)
+	second_offset = ((const_offset + 0x100) & 0x1ff) - 0x100;
+      else
+	second_offset = const_offset & (size < 4 ? 0xfff : 0x3ffc);
 
       if (second_offset == 0 || must_eq (orig_offset, second_offset))
 	return false;
@@ -6985,6 +7318,14 @@ aarch64_secondary_reload (bool in_p ATTRIBUTE_UNUSED, rtx x,
 			  machine_mode mode,
 			  secondary_reload_info *sri)
 {
+  if (BYTES_BIG_ENDIAN
+      && reg_class_subset_p (rclass, FP_REGS)
+      && (MEM_P (x) || (REG_P (x) && !HARD_REGISTER_P (x)))
+      && aarch64_sve_data_mode_p (mode))
+    {
+      sri->icode = CODE_FOR_aarch64_sve_reload_be;
+      return NO_REGS;
+    }
 
   /* If we have to disable direct literal pool loads and stores because the
      function is too big, then we need a scratch register.  */
@@ -7023,15 +7364,6 @@ aarch64_secondary_reload (bool in_p ATTRIBUTE_UNUSED, rtx x,
   return NO_REGS;
 }
 
-/* Implement TARGET_CANNOT_SUBSTITUTE_MEM_EQUIV_P.  */
-
-static bool
-aarch64_cannot_substitute_mem_equiv_p (rtx mem)
-{
-  /* See the comments above aarch64_sve_mov<mode> for details.  */
-  return BYTES_BIG_ENDIAN && aarch64_sve_data_mode_p (GET_MODE (mem));
-}
-
 static bool
 aarch64_can_eliminate (const int from, const int to)
 {
@@ -7058,6 +7390,7 @@ aarch64_can_eliminate (const int from, const int to)
 	 LR in the function, then we'll want a frame pointer after all, so
 	 prevent this elimination to ensure a frame pointer is used.  */
       if (to == STACK_POINTER_REGNUM
+	  && flag_omit_frame_pointer == 2
 	  && flag_omit_leaf_frame_pointer
 	  && df_regs_ever_live_p (LR_REGNUM))
 	return false;
@@ -7152,8 +7485,12 @@ aarch64_trampoline_init (rtx m_tramp, tree fndecl, rtx chain_value)
 static unsigned char
 aarch64_class_max_nregs (reg_class_t regclass, machine_mode mode)
 {
+  /* ??? Logically we should only need to provide a value when
+     HARD_REGNO_MODE_OK says that at least one register in REGCLASS
+     can hold MODE, but at the moment we need to handle all modes.
+     Just ignore any runtime parts for registers that can't store them.  */
+  HOST_WIDE_INT lowest_size = constant_lower_bound (GET_MODE_SIZE (mode));
   unsigned int nregs;
-  HOST_WIDE_INT size;
   switch (regclass)
     {
     case CALLER_SAVE_REGS:
@@ -7167,10 +7504,9 @@ aarch64_class_max_nregs (reg_class_t regclass, machine_mode mode)
 	  && constant_multiple_p (GET_MODE_SIZE (mode),
 				  BYTES_PER_SVE_VECTOR, &nregs))
 	return nregs;
-      size = constant_lower_bound (GET_MODE_SIZE (mode));
       return (aarch64_vector_data_mode_p (mode)
-	      ? CEIL (size, UNITS_PER_VREG)
-	      : CEIL (size, UNITS_PER_WORD));
+	      ? CEIL (lowest_size, UNITS_PER_VREG)
+	      : CEIL (lowest_size, UNITS_PER_WORD));
     case STACK_REG:
     case PR_REGS:
     case PR_LO_REGS:
@@ -8070,20 +8406,16 @@ aarch64_rtx_costs (rtx x, machine_mode mode, int outer ATTRIBUTE_UNUSED,
 	  /* The cost is one per vector-register copied.  */
 	  if (VECTOR_MODE_P (GET_MODE (op0)) && REG_P (op1))
 	    {
-	      int size = constant_lower_bound (GET_MODE_SIZE (GET_MODE (op0)));
-	      int n_minus_1 = (size - 1) / UNITS_PER_VREG;
-	      *cost = COSTS_N_INSNS (n_minus_1 + 1);
+	      int nregs = aarch64_hard_regno_nregs (V0_REGNUM, GET_MODE (op0));
+	      *cost = COSTS_N_INSNS (nregs);
 	    }
 	  /* const0_rtx is in general free, but we will use an
 	     instruction to set a register to 0.  */
 	  else if (REG_P (op1) || op1 == const0_rtx)
 	    {
-	      /* The cost is 1 per register copied.  The size in this
-		 case must be constant, since all variable-size modes
-		 are vectors.  */
-	      int size = GET_MODE_SIZE (GET_MODE (op0)).to_constant ();
-	      int n_minus_1 = (size - 1) / UNITS_PER_WORD;
-	      *cost = COSTS_N_INSNS (n_minus_1 + 1);
+	      /* The cost is 1 per register copied.  */
+	      int nregs = aarch64_hard_regno_nregs (R0_REGNUM, GET_MODE (op0));
+	      *cost = COSTS_N_INSNS (nregs);
 	    }
           else
 	    /* Cost is just the cost of the RHS of the set.  */
@@ -9694,9 +10026,11 @@ aarch64_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 	return costs->scalar_to_vec_cost;
 
       case unaligned_load:
+      case vector_gather_load:
 	return costs->vec_unalign_load_cost;
 
       case unaligned_store:
+      case vector_scatter_store:
 	return costs->vec_unalign_store_cost;
 
       case cond_branch_taken:
@@ -10098,24 +10432,16 @@ aarch64_parse_override_string (const char* input_string,
 static void
 aarch64_override_options_after_change_1 (struct gcc_options *opts)
 {
-  /* The logic here is that if we are disabling all frame pointer generation
-     then we do not need to disable leaf frame pointer generation as a
-     separate operation.  But if we are *only* disabling leaf frame pointer
-     generation then we set flag_omit_frame_pointer to true, but in
-     aarch64_frame_pointer_required we return false only for leaf functions.
-
-     PR 70044: We have to be careful about being called multiple times for the
-     same function.  Once we have decided to set flag_omit_frame_pointer just
-     so that we can omit leaf frame pointers, we must then not interpret a
-     second call as meaning that all frame pointer generation should be
-     omitted.  We do this by setting flag_omit_frame_pointer to a special,
-     non-zero value.  */
-  if (opts->x_flag_omit_frame_pointer == 2)
-    opts->x_flag_omit_frame_pointer = 0;
-
-  if (opts->x_flag_omit_frame_pointer)
-    opts->x_flag_omit_leaf_frame_pointer = false;
-  else if (opts->x_flag_omit_leaf_frame_pointer)
+  /* PR 70044: We have to be careful about being called multiple times for the
+     same function.  This means all changes should be repeatable.  */
+
+  /* If the frame pointer is enabled, set it to a special value that behaves
+     similar to frame pointer omission.  If we don't do this all leaf functions
+     will get a frame pointer even if flag_omit_leaf_frame_pointer is set.
+     If flag_omit_frame_pointer has this special value, we must force the
+     frame pointer if not in a leaf function.  We also need to force it in a
+     leaf function if flag_omit_frame_pointer is not set or if LR is used.  */
+  if (opts->x_flag_omit_frame_pointer == 0)
     opts->x_flag_omit_frame_pointer = 2;
 
   /* If not optimizing for size, set the default
@@ -10225,6 +10551,11 @@ aarch64_override_options_internal (struct gcc_options *opts)
 			   opts->x_param_values,
 			   global_options_set.x_param_values);
 
+  /* Use the alternative scheduling-pressure algorithm by default.  */
+  maybe_set_param_value (PARAM_SCHED_PRESSURE_ALGORITHM, SCHED_PRESSURE_MODEL,
+			 opts->x_param_values,
+			 global_options_set.x_param_values);
+
   /* Enable sw prefetching at specified optimization level for
      CPUS that have prefetch.  Lower optimization level threshold by 1
      when profiling is enabled.  */
@@ -10234,6 +10565,12 @@ aarch64_override_options_internal (struct gcc_options *opts)
       && opts->x_optimize >= aarch64_tune_params.prefetch->default_opt_level)
     opts->x_flag_prefetch_loop_arrays = 1;
 
+  /* We assume the guard page is 64k.  */
+  maybe_set_param_value (PARAM_STACK_CLASH_PROTECTION_GUARD_SIZE,
+			 16,
+			 opts->x_param_values,
+			 global_options_set.x_param_values);
+
   aarch64_override_options_after_change_1 (opts);
 }
 
@@ -10409,7 +10746,7 @@ static poly_uint16
 aarch64_convert_sve_vector_bits (aarch64_sve_vector_bits_enum value)
 {
   /* For now generate vector-length agnostic code for -msve-vector-bits=128.
-     This ensures we can clearly distinguish SVE and AdvSIMD modes when
+     This ensures we can clearly distinguish SVE and Advanced SIMD modes when
      deciding which .md file patterns to use and when deciding whether
      something is a legitimate address or constant.  */
   if (value == SVE_SCALABLE || value == SVE_128)
@@ -10712,9 +11049,8 @@ enum aarch64_attr_opt_type
    ATTR_TYPE specifies the type of behavior of the attribute as described
    in the definition of enum aarch64_attr_opt_type.
    ALLOW_NEG is true if the attribute supports a "no-" form.
-   HANDLER is the function that takes the attribute string and whether
-   it is a pragma or attribute and handles the option.  It is needed only
-   when the ATTR_TYPE is aarch64_attr_custom.
+   HANDLER is the function that takes the attribute string as an argument
+   It is needed only when the ATTR_TYPE is aarch64_attr_custom.
    OPT_NUM is the enum specifying the option that the attribute modifies.
    This is needed for attributes that mirror the behavior of a command-line
    option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
@@ -10725,15 +11061,14 @@ struct aarch64_attribute_info
   const char *name;
   enum aarch64_attr_opt_type attr_type;
   bool allow_neg;
-  bool (*handler) (const char *, const char *);
+  bool (*handler) (const char *);
   enum opt_code opt_num;
 };
 
-/* Handle the ARCH_STR argument to the arch= target attribute.
-   PRAGMA_OR_ATTR is used in potential error messages.  */
+/* Handle the ARCH_STR argument to the arch= target attribute.  */
 
 static bool
-aarch64_handle_attr_arch (const char *str, const char *pragma_or_attr)
+aarch64_handle_attr_arch (const char *str)
 {
   const struct processor *tmp_arch = NULL;
   enum aarch64_parse_opt_result parse_res
@@ -10750,15 +11085,14 @@ aarch64_handle_attr_arch (const char *str, const char *pragma_or_attr)
   switch (parse_res)
     {
       case AARCH64_PARSE_MISSING_ARG:
-	error ("missing architecture name in 'arch' target %s", pragma_or_attr);
+	error ("missing name in %<target(\"arch=\")%> pragma or attribute");
 	break;
       case AARCH64_PARSE_INVALID_ARG:
-	error ("unknown value %qs for 'arch' target %s", str, pragma_or_attr);
+	error ("invalid name (\"%s\") in %<target(\"arch=\")%> pragma or attribute", str);
 	aarch64_print_hint_for_arch (str);
 	break;
       case AARCH64_PARSE_INVALID_FEATURE:
-	error ("invalid feature modifier %qs for 'arch' target %s",
-	       str, pragma_or_attr);
+	error ("invalid value (\"%s\") in %<target()%> pragma or attribute", str);
 	break;
       default:
 	gcc_unreachable ();
@@ -10767,11 +11101,10 @@ aarch64_handle_attr_arch (const char *str, const char *pragma_or_attr)
   return false;
 }
 
-/* Handle the argument CPU_STR to the cpu= target attribute.
-   PRAGMA_OR_ATTR is used in potential error messages.  */
+/* Handle the argument CPU_STR to the cpu= target attribute.  */
 
 static bool
-aarch64_handle_attr_cpu (const char *str, const char *pragma_or_attr)
+aarch64_handle_attr_cpu (const char *str)
 {
   const struct processor *tmp_cpu = NULL;
   enum aarch64_parse_opt_result parse_res
@@ -10791,15 +11124,14 @@ aarch64_handle_attr_cpu (const char *str, const char *pragma_or_attr)
   switch (parse_res)
     {
       case AARCH64_PARSE_MISSING_ARG:
-	error ("missing cpu name in 'cpu' target %s", pragma_or_attr);
+	error ("missing name in %<target(\"cpu=\")%> pragma or attribute");
 	break;
       case AARCH64_PARSE_INVALID_ARG:
-	error ("unknown value %qs for 'cpu' target %s", str, pragma_or_attr);
+	error ("invalid name (\"%s\") in %<target(\"cpu=\")%> pragma or attribute", str);
 	aarch64_print_hint_for_core (str);
 	break;
       case AARCH64_PARSE_INVALID_FEATURE:
-	error ("invalid feature modifier %qs for 'cpu' target %s",
-	       str, pragma_or_attr);
+	error ("invalid value (\"%s\") in %<target()%> pragma or attribute", str);
 	break;
       default:
 	gcc_unreachable ();
@@ -10808,11 +11140,10 @@ aarch64_handle_attr_cpu (const char *str, const char *pragma_or_attr)
   return false;
 }
 
-/* Handle the argument STR to the tune= target attribute.
-   PRAGMA_OR_ATTR is used in potential error messages.  */
+/* Handle the argument STR to the tune= target attribute.  */
 
 static bool
-aarch64_handle_attr_tune (const char *str, const char *pragma_or_attr)
+aarch64_handle_attr_tune (const char *str)
 {
   const struct processor *tmp_tune = NULL;
   enum aarch64_parse_opt_result parse_res
@@ -10829,7 +11160,7 @@ aarch64_handle_attr_tune (const char *str, const char *pragma_or_attr)
   switch (parse_res)
     {
       case AARCH64_PARSE_INVALID_ARG:
-	error ("unknown value %qs for 'tune' target %s", str, pragma_or_attr);
+	error ("invalid name (\"%s\") in %<target(\"tune=\")%> pragma or attribute", str);
 	aarch64_print_hint_for_core (str);
 	break;
       default:
@@ -10842,11 +11173,10 @@ aarch64_handle_attr_tune (const char *str, const char *pragma_or_attr)
 /* Parse an architecture extensions target attribute string specified in STR.
    For example "+fp+nosimd".  Show any errors if needed.  Return TRUE
    if successful.  Update aarch64_isa_flags to reflect the ISA features
-   modified.
-   PRAGMA_OR_ATTR is used in potential error messages.  */
+   modified.  */
 
 static bool
-aarch64_handle_attr_isa_flags (char *str, const char *pragma_or_attr)
+aarch64_handle_attr_isa_flags (char *str)
 {
   enum aarch64_parse_opt_result parse_res;
   unsigned long isa_flags = aarch64_isa_flags;
@@ -10870,13 +11200,11 @@ aarch64_handle_attr_isa_flags (char *str, const char *pragma_or_attr)
   switch (parse_res)
     {
       case AARCH64_PARSE_MISSING_ARG:
-	error ("missing feature modifier in target %s %qs",
-	       pragma_or_attr, str);
+	error ("missing value in %<target()%> pragma or attribute");
 	break;
 
       case AARCH64_PARSE_INVALID_FEATURE:
-	error ("invalid feature modifier in target %s %qs",
-	       pragma_or_attr, str);
+	error ("invalid value (\"%s\") in %<target()%> pragma or attribute", str);
 	break;
 
       default:
@@ -10914,12 +11242,10 @@ static const struct aarch64_attribute_info aarch64_attributes[] =
 };
 
 /* Parse ARG_STR which contains the definition of one target attribute.
-   Show appropriate errors if any or return true if the attribute is valid.
-   PRAGMA_OR_ATTR holds the string to use in error messages about whether
-   we're processing a target attribute or pragma.  */
+   Show appropriate errors if any or return true if the attribute is valid.  */
 
 static bool
-aarch64_process_one_target_attr (char *arg_str, const char* pragma_or_attr)
+aarch64_process_one_target_attr (char *arg_str)
 {
   bool invert = false;
 
@@ -10927,7 +11253,7 @@ aarch64_process_one_target_attr (char *arg_str, const char* pragma_or_attr)
 
   if (len == 0)
     {
-      error ("malformed target %s", pragma_or_attr);
+      error ("malformed %<target()%> pragma or attribute");
       return false;
     }
 
@@ -10943,7 +11269,7 @@ aarch64_process_one_target_attr (char *arg_str, const char* pragma_or_attr)
      through the machinery for the rest of the target attributes in this
      function.  */
   if (*str_to_check == '+')
-    return aarch64_handle_attr_isa_flags (str_to_check, pragma_or_attr);
+    return aarch64_handle_attr_isa_flags (str_to_check);
 
   if (len > 3 && strncmp (str_to_check, "no-", 3) == 0)
     {
@@ -10975,8 +11301,7 @@ aarch64_process_one_target_attr (char *arg_str, const char* pragma_or_attr)
 
       if (attr_need_arg_p ^ (arg != NULL))
 	{
-	  error ("target %s %qs does not accept an argument",
-		  pragma_or_attr, str_to_check);
+	  error ("pragma or attribute %<target(\"%s\")%> does not accept an argument", str_to_check);
 	  return false;
 	}
 
@@ -10984,8 +11309,7 @@ aarch64_process_one_target_attr (char *arg_str, const char* pragma_or_attr)
 	 then we can't match.  */
       if (invert && !p_attr->allow_neg)
 	{
-	  error ("target %s %qs does not allow a negated form",
-		  pragma_or_attr, str_to_check);
+	  error ("pragma or attribute %<target(\"%s\")%> does not allow a negated form", str_to_check);
 	  return false;
 	}
 
@@ -10995,7 +11319,7 @@ aarch64_process_one_target_attr (char *arg_str, const char* pragma_or_attr)
 	   For example, cpu=, arch=, tune=.  */
 	  case aarch64_attr_custom:
 	    gcc_assert (p_attr->handler);
-	    if (!p_attr->handler (arg, pragma_or_attr))
+	    if (!p_attr->handler (arg))
 	      return false;
 	    break;
 
@@ -11039,8 +11363,7 @@ aarch64_process_one_target_attr (char *arg_str, const char* pragma_or_attr)
 		}
 	      else
 		{
-		  error ("target %s %s=%s is not valid",
-			 pragma_or_attr, str_to_check, arg);
+		  error ("pragma or attribute %<target(\"%s=%s\")%> is not valid", str_to_check, arg);
 		}
 	      break;
 	    }
@@ -11074,12 +11397,10 @@ num_occurences_in_str (char c, char *str)
 }
 
 /* Parse the tree in ARGS that contains the target attribute information
-   and update the global target options space.  PRAGMA_OR_ATTR is a string
-   to be used in error messages, specifying whether this is processing
-   a target attribute or a target pragma.  */
+   and update the global target options space.  */
 
 bool
-aarch64_process_target_attr (tree args, const char* pragma_or_attr)
+aarch64_process_target_attr (tree args)
 {
   if (TREE_CODE (args) == TREE_LIST)
     {
@@ -11088,7 +11409,7 @@ aarch64_process_target_attr (tree args, const char* pragma_or_attr)
 	  tree head = TREE_VALUE (args);
 	  if (head)
 	    {
-	      if (!aarch64_process_target_attr (head, pragma_or_attr))
+	      if (!aarch64_process_target_attr (head))
 		return false;
 	    }
 	  args = TREE_CHAIN (args);
@@ -11109,7 +11430,7 @@ aarch64_process_target_attr (tree args, const char* pragma_or_attr)
 
   if (len == 0)
     {
-      error ("malformed target %s value", pragma_or_attr);
+      error ("malformed %<target()%> pragma or attribute");
       return false;
     }
 
@@ -11124,9 +11445,9 @@ aarch64_process_target_attr (tree args, const char* pragma_or_attr)
   while (token)
     {
       num_attrs++;
-      if (!aarch64_process_one_target_attr (token, pragma_or_attr))
+      if (!aarch64_process_one_target_attr (token))
 	{
-	  error ("target %s %qs is invalid", pragma_or_attr, token);
+	  error ("pragma or attribute %<target(\"%s\")%> is not valid", token);
 	  return false;
 	}
 
@@ -11135,8 +11456,7 @@ aarch64_process_target_attr (tree args, const char* pragma_or_attr)
 
   if (num_attrs != num_commas + 1)
     {
-      error ("malformed target %s list %qs",
-	      pragma_or_attr, TREE_STRING_POINTER (args));
+      error ("malformed %<target(\"%s\")%> pragma or attribute", TREE_STRING_POINTER (args));
       return false;
     }
 
@@ -11195,8 +11515,7 @@ aarch64_option_valid_attribute_p (tree fndecl, tree, tree args, int)
     cl_target_option_restore (&global_options,
 			TREE_TARGET_OPTION (target_option_current_node));
 
-
-  ret = aarch64_process_target_attr (args, "attribute");
+  ret = aarch64_process_target_attr (args);
 
   /* Set up any additional state.  */
   if (ret)
@@ -11525,64 +11844,63 @@ aarch64_legitimate_pic_operand_p (rtx x)
   return true;
 }
 
-/* Return true if X holds either a quarter-precision or
-     floating-point +0.0 constant.  */
-static bool
-aarch64_valid_floating_const (rtx x)
-{
-  if (!CONST_DOUBLE_P (x))
-    return false;
-
-  /* This call determines which constants can be used in mov<mode>
-     as integer moves instead of constant loads.  */
-  if (aarch64_float_const_rtx_p (x))
-    return true;
-
-  return aarch64_float_const_representable_p (x);
-}
+/* Implement TARGET_LEGITIMATE_CONSTANT_P hook.  Return true for constants
+   that should be rematerialized rather than spilled.  */
 
 static bool
 aarch64_legitimate_constant_p (machine_mode mode, rtx x)
 {
-  /* Do not allow vector struct mode constants.  We could support
-     0 and -1 easily, but they need support in aarch64-simd.md.  */
+  /* Support CSE and rematerialization of common constants.  */
+  if (CONST_SCALAR_INT_P (x)
+      || CONST_DOUBLE_P (x)
+      || GET_CODE (x) == CONST_VECTOR)
+    return true;
+
+  /* Do not allow vector struct mode constants for Advanced SIMD.
+     We could support 0 and -1 easily, but they need support in
+     aarch64-simd.md.  */
   unsigned int vec_flags = aarch64_classify_vector_mode (mode);
   if (vec_flags == (VEC_ADVSIMD | VEC_STRUCT))
     return false;
 
-  /* For these cases we never want to use a literal load.
-     As such we have to prevent the compiler from forcing these
-     to memory.  */
-  rtx base, step;
-  if (aarch64_simd_valid_immediate (x, NULL)
-      || (vec_flags == VEC_SVE_DATA
-	  && (const_vec_series_p (x, &base, &step)
-	      || const_vec_duplicate_p (x)))
-      || CONST_INT_P (x)
-      || aarch64_valid_floating_const (x)
-      || aarch64_can_const_movi_rtx_p (x, mode)
-      || aarch64_float_const_rtx_p (x))
-	return !targetm.cannot_force_const_mem (mode, x);
+  /* Only accept variable-length vector constants if they can be
+     handled directly.
 
-  if (GET_CODE (x) == HIGH
-      && aarch64_valid_symref (XEXP (x, 0), GET_MODE (XEXP (x, 0))))
-    return true;
+     ??? It would be possible to handle rematerialization of other
+     constants via secondary reloads.  */
+  if (vec_flags & VEC_ANY_SVE)
+    return aarch64_simd_valid_immediate (x, NULL);
+
+  if (GET_CODE (x) == HIGH)
+    x = XEXP (x, 0);
 
-  /* Accept polynomials that can be calculated by using the destination
-     of a move as the sole temporary.  Constants that require a second
-     temporary cannot be rematerialized (they can't be forced to memory
-     and also aren't legitimate constants).  */
+  /* Accept polynomial constants that can be calculated by using the
+     destination of a move as the sole temporary.  Constants that
+     require a second temporary cannot be rematerialized (they can't be
+     forced to memory and also aren't legitimate constants).  */
   poly_int64 offset;
   if (poly_int_rtx_p (x, &offset))
     return aarch64_offset_temporaries (false, offset) <= 1;
 
+  /* If an offset is being added to something else, we need to allow the
+     base to be moved into the destination register, meaning that there
+     are no free temporaries for the offset.  */
+  x = strip_offset (x, &offset);
+  if (!offset.is_constant () && aarch64_offset_temporaries (true, offset) > 0)
+    return false;
+
+  /* Do not allow const (plus (anchor_symbol, const_int)).  */
+  if (may_ne (offset, 0) && SYMBOL_REF_P (x) && SYMBOL_REF_ANCHOR_P (x))
+    return false;
+
   /* Treat symbols as constants.  Avoid TLS symbols as they are complex,
      so spilling them is better than rematerialization.  */
   if (SYMBOL_REF_P (x) && !SYMBOL_REF_TLS_MODEL (x))
     return true;
 
-  if (SCALAR_INT_MODE_P (GET_MODE (x)))
-    return aarch64_constant_address_p (x);
+  /* Label references are always constant.  */
+  if (GET_CODE (x) == LABEL_REF)
+    return true;
 
   return false;
 }
@@ -11809,7 +12127,8 @@ aarch64_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
 					       &nregs,
 					       &is_ha))
     {
-      /* We don't support passing and returning SVE types.  */
+      /* No frontends can create types with variable-sized modes, so we
+	 shouldn't be asked to pass or return them.  */
       unsigned int ag_size = GET_MODE_SIZE (ag_mode).to_constant ();
 
       /* TYPE passed in fp/simd registers.  */
@@ -12430,12 +12749,14 @@ aarch64_simd_container_mode (scalar_mode mode, poly_int64 width)
   if (TARGET_SVE && must_eq (width, BITS_PER_SVE_VECTOR))
     switch (mode)
       {
-      case E_DImode:
-	return V4DImode;
       case E_DFmode:
 	return V4DFmode;
       case E_SFmode:
 	return V8SFmode;
+      case E_HFmode:
+	return V16HFmode;
+      case E_DImode:
+	return V4DImode;
       case E_SImode:
 	return V8SImode;
       case E_HImode:
@@ -12690,6 +13011,7 @@ aarch64_sve_arith_immediate_p (rtx x, bool negate_p)
   HOST_WIDE_INT val = INTVAL (elt);
   if (negate_p)
     val = -val;
+  val &= GET_MODE_MASK (GET_MODE_INNER (GET_MODE (x)));
 
   if (val & 0xff)
     return IN_RANGE (val, 0, 0xff);
@@ -12783,6 +13105,9 @@ aarch64_sve_float_mul_immediate_p (rtx x)
 	  && real_equal (CONST_DOUBLE_REAL_VALUE (elt), &dconsthalf));
 }
 
+/* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
+   for the Advanced SIMD operation described by WHICH and INSN.  If INFO
+   is nonnull, use it to describe valid immediates.  */
 static bool
 aarch64_advsimd_valid_immediate_hs (unsigned int val32,
 				    simd_immediate_info *info,
@@ -12829,9 +13154,9 @@ aarch64_advsimd_valid_immediate_hs (unsigned int val32,
   return false;
 }
 
-/* Return true if replicating VAL64 is a valid immediate for an AdvSIMD
-   MOVI or MVNI instruction.  If INFO is nonnull, use it to describe valid
-   immediates.  */
+/* Return true if replicating VAL64 is a valid immediate for the
+   Advanced SIMD operation described by WHICH.  If INFO is nonnull,
+   use it to describe valid immediates.  */
 static bool
 aarch64_advsimd_valid_immediate (unsigned HOST_WIDE_INT val64,
 				 simd_immediate_info *info,
@@ -12863,6 +13188,7 @@ aarch64_advsimd_valid_immediate (unsigned HOST_WIDE_INT val64,
 	  return true;
 	}
     }
+
   /* Try using a bit-to-bytemask.  */
   if (which == AARCH64_CHECK_MOV)
     {
@@ -12885,6 +13211,7 @@ aarch64_advsimd_valid_immediate (unsigned HOST_WIDE_INT val64,
 
 /* Return true if replicating VAL64 gives a valid immediate for an SVE MOV
    instruction.  If INFO is nonnull, use it to describe valid immediates.  */
+
 static bool
 aarch64_sve_valid_immediate (unsigned HOST_WIDE_INT val64,
 			     simd_immediate_info *info)
@@ -12928,8 +13255,9 @@ aarch64_sve_valid_immediate (unsigned HOST_WIDE_INT val64,
   return false;
 }
 
-/* Return true if OP is a valid SIMD immediate.  If INFO is nonnull,
-   use it to describe valid immediates.  */
+/* Return true if OP is a valid SIMD immediate for the operation
+   described by WHICH.  If INFO is nonnull, use it to describe valid
+   immediates.  */
 bool
 aarch64_simd_valid_immediate (rtx op, simd_immediate_info *info,
 			      enum simd_immediate_check which)
@@ -12983,7 +13311,8 @@ aarch64_simd_valid_immediate (rtx op, simd_immediate_info *info,
 
   scalar_int_mode elt_int_mode = int_mode_for_mode (elt_mode).require ();
 
-  /* Splat vector constant out into a byte vector.  */
+  /* Expand the vector constant out into a byte vector, with the least
+     significant byte of the register first.  */
   auto_vec<unsigned char, 16> bytes;
   bytes.reserve (n_elts * elt_size);
   for (unsigned int i = 0; i < n_elts; i++)
@@ -13014,7 +13343,8 @@ aarch64_simd_valid_immediate (rtx op, simd_immediate_info *info,
     if (bytes[i] != bytes[i - 8])
       return false;
 
-  /* Get the repeating 8-byte value as an integer.  */
+  /* Get the repeating 8-byte value as an integer.  No endian correction
+     is needed here because bytes is already in lsb-first order.  */
   unsigned HOST_WIDE_INT val64 = 0;
   for (unsigned int i = 0; i < 8; i++)
     val64 |= ((unsigned HOST_WIDE_INT) bytes[i % nbytes]
@@ -13126,7 +13456,8 @@ Architecture    3   2   1   0           3   2   1   0
 
 Low Mask:         { 2, 3 }                { 0, 1 }
 High Mask:        { 0, 1 }                { 2, 3 }
-*/
+
+   MODE Is the mode of the vector and NUNITS is the number of units in it.  */
 
 rtx
 aarch64_simd_vect_par_cnst_half (machine_mode mode, int nunits, bool high)
@@ -13206,12 +13537,13 @@ aarch64_simd_lane_bounds (rtx operand, HOST_WIDE_INT low, HOST_WIDE_INT high,
    of mode MODE, and return the result as an SImode rtx.  */
 
 rtx
-endian_lane_rtx (machine_mode mode, unsigned int n)
+aarch64_endian_lane_rtx (machine_mode mode, unsigned int n)
 {
   return gen_int_mode (ENDIAN_LANE_N (GET_MODE_NUNITS (mode), n), SImode);
 }
 
 /* Return TRUE if OP is a valid vector addressing mode.  */
+
 bool
 aarch64_simd_mem_operand_p (rtx op)
 {
@@ -13220,6 +13552,7 @@ aarch64_simd_mem_operand_p (rtx op)
 }
 
 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction.  */
+
 bool
 aarch64_sve_ld1r_operand_p (rtx op)
 {
@@ -13317,7 +13650,7 @@ aarch64_simd_emit_reg_reg_move (rtx *operands, machine_mode mode,
 int
 aarch64_simd_attr_length_rglist (machine_mode mode)
 {
-  /* This is only used (and only meaningful) for AdvSIMD, not SVE.  */
+  /* This is only used (and only meaningful) for Advanced SIMD, not SVE.  */
   return (GET_MODE_SIZE (mode).to_constant () / UNITS_PER_VREG) * 4;
 }
 
@@ -13360,8 +13693,8 @@ aarch64_simd_vector_alignment_reachable (const_tree type, bool is_packed)
     return false;
 
   /* We guarantee alignment for vectors up to 128-bits.  */
-  if (tree_int_cst_compare (TYPE_SIZE (type),
-			    bitsize_int (BIGGEST_ALIGNMENT)) > 0)
+  unsigned int align = aarch64_vectorize_preferred_vector_alignment (type);
+  if (align > BIGGEST_ALIGNMENT)
     return false;
 
   /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned.  */
@@ -14586,7 +14919,7 @@ aarch64_output_scalar_simd_mov_immediate (rtx immediate, scalar_int_mode mode)
 }
 
 /* Return the output string to use for moving immediate CONST_VECTOR
-   into an SVE register of mode MODE.  */
+   into an SVE register.  */
 
 char *
 aarch64_output_sve_mov_immediate (rtx const_vector)
@@ -14615,7 +14948,7 @@ aarch64_output_sve_mov_immediate (rtx const_vector)
       else
 	{
 	  const int buf_size = 20;
-	  char float_buf[buf_size] = { 0 };
+	  char float_buf[buf_size] = {};
 	  real_to_decimal_for_mode (float_buf,
 				    CONST_DOUBLE_REAL_VALUE (info.value),
 				    buf_size, buf_size, 1, info.elt_mode);
@@ -14638,7 +14971,7 @@ char *
 aarch64_output_ptrue (machine_mode mode, char suffix)
 {
   unsigned int nunits;
-  static char buf[sizeof ("ptrue\t%0.N, vlNNN")];
+  static char buf[sizeof ("ptrue\t%0.N, vlNNNNN")];
   if (GET_MODE_NUNITS (mode).is_constant (&nunits))
     snprintf (buf, sizeof (buf), "ptrue\t%%0.%c, vl%d", suffix, nunits);
   else
@@ -14757,6 +15090,9 @@ aarch64_expand_vec_perm_1 (rtx target, rtx op0, rtx op1, rtx sel)
     }
 }
 
+/* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
+   NELT is the number of elements in the vector.  */
+
 void
 aarch64_expand_vec_perm (rtx target, rtx op0, rtx op1, rtx sel,
 			 unsigned int nelt)
@@ -14804,11 +15140,11 @@ aarch64_expand_sve_vec_perm (rtx target, rtx op0, rtx op1, rtx sel)
   /* Enforced by the pattern condition.  */
   int nunits = GET_MODE_NUNITS (sel_mode).to_constant ();
 
-  /* Note: All sel indexes are wrapped when they are beyond the size of the
-	   two value vectors.  SVE TBL will produce 0 for any out of range
-	   sel indexes.  Therefore we need to modulo all the sel indexes to
-	   ensure they are all in range.  */
-
+  /* Note: vec_perm indices are supposed to wrap when they go beyond the
+     size of the two value vectors, i.e. the upper bits of the indices
+     are effectively ignored.  SVE TBL instead produces 0 for any
+     out-of-range indices, so we need to modulo all the vec_perm indices
+     to ensure they are all in range.  */
   rtx sel_reg = force_reg (sel_mode, sel);
 
   /* Check if the sel only references the first values vector.  */
@@ -14834,12 +15170,16 @@ aarch64_expand_sve_vec_perm (rtx target, rtx op0, rtx op1, rtx sel)
   rtx res0 = gen_reg_rtx (data_mode);
   rtx res1 = gen_reg_rtx (data_mode);
   rtx neg_num_elems = aarch64_simd_gen_const_vector_dup (sel_mode, -nunits);
-  rtx max_sel = aarch64_simd_gen_const_vector_dup (sel_mode, (2 * nunits) - 1);
-
-  rtx sel_mod = expand_simple_binop (sel_mode, AND, sel_reg, max_sel,
+  if (!const_vec_p (sel)
+      || !aarch64_const_vec_all_in_range_p (sel, 0, 2 * nunits - 1))
+    {
+      rtx max_sel = aarch64_simd_gen_const_vector_dup (sel_mode,
+						       2 * nunits - 1);
+      sel_reg = expand_simple_binop (sel_mode, AND, sel_reg, max_sel,
 				     NULL, 0, OPTAB_DIRECT);
-  emit_unspec2 (res0, UNSPEC_TBL, op0, sel_mod);
-  rtx sel_sub = expand_simple_binop (sel_mode, PLUS, sel_mod, neg_num_elems,
+    }
+  emit_unspec2 (res0, UNSPEC_TBL, op0, sel_reg);
+  rtx sel_sub = expand_simple_binop (sel_mode, PLUS, sel_reg, neg_num_elems,
 				     NULL, 0, OPTAB_DIRECT);
   emit_unspec2 (res1, UNSPEC_TBL, op1, sel_sub);
   if (GET_MODE_CLASS (data_mode) == MODE_VECTOR_INT)
@@ -14883,6 +15223,8 @@ aarch64_evpc_trn (struct expand_vec_perm_d *d)
 
   in0 = d->op0;
   in1 = d->op1;
+  /* We don't need a big-endian lane correction for SVE; see the comment
+     at the head of aarch64-sve.md for details.  */
   if (BYTES_BIG_ENDIAN && d->vec_flags == VEC_ADVSIMD)
     {
       x = in0, in0 = in1, in1 = x;
@@ -14929,6 +15271,8 @@ aarch64_evpc_uzp (struct expand_vec_perm_d *d)
 
   in0 = d->op0;
   in1 = d->op1;
+  /* We don't need a big-endian lane correction for SVE; see the comment
+     at the head of aarch64-sve.md for details.  */
   if (BYTES_BIG_ENDIAN && d->vec_flags == VEC_ADVSIMD)
     {
       x = in0, in0 = in1, in1 = x;
@@ -14980,6 +15324,8 @@ aarch64_evpc_zip (struct expand_vec_perm_d *d)
 
   in0 = d->op0;
   in1 = d->op1;
+  /* We don't need a big-endian lane correction for SVE; see the comment
+     at the head of aarch64-sve.md for details.  */
   if (BYTES_BIG_ENDIAN && d->vec_flags == VEC_ADVSIMD)
     {
       x = in0, in0 = in1, in1 = x;
@@ -15020,8 +15366,10 @@ aarch64_evpc_ext (struct expand_vec_perm_d *d)
     return true;
 
   /* The case where (location == 0) is a no-op for both big- and little-endian,
-     and is removed by the mid-end at optimization levels -O1 and higher.  */
+     and is removed by the mid-end at optimization levels -O1 and higher.
 
+     We don't need a big-endian lane correction for SVE; see the comment
+     at the head of aarch64-sve.md for details.  */
   if (BYTES_BIG_ENDIAN && location != 0 && d->vec_flags == VEC_ADVSIMD)
     {
       /* After setup, we want the high elements of the first vector (stored
@@ -15163,6 +15511,8 @@ aarch64_evpc_tbl (struct expand_vec_perm_d *d)
   return true;
 }
 
+/* Try to implement D using an SVE TBL instruction.  */
+
 static bool
 aarch64_evpc_sve_tbl (struct expand_vec_perm_d *d)
 {
@@ -15178,9 +15528,6 @@ aarch64_evpc_sve_tbl (struct expand_vec_perm_d *d)
   for (unsigned int i = 0; i < nelt; ++i)
     rperm[i] = GEN_INT (d->perm[i]);
   rtx sel = gen_rtx_CONST_VECTOR (sel_mode, gen_rtvec_v (nelt, rperm));
-  if (!aarch64_sve_vec_perm_operand (sel, sel_mode))
-    sel = force_reg (sel_mode, sel);
-
   aarch64_expand_sve_vec_perm (d->target, d->op0, d->op1, sel);
   return true;
 }
@@ -15224,7 +15571,8 @@ aarch64_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
   return false;
 }
 
-/* Expand a vec_perm_const pattern.  */
+/* Expand a vec_perm_const pattern with the operands given by TARGET,
+   OP0, OP1 and SEL.  NELT is the number of elements in the vector.  */
 
 bool
 aarch64_expand_vec_perm_const (rtx target, rtx op0, rtx op1, rtx sel,
@@ -15324,6 +15672,9 @@ aarch64_vectorize_vec_perm_const_ok (machine_mode vmode, vec_perm_indices sel)
   return ret;
 }
 
+/* Generate a byte permute mask for a register of mode MODE,
+   which has NUNITS units.  */
+
 rtx
 aarch64_reverse_mask (machine_mode mode, unsigned int nunits)
 {
@@ -17024,6 +17375,28 @@ aarch64_sched_can_speculate_insn (rtx_insn *insn)
     }
 }
 
+/* It has been decided that to allow up to 1kb of outgoing argument
+   space to be allocated w/o probing.  If more than 1kb of outgoing
+   argment space is allocated, then it must be probed and the last
+   probe must occur no more than 1kbyte away from the end of the
+   allocated space.
+
+   This implies that the residual part of an alloca allocation may
+   need probing in cases where the generic code might not otherwise
+   think a probe is needed.
+
+   This target hook returns TRUE when allocating RESIDUAL bytes of
+   alloca space requires an additional probe, otherwise FALSE is
+   returned.  */
+
+static bool
+aarch64_stack_clash_protection_final_dynamic_probe (rtx residual)
+{
+  return (residual == CONST0_RTX (Pmode)
+	  || GET_CODE (residual) != CONST_INT
+	  || INTVAL (residual) >= 1024);
+}
+
 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES.  */
 
 static int
@@ -17043,6 +17416,19 @@ aarch64_compute_pressure_classes (reg_class *classes)
   return i;
 }
 
+/* Implement TARGET_CAN_CHANGE_MODE_CLASS.  */
+
+static bool
+aarch64_can_change_mode_class (machine_mode from,
+			       machine_mode to, reg_class_t)
+{
+  /* See the comment at the head of aarch64-sve.md for details.  */
+  if (BYTES_BIG_ENDIAN
+      && (aarch64_sve_data_mode_p (from) != aarch64_sve_data_mode_p (to)))
+    return false;
+  return true;
+}
+
 /* Target-specific selftests.  */
 
 #if CHECKING_P
@@ -17292,9 +17678,6 @@ aarch64_libgcc_floating_mode_supported_p
 
 #undef TARGET_SECONDARY_RELOAD
 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
-#undef TARGET_CANNOT_SUBSTITUTE_MEM_EQUIV_P
-#define TARGET_CANNOT_SUBSTITUTE_MEM_EQUIV_P \
-  aarch64_cannot_substitute_mem_equiv_p
 
 #undef TARGET_SHIFT_TRUNCATION_MASK
 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
@@ -17517,9 +17900,16 @@ aarch64_libgcc_floating_mode_supported_p
 #undef TARGET_CONSTANT_ALIGNMENT
 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
 
+#undef TARGET_STACK_CLASH_PROTECTION_FINAL_DYNAMIC_PROBE
+#define TARGET_STACK_CLASH_PROTECTION_FINAL_DYNAMIC_PROBE \
+  aarch64_stack_clash_protection_final_dynamic_probe
+
 #undef TARGET_COMPUTE_PRESSURE_CLASSES
 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
 
+#undef TARGET_CAN_CHANGE_MODE_CLASS
+#define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
+
 #if CHECKING_P
 #undef TARGET_RUN_TARGET_SELFTESTS
 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index 6b9ffae6823..5816bc6c1e4 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -111,6 +111,9 @@
 
 #define STRUCTURE_SIZE_BOUNDARY		8
 
+/* Heap alignment (same as BIGGEST_ALIGNMENT and STACK_BOUNDARY).  */
+#define MALLOC_ABI_ALIGNMENT  128
+
 /* Defined by the ABI */
 #define WCHAR_TYPE "unsigned int"
 #define WCHAR_TYPE_SIZE			32
@@ -136,15 +139,16 @@ extern unsigned aarch64_architecture_version;
 #define AARCH64_FL_CRC        (1 << 3)	/* Has CRC.  */
 /* ARMv8.1-A architecture extensions.  */
 #define AARCH64_FL_LSE	      (1 << 4)  /* Has Large System Extensions.  */
-#define AARCH64_FL_RDMA	      (1 << 5)  /* Has Round Double Multiply Add.  */
-#define AARCH64_FL_V8_1	      (1 << 6)  /* Has ARMv8.1-A extensions.  */
+#define AARCH64_FL_RDMA       (1 << 5)  /* Has Round Double Multiply Add.  */
+#define AARCH64_FL_V8_1       (1 << 6)  /* Has ARMv8.1-A extensions.  */
 /* ARMv8.2-A architecture extensions.  */
-#define AARCH64_FL_V8_2	      (1 << 8)  /* Has ARMv8.2-A features.  */
+#define AARCH64_FL_V8_2       (1 << 8)  /* Has ARMv8.2-A features.  */
 #define AARCH64_FL_F16	      (1 << 9)  /* Has ARMv8.2-A FP16 extensions.  */
 #define AARCH64_FL_SVE        (1 << 10) /* Has Scalable Vector Extensions.  */
 /* ARMv8.3-A architecture extensions.  */
-#define AARCH64_FL_V8_3	      (1 << 11)  /* Has ARMv8.3-A features.  */
-#define AARCH64_FL_RCPC	      (1 << 12)  /* Has support for RCpc model.  */
+#define AARCH64_FL_V8_3       (1 << 11)  /* Has ARMv8.3-A features.  */
+#define AARCH64_FL_RCPC       (1 << 12)  /* Has support for RCpc model.  */
+#define AARCH64_FL_DOTPROD    (1 << 13)  /* Has ARMv8.2-A Dot Product ins.  */
 
 /* Has FP and SIMD.  */
 #define AARCH64_FL_FPSIMD     (AARCH64_FL_FP | AARCH64_FL_SIMD)
@@ -174,6 +178,7 @@ extern unsigned aarch64_architecture_version;
 #define AARCH64_ISA_F16		   (aarch64_isa_flags & AARCH64_FL_F16)
 #define AARCH64_ISA_SVE            (aarch64_isa_flags & AARCH64_FL_SVE)
 #define AARCH64_ISA_V8_3	   (aarch64_isa_flags & AARCH64_FL_V8_3)
+#define AARCH64_ISA_DOTPROD	   (aarch64_isa_flags & AARCH64_FL_DOTPROD)
 
 /* Crypto is an optional extension to AdvSIMD.  */
 #define TARGET_CRYPTO (TARGET_SIMD && AARCH64_ISA_CRYPTO)
@@ -188,6 +193,9 @@ extern unsigned aarch64_architecture_version;
 #define TARGET_FP_F16INST (TARGET_FLOAT && AARCH64_ISA_F16)
 #define TARGET_SIMD_F16INST (TARGET_SIMD && AARCH64_ISA_F16)
 
+/* Dot Product is an optional extension to AdvSIMD enabled through +dotprod.  */
+#define TARGET_DOTPROD (TARGET_SIMD && AARCH64_ISA_DOTPROD)
+
 /* SVE instructions, enabled through +sve.  */
 #define TARGET_SVE (AARCH64_ISA_SVE)
 
@@ -273,7 +281,7 @@ extern unsigned aarch64_architecture_version;
     0, 0, 0, 0,   0, 0, 0, 0,   /* V8 - V15 */		\
     0, 0, 0, 0,   0, 0, 0, 0,   /* V16 - V23 */         \
     0, 0, 0, 0,   0, 0, 0, 0,   /* V24 - V31 */         \
-    1, 1, 1,			/* SFP, AP, CC */	\
+    1, 1, 1, 1,			/* SFP, AP, CC, VG */	\
     0, 0, 0, 0,   0, 0, 0, 0,   /* P0 - P7 */           \
     0, 0, 0, 0,   0, 0, 0, 0,   /* P8 - P15 */          \
     1,				/* FFRT */		\
@@ -289,7 +297,7 @@ extern unsigned aarch64_architecture_version;
     0, 0, 0, 0,   0, 0, 0, 0,	/* V8 - V15 */		\
     1, 1, 1, 1,   1, 1, 1, 1,   /* V16 - V23 */         \
     1, 1, 1, 1,   1, 1, 1, 1,   /* V24 - V31 */         \
-    1, 1, 1,			/* SFP, AP, CC */	\
+    1, 1, 1, 1,			/* SFP, AP, CC, VG */	\
     1, 1, 1, 1,   1, 1, 1, 1,	/* P0 - P7 */		\
     1, 1, 1, 1,   1, 1, 1, 1,	/* P8 - P15 */		\
     1,				/* FFRT */		\
@@ -305,7 +313,7 @@ extern unsigned aarch64_architecture_version;
     "v8",  "v9",  "v10", "v11", "v12", "v13", "v14", "v15",	\
     "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",	\
     "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",	\
-    "sfp", "ap",  "cc",						\
+    "sfp", "ap",  "cc",  "vg",					\
     "p0",  "p1",  "p2",  "p3",  "p4",  "p5",  "p6",  "p7",	\
     "p8",  "p9",  "p10", "p11", "p12", "p13", "p14", "p15",	\
     "ffrt",							\
@@ -353,9 +361,9 @@ extern unsigned aarch64_architecture_version;
   (epilogue_completed && (REGNO) == LR_REGNUM)
 
 /* EXIT_IGNORE_STACK should be nonzero if, when returning from a function,
-   the stack pointer does not matter.  The value is tested only in
-   functions that have frame pointers.  */
-#define EXIT_IGNORE_STACK	1
+   the stack pointer does not matter.  This is only true if the function
+   uses alloca.  */
+#define EXIT_IGNORE_STACK	(cfun->calls_alloca)
 
 #define STATIC_CHAIN_REGNUM		R18_REGNUM
 #define HARD_FRAME_POINTER_REGNUM	R29_REGNUM
@@ -503,10 +511,10 @@ enum reg_class
   { 0x00000000, 0x0000ffff, 0x00000000 },       /* FP_LO_REGS  */	\
   { 0x00000000, 0xffffffff, 0x00000000 },       /* FP_REGS  */		\
   { 0xffffffff, 0xffffffff, 0x00000003 },	/* POINTER_AND_FP_REGS */\
-  { 0x00000000, 0x00000000, 0x000007f8 },	/* PR_LO_REGS */	\
-  { 0x00000000, 0x00000000, 0x0007f800 },	/* PR_HI_REGS */	\
-  { 0x00000000, 0x00000000, 0x0007fff8 },	/* PR_REGS */		\
-  { 0xffffffff, 0xffffffff, 0x000fffff }	/* ALL_REGS */		\
+  { 0x00000000, 0x00000000, 0x00000ff0 },	/* PR_LO_REGS */	\
+  { 0x00000000, 0x00000000, 0x000ff000 },	/* PR_HI_REGS */	\
+  { 0x00000000, 0x00000000, 0x000ffff0 },	/* PR_REGS */		\
+  { 0xffffffff, 0xffffffff, 0x001fffff }	/* ALL_REGS */		\
 }
 
 #define REGNO_REG_CLASS(REGNO)	aarch64_regno_regclass (REGNO)
@@ -622,6 +630,9 @@ struct GTY (()) aarch64_frame
   /* The size of the stack adjustment after saving callee-saves.  */
   poly_int64 final_adjust;
 
+  /* Store FP,LR and setup a frame pointer.  */
+  bool emit_frame_chain;
+
   unsigned wb_candidate1;
   unsigned wb_candidate2;
 
@@ -989,12 +1000,18 @@ extern tree aarch64_fp16_ptr_type_node;
 
 #ifndef USED_FOR_TARGET
 extern poly_uint16 aarch64_sve_vg;
+
+/* The number of bits and bytes in an SVE vector.  */
 #define BITS_PER_SVE_VECTOR (poly_uint16 (aarch64_sve_vg * 64))
 #define BYTES_PER_SVE_VECTOR (poly_uint16 (aarch64_sve_vg * 8))
+
+/* The number of bytes in an SVE predicate.  */
 #define BYTES_PER_SVE_PRED aarch64_sve_vg
+
+/* The SVE mode for a vector of bytes.  */
 #define SVE_BYTE_MODE V32QImode
 
-/* Maximum number of bytes in a fixed-size vector.  This is 256 bytes
+/* The maximum number of bytes in a fixed-size vector.  This is 256 bytes
    (for -msve-vector-bits=2048) multiplied by the maximum number of
    vectors in a structure mode (4).
 
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 5be96aaa92d..6a15ff0b61d 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -63,18 +63,15 @@
     (SFP_REGNUM		64)
     (AP_REGNUM		65)
     (CC_REGNUM		66)
-    (P0_REGNUM		67)
-    (P7_REGNUM		74)
-    (P15_REGNUM		82)
-    (FFRT_REGNUM	83)
+    ;; Defined only to make the DWARF description simpler.
+    (VG_REGNUM		67)
+    (P0_REGNUM		68)
+    (P7_REGNUM		75)
+    (P15_REGNUM		83)
+    (FFRT_REGNUM	84)
   ]
 )
 
-(define_c_enum "param" [
-  ; Parameter used for poly_ints.  Must be first.
-  CPARAM_VQ
-])
-
 (define_c_enum "unspec" [
     UNSPEC_AUTI1716
     UNSPEC_AUTISP
@@ -170,7 +167,6 @@
     UNSPEC_PACK
     UNSPEC_FLOAT_CONVERT
     UNSPEC_WHILE_LO
-    UNSPEC_PRED_MOVE
     UNSPEC_CLASTB
     UNSPEC_LDFF1
     UNSPEC_READ_NF
@@ -901,8 +897,7 @@
     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
       operands[1] = force_reg (<MODE>mode, operands[1]);
 
-    if (CONSTANT_P (operands[1])
-	&& !CONST_INT_P (operands[1]))
+    if (GET_CODE (operands[1]) == CONST_POLY_INT)
       {
 	aarch64_expand_mov_immediate (operands[0], operands[1]);
 	DONE;
@@ -1616,8 +1611,8 @@
   else if (operands[0] == stack_pointer_rtx
 	   && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
     {
-      aarch64_split_add_offset (<MODE>mode, operands[0], NULL_RTX, NULL_RTX,
-				operands[1], operands[2]);
+      aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
+				operands[2], NULL_RTX, NULL_RTX);
       DONE;
     }
 })
@@ -1734,8 +1729,8 @@
    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
   [(const_int 0)]
   {
-    aarch64_split_add_offset (<MODE>mode, operands[0], operands[0], NULL_RTX,
-			      operands[1], operands[2]);
+    aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
+			      operands[2], operands[0], NULL_RTX);
     DONE;
   }
   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
@@ -4123,7 +4118,7 @@
 (define_expand "<optab><mode>3"
   [(set (match_operand:GPI 0 "register_operand")
 	(ASHIFT:GPI (match_operand:GPI 1 "register_operand")
-		    (match_operand:QI 2 "nonmemory_operand")))]
+		    (match_operand:QI 2 "aarch64_reg_or_imm")))]
   ""
   {
     if (CONST_INT_P (operands[2]))
@@ -4159,7 +4154,7 @@
 (define_expand "rotr<mode>3"
   [(set (match_operand:GPI 0 "register_operand")
 	(rotatert:GPI (match_operand:GPI 1 "register_operand")
-		      (match_operand:QI 2 "nonmemory_operand")))]
+		      (match_operand:QI 2 "aarch64_reg_or_imm")))]
   ""
   {
     if (CONST_INT_P (operands[2]))
@@ -4179,7 +4174,7 @@
 (define_expand "rotl<mode>3"
   [(set (match_operand:GPI 0 "register_operand")
 	(rotatert:GPI (match_operand:GPI 1 "register_operand")
-		      (match_operand:QI 2 "nonmemory_operand")))]
+		      (match_operand:QI 2 "aarch64_reg_or_imm")))]
   ""
   {
     /* (SZ - cnt) % SZ == -cnt % SZ */
@@ -5014,11 +5009,37 @@
   [(set_attr "type" "f_cvt")]
 )
 
-(define_insn "<optab>_trunc<GPF_F16:mode><GPI:mode>2"
+;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
+;; and making r = w more expensive
+
+(define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
+  [(set (match_operand:GPI 0 "register_operand" "=?r,w")
+	(FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
+  "TARGET_FLOAT"
+  "@
+   fcvtz<su>\t%<w>0, %<s>1
+   fcvtz<su>\t%<s>0, %<s>1"
+  [(set_attr "type" "f_cvtf2i,neon_fp_to_int_s")]
+)
+
+;; Convert HF -> SI or DI
+
+(define_insn "<optab>_trunchf<GPI:mode>2"
+  [(set (match_operand:GPI 0 "register_operand" "=r")
+	(FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
+  "TARGET_FP_F16INST"
+  "fcvtz<su>\t%<w>0, %h1"
+  [(set_attr "type" "f_cvtf2i")]
+)
+
+;; Convert DF -> SI or SF -> DI which can only be accomplished with
+;; input in a fp register and output in a integer register
+
+(define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
   [(set (match_operand:GPI 0 "register_operand" "=r")
-	(FIXUORS:GPI (match_operand:GPF_F16 1 "register_operand" "w")))]
+	(FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
   "TARGET_FLOAT"
-  "fcvtz<su>\t%<GPI:w>0, %<GPF_F16:s>1"
+  "fcvtz<su>\t%<w>0, %<fpw>1"
   [(set_attr "type" "f_cvtf2i")]
 )
 
@@ -5320,7 +5341,9 @@
 (define_expand "lrint<GPF:mode><GPI:mode>2"
   [(match_operand:GPI 0 "register_operand")
    (match_operand:GPF 1 "register_operand")]
-  "TARGET_FLOAT"
+  "TARGET_FLOAT
+   && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
+   || !flag_trapping_math || flag_fp_int_builtin_inexact)"
 {
   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
@@ -5783,7 +5806,7 @@
 )
 
 (define_insn "probe_stack_range"
-  [(set (match_operand:DI 0 "register_operand" "=r")
+  [(set (match_operand:DI 0 "register_operand" "=rk")
 	(unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
 			     (match_operand:DI 2 "register_operand" "r")]
 			      UNSPECV_PROBE_STACK_RANGE))]
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index d7b30b0e5ee..96e740f91a7 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -31541,6 +31541,99 @@ vminnmvq_f16 (float16x8_t __a)
 
 #pragma GCC pop_options
 
+/* AdvSIMD Dot Product intrinsics.  */
+
+#pragma GCC push_options
+#pragma GCC target ("arch=armv8.2-a+dotprod")
+
+__extension__ extern __inline uint32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_u32 (uint32x2_t __r, uint8x8_t __a, uint8x8_t __b)
+{
+  return __builtin_aarch64_udotv8qi_uuuu (__r, __a, __b);
+}
+
+__extension__ extern __inline uint32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_u32 (uint32x4_t __r, uint8x16_t __a, uint8x16_t __b)
+{
+  return __builtin_aarch64_udotv16qi_uuuu (__r, __a, __b);
+}
+
+__extension__ extern __inline int32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_s32 (int32x2_t __r, int8x8_t __a, int8x8_t __b)
+{
+  return __builtin_aarch64_sdotv8qi (__r, __a, __b);
+}
+
+__extension__ extern __inline int32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_s32 (int32x4_t __r, int8x16_t __a, int8x16_t __b)
+{
+  return __builtin_aarch64_sdotv16qi (__r, __a, __b);
+}
+
+__extension__ extern __inline uint32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_lane_u32 (uint32x2_t __r, uint8x8_t __a, uint8x8_t __b, const int __index)
+{
+  return __builtin_aarch64_udot_lanev8qi_uuuus (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline uint32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_laneq_u32 (uint32x2_t __r, uint8x8_t __a, uint8x16_t __b,
+		const int __index)
+{
+  return __builtin_aarch64_udot_laneqv8qi_uuuus (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline uint32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_lane_u32 (uint32x4_t __r, uint8x16_t __a, uint8x8_t __b,
+		const int __index)
+{
+  return __builtin_aarch64_udot_lanev16qi_uuuus (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline uint32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_laneq_u32 (uint32x4_t __r, uint8x16_t __a, uint8x16_t __b,
+		 const int __index)
+{
+  return __builtin_aarch64_udot_laneqv16qi_uuuus (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline int32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_lane_s32 (int32x2_t __r, int8x8_t __a, int8x8_t __b, const int __index)
+{
+  return __builtin_aarch64_sdot_lanev8qi (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline int32x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdot_laneq_s32 (int32x2_t __r, int8x8_t __a, int8x16_t __b, const int __index)
+{
+  return __builtin_aarch64_sdot_laneqv8qi (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline int32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_lane_s32 (int32x4_t __r, int8x16_t __a, int8x8_t __b, const int __index)
+{
+  return __builtin_aarch64_sdot_lanev16qi (__r, __a, __b, __index);
+}
+
+__extension__ extern __inline int32x4_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vdotq_laneq_s32 (int32x4_t __r, int8x16_t __a, int8x16_t __b, const int __index)
+{
+  return __builtin_aarch64_sdot_laneqv16qi (__r, __a, __b, __index);
+}
+#pragma GCC pop_options
+
 #undef __aarch64_vget_lane_any
 
 #undef __aarch64_vdup_lane_any
diff --git a/gcc/config/aarch64/constraints.md b/gcc/config/aarch64/constraints.md
index 0f7e3099a4c..2a8722c4c86 100644
--- a/gcc/config/aarch64/constraints.md
+++ b/gcc/config/aarch64/constraints.md
@@ -160,14 +160,14 @@
 ;; "ad" for "(A)DDVL/ADDPL (D)irect".
 (define_constraint "Uad"
   "@internal
- A constraint that matches a VG-based constant that can be added by
- a single ADDVL or ADDPL."
+   A constraint that matches a VG-based constant that can be added by
+   a single ADDVL or ADDPL."
  (match_operand 0 "aarch64_sve_addvl_addpl_immediate"))
 
 (define_constraint "Ua1"
   "@internal
- A constraint that matches a VG-based constant that can be added by
- using multiple instructions, with one temporary register."
+   A constraint that matches a VG-based constant that can be added by
+   using multiple instructions, with one temporary register."
  (match_operand 0 "aarch64_split_add_offset_immediate"))
 
 (define_memory_constraint "Q"
@@ -297,7 +297,7 @@
 
 (define_constraint "Dm"
   "@internal
- A constraint that matches a vector of immediate minus one."
+   A constraint that matches a vector of immediate minus one."
  (and (match_code "const,const_vector")
       (match_test "op == CONST1_RTX (GET_MODE (op))")))
 
@@ -322,65 +322,65 @@
 
 (define_constraint "Dv"
   "@internal
- A constraint that matches a VG-based constant that can be loaded by
- a single CNT[BHWD]."
+   A constraint that matches a VG-based constant that can be loaded by
+   a single CNT[BHWD]."
  (match_operand 0 "aarch64_sve_cnt_immediate"))
 
 (define_constraint "vsa"
   "@internal
- A constraint that matches a signed immediate operand valid for SVE
- arithmetic instructions."
+   A constraint that matches an immediate operand valid for SVE
+   arithmetic instructions."
  (match_operand 0 "aarch64_sve_arith_immediate"))
 
 (define_constraint "vsc"
   "@internal
- A constraint that matches a signed immediate operand valid for SVE
- CMP instructions."
+   A constraint that matches a signed immediate operand valid for SVE
+   CMP instructions."
  (match_operand 0 "aarch64_sve_cmp_vsc_immediate"))
 
 (define_constraint "vsd"
   "@internal
- A constraint that matches an unsigned immediate operand valid for SVE
- CMP instructions."
+   A constraint that matches an unsigned immediate operand valid for SVE
+   CMP instructions."
  (match_operand 0 "aarch64_sve_cmp_vsd_immediate"))
 
 (define_constraint "vsi"
   "@internal
- A constraint that matches a vector count operand valid for SVE INC and
- DEC instructions."
+   A constraint that matches a vector count operand valid for SVE INC and
+   DEC instructions."
  (match_operand 0 "aarch64_sve_inc_dec_immediate"))
 
 (define_constraint "vsn"
   "@internal
- A constraint that matches a signed immediate operand whose negative
- is valid for SVE SUB instructions."
+   A constraint that matches an immediate operand whose negative
+   is valid for SVE SUB instructions."
  (match_operand 0 "aarch64_sve_sub_arith_immediate"))
 
 (define_constraint "vsl"
   "@internal
- A constraint that matches an immediate operand valid for SVE logical
- operations."
+   A constraint that matches an immediate operand valid for SVE logical
+   operations."
  (match_operand 0 "aarch64_sve_logical_immediate"))
 
 (define_constraint "vsm"
   "@internal
- A constraint that matches an immediate operand valid for SVE MUL
- operations."
+   A constraint that matches an immediate operand valid for SVE MUL
+   operations."
  (match_operand 0 "aarch64_sve_mul_immediate"))
 
 (define_constraint "vfa"
   "@internal
- A constraint that matches an immediate operand valid for SVE FADD
- and FSUB operations."
+   A constraint that matches an immediate operand valid for SVE FADD
+   and FSUB operations."
  (match_operand 0 "aarch64_sve_float_arith_immediate"))
 
 (define_constraint "vfm"
   "@internal
- A constraint that matches an imediate operand valid for SVE FMUL
- operations."
+   A constraint that matches an imediate operand valid for SVE FMUL
+   operations."
  (match_operand 0 "aarch64_sve_float_mul_immediate"))
 
 (define_constraint "vfn"
   "@internal
- A constraint that matches the negative of vfa"
+   A constraint that matches the negative of vfa"
  (match_operand 0 "aarch64_sve_float_arith_with_sub_immediate"))
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 4cb4babe024..2a4e26fb940 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -56,20 +56,20 @@
 ;; Iterator for all scalar floating point modes (SF, DF and TF)
 (define_mode_iterator GPF_TF [SF DF TF])
 
-;; Integer AdvSIMD modes.
+;; Integer Advanced SIMD modes.
 (define_mode_iterator VDQ_I [V8QI V16QI V4HI V8HI V2SI V4SI V2DI])
 
-;; AdvSIMD and scalar, 64 & 128-bit container, all integer modes
+;; Advanced SIMD and scalar, 64 & 128-bit container, all integer modes.
 (define_mode_iterator VSDQ_I [V8QI V16QI V4HI V8HI V2SI V4SI V2DI QI HI SI DI])
 
-;; AdvSIMD and scalar, 64 & 128-bit container: all AdvSIMD integer modes;
-;; 64-bit scalar integer mode
+;; Advanced SIMD and scalar, 64 & 128-bit container: all Advanced SIMD
+;; integer modes; 64-bit scalar integer mode.
 (define_mode_iterator VSDQ_I_DI [V8QI V16QI V4HI V8HI V2SI V4SI V2DI DI])
 
 ;; Double vector modes.
 (define_mode_iterator VD [V8QI V4HI V4HF V2SI V2SF])
 
-;; AdvSIMD, 64-bit container, all integer modes
+;; Advanced SIMD, 64-bit container, all integer modes.
 (define_mode_iterator VD_BHSI [V8QI V4HI V2SI])
 
 ;; 128 and 64-bit container; 8, 16, 32-bit vector integer modes
@@ -94,16 +94,16 @@
 ;; pointer-sized quantities.  Exactly one of the two alternatives will match.
 (define_mode_iterator PTR [(SI "ptr_mode == SImode") (DI "ptr_mode == DImode")])
 
-;; AdvSIMD Float modes suitable for moving, loading and storing.
+;; Advanced SIMD Float modes suitable for moving, loading and storing.
 (define_mode_iterator VDQF_F16 [V4HF V8HF V2SF V4SF V2DF])
 
-;; AdvSIMD Float modes.
+;; Advanced SIMD Float modes.
 (define_mode_iterator VDQF [V2SF V4SF V2DF])
 (define_mode_iterator VHSDF [(V4HF "TARGET_SIMD_F16INST")
 			     (V8HF "TARGET_SIMD_F16INST")
 			     V2SF V4SF V2DF])
 
-;; AdvSIMD Float modes, and DF.
+;; Advanced SIMD Float modes, and DF.
 (define_mode_iterator VHSDF_DF [(V4HF "TARGET_SIMD_F16INST")
 				(V8HF "TARGET_SIMD_F16INST")
 				V2SF V4SF V2DF DF])
@@ -113,7 +113,7 @@
 				  (HF "TARGET_SIMD_F16INST")
 				  SF DF])
 
-;; AdvSIMD single Float modes.
+;; Advanced SIMD single Float modes.
 (define_mode_iterator VDQSF [V2SF V4SF])
 
 ;; Quad vector Float modes with half/single elements.
@@ -122,16 +122,16 @@
 ;; Modes suitable to use as the return type of a vcond expression.
 (define_mode_iterator VDQF_COND [V2SF V2SI V4SF V4SI V2DF V2DI])
 
-;; All scalar and AdvSIMD Float modes.
+;; All scalar and Advanced SIMD Float modes.
 (define_mode_iterator VALLF [V2SF V4SF V2DF SF DF])
 
-;; AdvSIMD Float modes with 2 elements.
+;; Advanced SIMD Float modes with 2 elements.
 (define_mode_iterator V2F [V2SF V2DF])
 
-;; All AdvSIMD modes on which we support any arithmetic operations.
+;; All Advanced SIMD modes on which we support any arithmetic operations.
 (define_mode_iterator VALL [V8QI V16QI V4HI V8HI V2SI V4SI V2DI V2SF V4SF V2DF])
 
-;; All AdvSIMD modes suitable for moving, loading, and storing.
+;; All Advanced SIMD modes suitable for moving, loading, and storing.
 (define_mode_iterator VALL_F16 [V8QI V16QI V4HI V8HI V2SI V4SI V2DI
 				V4HF V8HF V2SF V4SF V2DF])
 
@@ -139,21 +139,21 @@
 (define_mode_iterator VALL_F16_NO_V2Q [V8QI V16QI V4HI V8HI V2SI V4SI
 				V4HF V8HF V2SF V4SF])
 
-;; All AdvSIMD modes barring HF modes, plus DI.
+;; All Advanced SIMD modes barring HF modes, plus DI.
 (define_mode_iterator VALLDI [V8QI V16QI V4HI V8HI V2SI V4SI V2DI V2SF V4SF V2DF DI])
 
-;; All AdvSIMD modes and DI.
+;; All Advanced SIMD modes and DI.
 (define_mode_iterator VALLDI_F16 [V8QI V16QI V4HI V8HI V2SI V4SI V2DI
 				  V4HF V8HF V2SF V4SF V2DF DI])
 
-;; All AdvSIMD modes, plus DI and DF.
+;; All Advanced SIMD modes, plus DI and DF.
 (define_mode_iterator VALLDIF [V8QI V16QI V4HI V8HI V2SI V4SI
 			       V2DI V4HF V8HF V2SF V4SF V2DF DI DF])
 
-;; AdvSIMD modes for Integer reduction across lanes.
+;; Advanced SIMD modes for Integer reduction across lanes.
 (define_mode_iterator VDQV [V8QI V16QI V4HI V8HI V4SI V2DI])
 
-;; AdvSIMD modes (except V2DI) for Integer reduction across lanes.
+;; Advanced SIMD modes (except V2DI) for Integer reduction across lanes.
 (define_mode_iterator VDQV_S [V8QI V16QI V4HI V8HI V4SI])
 
 ;; All double integer narrow-able modes.
@@ -162,8 +162,8 @@
 ;; All quad integer narrow-able modes.
 (define_mode_iterator VQN [V8HI V4SI V2DI])
 
-;; AdvSIMD and scalar 128-bit container: narrowable 16, 32, 64-bit integer
-;; modes
+;; Advanced SIMD and scalar 128-bit container: narrowable 16, 32, 64-bit
+;; integer modes
 (define_mode_iterator VSQN_HSDI [V8HI V4SI V2DI HI SI DI])
 
 ;; All quad integer widen-able modes.
@@ -172,54 +172,54 @@
 ;; Double vector modes for combines.
 (define_mode_iterator VDC [V8QI V4HI V4HF V2SI V2SF DI DF])
 
-;; AdvSIMD modes except double int.
+;; Advanced SIMD modes except double int.
 (define_mode_iterator VDQIF [V8QI V16QI V4HI V8HI V2SI V4SI V2SF V4SF V2DF])
 (define_mode_iterator VDQIF_F16 [V8QI V16QI V4HI V8HI V2SI V4SI
                                  V4HF V8HF V2SF V4SF V2DF])
 
-;; AdvSIMD modes for S type.
+;; Advanced SIMD modes for S type.
 (define_mode_iterator VDQ_SI [V2SI V4SI])
 
-;; AdvSIMD modes for S and D
+;; Advanced SIMD modes for S and D.
 (define_mode_iterator VDQ_SDI [V2SI V4SI V2DI])
 
-;; AdvSIMD modes for H, S and D
+;; Advanced SIMD modes for H, S and D.
 (define_mode_iterator VDQ_HSDI [(V4HI "TARGET_SIMD_F16INST")
 				(V8HI "TARGET_SIMD_F16INST")
 				V2SI V4SI V2DI])
 
-;; Scalar and AdvSIMD modes for S and D
+;; Scalar and Advanced SIMD modes for S and D.
 (define_mode_iterator VSDQ_SDI [V2SI V4SI V2DI SI DI])
 
-;; Scalar and AdvSIMD modes for S and D, AdvSIMD modes for H.
+;; Scalar and Advanced SIMD modes for S and D, Advanced SIMD modes for H.
 (define_mode_iterator VSDQ_HSDI [(V4HI "TARGET_SIMD_F16INST")
 				 (V8HI "TARGET_SIMD_F16INST")
 				 V2SI V4SI V2DI
 				 (HI "TARGET_SIMD_F16INST")
 				 SI DI])
 
-;; AdvSIMD modes for Q and H types.
+;; Advanced SIMD modes for Q and H types.
 (define_mode_iterator VDQQH [V8QI V16QI V4HI V8HI])
 
-;; AdvSIMD modes for H and S types.
+;; Advanced SIMD modes for H and S types.
 (define_mode_iterator VDQHS [V4HI V8HI V2SI V4SI])
 
-;; AdvSIMD modes for H, S and D types.
+;; Advanced SIMD modes for H, S and D types.
 (define_mode_iterator VDQHSD [V4HI V8HI V2SI V4SI V2DI])
 
-;; AdvSIMD and scalar integer modes for H and S
+;; Advanced SIMD and scalar integer modes for H and S.
 (define_mode_iterator VSDQ_HSI [V4HI V8HI V2SI V4SI HI SI])
 
-;; AdvSIMD and scalar 64-bit container: 16, 32-bit integer modes
+;; Advanced SIMD and scalar 64-bit container: 16, 32-bit integer modes.
 (define_mode_iterator VSD_HSI [V4HI V2SI HI SI])
 
-;; AdvSIMD 64-bit container: 16, 32-bit integer modes
+;; Advanced SIMD 64-bit container: 16, 32-bit integer modes.
 (define_mode_iterator VD_HSI [V4HI V2SI])
 
 ;; Scalar 64-bit container: 16, 32-bit integer modes
 (define_mode_iterator SD_HSI [HI SI])
 
-;; AdvSIMD 64-bit container: 16, 32-bit integer modes
+;; Advanced SIMD 64-bit container: 16, 32-bit integer modes.
 (define_mode_iterator VQ_HSI [V8HI V4SI])
 
 ;; All byte modes.
@@ -230,38 +230,45 @@
 
 (define_mode_iterator TX [TI TF])
 
-;; AdvSIMD opaque structure modes.
+;; Advanced SIMD opaque structure modes.
 (define_mode_iterator VSTRUCT [OI CI XI])
 
 ;; Double scalar modes
 (define_mode_iterator DX [DI DF])
 
-;; Modes available for AdvSIMD <f>mul lane operations.
+;; Modes available for Advanced SIMD <f>mul lane operations.
 (define_mode_iterator VMUL [V4HI V8HI V2SI V4SI
 			    (V4HF "TARGET_SIMD_F16INST")
 			    (V8HF "TARGET_SIMD_F16INST")
 			    V2SF V4SF V2DF])
 
-;; Modes available for AdvSIMD <f>mul lane operations changing lane count.
+;; Modes available for Advanced SIMD <f>mul lane operations changing lane
+;; count.
 (define_mode_iterator VMUL_CHANGE_NLANES [V4HI V8HI V2SI V4SI V2SF V4SF])
 
 ;; All SVE vector modes.
-(define_mode_iterator SVE_ALL [V32QI V16HI V8SI V4DI V8SF V4DF])
+(define_mode_iterator SVE_ALL [V32QI V16HI V8SI V4DI V16HF V8SF V4DF])
 
 ;; All SVE vector structure modes.
-(define_mode_iterator SVE_STRUCT [V64QI V32HI V16SI V8DI V16SF V8DF
-				  V96QI V48HI V24SI V12DI V24SF V12DF
-				  V128QI V64HI V32SI V16DI V32SF V16DF])
+(define_mode_iterator SVE_STRUCT [V64QI V32HI V16SI V8DI V32HF V16SF V8DF
+				  V96QI V48HI V24SI V12DI V48HF V24SF V12DF
+				  V128QI V64HI V32SI V16DI V64HF V32SF V16DF])
 
 ;; All SVE vector modes that have 8-bit or 16-bit elements.
-(define_mode_iterator SVE_BH [V32QI V16HI])
+(define_mode_iterator SVE_BH [V32QI V16HI V16HF])
 
 ;; All SVE vector modes that have 8-bit, 16-bit or 32-bit elements.
-(define_mode_iterator SVE_BHS [V32QI V16HI V8SI V8SF])
+(define_mode_iterator SVE_BHS [V32QI V16HI V8SI V16HF V8SF])
 
 ;; All SVE integer vector modes that have 8-bit, 16-bit or 32-bit elements.
 (define_mode_iterator SVE_BHSI [V32QI V16HI V8SI])
 
+;; All SVE integer vector modes that have 16-bit, 32-bit or 64-bit elements.
+(define_mode_iterator SVE_HSDI [V32QI V16HI V8SI])
+
+;; All SVE floating-point vector modes that have 16-bit or 32-bit elements.
+(define_mode_iterator SVE_HSF [V16HF V8SF])
+
 ;; All SVE vector modes that have 32-bit or 64-bit elements.
 (define_mode_iterator SVE_SD [V8SI V4DI V8SF V4DF])
 
@@ -272,7 +279,7 @@
 (define_mode_iterator SVE_I [V32QI V16HI V8SI V4DI])
 
 ;; All SVE floating-point vector modes.
-(define_mode_iterator SVE_F [V8SF V4DF])
+(define_mode_iterator SVE_F [V16HF V8SF V4DF])
 
 ;; All SVE predicate modes.
 (define_mode_iterator PRED_ALL [V32BI V16BI V8BI V4BI])
@@ -358,16 +365,21 @@
     UNSPEC_TBL		; Used in vector permute patterns.
     UNSPEC_TBX		; Used in vector permute patterns.
     UNSPEC_CONCAT	; Used in vector permute patterns.
+
+    ;; The following permute unspecs are generated directly by
+    ;; aarch64_expand_vec_perm_const, so any changes to the underlying
+    ;; instructions would need a corresponding change there.
     UNSPEC_ZIP1		; Used in vector permute patterns.
     UNSPEC_ZIP2		; Used in vector permute patterns.
     UNSPEC_UZP1		; Used in vector permute patterns.
     UNSPEC_UZP2		; Used in vector permute patterns.
     UNSPEC_TRN1		; Used in vector permute patterns.
     UNSPEC_TRN2		; Used in vector permute patterns.
-    UNSPEC_EXT		; Used in aarch64-simd.md.
+    UNSPEC_EXT		; Used in vector permute patterns.
     UNSPEC_REV64	; Used in vector reverse patterns (permute).
     UNSPEC_REV32	; Used in vector reverse patterns (permute).
     UNSPEC_REV16	; Used in vector reverse patterns (permute).
+
     UNSPEC_AESE		; Used in aarch64-simd.md.
     UNSPEC_AESD         ; Used in aarch64-simd.md.
     UNSPEC_AESMC        ; Used in aarch64-simd.md.
@@ -390,6 +402,8 @@
     UNSPEC_SQRDMLSH     ; Used in aarch64-simd.md.
     UNSPEC_FMAXNM       ; Used in aarch64-simd.md.
     UNSPEC_FMINNM       ; Used in aarch64-simd.md.
+    UNSPEC_SDOT		; Used in aarch64-simd.md.
+    UNSPEC_UDOT		; Used in aarch64-simd.md.
     UNSPEC_SEL		; Used in aarch64-sve.md.
     UNSPEC_ANDV		; Used in aarch64-sve.md.
     UNSPEC_IORV		; Used in aarch64-sve.md.
@@ -467,6 +481,9 @@
 (define_mode_attr w1 [(HF "w") (SF "w") (DF "x")])
 (define_mode_attr w2 [(HF "x") (SF "x") (DF "w")])
 
+;; For width of fp registers in fcvt instruction
+(define_mode_attr fpw [(DI "s") (SI "d")])
+
 (define_mode_attr short_mask [(HI "65535") (QI "255")])
 
 ;; For constraints used in scalar immediate vector moves
@@ -475,6 +492,10 @@
 ;; For doubling width of an integer mode
 (define_mode_attr DWI [(QI "HI") (HI "SI") (SI "DI") (DI "TI")])
 
+(define_mode_attr fcvt_change_mode [(SI "df") (DI "sf")])
+
+(define_mode_attr FCVT_CHANGE_MODE [(SI "DF") (DI "SF")])
+
 ;; For scalar usage of vector/FP registers
 (define_mode_attr v [(QI "b") (HI "h") (SI "s") (DI "d")
 		    (HF  "h") (SF "s") (DF "d")
@@ -507,7 +528,8 @@
 (define_mode_attr rtn [(DI "d") (SI "")])
 (define_mode_attr vas [(DI "") (SI ".2s")])
 
-;; Map a vector to the number of units.
+;; Map a vector to the number of units in it, if the size of the mode
+;; is constant.
 (define_mode_attr nunits [(V8QI "8") (V16QI "16")
 			  (V4HI "4") (V8HI "8")
 			  (V2SI "2") (V4SI "4")
@@ -517,8 +539,15 @@
 			  (V1DF "1") (V2DF "2")
 			  (DI "1") (DF "1")])
 
-;; Map a floating point mode to the appropriate register name prefix
-(define_mode_attr s [(HF "h") (SF "s") (DF "d")])
+;; Map a mode to the number of bits in it, if the size of the mode
+;; is constant.
+(define_mode_attr bitsize [(V8QI "64") (V16QI "128")
+			   (V4HI "64") (V8HI "128")
+			   (V2SI "64") (V4SI "128")
+				       (V2DI "128")])
+
+;; Map a floating point or integer mode to the appropriate register name prefix
+(define_mode_attr s [(HF "h") (SF "s") (DF "d") (SI "s") (DI "d")])
 
 ;; Give the length suffix letter for a sign- or zero-extension.
 (define_mode_attr size [(QI "b") (HI "h") (SI "w")])
@@ -571,7 +600,7 @@
 			  (V4HI "h") (V8HI  "h") (V16HI "h") (V16BI "h")
 			  (V2SI "s") (V4SI  "s") (V8SI  "s") (V8BI "s")
 			  (V2DI "d") (V4DI  "d") (V4BI  "d")
-			  (V4HF "h") (V8HF  "h")
+			  (V4HF "h") (V8HF  "h") (V16HF "h")
 			  (V2SF "s") (V4SF  "s") (V8SF  "s")
 			  (V2DF "d") (V4DF  "d")
 			  (HF   "h")
@@ -580,7 +609,8 @@
 			  (SI "s")   (DI "d")])
 
 ;; Equivalent of "size" for a vector element.
-(define_mode_attr Vesize [(V32QI "b") (V16HI "h")
+(define_mode_attr Vesize [(V32QI "b")
+			  (V16HI "h") (V16HF "h")
 			  (V8SI  "w") (V8SF  "w")
 			  (V4DI  "d") (V4DF  "d")])
 
@@ -609,23 +639,41 @@
 			(V4HI "HI") (V8HI  "HI") (V16HI "HI")
 			(V2SI "SI") (V4SI  "SI") (V8SI  "SI")
 			(DI   "DI") (V2DI  "DI") (V4DI  "DI")
-			(V4HF "HF") (V8HF  "HF")
+			(V4HF "HF") (V8HF  "HF") (V16HF "HF")
 			(V2SF "SF") (V4SF  "SF") (V8SF  "SF")
 			(DF   "DF") (V2DF  "DF") (V4DF  "DF")
 			(SI   "SI") (HI    "HI")
 			(QI   "QI")])
 
 ;; Define element mode for each vector mode (lower case).
-(define_mode_attr Vel [(V8QI "qi") (V16QI "qi")
-			(V4HI "hi") (V8HI "hi")
-			(V2SI "si") (V4SI "si")
-			(DI "di")   (V2DI "di")
-			(V4HF "hf") (V8HF "hf")
-			(V2SF "sf") (V4SF "sf")
-			(V2DF "df") (DF "df")
+(define_mode_attr Vel [(V8QI "qi") (V16QI "qi") (V32QI "qi")
+			(V4HI "hi") (V8HI "hi") (V16HI "hi")
+			(V2SI "si") (V4SI "si") (V8SI  "si")
+			(DI "di")   (V2DI "di") (V4DI  "di")
+			(V4HF "hf") (V8HF "hf") (V16HF "hf")
+			(V2SF "sf") (V4SF "sf") (V8SF  "sf")
+			(V2DF "df") (DF "df")   (V4DF  "df")
 			(SI   "si") (HI   "hi")
 			(QI   "qi")])
 
+;; Element mode with floating-point values replaced by like-sized integers.
+(define_mode_attr VEL_INT [(V32QI "QI")
+			   (V16HI "HI") (V16HF "HI")
+			   (V8SI  "SI") (V8SF  "SI")
+			   (V4DI  "DI") (V4DF  "DI")])
+
+;; Gives the mode of the 128-bit lowpart of an SVE vector.
+(define_mode_attr V128 [(V32QI "V16QI")
+			(V16HI "V8HI") (V16HF "V8HF")
+			(V8SI "V4SI") (V8SF "V4SF")
+			(V4DI "V2DI") (V4DF "V2DF")])
+
+;; ...and again in lower case.
+(define_mode_attr v128 [(V32QI "v16qi")
+			(V16HI "v8hi") (V16HF "v8hf")
+			(V8SI "v4si") (V8SF "v4sf")
+			(V4DI "v2di") (V4DF "v2df")])
+
 ;; 64-bit container modes the inner or scalar source mode.
 (define_mode_attr VCOND [(HI "V4HI") (SI "V2SI")
 			 (V4HI "V4HI") (V8HI "V4HI")
@@ -710,15 +758,20 @@
 			 (HI    "SI")    (SI    "DI")
 			 (V8HF  "V4SF")  (V4SF  "V2DF")
 			 (V4HF  "V4SF")  (V2SF  "V2DF")
+			 (V16HF "V8SF")  (V8SF  "V4DF")
 			 (V32QI "V16HI") (V16HI "V8SI")
 			 (V8SI  "V4DI")
 			 (V32BI "V16BI") (V16BI "V8BI")
 			 (V8BI  "V4BI")])
 
+;; Predicate mode associated with VWIDE.
+(define_mode_attr VWIDE_PRED [(V16HF "V8BI") (V8SF "V4BI")])
+
 ;; Widened modes of vector modes, lowercase
 (define_mode_attr Vwide [(V2SF "v2df") (V4HF "v4sf")
 			 (V32QI "v16hi") (V16HI "v8si")
 			 (V8SI  "v4di")
+			 (V16HF "v8sf") (V8SF "v4df")
 			 (V32BI "v16bi") (V16BI "v8bi")
 			 (V8BI  "v4bi")])
 
@@ -729,7 +782,9 @@
 			  (V8HF "4s") (V4SF "2d")])
 
 ;; SVE vector after widening
-(define_mode_attr Vewtype [(V32QI "h") (V16HI "s") (V8SI "d")])
+(define_mode_attr Vewtype [(V32QI "h")
+			   (V16HI "s") (V16HF "s")
+			   (V8SI "d") (V8SF "d")])
 
 ;; Widened mode register suffixes for VDW/VQW.
 (define_mode_attr Vmwtype [(V8QI ".8h") (V4HI ".4s")
@@ -748,6 +803,7 @@
 		      (V4HI "w") (V8HI "w") (V16HI "w")
 		      (V2SI "w") (V4SI "w") (V8SI "w")
 		      (DI   "x") (V2DI "x") (V4DI "x")
+		      (V16HF "h")
 		      (V2SF "s") (V4SF "s") (V8SF "s")
 		      (V2DF "d") (V4DF "d")])
 
@@ -757,7 +813,7 @@
 			  (V4HI "w") (V8HI "w") (V16HI "w")
 			  (V2SI "w") (V4SI "w") (V8SI "w")
 			  (DI   "x") (V2DI "x") (V4DI "x")
-			  (V4HF "w") (V8HF "w")
+			  (V4HF "w") (V8HF "w") (V16HF "w")
 			  (V2SF "w") (V4SF "w") (V8SF "w")
 			  (V2DF "x") (V4DF "x")])
 
@@ -769,7 +825,7 @@
 			       (V4HI "V4HI") (V8HI  "V8HI")  (V16HI "V16HI")
 			       (V2SI "V2SI") (V4SI  "V4SI")  (V8SI  "V8SI")
 			       (DI   "DI")   (V2DI  "V2DI")  (V4DI  "V4DI")
-			       (V4HF "V4HI") (V8HF  "V8HI")
+			       (V4HF "V4HI") (V8HF  "V8HI")  (V16HF "V16HI")
 			       (V2SF "V2SI") (V4SF  "V4SI")  (V8SF  "V8SI")
 			       (DF   "DI")   (V2DF  "V2DI")  (V4DF  "V4DI")
 			       (SF   "SI")   (HF    "HI")])
@@ -779,7 +835,7 @@
 			       (V4HI "v4hi") (V8HI  "v8hi")  (V16HI "v16hi")
 			       (V2SI "v2si") (V4SI  "v4si")  (V8SI  "v8si")
 			       (DI   "di")   (V2DI  "v2di")  (V4DI  "v4di")
-			       (V4HF "v4hi") (V8HF  "v8hi")
+			       (V4HF "v4hi") (V8HF  "v8hi")  (V16HF "v16hi")
 			       (V2SF "v2si") (V4SF  "v4si")  (V8SF  "v8si")
 			       (DF   "di")   (V2DF  "v2di")  (V4DF  "v4di")
 			       (SF   "si")])
@@ -820,29 +876,29 @@
 (define_mode_attr nregs [(OI "2") (CI "3") (XI "4")])
 
 ;; Map the mode of a single vector to a list of two vectors.
-(define_mode_attr VRL2 [(V32QI "V64QI") (V16HI "V32HI")
+(define_mode_attr VRL2 [(V32QI "V64QI") (V16HI "V32HI") (V16HF "V32HF")
 			(V8SI  "V16SI") (V8SF  "V16SF")
 			(V4DI  "V8DI")  (V4DF  "V8DF")])
 
-(define_mode_attr vrl2 [(V32QI "v64qi") (V16HI "v32hi")
+(define_mode_attr vrl2 [(V32QI "v64qi") (V16HI "v32hi") (V16HF "v32hf")
 			(V8SI  "v16si") (V8SF  "v16sf")
 			(V4DI  "v8di")  (V4DF  "v8df")])
 
 ;; Map the mode of a single vector to a list of three vectors.
-(define_mode_attr VRL3 [(V32QI "V96QI") (V16HI "V48HI")
+(define_mode_attr VRL3 [(V32QI "V96QI") (V16HI "V48HI") (V16HF "V48HF")
 			(V8SI  "V24SI") (V8SF  "V24SF")
 			(V4DI  "V12DI") (V4DF  "V12DF")])
 
-(define_mode_attr vrl3 [(V32QI "v96qi") (V16HI "v48hi")
+(define_mode_attr vrl3 [(V32QI "v96qi") (V16HI "v48hi") (V16HF "v48hf")
 			(V8SI  "v24si") (V8SF  "v24sf")
 			(V4DI  "v12di") (V4DF  "v12df")])
 
 ;; Map the mode of a single vector to a list of four vectors.
-(define_mode_attr VRL4 [(V32QI "V128QI") (V16HI "V64HI")
+(define_mode_attr VRL4 [(V32QI "V128QI") (V16HI "V64HI") (V16HF "V64HF")
 			(V8SI  "V32SI")  (V8SF  "V32SF")
 			(V4DI  "V16DI")  (V4DF  "V16DF")])
 
-(define_mode_attr vrl4 [(V32QI "v128qi") (V16HI "v64hi")
+(define_mode_attr vrl4 [(V32QI "v128qi") (V16HI "v64hi") (V16HF "v64hf")
 			(V8SI  "v32si")  (V8SF  "v32sf")
 			(V4DI  "v16di")  (V4DF  "v16df")])
 
@@ -940,6 +996,10 @@
 (define_mode_attr vsi2qi [(V2SI "v8qi") (V4SI "v16qi")])
 (define_mode_attr VSI2QI [(V2SI "V8QI") (V4SI "V16QI")])
 
+
+;; Register suffix for DOTPROD input types from the return type.
+(define_mode_attr Vdottype [(V2SI "8b") (V4SI "16b")])
+
 ;; Sum of lengths of instructions needed to move vector registers of a mode.
 (define_mode_attr insn_count [(OI "8") (CI "12") (XI "16")])
 
@@ -948,72 +1008,84 @@
 (define_mode_attr got_modifier [(SI "gotpage_lo14") (DI "gotpage_lo15")])
 
 ;; The number of subvectors in an SVE_STRUCT.
-(define_mode_attr vector_count [(V64QI "2") (V32HI "2") (V16SI "2")
-				(V8DI "2") (V16SF "2") (V8DF "2")
-				(V96QI "3") (V48HI "3") (V24SI "3")
-				(V12DI "3") (V24SF "3") (V12DF "3")
-				(V128QI "4") (V64HI "4") (V32SI "4")
-				(V16DI "4") (V32SF "4") (V16DF "4")])
+(define_mode_attr vector_count [(V64QI "2") (V32HI "2")
+				(V16SI "2") (V8DI "2")
+				(V32HF "2") (V16SF "2") (V8DF "2")
+				(V96QI "3") (V48HI "3")
+				(V24SI "3") (V12DI "3")
+				(V48HF "3") (V24SF "3") (V12DF "3")
+				(V128QI "4") (V64HI "4")
+				(V32SI "4") (V16DI "4")
+				(V64HF "4") (V32SF "4") (V16DF "4")])
 
 ;; The number of instruction bytes needed for an SVE_STRUCT move.  This is
 ;; equal to vector_count * 4.
-(define_mode_attr insn_length [(V64QI "8") (V32HI "8") (V16SI "8")
-			       (V8DI "8") (V16SF "8") (V8DF "8")
-			       (V96QI "12") (V48HI "12") (V24SI "12")
-			       (V12DI "12") (V24SF "12") (V12DF "12")
-			       (V128QI "16") (V64HI "16") (V32SI "16")
-			       (V16DI "16") (V32SF "16") (V16DF "16")])
+(define_mode_attr insn_length [(V64QI "8") (V32HI "8")
+			       (V16SI "8") (V8DI "8")
+			       (V32HF "8") (V16SF "8") (V8DF "8")
+			       (V96QI "12") (V48HI "12")
+			       (V24SI "12") (V12DI "12")
+			       (V48HF "12") (V24SF "12") (V12DF "12")
+			       (V128QI "16") (V64HI "16")
+			       (V32SI "16") (V16DI "16")
+			       (V64HF "16") (V32SF "16") (V16DF "16")])
 
 ;; The type of a subvector in an SVE_STRUCT.
-(define_mode_attr VSINGLE [(V64QI "V32QI") (V32HI "V16HI") (V16SI "V8SI")
-			   (V8DI "V4DI") (V16SF "V8SF") (V8DF "V4DF")
-			   (V96QI "V32QI") (V48HI "V16HI") (V24SI "V8SI")
-			   (V12DI "V4DI") (V24SF "V8SF") (V12DF "V4DF")
-			   (V128QI "V32QI") (V64HI "V16HI") (V32SI "V8SI")
-			   (V16DI "V4DI") (V32SF "V8SF") (V16DF "V4DF")])
+(define_mode_attr VSINGLE [(V64QI "V32QI") (V32HI "V16HI")
+			   (V16SI "V8SI") (V8DI "V4DI")
+			   (V32HF "V16HF") (V16SF "V8SF") (V8DF "V4DF")
+			   (V96QI "V32QI") (V48HI "V16HI")
+			   (V24SI "V8SI") (V12DI "V4DI")
+			   (V48HF "V16HF") (V24SF "V8SF") (V12DF "V4DF")
+			   (V128QI "V32QI") (V64HI "V16HI")
+			   (V32SI "V8SI") (V16DI "V4DI")
+			   (V64HF "V16HF") (V32SF "V8SF") (V16DF "V4DF")])
 
 ;; ...and again in lower case.
-(define_mode_attr vsingle [(V64QI "v32qi") (V32HI "v16hi") (V16SI "v8si")
-			   (V8DI "v4di") (V16SF "v8sf") (V8DF "v4df")
-			   (V96QI "v32qi") (V48HI "v16hi") (V24SI "v8si")
-			   (V12DI "v4di") (V24SF "v8sf") (V12DF "v4df")
-			   (V128QI "v32qi") (V64HI "v16hi") (V32SI "v8si")
-			   (V16DI "v4di") (V32SF "v8sf") (V16DF "v4df")])
+(define_mode_attr vsingle [(V64QI "v32qi") (V32HI "v16hi")
+			   (V16SI "v8si") (V8DI "v4di")
+			   (V32HF "v16hf") (V16SF "v8sf") (V8DF "v4df")
+			   (V96QI "v32qi") (V48HI "v16hi")
+			   (V24SI "v8si") (V12DI "v4di")
+			   (V48HF "v16hf") (V24SF "v8sf") (V12DF "v4df")
+			   (V128QI "v32qi") (V64HI "v16hi")
+			   (V32SI "v8si") (V16DI "v4di")
+			   (V64HF "v16hf") (V32SF "v8sf") (V16DF "v4df")])
 
 ;; The predicate mode associated with an SVE data mode.  For structure modes
 ;; this is equivalent to the <VPRED> of the subvector mode.
 (define_mode_attr VPRED [(V32QI "V32BI")
-			 (V16HI "V16BI")
+			 (V16HI "V16BI") (V16HF "V16BI")
 			 (V8SI "V8BI") (V8SF "V8BI")
 			 (V4DI "V4BI") (V4DF "V4BI")
 			 (V64QI "V32BI")
-			 (V32HI "V16BI")
+			 (V32HI "V16BI") (V32HF "V16BI")
 			 (V16SI "V8BI") (V16SF "V8BI")
 			 (V8DI "V4BI") (V8DF "V4BI")
 			 (V96QI "V32BI")
-			 (V48HI "V16BI")
+			 (V48HI "V16BI") (V48HF "V16BI")
 			 (V24SI "V8BI") (V24SF "V8BI")
 			 (V12DI "V4BI") (V12DF "V4BI")
 			 (V128QI "V32BI")
-			 (V64HI "V16BI")
+			 (V64HI "V16BI") (V64HF "V16BI")
 			 (V32SI "V8BI") (V32SF "V8BI")
 			 (V16DI "V4BI") (V16DF "V4BI")])
 
 ;; ...and again in lower case.
 (define_mode_attr vpred [(V32QI "v32bi")
-			 (V16HI "v16bi")
+			 (V16HI "v16bi") (V16HF "v16bi")
 			 (V8SI "v8bi") (V8SF "v8bi")
 			 (V4DI "v4bi") (V4DF "v4bi")
 			 (V64QI "v32bi")
-			 (V32HI "v16bi")
+			 (V32HI "v16bi") (V32HF "v16bi")
 			 (V16SI "v8bi") (V16SF "v8bi")
 			 (V8DI "v4bi") (V8DF "v4bi")
 			 (V96QI "v32bi")
-			 (V48HI "v16bi")
+			 (V48HI "v16bi") (V48HF "v16bi")
 			 (V24SI "v8bi") (V24SF "v8bi")
 			 (V12DI "v4bi") (V12DF "v4bi")
 			 (V128QI "v32bi")
-			 (V64HI "v16bi")
+			 (V64HI "v16bi") (V64HF "v8bi")
 			 (V32SI "v8bi") (V32SF "v8bi")
 			 (V16DI "v4bi") (V16DF "v4bi")])
 
@@ -1099,6 +1171,7 @@
 ;; SVE integer unary operations.
 (define_code_iterator SVE_INT_UNARY [neg not popcount])
 
+;; SVE floating-point unary operations.
 (define_code_iterator SVE_FP_UNARY [neg abs sqrt])
 
 ;; -------------------------------------------------------------------
@@ -1251,6 +1324,7 @@
 ;; Attribute to describe constants acceptable in atomic logical operations
 (define_mode_attr lconst_atomic [(QI "K") (HI "K") (SI "K") (DI "L")])
 
+;; The integer SVE instruction that implements an rtx code.
 (define_code_attr sve_int_op [(plus "add")
 			      (neg "neg")
 			      (smin "smin")
@@ -1263,6 +1337,7 @@
 			      (not "not")
 			      (popcount "cnt")])
 
+;; The floating-point SVE instruction that implements an rtx code.
 (define_code_attr sve_fp_op [(plus "fadd")
 			     (neg "fneg")
 			     (abs "fabs")
@@ -1286,6 +1361,7 @@
 			      UNSPEC_SHSUB UNSPEC_UHSUB
 			      UNSPEC_SRHSUB UNSPEC_URHSUB])
 
+(define_int_iterator DOTPROD [UNSPEC_SDOT UNSPEC_UDOT])
 
 (define_int_iterator ADDSUBHN [UNSPEC_ADDHN UNSPEC_RADDHN
 			       UNSPEC_SUBHN UNSPEC_RSUBHN])
@@ -1407,6 +1483,9 @@
 ;; Int Iterators Attributes.
 ;; -------------------------------------------------------------------
 
+;; The optab associated with an operation.  Note that for ANDF, IORF
+;; and XORF, the optab should not actually be defined; we just use this
+;; name for consistency with the integer patterns.
 (define_int_attr optab [(UNSPEC_ANDF "and")
 			(UNSPEC_IORF "ior")
 			(UNSPEC_XORF "xor")
@@ -1459,10 +1538,12 @@
 			       (UNSPEC_IORV "orv")
 			       (UNSPEC_XORV "eorv")])
 
+;; The SVE logical instruction that implements an unspec.
 (define_int_attr logicalf_op [(UNSPEC_ANDF "and")
 		 	      (UNSPEC_IORF "orr")
 			      (UNSPEC_XORF "eor")])
 
+;; "s" for signed operations and "u" for unsigned ones.
 (define_int_attr su [(UNSPEC_UNPACKSHI "s")
 		     (UNSPEC_UNPACKUHI "u")
 		     (UNSPEC_UNPACKSLO "s")
@@ -1492,6 +1573,7 @@
 		      (UNSPEC_USHLL  "u")  (UNSPEC_SSHLL "s")
 		      (UNSPEC_URSHL  "ur") (UNSPEC_SRSHL  "sr")
 		      (UNSPEC_UQRSHL  "u") (UNSPEC_SQRSHL  "s")
+		      (UNSPEC_SDOT "s") (UNSPEC_UDOT "u")
 ])
 
 (define_int_attr r [(UNSPEC_SQDMULH "") (UNSPEC_SQRDMULH "r")
@@ -1608,6 +1690,7 @@
 
 (define_int_attr rdma_as [(UNSPEC_SQRDMLAH "a") (UNSPEC_SQRDMLSH "s")])
 
+;; The condition associated with an UNSPEC_COND_<xx>.
 (define_int_attr cmp_op [(UNSPEC_COND_LT "lt")
 			 (UNSPEC_COND_LE "le")
 			 (UNSPEC_COND_EQ "eq")
@@ -1619,6 +1702,7 @@
 			 (UNSPEC_COND_HS "hs")
 			 (UNSPEC_COND_HI "hi")])
 
+;; The constraint to use for an UNSPEC_COND_<xx>.
 (define_int_attr imm_con [(UNSPEC_COND_EQ "vsc")
 			  (UNSPEC_COND_NE "vsc")
 			  (UNSPEC_COND_LT "vsc")
diff --git a/gcc/config/aarch64/predicates.md b/gcc/config/aarch64/predicates.md
index 4bb473b80c4..972ab2182d5 100644
--- a/gcc/config/aarch64/predicates.md
+++ b/gcc/config/aarch64/predicates.md
@@ -267,16 +267,20 @@
 	    (ior (match_operand 0 "memory_operand")
 		 (match_test "aarch64_mov_operand_p (op, mode)")))))
 
-(define_predicate "aarch64_movti_operand"
-  (and (match_code "reg,subreg,mem,const_int")
+(define_predicate "aarch64_nonmemory_operand"
+  (and (match_code "reg,subreg,const,const_int,symbol_ref,label_ref,high,
+		    const_poly_int,const_vector")
        (ior (match_operand 0 "register_operand")
-	    (ior (match_operand 0 "memory_operand")
-		 (match_operand 0 "const_int_operand")))))
+	    (match_test "aarch64_mov_operand_p (op, mode)"))))
+
+(define_predicate "aarch64_movti_operand"
+  (ior (match_operand 0 "register_operand")
+       (match_operand 0 "memory_operand")
+       (match_operand 0 "const_scalar_int_operand")))
 
 (define_predicate "aarch64_reg_or_imm"
-  (and (match_code "reg,subreg,const_int")
-       (ior (match_operand 0 "register_operand")
-	    (match_operand 0 "const_int_operand"))))
+  (ior (match_operand 0 "register_operand")
+       (match_operand 0 "const_scalar_int_operand")))
 
 ;; True for integer comparisons and for FP comparisons other than LTGT or UNEQ.
 (define_special_predicate "aarch64_comparison_operator"
@@ -366,8 +370,8 @@
 (define_predicate "aarch64_simd_reg_or_zero"
   (and (match_code "reg,subreg,const_int,const_double,const,const_vector")
        (ior (match_operand 0 "register_operand")
-	    (match_operand 0 "aarch64_simd_imm_zero")
-	    (match_test "op == const0_rtx"))))
+	    (match_test "op == const0_rtx")
+	    (match_operand 0 "aarch64_simd_imm_zero"))))
 
 (define_predicate "aarch64_simd_struct_operand"
   (and (match_code "mem")
diff --git a/gcc/config/alpha/sync.md b/gcc/config/alpha/sync.md
index 69c9d249b97..a0e67a99e88 100644
--- a/gcc/config/alpha/sync.md
+++ b/gcc/config/alpha/sync.md
@@ -24,7 +24,7 @@
   [(plus "add_operand") (minus "reg_or_8bit_operand")
    (ior "or_operand") (xor "or_operand") (and "and_operand")])
 (define_code_attr fetchop_constr
-  [(plus "rKL") (minus "rI") (ior "rIN") (xor "rIN") (and "riNM")])
+  [(plus "rKL") (minus "rI") (ior "rIN") (xor "rIN") (and "rINM")])
 
 
 (define_expand "memory_barrier"
diff --git a/gcc/config/arc/arc.c b/gcc/config/arc/arc.c
index 0a68db8e889..90a9a56e016 100644
--- a/gcc/config/arc/arc.c
+++ b/gcc/config/arc/arc.c
@@ -2829,12 +2829,23 @@ arc_save_restore (rtx base_reg,
 	  else
 	    {
 	      insn = frame_insn (insn);
-	      if (epilogue_p)
-		for (r = start_call; r <= end_call; r++)
-		  {
-		    rtx reg = gen_rtx_REG (SImode, r);
-		    add_reg_note (insn, REG_CFA_RESTORE, reg);
-		  }
+	      for (r = start_call, off = 0;
+		   r <= end_call;
+		   r++, off += UNITS_PER_WORD)
+		{
+		  rtx reg = gen_rtx_REG (SImode, r);
+		  if (epilogue_p)
+		      add_reg_note (insn, REG_CFA_RESTORE, reg);
+		  else
+		    {
+		      rtx mem = gen_rtx_MEM (SImode, plus_constant (Pmode,
+								    base_reg,
+								    off));
+
+		      add_reg_note (insn, REG_CFA_OFFSET,
+				    gen_rtx_SET (mem, reg));
+		    }
+		}
 	    }
 	  offset += off;
 	}
@@ -3076,6 +3087,19 @@ arc_expand_prologue (void)
       frame_size_to_allocate -= cfun->machine->frame_info.reg_size;
     }
 
+  /* In the case of millicode thunk, we need to restore the clobbered
+     blink register.  */
+  if (cfun->machine->frame_info.millicode_end_reg > 0
+      && arc_must_save_return_addr (cfun))
+    {
+      HOST_WIDE_INT tmp = cfun->machine->frame_info.reg_size;
+      emit_insn (gen_rtx_SET (gen_rtx_REG (Pmode, RETURN_ADDR_REGNUM),
+			      gen_rtx_MEM (Pmode,
+					   plus_constant (Pmode,
+							  stack_pointer_rtx,
+							  tmp))));
+    }
+
   /* Save frame pointer if needed.  First save the FP on stack, if not
      autosaved.  */
   if (arc_frame_pointer_needed ()
@@ -7189,6 +7213,12 @@ hwloop_optimize (hwloop_info loop)
 	fprintf (dump_file, ";; loop %d too long\n", loop->loop_no);
       return false;
     }
+  else if (!loop->length)
+    {
+      if (dump_file)
+	fprintf (dump_file, ";; loop %d is empty\n", loop->loop_no);
+      return false;
+    }
 
   /* Check if we use a register or not.  */
   if (!REG_P (loop->iter_reg))
@@ -7260,8 +7290,11 @@ hwloop_optimize (hwloop_info loop)
       && INSN_P (last_insn)
       && (JUMP_P (last_insn) || CALL_P (last_insn)
 	  || GET_CODE (PATTERN (last_insn)) == SEQUENCE
-	  || get_attr_type (last_insn) == TYPE_BRCC
-	  || get_attr_type (last_insn) == TYPE_BRCC_NO_DELAY_SLOT))
+	  /* At this stage we can have (insn (clobber (mem:BLK
+	     (reg)))) instructions, ignore them.  */
+	  || (GET_CODE (PATTERN (last_insn)) != CLOBBER
+	      && (get_attr_type (last_insn) == TYPE_BRCC
+		  || get_attr_type (last_insn) == TYPE_BRCC_NO_DELAY_SLOT))))
     {
       if (loop->length + 2 > ARC_MAX_LOOP_LENGTH)
 	{
diff --git a/gcc/config/arc/linux.h b/gcc/config/arc/linux.h
index d8e006307fc..707347183ca 100644
--- a/gcc/config/arc/linux.h
+++ b/gcc/config/arc/linux.h
@@ -91,3 +91,11 @@ along with GCC; see the file COPYING3.  If not see
 /* Pre/post modify with register displacement are default off.  */
 #undef TARGET_AUTO_MODIFY_REG_DEFAULT
 #define TARGET_AUTO_MODIFY_REG_DEFAULT 0
+
+#if DEFAULT_LIBC == LIBC_GLIBC
+/* Override linux.h LINK_EH_SPEC definition.
+   Signalize that because we have fde-glibc, we don't need all C shared libs
+   linked against -lgcc_s.  */
+#undef LINK_EH_SPEC
+#define LINK_EH_SPEC "--eh-frame-hdr"
+#endif
diff --git a/gcc/config/arm/arm-builtins.c b/gcc/config/arm/arm-builtins.c
index 692496d49d5..d09c6e371de 100644
--- a/gcc/config/arm/arm-builtins.c
+++ b/gcc/config/arm/arm-builtins.c
@@ -107,6 +107,13 @@ arm_ternop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_none, qualifier_none, qualifier_none, qualifier_none };
 #define TERNOP_QUALIFIERS (arm_ternop_qualifiers)
 
+/* unsigned T (unsigned T, unsigned T, unsigned T).  */
+static enum arm_type_qualifiers
+arm_unsigned_uternop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
+      qualifier_unsigned };
+#define UTERNOP_QUALIFIERS (arm_unsigned_uternop_qualifiers)
+
 /* T (T, immediate).  */
 static enum arm_type_qualifiers
 arm_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
@@ -133,6 +140,13 @@ arm_mac_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
       qualifier_none, qualifier_lane_index };
 #define MAC_LANE_QUALIFIERS (arm_mac_lane_qualifiers)
 
+/* unsigned T (unsigned T, unsigned T, unsigend T, lane index).  */
+static enum arm_type_qualifiers
+arm_umac_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
+      qualifier_unsigned, qualifier_lane_index };
+#define UMAC_LANE_QUALIFIERS (arm_umac_lane_qualifiers)
+
 /* T (T, T, immediate).  */
 static enum arm_type_qualifiers
 arm_ternop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
diff --git a/gcc/config/arm/arm-c.c b/gcc/config/arm/arm-c.c
index b2e9af6c45d..635bc3c1c38 100644
--- a/gcc/config/arm/arm-c.c
+++ b/gcc/config/arm/arm-c.c
@@ -74,11 +74,11 @@ arm_cpu_builtins (struct cpp_reader* pfile)
 
   def_or_undef_macro (pfile, "__ARM_FEATURE_QRDMX", TARGET_NEON_RDMA);
 
-  if (TARGET_CRC32)
-    builtin_define ("__ARM_FEATURE_CRC32");
-
+  def_or_undef_macro (pfile, "__ARM_FEATURE_CRC32", TARGET_CRC32);
+  def_or_undef_macro (pfile, "__ARM_FEATURE_DOTPROD", TARGET_DOTPROD);
   def_or_undef_macro (pfile, "__ARM_32BIT_STATE", TARGET_32BIT);
 
+  cpp_undef (pfile, "__ARM_FEATURE_CMSE");
   if (arm_arch8 && !arm_arch_notm)
     {
       if (arm_arch_cmse && use_cmse)
diff --git a/gcc/config/arm/arm-cpus.in b/gcc/config/arm/arm-cpus.in
index 07de4c9375b..0820ad74c2e 100644
--- a/gcc/config/arm/arm-cpus.in
+++ b/gcc/config/arm/arm-cpus.in
@@ -156,6 +156,8 @@ define feature crypto
 # FP16 data processing (half-precision float).
 define feature fp16
 
+# Dot Product instructions extension to ARMv8.2-a.
+define feature dotprod
 
 # ISA Quirks (errata?).  Don't forget to add this to the fgroup
 # ALL_QUIRKS below.
@@ -173,6 +175,17 @@ define feature quirk_cm3_ldrd
 define feature smallmul
 
 # Feature groups.  Conventionally all (or mostly) upper case.
+# ALL_FPU lists all the feature bits associated with the floating-point
+# unit; these will all be removed if the floating-point unit is disabled
+# (eg -mfloat-abi=soft).  ALL_FPU_INTERNAL must ONLY contain features that
+# form part of a named -mfpu option; it is used to map the capabilities
+# back to a named FPU for the benefit of the assembler.
+#
+# ALL_SIMD_INTERNAL and ALL_SIMD are similarly defined to help with the
+# construction of ALL_FPU and ALL_FPU_INTERNAL; they describe the SIMD
+# extensions that are either part of a named FPU or optional extensions
+# respectively.
+
 
 # List of all cryptographic extensions to stripout if crypto is
 # disabled.  Currently, that's trivial, but we define it anyway for
@@ -182,11 +195,12 @@ define fgroup ALL_CRYPTO	crypto
 # List of all SIMD bits to strip out if SIMD is disabled.  This does
 # strip off 32 D-registers, but does not remove support for
 # double-precision FP.
-define fgroup ALL_SIMD	fp_d32 neon ALL_CRYPTO
+define fgroup ALL_SIMD_INTERNAL	fp_d32 neon ALL_CRYPTO
+define fgroup ALL_SIMD	ALL_SIMD_INTERNAL dotprod
 
 # List of all FPU bits to strip out if -mfpu is used to override the
 # default.  fp16 is deliberately missing from this list.
-define fgroup ALL_FPU_INTERNAL	vfpv2 vfpv3 vfpv4 fpv5 fp16conv fp_dbl ALL_SIMD
+define fgroup ALL_FPU_INTERNAL	vfpv2 vfpv3 vfpv4 fpv5 fp16conv fp_dbl ALL_SIMD_INTERNAL
 
 # Similarly, but including fp16 and other extensions that aren't part of
 # -mfpu support.
@@ -239,6 +253,7 @@ define fgroup FP_D32	FP_DBL fp_d32
 define fgroup FP_ARMv8	FPv5 FP_D32
 define fgroup NEON	FP_D32 neon
 define fgroup CRYPTO	NEON crypto
+define fgroup DOTPROD	NEON dotprod
 
 # List of all quirk bits to strip out when comparing CPU features with
 # architectures.
@@ -561,6 +576,7 @@ begin arch armv8.2-a
  option crypto add FP_ARMv8 CRYPTO
  option nocrypto remove ALL_CRYPTO
  option nofp remove ALL_FP
+ option dotprod add FP_ARMv8 DOTPROD
 end arch armv8.2-a
 
 begin arch armv8-m.base
@@ -1473,7 +1489,7 @@ begin cpu cortex-a55
  cname cortexa55
  tune for cortex-a53
  tune flags LDSCHED
- architecture armv8.2-a+fp16
+ architecture armv8.2-a+fp16+dotprod
  fpu neon-fp-armv8
  option crypto add FP_ARMv8 CRYPTO
  option nofp remove ALL_FP
@@ -1484,7 +1500,7 @@ begin cpu cortex-a75
  cname cortexa75
  tune for cortex-a57
  tune flags LDSCHED
- architecture armv8.2-a+fp16
+ architecture armv8.2-a+fp16+dotprod
  fpu neon-fp-armv8
  option crypto add FP_ARMv8 CRYPTO
  costs cortex_a73
@@ -1496,7 +1512,7 @@ begin cpu cortex-a75.cortex-a55
  cname cortexa75cortexa55
  tune for cortex-a53
  tune flags LDSCHED
- architecture armv8.2-a+fp16
+ architecture armv8.2-a+fp16+dotprod
  fpu neon-fp-armv8
  option crypto add FP_ARMv8 CRYPTO
  costs cortex_a73
@@ -1516,6 +1532,7 @@ begin cpu cortex-m33
  architecture armv8-m.main+dsp
  fpu fpv5-sp-d16
  option nofp remove ALL_FP
+ option nodsp remove armv7em
  costs v7m
 end cpu cortex-m33
 
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index ce3aaeb04e0..47ba0dd09e3 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -973,6 +973,9 @@ int arm_condexec_masklen = 0;
 /* Nonzero if chip supports the ARMv8 CRC instructions.  */
 int arm_arch_crc = 0;
 
+/* Nonzero if chip supports the AdvSIMD Dot Product instructions.  */
+int arm_arch_dotprod = 0;
+
 /* Nonzero if chip supports the ARMv8-M security extensions.  */
 int arm_arch_cmse = 0;
 
@@ -9429,6 +9432,9 @@ arm_rtx_costs_internal (rtx x, enum rtx_code code, enum rtx_code outer_code,
 		   + rtx_cost (XEXP (x, 0), mode, code, 0, speed_p));
 	  if (speed_p)
 	    *cost += 2 * extra_cost->alu.shift;
+	  /* Slightly disparage left shift by 1 at so we prefer adddi3.  */
+	  if (code == ASHIFT && XEXP (x, 1) == CONST1_RTX (SImode))
+	    *cost += 1;
 	  return true;
 	}
       else if (mode == SImode)
@@ -11252,9 +11258,11 @@ arm_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
         return current_tune->vec_costs->scalar_to_vec_cost;
 
       case unaligned_load:
+      case vector_gather_load:
         return current_tune->vec_costs->vec_unalign_load_cost;
 
       case unaligned_store:
+      case vector_scatter_store:
         return current_tune->vec_costs->vec_unalign_store_cost;
 
       case cond_branch_taken:
@@ -15293,12 +15301,23 @@ operands_ok_ldrd_strd (rtx rt, rtx rt2, rtx rn, HOST_WIDE_INT offset,
   return true;
 }
 
+/* Return true if a 64-bit access with alignment ALIGN and with a
+   constant offset OFFSET from the base pointer is permitted on this
+   architecture.  */
+static bool
+align_ok_ldrd_strd (HOST_WIDE_INT align, HOST_WIDE_INT offset)
+{
+  return (unaligned_access
+	  ? (align >= BITS_PER_WORD && (offset & 3) == 0)
+	  : (align >= 2 * BITS_PER_WORD && (offset & 7) == 0));
+}
+
 /* Helper for gen_operands_ldrd_strd.  Returns true iff the memory
    operand MEM's address contains an immediate offset from the base
-   register and has no side effects, in which case it sets BASE and
-   OFFSET accordingly.  */
+   register and has no side effects, in which case it sets BASE,
+   OFFSET and ALIGN accordingly.  */
 static bool
-mem_ok_for_ldrd_strd (rtx mem, rtx *base, rtx *offset)
+mem_ok_for_ldrd_strd (rtx mem, rtx *base, rtx *offset, HOST_WIDE_INT *align)
 {
   rtx addr;
 
@@ -15317,6 +15336,7 @@ mem_ok_for_ldrd_strd (rtx mem, rtx *base, rtx *offset)
   gcc_assert (MEM_P (mem));
 
   *offset = const0_rtx;
+  *align = MEM_ALIGN (mem);
 
   addr = XEXP (mem, 0);
 
@@ -15357,7 +15377,7 @@ gen_operands_ldrd_strd (rtx *operands, bool load,
                         bool const_store, bool commute)
 {
   int nops = 2;
-  HOST_WIDE_INT offsets[2], offset;
+  HOST_WIDE_INT offsets[2], offset, align[2];
   rtx base = NULL_RTX;
   rtx cur_base, cur_offset, tmp;
   int i, gap;
@@ -15369,7 +15389,8 @@ gen_operands_ldrd_strd (rtx *operands, bool load,
      registers, and the corresponding memory offsets.  */
   for (i = 0; i < nops; i++)
     {
-      if (!mem_ok_for_ldrd_strd (operands[nops+i], &cur_base, &cur_offset))
+      if (!mem_ok_for_ldrd_strd (operands[nops+i], &cur_base, &cur_offset,
+				 &align[i]))
         return false;
 
       if (i == 0)
@@ -15483,6 +15504,7 @@ gen_operands_ldrd_strd (rtx *operands, bool load,
       /* Swap the instructions such that lower memory is accessed first.  */
       std::swap (operands[0], operands[1]);
       std::swap (operands[2], operands[3]);
+      std::swap (align[0], align[1]);
       if (const_store)
         std::swap (operands[4], operands[5]);
     }
@@ -15496,6 +15518,9 @@ gen_operands_ldrd_strd (rtx *operands, bool load,
   if (gap != 4)
     return false;
 
+  if (!align_ok_ldrd_strd (align[0], offset))
+    return false;
+
   /* Make sure we generate legal instructions.  */
   if (operands_ok_ldrd_strd (operands[0], operands[1], base, offset,
                              false, load))
@@ -30365,6 +30390,8 @@ arm_const_not_ok_for_debug_p (rtx p)
   tree decl_op0 = NULL;
   tree decl_op1 = NULL;
 
+  if (GET_CODE (p) == UNSPEC)
+    return true;
   if (GET_CODE (p) == MINUS)
     {
       if (GET_CODE (XEXP (p, 1)) == SYMBOL_REF)
diff --git a/gcc/config/arm/arm.h b/gcc/config/arm/arm.h
index 336db4b042d..65d6db4d086 100644
--- a/gcc/config/arm/arm.h
+++ b/gcc/config/arm/arm.h
@@ -210,6 +210,11 @@ extern tree arm_fp16_type_node;
 /* FPU supports ARMv8.1 Adv.SIMD extensions.  */
 #define TARGET_NEON_RDMA (TARGET_NEON && arm_arch8_1)
 
+/* Supports for Dot Product AdvSIMD extensions.  */
+#define TARGET_DOTPROD (TARGET_NEON					\
+			&& bitmap_bit_p (arm_active_target.isa,		\
+					isa_bit_dotprod))
+
 /* FPU supports the floating point FP16 instructions for ARMv8.2 and later.  */
 #define TARGET_VFP_FP16INST \
   (TARGET_32BIT && TARGET_HARD_FLOAT && TARGET_VFP5 && arm_fp16_inst)
@@ -1248,7 +1253,7 @@ enum reg_class
    couldn't convert a direct call into an indirect one.  */
 #define CALLER_INTERWORKING_SLOT_SIZE			\
   (TARGET_CALLER_INTERWORKING				\
-   && maybe_nonzero (crtl->outgoing_args_size)		\
+   && may_ne (crtl->outgoing_args_size, 0)		\
    ? UNITS_PER_WORD : 0)
 
 /* If we generate an insn to push BYTES bytes,
diff --git a/gcc/config/arm/arm.md b/gcc/config/arm/arm.md
index f241f9d0b7d..ddb9d8f3590 100644
--- a/gcc/config/arm/arm.md
+++ b/gcc/config/arm/arm.md
@@ -4059,12 +4059,6 @@
     {
       rtx scratch1, scratch2;
 
-      if (operands[2] == CONST1_RTX (SImode))
-        {
-          emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1]));
-          DONE;
-        }
-
       /* Ideally we should use iwmmxt here if we could know that operands[1]
          ends up already living in an iwmmxt register. Otherwise it's
          cheaper to have the alternate code being generated than moving
@@ -4081,18 +4075,6 @@
   "
 )
 
-(define_insn "arm_ashldi3_1bit"
-  [(set (match_operand:DI            0 "s_register_operand" "=r,&r")
-        (ashift:DI (match_operand:DI 1 "s_register_operand" "0,r")
-                   (const_int 1)))
-   (clobber (reg:CC CC_REGNUM))]
-  "TARGET_32BIT"
-  "movs\\t%Q0, %Q1, asl #1\;adc\\t%R0, %R1, %R1"
-  [(set_attr "conds" "clob")
-   (set_attr "length" "8")
-   (set_attr "type" "multiple")]
-)
-
 (define_expand "ashlsi3"
   [(set (match_operand:SI            0 "s_register_operand" "")
 	(ashift:SI (match_operand:SI 1 "s_register_operand" "")
@@ -4128,12 +4110,6 @@
     {
       rtx scratch1, scratch2;
 
-      if (operands[2] == CONST1_RTX (SImode))
-        {
-          emit_insn (gen_arm_ashrdi3_1bit (operands[0], operands[1]));
-          DONE;
-        }
-
       /* Ideally we should use iwmmxt here if we could know that operands[1]
          ends up already living in an iwmmxt register. Otherwise it's
          cheaper to have the alternate code being generated than moving
@@ -4150,18 +4126,6 @@
   "
 )
 
-(define_insn "arm_ashrdi3_1bit"
-  [(set (match_operand:DI              0 "s_register_operand" "=r,&r")
-        (ashiftrt:DI (match_operand:DI 1 "s_register_operand" "0,r")
-                     (const_int 1)))
-   (clobber (reg:CC CC_REGNUM))]
-  "TARGET_32BIT"
-  "movs\\t%R0, %R1, asr #1\;mov\\t%Q0, %Q1, rrx"
-  [(set_attr "conds" "clob")
-   (set_attr "length" "8")
-   (set_attr "type" "multiple")]
-)
-
 (define_expand "ashrsi3"
   [(set (match_operand:SI              0 "s_register_operand" "")
 	(ashiftrt:SI (match_operand:SI 1 "s_register_operand" "")
@@ -4194,12 +4158,6 @@
     {
       rtx scratch1, scratch2;
 
-      if (operands[2] == CONST1_RTX (SImode))
-        {
-          emit_insn (gen_arm_lshrdi3_1bit (operands[0], operands[1]));
-          DONE;
-        }
-
       /* Ideally we should use iwmmxt here if we could know that operands[1]
          ends up already living in an iwmmxt register. Otherwise it's
          cheaper to have the alternate code being generated than moving
@@ -4216,18 +4174,6 @@
   "
 )
 
-(define_insn "arm_lshrdi3_1bit"
-  [(set (match_operand:DI              0 "s_register_operand" "=r,&r")
-        (lshiftrt:DI (match_operand:DI 1 "s_register_operand" "0,r")
-                     (const_int 1)))
-   (clobber (reg:CC CC_REGNUM))]
-  "TARGET_32BIT"
-  "movs\\t%R0, %R1, lsr #1\;mov\\t%Q0, %Q1, rrx"
-  [(set_attr "conds" "clob")
-   (set_attr "length" "8")
-   (set_attr "type" "multiple")]
-)
-
 (define_expand "lshrsi3"
   [(set (match_operand:SI              0 "s_register_operand" "")
 	(lshiftrt:SI (match_operand:SI 1 "s_register_operand" "")
diff --git a/gcc/config/arm/arm_neon_builtins.def b/gcc/config/arm/arm_neon_builtins.def
index 07f0368343a..982eec810da 100644
--- a/gcc/config/arm/arm_neon_builtins.def
+++ b/gcc/config/arm/arm_neon_builtins.def
@@ -331,3 +331,7 @@ VAR11 (STORE1, vst4,
 	v8qi, v4hi, v4hf, v2si, v2sf, di, v16qi, v8hi, v8hf, v4si, v4sf)
 VAR9 (STORE1LANE, vst4_lane,
 	v8qi, v4hi, v4hf, v2si, v2sf, v8hi, v8hf, v4si, v4sf)
+VAR2 (TERNOP, sdot, v8qi, v16qi)
+VAR2 (UTERNOP, udot, v8qi, v16qi)
+VAR2 (MAC_LANE, sdot_lane, v8qi, v16qi)
+VAR2 (UMAC_LANE, udot_lane, v8qi, v16qi)
diff --git a/gcc/config/arm/iterators.md b/gcc/config/arm/iterators.md
index 7acbaf1bb40..a4fb234a846 100644
--- a/gcc/config/arm/iterators.md
+++ b/gcc/config/arm/iterators.md
@@ -410,6 +410,8 @@
 
 (define_int_iterator VFM_LANE_AS [UNSPEC_VFMA_LANE UNSPEC_VFMS_LANE])
 
+(define_int_iterator DOTPROD [UNSPEC_DOT_S UNSPEC_DOT_U])
+
 ;;----------------------------------------------------------------------------
 ;; Mode attributes
 ;;----------------------------------------------------------------------------
@@ -720,6 +722,9 @@
 
 (define_mode_attr pf [(V8QI "p") (V16QI "p") (V2SF "f") (V4SF "f")])
 
+(define_mode_attr VSI2QI [(V2SI "V8QI") (V4SI "V16QI")])
+(define_mode_attr vsi2qi [(V2SI "v8qi") (V4SI "v16qi")])
+
 ;;----------------------------------------------------------------------------
 ;; Code attributes
 ;;----------------------------------------------------------------------------
@@ -816,6 +821,7 @@
   (UNSPEC_VSRA_S_N "s") (UNSPEC_VSRA_U_N "u")
   (UNSPEC_VRSRA_S_N "s") (UNSPEC_VRSRA_U_N "u")
   (UNSPEC_VCVTH_S "s") (UNSPEC_VCVTH_U "u")
+  (UNSPEC_DOT_S "s") (UNSPEC_DOT_U "u")
 ])
 
 (define_int_attr vcvth_op
@@ -1003,3 +1009,6 @@
 
 (define_int_attr mrrc [(VUNSPEC_MRRC "mrrc") (VUNSPEC_MRRC2 "mrrc2")])
 (define_int_attr MRRC [(VUNSPEC_MRRC "MRRC") (VUNSPEC_MRRC2 "MRRC2")])
+
+(define_int_attr opsuffix [(UNSPEC_DOT_S "s8")
+			   (UNSPEC_DOT_U "u8")])
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md
index 6590e8cd894..073c26580dd 100644
--- a/gcc/config/arm/neon.md
+++ b/gcc/config/arm/neon.md
@@ -1221,12 +1221,8 @@
 	gcc_assert (!reg_overlap_mentioned_p (operands[0], operands[1])
 		    || REGNO (operands[0]) == REGNO (operands[1]));
 
-	if (operands[2] == CONST1_RTX (SImode))
-	  /* This clobbers CC.  */
-	  emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1]));
-	else
-	  arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1],
-					 operands[2], operands[3], operands[4]);
+	arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1],
+				       operands[2], operands[3], operands[4]);
       }
     DONE;
   }"
@@ -1325,13 +1321,9 @@
 	gcc_assert (!reg_overlap_mentioned_p (operands[0], operands[1])
 		    || REGNO (operands[0]) == REGNO (operands[1]));
 
-	if (operands[2] == CONST1_RTX (SImode))
-	  /* This clobbers CC.  */
-	  emit_insn (gen_arm_<shift>di3_1bit (operands[0], operands[1]));
-	else
-	  /* This clobbers CC (ASHIFTRT by register only).  */
-	  arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1],
-				 	 operands[2], operands[3], operands[4]);
+	/* This clobbers CC (ASHIFTRT by register only).  */
+	arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1],
+				       operands[2], operands[3], operands[4]);
       }
 
     DONE;
@@ -3044,6 +3036,76 @@
   DONE;
 })
 
+;; These instructions map to the __builtins for the Dot Product operations.
+(define_insn "neon_<sup>dot<vsi2qi>"
+  [(set (match_operand:VCVTI 0 "register_operand" "=w")
+	(plus:VCVTI (match_operand:VCVTI 1 "register_operand" "0")
+		    (unspec:VCVTI [(match_operand:<VSI2QI> 2
+							"register_operand" "w")
+				   (match_operand:<VSI2QI> 3
+							"register_operand" "w")]
+		DOTPROD)))]
+  "TARGET_DOTPROD"
+  "v<sup>dot.<opsuffix>\\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
+  [(set_attr "type" "neon_dot")]
+)
+
+;; These instructions map to the __builtins for the Dot Product
+;; indexed operations.
+(define_insn "neon_<sup>dot_lane<vsi2qi>"
+  [(set (match_operand:VCVTI 0 "register_operand" "=w")
+	(plus:VCVTI (match_operand:VCVTI 1 "register_operand" "0")
+		    (unspec:VCVTI [(match_operand:<VSI2QI> 2
+							"register_operand" "w")
+				   (match_operand:V8QI 3 "register_operand" "t")
+				   (match_operand:SI 4 "immediate_operand" "i")]
+		DOTPROD)))]
+  "TARGET_DOTPROD"
+  {
+    operands[4]
+      = GEN_INT (NEON_ENDIAN_LANE_N (V8QImode, INTVAL (operands[4])));
+    return "v<sup>dot.<opsuffix>\\t%<V_reg>0, %<V_reg>2, %P3[%c4]";
+  }
+  [(set_attr "type" "neon_dot")]
+)
+
+;; These expands map to the Dot Product optab the vectorizer checks for.
+;; The auto-vectorizer expects a dot product builtin that also does an
+;; accumulation into the provided register.
+;; Given the following pattern
+;;
+;; for (i=0; i<len; i++) {
+;;     c = a[i] * b[i];
+;;     r += c;
+;; }
+;; return result;
+;;
+;; This can be auto-vectorized to
+;; r  = a[0]*b[0] + a[1]*b[1] + a[2]*b[2] + a[3]*b[3];
+;;
+;; given enough iterations.  However the vectorizer can keep unrolling the loop
+;; r += a[4]*b[4] + a[5]*b[5] + a[6]*b[6] + a[7]*b[7];
+;; r += a[8]*b[8] + a[9]*b[9] + a[10]*b[10] + a[11]*b[11];
+;; ...
+;;
+;; and so the vectorizer provides r, in which the result has to be accumulated.
+(define_expand "<sup>dot_prod<vsi2qi>"
+  [(set (match_operand:VCVTI 0 "register_operand")
+	(plus:VCVTI (unspec:VCVTI [(match_operand:<VSI2QI> 1
+							"register_operand")
+				   (match_operand:<VSI2QI> 2
+							"register_operand")]
+		     DOTPROD)
+		    (match_operand:VCVTI 3 "register_operand")))]
+  "TARGET_DOTPROD"
+{
+  emit_insn (
+    gen_neon_<sup>dot<vsi2qi> (operands[3], operands[3], operands[1],
+				 operands[2]));
+  emit_insn (gen_rtx_SET (operands[0], operands[3]));
+  DONE;
+})
+
 (define_expand "neon_copysignf<mode>"
   [(match_operand:VCVTF 0 "register_operand")
    (match_operand:VCVTF 1 "register_operand")
diff --git a/gcc/config/arm/t-multilib b/gcc/config/arm/t-multilib
index ec4b76dbc8f..47f3673160a 100644
--- a/gcc/config/arm/t-multilib
+++ b/gcc/config/arm/t-multilib
@@ -68,7 +68,7 @@ v7ve_vfpv4_simd_variants := +simd
 v8_a_nosimd_variants	:= +crc
 v8_a_simd_variants	:= $(call all_feat_combs, simd crypto)
 v8_1_a_simd_variants	:= $(call all_feat_combs, simd crypto)
-v8_2_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto)
+v8_2_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto dotprod)
 
 
 ifneq (,$(HAS_APROFILE))
diff --git a/gcc/config/arm/types.md b/gcc/config/arm/types.md
index 22d993d46a3..03e9cdebb75 100644
--- a/gcc/config/arm/types.md
+++ b/gcc/config/arm/types.md
@@ -316,6 +316,8 @@
 ; neon_cls_q
 ; neon_cnt
 ; neon_cnt_q
+; neon_dot
+; neon_dot_q
 ; neon_ext
 ; neon_ext_q
 ; neon_rbit
@@ -764,6 +766,8 @@
 \
   neon_abs,\
   neon_abs_q,\
+  neon_dot,\
+  neon_dot_q,\
   neon_neg,\
   neon_neg_q,\
   neon_qneg,\
@@ -1110,8 +1114,8 @@
           neon_sub, neon_sub_q, neon_sub_widen, neon_sub_long, neon_qsub,\
           neon_qsub_q, neon_sub_halve, neon_sub_halve_q,\
           neon_sub_halve_narrow_q,\
-          neon_abs, neon_abs_q, neon_neg, neon_neg_q, neon_qneg,\
-          neon_qneg_q, neon_qabs, neon_qabs_q, neon_abd, neon_abd_q,\
+	  neon_abs, neon_abs_q, neon_dot, neon_dot_q, neon_neg, neon_neg_q,\
+	  neon_qneg, neon_qneg_q, neon_qabs, neon_qabs_q, neon_abd, neon_abd_q,\
           neon_abd_long, neon_minmax, neon_minmax_q, neon_compare,\
           neon_compare_q, neon_compare_zero, neon_compare_zero_q,\
           neon_arith_acc, neon_arith_acc_q, neon_reduc_add,\
diff --git a/gcc/config/arm/unspecs.md b/gcc/config/arm/unspecs.md
index 99cfa41b08d..c474f4bb5db 100644
--- a/gcc/config/arm/unspecs.md
+++ b/gcc/config/arm/unspecs.md
@@ -410,4 +410,6 @@
   UNSPEC_VRNDN
   UNSPEC_VRNDP
   UNSPEC_VRNDX
+  UNSPEC_DOT_S
+  UNSPEC_DOT_U
 ])
diff --git a/gcc/config/arm/vfp.md b/gcc/config/arm/vfp.md
index 9521e904d21..a541413c263 100644
--- a/gcc/config/arm/vfp.md
+++ b/gcc/config/arm/vfp.md
@@ -304,9 +304,9 @@
 ;; DImode moves
 
 (define_insn "*movdi_vfp"
-  [(set (match_operand:DI 0 "nonimmediate_di_operand" "=r,r,r,r,q,q,m,w,r,w,w, Uv")
+  [(set (match_operand:DI 0 "nonimmediate_di_operand" "=r,r,r,r,q,q,m,w,!r,w,w, Uv")
        (match_operand:DI 1 "di_operand"              "r,rDa,Db,Dc,mi,mi,q,r,w,w,Uvi,w"))]
-  "TARGET_32BIT && TARGET_HARD_FLOAT && arm_tune != TARGET_CPU_cortexa8
+  "TARGET_32BIT && TARGET_HARD_FLOAT
    && (   register_operand (operands[0], DImode)
        || register_operand (operands[1], DImode))
    && !(TARGET_NEON && CONST_INT_P (operands[1])
@@ -339,71 +339,25 @@
     }
   "
   [(set_attr "type" "multiple,multiple,multiple,multiple,load_8,load_8,store_8,f_mcrr,f_mrrc,ffarithd,f_loadd,f_stored")
-   (set (attr "length") (cond [(eq_attr "alternative" "1,4,5,6") (const_int 8)
+   (set (attr "length") (cond [(eq_attr "alternative" "1") (const_int 8)
                               (eq_attr "alternative" "2") (const_int 12)
                               (eq_attr "alternative" "3") (const_int 16)
+			      (eq_attr "alternative" "4,5,6")
+			       (symbol_ref "arm_count_output_move_double_insns (operands) * 4")
                               (eq_attr "alternative" "9")
                                (if_then_else
                                  (match_test "TARGET_VFP_SINGLE")
                                  (const_int 8)
                                  (const_int 4))]
                               (const_int 4)))
+   (set_attr "predicable"    "yes")
    (set_attr "arm_pool_range"     "*,*,*,*,1020,4096,*,*,*,*,1020,*")
    (set_attr "thumb2_pool_range"     "*,*,*,*,1018,4094,*,*,*,*,1018,*")
    (set_attr "neg_pool_range" "*,*,*,*,1004,0,*,*,*,*,1004,*")
+   (set (attr "ce_count") (symbol_ref "get_attr_length (insn) / 4"))
    (set_attr "arch"           "t2,any,any,any,a,t2,any,any,any,any,any,any")]
 )
 
-(define_insn "*movdi_vfp_cortexa8"
-  [(set (match_operand:DI 0 "nonimmediate_di_operand" "=r,r,r,r,q,q,m,w,!r,w,w, Uv")
-	(match_operand:DI 1 "di_operand"		"r,rDa,Db,Dc,mi,mi,q,r,w,w,Uvi,w"))]
-  "TARGET_32BIT && TARGET_HARD_FLOAT && arm_tune == TARGET_CPU_cortexa8
-    && (   register_operand (operands[0], DImode)
-        || register_operand (operands[1], DImode))
-    && !(TARGET_NEON && CONST_INT_P (operands[1])
-	 && neon_immediate_valid_for_move (operands[1], DImode, NULL, NULL))"
-  "*
-  switch (which_alternative)
-    {
-    case 0: 
-    case 1:
-    case 2:
-    case 3:
-      return \"#\";
-    case 4:
-    case 5:
-    case 6:
-      return output_move_double (operands, true, NULL);
-    case 7:
-      return \"vmov%?\\t%P0, %Q1, %R1\\t%@ int\";
-    case 8:
-      return \"vmov%?\\t%Q0, %R0, %P1\\t%@ int\";
-    case 9:
-      return \"vmov%?.f64\\t%P0, %P1\\t%@ int\";
-    case 10: case 11:
-      return output_move_vfp (operands);
-    default:
-      gcc_unreachable ();
-    }
-  "
-  [(set_attr "type" "multiple,multiple,multiple,multiple,load_8,load_8,store_8,f_mcrr,f_mrrc,ffarithd,f_loadd,f_stored")
-   (set (attr "length") (cond [(eq_attr "alternative" "1") (const_int 8)
-                               (eq_attr "alternative" "2") (const_int 12)
-                               (eq_attr "alternative" "3") (const_int 16)
-                               (eq_attr "alternative" "4,5,6") 
-			       (symbol_ref 
-				"arm_count_output_move_double_insns (operands) \
-                                 * 4")]
-                              (const_int 4)))
-   (set_attr "predicable"    "yes")
-   (set_attr "arm_pool_range"     "*,*,*,*,1018,4094,*,*,*,*,1018,*")
-   (set_attr "thumb2_pool_range"     "*,*,*,*,1018,4094,*,*,*,*,1018,*")
-   (set_attr "neg_pool_range" "*,*,*,*,1004,0,*,*,*,*,1004,*")
-   (set (attr "ce_count") 
-	(symbol_ref "get_attr_length (insn) / 4"))
-   (set_attr "arch"           "t2,any,any,any,a,t2,any,any,any,any,any,any")]
- )
-
 ;; HFmode moves
 
 (define_insn "*movhf_vfp_fp16"
diff --git a/gcc/config/cris/cris.h b/gcc/config/cris/cris.h
index f9149c717a7..892a3724393 100644
--- a/gcc/config/cris/cris.h
+++ b/gcc/config/cris/cris.h
@@ -998,7 +998,7 @@ enum cris_symbol_type
 /* (no definitions) */
 
 
-/* Node: SDB and DWARF */
+/* Node: DWARF */
 /* (no definitions) */
 
 /* Node: Misc */
diff --git a/gcc/config/dbxcoff.h b/gcc/config/dbxcoff.h
index e5eef64f60e..c20b4fe77b1 100644
--- a/gcc/config/dbxcoff.h
+++ b/gcc/config/dbxcoff.h
@@ -25,10 +25,10 @@ along with GCC; see the file COPYING3.  If not see
 
 #define DBX_DEBUGGING_INFO 1
 
-/* Generate SDB debugging information by default.  */
+/* Generate DBX debugging information by default.  */
 
 #ifndef PREFERRED_DEBUGGING_TYPE
-#define PREFERRED_DEBUGGING_TYPE SDB_DEBUG
+#define PREFERRED_DEBUGGING_TYPE DBX_DEBUG
 #endif
 
 /* Be function-relative for block and source line stab directives.  */
diff --git a/gcc/config/ft32/ft32.c b/gcc/config/ft32/ft32.c
index 99e93821b3a..8a041f167a8 100644
--- a/gcc/config/ft32/ft32.c
+++ b/gcc/config/ft32/ft32.c
@@ -869,6 +869,8 @@ static bool
 ft32_addr_space_legitimate_address_p (machine_mode mode, rtx x, bool strict,
                                       addr_space_t as ATTRIBUTE_UNUSED)
 {
+  int max_offset = TARGET_FT32B ? 16384 : 128;
+
   if (mode != BLKmode)
     {
       if (GET_CODE (x) == PLUS)
@@ -878,8 +880,9 @@ ft32_addr_space_legitimate_address_p (machine_mode mode, rtx x, bool strict,
           op2 = XEXP (x, 1);
           if (GET_CODE (op1) == REG
               && CONST_INT_P (op2)
-              && INTVAL (op2) >= -128
-              && INTVAL (op2) < 128 && reg_ok_for_base_p (op1, strict))
+              && (-max_offset <= INTVAL (op2))
+              && (INTVAL (op2) < max_offset)
+              && reg_ok_for_base_p (op1, strict))
             goto yes;
           if (GET_CODE (op1) == SYMBOL_REF && CONST_INT_P (op2))
             goto yes;
diff --git a/gcc/config/ft32/ft32.h b/gcc/config/ft32/ft32.h
index d52bb9af17c..8bb0d399a0c 100644
--- a/gcc/config/ft32/ft32.h
+++ b/gcc/config/ft32/ft32.h
@@ -39,6 +39,7 @@
 
 #undef LIB_SPEC
 #define LIB_SPEC "%{!shared:%{!symbolic:-lc}} \
+                   %{mcompress:--relax} \
                    %{msim:-Tsim.ld}"
 
 #undef  LINK_SPEC
@@ -199,12 +200,12 @@ enum reg_class
 
 #define GLOBAL_ASM_OP "\t.global\t"
 
-#define JUMP_TABLES_IN_TEXT_SECTION 1
+#define JUMP_TABLES_IN_TEXT_SECTION (TARGET_NOPM ? 0 : 1)
 
 /* This is how to output an element of a case-vector that is absolute.  */
 
 #define ASM_OUTPUT_ADDR_VEC_ELT(FILE, VALUE)  \
-    fprintf (FILE, "\tjmp\t.L%d\n", VALUE);				\
+    fprintf (FILE, "\t.long\t.L%d\n", VALUE);				\
 
 /* Passing Arguments in Registers */
 
@@ -469,7 +470,7 @@ do { \
 #define ADDR_SPACE_PM 1
 
 #define REGISTER_TARGET_PRAGMAS() do { \
-  c_register_addr_space ("__flash__", ADDR_SPACE_PM); \
+  c_register_addr_space ("__flash__", TARGET_NOPM ? 0 : ADDR_SPACE_PM); \
 } while (0);
 
 extern int ft32_is_mem_pm(rtx o);
diff --git a/gcc/config/ft32/ft32.md b/gcc/config/ft32/ft32.md
index 984c3b67e32..2e772faf72f 100644
--- a/gcc/config/ft32/ft32.md
+++ b/gcc/config/ft32/ft32.md
@@ -777,8 +777,12 @@
    (clobber (match_scratch:SI 2 "=&r"))
   ]
   ""
-  "ldk.l\t$cc,%l1\;ashl.l\t%2,%0,2\;add.l\t%2,%2,$cc\;jmpi\t%2"
-  )
+  {
+    if (TARGET_NOPM)
+      return \"ldk.l\t$cc,%l1\;ashl.l\t%2,%0,2\;add.l\t%2,%2,$cc\;ldi.l\t%2,%2,0\;jmpi\t%2\";
+    else
+      return \"ldk.l\t$cc,%l1\;ashl.l\t%2,%0,2\;add.l\t%2,%2,$cc\;lpmi.l\t%2,%2,0\;jmpi\t%2\";
+  })
 
 ;; -------------------------------------------------------------------------
 ;; Atomic exchange instruction
diff --git a/gcc/config/ft32/ft32.opt b/gcc/config/ft32/ft32.opt
index ba01c81ecf1..9e75f340335 100644
--- a/gcc/config/ft32/ft32.opt
+++ b/gcc/config/ft32/ft32.opt
@@ -29,3 +29,15 @@ Use LRA instead of reload.
 mnodiv
 Target Report Mask(NODIV)
 Avoid use of the DIV and MOD instructions
+
+mft32b
+Target Report Mask(FT32B)
+target the FT32B architecture
+
+mcompress
+Target Report Mask(COMPRESS)
+enable FT32B code compression
+
+mnopm
+Target Report Mask(NOPM)
+Avoid placing any readable data in program memory
diff --git a/gcc/config/gnu-user.h b/gcc/config/gnu-user.h
index a967b69a350..df17b180906 100644
--- a/gcc/config/gnu-user.h
+++ b/gcc/config/gnu-user.h
@@ -162,11 +162,13 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
   LD_STATIC_OPTION " --whole-archive -lasan --no-whole-archive " \
   LD_DYNAMIC_OPTION "}}%{!static-libasan:-lasan}"
 #undef LIBTSAN_EARLY_SPEC
-#define LIBTSAN_EARLY_SPEC "%{static-libtsan:%{!shared:" \
+#define LIBTSAN_EARLY_SPEC "%{!shared:libtsan_preinit%O%s} " \
+  "%{static-libtsan:%{!shared:" \
   LD_STATIC_OPTION " --whole-archive -ltsan --no-whole-archive " \
   LD_DYNAMIC_OPTION "}}%{!static-libtsan:-ltsan}"
 #undef LIBLSAN_EARLY_SPEC
-#define LIBLSAN_EARLY_SPEC "%{static-liblsan:%{!shared:" \
+#define LIBLSAN_EARLY_SPEC "%{!shared:liblsan_preinit%O%s} " \
+  "%{static-liblsan:%{!shared:" \
   LD_STATIC_OPTION " --whole-archive -llsan --no-whole-archive " \
   LD_DYNAMIC_OPTION "}}%{!static-liblsan:-llsan}"
 #endif
diff --git a/gcc/config/i386/avx512dqintrin.h b/gcc/config/i386/avx512dqintrin.h
index 88e8adb18c5..8e887d8c5ba 100644
--- a/gcc/config/i386/avx512dqintrin.h
+++ b/gcc/config/i386/avx512dqintrin.h
@@ -1160,16 +1160,63 @@ extern __inline __m128d
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_reduce_sd (__m128d __A, __m128d __B, int __C)
 {
-  return (__m128d) __builtin_ia32_reducesd ((__v2df) __A,
-						 (__v2df) __B, __C);
+  return (__m128d) __builtin_ia32_reducesd_mask ((__v2df) __A,
+						 (__v2df) __B, __C,
+						 (__v2df) _mm_setzero_pd (),
+						 (__mmask8) -1);
+}
+
+extern __inline __m128d
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mask_reduce_sd (__m128d __W,  __mmask8 __U, __m128d __A,
+		    __m128d __B, int __C)
+{
+  return (__m128d) __builtin_ia32_reducesd_mask ((__v2df) __A,
+						 (__v2df) __B, __C,
+						 (__v2df) __W,
+						 (__mmask8) __U);
+}
+
+extern __inline __m128d
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskz_reduce_sd (__mmask8 __U, __m128d __A, __m128d __B, int __C)
+{
+  return (__m128d) __builtin_ia32_reducesd_mask ((__v2df) __A,
+						 (__v2df) __B, __C,
+						 (__v2df) _mm_setzero_pd (),
+						 (__mmask8) __U);
 }
 
 extern __inline __m128
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_reduce_ss (__m128 __A, __m128 __B, int __C)
 {
-  return (__m128) __builtin_ia32_reducess ((__v4sf) __A,
-						(__v4sf) __B, __C);
+  return (__m128) __builtin_ia32_reducess_mask ((__v4sf) __A,
+						(__v4sf) __B, __C,
+						(__v4sf) _mm_setzero_ps (),
+						(__mmask8) -1);
+}
+
+
+extern __inline __m128
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mask_reduce_ss (__m128 __W,  __mmask8 __U, __m128 __A,
+		    __m128 __B, int __C)
+{
+  return (__m128) __builtin_ia32_reducess_mask ((__v4sf) __A,
+						(__v4sf) __B, __C,
+						(__v4sf) __W,
+						(__mmask8) __U);
+}
+
+extern __inline __m128
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskz_reduce_ss (__mmask8 __U, __m128 __A, __m128 __B, int __C)
+{
+  return (__m128) __builtin_ia32_reducess_mask ((__v4sf) __A,
+						(__v4sf) __B, __C,
+						(__v4sf) _mm_setzero_ps (),
+						(__mmask8) __U);
 }
 
 extern __inline __m128d
@@ -2449,12 +2496,34 @@ _mm512_fpclass_ps_mask (__m512 __A, const int __imm)
 						 (int) (c),(__mmask8)-1))
 
 #define _mm_reduce_sd(A, B, C)						\
-  ((__m128d) __builtin_ia32_reducesd ((__v2df)(__m128d)(A),	\
-    (__v2df)(__m128d)(B), (int)(C)))					\
+  ((__m128d) __builtin_ia32_reducesd_mask ((__v2df)(__m128d)(A),	\
+    (__v2df)(__m128d)(B), (int)(C), (__v2df) _mm_setzero_pd (),		\
+    (__mmask8)-1))
+
+#define _mm_mask_reduce_sd(W, U, A, B, C)				\
+  ((__m128d) __builtin_ia32_reducesd_mask ((__v2df)(__m128d)(A),	\
+    (__v2df)(__m128d)(B), (int)(C), (__v2df)(__m128d)(W), (__mmask8)(U)))
+
+#define _mm_maskz_reduce_sd(U, A, B, C)					\
+  ((__m128d) __builtin_ia32_reducesd_mask ((__v2df)(__m128d)(A),	\
+    (__v2df)(__m128d)(B), (int)(C), (__v2df) _mm_setzero_pd (),		\
+    (__mmask8)(U)))
 
 #define _mm_reduce_ss(A, B, C)						\
-  ((__m128) __builtin_ia32_reducess ((__v4sf)(__m128)(A),		\
-    (__v4sf)(__m128)(A), (int)(C)))					\
+  ((__m128) __builtin_ia32_reducess_mask ((__v4sf)(__m128)(A),		\
+    (__v4sf)(__m128)(B), (int)(C), (__v4sf) _mm_setzero_ps (),		\
+    (__mmask8)-1))
+
+#define _mm_mask_reduce_ss(W, U, A, B, C)				\
+  ((__m128) __builtin_ia32_reducess_mask ((__v4sf)(__m128)(A),		\
+    (__v4sf)(__m128)(B), (int)(C), (__v4sf)(__m128)(W), (__mmask8)(U)))
+
+#define _mm_maskz_reduce_ss(U, A, B, C)					\
+  ((__m128) __builtin_ia32_reducess_mask ((__v4sf)(__m128)(A),		\
+    (__v4sf)(__m128)(B), (int)(C), (__v4sf) _mm_setzero_ps (),		\
+    (__mmask8)(U)))
+
+
 
 #endif
 
diff --git a/gcc/config/i386/avx512fintrin.h b/gcc/config/i386/avx512fintrin.h
index 72f57f7b6c9..5dc5fae1081 100644
--- a/gcc/config/i386/avx512fintrin.h
+++ b/gcc/config/i386/avx512fintrin.h
@@ -14005,6 +14005,326 @@ _mm512_mask_cmp_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y, const int __P)
 
 extern __inline __mmask8
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpeq_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_EQ_OQ,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpeq_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_EQ_OQ,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmplt_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_LT_OS,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmplt_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_LT_OS,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmple_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_LE_OS,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmple_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_LE_OS,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpunord_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_UNORD_Q,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpunord_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_UNORD_Q,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpneq_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_NEQ_UQ,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpneq_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_NEQ_UQ,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpnlt_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_NLT_US,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpnlt_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_NLT_US,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpnle_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_NLE_US,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpnle_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_NLE_US,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpord_pd_mask (__m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_ORD_Q,
+						  (__mmask8) -1,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpord_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y)
+{
+  return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X,
+						  (__v8df) __Y, _CMP_ORD_Q,
+						  (__mmask8) __U,
+						  _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpeq_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_EQ_OQ,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpeq_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_EQ_OQ,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmplt_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_LT_OS,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmplt_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_LT_OS,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmple_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_LE_OS,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmple_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_LE_OS,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpunord_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_UNORD_Q,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpunord_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_UNORD_Q,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpneq_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_NEQ_UQ,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpneq_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_NEQ_UQ,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpnlt_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_NLT_US,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpnlt_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_NLT_US,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpnle_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_NLE_US,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpnle_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_NLE_US,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_cmpord_ps_mask (__m512 __X, __m512 __Y)
+{
+  return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_ORD_Q,
+						   (__mmask16) -1,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask16
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_cmpord_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y)
+{
+   return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X,
+						   (__v16sf) __Y, _CMP_ORD_Q,
+						   (__mmask16) __U,
+						   _MM_FROUND_CUR_DIRECTION);
+}
+
+extern __inline __mmask8
+__attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_cmp_sd_mask (__m128d __X, __m128d __Y, const int __P)
 {
   return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X,
diff --git a/gcc/config/i386/cet.c b/gcc/config/i386/cet.c
new file mode 100644
index 00000000000..a53c499fd92
--- /dev/null
+++ b/gcc/config/i386/cet.c
@@ -0,0 +1,76 @@
+/* Functions for CET/x86.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "tm.h"
+#include "output.h"
+#include "linux-common.h"
+
+void
+file_end_indicate_exec_stack_and_cet (void)
+{
+  file_end_indicate_exec_stack ();
+
+  if (flag_cf_protection == CF_NONE)
+    return;
+
+  unsigned int feature_1 = 0;
+
+  if (TARGET_IBT)
+    /* GNU_PROPERTY_X86_FEATURE_1_IBT.  */
+    feature_1 |= 0x1;
+
+  if (TARGET_SHSTK)
+    /* GNU_PROPERTY_X86_FEATURE_1_SHSTK.  */
+    feature_1 |= 0x2;
+
+  if (feature_1)
+    {
+      int p2align = ptr_mode == SImode ? 2 : 3;
+
+      /* Generate GNU_PROPERTY_X86_FEATURE_1_XXX.  */
+      switch_to_section (get_section (".note.gnu.property",
+				      SECTION_NOTYPE, NULL));
+
+      ASM_OUTPUT_ALIGN (asm_out_file, p2align);
+      /* name length.  */
+      fprintf (asm_out_file, ASM_LONG " 1f - 0f\n");
+      /* data length.  */
+      fprintf (asm_out_file, ASM_LONG " 4f - 1f\n");
+      /* note type: NT_GNU_PROPERTY_TYPE_0.  */
+      fprintf (asm_out_file, ASM_LONG " 5\n");
+      ASM_OUTPUT_LABEL (asm_out_file, "0");
+      /* vendor name: "GNU".  */
+      fprintf (asm_out_file, STRING_ASM_OP " \"GNU\"\n");
+      ASM_OUTPUT_LABEL (asm_out_file, "1");
+      ASM_OUTPUT_ALIGN (asm_out_file, p2align);
+      /* pr_type: GNU_PROPERTY_X86_FEATURE_1_AND.  */
+      fprintf (asm_out_file, ASM_LONG " 0xc0000002\n");
+      /* pr_datasz.  */\
+      fprintf (asm_out_file, ASM_LONG " 3f - 2f\n");
+      ASM_OUTPUT_LABEL (asm_out_file, "2");
+      /* GNU_PROPERTY_X86_FEATURE_1_XXX.  */
+      fprintf (asm_out_file, ASM_LONG " 0x%x\n", feature_1);
+      ASM_OUTPUT_LABEL (asm_out_file, "3");
+      ASM_OUTPUT_ALIGN (asm_out_file, p2align);
+      ASM_OUTPUT_LABEL (asm_out_file, "4");
+    }
+}
diff --git a/gcc/config/i386/cetintrin.h b/gcc/config/i386/cetintrin.h
new file mode 100644
index 00000000000..b15a776d7f8
--- /dev/null
+++ b/gcc/config/i386/cetintrin.h
@@ -0,0 +1,134 @@
+/* Copyright (C) 2015-2017 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#if !defined _IMMINTRIN_H_INCLUDED
+# error "Never use <cetintrin.h> directly; include <x86intrin.h> instead."
+#endif
+
+#ifndef _CETINTRIN_H_INCLUDED
+#define _CETINTRIN_H_INCLUDED
+
+#ifndef __SHSTK__
+#pragma GCC push_options
+#pragma GCC target ("shstk")
+#define __DISABLE_SHSTK__
+#endif /* __SHSTK__ */
+
+extern __inline unsigned int
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_rdsspd (unsigned int __B)
+{
+  return __builtin_ia32_rdsspd (__B);
+}
+
+#ifdef __x86_64__
+extern __inline unsigned long long
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_rdsspq (unsigned long long __B)
+{
+  return __builtin_ia32_rdsspq (__B);
+}
+#endif
+
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_incsspd (unsigned int __B)
+{
+  __builtin_ia32_incsspd (__B);
+}
+
+#ifdef __x86_64__
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_incsspq (unsigned long long __B)
+{
+  __builtin_ia32_incsspq (__B);
+}
+#endif
+
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_saveprevssp (void)
+{
+  __builtin_ia32_saveprevssp ();
+}
+
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_rstorssp (void *__B)
+{
+  __builtin_ia32_rstorssp (__B);
+}
+
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_wrssd (unsigned int __B, void *__C)
+{
+  __builtin_ia32_wrssd (__B, __C);
+}
+
+#ifdef __x86_64__
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_wrssq (unsigned long long __B, void *__C)
+{
+  __builtin_ia32_wrssq (__B, __C);
+}
+#endif
+
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_wrussd (unsigned int __B, void *__C)
+{
+  __builtin_ia32_wrussd (__B, __C);
+}
+
+#ifdef __x86_64__
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_wrussq (unsigned long long __B, void *__C)
+{
+  __builtin_ia32_wrussq (__B, __C);
+}
+#endif
+
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_setssbsy (void)
+{
+  __builtin_ia32_setssbsy ();
+}
+
+extern __inline void
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_clrssbsy (void *__B)
+{
+  __builtin_ia32_clrssbsy (__B);
+}
+
+#ifdef __DISABLE_SHSTK__
+#undef __DISABLE_SHSTK__
+#pragma GCC pop_options
+#endif /* __DISABLE_SHSTK__ */
+
+#endif /* _CETINTRIN_H_INCLUDED.  */
diff --git a/gcc/config/i386/constraints.md b/gcc/config/i386/constraints.md
index 98c05c9ebab..619b465f059 100644
--- a/gcc/config/i386/constraints.md
+++ b/gcc/config/i386/constraints.md
@@ -332,6 +332,11 @@
    of it satisfies the e constraint."
   (match_operand 0 "x86_64_hilo_int_operand"))
 
+(define_constraint "Wf"
+  "32-bit signed integer constant zero extended from word size
+   to double word size."
+  (match_operand 0 "x86_64_dwzext_immediate_operand"))
+
 (define_constraint "Z"
   "32-bit unsigned integer constant, or a symbolic reference known
    to fit that range (for immediate operands in zero-extending x86-64
diff --git a/gcc/config/i386/cpuid.h b/gcc/config/i386/cpuid.h
index b3b0f912c98..8cb1848dff5 100644
--- a/gcc/config/i386/cpuid.h
+++ b/gcc/config/i386/cpuid.h
@@ -97,12 +97,15 @@
 #define bit_AVX512VBMI	(1 << 1)
 #define bit_PKU	(1 << 3)
 #define bit_OSPKE	(1 << 4)
+#define bit_SHSTK	(1 << 7)
+#define bit_GFNI	(1 << 8)
 #define bit_AVX512VPOPCNTDQ	(1 << 14)
 #define bit_RDPID	(1 << 22)
 
 /* %edx */
 #define bit_AVX5124VNNIW (1 << 2)
 #define bit_AVX5124FMAPS (1 << 3)
+#define bit_IBT	(1 << 20)
 
 /* XFEATURE_ENABLED_MASK register bits (%eax == 13, %ecx == 0) */
 #define bit_BNDREGS     (1 << 3)
diff --git a/gcc/config/i386/cygming.h b/gcc/config/i386/cygming.h
index a731e2f6c6a..1ed9b170d43 100644
--- a/gcc/config/i386/cygming.h
+++ b/gcc/config/i386/cygming.h
@@ -19,7 +19,6 @@ along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
 #define DBX_DEBUGGING_INFO 1
-#define SDB_DEBUGGING_INFO 1
 #if TARGET_64BIT_DEFAULT || defined (HAVE_GAS_PE_SECREL32_RELOC)
 #define DWARF2_DEBUGGING_INFO 1
 #endif
@@ -308,8 +307,7 @@ do {						\
 #define TARGET_SECTION_TYPE_FLAGS  i386_pe_section_type_flags
 
 /* Write the extra assembler code needed to declare a function
-   properly.  If we are generating SDB debugging information, this
-   will happen automatically, so we only need to handle other cases.  */
+   properly.  */
 #undef ASM_DECLARE_FUNCTION_NAME
 #define ASM_DECLARE_FUNCTION_NAME(FILE, NAME, DECL) \
   i386_pe_start_function (FILE, NAME, DECL)
diff --git a/gcc/config/i386/driver-i386.c b/gcc/config/i386/driver-i386.c
index 4e7fda68281..973abddcc67 100644
--- a/gcc/config/i386/driver-i386.c
+++ b/gcc/config/i386/driver-i386.c
@@ -417,6 +417,8 @@ const char *host_detect_local_cpu (int argc, const char **argv)
   unsigned int has_avx512vbmi = 0, has_avx512ifma = 0, has_clwb = 0;
   unsigned int has_mwaitx = 0, has_clzero = 0, has_pku = 0, has_rdpid = 0;
   unsigned int has_avx5124fmaps = 0, has_avx5124vnniw = 0;
+  unsigned int has_gfni = 0;
+  unsigned int has_ibt = 0, has_shstk = 0;
 
   bool arch;
 
@@ -506,9 +508,13 @@ const char *host_detect_local_cpu (int argc, const char **argv)
       has_avx512vbmi = ecx & bit_AVX512VBMI;
       has_pku = ecx & bit_OSPKE;
       has_rdpid = ecx & bit_RDPID;
+      has_gfni = ecx & bit_GFNI;
 
       has_avx5124vnniw = edx & bit_AVX5124VNNIW;
       has_avx5124fmaps = edx & bit_AVX5124FMAPS;
+
+      has_shstk = ecx & bit_SHSTK;
+      has_ibt = edx & bit_IBT;
     }
 
   if (max_level >= 13)
@@ -1050,6 +1056,9 @@ const char *host_detect_local_cpu (int argc, const char **argv)
       const char *clzero  = has_clzero  ? " -mclzero"  : " -mno-clzero";
       const char *pku = has_pku ? " -mpku" : " -mno-pku";
       const char *rdpid = has_rdpid ? " -mrdpid" : " -mno-rdpid";
+      const char *gfni = has_gfni ? " -mgfni" : " -mno-gfni";
+      const char *ibt = has_ibt ? " -mibt" : " -mno-ibt";
+      const char *shstk = has_shstk ? " -mshstk" : " -mno-shstk";
       options = concat (options, mmx, mmx3dnow, sse, sse2, sse3, ssse3,
 			sse4a, cx16, sahf, movbe, aes, sha, pclmul,
 			popcnt, abm, lwp, fma, fma4, xop, bmi, sgx, bmi2,
@@ -1059,7 +1068,7 @@ const char *host_detect_local_cpu (int argc, const char **argv)
 			avx512cd, avx512pf, prefetchwt1, clflushopt,
 			xsavec, xsaves, avx512dq, avx512bw, avx512vl,
 			avx512ifma, avx512vbmi, avx5124fmaps, avx5124vnniw,
-			clwb, mwaitx, clzero, pku, rdpid, NULL);
+			clwb, mwaitx, clzero, pku, rdpid, gfni, ibt, shstk, NULL);
     }
 
 done:
diff --git a/gcc/config/i386/gas.h b/gcc/config/i386/gas.h
index 9b42787f6b1..862c1c2cb83 100644
--- a/gcc/config/i386/gas.h
+++ b/gcc/config/i386/gas.h
@@ -40,10 +40,6 @@ along with GCC; see the file COPYING3.  If not see
 #undef DBX_NO_XREFS
 #undef DBX_CONTIN_LENGTH
 
-/* Ask for COFF symbols.  */
-
-#define SDB_DEBUGGING_INFO 1
-
 /* Output #ident as a .ident.  */
 
 #undef TARGET_ASM_OUTPUT_IDENT
diff --git a/gcc/config/i386/gfniintrin.h b/gcc/config/i386/gfniintrin.h
new file mode 100644
index 00000000000..f4ca01c5b11
--- /dev/null
+++ b/gcc/config/i386/gfniintrin.h
@@ -0,0 +1,229 @@
+/* Copyright (C) 2017 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef _IMMINTRIN_H_INCLUDED
+#error "Never use <gfniintrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#ifndef _GFNIINTRIN_H_INCLUDED
+#define _GFNIINTRIN_H_INCLUDED
+
+#ifndef __GFNI__
+#pragma GCC push_options
+#pragma GCC target("gfni")
+#define __DISABLE_GFNI__
+#endif /* __GFNI__ */
+
+#ifdef __OPTIMIZE__
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_gf2p8affineinv_epi64_epi8 (__m128i __A, __m128i __B, const int __C)
+{
+  return (__m128i) __builtin_ia32_vgf2p8affineinvqb_v16qi ((__v16qi) __A,
+							   (__v16qi) __B,
+							    __C);
+}
+#else
+#define _mm_gf2p8affineinv_epi64_epi8(A, B, C)				   \
+  ((__m128i) __builtin_ia32_vgf2p8affineinvqb_v16qi((__v16qi)(__m128i)(A), \
+					   (__v16qi)(__m128i)(B), (int)(C)))
+#endif
+
+#ifdef __DISABLE_GFNI__
+#undef __DISABLE_GFNI__
+#pragma GCC pop_options
+#endif /* __DISABLE_GFNI__ */
+
+#if !defined(__GFNI__) || !defined(__AVX__)
+#pragma GCC push_options
+#pragma GCC target("gfni,avx")
+#define __DISABLE_GFNIAVX__
+#endif /* __GFNIAVX__ */
+
+#ifdef __OPTIMIZE__
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_gf2p8affineinv_epi64_epi8 (__m256i __A, __m256i __B, const int __C)
+{
+  return (__m256i) __builtin_ia32_vgf2p8affineinvqb_v32qi ((__v32qi) __A,
+							   (__v32qi) __B,
+							    __C);
+}
+#else
+#define _mm256_gf2p8affineinv_epi64_epi8(A, B, C)			   \
+  ((__m256i) __builtin_ia32_vgf2p8affineinvqb_v32qi((__v32qi)(__m256i)(A), \
+						    (__v32qi)(__m256i)(B), \
+						    (int)(C)))
+#endif
+
+#ifdef __DISABLE_GFNIAVX__
+#undef __DISABLE_GFNIAVX__
+#pragma GCC pop_options
+#endif /* __GFNIAVX__ */
+
+#if !defined(__GFNI__) || !defined(__AVX512VL__)
+#pragma GCC push_options
+#pragma GCC target("gfni,avx512vl")
+#define __DISABLE_GFNIAVX512VL__
+#endif /* __GFNIAVX512VL__ */
+
+#ifdef __OPTIMIZE__
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mask_gf2p8affineinv_epi64_epi8 (__m128i __A, __mmask16 __B, __m128i __C,
+				    __m128i __D, const int __E)
+{
+  return (__m128i) __builtin_ia32_vgf2p8affineinvqb_v16qi_mask ((__v16qi) __C,
+								(__v16qi) __D,
+								 __E,
+								(__v16qi)__A,
+								 __B);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskz_gf2p8affineinv_epi64_epi8 (__mmask16 __A, __m128i __B, __m128i __C,
+				     const int __D)
+{
+  return (__m128i) __builtin_ia32_vgf2p8affineinvqb_v16qi_mask ((__v16qi) __B,
+						(__v16qi) __C, __D,
+						(__v16qi) _mm_setzero_si128 (),
+						 __A);
+}
+#else
+#define _mm_mask_gf2p8affineinv_epi64_epi8(A, B, C, D, E) 		   \
+  ((__m128i) __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(		   \
+			(__v16qi)(__m128i)(C), (__v16qi)(__m128i)(D),      \
+			(int)(E), (__v16qi)(__m128i)(A), (__mmask16)(B)))
+#define _mm_maskz_gf2p8affineinv_epi64_epi8(A, B, C, D) \
+  ((__m128i) __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(		   \
+			(__v16qi)(__m128i)(B), (__v16qi)(__m128i)(C),	   \
+			(int)(D), (__v16qi)(__m128i) _mm_setzero_si128 (), \
+			(__mmask16)(A)))
+#endif
+
+#ifdef __DISABLE_GFNIAVX512VL__
+#undef __DISABLE_GFNIAVX512VL__
+#pragma GCC pop_options
+#endif /* __GFNIAVX512VL__ */
+
+#if !defined(__GFNI__) || !defined(__AVX512VL__) || !defined(__AVX512BW__)
+#pragma GCC push_options
+#pragma GCC target("gfni,avx512vl,avx512bw")
+#define __DISABLE_GFNIAVX512VLBW__
+#endif /* __GFNIAVX512VLBW__ */
+
+#ifdef __OPTIMIZE__
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_mask_gf2p8affineinv_epi64_epi8 (__m256i __A, __mmask32 __B,
+				       __m256i __C, __m256i __D, const int __E)
+{
+  return (__m256i) __builtin_ia32_vgf2p8affineinvqb_v32qi_mask ((__v32qi) __C,
+								(__v32qi) __D,
+							 	 __E,
+								(__v32qi)__A,
+								 __B);
+}
+
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskz_gf2p8affineinv_epi64_epi8 (__mmask32 __A, __m256i __B,
+					__m256i __C, const int __D)
+{
+  return (__m256i) __builtin_ia32_vgf2p8affineinvqb_v32qi_mask ((__v32qi) __B,
+				      (__v32qi) __C, __D,
+				      (__v32qi) _mm256_setzero_si256 (), __A);
+}
+#else
+#define _mm256_mask_gf2p8affineinv_epi64_epi8(A, B, C, D, E)		\
+  ((__m256i) __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(		\
+	(__v32qi)(__m256i)(C), (__v32qi)(__m256i)(D), (int)(E),		\
+	(__v32qi)(__m256i)(A), (__mmask32)(B)))
+#define _mm256_maskz_gf2p8affineinv_epi64_epi8(A, B, C, D)		\
+  ((__m256i) __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(		\
+	(__v32qi)(__m256i)(B), (__v32qi)(__m256i)(C), (int)(D),		\
+	(__v32qi)(__m256i) _mm256_setzero_si256 (), (__mmask32)(A)))
+#endif
+
+#ifdef __DISABLE_GFNIAVX512VLBW__
+#undef __DISABLE_GFNIAVX512VLBW__
+#pragma GCC pop_options
+#endif /* __GFNIAVX512VLBW__ */
+
+#if !defined(__GFNI__) || !defined(__AVX512F__) || !defined(__AVX512BW__)
+#pragma GCC push_options
+#pragma GCC target("gfni,avx512f,avx512bw")
+#define __DISABLE_GFNIAVX512FBW__
+#endif /* __GFNIAVX512FBW__ */
+
+#ifdef __OPTIMIZE__
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_gf2p8affineinv_epi64_epi8 (__m512i __A, __mmask64 __B, __m512i __C,
+				       __m512i __D, const int __E)
+{
+  return (__m512i) __builtin_ia32_vgf2p8affineinvqb_v64qi_mask ((__v64qi) __C,
+								(__v64qi) __D,
+								 __E,
+								(__v64qi)__A,
+								 __B);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_maskz_gf2p8affineinv_epi64_epi8 (__mmask64 __A, __m512i __B,
+					__m512i __C, const int __D)
+{
+  return (__m512i) __builtin_ia32_vgf2p8affineinvqb_v64qi_mask ((__v64qi) __B,
+				(__v64qi) __C, __D,
+				(__v64qi) _mm512_setzero_si512 (), __A);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_gf2p8affineinv_epi64_epi8 (__m512i __A, __m512i __B, const int __C)
+{
+  return (__m512i) __builtin_ia32_vgf2p8affineinvqb_v64qi ((__v64qi) __A,
+							   (__v64qi) __B, __C);
+}
+#else
+#define _mm512_mask_gf2p8affineinv_epi64_epi8(A, B, C, D, E) 		\
+  ((__m512i) __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(		\
+	(__v64qi)(__m512i)(C), (__v64qi)(__m512i)(D), (int)(E),		\
+	(__v64qi)(__m512i)(A), (__mmask64)(B)))
+#define _mm512_maskz_gf2p8affineinv_epi64_epi8(A, B, C, D)		\
+  ((__m512i) __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(		\
+	(__v64qi)(__m512i)(B), (__v64qi)(__m512i)(C), (int)(D),		\
+	(__v64qi)(__m512i) _mm512_setzero_si512 (), (__mmask64)(A)))
+#define _mm512_gf2p8affineinv_epi64_epi8(A, B, C)			\
+  ((__m512i) __builtin_ia32_vgf2p8affineinvqb_v64qi (			\
+	(__v64qi)(__m512i)(A), (__v64qi)(__m512i)(B), (int)(C)))
+#endif
+
+#ifdef __DISABLE_GFNIAVX512FBW__
+#undef __DISABLE_GFNIAVX512FBW__
+#pragma GCC pop_options
+#endif /* __GFNIAVX512FBW__ */
+
+#endif /* _GFNIINTRIN_H_INCLUDED */
diff --git a/gcc/config/i386/i386-builtin-types.def b/gcc/config/i386/i386-builtin-types.def
index 8d584dbe940..5b3b96ea2d0 100644
--- a/gcc/config/i386/i386-builtin-types.def
+++ b/gcc/config/i386/i386-builtin-types.def
@@ -286,7 +286,9 @@ DEF_FUNCTION_TYPE (V8SI, V8SI)
 DEF_FUNCTION_TYPE (VOID, PCVOID)
 DEF_FUNCTION_TYPE (VOID, PVOID)
 DEF_FUNCTION_TYPE (VOID, UINT64)
+DEF_FUNCTION_TYPE (VOID, UINT64, PVOID)
 DEF_FUNCTION_TYPE (VOID, UNSIGNED)
+DEF_FUNCTION_TYPE (VOID, UNSIGNED, PVOID)
 DEF_FUNCTION_TYPE (INT, PUSHORT)
 DEF_FUNCTION_TYPE (INT, PUNSIGNED)
 DEF_FUNCTION_TYPE (INT, PULONGLONG)
@@ -1210,3 +1212,9 @@ DEF_FUNCTION_TYPE (BND, BND, BND)
 DEF_FUNCTION_TYPE (PVOID, PCVOID, BND, ULONG)
 DEF_FUNCTION_TYPE (ULONG, VOID)
 DEF_FUNCTION_TYPE (PVOID, BND)
+
+#GFNI builtins
+DEF_FUNCTION_TYPE (V64QI, V64QI, V64QI, INT)
+DEF_FUNCTION_TYPE (V64QI, V64QI, V64QI, INT, V64QI, UDI)
+DEF_FUNCTION_TYPE (V32QI, V32QI, V32QI, INT, V32QI, USI)
+DEF_FUNCTION_TYPE (V16QI, V16QI, V16QI, INT, V16QI, UHI)
diff --git a/gcc/config/i386/i386-builtin.def b/gcc/config/i386/i386-builtin.def
index 0d5d5b74675..76e5f0fafdd 100644
--- a/gcc/config/i386/i386-builtin.def
+++ b/gcc/config/i386/i386-builtin.def
@@ -1666,8 +1666,8 @@ BDESC (OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv4df
 BDESC (OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv2df_mask, "__builtin_ia32_reducepd128_mask", IX86_BUILTIN_REDUCEPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_UQI)
 BDESC (OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv8sf_mask, "__builtin_ia32_reduceps256_mask", IX86_BUILTIN_REDUCEPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_UQI)
 BDESC (OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv4sf_mask, "__builtin_ia32_reduceps128_mask", IX86_BUILTIN_REDUCEPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_UQI)
-BDESC (OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv2df, "__builtin_ia32_reducesd", IX86_BUILTIN_REDUCESD_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT)
-BDESC (OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv4sf, "__builtin_ia32_reducess", IX86_BUILTIN_REDUCESS_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT)
+BDESC (OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv2df_mask, "__builtin_ia32_reducesd_mask", IX86_BUILTIN_REDUCESD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI)
+BDESC (OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv4sf_mask, "__builtin_ia32_reducess_mask", IX86_BUILTIN_REDUCESS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI)
 BDESC (OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv16hi_mask, "__builtin_ia32_permvarhi256_mask", IX86_BUILTIN_VPERMVARHI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_UHI)
 BDESC (OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv8hi_mask, "__builtin_ia32_permvarhi128_mask", IX86_BUILTIN_VPERMVARHI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_UQI)
 BDESC (OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16hi3_mask, "__builtin_ia32_vpermt2varhi256_mask", IX86_BUILTIN_VPERMT2VARHI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_UHI)
@@ -2589,6 +2589,13 @@ BDESC (OPTION_MASK_ISA_AVX512VPOPCNTDQ, CODE_FOR_vpopcountv8di_mask, "__builtin_
 /* RDPID */
 BDESC (OPTION_MASK_ISA_RDPID, CODE_FOR_rdpid, "__builtin_ia32_rdpid", IX86_BUILTIN_RDPID, UNKNOWN, (int) UNSIGNED_FTYPE_VOID)
 
+/* GFNI */
+BDESC (OPTION_MASK_ISA_GFNI, CODE_FOR_vgf2p8affineinvqb_v64qi, "__builtin_ia32_vgf2p8affineinvqb_v64qi", IX86_BUILTIN_VGF2P8AFFINEINVQB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_INT)
+BDESC (OPTION_MASK_ISA_GFNI | OPTION_MASK_ISA_AVX512BW, CODE_FOR_vgf2p8affineinvqb_v64qi_mask, "__builtin_ia32_vgf2p8affineinvqb_v64qi_mask", IX86_BUILTIN_VGF2P8AFFINEINVQB512MASK, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI)
+BDESC (OPTION_MASK_ISA_GFNI, CODE_FOR_vgf2p8affineinvqb_v32qi, "__builtin_ia32_vgf2p8affineinvqb_v32qi", IX86_BUILTIN_VGF2P8AFFINEINVQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT)
+BDESC (OPTION_MASK_ISA_GFNI | OPTION_MASK_ISA_AVX512BW, CODE_FOR_vgf2p8affineinvqb_v32qi_mask, "__builtin_ia32_vgf2p8affineinvqb_v32qi_mask", IX86_BUILTIN_VGF2P8AFFINEINVQB256MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI)
+BDESC (OPTION_MASK_ISA_GFNI, CODE_FOR_vgf2p8affineinvqb_v16qi, "__builtin_ia32_vgf2p8affineinvqb_v16qi", IX86_BUILTIN_VGF2P8AFFINEINVQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT)
+BDESC (OPTION_MASK_ISA_GFNI | OPTION_MASK_ISA_AVX512BW, CODE_FOR_vgf2p8affineinvqb_v16qi_mask, "__builtin_ia32_vgf2p8affineinvqb_v16qi_mask", IX86_BUILTIN_VGF2P8AFFINEINVQB128MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI)
 BDESC_END (ARGS2, MPX)
 
 /* Builtins for MPX.  */
@@ -2779,4 +2786,25 @@ BDESC (OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3,     "__builtin_ia32_vper
 BDESC (OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3,     "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1)
 BDESC (OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3,     "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1)
 
-BDESC_END (MULTI_ARG, MAX)
+BDESC_END (MULTI_ARG, CET)
+
+/* CET.  */
+BDESC_FIRST (cet, CET,
+       OPTION_MASK_ISA_SHSTK, CODE_FOR_incsspsi, "__builtin_ia32_incsspd", IX86_BUILTIN_INCSSPD, UNKNOWN, (int) VOID_FTYPE_UNSIGNED)
+BDESC (OPTION_MASK_ISA_SHSTK | OPTION_MASK_ISA_64BIT, CODE_FOR_incsspdi, "__builtin_ia32_incsspq", IX86_BUILTIN_INCSSPQ, UNKNOWN, (int) VOID_FTYPE_UINT64)
+BDESC (OPTION_MASK_ISA_SHSTK, CODE_FOR_saveprevssp, "__builtin_ia32_saveprevssp", IX86_BUILTIN_SAVEPREVSSP, UNKNOWN, (int) VOID_FTYPE_VOID)
+BDESC (OPTION_MASK_ISA_SHSTK, CODE_FOR_rstorssp, "__builtin_ia32_rstorssp", IX86_BUILTIN_RSTORSSP, UNKNOWN, (int) VOID_FTYPE_PVOID)
+BDESC (OPTION_MASK_ISA_SHSTK, CODE_FOR_wrsssi, "__builtin_ia32_wrssd", IX86_BUILTIN_WRSSD, UNKNOWN, (int) VOID_FTYPE_UNSIGNED_PVOID)
+BDESC (OPTION_MASK_ISA_SHSTK | OPTION_MASK_ISA_64BIT, CODE_FOR_wrssdi, "__builtin_ia32_wrssq", IX86_BUILTIN_WRSSQ, UNKNOWN, (int) VOID_FTYPE_UINT64_PVOID)
+BDESC (OPTION_MASK_ISA_SHSTK, CODE_FOR_wrusssi, "__builtin_ia32_wrussd", IX86_BUILTIN_WRUSSD, UNKNOWN, (int) VOID_FTYPE_UNSIGNED_PVOID)
+BDESC (OPTION_MASK_ISA_SHSTK | OPTION_MASK_ISA_64BIT, CODE_FOR_wrussdi, "__builtin_ia32_wrussq", IX86_BUILTIN_WRUSSQ, UNKNOWN, (int) VOID_FTYPE_UINT64_PVOID)
+BDESC (OPTION_MASK_ISA_SHSTK, CODE_FOR_setssbsy, "__builtin_ia32_setssbsy", IX86_BUILTIN_SETSSBSY, UNKNOWN, (int) VOID_FTYPE_VOID)
+BDESC (OPTION_MASK_ISA_SHSTK, CODE_FOR_clrssbsy, "__builtin_ia32_clrssbsy", IX86_BUILTIN_CLRSSBSY, UNKNOWN, (int) VOID_FTYPE_PVOID)
+
+BDESC_END (CET, CET_NORMAL)
+
+BDESC_FIRST (cet_rdssp, CET_NORMAL,
+       OPTION_MASK_ISA_SHSTK, CODE_FOR_rdsspsi, "__builtin_ia32_rdsspd", IX86_BUILTIN_RDSSPD, UNKNOWN, (int) UINT_FTYPE_UINT)
+BDESC (OPTION_MASK_ISA_SHSTK | OPTION_MASK_ISA_64BIT, CODE_FOR_rdsspdi, "__builtin_ia32_rdsspq", IX86_BUILTIN_RDSSPQ, UNKNOWN, (int) UINT64_FTYPE_UINT64)
+
+BDESC_END (CET_NORMAL, MAX)
diff --git a/gcc/config/i386/i386-c.c b/gcc/config/i386/i386-c.c
index 9bed360c43b..be99d01f110 100644
--- a/gcc/config/i386/i386-c.c
+++ b/gcc/config/i386/i386-c.c
@@ -459,6 +459,20 @@ ix86_target_macros_internal (HOST_WIDE_INT isa_flag,
     def_or_undef (parse_in, "__PKU__");
   if (isa_flag2 & OPTION_MASK_ISA_RDPID)
     def_or_undef (parse_in, "__RDPID__");
+  if (isa_flag2 & OPTION_MASK_ISA_GFNI)
+    def_or_undef (parse_in, "__GFNI__");
+  if (isa_flag2 & OPTION_MASK_ISA_IBT)
+    {
+      def_or_undef (parse_in, "__IBT__");
+      if (flag_cf_protection != CF_NONE)
+	def_or_undef (parse_in, "__CET__");
+    }
+  if (isa_flag2 & OPTION_MASK_ISA_SHSTK)
+    {
+      def_or_undef (parse_in, "__SHSTK__");
+      if (flag_cf_protection != CF_NONE)
+	def_or_undef (parse_in, "__CET__");
+    }
   if (TARGET_IAMCU)
     {
       def_or_undef (parse_in, "__iamcu");
diff --git a/gcc/config/i386/i386-modes.def b/gcc/config/i386/i386-modes.def
index 83216e38758..dcf6854b57d 100644
--- a/gcc/config/i386/i386-modes.def
+++ b/gcc/config/i386/i386-modes.def
@@ -39,19 +39,22 @@ ADJUST_ALIGNMENT (XF, TARGET_128BIT_LONG_DOUBLE ? 16 : 4);
    For the i386, we need separate modes when floating-point
    equality comparisons are being done.
 
-   Add CCNO to indicate comparisons against zero that requires
+   Add CCNO to indicate comparisons against zero that require
    Overflow flag to be unset.  Sign bit test is used instead and
    thus can be used to form "a&b>0" type of tests.
 
-   Add CCGC to indicate comparisons against zero that allows
+   Add CCGC to indicate comparisons against zero that allow
    unspecified garbage in the Carry flag.  This mode is used
    by inc/dec instructions.
 
-   Add CCGOC to indicate comparisons against zero that allows
+   Add CCGOC to indicate comparisons against zero that allow
    unspecified garbage in the Carry and Overflow flag. This
    mode is used to simulate comparisons of (a-b) and (a+b)
    against zero using sub/cmp/add operations.
 
+   Add CCGZ to indicate comparisons that allow unspecified garbage
+   in the Zero flag.  This mode is used in double-word comparisons.
+
    Add CCA to indicate that only the Above flag is valid.
    Add CCC to indicate that only the Carry flag is valid.
    Add CCO to indicate that only the Overflow flag is valid.
@@ -62,14 +65,15 @@ ADJUST_ALIGNMENT (XF, TARGET_128BIT_LONG_DOUBLE ? 16 : 4);
 CC_MODE (CCGC);
 CC_MODE (CCGOC);
 CC_MODE (CCNO);
+CC_MODE (CCGZ);
 CC_MODE (CCA);
 CC_MODE (CCC);
 CC_MODE (CCO);
 CC_MODE (CCP);
 CC_MODE (CCS);
 CC_MODE (CCZ);
+
 CC_MODE (CCFP);
-CC_MODE (CCFPU);
 
 /* Vector modes.  Note that VEC_CONCAT patterns require vector
    sizes twice as big as implemented in hardware.  */
diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
index 49534619221..5c6e9c3494e 100644
--- a/gcc/config/i386/i386-passes.def
+++ b/gcc/config/i386/i386-passes.def
@@ -29,3 +29,5 @@ along with GCC; see the file COPYING3.  If not see
   /* Run the 64-bit STV pass before the CSE pass so that CONST0_RTX and
      CONSTM1_RTX generated by the STV pass can be CSEed.  */
   INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
+
+  INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbranch);
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index a5d7a6c75bb..eca6d5cf196 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -158,8 +158,6 @@ extern int ix86_attr_length_immediate_default (rtx_insn *, bool);
 extern int ix86_attr_length_address_default (rtx_insn *);
 extern int ix86_attr_length_vex_default (rtx_insn *, bool, bool);
 
-extern machine_mode ix86_fp_compare_mode (enum rtx_code);
-
 extern rtx ix86_libcall_value (machine_mode);
 extern bool ix86_function_arg_regno_p (int);
 extern void ix86_asm_output_function_label (FILE *, const char *, tree);
@@ -277,8 +275,6 @@ extern bool i386_pe_type_dllexport_p (tree);
 
 extern int i386_pe_reloc_rw_mask (void);
 
-extern rtx maybe_get_pool_constant (rtx);
-
 extern char internal_label_prefix[16];
 extern int internal_label_prefix_len;
 
@@ -356,3 +352,4 @@ class rtl_opt_pass;
 
 extern rtl_opt_pass *make_pass_insert_vzeroupper (gcc::context *);
 extern rtl_opt_pass *make_pass_stv (gcc::context *);
+extern rtl_opt_pass *make_pass_insert_endbranch (gcc::context *);
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 619b13b3d09..4b684522082 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -102,6 +102,9 @@ static rtx legitimize_pe_coff_symbol (rtx, bool);
 static void ix86_print_operand_address_as (FILE *, rtx, addr_space_t, bool);
 static bool ix86_save_reg (unsigned int, bool, bool);
 static bool ix86_function_naked (const_tree);
+static bool ix86_notrack_prefixed_insn_p (rtx);
+static void ix86_emit_restore_reg_using_pop (rtx);
+
 
 #ifndef CHECK_STACK_LIMIT
 #define CHECK_STACK_LIMIT (-1)
@@ -302,7 +305,7 @@ int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
 	7 for %edi (gcc regno = 5)
    The following three DWARF register numbers are never generated by
    the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
-   believes these numbers have these meanings.
+   believed these numbers have these meanings.
 	8  for %eip    (no gcc equivalent)
 	9  for %eflags (gcc regno = 17)
 	10 for %trapno (no gcc equivalent)
@@ -310,20 +313,20 @@ int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
    for the x86 architecture.  If the version of SDB on x86/svr4 were
    a bit less brain dead with respect to floating-point then we would
    have a precedent to follow with respect to DWARF register numbers
-   for x86 FP registers, but the SDB on x86/svr4 is so completely
+   for x86 FP registers, but the SDB on x86/svr4 was so completely
    broken with respect to FP registers that it is hardly worth thinking
    of it as something to strive for compatibility with.
-   The version of x86/svr4 SDB I have at the moment does (partially)
+   The version of x86/svr4 SDB I had does (partially)
    seem to believe that DWARF register number 11 is associated with
    the x86 register %st(0), but that's about all.  Higher DWARF
    register numbers don't seem to be associated with anything in
-   particular, and even for DWARF regno 11, SDB only seems to under-
+   particular, and even for DWARF regno 11, SDB only seemed to under-
    stand that it should say that a variable lives in %st(0) (when
    asked via an `=' command) if we said it was in DWARF regno 11,
-   but SDB still prints garbage when asked for the value of the
+   but SDB still printed garbage when asked for the value of the
    variable in question (via a `/' command).
-   (Also note that the labels SDB prints for various FP stack regs
-   when doing an `x' command are all wrong.)
+   (Also note that the labels SDB printed for various FP stack regs
+   when doing an `x' command were all wrong.)
    Note that these problems generally don't affect the native SVR4
    C compiler because it doesn't allow the use of -O with -g and
    because when it is *not* optimizing, it allocates a memory
@@ -1602,7 +1605,7 @@ dimode_scalar_chain::compute_convert_gain ()
       rtx dst = SET_DEST (def_set);
 
       if (REG_P (src) && REG_P (dst))
-	gain += COSTS_N_INSNS (2) - ix86_cost->sse_move;
+	gain += COSTS_N_INSNS (2) - ix86_cost->xmm_move;
       else if (REG_P (src) && MEM_P (dst))
 	gain += 2 * ix86_cost->int_store[2] - ix86_cost->sse_store[1];
       else if (MEM_P (src) && REG_P (dst))
@@ -2570,6 +2573,151 @@ make_pass_stv (gcc::context *ctxt)
   return new pass_stv (ctxt);
 }
 
+/* Inserting ENDBRANCH instructions.  */
+
+static unsigned int
+rest_of_insert_endbranch (void)
+{
+  timevar_push (TV_MACH_DEP);
+
+  rtx cet_eb;
+  rtx_insn *insn;
+  basic_block bb;
+
+  /* Currently emit EB if it's a tracking function, i.e. 'nocf_check' is
+     absent among function attributes.  Later an optimization will be
+     introduced to make analysis if an address of a static function is
+     taken.  A static function whose address is not taken will get a
+     nocf_check attribute.  This will allow to reduce the number of EB.  */
+
+  if (!lookup_attribute ("nocf_check",
+			 TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
+      && !cgraph_node::get (cfun->decl)->only_called_directly_p ())
+    {
+      cet_eb = gen_nop_endbr ();
+
+      bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
+      insn = BB_HEAD (bb);
+      emit_insn_before (cet_eb, insn);
+    }
+
+  bb = 0;
+  FOR_EACH_BB_FN (bb, cfun)
+    {
+      for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb));
+	   insn = NEXT_INSN (insn))
+	{
+	  if (INSN_P (insn) && GET_CODE (insn) == CALL_INSN)
+	    {
+	      rtx_insn *next_insn = insn;
+
+	      while ((next_insn != BB_END (bb))
+		      && (DEBUG_INSN_P (NEXT_INSN (next_insn))
+			  || NOTE_P (NEXT_INSN (next_insn))
+			  || BARRIER_P (NEXT_INSN (next_insn))))
+		next_insn = NEXT_INSN (next_insn);
+
+	      /* Generate ENDBRANCH after CALL, which can return more than
+		 twice, setjmp-like functions.  */
+	      if (find_reg_note (insn, REG_SETJMP, NULL) != NULL)
+		{
+		  cet_eb = gen_nop_endbr ();
+		  emit_insn_after (cet_eb, next_insn);
+		}
+	      continue;
+	    }
+
+	  if (INSN_P (insn) && JUMP_P (insn) && flag_cet_switch)
+	    {
+	      rtx target = JUMP_LABEL (insn);
+	      if (target == NULL_RTX || ANY_RETURN_P (target))
+		continue;
+
+	      /* Check the jump is a switch table.  */
+	      rtx_insn *label = as_a<rtx_insn *> (target);
+	      rtx_insn *table = next_insn (label);
+	      if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
+		continue;
+
+	      /* For the indirect jump find out all places it jumps and insert
+		 ENDBRANCH there.  It should be done under a special flag to
+		 control ENDBRANCH generation for switch stmts.  */
+	      edge_iterator ei;
+	      edge e;
+	      basic_block dest_blk;
+
+	      FOR_EACH_EDGE (e, ei, bb->succs)
+		{
+		  rtx_insn *insn;
+
+		  dest_blk = e->dest;
+		  insn = BB_HEAD (dest_blk);
+		  gcc_assert (LABEL_P (insn));
+		  cet_eb = gen_nop_endbr ();
+		  emit_insn_after (cet_eb, insn);
+		}
+	      continue;
+	    }
+
+	  if ((LABEL_P (insn) && LABEL_PRESERVE_P (insn))
+	      || (NOTE_P (insn)
+		  && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL))
+/* TODO.  Check /s bit also.  */
+	    {
+	      cet_eb = gen_nop_endbr ();
+	      emit_insn_after (cet_eb, insn);
+	      continue;
+	    }
+	}
+    }
+
+  timevar_pop (TV_MACH_DEP);
+  return 0;
+}
+
+namespace {
+
+const pass_data pass_data_insert_endbranch =
+{
+  RTL_PASS, /* type.  */
+  "cet", /* name.  */
+  OPTGROUP_NONE, /* optinfo_flags.  */
+  TV_MACH_DEP, /* tv_id.  */
+  0, /* properties_required.  */
+  0, /* properties_provided.  */
+  0, /* properties_destroyed.  */
+  0, /* todo_flags_start.  */
+  0, /* todo_flags_finish.  */
+};
+
+class pass_insert_endbranch : public rtl_opt_pass
+{
+public:
+  pass_insert_endbranch (gcc::context *ctxt)
+    : rtl_opt_pass (pass_data_insert_endbranch, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  virtual bool gate (function *)
+    {
+      return ((flag_cf_protection & CF_BRANCH) && TARGET_IBT);
+    }
+
+  virtual unsigned int execute (function *)
+    {
+      return rest_of_insert_endbranch ();
+    }
+
+}; // class pass_insert_endbranch
+
+} // anon namespace
+
+rtl_opt_pass *
+make_pass_insert_endbranch (gcc::context *ctxt)
+{
+  return new pass_insert_endbranch (ctxt);
+}
+
 /* Return true if a red-zone is in use.  */
 
 bool
@@ -2597,11 +2745,14 @@ ix86_target_string (HOST_WIDE_INT isa, HOST_WIDE_INT isa2,
      ISAs come first.  Target string will be displayed in the same order.  */
   static struct ix86_target_opts isa2_opts[] =
   {
+    { "-mgfni",		OPTION_MASK_ISA_GFNI },
     { "-mrdpid",	OPTION_MASK_ISA_RDPID },
     { "-msgx",		OPTION_MASK_ISA_SGX },
     { "-mavx5124vnniw", OPTION_MASK_ISA_AVX5124VNNIW },
     { "-mavx5124fmaps", OPTION_MASK_ISA_AVX5124FMAPS },
-    { "-mavx512vpopcntdq", OPTION_MASK_ISA_AVX512VPOPCNTDQ }
+    { "-mavx512vpopcntdq", OPTION_MASK_ISA_AVX512VPOPCNTDQ },
+    { "-mibt",	OPTION_MASK_ISA_IBT },
+    { "-mshstk",	OPTION_MASK_ISA_SHSTK }
   };
   static struct ix86_target_opts isa_opts[] =
   {
@@ -4694,6 +4845,37 @@ ix86_option_override_internal (bool main_args_p,
     target_option_default_node = target_option_current_node
       = build_target_option_node (opts);
 
+  /* Do not support control flow instrumentation if CET is not enabled.  */
+  if (opts->x_flag_cf_protection != CF_NONE)
+    {
+      if (!(TARGET_IBT_P (opts->x_ix86_isa_flags2)
+	    || TARGET_SHSTK_P (opts->x_ix86_isa_flags2)))
+	{
+	  if (flag_cf_protection == CF_FULL)
+	    {
+	      error ("%<-fcf-protection=full%> requires CET support "
+		     "on this target. Use -mcet or one of -mibt, "
+		     "-mshstk options to enable CET");
+	    }
+	  else if (flag_cf_protection == CF_BRANCH)
+	    {
+	      error ("%<-fcf-protection=branch%> requires CET support "
+		     "on this target. Use -mcet or one of -mibt, "
+		     "-mshstk options to enable CET");
+	    }
+	  else if (flag_cf_protection == CF_RETURN)
+	    {
+	      error ("%<-fcf-protection=return%> requires CET support "
+		     "on this target. Use -mcet or one of -mibt, "
+		     "-mshstk options to enable CET");
+	    }
+	  flag_cf_protection = CF_NONE;
+	  return false;
+	}
+      opts->x_flag_cf_protection =
+	(cf_protection_level) (opts->x_flag_cf_protection | CF_SET);
+    }
+
   return true;
 }
 
@@ -5123,6 +5305,9 @@ ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
     IX86_ATTR_ISA ("mpx",	OPT_mmpx),
     IX86_ATTR_ISA ("clwb",	OPT_mclwb),
     IX86_ATTR_ISA ("rdpid",	OPT_mrdpid),
+    IX86_ATTR_ISA ("gfni",	OPT_mgfni),
+    IX86_ATTR_ISA ("ibt",	OPT_mibt),
+    IX86_ATTR_ISA ("shstk",	OPT_mshstk),
 
     /* enum options */
     IX86_ATTR_ENUM ("fpmath=",	OPT_mfpmath_),
@@ -11943,8 +12128,14 @@ ix86_adjust_stack_and_probe_stack_clash (const HOST_WIDE_INT size)
      we just probe when we cross PROBE_INTERVAL.  */
   if (TREE_THIS_VOLATILE (cfun->decl))
     {
-      emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
-				       -GET_MODE_SIZE (word_mode)));
+      /* We can safely use any register here since we're just going to push
+	 its value and immediately pop it back.  But we do try and avoid
+	 argument passing registers so as not to introduce dependencies in
+	 the pipeline.  For 32 bit we use %esi and for 64 bit we use %rax.  */
+      rtx dummy_reg = gen_rtx_REG (word_mode, TARGET_64BIT ? AX_REG : SI_REG);
+      rtx_insn *insn = emit_insn (gen_push (dummy_reg));
+      RTX_FRAME_RELATED_P (insn) = 1;
+      ix86_emit_restore_reg_using_pop (dummy_reg);
       emit_insn (gen_blockage ());
     }
 
@@ -12512,10 +12703,13 @@ ix86_finalize_stack_frame_flags (void)
 	      for (ref = DF_REG_USE_CHAIN (HARD_FRAME_POINTER_REGNUM);
 		   ref; ref = next)
 		{
-		  rtx_insn *insn = DF_REF_INSN (ref);
+		  next = DF_REF_NEXT_REG (ref);
+		  if (!DF_REF_INSN_INFO (ref))
+		    continue;
+
 		  /* Make sure the next ref is for a different instruction,
 		     so that we're not affected by the rescan.  */
-		  next = DF_REF_NEXT_REG (ref);
+		  rtx_insn *insn = DF_REF_INSN (ref);
 		  while (next && DF_REF_INSN (next) == insn)
 		    next = DF_REF_NEXT_REG (next);
 
@@ -12836,7 +13030,7 @@ ix86_expand_prologue (void)
   if (frame_pointer_needed && !m->fs.fp_valid)
     {
       /* Note: AT&T enter does NOT have reversed args.  Enter is probably
-         slower on all targets.  Also sdb doesn't like it.  */
+         slower on all targets.  Also sdb didn't like it.  */
       insn = emit_insn (gen_push (hard_frame_pointer_rtx));
       RTX_FRAME_RELATED_P (insn) = 1;
 
@@ -12983,8 +13177,12 @@ ix86_expand_prologue (void)
       && (flag_stack_check == STATIC_BUILTIN_STACK_CHECK
 	  || flag_stack_clash_protection))
     {
-      /* We expect the GP registers to be saved when probes are used.  */
-      gcc_assert (int_registers_saved);
+      /* This assert wants to verify that integer registers were saved
+	 prior to probing.  This is necessary when probing may be implemented
+	 as a function call (Windows).  It is not necessary for stack clash
+	 protection probing.  */
+      if (!flag_stack_clash_protection)
+	gcc_assert (int_registers_saved);
 
       if (flag_stack_clash_protection)
 	{
@@ -13628,7 +13826,7 @@ ix86_expand_epilogue (int style)
 	 the stack pointer, if we will restore SSE regs via sp.  */
       if (TARGET_64BIT
 	  && m->fs.sp_offset > 0x7fffffff
-	  && sp_valid_at (frame.stack_realign_offset)
+	  && sp_valid_at (frame.stack_realign_offset + 1)
 	  && (frame.nsseregs + frame.nregs) != 0)
 	{
 	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
@@ -14895,10 +15093,16 @@ legitimate_pic_address_disp_p (rtx disp)
 	    break;
 	  op0 = XEXP (XEXP (disp, 0), 0);
 	  op1 = XEXP (XEXP (disp, 0), 1);
-	  if (!CONST_INT_P (op1)
-	      || INTVAL (op1) >= 16*1024*1024
+	  if (!CONST_INT_P (op1))
+	    break;
+	  if (GET_CODE (op0) == UNSPEC
+	      && (XINT (op0, 1) == UNSPEC_DTPOFF
+		  || XINT (op0, 1) == UNSPEC_NTPOFF)
+	      && trunc_int_for_mode (INTVAL (op1), SImode) == INTVAL (op1))
+	    return true;
+	  if (INTVAL (op1) >= 16*1024*1024
 	      || INTVAL (op1) < -16*1024*1024)
-            break;
+	    break;
 	  if (GET_CODE (op0) == LABEL_REF)
 	    return true;
 	  if (GET_CODE (op0) == CONST
@@ -16657,13 +16861,17 @@ ix86_delegitimize_address_1 (rtx x, bool base_term_p)
 	 movl foo@GOTOFF(%ecx), %edx
 	 in which case we return (%ecx - %ebx) + foo
 	 or (%ecx - _GLOBAL_OFFSET_TABLE_) + foo if pseudo_pic_reg
-	 and reload has completed.  */
+	 and reload has completed.  Don't do the latter for debug,
+	 as _GLOBAL_OFFSET_TABLE_ can't be expressed in the assembly.  */
       if (pic_offset_table_rtx
 	  && (!reload_completed || !ix86_use_pseudo_pic_reg ()))
         result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend),
 						     pic_offset_table_rtx),
 			       result);
-      else if (pic_offset_table_rtx && !TARGET_MACHO && !TARGET_VXWORKS_RTP)
+      else if (base_term_p
+	       && pic_offset_table_rtx
+	       && !TARGET_MACHO
+	       && !TARGET_VXWORKS_RTP)
 	{
 	  rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
 	  tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
@@ -16716,6 +16924,25 @@ ix86_find_base_term (rtx x)
 
   return ix86_delegitimize_address_1 (x, true);
 }
+
+/* Return true if X shouldn't be emitted into the debug info.
+   Disallow UNSPECs other than @gotoff - we can't emit _GLOBAL_OFFSET_TABLE_
+   symbol easily into the .debug_info section, so we need not to
+   delegitimize, but instead assemble as @gotoff.
+   Disallow _GLOBAL_OFFSET_TABLE_ SYMBOL_REF - the assembler magically
+   assembles that as _GLOBAL_OFFSET_TABLE_-. expression.  */
+
+static bool
+ix86_const_not_ok_for_debug_p (rtx x)
+{
+  if (GET_CODE (x) == UNSPEC && XINT (x, 1) != UNSPEC_GOTOFF)
+    return true;
+
+  if (SYMBOL_REF_P (x) && strcmp (XSTR (x, 0), GOT_SYMBOL_NAME) == 0)
+    return true;
+
+  return false;
+}
 
 static void
 put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
@@ -16723,7 +16950,7 @@ put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
 {
   const char *suffix;
 
-  if (mode == CCFPmode || mode == CCFPUmode)
+  if (mode == CCFPmode)
     {
       code = ix86_fp_compare_code_to_integer (code);
       mode = CCmode;
@@ -16734,6 +16961,7 @@ put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
   switch (code)
     {
     case EQ:
+      gcc_assert (mode != CCGZmode);
       switch (mode)
 	{
 	case E_CCAmode:
@@ -16757,6 +16985,7 @@ put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
 	}
       break;
     case NE:
+      gcc_assert (mode != CCGZmode);
       switch (mode)
 	{
 	case E_CCAmode:
@@ -16801,6 +17030,7 @@ put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
 
 	case E_CCmode:
 	case E_CCGCmode:
+	case E_CCGZmode:
 	  suffix = "l";
 	  break;
 
@@ -16809,7 +17039,7 @@ put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
 	}
       break;
     case LTU:
-      if (mode == CCmode)
+      if (mode == CCmode || mode == CCGZmode)
 	suffix = "b";
       else if (mode == CCCmode)
 	suffix = fp ? "b" : "c";
@@ -16826,6 +17056,7 @@ put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
 
 	case E_CCmode:
 	case E_CCGCmode:
+	case E_CCGZmode:
 	  suffix = "ge";
 	  break;
 
@@ -16834,7 +17065,7 @@ put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
 	}
       break;
     case GEU:
-      if (mode == CCmode)
+      if (mode == CCmode || mode == CCGZmode)
 	suffix = "nb";
       else if (mode == CCCmode)
 	suffix = fp ? "nb" : "nc";
@@ -17613,6 +17844,8 @@ ix86_print_operand (FILE *file, rtx x, int code)
 	case '!':
 	  if (ix86_bnd_prefixed_insn_p (current_output_insn))
 	    fputs ("bnd ", file);
+	  if (ix86_notrack_prefixed_insn_p (current_output_insn))
+	    fputs ("notrack ", file);
 	  return;
 
 	default:
@@ -18028,6 +18261,10 @@ i386_asm_output_addr_const_extra (FILE *file, rtx x)
   op = XVECEXP (x, 0, 0);
   switch (XINT (x, 1))
     {
+    case UNSPEC_GOTOFF:
+      output_addr_const (file, op);
+      fputs ("@gotoff", file);
+      break;
     case UNSPEC_GOTTPOFF:
       output_addr_const (file, op);
       /* FIXME: This might be @TPOFF in Sun ld.  */
@@ -18147,89 +18384,66 @@ output_387_binary_op (rtx_insn *insn, rtx *operands)
 {
   static char buf[40];
   const char *p;
-  const char *ssep;
-  int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]);
+  bool is_sse
+    = (SSE_REG_P (operands[0])
+       || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]));
 
-  /* Even if we do not want to check the inputs, this documents input
-     constraints.  Which helps in understanding the following code.  */
-  if (flag_checking)
-    {
-      if (STACK_REG_P (operands[0])
-	  && ((REG_P (operands[1])
-	       && REGNO (operands[0]) == REGNO (operands[1])
-	       && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
-	      || (REG_P (operands[2])
-		  && REGNO (operands[0]) == REGNO (operands[2])
-		  && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
-	  && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
-	; /* ok */
-      else
-	gcc_assert (is_sse);
-    }
+  if (is_sse)
+    p = "%v";
+  else if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
+	   || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
+    p = "fi";
+  else
+    p = "f";
+
+  strcpy (buf, p);
 
   switch (GET_CODE (operands[3]))
     {
     case PLUS:
-      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
-	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
-	p = "fiadd";
-      else
-	p = "fadd";
-      ssep = "vadd";
-      break;
-
+      p = "add"; break;
     case MINUS:
-      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
-	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
-	p = "fisub";
-      else
-	p = "fsub";
-      ssep = "vsub";
-      break;
-
+      p = "sub"; break;
     case MULT:
-      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
-	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
-	p = "fimul";
-      else
-	p = "fmul";
-      ssep = "vmul";
-      break;
-
+      p = "mul"; break;
     case DIV:
-      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
-	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
-	p = "fidiv";
-      else
-	p = "fdiv";
-      ssep = "vdiv";
-      break;
-
+      p = "div"; break;
     default:
       gcc_unreachable ();
     }
 
+  strcat (buf, p);
+
   if (is_sse)
    {
+     p = (GET_MODE (operands[0]) == SFmode) ? "ss" : "sd";
+     strcat (buf, p);
+
      if (TARGET_AVX)
-       {
-	 strcpy (buf, ssep);
-	 if (GET_MODE (operands[0]) == SFmode)
-	   strcat (buf, "ss\t{%2, %1, %0|%0, %1, %2}");
-	 else
-	   strcat (buf, "sd\t{%2, %1, %0|%0, %1, %2}");
-       }
+       p = "\t{%2, %1, %0|%0, %1, %2}";
      else
-       {
-	 strcpy (buf, ssep + 1);
-	 if (GET_MODE (operands[0]) == SFmode)
-	   strcat (buf, "ss\t{%2, %0|%0, %2}");
-	 else
-	   strcat (buf, "sd\t{%2, %0|%0, %2}");
-       }
-      return buf;
+       p = "\t{%2, %0|%0, %2}";
+
+     strcat (buf, p);
+     return buf;
    }
-  strcpy (buf, p);
+
+  /* Even if we do not want to check the inputs, this documents input
+     constraints.  Which helps in understanding the following code.  */
+  if (flag_checking)
+    {
+      if (STACK_REG_P (operands[0])
+	  && ((REG_P (operands[1])
+	       && REGNO (operands[0]) == REGNO (operands[1])
+	       && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
+	      || (REG_P (operands[2])
+		  && REGNO (operands[0]) == REGNO (operands[2])
+		  && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
+	  && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
+	; /* ok */
+      else
+	gcc_unreachable ();
+    }
 
   switch (GET_CODE (operands[3]))
     {
@@ -18818,10 +19032,13 @@ ix86_emit_mode_set (int entity, int mode, int prev_mode ATTRIBUTE_UNUSED,
 const char *
 output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
 {
-  int stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
-  int dimode_p = GET_MODE (operands[0]) == DImode;
+  bool stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);
+  bool dimode_p = GET_MODE (operands[0]) == DImode;
   int round_mode = get_attr_i387_cw (insn);
 
+  static char buf[40];
+  const char *p;
+
   /* Jump through a hoop or two for DImode, since the hardware has no
      non-popping instruction.  We used to do this a different way, but
      that was somewhat fragile and broke with post-reload splitters.  */
@@ -18833,18 +19050,20 @@ output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
   gcc_assert (GET_MODE (operands[1]) != TFmode);
 
   if (fisttp)
-      output_asm_insn ("fisttp%Z0\t%0", operands);
-  else
-    {
-      if (round_mode != I387_CW_ANY)
-	output_asm_insn ("fldcw\t%3", operands);
-      if (stack_top_dies || dimode_p)
-	output_asm_insn ("fistp%Z0\t%0", operands);
-      else
-	output_asm_insn ("fist%Z0\t%0", operands);
-      if (round_mode != I387_CW_ANY)
-	output_asm_insn ("fldcw\t%2", operands);
-    }
+    return "fisttp%Z0\t%0";
+
+  strcpy (buf, "fist");
+
+  if (round_mode != I387_CW_ANY)
+    output_asm_insn ("fldcw\t%3", operands);
+
+  p = "p%Z0\t%0";
+  strcat (buf, p + !(stack_top_dies || dimode_p));
+
+  output_asm_insn (buf, operands);
+
+  if (round_mode != I387_CW_ANY)
+    output_asm_insn ("fldcw\t%2", operands);
 
   return "";
 }
@@ -18881,120 +19100,65 @@ output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
    should be used.  UNORDERED_P is true when fucom should be used.  */
 
 const char *
-output_fp_compare (rtx_insn *insn, rtx *operands, bool eflags_p, bool unordered_p)
+output_fp_compare (rtx_insn *insn, rtx *operands,
+		   bool eflags_p, bool unordered_p)
 {
-  int stack_top_dies;
-  rtx cmp_op0, cmp_op1;
-  int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]);
-
-  if (eflags_p)
-    {
-      cmp_op0 = operands[0];
-      cmp_op1 = operands[1];
-    }
-  else
-    {
-      cmp_op0 = operands[1];
-      cmp_op1 = operands[2];
-    }
+  rtx *xops = eflags_p ? &operands[0] : &operands[1];
+  bool stack_top_dies;
 
-  if (is_sse)
-    {
-      if (GET_MODE (operands[0]) == SFmode)
-	if (unordered_p)
-	  return "%vucomiss\t{%1, %0|%0, %1}";
-	else
-	  return "%vcomiss\t{%1, %0|%0, %1}";
-      else
-	if (unordered_p)
-	  return "%vucomisd\t{%1, %0|%0, %1}";
-	else
-	  return "%vcomisd\t{%1, %0|%0, %1}";
-    }
+  static char buf[40];
+  const char *p;
 
-  gcc_assert (STACK_TOP_P (cmp_op0));
+  gcc_assert (STACK_TOP_P (xops[0]));
 
-  stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
+  stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);
 
-  if (cmp_op1 == CONST0_RTX (GET_MODE (cmp_op1)))
+  if (eflags_p)
     {
-      if (stack_top_dies)
-	{
-	  output_asm_insn ("ftst\n\tfnstsw\t%0", operands);
-	  return output_387_ffreep (operands, 1);
-	}
-      else
-	return "ftst\n\tfnstsw\t%0";
+      p = unordered_p ? "fucomi" : "fcomi";
+      strcpy (buf, p);
+
+      p = "p\t{%y1, %0|%0, %y1}";
+      strcat (buf, p + !stack_top_dies);
+
+      return buf;
     }
 
-  if (STACK_REG_P (cmp_op1)
+  if (STACK_REG_P (xops[1])
       && stack_top_dies
-      && find_regno_note (insn, REG_DEAD, REGNO (cmp_op1))
-      && REGNO (cmp_op1) != FIRST_STACK_REG)
+      && find_regno_note (insn, REG_DEAD, FIRST_STACK_REG + 1))
     {
-      /* If both the top of the 387 stack dies, and the other operand
-	 is also a stack register that dies, then this must be a
-	 `fcompp' float compare */
+      gcc_assert (REGNO (xops[1]) == FIRST_STACK_REG + 1);
 
-      if (eflags_p)
-	{
-	  /* There is no double popping fcomi variant.  Fortunately,
-	     eflags is immune from the fstp's cc clobbering.  */
-	  if (unordered_p)
-	    output_asm_insn ("fucomip\t{%y1, %0|%0, %y1}", operands);
-	  else
-	    output_asm_insn ("fcomip\t{%y1, %0|%0, %y1}", operands);
-	  return output_387_ffreep (operands, 0);
-	}
-      else
-	{
-	  if (unordered_p)
-	    return "fucompp\n\tfnstsw\t%0";
-	  else
-	    return "fcompp\n\tfnstsw\t%0";
-	}
+      /* If both the top of the 387 stack die, and the other operand
+	 is also a stack register that dies, then this must be a
+	 `fcompp' float compare.  */
+      p = unordered_p ? "fucompp" : "fcompp";
+      strcpy (buf, p);
+    }
+  else if (const0_operand (xops[1], VOIDmode))
+    {
+      gcc_assert (!unordered_p);
+      strcpy (buf, "ftst");
     }
   else
     {
-      /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies.  */
-
-      static const char * const alt[16] =
-      {
-	"fcom%Z2\t%y2\n\tfnstsw\t%0",
-	"fcomp%Z2\t%y2\n\tfnstsw\t%0",
-	"fucom%Z2\t%y2\n\tfnstsw\t%0",
-	"fucomp%Z2\t%y2\n\tfnstsw\t%0",
-
-	"ficom%Z2\t%y2\n\tfnstsw\t%0",
-	"ficomp%Z2\t%y2\n\tfnstsw\t%0",
-	NULL,
-	NULL,
-
-	"fcomi\t{%y1, %0|%0, %y1}",
-	"fcomip\t{%y1, %0|%0, %y1}",
-	"fucomi\t{%y1, %0|%0, %y1}",
-	"fucomip\t{%y1, %0|%0, %y1}",
-
-	NULL,
-	NULL,
-	NULL,
-	NULL
-      };
-
-      int mask;
-      const char *ret;
-
-      mask  = eflags_p << 3;
-      mask |= (GET_MODE_CLASS (GET_MODE (cmp_op1)) == MODE_INT) << 2;
-      mask |= unordered_p << 1;
-      mask |= stack_top_dies;
+      if (GET_MODE_CLASS (GET_MODE (xops[1])) == MODE_INT)
+	{
+	  gcc_assert (!unordered_p);
+	  p = "ficom";
+	}
+      else
+	p = unordered_p ? "fucom" : "fcom";
 
-      gcc_assert (mask < 16);
-      ret = alt[mask];
-      gcc_assert (ret);
+      strcpy (buf, p);
 
-      return ret;
+      p = "p%Z2\t%y2";
+      strcat (buf, p + !stack_top_dies);
     }
+
+  output_asm_insn (buf, operands);
+  return "fnstsw\t%0";
 }
 
 void
@@ -19067,20 +19231,6 @@ ix86_expand_clear (rtx dest)
   emit_insn (tmp);
 }
 
-/* X is an unchanging MEM.  If it is a constant pool reference, return
-   the constant pool rtx, else NULL.  */
-
-rtx
-maybe_get_pool_constant (rtx x)
-{
-  x = ix86_delegitimize_address (XEXP (x, 0));
-
-  if (GET_CODE (x) == SYMBOL_REF && CONSTANT_POOL_ADDRESS_P (x))
-    return get_pool_constant (x);
-
-  return NULL_RTX;
-}
-
 void
 ix86_expand_move (machine_mode mode, rtx operands[])
 {
@@ -21526,6 +21676,8 @@ ix86_match_ccmode (rtx insn, machine_mode req_mode)
     case E_CCZmode:
       break;
 
+    case E_CCGZmode:
+
     case E_CCAmode:
     case E_CCCmode:
     case E_CCOmode:
@@ -21563,18 +21715,38 @@ ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
   return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
 }
 
-/* Figure out whether to use ordered or unordered fp comparisons.
-   Return the appropriate mode to use.  */
+/* Figure out whether to use unordered fp comparisons.  */
 
-machine_mode
-ix86_fp_compare_mode (enum rtx_code)
+static bool
+ix86_unordered_fp_compare (enum rtx_code code)
 {
-  /* ??? In order to make all comparisons reversible, we do all comparisons
-     non-trapping when compiling for IEEE.  Once gcc is able to distinguish
-     all forms trapping and nontrapping comparisons, we can make inequality
-     comparisons trapping again, since it results in better code when using
-     FCOM based compares.  */
-  return TARGET_IEEE_FP ? CCFPUmode : CCFPmode;
+  if (!TARGET_IEEE_FP)
+    return false;
+
+  switch (code)
+    {
+    case GT:
+    case GE:
+    case LT:
+    case LE:
+      return false;
+
+    case EQ:
+    case NE:
+
+    case LTGT:
+    case UNORDERED:
+    case ORDERED:
+    case UNLT:
+    case UNLE:
+    case UNGT:
+    case UNGE:
+    case UNEQ:
+      return true;
+
+    default:
+      gcc_unreachable ();
+    }
 }
 
 machine_mode
@@ -21585,7 +21757,7 @@ ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
   if (SCALAR_FLOAT_MODE_P (mode))
     {
       gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
-      return ix86_fp_compare_mode (code);
+      return CCFPmode;
     }
 
   switch (code)
@@ -21707,7 +21879,6 @@ ix86_cc_modes_compatible (machine_mode m1, machine_mode m2)
 	}
 
     case E_CCFPmode:
-    case E_CCFPUmode:
       /* These are only compatible with themselves, which we already
 	 checked above.  */
       return VOIDmode;
@@ -21811,10 +21982,10 @@ ix86_fp_comparison_strategy (enum rtx_code)
 static enum rtx_code
 ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
 {
-  machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
+  bool unordered_compare = ix86_unordered_fp_compare (code);
   rtx op0 = *pop0, op1 = *pop1;
   machine_mode op_mode = GET_MODE (op0);
-  int is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode);
+  bool is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode);
 
   /* All of the unordered compare instructions only work on registers.
      The same is true of the fcomi compare instructions.  The XFmode
@@ -21823,7 +21994,7 @@ ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
      floating point.  */
 
   if (!is_sse
-      && (fpcmp_mode == CCFPUmode
+      && (unordered_compare
 	  || (op_mode == XFmode
 	      && ! (standard_80387_constant_p (op0) == 1
 		    || standard_80387_constant_p (op1) == 1)
@@ -21920,27 +22091,29 @@ ix86_fp_compare_code_to_integer (enum rtx_code code)
 static rtx
 ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch)
 {
-  machine_mode fpcmp_mode, intcmp_mode;
+  bool unordered_compare = ix86_unordered_fp_compare (code);
+  machine_mode intcmp_mode;
   rtx tmp, tmp2;
 
-  fpcmp_mode = ix86_fp_compare_mode (code);
   code = ix86_prepare_fp_compare_args (code, &op0, &op1);
 
   /* Do fcomi/sahf based test when profitable.  */
   switch (ix86_fp_comparison_strategy (code))
     {
     case IX86_FPCMP_COMI:
-      intcmp_mode = fpcmp_mode;
-      tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
-      tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp);
-      emit_insn (tmp);
+      intcmp_mode = CCFPmode;
+      tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
+      if (unordered_compare)
+	tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
+      emit_insn (gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp));
       break;
 
     case IX86_FPCMP_SAHF:
-      intcmp_mode = fpcmp_mode;
-      tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
-      tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp);
-
+      intcmp_mode = CCFPmode;
+      tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
+      if (unordered_compare)
+	tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
+      tmp = gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp);
       if (!scratch)
 	scratch = gen_reg_rtx (HImode);
       tmp2 = gen_rtx_CLOBBER (VOIDmode, scratch);
@@ -21949,11 +22122,13 @@ ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch)
 
     case IX86_FPCMP_ARITH:
       /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first.  */
-      tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
-      tmp2 = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
+      tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
+      if (unordered_compare)
+	tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
+      tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
       if (!scratch)
 	scratch = gen_reg_rtx (HImode);
-      emit_insn (gen_rtx_SET (scratch, tmp2));
+      emit_insn (gen_rtx_SET (scratch, tmp));
 
       /* In the unordered case, we have to check C2 for NaN's, which
 	 doesn't happen to work out to anything nice combination-wise.
@@ -22234,6 +22409,62 @@ ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
 	      break;
 	    }
 
+	/* Emulate comparisons that do not depend on Zero flag with
+	   double-word subtraction.  Note that only Overflow, Sign
+	   and Carry flags are valid, so swap arguments and condition
+	   of comparisons that would otherwise test Zero flag.  */
+
+	switch (code)
+	  {
+	  case LE: case LEU: case GT: case GTU:
+	    std::swap (lo[0], lo[1]);
+	    std::swap (hi[0], hi[1]);
+	    code = swap_condition (code);
+	    /* FALLTHRU */
+
+	  case LT: case LTU: case GE: case GEU:
+	    {
+	      rtx (*cmp_insn) (rtx, rtx);
+	      rtx (*sbb_insn) (rtx, rtx, rtx);
+	      bool uns = (code == LTU || code == GEU);
+
+	      if (TARGET_64BIT)
+		{
+		  cmp_insn = gen_cmpdi_1;
+		  sbb_insn
+		    = uns ? gen_subdi3_carry_ccc : gen_subdi3_carry_ccgz;
+		}
+	      else
+		{
+		  cmp_insn = gen_cmpsi_1;
+		  sbb_insn
+		    = uns ? gen_subsi3_carry_ccc : gen_subsi3_carry_ccgz;
+		}
+
+	      if (!nonimmediate_operand (lo[0], submode))
+		lo[0] = force_reg (submode, lo[0]);
+	      if (!x86_64_general_operand (lo[1], submode))
+		lo[1] = force_reg (submode, lo[1]);
+
+	      if (!register_operand (hi[0], submode))
+		hi[0] = force_reg (submode, hi[0]);
+	      if ((uns && !nonimmediate_operand (hi[1], submode))
+		  || (!uns && !x86_64_general_operand (hi[1], submode)))
+		hi[1] = force_reg (submode, hi[1]);
+
+	      emit_insn (cmp_insn (lo[0], lo[1]));
+	      emit_insn (sbb_insn (gen_rtx_SCRATCH (submode), hi[0], hi[1]));
+
+	      tmp = gen_rtx_REG (uns ? CCCmode : CCGZmode, FLAGS_REG);
+
+	      ix86_expand_branch (code, tmp, const0_rtx, label);
+	      return;
+	    }
+
+	  default:
+	    break;
+	  }
+
 	/* Otherwise, we need two or three jumps.  */
 
 	label2 = gen_label_rtx ();
@@ -22339,8 +22570,7 @@ ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
       compare_seq = get_insns ();
       end_sequence ();
 
-      if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
-	  || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
+      if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode)
         code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
       else
 	code = GET_CODE (compare_op);
@@ -22480,8 +22710,7 @@ ix86_expand_int_movcc (rtx operands[])
 
 	      flags = XEXP (compare_op, 0);
 
-	      if (GET_MODE (flags) == CCFPmode
-		  || GET_MODE (flags) == CCFPUmode)
+	      if (GET_MODE (flags) == CCFPmode)
 		{
 		  fpcmp = true;
 		  compare_code
@@ -23826,10 +24055,10 @@ struct expand_vec_perm_d
 };
 
 static bool
-ix86_expand_vec_perm_vpermi2 (rtx target, rtx op0, rtx mask, rtx op1,
+ix86_expand_vec_perm_vpermt2 (rtx target, rtx mask, rtx op0, rtx op1,
 			      struct expand_vec_perm_d *d)
 {
-  /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
+  /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
      expander, so args are either in d, or in op0, op1 etc.  */
   machine_mode mode = GET_MODE (d ? d->op0 : op0);
   machine_mode maskmode = mode;
@@ -23839,83 +24068,83 @@ ix86_expand_vec_perm_vpermi2 (rtx target, rtx op0, rtx mask, rtx op1,
     {
     case E_V8HImode:
       if (TARGET_AVX512VL && TARGET_AVX512BW)
-	gen = gen_avx512vl_vpermi2varv8hi3;
+	gen = gen_avx512vl_vpermt2varv8hi3;
       break;
     case E_V16HImode:
       if (TARGET_AVX512VL && TARGET_AVX512BW)
-	gen = gen_avx512vl_vpermi2varv16hi3;
+	gen = gen_avx512vl_vpermt2varv16hi3;
       break;
     case E_V64QImode:
       if (TARGET_AVX512VBMI)
-	gen = gen_avx512bw_vpermi2varv64qi3;
+	gen = gen_avx512bw_vpermt2varv64qi3;
       break;
     case E_V32HImode:
       if (TARGET_AVX512BW)
-	gen = gen_avx512bw_vpermi2varv32hi3;
+	gen = gen_avx512bw_vpermt2varv32hi3;
       break;
     case E_V4SImode:
       if (TARGET_AVX512VL)
-	gen = gen_avx512vl_vpermi2varv4si3;
+	gen = gen_avx512vl_vpermt2varv4si3;
       break;
     case E_V8SImode:
       if (TARGET_AVX512VL)
-	gen = gen_avx512vl_vpermi2varv8si3;
+	gen = gen_avx512vl_vpermt2varv8si3;
       break;
     case E_V16SImode:
       if (TARGET_AVX512F)
-	gen = gen_avx512f_vpermi2varv16si3;
+	gen = gen_avx512f_vpermt2varv16si3;
       break;
     case E_V4SFmode:
       if (TARGET_AVX512VL)
 	{
-	  gen = gen_avx512vl_vpermi2varv4sf3;
+	  gen = gen_avx512vl_vpermt2varv4sf3;
 	  maskmode = V4SImode;
 	}
       break;
     case E_V8SFmode:
       if (TARGET_AVX512VL)
 	{
-	  gen = gen_avx512vl_vpermi2varv8sf3;
+	  gen = gen_avx512vl_vpermt2varv8sf3;
 	  maskmode = V8SImode;
 	}
       break;
     case E_V16SFmode:
       if (TARGET_AVX512F)
 	{
-	  gen = gen_avx512f_vpermi2varv16sf3;
+	  gen = gen_avx512f_vpermt2varv16sf3;
 	  maskmode = V16SImode;
 	}
       break;
     case E_V2DImode:
       if (TARGET_AVX512VL)
-	gen = gen_avx512vl_vpermi2varv2di3;
+	gen = gen_avx512vl_vpermt2varv2di3;
       break;
     case E_V4DImode:
       if (TARGET_AVX512VL)
-	gen = gen_avx512vl_vpermi2varv4di3;
+	gen = gen_avx512vl_vpermt2varv4di3;
       break;
     case E_V8DImode:
       if (TARGET_AVX512F)
-	gen = gen_avx512f_vpermi2varv8di3;
+	gen = gen_avx512f_vpermt2varv8di3;
       break;
     case E_V2DFmode:
       if (TARGET_AVX512VL)
 	{
-	  gen = gen_avx512vl_vpermi2varv2df3;
+	  gen = gen_avx512vl_vpermt2varv2df3;
 	  maskmode = V2DImode;
 	}
       break;
     case E_V4DFmode:
       if (TARGET_AVX512VL)
 	{
-	  gen = gen_avx512vl_vpermi2varv4df3;
+	  gen = gen_avx512vl_vpermt2varv4df3;
 	  maskmode = V4DImode;
 	}
       break;
     case E_V8DFmode:
       if (TARGET_AVX512F)
 	{
-	  gen = gen_avx512f_vpermi2varv8df3;
+	  gen = gen_avx512f_vpermt2varv8df3;
 	  maskmode = V8DImode;
 	}
       break;
@@ -23926,7 +24155,7 @@ ix86_expand_vec_perm_vpermi2 (rtx target, rtx op0, rtx mask, rtx op1,
   if (gen == NULL)
     return false;
 
-  /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
+  /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
      expander, so args are either in d, or in op0, op1 etc.  */
   if (d)
     {
@@ -23939,7 +24168,7 @@ ix86_expand_vec_perm_vpermi2 (rtx target, rtx op0, rtx mask, rtx op1,
       mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
     }
 
-  emit_insn (gen (target, op0, force_reg (maskmode, mask), op1));
+  emit_insn (gen (target, force_reg (maskmode, mask), op0, op1));
   return true;
 }
 
@@ -23990,7 +24219,7 @@ ix86_expand_vec_perm (rtx operands[])
 	}
     }
 
-  if (ix86_expand_vec_perm_vpermi2 (target, op0, mask, op1, NULL))
+  if (ix86_expand_vec_perm_vpermt2 (target, mask, op0, op1, NULL))
     return;
 
   if (TARGET_AVX2)
@@ -24515,8 +24744,7 @@ ix86_expand_int_addcc (rtx operands[])
 
   flags = XEXP (compare_op, 0);
 
-  if (GET_MODE (flags) == CCFPmode
-      || GET_MODE (flags) == CCFPUmode)
+  if (GET_MODE (flags) == CCFPmode)
     {
       fpcmp = true;
       code = ix86_fp_compare_code_to_integer (code);
@@ -24603,11 +24831,7 @@ ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
   /* Optimize constant pool reference to immediates.  This is used by fp
      moves, that force all constants to memory to allow combining.  */
   if (MEM_P (operand) && MEM_READONLY_P (operand))
-    {
-      rtx tmp = maybe_get_pool_constant (operand);
-      if (tmp)
-	operand = tmp;
-    }
+    operand = avoid_constant_pool_reference (operand);
 
   if (MEM_P (operand) && !offsettable_memref_p (operand))
     {
@@ -29804,8 +30028,12 @@ BDESC_VERIFYS (IX86_BUILTIN__BDESC_MPX_CONST_FIRST,
 	       IX86_BUILTIN__BDESC_MPX_LAST, 1);
 BDESC_VERIFYS (IX86_BUILTIN__BDESC_MULTI_ARG_FIRST,
 	       IX86_BUILTIN__BDESC_MPX_CONST_LAST, 1);
-BDESC_VERIFYS (IX86_BUILTIN_MAX,
+BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_FIRST,
 	       IX86_BUILTIN__BDESC_MULTI_ARG_LAST, 1);
+BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_NORMAL_FIRST,
+	       IX86_BUILTIN__BDESC_CET_LAST, 1);
+BDESC_VERIFYS (IX86_BUILTIN_MAX,
+	       IX86_BUILTIN__BDESC_CET_NORMAL_LAST, 1);
 
 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
    in the current target ISA to allow the user to compile particular modules
@@ -30472,6 +30700,35 @@ ix86_init_mmx_sse_builtins (void)
   BDESC_VERIFYS (IX86_BUILTIN__BDESC_MULTI_ARG_LAST,
 		 IX86_BUILTIN__BDESC_MULTI_ARG_FIRST,
 		 ARRAY_SIZE (bdesc_multi_arg) - 1);
+
+  /* Add CET inrinsics.  */
+  for (i = 0, d = bdesc_cet; i < ARRAY_SIZE (bdesc_cet); i++, d++)
+    {
+      BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_CET_FIRST, i);
+      if (d->name == 0)
+	continue;
+
+      ftype = (enum ix86_builtin_func_type) d->flag;
+      def_builtin2 (d->mask, d->name, ftype, d->code);
+    }
+  BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_LAST,
+		 IX86_BUILTIN__BDESC_CET_FIRST,
+		 ARRAY_SIZE (bdesc_cet) - 1);
+
+  for (i = 0, d = bdesc_cet_rdssp;
+       i < ARRAY_SIZE (bdesc_cet_rdssp);
+       i++, d++)
+    {
+      BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_CET_NORMAL_FIRST, i);
+      if (d->name == 0)
+	continue;
+
+      ftype = (enum ix86_builtin_func_type) d->flag;
+      def_builtin2 (d->mask, d->name, ftype, d->code);
+    }
+  BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_NORMAL_LAST,
+		 IX86_BUILTIN__BDESC_CET_NORMAL_FIRST,
+		 ARRAY_SIZE (bdesc_cet_rdssp) - 1);
 }
 
 static void
@@ -33425,6 +33682,7 @@ ix86_expand_args_builtin (const struct builtin_description *d,
     case UQI_FTYPE_V4SF_V4SF_INT:
     case UHI_FTYPE_V16SI_V16SI_INT:
     case UHI_FTYPE_V16SF_V16SF_INT:
+    case V64QI_FTYPE_V64QI_V64QI_INT:
       nargs = 3;
       nargs_constant = 1;
       break;
@@ -33652,6 +33910,13 @@ ix86_expand_args_builtin (const struct builtin_description *d,
       mask_pos = 1;
       nargs_constant = 1;
       break;
+    case V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI:
+    case V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI:
+    case V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI:
+      nargs = 5;
+      mask_pos = 1;
+      nargs_constant = 2;
+      break;
 
     default:
       gcc_unreachable ();
@@ -34830,10 +35095,10 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
 		     machine_mode mode, int ignore)
 {
   size_t i;
-  enum insn_code icode;
+  enum insn_code icode, icode2;
   tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
   tree arg0, arg1, arg2, arg3, arg4;
-  rtx op0, op1, op2, op3, op4, pat, insn;
+  rtx op0, op1, op2, op3, op4, pat, pat2, insn;
   machine_mode mode0, mode1, mode2, mode3, mode4;
   unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
 
@@ -35808,22 +36073,34 @@ rdseed_step:
 
     case IX86_BUILTIN_SBB32:
       icode = CODE_FOR_subborrowsi;
+      icode2 = CODE_FOR_subborrowsi_0;
       mode0 = SImode;
+      mode1 = DImode;
+      mode2 = CCmode;
       goto handlecarry;
 
     case IX86_BUILTIN_SBB64:
       icode = CODE_FOR_subborrowdi;
+      icode2 = CODE_FOR_subborrowdi_0;
       mode0 = DImode;
+      mode1 = TImode;
+      mode2 = CCmode;
       goto handlecarry;
 
     case IX86_BUILTIN_ADDCARRYX32:
       icode = CODE_FOR_addcarrysi;
+      icode2 = CODE_FOR_addcarrysi_0;
       mode0 = SImode;
+      mode1 = DImode;
+      mode2 = CCCmode;
       goto handlecarry;
 
     case IX86_BUILTIN_ADDCARRYX64:
       icode = CODE_FOR_addcarrydi;
+      icode2 = CODE_FOR_addcarrydi_0;
       mode0 = DImode;
+      mode1 = TImode;
+      mode2 = CCCmode;
 
     handlecarry:
       arg0 = CALL_EXPR_ARG (exp, 0); /* unsigned char c_in.  */
@@ -35832,7 +36109,8 @@ rdseed_step:
       arg3 = CALL_EXPR_ARG (exp, 3); /* unsigned int *sum_out.  */
 
       op1 = expand_normal (arg0);
-      op1 = copy_to_mode_reg (QImode, convert_to_mode (QImode, op1, 1));
+      if (!integer_zerop (arg0))
+	op1 = copy_to_mode_reg (QImode, convert_to_mode (QImode, op1, 1));
 
       op2 = expand_normal (arg1);
       if (!register_operand (op2, mode0))
@@ -35849,21 +36127,31 @@ rdseed_step:
 	  op4 = copy_addr_to_reg (op4);
 	}
 
-      /* Generate CF from input operand.  */
-      emit_insn (gen_addqi3_cconly_overflow (op1, constm1_rtx));
-
-      /* Generate instruction that consumes CF.  */
       op0 = gen_reg_rtx (mode0);
+      if (integer_zerop (arg0))
+	{
+	  /* If arg0 is 0, optimize right away into add or sub
+	     instruction that sets CCCmode flags.  */
+	  op1 = gen_rtx_REG (mode2, FLAGS_REG);
+	  emit_insn (GEN_FCN (icode2) (op0, op2, op3));
+	}
+      else
+	{
+	  /* Generate CF from input operand.  */
+	  emit_insn (gen_addqi3_cconly_overflow (op1, constm1_rtx));
 
-      op1 = gen_rtx_REG (CCCmode, FLAGS_REG);
-      pat = gen_rtx_LTU (mode0, op1, const0_rtx);
-      emit_insn (GEN_FCN (icode) (op0, op2, op3, op1, pat));
+	  /* Generate instruction that consumes CF.  */
+	  op1 = gen_rtx_REG (CCCmode, FLAGS_REG);
+	  pat = gen_rtx_LTU (mode1, op1, const0_rtx);
+	  pat2 = gen_rtx_LTU (mode0, op1, const0_rtx);
+	  emit_insn (GEN_FCN (icode) (op0, op2, op3, op1, pat, pat2));
+	}
 
       /* Return current CF value.  */
       if (target == 0)
         target = gen_reg_rtx (QImode);
 
-      PUT_MODE (pat, QImode);
+      pat = gen_rtx_LTU (QImode, op1, const0_rtx);
       emit_insn (gen_rtx_SET (target, pat));
 
       /* Store the result.  */
@@ -36656,6 +36944,57 @@ rdseed_step:
       emit_insn (gen_xabort (op0));
       return 0;
 
+    case IX86_BUILTIN_RSTORSSP:
+    case IX86_BUILTIN_CLRSSBSY:
+      arg0 = CALL_EXPR_ARG (exp, 0);
+      op0 = expand_normal (arg0);
+      icode = (fcode == IX86_BUILTIN_RSTORSSP
+	  ? CODE_FOR_rstorssp
+	  : CODE_FOR_clrssbsy);
+      if (!address_operand (op0, VOIDmode))
+	{
+	  op1 = convert_memory_address (Pmode, op0);
+	  op0 = copy_addr_to_reg (op1);
+	}
+      emit_insn (GEN_FCN (icode) (gen_rtx_MEM (Pmode, op0)));
+      return 0;
+
+    case IX86_BUILTIN_WRSSD:
+    case IX86_BUILTIN_WRSSQ:
+    case IX86_BUILTIN_WRUSSD:
+    case IX86_BUILTIN_WRUSSQ:
+      arg0 = CALL_EXPR_ARG (exp, 0);
+      op0 = expand_normal (arg0);
+      arg1 = CALL_EXPR_ARG (exp, 1);
+      op1 = expand_normal (arg1);
+      switch (fcode)
+	{
+	case IX86_BUILTIN_WRSSD:
+	  icode = CODE_FOR_wrsssi;
+	  mode = SImode;
+	  break;
+	case IX86_BUILTIN_WRSSQ:
+	  icode = CODE_FOR_wrssdi;
+	  mode = DImode;
+	  break;
+	case IX86_BUILTIN_WRUSSD:
+	  icode = CODE_FOR_wrusssi;
+	  mode = SImode;
+	  break;
+	case IX86_BUILTIN_WRUSSQ:
+	  icode = CODE_FOR_wrussdi;
+	  mode = DImode;
+	  break;
+	}
+      op0 = force_reg (mode, op0);
+      if (!address_operand (op1, VOIDmode))
+	{
+	  op2 = convert_memory_address (Pmode, op1);
+	  op1 = copy_addr_to_reg (op2);
+	}
+      emit_insn (GEN_FCN (icode) (op0, gen_rtx_MEM (mode, op1)));
+      return 0;
+
     default:
       break;
     }
@@ -36958,6 +37297,22 @@ s4fma_expand:
 					    d->flag, d->comparison);
     }
 
+  if (fcode >= IX86_BUILTIN__BDESC_CET_FIRST
+      && fcode <= IX86_BUILTIN__BDESC_CET_LAST)
+    {
+      i = fcode - IX86_BUILTIN__BDESC_CET_FIRST;
+      return ix86_expand_special_args_builtin (bdesc_cet + i, exp,
+					       target);
+    }
+
+  if (fcode >= IX86_BUILTIN__BDESC_CET_NORMAL_FIRST
+      && fcode <= IX86_BUILTIN__BDESC_CET_NORMAL_LAST)
+    {
+      i = fcode - IX86_BUILTIN__BDESC_CET_NORMAL_FIRST;
+      return ix86_expand_args_builtin (bdesc_cet_rdssp + i, exp,
+				       target);
+    }
+
   gcc_unreachable ();
 }
 
@@ -38347,6 +38702,28 @@ ix86_can_change_mode_class (machine_mode from, machine_mode to,
   return true;
 }
 
+/* Return index of MODE in the sse load/store tables.  */
+
+static inline int
+sse_store_index (machine_mode mode)
+{
+      switch (GET_MODE_SIZE (mode))
+	{
+	  case 4:
+	    return 0;
+	  case 8:
+	    return 1;
+	  case 16:
+	    return 2;
+	  case 32:
+	    return 3;
+	  case 64:
+	    return 4;
+	  default:
+	    return -1;
+	}
+}
+
 /* Return the cost of moving data of mode M between a
    register and memory.  A value of 2 is the default; this cost is
    relative to those in `REGISTER_MOVE_COST'.
@@ -38390,21 +38767,9 @@ inline_memory_move_cost (machine_mode mode, enum reg_class regclass,
     }
   if (SSE_CLASS_P (regclass))
     {
-      int index;
-      switch (GET_MODE_SIZE (mode))
-	{
-	  case 4:
-	    index = 0;
-	    break;
-	  case 8:
-	    index = 1;
-	    break;
-	  case 16:
-	    index = 2;
-	    break;
-	  default:
-	    return 100;
-	}
+      int index = sse_store_index (mode);
+      if (index == -1)
+	return 100;
       if (in == 2)
         return MAX (ix86_cost->sse_load [index], ix86_cost->sse_store [index]);
       return in ? ix86_cost->sse_load [index] : ix86_cost->sse_store [index];
@@ -38507,8 +38872,10 @@ ix86_register_move_cost (machine_mode mode, reg_class_t class1_i,
       /* In case of copying from general_purpose_register we may emit multiple
          stores followed by single load causing memory size mismatch stall.
          Count this as arbitrarily high cost of 20.  */
-      if (targetm.class_max_nregs (class1, mode)
-	  > targetm.class_max_nregs (class2, mode))
+      if (GET_MODE_BITSIZE (mode) > BITS_PER_WORD
+	  && TARGET_MEMORY_MISMATCH_STALL
+	  && targetm.class_max_nregs (class1, mode)
+	     > targetm.class_max_nregs (class2, mode))
 	cost += 20;
 
       /* In the case of FP/MMX moves, the registers actually overlap, and we
@@ -38530,12 +38897,19 @@ ix86_register_move_cost (machine_mode mode, reg_class_t class1_i,
        where integer modes in MMX/SSE registers are not tieable
        because of missing QImode and HImode moves to, from or between
        MMX/SSE registers.  */
-    return MAX (8, ix86_cost->mmxsse_to_integer);
+    return MAX (8, MMX_CLASS_P (class1) || MMX_CLASS_P (class2)
+		? ix86_cost->mmxsse_to_integer : ix86_cost->ssemmx_to_integer);
 
   if (MAYBE_FLOAT_CLASS_P (class1))
     return ix86_cost->fp_move;
   if (MAYBE_SSE_CLASS_P (class1))
-    return ix86_cost->sse_move;
+    {
+      if (GET_MODE_BITSIZE (mode) <= 128)
+	return ix86_cost->xmm_move;
+      if (GET_MODE_BITSIZE (mode) <= 256)
+	return ix86_cost->ymm_move;
+      return ix86_cost->zmm_move;
+    }
   if (MAYBE_MMX_CLASS_P (class1))
     return ix86_cost->mmx_move;
   return 2;
@@ -38806,6 +39180,27 @@ ix86_set_reg_reg_cost (machine_mode mode)
   return COSTS_N_INSNS (CEIL (GET_MODE_SIZE (mode), units));
 }
 
+/* Return cost of vector operation in MODE given that scalar version has
+   COST.  If PARALLEL is true assume that CPU has more than one unit
+   performing the operation.  */
+
+static int
+ix86_vec_cost (machine_mode mode, int cost, bool parallel)
+{
+  if (!VECTOR_MODE_P (mode))
+    return cost;
+ 
+  if (!parallel)
+    return cost * GET_MODE_NUNITS (mode);
+  if (GET_MODE_BITSIZE (mode) == 128
+      && TARGET_SSE_SPLIT_REGS)
+    return cost * 2;
+  if (GET_MODE_BITSIZE (mode) > 128
+      && TARGET_AVX128_OPTIMAL)
+    return cost * GET_MODE_BITSIZE (mode) / 128;
+  return cost;
+}
+
 /* Compute a (partial) cost for rtx X.  Return true if the complete
    cost has been computed, and false if subexpressions should be
    scanned.  In either case, *TOTAL contains the cost result.  */
@@ -38819,6 +39214,9 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
   enum rtx_code outer_code = (enum rtx_code) outer_code_i;
   const struct processor_costs *cost = speed ? ix86_cost : &ix86_size_cost;
   int src_cost;
+  machine_mode inner_mode = mode;
+  if (VECTOR_MODE_P (mode))
+    inner_mode = GET_MODE_INNER (mode);
 
   switch (code)
     {
@@ -38963,19 +39361,20 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 		     shift with one insn set the cost to prefer paddb.  */
 		  if (CONSTANT_P (XEXP (x, 1)))
 		    {
-		      *total = (cost->fabs
+		      *total = ix86_vec_cost (mode,
+				cost->sse_op
 				+ rtx_cost (XEXP (x, 0), mode, code, 0, speed)
-				+ (speed ? 2 : COSTS_N_BYTES (16)));
+				+ (speed ? 2 : COSTS_N_BYTES (16)), true);
 		      return true;
 		    }
 		  count = 3;
 		}
 	      else if (TARGET_SSSE3)
 		count = 7;
-	      *total = cost->fabs * count;
+	      *total = ix86_vec_cost (mode, cost->sse_op * count, true);
 	    }
 	  else
-	    *total = cost->fabs;
+	    *total = ix86_vec_cost (mode, cost->sse_op, true);
 	}
       else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
 	{
@@ -39017,9 +39416,9 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
         gcc_assert (FLOAT_MODE_P (mode));
         gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F);
 
-        /* ??? SSE scalar/vector cost should be used here.  */
-        /* ??? Bald assumption that fma has the same cost as fmul.  */
-        *total = cost->fmul;
+        *total = ix86_vec_cost (mode,
+				mode == SFmode ? cost->fmass : cost->fmasd,
+				true);
 	*total += rtx_cost (XEXP (x, 1), mode, FMA, 1, speed);
 
         /* Negate in op0 or op2 is free: FMS, FNMA, FNMS.  */
@@ -39038,8 +39437,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
     case MULT:
       if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
 	{
-	  /* ??? SSE scalar cost should be used here.  */
-	  *total = cost->fmul;
+	  *total = inner_mode == DFmode ? cost->mulsd : cost->mulss;
 	  return false;
 	}
       else if (X87_FLOAT_MODE_P (mode))
@@ -39049,8 +39447,9 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 	}
       else if (FLOAT_MODE_P (mode))
 	{
-	  /* ??? SSE vector cost should be used here.  */
-	  *total = cost->fmul;
+	  *total = ix86_vec_cost (mode,
+				  inner_mode == DFmode
+				  ? cost->mulsd : cost->mulss, true);
 	  return false;
 	}
       else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
@@ -39063,22 +39462,29 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 		extra = 5;
 	      else if (TARGET_SSSE3)
 		extra = 6;
-	      *total = cost->fmul * 2 + cost->fabs * extra;
+	      *total = ix86_vec_cost (mode,
+				      cost->mulss * 2 + cost->sse_op * extra,
+				      true);
 	    }
 	  /* V*DImode is emulated with 5-8 insns.  */
 	  else if (mode == V2DImode || mode == V4DImode)
 	    {
 	      if (TARGET_XOP && mode == V2DImode)
-		*total = cost->fmul * 2 + cost->fabs * 3;
+		*total = ix86_vec_cost (mode,
+					cost->mulss * 2 + cost->sse_op * 3,
+					true);
 	      else
-		*total = cost->fmul * 3 + cost->fabs * 5;
+		*total = ix86_vec_cost (mode,
+					cost->mulss * 3 + cost->sse_op * 5,
+					true);
 	    }
 	  /* Without sse4.1, we don't have PMULLD; it's emulated with 7
 	     insns, including two PMULUDQ.  */
 	  else if (mode == V4SImode && !(TARGET_SSE4_1 || TARGET_AVX))
-	    *total = cost->fmul * 2 + cost->fabs * 5;
+	    *total = ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 5,
+				    true);
 	  else
-	    *total = cost->fmul;
+	    *total = ix86_vec_cost (mode, cost->mulss, true);
 	  return false;
 	}
       else
@@ -39132,13 +39538,13 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
     case MOD:
     case UMOD:
       if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
-	/* ??? SSE cost should be used here.  */
-	*total = cost->fdiv;
+	*total = inner_mode == DFmode ? cost->divsd : cost->divss;
       else if (X87_FLOAT_MODE_P (mode))
 	*total = cost->fdiv;
       else if (FLOAT_MODE_P (mode))
-	/* ??? SSE vector cost should be used here.  */
-	*total = cost->fdiv;
+	*total = ix86_vec_cost (mode,
+			        inner_mode == DFmode ? cost->divsd : cost->divss,
+				true);
       else
 	*total = cost->divide[MODE_INDEX (mode)];
       return false;
@@ -39217,8 +39623,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 
       if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
 	{
-	  /* ??? SSE cost should be used here.  */
-	  *total = cost->fadd;
+	  *total = cost->addss;
 	  return false;
 	}
       else if (X87_FLOAT_MODE_P (mode))
@@ -39228,8 +39633,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 	}
       else if (FLOAT_MODE_P (mode))
 	{
-	  /* ??? SSE vector cost should be used here.  */
-	  *total = cost->fadd;
+	  *total = ix86_vec_cost (mode, cost->addss, true);
 	  return false;
 	}
       /* FALLTHRU */
@@ -39252,8 +39656,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
     case NEG:
       if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
 	{
-	  /* ??? SSE cost should be used here.  */
-	  *total = cost->fchs;
+	  *total = cost->sse_op;
 	  return false;
 	}
       else if (X87_FLOAT_MODE_P (mode))
@@ -39263,20 +39666,14 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 	}
       else if (FLOAT_MODE_P (mode))
 	{
-	  /* ??? SSE vector cost should be used here.  */
-	  *total = cost->fchs;
+	  *total = ix86_vec_cost (mode, cost->sse_op, true);
 	  return false;
 	}
       /* FALLTHRU */
 
     case NOT:
       if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
-	{
-	  /* ??? Should be SSE vector operation cost.  */
-	  /* At least for published AMD latencies, this really is the same
-	     as the latency for a simple fpu operation like fabs.  */
-	  *total = cost->fabs;
-	}
+	*total = ix86_vec_cost (mode, cost->sse_op, true);
       else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
 	*total = cost->add * 2;
       else
@@ -39309,28 +39706,38 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
     case FLOAT_EXTEND:
       if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH))
 	*total = 0;
+      else
+        *total = ix86_vec_cost (mode, cost->addss, true);
+      return false;
+
+    case FLOAT_TRUNCATE:
+      if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH))
+	*total = cost->fadd;
+      else
+        *total = ix86_vec_cost (mode, cost->addss, true);
       return false;
 
     case ABS:
+      /* SSE requires memory load for the constant operand. It may make
+	 sense to account for this.  Of course the constant operand may or
+	 may not be reused. */
       if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
-	/* ??? SSE cost should be used here.  */
-	*total = cost->fabs;
+	*total = cost->sse_op;
       else if (X87_FLOAT_MODE_P (mode))
 	*total = cost->fabs;
       else if (FLOAT_MODE_P (mode))
-	/* ??? SSE vector cost should be used here.  */
-	*total = cost->fabs;
+	*total = ix86_vec_cost (mode, cost->sse_op, true);
       return false;
 
     case SQRT:
       if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
-	/* ??? SSE cost should be used here.  */
-	*total = cost->fsqrt;
+	*total = mode == SFmode ? cost->sqrtss : cost->sqrtsd;
       else if (X87_FLOAT_MODE_P (mode))
 	*total = cost->fsqrt;
       else if (FLOAT_MODE_P (mode))
-	/* ??? SSE vector cost should be used here.  */
-	*total = cost->fsqrt;
+	*total = ix86_vec_cost (mode,
+				mode == SFmode ? cost->sqrtss : cost->sqrtsd,
+				true);
       return false;
 
     case UNSPEC:
@@ -39344,7 +39751,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
       /* ??? Assume all of these vector manipulation patterns are
 	 recognizable.  In which case they all pretty much have the
 	 same cost.  */
-     *total = cost->fabs;
+     *total = cost->sse_op;
      return true;
     case VEC_MERGE:
       mask = XEXP (x, 2);
@@ -39353,7 +39760,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
       if (TARGET_AVX512F && register_operand (mask, GET_MODE (mask)))
 	*total = rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed);
       else
-	*total = cost->fabs;
+	*total = cost->sse_op;
       return true;
 
     default:
@@ -39818,6 +40225,10 @@ x86_output_mi_thunk (FILE *file, tree, HOST_WIDE_INT delta,
 
   emit_note (NOTE_INSN_PROLOGUE_END);
 
+  /* CET is enabled, insert EB instruction.  */
+  if ((flag_cf_protection & CF_BRANCH) && TARGET_IBT)
+    emit_insn (gen_nop_endbr ());
+
   /* If VCALL_OFFSET, we'll need THIS in a register.  Might as well
      pull it in now and let DELTA benefit.  */
   if (REG_P (this_param))
@@ -40835,7 +41246,7 @@ ix86_vector_duplicate_value (machine_mode mode, rtx target, rtx val)
       reg = force_reg (innermode, val);
       if (GET_MODE (reg) != innermode)
 	reg = gen_lowpart (innermode, reg);
-      XEXP (dup, 0) = reg;
+      SET_SRC (PATTERN (insn)) = gen_vec_duplicate (mode, reg);
       seq = get_insns ();
       end_sequence ();
       if (seq)
@@ -42800,9 +43211,9 @@ ix86_encode_section_info (tree decl, rtx rtl, int first)
 enum rtx_code
 ix86_reverse_condition (enum rtx_code code, machine_mode mode)
 {
-  return (mode != CCFPmode && mode != CCFPUmode
-	  ? reverse_condition (code)
-	  : reverse_condition_maybe_unordered (code));
+  return (mode == CCFPmode
+	  ? reverse_condition_maybe_unordered (code)
+	  : reverse_condition (code));
 }
 
 /* Output code to perform an x87 FP register move, from OPERANDS[1]
@@ -43415,17 +43826,20 @@ static rtx_code_label *
 ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
                                   bool swap_operands)
 {
-  machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
+  bool unordered_compare = ix86_unordered_fp_compare (code);
   rtx_code_label *label;
-  rtx tmp;
+  rtx tmp, reg;
 
   if (swap_operands)
     std::swap (op0, op1);
 
   label = gen_label_rtx ();
-  tmp = gen_rtx_REG (fpcmp_mode, FLAGS_REG);
-  emit_insn (gen_rtx_SET (tmp, gen_rtx_COMPARE (fpcmp_mode, op0, op1)));
-  tmp = gen_rtx_fmt_ee (code, VOIDmode, tmp, const0_rtx);
+  tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
+  if (unordered_compare)
+    tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
+  reg = gen_rtx_REG (CCFPmode, FLAGS_REG);
+  emit_insn (gen_rtx_SET (reg, tmp));
+  tmp = gen_rtx_fmt_ee (code, VOIDmode, reg, const0_rtx);
   tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
 			      gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
   tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
@@ -44044,35 +44458,83 @@ static int
 ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
                                  tree vectype, int)
 {
+  bool fp = false;
+  machine_mode mode = TImode;
+  int index;
+  if (vectype != NULL)
+    {
+      fp = FLOAT_TYPE_P (vectype);
+      mode = TYPE_MODE (vectype);
+    }
+
   switch (type_of_cost)
     {
       case scalar_stmt:
-        return ix86_cost->scalar_stmt_cost;
+        return fp ? ix86_cost->addss : COSTS_N_INSNS (1);
 
       case scalar_load:
-        return ix86_cost->scalar_load_cost;
+	/* load/store costs are relative to register move which is 2. Recompute
+ 	   it to COSTS_N_INSNS so everything have same base.  */
+        return COSTS_N_INSNS (fp ? ix86_cost->sse_load[0]
+			      : ix86_cost->int_load [2]) / 2;
 
       case scalar_store:
-        return ix86_cost->scalar_store_cost;
+        return COSTS_N_INSNS (fp ? ix86_cost->sse_store[0]
+			      : ix86_cost->int_store [2]) / 2;
 
       case vector_stmt:
-        return ix86_cost->vec_stmt_cost;
+        return ix86_vec_cost (mode,
+			      fp ? ix86_cost->addss : ix86_cost->sse_op,
+			      true);
 
       case vector_load:
-        return ix86_cost->vec_align_load_cost;
+	index = sse_store_index (mode);
+	gcc_assert (index >= 0);
+        return ix86_vec_cost (mode,
+			      COSTS_N_INSNS (ix86_cost->sse_load[index]) / 2,
+			      true);
 
       case vector_store:
-        return ix86_cost->vec_store_cost;
+	index = sse_store_index (mode);
+        return ix86_vec_cost (mode,
+			      COSTS_N_INSNS (ix86_cost->sse_store[index]) / 2,
+			      true);
 
       case vec_to_scalar:
-        return ix86_cost->vec_to_scalar_cost;
-
       case scalar_to_vec:
-        return ix86_cost->scalar_to_vec_cost;
+        return ix86_vec_cost (mode, ix86_cost->sse_op, true);
 
+      /* We should have separate costs for unaligned loads and gather/scatter.
+	 Do that incrementally.  */
       case unaligned_load:
+	index = sse_store_index (mode);
+        return ix86_vec_cost (mode,
+			      COSTS_N_INSNS
+				 (ix86_cost->sse_unaligned_load[index]) / 2,
+			      true);
+
       case unaligned_store:
-        return ix86_cost->vec_unalign_load_cost;
+	index = sse_store_index (mode);
+        return ix86_vec_cost (mode,
+			      COSTS_N_INSNS
+				 (ix86_cost->sse_unaligned_store[index]) / 2,
+			      true);
+
+      case vector_gather_load:
+        return ix86_vec_cost (mode,
+			      COSTS_N_INSNS
+				 (ix86_cost->gather_static
+				  + ix86_cost->gather_per_elt
+				    * TYPE_VECTOR_SUBPARTS (vectype)) / 2,
+			      true);
+
+      case vector_scatter_store:
+        return ix86_vec_cost (mode,
+			      COSTS_N_INSNS
+				 (ix86_cost->scatter_static
+				  + ix86_cost->scatter_per_elt
+				    * TYPE_VECTOR_SUBPARTS (vectype)) / 2,
+			      true);
 
       case cond_branch_taken:
         return ix86_cost->cond_taken_branch_cost;
@@ -44082,10 +44544,11 @@ ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 
       case vec_perm:
       case vec_promote_demote:
-        return ix86_cost->vec_stmt_cost;
+        return ix86_vec_cost (mode,
+			      ix86_cost->sse_op, true);
 
       case vec_construct:
-	return ix86_cost->vec_stmt_cost * (TYPE_VECTOR_SUBPARTS (vectype) - 1);
+	return ix86_vec_cost (mode, ix86_cost->sse_op, false);
 
       default:
         gcc_unreachable ();
@@ -44963,8 +45426,8 @@ expand_vec_perm_1 (struct expand_vec_perm_d *d)
   if (ix86_expand_vec_one_operand_perm_avx512 (d))
     return true;
 
-  /* Try the AVX512F vpermi2 instructions.  */
-  if (ix86_expand_vec_perm_vpermi2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
+  /* Try the AVX512F vpermt2/vpermi2 instructions.  */
+  if (ix86_expand_vec_perm_vpermt2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
     return true;
 
   /* See if we can get the same permutation in different vector integer
@@ -46621,9 +47084,9 @@ expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
 }
 
 /* Implement arbitrary permutations of two V64QImode operands
-   will 2 vpermi2w, 2 vpshufb and one vpor instruction.  */
+   with 2 vperm[it]2w, 2 vpshufb and one vpor instruction.  */
 static bool
-expand_vec_perm_vpermi2_vpshub2 (struct expand_vec_perm_d *d)
+expand_vec_perm_vpermt2_vpshub2 (struct expand_vec_perm_d *d)
 {
   if (!TARGET_AVX512BW || !(d->vmode == V64QImode))
     return false;
@@ -46868,7 +47331,7 @@ ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
   if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
     return true;
 
-  if (expand_vec_perm_vpermi2_vpshub2 (d))
+  if (expand_vec_perm_vpermt2_vpshub2 (d))
     return true;
 
   /* ??? Look for narrow permutations whose element orderings would
@@ -47016,17 +47479,17 @@ ix86_vectorize_vec_perm_const_ok (machine_mode vmode, vec_perm_indices sel)
     case E_V8DImode:
     case E_V8DFmode:
       if (TARGET_AVX512F)
-	/* All implementable with a single vpermi2 insn.  */
+	/* All implementable with a single vperm[it]2 insn.  */
 	return true;
       break;
     case E_V32HImode:
       if (TARGET_AVX512BW)
-	/* All implementable with a single vpermi2 insn.  */
+	/* All implementable with a single vperm[it]2 insn.  */
 	return true;
       break;
     case E_V64QImode:
       if (TARGET_AVX512BW)
-	/* Implementable with 2 vpermi2, 2 vpshufb and 1 or insn.  */
+	/* Implementable with 2 vperm[it]2, 2 vpshufb and 1 or insn.  */
 	return true;
       break;
     case E_V8SImode:
@@ -47034,7 +47497,7 @@ ix86_vectorize_vec_perm_const_ok (machine_mode vmode, vec_perm_indices sel)
     case E_V4DFmode:
     case E_V4DImode:
       if (TARGET_AVX512VL)
-	/* All implementable with a single vpermi2 insn.  */
+	/* All implementable with a single vperm[it]2 insn.  */
 	return true;
       break;
     case E_V16HImode:
@@ -47204,7 +47667,6 @@ ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
       op2_h = gen_reg_rtx (qimode);
       emit_insn (gen_il (op2_l, op2, op2));
       emit_insn (gen_ih (op2_h, op2, op2));
-      /* FALLTHRU */
 
       op1_l = gen_reg_rtx (qimode);
       op1_h = gen_reg_rtx (qimode);
@@ -47632,6 +48094,46 @@ ix86_bnd_prefixed_insn_p (rtx insn)
   return chkp_function_instrumented_p (current_function_decl);
 }
 
+/* Return 1 if control tansfer instruction INSN
+   should be encoded with notrack prefix.  */
+
+static bool
+ix86_notrack_prefixed_insn_p (rtx insn)
+{
+  if (!insn || !((flag_cf_protection & CF_BRANCH) && TARGET_IBT))
+    return false;
+
+  if (CALL_P (insn))
+    {
+      rtx call = get_call_rtx_from (insn);
+      gcc_assert (call != NULL_RTX);
+      rtx addr = XEXP (call, 0);
+
+      /* Do not emit 'notrack' if it's not an indirect call.  */
+      if (MEM_P (addr)
+	  && GET_CODE (XEXP (addr, 0)) == SYMBOL_REF)
+	return false;
+      else
+	return find_reg_note (insn, REG_CALL_NOCF_CHECK, 0);
+    }
+
+  if (JUMP_P (insn) && !flag_cet_switch)
+    {
+      rtx target = JUMP_LABEL (insn);
+      if (target == NULL_RTX || ANY_RETURN_P (target))
+	return false;
+
+      /* Check the jump is a switch table.  */
+      rtx_insn *label = as_a<rtx_insn *> (target);
+      rtx_insn *table = next_insn (label);
+      if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
+	return false;
+      else
+	return true;
+    }
+  return false;
+}
+
 /* Calculate integer abs() using only SSE2 instructions.  */
 
 void
@@ -49420,6 +49922,9 @@ ix86_run_selftests (void)
 #undef TARGET_DELEGITIMIZE_ADDRESS
 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
 
+#undef TARGET_CONST_NOT_OK_FOR_DEBUG_P
+#define TARGET_CONST_NOT_OK_FOR_DEBUG_P ix86_const_not_ok_for_debug_p
+
 #undef TARGET_MS_BITFIELD_LAYOUT_P
 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
 
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index 1c796ef392d..4855105c4ac 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -103,6 +103,8 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define TARGET_SGX_P(x)	TARGET_ISA_SGX_P(x)
 #define TARGET_RDPID	TARGET_ISA_RDPID
 #define TARGET_RDPID_P(x)	TARGET_ISA_RDPID_P(x)
+#define TARGET_GFNI	TARGET_ISA_GFNI
+#define TARGET_GFNI_P(x)	TARGET_ISA_GFNI_P(x)
 #define TARGET_BMI	TARGET_ISA_BMI
 #define TARGET_BMI_P(x)	TARGET_ISA_BMI_P(x)
 #define TARGET_BMI2	TARGET_ISA_BMI2
@@ -167,6 +169,10 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define TARGET_MWAITX_P(x)	TARGET_ISA_MWAITX_P(x)
 #define TARGET_PKU	TARGET_ISA_PKU
 #define TARGET_PKU_P(x)	TARGET_ISA_PKU_P(x)
+#define TARGET_IBT	TARGET_ISA_IBT
+#define TARGET_IBT_P(x)	TARGET_ISA_IBT_P(x)
+#define TARGET_SHSTK	TARGET_ISA_SHSTK
+#define TARGET_SHSTK_P(x)	TARGET_ISA_SHSTK_P(x)
 
 #define TARGET_LP64	TARGET_ABI_64
 #define TARGET_LP64_P(x)	TARGET_ABI_64_P(x)
@@ -236,13 +242,21 @@ struct processor_costs {
 				   in SImode and DImode */
   const int mmx_store[2];	/* cost of storing MMX register
 				   in SImode and DImode */
-  const int sse_move;		/* cost of moving SSE register.  */
-  const int sse_load[3];	/* cost of loading SSE register
-				   in SImode, DImode and TImode*/
-  const int sse_store[3];	/* cost of storing SSE register
-				   in SImode, DImode and TImode*/
+  const int xmm_move, ymm_move, /* cost of moving XMM and YMM register.  */
+	    zmm_move;
+  const int sse_load[5];	/* cost of loading SSE register
+				   in 32bit, 64bit, 128bit, 256bit and 512bit */
+  const int sse_unaligned_load[5];/* cost of unaligned load.  */
+  const int sse_store[5];	/* cost of storing SSE register
+				   in SImode, DImode and TImode.  */
+  const int sse_unaligned_store[5];/* cost of unaligned store.  */
   const int mmxsse_to_integer;	/* cost of moving mmxsse register to
-				   integer and vice versa.  */
+				   integer.  */
+  const int ssemmx_to_integer;  /* cost of moving integer to mmxsse register. */
+  const int gather_static, gather_per_elt; /* Cost of gather load is computed
+				   as static + per_item * nelts. */
+  const int scatter_static, scatter_per_elt; /* Cost of gather store is
+				   computed as static + per_item * nelts.  */
   const int l1_cache_size;	/* size of l1 cache, in kilobytes.  */
   const int l2_cache_size;	/* size of l2 cache, in kilobytes.  */
   const int prefetch_block;	/* bytes moved to cache for prefetch.  */
@@ -257,6 +271,16 @@ struct processor_costs {
   const int fsqrt;		/* cost of FSQRT instruction.  */
 				/* Specify what algorithm
 				   to use for stringops on unknown size.  */
+  const int sse_op;		/* cost of cheap SSE instruction.  */
+  const int addss;		/* cost of ADDSS/SD SUBSS/SD instructions.  */
+  const int mulss;		/* cost of MULSS instructions.  */
+  const int mulsd;		/* cost of MULSD instructions.  */
+  const int fmass;		/* cost of FMASS instructions.  */
+  const int fmasd;		/* cost of FMASD instructions.  */
+  const int divss;		/* cost of DIVSS instructions.  */
+  const int divsd;		/* cost of DIVSD instructions.  */
+  const int sqrtss;		/* cost of SQRTSS instructions.  */
+  const int sqrtsd;		/* cost of SQRTSD instructions.  */
   const int reassoc_int, reassoc_fp, reassoc_vec_int, reassoc_vec_fp;
 				/* Specify reassociation width for integer,
 				   fp, vector integer and vector fp
@@ -265,18 +289,6 @@ struct processor_costs {
 				   parallel.  See also
 				   ix86_reassociation_width.  */
   struct stringop_algs *memcpy, *memset;
-  const int scalar_stmt_cost;   /* Cost of any scalar operation, excluding
-				   load and store.  */
-  const int scalar_load_cost;   /* Cost of scalar load.  */
-  const int scalar_store_cost;  /* Cost of scalar store.  */
-  const int vec_stmt_cost;      /* Cost of any vector operation, excluding
-                                   load, store, vector-to-scalar and
-                                   scalar-to-vector operation.  */
-  const int vec_to_scalar_cost;    /* Cost of vect-to-scalar operation.  */
-  const int scalar_to_vec_cost;    /* Cost of scalar-to-vector operation.  */
-  const int vec_align_load_cost;   /* Cost of aligned vector load.  */
-  const int vec_unalign_load_cost; /* Cost of unaligned vector load.  */
-  const int vec_store_cost;        /* Cost of vector store.  */
   const int cond_taken_branch_cost;    /* Cost of taken branch for vectorizer
 					  cost model.  */
   const int cond_not_taken_branch_cost;/* Cost of not taken branch for
@@ -2589,6 +2601,7 @@ struct GTY(()) machine_function {
 #define ix86_current_function_calls_tls_descriptor \
   (ix86_tls_descriptor_calls_expanded_in_cfun && df_regs_ever_live_p (SP_REG))
 #define ix86_static_chain_on_stack (cfun->machine->static_chain_on_stack)
+#define ix86_red_zone_size (cfun->machine->frame.red_zone_size)
 
 /* Control behavior of x86_file_start.  */
 #define X86_FILE_START_VERSION_DIRECTIVE false
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 3413b90028f..d48decbb7d9 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -62,7 +62,7 @@
 ;; ; -- print a semicolon (after prefixes due to bug in older gas).
 ;; ~ -- print "i" if TARGET_AVX2, "f" otherwise.
 ;; ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
-;; ! -- print MPX prefix for jxx/call/ret instructions if required.
+;; ! -- print MPX or NOTRACK prefix for jxx/call/ret instructions if required.
 
 (define_c_enum "unspec" [
   ;; Relocation specifiers
@@ -99,6 +99,7 @@
   UNSPEC_SCAS
   UNSPEC_FNSTSW
   UNSPEC_SAHF
+  UNSPEC_NOTRAP
   UNSPEC_PARITY
   UNSPEC_FSTCW
   UNSPEC_FLDCW
@@ -112,6 +113,7 @@
   UNSPEC_STOS
   UNSPEC_PEEPSIB
   UNSPEC_INSN_FALSE_DEP
+  UNSPEC_SBB
 
   ;; For SSE/MMX support:
   UNSPEC_FIX_NOTRUNC
@@ -274,6 +276,17 @@
 
   ;; For RDPID support
   UNSPECV_RDPID
+
+  ;; For CET support
+  UNSPECV_NOP_ENDBR
+  UNSPECV_NOP_RDSSP
+  UNSPECV_INCSSP
+  UNSPECV_SAVEPREVSSP
+  UNSPECV_RSTORSSP
+  UNSPECV_WRSS
+  UNSPECV_WRUSS
+  UNSPECV_SETSSBSY
+  UNSPECV_CLRSSBSY
 ])
 
 ;; Constants to represent rounding modes in the ROUND instruction
@@ -798,7 +811,7 @@
 (define_attr "isa" "base,x64,x64_sse4,x64_sse4_noavx,x64_avx,nox64,
 		    sse2,sse2_noavx,sse3,sse4,sse4_noavx,avx,noavx,
 		    avx2,noavx2,bmi,bmi2,fma4,fma,avx512f,noavx512f,
-		    fma_avx512f,avx512bw,noavx512bw,avx512dq,noavx512dq,
+		    avx512bw,noavx512bw,avx512dq,noavx512dq,
 		    avx512vl,noavx512vl,x64_avx512dq,x64_avx512bw"
   (const_string "base"))
 
@@ -832,8 +845,6 @@
 	 (eq_attr "isa" "fma") (symbol_ref "TARGET_FMA")
 	 (eq_attr "isa" "avx512f") (symbol_ref "TARGET_AVX512F")
 	 (eq_attr "isa" "noavx512f") (symbol_ref "!TARGET_AVX512F")
-	 (eq_attr "isa" "fma_avx512f")
-	   (symbol_ref "TARGET_FMA || TARGET_AVX512F")
 	 (eq_attr "isa" "avx512bw") (symbol_ref "TARGET_AVX512BW")
 	 (eq_attr "isa" "noavx512bw") (symbol_ref "!TARGET_AVX512BW")
 	 (eq_attr "isa" "avx512dq") (symbol_ref "TARGET_AVX512DQ")
@@ -1468,9 +1479,6 @@
 
 ;; FP compares, step 1:
 ;; Set the FP condition codes.
-;;
-;; CCFPmode	compare with exceptions
-;; CCFPUmode	compare with no exceptions
 
 ;; We may not use "#" to split and emit these, since the REG_DEAD notes
 ;; used to manage the reg stack popping would not be preserved.
@@ -1577,9 +1585,11 @@
 (define_insn "*cmpu<mode>_i387"
   [(set (match_operand:HI 0 "register_operand" "=a")
 	(unspec:HI
-	  [(compare:CCFPU
-	     (match_operand:X87MODEF 1 "register_operand" "f")
-	     (match_operand:X87MODEF 2 "register_operand" "f"))]
+	  [(unspec:CCFP
+	     [(compare:CCFP
+		(match_operand:X87MODEF 1 "register_operand" "f")
+		(match_operand:X87MODEF 2 "register_operand" "f"))]
+	     UNSPEC_NOTRAP)]
 	  UNSPEC_FNSTSW))]
   "TARGET_80387"
   "* return output_fp_compare (insn, operands, false, true);"
@@ -1588,18 +1598,22 @@
    (set_attr "mode" "<MODE>")])
 
 (define_insn_and_split "*cmpu<mode>_cc_i387"
-  [(set (reg:CCFPU FLAGS_REG)
-	(compare:CCFPU
-	  (match_operand:X87MODEF 1 "register_operand" "f")
-	  (match_operand:X87MODEF 2 "register_operand" "f")))
+  [(set (reg:CCFP FLAGS_REG)
+	(unspec:CCFP
+	  [(compare:CCFP
+	     (match_operand:X87MODEF 1 "register_operand" "f")
+	     (match_operand:X87MODEF 2 "register_operand" "f"))]
+	  UNSPEC_NOTRAP))
    (clobber (match_operand:HI 0 "register_operand" "=a"))]
   "TARGET_80387 && TARGET_SAHF && !TARGET_CMOVE"
   "#"
   "&& reload_completed"
   [(set (match_dup 0)
 	(unspec:HI
-	  [(compare:CCFPU (match_dup 1)(match_dup 2))]
-	UNSPEC_FNSTSW))
+	  [(unspec:CCFP
+	     [(compare:CCFP (match_dup 1)(match_dup 2))]
+	     UNSPEC_NOTRAP)]
+	  UNSPEC_FNSTSW))
    (set (reg:CC FLAGS_REG)
 	(unspec:CC [(match_dup 0)] UNSPEC_SAHF))]
   ""
@@ -1685,21 +1699,30 @@
    (set_attr "mode" "SI")])
 
 ;; Pentium Pro can do steps 1 through 3 in one go.
-;; comi*, ucomi*, fcomi*, ficomi*, fucomi*
-;; (these i387 instructions set flags directly)
+;; (these instructions set flags directly)
 
-(define_mode_iterator FPCMP [CCFP CCFPU])
-(define_mode_attr unord [(CCFP "") (CCFPU "u")])
+(define_subst_attr "unord" "unord_subst" "" "u")
+(define_subst_attr "unordered" "unord_subst" "false" "true")
+
+(define_subst "unord_subst"
+  [(set (match_operand:CCFP 0)
+        (match_operand:CCFP 1))]
+  ""
+  [(set (match_dup 0)
+        (unspec:CCFP
+	  [(match_dup 1)]
+	  UNSPEC_NOTRAP))])
 
-(define_insn "*cmpi<FPCMP:unord><MODEF:mode>"
-  [(set (reg:FPCMP FLAGS_REG)
-	(compare:FPCMP
+(define_insn "*cmpi<unord><MODEF:mode>"
+  [(set (reg:CCFP FLAGS_REG)
+	(compare:CCFP
 	  (match_operand:MODEF 0 "register_operand" "f,v")
 	  (match_operand:MODEF 1 "register_ssemem_operand" "f,vm")))]
   "(SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH)
    || (TARGET_80387 && TARGET_CMOVE)"
-  "* return output_fp_compare (insn, operands, true,
-			       <FPCMP:MODE>mode == CCFPUmode);"
+  "@
+   * return output_fp_compare (insn, operands, true, <unordered>);
+   %v<unord>comi<MODEF:ssemodesuffix>\t{%1, %0|%0, %1}"
   [(set_attr "type" "fcmp,ssecomi")
    (set_attr "prefix" "orig,maybe_vex")
    (set_attr "mode" "<MODEF:MODE>")
@@ -1728,13 +1751,12 @@
 	 (symbol_ref "false"))))])
 
 (define_insn "*cmpi<unord>xf_i387"
-  [(set (reg:FPCMP FLAGS_REG)
-	(compare:FPCMP
+  [(set (reg:CCFP FLAGS_REG)
+	(compare:CCFP
 	  (match_operand:XF 0 "register_operand" "f")
 	  (match_operand:XF 1 "register_operand" "f")))]
   "TARGET_80387 && TARGET_CMOVE"
-  "* return output_fp_compare (insn, operands, true,
-			       <MODE>mode == CCFPUmode);"
+  "* return output_fp_compare (insn, operands, true, <unordered>);"
   [(set_attr "type" "fcmp")
    (set_attr "mode" "XF")
    (set_attr "athlon_decode" "vector")
@@ -3564,8 +3586,10 @@
 
 	       /* movaps is one byte shorter for non-AVX targets.  */
 	       (eq_attr "alternative" "13,17")
-		 (cond [(ior (match_operand 0 "ext_sse_reg_operand")
-			     (match_operand 1 "ext_sse_reg_operand"))
+		 (cond [(and (ior (not (match_test "TARGET_PREFER_AVX256"))
+				  (not (match_test "TARGET_AVX512VL")))
+			     (ior (match_operand 0 "ext_sse_reg_operand")
+				  (match_operand 1 "ext_sse_reg_operand")))
 			  (const_string "V8DF")
 			(ior (not (match_test "TARGET_SSE2"))
 			     (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
@@ -3739,8 +3763,10 @@
 		  better to maintain the whole registers in single format
 		  to avoid problems on using packed logical operations.  */
 	       (eq_attr "alternative" "6")
-		 (cond [(ior  (match_operand 0 "ext_sse_reg_operand")
-			      (match_operand 1 "ext_sse_reg_operand"))
+		 (cond [(and (ior (not (match_test "TARGET_PREFER_AVX256"))
+				  (not (match_test "TARGET_AVX512VL")))
+			     (ior (match_operand 0 "ext_sse_reg_operand")
+				  (match_operand 1 "ext_sse_reg_operand")))
 			  (const_string "V16SF")
 			(ior (match_test "TARGET_SSE_PARTIAL_REG_DEPENDENCY")
 			     (match_test "TARGET_SSE_SPLIT_REGS"))
@@ -6765,6 +6791,17 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "<MODE>")])
 
+(define_peephole2
+  [(parallel
+     [(set (reg:CC FLAGS_REG)
+	   (compare:CC (match_operand:SWI 0 "general_reg_operand")
+		       (match_operand:SWI 1 "general_gr_operand")))
+      (set (match_dup 0)
+	   (minus:SWI (match_dup 0) (match_dup 1)))])]
+  "find_regno_note (peep2_next_insn (0), REG_UNUSED, REGNO (operands[0])) != 0"
+  [(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 0) (match_dup 1)))])
+
 (define_insn "*subsi_3_zext"
   [(set (reg FLAGS_REG)
 	(compare (match_operand:SI 1 "register_operand" "0")
@@ -6818,15 +6855,19 @@
 (define_insn "addcarry<mode>"
   [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
-	  (plus:SWI48
+	  (zero_extend:<DWI>
 	    (plus:SWI48
-	      (match_operator:SWI48 4 "ix86_carry_flag_operator"
-	       [(match_operand 3 "flags_reg_operand") (const_int 0)])
-	      (match_operand:SWI48 1 "nonimmediate_operand" "%0"))
-	    (match_operand:SWI48 2 "nonimmediate_operand" "rm"))
-	  (match_dup 1)))
+	      (plus:SWI48
+		(match_operator:SWI48 5 "ix86_carry_flag_operator"
+		  [(match_operand 3 "flags_reg_operand") (const_int 0)])
+		(match_operand:SWI48 1 "nonimmediate_operand" "%0"))
+	      (match_operand:SWI48 2 "nonimmediate_operand" "rm")))
+	  (plus:<DWI>
+	    (zero_extend:<DWI> (match_dup 2))
+	    (match_operator:<DWI> 4 "ix86_carry_flag_operator"
+	      [(match_dup 3) (const_int 0)]))))
    (set (match_operand:SWI48 0 "register_operand" "=r")
-	(plus:SWI48 (plus:SWI48 (match_op_dup 4
+	(plus:SWI48 (plus:SWI48 (match_op_dup 5
 				 [(match_dup 3) (const_int 0)])
 				(match_dup 1))
 		    (match_dup 2)))]
@@ -6837,6 +6878,18 @@
    (set_attr "pent_pair" "pu")
    (set_attr "mode" "<MODE>")])
 
+(define_expand "addcarry<mode>_0"
+  [(parallel
+     [(set (reg:CCC FLAGS_REG)
+	   (compare:CCC
+	     (plus:SWI48
+	       (match_operand:SWI48 1 "nonimmediate_operand")
+	       (match_operand:SWI48 2 "x86_64_general_operand"))
+	     (match_dup 1)))
+      (set (match_operand:SWI48 0 "register_operand")
+	   (plus:SWI48 (match_dup 1) (match_dup 2)))])]
+  "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)")
+
 (define_insn "sub<mode>3_carry"
   [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(minus:SWI
@@ -6870,18 +6923,67 @@
    (set_attr "pent_pair" "pu")
    (set_attr "mode" "SI")])
 
+(define_insn "sub<mode>3_carry_ccc"
+  [(set (reg:CCC FLAGS_REG)
+	(compare:CCC
+	  (zero_extend:<DWI> (match_operand:DWIH 1 "register_operand" "0"))
+	  (plus:<DWI>
+	    (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0))
+	    (zero_extend:<DWI>
+	      (match_operand:DWIH 2 "x86_64_sext_operand" "rmWe")))))
+   (clobber (match_scratch:DWIH 0 "=r"))]
+  ""
+  "sbb{<imodesuffix>}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "<MODE>")])
+
+(define_insn "*sub<mode>3_carry_ccc_1"
+  [(set (reg:CCC FLAGS_REG)
+	(compare:CCC
+	  (zero_extend:<DWI> (match_operand:DWIH 1 "register_operand" "0"))
+	  (plus:<DWI>
+	    (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0))
+	    (match_operand:<DWI> 2 "x86_64_dwzext_immediate_operand" "Wf"))))
+   (clobber (match_scratch:DWIH 0 "=r"))]
+  ""
+{
+  operands[3] = simplify_subreg (<MODE>mode, operands[2], <DWI>mode, 0);
+  return "sbb{<imodesuffix>}\t{%3, %0|%0, %3}";
+}
+  [(set_attr "type" "alu")
+   (set_attr "mode" "<MODE>")])
+
+;; The sign flag is set from the
+;; (compare (match_dup 1) (plus:DWIH (ltu:DWIH ...) (match_dup 2)))
+;; result, the overflow flag likewise, but the overflow flag is also
+;; set if the (plus:DWIH (ltu:DWIH ...) (match_dup 2)) overflows.
+(define_insn "sub<mode>3_carry_ccgz"
+  [(set (reg:CCGZ FLAGS_REG)
+	(unspec:CCGZ [(match_operand:DWIH 1 "register_operand" "0")
+		      (match_operand:DWIH 2 "x86_64_general_operand" "rme")
+		      (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))]
+		     UNSPEC_SBB))
+   (clobber (match_scratch:DWIH 0 "=r"))]
+  ""
+  "sbb{<imodesuffix>}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "<MODE>")])
+
 (define_insn "subborrow<mode>"
   [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
-	  (match_operand:SWI48 1 "nonimmediate_operand" "0")
-	  (plus:SWI48
-	    (match_operator:SWI48 4 "ix86_carry_flag_operator"
-	     [(match_operand 3 "flags_reg_operand") (const_int 0)])
-	    (match_operand:SWI48 2 "nonimmediate_operand" "rm"))))
+	  (zero_extend:<DWI>
+	    (match_operand:SWI48 1 "nonimmediate_operand" "0"))
+	  (plus:<DWI>
+	    (match_operator:<DWI> 4 "ix86_carry_flag_operator"
+	      [(match_operand 3 "flags_reg_operand") (const_int 0)])
+	    (zero_extend:<DWI>
+	      (match_operand:SWI48 2 "nonimmediate_operand" "rm")))))
    (set (match_operand:SWI48 0 "register_operand" "=r")
-	(minus:SWI48 (minus:SWI48 (match_dup 1)
-				  (match_op_dup 4
-				   [(match_dup 3) (const_int 0)]))
+	(minus:SWI48 (minus:SWI48
+		       (match_dup 1)
+		       (match_operator:SWI48 5 "ix86_carry_flag_operator"
+			 [(match_dup 3) (const_int 0)]))
 		     (match_dup 2)))]
   "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
   "sbb{<imodesuffix>}\t{%2, %0|%0, %2}"
@@ -6889,6 +6991,16 @@
    (set_attr "use_carry" "1")
    (set_attr "pent_pair" "pu")
    (set_attr "mode" "<MODE>")])
+
+(define_expand "subborrow<mode>_0"
+  [(parallel
+     [(set (reg:CC FLAGS_REG)
+	   (compare:CC
+	     (match_operand:SWI48 1 "nonimmediate_operand")
+	     (match_operand:SWI48 2 "<general_operand>")))
+      (set (match_operand:SWI48 0 "register_operand")
+	   (minus:SWI48 (match_dup 1) (match_dup 2)))])]
+  "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)")
 
 ;; Overflow setting add instructions
 
@@ -12286,6 +12398,34 @@
   ix86_expand_clear (operands[3]);
 })
 
+(define_peephole2
+  [(set (reg FLAGS_REG) (match_operand 0))
+   (parallel [(set (reg FLAGS_REG) (match_operand 1))
+	      (match_operand 5)])
+   (set (match_operand:QI 2 "register_operand")
+	(match_operator:QI 3 "ix86_comparison_operator"
+	  [(reg FLAGS_REG) (const_int 0)]))
+   (set (match_operand 4 "any_QIreg_operand")
+	(zero_extend (match_dup 2)))]
+  "(peep2_reg_dead_p (4, operands[2])
+    || operands_match_p (operands[2], operands[4]))
+   && ! reg_overlap_mentioned_p (operands[4], operands[0])
+   && ! reg_overlap_mentioned_p (operands[4], operands[1])
+   && ! reg_set_p (operands[4], operands[5])
+   && refers_to_regno_p (FLAGS_REG, operands[1], (rtx *)NULL)
+   && peep2_regno_dead_p (0, FLAGS_REG)"
+  [(set (match_dup 6) (match_dup 0))
+   (parallel [(set (match_dup 7) (match_dup 1))
+	      (match_dup 5)])
+   (set (strict_low_part (match_dup 8))
+	(match_dup 3))]
+{
+  operands[6] = gen_rtx_REG (GET_MODE (operands[0]), FLAGS_REG);
+  operands[7] = gen_rtx_REG (GET_MODE (operands[1]), FLAGS_REG);
+  operands[8] = gen_lowpart (QImode, operands[4]);
+  ix86_expand_clear (operands[4]);
+})
+
 ;; Similar, but match zero extend with andsi3.
 
 (define_peephole2
@@ -12331,6 +12471,35 @@
   operands[6] = gen_lowpart (QImode, operands[3]);
   ix86_expand_clear (operands[3]);
 })
+
+(define_peephole2
+  [(set (reg FLAGS_REG) (match_operand 0))
+   (parallel [(set (reg FLAGS_REG) (match_operand 1))
+	      (match_operand 5)])
+   (set (match_operand:QI 2 "register_operand")
+	(match_operator:QI 3 "ix86_comparison_operator"
+	  [(reg FLAGS_REG) (const_int 0)]))
+   (parallel [(set (match_operand 4 "any_QIreg_operand")
+		   (zero_extend (match_dup 2)))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "(peep2_reg_dead_p (4, operands[2])
+    || operands_match_p (operands[2], operands[4]))
+   && ! reg_overlap_mentioned_p (operands[4], operands[0])
+   && ! reg_overlap_mentioned_p (operands[4], operands[1])
+   && ! reg_set_p (operands[4], operands[5])
+   && refers_to_regno_p (FLAGS_REG, operands[1], (rtx *)NULL)
+   && peep2_regno_dead_p (0, FLAGS_REG)"
+  [(set (match_dup 6) (match_dup 0))
+   (parallel [(set (match_dup 7) (match_dup 1))
+	      (match_dup 5)])
+   (set (strict_low_part (match_dup 8))
+	(match_dup 3))]
+{
+  operands[6] = gen_rtx_REG (GET_MODE (operands[0]), FLAGS_REG);
+  operands[7] = gen_rtx_REG (GET_MODE (operands[1]), FLAGS_REG);
+  operands[8] = gen_lowpart (QImode, operands[4]);
+  ix86_expand_clear (operands[4]);
+})
 
 ;; Call instructions.
 
@@ -18135,6 +18304,28 @@
   "* return output_probe_stack_range (operands[0], operands[2]);"
   [(set_attr "type" "multi")])
 
+/* Additional processing for builtin_setjmp.  Store the shadow stack pointer
+   as a forth element in jmpbuf.  */
+(define_expand "builtin_setjmp_setup"
+  [(match_operand 0 "address_operand")]
+  "TARGET_SHSTK"
+{
+  if (flag_cf_protection & CF_RETURN)
+    {
+      rtx mem, reg_ssp;
+
+      mem = gen_rtx_MEM (Pmode, plus_constant (Pmode, operands[0],
+					       3 * GET_MODE_SIZE (Pmode)));
+      reg_ssp = gen_reg_rtx (Pmode);
+      emit_insn (gen_rtx_SET (reg_ssp, const0_rtx));
+      emit_insn ((Pmode == SImode)
+		  ? gen_rdsspsi (reg_ssp, reg_ssp)
+		  : gen_rdsspdi (reg_ssp, reg_ssp));
+      emit_move_insn (mem, reg_ssp);
+    }
+  DONE;
+})
+
 (define_expand "builtin_setjmp_receiver"
   [(label_ref (match_operand 0))]
   "!TARGET_64BIT && flag_pic"
@@ -18155,6 +18346,83 @@
   DONE;
 })
 
+(define_expand "builtin_longjmp"
+  [(match_operand 0 "address_operand")]
+  "TARGET_SHSTK"
+{
+  rtx fp, lab, stack;
+  rtx jump, label, reg_adj, reg_ssp, reg_minus, mem_buf, tmp, clob;
+  machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
+
+  /* Adjust the shadow stack pointer (ssp) to the value saved in the
+     jmp_buf.  The saving was done in the builtin_setjmp_setup.  */
+  if (flag_cf_protection & CF_RETURN)
+    {
+      /* Get current shadow stack pointer.  The code below will check if
+	 SHSTK feature is enabled.  If it's not enabled RDSSP instruction
+	 is a NOP.  */
+      reg_ssp = gen_reg_rtx (Pmode);
+      emit_insn (gen_rtx_SET (reg_ssp, const0_rtx));
+      emit_insn ((Pmode == SImode)
+		 ? gen_rdsspsi (reg_ssp, reg_ssp)
+		 : gen_rdsspdi (reg_ssp, reg_ssp));
+      mem_buf = gen_rtx_MEM (Pmode, plus_constant (Pmode, operands[0],
+						   3 * GET_MODE_SIZE (Pmode))),
+
+      /* Compare through substraction the saved and the current ssp to decide
+	 if ssp has to be adjusted.  */
+      reg_minus = gen_reg_rtx (Pmode);
+      tmp = gen_rtx_SET (reg_minus, gen_rtx_MINUS (Pmode, reg_ssp, mem_buf));
+      clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
+      tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
+      emit_insn (tmp);
+
+      /* Jump over adjustment code.  */
+      label = gen_label_rtx ();
+      tmp = gen_rtx_REG (CCmode, FLAGS_REG);
+      tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
+      tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
+				  gen_rtx_LABEL_REF (VOIDmode, label),
+				  pc_rtx);
+      jump = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
+      JUMP_LABEL (jump) = label;
+
+      /* Adjust the ssp.  */
+      reg_adj = gen_reg_rtx (Pmode);
+      tmp = gen_rtx_SET (reg_adj,
+			 gen_rtx_LSHIFTRT (Pmode, negate_rtx (Pmode, reg_minus),
+					   GEN_INT (3)));
+      clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
+      tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
+      emit_insn (tmp);
+      emit_insn ((Pmode == SImode)
+		 ? gen_incsspsi (reg_adj)
+		 : gen_incsspdi (reg_adj));
+
+      emit_label (label);
+      LABEL_NUSES (label) = 1;
+    }
+
+  /* This code is the same as in expand_buildin_longjmp.  */
+  fp = gen_rtx_MEM (Pmode, operands[0]);
+  lab = gen_rtx_MEM (Pmode, plus_constant (Pmode, operands[0],
+					   GET_MODE_SIZE (Pmode)));
+  stack = gen_rtx_MEM (sa_mode, plus_constant (Pmode, operands[0],
+					       2 * GET_MODE_SIZE (Pmode)));
+  lab = copy_to_reg (lab);
+
+  emit_clobber (gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode)));
+  emit_clobber (gen_rtx_MEM (BLKmode, hard_frame_pointer_rtx));
+
+  emit_move_insn (hard_frame_pointer_rtx, fp);
+  emit_stack_restore (SAVE_NONLOCAL, stack);
+
+  emit_use (hard_frame_pointer_rtx);
+  emit_use (stack_pointer_rtx);
+  emit_indirect_jump (lab);
+})
+
+
 ;; Avoid redundant prefixes by splitting HImode arithmetic to SImode.
 ;; Do not split instructions with mask registers.
 (define_split
@@ -18814,7 +19082,7 @@
 	      (clobber (mem:BLK (scratch)))])]
   "(TARGET_SINGLE_PUSH || optimize_insn_for_size_p ())
    && INTVAL (operands[0]) == -GET_MODE_SIZE (word_mode)
-   && !ix86_using_red_zone ()"
+   && ix86_red_zone_size == 0"
   [(clobber (match_dup 1))
    (parallel [(set (mem:W (pre_dec:P (reg:P SP_REG))) (match_dup 1))
 	      (clobber (mem:BLK (scratch)))])])
@@ -18828,7 +19096,7 @@
 	      (clobber (mem:BLK (scratch)))])]
   "(TARGET_DOUBLE_PUSH || optimize_insn_for_size_p ())
    && INTVAL (operands[0]) == -2*GET_MODE_SIZE (word_mode)
-   && !ix86_using_red_zone ()"
+   && ix86_red_zone_size == 0"
   [(clobber (match_dup 1))
    (set (mem:W (pre_dec:P (reg:P SP_REG))) (match_dup 1))
    (parallel [(set (mem:W (pre_dec:P (reg:P SP_REG))) (match_dup 1))
@@ -18843,7 +19111,7 @@
 	      (clobber (reg:CC FLAGS_REG))])]
   "(TARGET_SINGLE_PUSH || optimize_insn_for_size_p ())
    && INTVAL (operands[0]) == -GET_MODE_SIZE (word_mode)
-   && !ix86_using_red_zone ()"
+   && ix86_red_zone_size == 0"
   [(clobber (match_dup 1))
    (set (mem:W (pre_dec:P (reg:P SP_REG))) (match_dup 1))])
 
@@ -18855,7 +19123,7 @@
 	      (clobber (reg:CC FLAGS_REG))])]
   "(TARGET_DOUBLE_PUSH || optimize_insn_for_size_p ())
    && INTVAL (operands[0]) == -2*GET_MODE_SIZE (word_mode)
-   && !ix86_using_red_zone ()"
+   && ix86_red_zone_size == 0"
   [(clobber (match_dup 1))
    (set (mem:W (pre_dec:P (reg:P SP_REG))) (match_dup 1))
    (set (mem:W (pre_dec:P (reg:P SP_REG))) (match_dup 1))])
@@ -19775,6 +20043,83 @@
   [(set_attr "length" "2")
    (set_attr "memory" "unknown")])
 
+;; CET instructions
+(define_insn "rdssp<mode>"
+  [(set (match_operand:SWI48x 0 "register_operand" "=r")
+	(unspec_volatile:SWI48x
+	  [(match_operand:SWI48x 1 "register_operand" "0")]
+	  UNSPECV_NOP_RDSSP))]
+  "TARGET_SHSTK"
+  "rdssp<mskmodesuffix>\t%0"
+  [(set_attr "length" "4")
+   (set_attr "type" "other")])
+
+(define_insn "incssp<mode>"
+  [(unspec_volatile [(match_operand:SWI48x 0 "register_operand" "r")]
+		   UNSPECV_INCSSP)]
+  "TARGET_SHSTK"
+  "incssp<mskmodesuffix>\t%0"
+  [(set_attr "length" "4")
+   (set_attr "type" "other")])
+
+(define_insn "saveprevssp"
+  [(unspec_volatile [(const_int 0)] UNSPECV_SAVEPREVSSP)]
+  "TARGET_SHSTK"
+  "saveprevssp"
+  [(set_attr "length" "5")
+   (set_attr "type" "other")])
+
+(define_insn "rstorssp"
+  [(unspec_volatile [(match_operand 0 "memory_operand" "m")]
+		   UNSPECV_RSTORSSP)]
+  "TARGET_SHSTK"
+  "rstorssp\t%0"
+  [(set_attr "length" "5")
+   (set_attr "type" "other")])
+
+(define_insn "wrss<mode>"
+  [(unspec_volatile [(match_operand:SWI48x 0 "register_operand" "r")
+		     (match_operand:SWI48x 1 "memory_operand" "m")]
+		   UNSPECV_WRSS)]
+  "TARGET_SHSTK"
+  "wrss<mskmodesuffix>\t%0, %1"
+  [(set_attr "length" "3")
+   (set_attr "type" "other")])
+
+(define_insn "wruss<mode>"
+  [(unspec_volatile [(match_operand:SWI48x 0 "register_operand" "r")
+		     (match_operand:SWI48x 1 "memory_operand" "m")]
+		   UNSPECV_WRUSS)]
+  "TARGET_SHSTK"
+  "wruss<mskmodesuffix>\t%0, %1"
+  [(set_attr "length" "4")
+   (set_attr "type" "other")])
+
+(define_insn "setssbsy"
+  [(unspec_volatile [(const_int 0)] UNSPECV_SETSSBSY)]
+  "TARGET_SHSTK"
+  "setssbsy"
+  [(set_attr "length" "4")
+   (set_attr "type" "other")])
+
+(define_insn "clrssbsy"
+  [(unspec_volatile [(match_operand 0 "memory_operand" "m")]
+		   UNSPECV_CLRSSBSY)]
+  "TARGET_SHSTK"
+  "clrssbsy\t%0"
+  [(set_attr "length" "4")
+   (set_attr "type" "other")])
+
+(define_insn "nop_endbr"
+  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_ENDBR)]
+  "TARGET_IBT"
+  "*
+{ return (TARGET_64BIT)? \"endbr64\" : \"endbr32\"; }"
+  [(set_attr "length" "4")
+   (set_attr "length_immediate" "0")
+   (set_attr "modrm" "0")])
+
+;; For RTM support
 (define_expand "xbegin"
   [(set (match_operand:SI 0 "register_operand")
 	(unspec_volatile:SI [(const_int 0)] UNSPECV_XBEGIN))]
diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
index 9064bf09eb5..7c9dd471686 100644
--- a/gcc/config/i386/i386.opt
+++ b/gcc/config/i386/i386.opt
@@ -753,6 +753,10 @@ mrdpid
 Target Report Mask(ISA_RDPID) Var(ix86_isa_flags2) Save
 Support RDPID built-in functions and code generation.
 
+mgfni
+Target Report Mask(ISA_GFNI) Var(ix86_isa_flags2) Save
+Support GFNI built-in functions and code generation.
+
 mbmi
 Target Report Mask(ISA_BMI) Var(ix86_isa_flags) Save
 Support BMI built-in functions and code generation.
@@ -953,3 +957,23 @@ Attempt to avoid generating instruction sequences containing ret bytes.
 mgeneral-regs-only
 Target Report RejectNegative Mask(GENERAL_REGS_ONLY) Var(ix86_target_flags) Save
 Generate code which uses only the general registers.
+
+mcet
+Target Report Var(flag_cet) Init(0)
+Support Control-flow  Enforcment Technology (CET) built-in functions
+and code generation.
+
+mibt
+Target Report Mask(ISA_IBT) Var(ix86_isa_flags2) Save
+Specifically enables an indirect branch tracking feature from Control-flow
+Enforcment Technology (CET).
+
+mshstk
+Target Report Mask(ISA_SHSTK) Var(ix86_isa_flags2) Save
+Specifically enables an shadow stack support feature from Control-flow
+Enforcment Technology (CET).
+
+mcet-switch
+Target Report Undocumented Var(flag_cet_switch) Init(0)
+Turn on CET instrumentation for switch statements, which use jump table and
+indirect jump.
diff --git a/gcc/config/i386/immintrin.h b/gcc/config/i386/immintrin.h
index b52f58efa40..365d2db7dd0 100644
--- a/gcc/config/i386/immintrin.h
+++ b/gcc/config/i386/immintrin.h
@@ -90,6 +90,10 @@
 
 #include <xtestintrin.h>
 
+#include <cetintrin.h>
+
+#include <gfniintrin.h>
+
 #ifndef __RDRND__
 #pragma GCC push_options
 #pragma GCC target("rdrnd")
diff --git a/gcc/config/i386/linux-common.h b/gcc/config/i386/linux-common.h
index 6380639b204..6613807180e 100644
--- a/gcc/config/i386/linux-common.h
+++ b/gcc/config/i386/linux-common.h
@@ -121,3 +121,8 @@ along with GCC; see the file COPYING3.  If not see
 #define CHKP_SPEC "\
 %{!nostdlib:%{!nodefaultlibs:" LIBMPX_SPEC LIBMPXWRAPPERS_SPEC "}}" MPX_SPEC
 #endif
+
+extern void file_end_indicate_exec_stack_and_cet (void);
+
+#undef TARGET_ASM_FILE_END
+#define TARGET_ASM_FILE_END file_end_indicate_exec_stack_and_cet
diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index 0917fad15d4..c3f442eb8ac 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -366,6 +366,31 @@
     }
 })
 
+;; Return true if VALUE is a constant integer whose value is
+;; x86_64_immediate_operand value zero extended from word mode to mode.
+(define_predicate "x86_64_dwzext_immediate_operand"
+  (match_code "const_int,const_wide_int")
+{
+  switch (GET_CODE (op))
+    {
+    case CONST_INT:
+      if (!TARGET_64BIT)
+	return UINTVAL (op) <= HOST_WIDE_INT_UC (0xffffffff);
+      return UINTVAL (op) <= HOST_WIDE_INT_UC (0x7fffffff);
+
+    case CONST_WIDE_INT:
+      if (!TARGET_64BIT)
+	return false;
+      return (CONST_WIDE_INT_NUNITS (op) == 2
+	      && CONST_WIDE_INT_ELT (op, 1) == 0
+	      && (trunc_int_for_mode (CONST_WIDE_INT_ELT (op, 0), SImode)
+		  == (HOST_WIDE_INT) CONST_WIDE_INT_ELT (op, 0)));
+
+    default:
+      gcc_unreachable ();
+    }
+})
+
 ;; Return true if size of VALUE can be stored in a sign
 ;; extended immediate field.
 (define_predicate "x86_64_immediate_size_operand"
@@ -979,9 +1004,9 @@
   (match_code "mem")
 {
   unsigned n_elts;
-  op = maybe_get_pool_constant (op);
+  op = avoid_constant_pool_reference (op);
 
-  if (!(op && GET_CODE (op) == CONST_VECTOR))
+  if (GET_CODE (op) != CONST_VECTOR)
     return false;
 
   n_elts = CONST_VECTOR_NUNITS (op);
@@ -1276,7 +1301,7 @@
   machine_mode inmode = GET_MODE (XEXP (op, 0));
   enum rtx_code code = GET_CODE (op);
 
-  if (inmode == CCFPmode || inmode == CCFPUmode)
+  if (inmode == CCFPmode)
     {
       if (!ix86_trivial_fp_comparison_operator (op, mode))
 	return false;
@@ -1286,8 +1311,7 @@
   switch (code)
     {
     case LTU: case GTU: case LEU: case GEU:
-      if (inmode == CCmode || inmode == CCFPmode || inmode == CCFPUmode
-	  || inmode == CCCmode)
+      if (inmode == CCmode || inmode == CCFPmode || inmode == CCCmode)
 	return true;
       return false;
     case ORDERED: case UNORDERED:
@@ -1323,20 +1347,26 @@
   machine_mode inmode = GET_MODE (XEXP (op, 0));
   enum rtx_code code = GET_CODE (op);
 
-  if (inmode == CCFPmode || inmode == CCFPUmode)
+  if (inmode == CCFPmode)
     return ix86_trivial_fp_comparison_operator (op, mode);
 
   switch (code)
     {
     case EQ: case NE:
+      if (inmode == CCGZmode)
+	return false;
       return true;
-    case LT: case GE:
+    case GE: case LT:
       if (inmode == CCmode || inmode == CCGCmode
-	  || inmode == CCGOCmode || inmode == CCNOmode)
+	  || inmode == CCGOCmode || inmode == CCNOmode || inmode == CCGZmode)
 	return true;
       return false;
-    case LTU: case GTU: case LEU: case GEU:
-      if (inmode == CCmode || inmode == CCCmode)
+    case GEU: case LTU:
+      if (inmode == CCGZmode)
+	return true;
+      /* FALLTHRU */
+    case GTU: case LEU:
+      if (inmode == CCmode || inmode == CCCmode || inmode == CCGZmode)
 	return true;
       return false;
     case ORDERED: case UNORDERED:
@@ -1360,7 +1390,7 @@
   machine_mode inmode = GET_MODE (XEXP (op, 0));
   enum rtx_code code = GET_CODE (op);
 
-  if (inmode == CCFPmode || inmode == CCFPUmode)
+  if (inmode == CCFPmode)
     {
       if (!ix86_trivial_fp_comparison_operator (op, mode))
 	return false;
diff --git a/gcc/config/i386/sol2.h b/gcc/config/i386/sol2.h
index 61733603fa2..05e5e1a4949 100644
--- a/gcc/config/i386/sol2.h
+++ b/gcc/config/i386/sol2.h
@@ -65,8 +65,16 @@ along with GCC; see the file COPYING3.  If not see
 #define ASM_CPU64_DEFAULT_SPEC "-xarch=generic64"
 #endif
 
+/* Since Studio 12.6, as needs -xbrace_comment=no so its AVX512 syntax is
+   fully compatible with gas.  */
+#ifdef HAVE_AS_XBRACE_COMMENT_OPTION
+#define ASM_XBRACE_COMMENT_SPEC "-xbrace_comment=no"
+#else
+#define ASM_XBRACE_COMMENT_SPEC ""
+#endif
+
 #undef ASM_CPU_SPEC
-#define ASM_CPU_SPEC "%(asm_cpu_default)"
+#define ASM_CPU_SPEC "%(asm_cpu_default) " ASM_XBRACE_COMMENT_SPEC
 
 /* Don't include ASM_PIC_SPEC.  While the Solaris 10+ assembler accepts -K PIC,
    it gives many warnings: 
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index d5e2ec00237..4dfb2f8d3b3 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -83,9 +83,7 @@
   UNSPEC_VSIBADDR
 
   ;; For AVX512F support
-  UNSPEC_VPERMI2
   UNSPEC_VPERMT2
-  UNSPEC_VPERMI2_MASK
   UNSPEC_UNSIGNED_FIX_NOTRUNC
   UNSPEC_UNSIGNED_PCMP
   UNSPEC_TESTM
@@ -157,6 +155,9 @@
   UNSPEC_VP4FNMADD
   UNSPEC_VP4DPWSSD
   UNSPEC_VP4DPWSSDS
+
+  ;; For GFNI support
+  UNSPEC_GF2P8AFFINEINV
 ])
 
 (define_c_enum "unspecv" [
@@ -325,6 +326,9 @@
 (define_mode_iterator VI1_AVX512
   [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI])
 
+(define_mode_iterator VI1_AVX512F
+  [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI])
+
 (define_mode_iterator VI2_AVX2
   [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
 
@@ -371,10 +375,17 @@
   [V16SF V16SI])
 
 ;; ??? We should probably use TImode instead.
-(define_mode_iterator VIMAX_AVX2
+(define_mode_iterator VIMAX_AVX2_AVX512BW
   [(V4TI "TARGET_AVX512BW") (V2TI "TARGET_AVX2") V1TI])
 
-;; ??? This should probably be dropped in favor of VIMAX_AVX2.
+;; Suppose TARGET_AVX512BW as baseline
+(define_mode_iterator VIMAX_AVX512VL
+  [V4TI (V2TI "TARGET_AVX512VL") (V1TI "TARGET_AVX512VL")])
+
+(define_mode_iterator VIMAX_AVX2
+  [(V2TI "TARGET_AVX2") V1TI])
+
+;; ??? This should probably be dropped in favor of VIMAX_AVX2_AVX512BW.
 (define_mode_iterator SSESCALARMODE
   [(V4TI "TARGET_AVX512BW") (V2TI "TARGET_AVX2") TI])
 
@@ -403,11 +414,19 @@
   [(V8SI "TARGET_AVX2") V4SI
    (V4DI "TARGET_AVX2") V2DI])
 
+(define_mode_iterator VI248_AVX2
+  [(V16HI "TARGET_AVX2") V8HI
+   (V8SI "TARGET_AVX2") V4SI
+   (V4DI "TARGET_AVX2") V2DI])
+
 (define_mode_iterator VI248_AVX2_8_AVX512F_24_AVX512BW
   [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
    (V16SI "TARGET_AVX512BW") (V8SI "TARGET_AVX2") V4SI
    (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
 
+(define_mode_iterator VI248_AVX512BW
+  [(V32HI "TARGET_AVX512BW") V16SI V8DI])
+
 (define_mode_iterator VI248_AVX512BW_AVX512VL
   [(V32HI "TARGET_AVX512BW") 
    (V4DI "TARGET_AVX512VL") V16SI V8DI])
@@ -418,6 +437,11 @@
   V8SI V4SI
   V2DI])
    
+(define_mode_iterator VI248_AVX512BW_2
+ [(V16HI "TARGET_AVX512BW") (V8HI "TARGET_AVX512BW")
+  V8SI V4SI
+  V4DI V2DI])
+   
 (define_mode_iterator VI48_AVX512F
   [(V16SI "TARGET_AVX512F") V8SI V4SI
    (V8DI "TARGET_AVX512F") V4DI V2DI])
@@ -2522,7 +2546,7 @@
    (set_attr "prefix" "evex")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "reduces<mode>"
+(define_insn "reduces<mode><mask_scalar_name>"
   [(set (match_operand:VF_128 0 "register_operand" "=v")
 	(vec_merge:VF_128
 	  (unspec:VF_128
@@ -2533,7 +2557,7 @@
 	  (match_dup 1)
 	  (const_int 1)))]
   "TARGET_AVX512DQ"
-  "vreduce<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+  "vreduce<ssescalarmodesuffix>\t{%3, %2, %1, %0<mask_scalar_operand4>|%0<mask_scalar_operand4>, %1, %2, %3}"
   [(set_attr "type" "sse")
    (set_attr "prefix" "evex")
    (set_attr "mode" "<MODE>")])
@@ -2737,7 +2761,7 @@
    (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_insn "<sse>_comi<round_saeonly_name>"
+(define_insn "<sse>_<unord>comi<round_saeonly_name>"
   [(set (reg:CCFP FLAGS_REG)
 	(compare:CCFP
 	  (vec_select:MODEF
@@ -2747,27 +2771,7 @@
 	    (match_operand:<ssevecmode> 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
 	    (parallel [(const_int 0)]))))]
   "SSE_FLOAT_MODE_P (<MODE>mode)"
-  "%vcomi<ssemodesuffix>\t{<round_saeonly_op2>%1, %0|%0, %<iptr>1<round_saeonly_op2>}"
-  [(set_attr "type" "ssecomi")
-   (set_attr "prefix" "maybe_vex")
-   (set_attr "prefix_rep" "0")
-   (set (attr "prefix_data16")
-	(if_then_else (eq_attr "mode" "DF")
-		      (const_string "1")
-		      (const_string "0")))
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "<sse>_ucomi<round_saeonly_name>"
-  [(set (reg:CCFPU FLAGS_REG)
-	(compare:CCFPU
-	  (vec_select:MODEF
-	    (match_operand:<ssevecmode> 0 "register_operand" "v")
-	    (parallel [(const_int 0)]))
-	  (vec_select:MODEF
-	    (match_operand:<ssevecmode> 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
-	    (parallel [(const_int 0)]))))]
-  "SSE_FLOAT_MODE_P (<MODE>mode)"
-  "%vucomi<ssemodesuffix>\t{<round_saeonly_op2>%1, %0|%0, %<iptr>1<round_saeonly_op2>}"
+  "%v<unord>comi<ssemodesuffix>\t{<round_saeonly_op2>%1, %0|%0, %<iptr>1<round_saeonly_op2>}"
   [(set_attr "type" "ssecomi")
    (set_attr "prefix" "maybe_vex")
    (set_attr "prefix_rep" "0")
@@ -3700,8 +3704,7 @@
   "@
    vfmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
    vfmadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
-  [(set_attr "isa" "fma_avx512f,fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "<avx512>_fmadd_<mode>_mask3<round_name>"
@@ -3715,8 +3718,7 @@
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
   "TARGET_AVX512F"
   "vfmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
-  [(set_attr "isa" "fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*fma_fmsub_<mode>"
@@ -3766,8 +3768,7 @@
   "@
    vfmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
    vfmsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
-  [(set_attr "isa" "fma_avx512f,fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "<avx512>_fmsub_<mode>_mask3<round_name>"
@@ -3782,8 +3783,7 @@
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
   "TARGET_AVX512F && <round_mode512bit_condition>"
   "vfmsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
-  [(set_attr "isa" "fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*fma_fnmadd_<mode>"
@@ -3833,8 +3833,7 @@
   "@
    vfnmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
    vfnmadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
-  [(set_attr "isa" "fma_avx512f,fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "<avx512>_fnmadd_<mode>_mask3<round_name>"
@@ -3849,8 +3848,7 @@
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
   "TARGET_AVX512F && <round_mode512bit_condition>"
   "vfnmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
-  [(set_attr "isa" "fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*fma_fnmsub_<mode>"
@@ -3903,8 +3901,7 @@
   "@
    vfnmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
    vfnmsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
-  [(set_attr "isa" "fma_avx512f,fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "<avx512>_fnmsub_<mode>_mask3<round_name>"
@@ -3920,8 +3917,7 @@
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
   "TARGET_AVX512F"
   "vfnmsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
-  [(set_attr "isa" "fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 ;; FMA parallel floating point multiply addsub and subadd operations.
@@ -4005,8 +4001,7 @@
   "@
    vfmaddsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
    vfmaddsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
-  [(set_attr "isa" "fma_avx512f,fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "<avx512>_fmaddsub_<mode>_mask3<round_name>"
@@ -4021,8 +4016,7 @@
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
   "TARGET_AVX512F"
   "vfmaddsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
-  [(set_attr "isa" "fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*fma_fmsubadd_<mode>"
@@ -4075,8 +4069,7 @@
   "@
    vfmsubadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
    vfmsubadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
-  [(set_attr "isa" "fma_avx512f,fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "<avx512>_fmsubadd_<mode>_mask3<round_name>"
@@ -4092,8 +4085,7 @@
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
   "TARGET_AVX512F"
   "vfmsubadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
-  [(set_attr "isa" "fma_avx512f")
-   (set_attr "type" "ssemuladd")
+  [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
 ;; FMA3 floating point scalar intrinsics. These merge result with
@@ -10168,8 +10160,7 @@
                          (const_int 12) (const_int 14)])))))]
   "TARGET_AVX512F && ix86_binary_operator_ok (MULT, V16SImode, operands)"
   "vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
-  [(set_attr "isa" "avx512f")
-   (set_attr "type" "sseimul")
+  [(set_attr "type" "sseimul")
    (set_attr "prefix_extra" "1")
    (set_attr "prefix" "evex")
    (set_attr "mode" "XI")])
@@ -10285,8 +10276,7 @@
                          (const_int 12) (const_int 14)])))))]
   "TARGET_AVX512F && ix86_binary_operator_ok (MULT, V16SImode, operands)"
   "vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
-  [(set_attr "isa" "avx512f")
-   (set_attr "type" "sseimul")
+  [(set_attr "type" "sseimul")
    (set_attr "prefix_extra" "1")
    (set_attr "prefix" "evex")
    (set_attr "mode" "XI")])
@@ -10731,65 +10721,57 @@
        (const_string "0")))
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_insn "<shift_insn><mode>3<mask_name>"
-  [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,v")
-	(any_lshift:VI2_AVX2_AVX512BW
-	  (match_operand:VI2_AVX2_AVX512BW 1 "register_operand" "0,v")
-	  (match_operand:DI 2 "nonmemory_operand" "xN,vN")))]
-  "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
-  "@
-   p<vshift><ssemodesuffix>\t{%2, %0|%0, %2}
-   vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
-  [(set_attr "isa" "noavx,avx")
-   (set_attr "type" "sseishft")
+(define_insn "<mask_codefor><shift_insn><mode>3<mask_name>"
+  [(set (match_operand:VI248_AVX512BW_2 0 "register_operand" "=v,v")
+	(any_lshift:VI248_AVX512BW_2
+	  (match_operand:VI248_AVX512BW_2 1 "nonimmediate_operand" "v,vm")
+	  (match_operand:DI 2 "nonmemory_operand" "v,N")))]
+  "TARGET_AVX512VL"
+  "vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
+  [(set_attr "type" "sseishft")
    (set (attr "length_immediate")
      (if_then_else (match_operand 2 "const_int_operand")
        (const_string "1")
        (const_string "0")))
-   (set_attr "prefix_data16" "1,*")
-   (set_attr "prefix" "orig,vex")
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_insn "<shift_insn><mode>3<mask_name>"
-  [(set (match_operand:VI48_AVX2 0 "register_operand" "=x,x,v")
-	(any_lshift:VI48_AVX2
-	  (match_operand:VI48_AVX2 1 "register_operand" "0,x,v")
-	  (match_operand:DI 2 "nonmemory_operand" "xN,xN,vN")))]
-  "TARGET_SSE2 && <mask_mode512bit_condition>"
+(define_insn "<shift_insn><mode>3"
+  [(set (match_operand:VI248_AVX2 0 "register_operand" "=x,x")
+	(any_lshift:VI248_AVX2
+	  (match_operand:VI248_AVX2 1 "register_operand" "0,x")
+	  (match_operand:DI 2 "nonmemory_operand" "xN,xN")))]
+  "TARGET_SSE2"
   "@
    p<vshift><ssemodesuffix>\t{%2, %0|%0, %2}
-   vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
-   vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"  
-  [(set_attr "isa" "noavx,avx,avx512bw")
+   vp<vshift><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+  [(set_attr "isa" "noavx,avx")
    (set_attr "type" "sseishft")
    (set (attr "length_immediate")
      (if_then_else (match_operand 2 "const_int_operand")
        (const_string "1")
        (const_string "0")))
-   (set_attr "prefix_data16" "1,*,*")
-   (set_attr "prefix" "orig,vex,evex")
+   (set_attr "prefix_data16" "1,*")
+   (set_attr "prefix" "orig,vex")
    (set_attr "mode" "<sseinsnmode>")])
 
 (define_insn "<shift_insn><mode>3<mask_name>"
-  [(set (match_operand:VI48_512 0 "register_operand" "=v,v")
-	(any_lshift:VI48_512
-	  (match_operand:VI48_512 1 "nonimmediate_operand" "v,m")
+  [(set (match_operand:VI248_AVX512BW 0 "register_operand" "=v,v")
+	(any_lshift:VI248_AVX512BW
+	  (match_operand:VI248_AVX512BW 1 "nonimmediate_operand" "v,m")
 	  (match_operand:DI 2 "nonmemory_operand" "vN,N")))]
-  "TARGET_AVX512F && <mask_mode512bit_condition>"
+  "TARGET_AVX512F"
   "vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
-  [(set_attr "isa" "avx512f")
-   (set_attr "type" "sseishft")
+  [(set_attr "type" "sseishft")
    (set (attr "length_immediate")
      (if_then_else (match_operand 2 "const_int_operand")
        (const_string "1")
        (const_string "0")))
-   (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
 
 
-(define_expand "vec_shl_<mode>"
+(define_expand "vec_shr_<mode>"
   [(set (match_dup 3)
-	(ashift:V1TI
+	(lshiftrt:V1TI
 	 (match_operand:VI_128 1 "register_operand")
 	 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
    (set (match_operand:VI_128 0 "register_operand") (match_dup 4))]
@@ -10800,48 +10782,24 @@
   operands[4] = gen_lowpart (<MODE>mode, operands[3]);
 })
 
-(define_insn "<sse2_avx2>_ashl<mode>3"
-  [(set (match_operand:VIMAX_AVX2 0 "register_operand" "=x,v")
-	(ashift:VIMAX_AVX2
-	 (match_operand:VIMAX_AVX2 1 "register_operand" "0,v")
-	 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n,n")))]
-  "TARGET_SSE2"
+(define_insn "avx512bw_<shift_insn><mode>3"
+  [(set (match_operand:VIMAX_AVX512VL 0 "register_operand" "=v")
+	(any_lshift:VIMAX_AVX512VL
+	 (match_operand:VIMAX_AVX512VL 1 "nonimmediate_operand" "vm")
+	 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
+  "TARGET_AVX512BW"
 {
   operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
-
-  switch (which_alternative)
-    {
-    case 0:
-      return "pslldq\t{%2, %0|%0, %2}";
-    case 1:
-      return "vpslldq\t{%2, %1, %0|%0, %1, %2}";
-    default:
-      gcc_unreachable ();
-    }
+  return "vp<vshift>dq\t{%2, %1, %0|%0, %1, %2}";
 }
-  [(set_attr "isa" "noavx,avx")
-   (set_attr "type" "sseishft")
+  [(set_attr "type" "sseishft")
    (set_attr "length_immediate" "1")
-   (set_attr "prefix_data16" "1,*")
-   (set_attr "prefix" "orig,vex")
+   (set_attr "prefix" "maybe_evex")
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_expand "vec_shr_<mode>"
-  [(set (match_dup 3)
-	(lshiftrt:V1TI
-	 (match_operand:VI_128 1 "register_operand")
-	 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
-   (set (match_operand:VI_128 0 "register_operand") (match_dup 4))]
-  "TARGET_SSE2"
-{
-  operands[1] = gen_lowpart (V1TImode, operands[1]);
-  operands[3] = gen_reg_rtx (V1TImode);
-  operands[4] = gen_lowpart (<MODE>mode, operands[3]);
-})
-
-(define_insn "<sse2_avx2>_lshr<mode>3"
+(define_insn "<sse2_avx2>_<shift_insn><mode>3"
   [(set (match_operand:VIMAX_AVX2 0 "register_operand" "=x,v")
-	(lshiftrt:VIMAX_AVX2
+	(any_lshift:VIMAX_AVX2
 	 (match_operand:VIMAX_AVX2 1 "register_operand" "0,v")
 	 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n,n")))]
   "TARGET_SSE2"
@@ -10851,9 +10809,9 @@
   switch (which_alternative)
     {
     case 0:
-      return "psrldq\t{%2, %0|%0, %2}";
+      return "p<vshift>dq\t{%2, %0|%0, %2}";
     case 1:
-      return "vpsrldq\t{%2, %1, %0|%0, %1, %2}";
+      return "vp<vshift>dq\t{%2, %1, %0|%0, %1, %2}";
     default:
       gcc_unreachable ();
     }
@@ -11562,10 +11520,10 @@
   "TARGET_AVX512BW")
 
 (define_insn "*andnot<mode>3"
-  [(set (match_operand:VI 0 "register_operand" "=x,v")
+  [(set (match_operand:VI 0 "register_operand" "=x,x,v")
 	(and:VI
-	  (not:VI (match_operand:VI 1 "register_operand" "0,v"))
-	  (match_operand:VI 2 "vector_operand" "xBm,vm")))]
+	  (not:VI (match_operand:VI 1 "register_operand" "0,x,v"))
+	  (match_operand:VI 2 "vector_operand" "xBm,xm,vm")))]
   "TARGET_SSE"
 {
   static char buf[64];
@@ -11600,10 +11558,11 @@
 	case E_V4DImode:
 	case E_V4SImode:
 	case E_V2DImode:
-	  ssesuffix = TARGET_AVX512VL ? "<ssemodesuffix>" : "";
+	  ssesuffix = (TARGET_AVX512VL && which_alternative == 2
+		       ? "<ssemodesuffix>" : "");
 	  break;
 	default:
-	  ssesuffix = TARGET_AVX512VL ? "q" : "";
+	  ssesuffix = TARGET_AVX512VL && which_alternative == 2 ? "q" : "";
 	}
       break;
 
@@ -11629,6 +11588,7 @@
       ops = "%s%s\t{%%2, %%0|%%0, %%2}";
       break;
     case 1:
+    case 2:
       ops = "v%s%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
       break;
     default:
@@ -11638,7 +11598,7 @@
   snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
   return buf;
 }
-  [(set_attr "isa" "noavx,avx")
+  [(set_attr "isa" "noavx,avx,avx")
    (set_attr "type" "sselog")
    (set (attr "prefix_data16")
      (if_then_else
@@ -11646,7 +11606,7 @@
 	    (eq_attr "mode" "TI"))
        (const_string "1")
        (const_string "*")))
-   (set_attr "prefix" "orig,vex")
+   (set_attr "prefix" "orig,vex,evex")
    (set (attr "mode")
 	(cond [(and (match_test "<MODE_SIZE> == 16")
 		    (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
@@ -11691,10 +11651,10 @@
 })
 
 (define_insn "<mask_codefor><code><mode>3<mask_name>"
-  [(set (match_operand:VI48_AVX_AVX512F 0 "register_operand" "=x,v")
+  [(set (match_operand:VI48_AVX_AVX512F 0 "register_operand" "=x,x,v")
 	(any_logic:VI48_AVX_AVX512F
-	  (match_operand:VI48_AVX_AVX512F 1 "vector_operand" "%0,v")
-	  (match_operand:VI48_AVX_AVX512F 2 "vector_operand" "xBm,vm")))]
+	  (match_operand:VI48_AVX_AVX512F 1 "vector_operand" "%0,x,v")
+	  (match_operand:VI48_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))]
   "TARGET_SSE && <mask_mode512bit_condition>
    && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
 {
@@ -11724,7 +11684,9 @@
 	case E_V4DImode:
 	case E_V4SImode:
 	case E_V2DImode:
-	  ssesuffix = TARGET_AVX512VL ? "<ssemodesuffix>" : "";
+	  ssesuffix = (TARGET_AVX512VL
+		       && (<mask_applied> || which_alternative == 2)
+		       ? "<ssemodesuffix>" : "");
 	  break;
 	default:
 	  gcc_unreachable ();
@@ -11753,6 +11715,7 @@
         ops = "%s%s\t{%%2, %%0|%%0, %%2}";
       break;
     case 1:
+    case 2:
       ops = "v%s%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
       break;
     default:
@@ -11762,7 +11725,7 @@
   snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
   return buf;
 }
-  [(set_attr "isa" "noavx,avx")
+  [(set_attr "isa" "noavx,avx,avx")
    (set_attr "type" "sselog")
    (set (attr "prefix_data16")
      (if_then_else
@@ -11770,7 +11733,7 @@
 	    (eq_attr "mode" "TI"))
        (const_string "1")
        (const_string "*")))
-   (set_attr "prefix" "<mask_prefix3>")
+   (set_attr "prefix" "<mask_prefix3>,evex")
    (set (attr "mode")
 	(cond [(and (match_test "<MODE_SIZE> == 16")
 		    (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
@@ -11789,10 +11752,10 @@
 	      (const_string "<sseinsnmode>")))])
 
 (define_insn "*<code><mode>3"
-  [(set (match_operand:VI12_AVX_AVX512F 0 "register_operand" "=x,v")
+  [(set (match_operand:VI12_AVX_AVX512F 0 "register_operand" "=x,x,v")
 	(any_logic: VI12_AVX_AVX512F
-	  (match_operand:VI12_AVX_AVX512F 1 "vector_operand" "%0,v")
-	  (match_operand:VI12_AVX_AVX512F 2 "vector_operand" "xBm,vm")))]
+	  (match_operand:VI12_AVX_AVX512F 1 "vector_operand" "%0,x,v")
+	  (match_operand:VI12_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))]
   "TARGET_SSE && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
 {
   static char buf[64];
@@ -11821,7 +11784,7 @@
 	case E_V16HImode:
 	case E_V16QImode:
 	case E_V8HImode:
-	  ssesuffix = TARGET_AVX512VL ? "q" : "";
+	  ssesuffix = TARGET_AVX512VL && which_alternative == 2 ? "q" : "";
 	  break;
 	default:
 	  gcc_unreachable ();
@@ -11847,6 +11810,7 @@
       ops = "%s%s\t{%%2, %%0|%%0, %%2}";
       break;
     case 1:
+    case 2:
       ops = "v%s%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
       break;
     default:
@@ -11856,7 +11820,7 @@
   snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
   return buf;
 }
-  [(set_attr "isa" "noavx,avx")
+  [(set_attr "isa" "noavx,avx,avx")
    (set_attr "type" "sselog")
    (set (attr "prefix_data16")
      (if_then_else
@@ -11864,7 +11828,7 @@
 	    (eq_attr "mode" "TI"))
        (const_string "1")
        (const_string "*")))
-   (set_attr "prefix" "<mask_prefix3>")
+   (set_attr "prefix" "<mask_prefix3>,evex")
    (set (attr "mode")
 	(cond [(and (match_test "<MODE_SIZE> == 16")
 		    (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
@@ -18099,96 +18063,48 @@
    (set_attr "prefix" "<mask_prefix>")
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_expand "<avx512>_vpermi2var<mode>3_maskz"
-  [(match_operand:VI48F 0 "register_operand")
-   (match_operand:VI48F 1 "register_operand")
-   (match_operand:<sseintvecmode> 2 "register_operand")
-   (match_operand:VI48F 3 "nonimmediate_operand")
-   (match_operand:<avx512fmaskmode> 4 "register_operand")]
-  "TARGET_AVX512F"
-{
-  emit_insn (gen_<avx512>_vpermi2var<mode>3_maskz_1 (
-	operands[0], operands[1], operands[2], operands[3],
-	CONST0_RTX (<MODE>mode), operands[4]));
-  DONE;
-})
-
-(define_expand "<avx512>_vpermi2var<mode>3_maskz"
-  [(match_operand:VI1_AVX512VL 0 "register_operand")
-   (match_operand:VI1_AVX512VL 1 "register_operand")
-   (match_operand:<sseintvecmode> 2 "register_operand")
-   (match_operand:VI1_AVX512VL 3 "nonimmediate_operand")
-   (match_operand:<avx512fmaskmode> 4 "register_operand")]
-  "TARGET_AVX512VBMI"
-{
-  emit_insn (gen_<avx512>_vpermi2var<mode>3_maskz_1 (
-	operands[0], operands[1], operands[2], operands[3],
-	CONST0_RTX (<MODE>mode), operands[4]));
-  DONE;
-})
-
-(define_expand "<avx512>_vpermi2var<mode>3_maskz"
-  [(match_operand:VI2_AVX512VL 0 "register_operand")
-   (match_operand:VI2_AVX512VL 1 "register_operand")
-   (match_operand:<sseintvecmode> 2 "register_operand")
-   (match_operand:VI2_AVX512VL 3 "nonimmediate_operand")
-   (match_operand:<avx512fmaskmode> 4 "register_operand")]
-  "TARGET_AVX512BW"
-{
-  emit_insn (gen_<avx512>_vpermi2var<mode>3_maskz_1 (
-	operands[0], operands[1], operands[2], operands[3],
-	CONST0_RTX (<MODE>mode), operands[4]));
-  DONE;
-})
-
-(define_insn "<avx512>_vpermi2var<mode>3<sd_maskz_name>"
-  [(set (match_operand:VI48F 0 "register_operand" "=v")
-	(unspec:VI48F
-	  [(match_operand:VI48F 1 "register_operand" "v")
-	   (match_operand:<sseintvecmode> 2 "register_operand" "0")
-	   (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
-	  UNSPEC_VPERMI2))]
+(define_mode_iterator VPERMI2
+  [V16SI V16SF V8DI V8DF
+   (V8SI "TARGET_AVX512VL") (V8SF "TARGET_AVX512VL")
+   (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")
+   (V4SI "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
+   (V2DI "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")
+   (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX512BW && TARGET_AVX512VL")
+   (V8HI "TARGET_AVX512BW && TARGET_AVX512VL")
+   (V64QI "TARGET_AVX512VBMI") (V32QI "TARGET_AVX512VBMI && TARGET_AVX512VL")
+   (V16QI "TARGET_AVX512VBMI && TARGET_AVX512VL")])
+
+(define_mode_iterator VPERMI2I
+  [V16SI V8DI
+   (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
+   (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
+   (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX512BW && TARGET_AVX512VL")
+   (V8HI "TARGET_AVX512BW && TARGET_AVX512VL")
+   (V64QI "TARGET_AVX512VBMI") (V32QI "TARGET_AVX512VBMI && TARGET_AVX512VL")
+   (V16QI "TARGET_AVX512VBMI && TARGET_AVX512VL")])
+
+(define_expand "<avx512>_vpermi2var<mode>3_mask"
+  [(set (match_operand:VPERMI2 0 "register_operand")
+	(vec_merge:VPERMI2
+	  (unspec:VPERMI2
+	    [(match_operand:<sseintvecmode> 2 "register_operand")
+	     (match_operand:VPERMI2 1 "register_operand")
+	     (match_operand:VPERMI2 3 "nonimmediate_operand")]
+	    UNSPEC_VPERMT2)
+	  (match_dup 5)
+	  (match_operand:<avx512fmaskmode> 4 "register_operand")))]
   "TARGET_AVX512F"
-  "vpermi2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "<avx512>_vpermi2var<mode>3<sd_maskz_name>"
-  [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
-	(unspec:VI1_AVX512VL
-	  [(match_operand:VI1_AVX512VL 1 "register_operand" "v")
-	   (match_operand:<sseintvecmode> 2 "register_operand" "0")
-	   (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
-	  UNSPEC_VPERMI2))]
-  "TARGET_AVX512VBMI"
-  "vpermi2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "<avx512>_vpermi2var<mode>3<sd_maskz_name>"
-  [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
-	(unspec:VI2_AVX512VL
-	  [(match_operand:VI2_AVX512VL 1 "register_operand" "v")
-	   (match_operand:<sseintvecmode> 2 "register_operand" "0")
-	   (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
-	  UNSPEC_VPERMI2))]
-  "TARGET_AVX512BW"
-  "vpermi2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "<avx512>_vpermi2var<mode>3_mask"
-  [(set (match_operand:VI48F 0 "register_operand" "=v")
-	(vec_merge:VI48F
-	  (unspec:VI48F
-	    [(match_operand:VI48F 1 "register_operand" "v")
-	    (match_operand:<sseintvecmode> 2 "register_operand" "0")
-	    (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
-	    UNSPEC_VPERMI2_MASK)
-	  (match_dup 0)
+  "operands[5] = gen_lowpart (<MODE>mode, operands[2]);")
+
+(define_insn "*<avx512>_vpermi2var<mode>3_mask"
+  [(set (match_operand:VPERMI2I 0 "register_operand" "=v")
+	(vec_merge:VPERMI2I
+	  (unspec:VPERMI2I
+	    [(match_operand:<sseintvecmode> 2 "register_operand" "0")
+	     (match_operand:VPERMI2I 1 "register_operand" "v")
+	     (match_operand:VPERMI2I 3 "nonimmediate_operand" "vm")]
+	    UNSPEC_VPERMT2)
+	  (match_dup 2)
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
   "TARGET_AVX512F"
   "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
@@ -18196,43 +18112,27 @@
    (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_insn "<avx512>_vpermi2var<mode>3_mask"
-  [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
-	(vec_merge:VI1_AVX512VL
-	  (unspec:VI1_AVX512VL
-	    [(match_operand:VI1_AVX512VL 1 "register_operand" "v")
-	    (match_operand:<sseintvecmode> 2 "register_operand" "0")
-	    (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
-	    UNSPEC_VPERMI2_MASK)
-	  (match_dup 0)
-	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
-  "TARGET_AVX512VBMI"
-  "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "<avx512>_vpermi2var<mode>3_mask"
-  [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
-	(vec_merge:VI2_AVX512VL
-	  (unspec:VI2_AVX512VL
-	    [(match_operand:VI2_AVX512VL 1 "register_operand" "v")
-	    (match_operand:<sseintvecmode> 2 "register_operand" "0")
-	    (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
-	    UNSPEC_VPERMI2_MASK)
-	  (match_dup 0)
+(define_insn "*<avx512>_vpermi2var<mode>3_mask"
+  [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
+	(vec_merge:VF_AVX512VL
+	  (unspec:VF_AVX512VL
+	    [(match_operand:<sseintvecmode> 2 "register_operand" "0")
+	     (match_operand:VF_AVX512VL 1 "register_operand" "v")
+	     (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "vm")]
+	    UNSPEC_VPERMT2)
+	  (subreg:VF_AVX512VL (match_dup 2) 0)
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
-  "TARGET_AVX512BW"
+  "TARGET_AVX512F"
   "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
 
 (define_expand "<avx512>_vpermt2var<mode>3_maskz"
-  [(match_operand:VI48F 0 "register_operand")
+  [(match_operand:VPERMI2 0 "register_operand")
    (match_operand:<sseintvecmode> 1 "register_operand")
-   (match_operand:VI48F 2 "register_operand")
-   (match_operand:VI48F 3 "nonimmediate_operand")
+   (match_operand:VPERMI2 2 "register_operand")
+   (match_operand:VPERMI2 3 "nonimmediate_operand")
    (match_operand:<avx512fmaskmode> 4 "register_operand")]
   "TARGET_AVX512F"
 {
@@ -18242,80 +18142,28 @@
   DONE;
 })
 
-(define_expand "<avx512>_vpermt2var<mode>3_maskz"
-  [(match_operand:VI1_AVX512VL 0 "register_operand")
-   (match_operand:<sseintvecmode> 1 "register_operand")
-   (match_operand:VI1_AVX512VL 2 "register_operand")
-   (match_operand:VI1_AVX512VL 3 "nonimmediate_operand")
-   (match_operand:<avx512fmaskmode> 4 "register_operand")]
-  "TARGET_AVX512VBMI"
-{
-  emit_insn (gen_<avx512>_vpermt2var<mode>3_maskz_1 (
-	operands[0], operands[1], operands[2], operands[3],
-	CONST0_RTX (<MODE>mode), operands[4]));
-  DONE;
-})
-
-(define_expand "<avx512>_vpermt2var<mode>3_maskz"
-  [(match_operand:VI2_AVX512VL 0 "register_operand")
-   (match_operand:<sseintvecmode> 1 "register_operand")
-   (match_operand:VI2_AVX512VL 2 "register_operand")
-   (match_operand:VI2_AVX512VL 3 "nonimmediate_operand")
-   (match_operand:<avx512fmaskmode> 4 "register_operand")]
-  "TARGET_AVX512BW"
-{
-  emit_insn (gen_<avx512>_vpermt2var<mode>3_maskz_1 (
-	operands[0], operands[1], operands[2], operands[3],
-	CONST0_RTX (<MODE>mode), operands[4]));
-  DONE;
-})
-
 (define_insn "<avx512>_vpermt2var<mode>3<sd_maskz_name>"
-  [(set (match_operand:VI48F 0 "register_operand" "=v")
-	(unspec:VI48F
-	  [(match_operand:<sseintvecmode> 1 "register_operand" "v")
-	   (match_operand:VI48F 2 "register_operand" "0")
-	   (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
+  [(set (match_operand:VPERMI2 0 "register_operand" "=v,v")
+	(unspec:VPERMI2
+	  [(match_operand:<sseintvecmode> 1 "register_operand" "v,0")
+	   (match_operand:VPERMI2 2 "register_operand" "0,v")
+	   (match_operand:VPERMI2 3 "nonimmediate_operand" "vm,vm")]
 	  UNSPEC_VPERMT2))]
   "TARGET_AVX512F"
-  "vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "<avx512>_vpermt2var<mode>3<sd_maskz_name>"
-  [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
-	(unspec:VI1_AVX512VL
-	  [(match_operand:<sseintvecmode> 1 "register_operand" "v")
-	   (match_operand:VI1_AVX512VL 2 "register_operand" "0")
-	   (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
-	  UNSPEC_VPERMT2))]
-  "TARGET_AVX512VBMI"
-  "vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "<avx512>_vpermt2var<mode>3<sd_maskz_name>"
-  [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
-	(unspec:VI2_AVX512VL
-	  [(match_operand:<sseintvecmode> 1 "register_operand" "v")
-	   (match_operand:VI2_AVX512VL 2 "register_operand" "0")
-	   (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
-	  UNSPEC_VPERMT2))]
-  "TARGET_AVX512BW"
-  "vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
+  "@
+   vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}
+   vpermi2<ssemodesuffix>\t{%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
 
 (define_insn "<avx512>_vpermt2var<mode>3_mask"
-  [(set (match_operand:VI48F 0 "register_operand" "=v")
-	(vec_merge:VI48F
-	  (unspec:VI48F
+  [(set (match_operand:VPERMI2 0 "register_operand" "=v")
+	(vec_merge:VPERMI2
+	  (unspec:VPERMI2
 	    [(match_operand:<sseintvecmode> 1 "register_operand" "v")
-	    (match_operand:VI48F 2 "register_operand" "0")
-	    (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
+	    (match_operand:VPERMI2 2 "register_operand" "0")
+	    (match_operand:VPERMI2 3 "nonimmediate_operand" "vm")]
 	    UNSPEC_VPERMT2)
 	  (match_dup 2)
 	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
@@ -18325,38 +18173,6 @@
    (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
 
-(define_insn "<avx512>_vpermt2var<mode>3_mask"
-  [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
-	(vec_merge:VI1_AVX512VL
-	  (unspec:VI1_AVX512VL
-	    [(match_operand:<sseintvecmode> 1 "register_operand" "v")
-	    (match_operand:VI1_AVX512VL 2 "register_operand" "0")
-	    (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
-	    UNSPEC_VPERMT2)
-	  (match_dup 2)
-	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
-  "TARGET_AVX512VBMI"
-  "vpermt2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "<avx512>_vpermt2var<mode>3_mask"
-  [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
-	(vec_merge:VI2_AVX512VL
-	  (unspec:VI2_AVX512VL
-	    [(match_operand:<sseintvecmode> 1 "register_operand" "v")
-	    (match_operand:VI2_AVX512VL 2 "register_operand" "0")
-	    (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
-	    UNSPEC_VPERMT2)
-	  (match_dup 2)
-	  (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
-  "TARGET_AVX512BW"
-  "vpermt2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
-  [(set_attr "type" "sselog")
-   (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
-
 (define_expand "avx_vperm2f128<mode>3"
   [(set (match_operand:AVX256MODE2P 0 "register_operand")
 	(unspec:AVX256MODE2P
@@ -19613,8 +19429,7 @@
 	  UNSPEC_DBPSADBW))]
    "TARGET_AVX512BW"
   "vdbpsadbw\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3}"
-  [(set_attr "isa" "avx")
-   (set_attr "type" "sselog1")
+  [(set_attr "type" "sselog1")
    (set_attr "length_immediate" "1")
    (set_attr "prefix" "evex")
    (set_attr "mode" "<sseinsnmode>")])
@@ -20159,3 +19974,20 @@
     ])]
   "TARGET_SSE && TARGET_64BIT"
   "jmp\t%P1")
+
+(define_insn "vgf2p8affineinvqb_<mode><mask_name>"
+  [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,x,v")
+	(unspec:VI1_AVX512F [(match_operand:VI1_AVX512F 1 "register_operand" "%0,x,v")
+			       (match_operand:VI1_AVX512F 2 "nonimmediate_operand" "xBm,xm,vm")
+			       (match_operand:QI 3 "const_0_to_255_operand" "n,n,n")]
+			      UNSPEC_GF2P8AFFINEINV))]
+  "TARGET_GFNI"
+  "@
+   gf2p8affineinvqb\t{%3, %2, %0| %0, %2, %3}
+   vgf2p8affineinvqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}
+   vgf2p8affineinvqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}"
+  [(set_attr "isa" "noavx,avx,avx512bw")
+   (set_attr "prefix_data16" "1,*,*")
+   (set_attr "prefix_extra" "1")
+   (set_attr "prefix" "orig,maybe_evex,evex")
+   (set_attr "mode" "<sseinsnmode>")])
diff --git a/gcc/config/i386/subst.md b/gcc/config/i386/subst.md
index a318a8d4c80..d9100c8d6b0 100644
--- a/gcc/config/i386/subst.md
+++ b/gcc/config/i386/subst.md
@@ -37,8 +37,7 @@
    V8DI  V4DI  V2DI
    V16SF V8SF  V4SF
    V8DF  V4DF  V2DF
-   QI HI SI DI SF DF
-   CCFP CCFPU])
+   QI HI SI DI SF DF])
 
 (define_subst_attr "mask_name" "mask" "" "_mask")
 (define_subst_attr "mask_applied" "mask" "false" "true")
@@ -62,8 +61,8 @@
 (define_subst_attr "store_mask_predicate" "mask" "nonimmediate_operand" "register_operand")
 (define_subst_attr "mask_prefix" "mask" "vex" "evex")
 (define_subst_attr "mask_prefix2" "mask" "maybe_vex" "evex")
-(define_subst_attr "mask_prefix3" "mask" "orig,vex" "evex")
-(define_subst_attr "mask_prefix4" "mask" "orig,orig,vex" "evex")
+(define_subst_attr "mask_prefix3" "mask" "orig,vex" "evex,evex")
+(define_subst_attr "mask_prefix4" "mask" "orig,orig,vex" "evex,evex,evex")
 (define_subst_attr "mask_expand_op3" "mask" "3" "5")
 
 (define_subst "mask"
@@ -183,6 +182,16 @@
 	  UNSPEC_EMBEDDED_ROUNDING))
 ])
 
+(define_subst "round_saeonly"
+  [(set (match_operand:CCFP 0)
+        (match_operand:CCFP 1))]
+  "TARGET_AVX512F"
+  [(set (match_dup 0)
+	(unspec:CCFP [(match_dup 1)
+	  (match_operand:SI 2 "const48_operand")]
+	  UNSPEC_EMBEDDED_ROUNDING))
+])
+
 (define_subst_attr "round_expand_name" "round_expand" "" "_round")
 (define_subst_attr "round_expand_nimm_predicate" "round_expand" "nonimmediate_operand" "register_operand")
 (define_subst_attr "round_expand_operand" "round_expand" "" ", operands[5]")
diff --git a/gcc/config/i386/sync.md b/gcc/config/i386/sync.md
index 29b82f86d43..eceaa73a679 100644
--- a/gcc/config/i386/sync.md
+++ b/gcc/config/i386/sync.md
@@ -219,29 +219,71 @@
    (set (match_operand:DI 2 "memory_operand")
 	(unspec:DI [(match_dup 0)]
 		   UNSPEC_FIST_ATOMIC))
-   (set (match_operand:DF 3 "fp_register_operand")
+   (set (match_operand:DF 3 "any_fp_register_operand")
 	(match_operand:DF 4 "memory_operand"))]
   "!TARGET_64BIT
    && peep2_reg_dead_p (2, operands[0])
-   && rtx_equal_p (operands[4], adjust_address_nv (operands[2], DFmode, 0))"
+   && rtx_equal_p (XEXP (operands[4], 0), XEXP (operands[2], 0))"
   [(set (match_dup 3) (match_dup 5))]
   "operands[5] = gen_lowpart (DFmode, operands[1]);")
 
 (define_peephole2
+  [(set (match_operand:DF 0 "fp_register_operand")
+	(unspec:DF [(match_operand:DI 1 "memory_operand")]
+		   UNSPEC_FILD_ATOMIC))
+   (set (match_operand:DI 2 "memory_operand")
+	(unspec:DI [(match_dup 0)]
+		   UNSPEC_FIST_ATOMIC))
+   (set (mem:BLK (scratch:SI))
+	(unspec:BLK [(mem:BLK (scratch:SI))] UNSPEC_MEMORY_BLOCKAGE))
+   (set (match_operand:DF 3 "any_fp_register_operand")
+	(match_operand:DF 4 "memory_operand"))]
+  "!TARGET_64BIT
+   && peep2_reg_dead_p (2, operands[0])
+   && rtx_equal_p (XEXP (operands[4], 0), XEXP (operands[2], 0))"
+  [(const_int 0)]
+{
+  emit_move_insn (operands[3], gen_lowpart (DFmode, operands[1]));
+  emit_insn (gen_memory_blockage ());
+  DONE;
+})
+
+(define_peephole2
   [(set (match_operand:DF 0 "sse_reg_operand")
 	(unspec:DF [(match_operand:DI 1 "memory_operand")]
 		   UNSPEC_LDX_ATOMIC))
    (set (match_operand:DI 2 "memory_operand")
 	(unspec:DI [(match_dup 0)]
 		   UNSPEC_STX_ATOMIC))
-   (set (match_operand:DF 3 "fp_register_operand")
+   (set (match_operand:DF 3 "any_fp_register_operand")
 	(match_operand:DF 4 "memory_operand"))]
   "!TARGET_64BIT
    && peep2_reg_dead_p (2, operands[0])
-   && rtx_equal_p (operands[4], adjust_address_nv (operands[2], DFmode, 0))"
+   && rtx_equal_p (XEXP (operands[4], 0), XEXP (operands[2], 0))"
   [(set (match_dup 3) (match_dup 5))]
   "operands[5] = gen_lowpart (DFmode, operands[1]);")
 
+(define_peephole2
+  [(set (match_operand:DF 0 "sse_reg_operand")
+	(unspec:DF [(match_operand:DI 1 "memory_operand")]
+		   UNSPEC_LDX_ATOMIC))
+   (set (match_operand:DI 2 "memory_operand")
+	(unspec:DI [(match_dup 0)]
+		   UNSPEC_STX_ATOMIC))
+   (set (mem:BLK (scratch:SI))
+	(unspec:BLK [(mem:BLK (scratch:SI))] UNSPEC_MEMORY_BLOCKAGE))
+   (set (match_operand:DF 3 "any_fp_register_operand")
+	(match_operand:DF 4 "memory_operand"))]
+  "!TARGET_64BIT
+   && peep2_reg_dead_p (2, operands[0])
+   && rtx_equal_p (XEXP (operands[4], 0), XEXP (operands[2], 0))"
+  [(const_int 0)]
+{
+  emit_move_insn (operands[3], gen_lowpart (DFmode, operands[1]));
+  emit_insn (gen_memory_blockage ());
+  DONE;
+})
+
 (define_expand "atomic_store<mode>"
   [(set (match_operand:ATOMIC 0 "memory_operand")
 	(unspec:ATOMIC [(match_operand:ATOMIC 1 "nonimmediate_operand")
@@ -331,7 +373,7 @@
 
 (define_peephole2
   [(set (match_operand:DF 0 "memory_operand")
-	(match_operand:DF 1 "fp_register_operand"))
+	(match_operand:DF 1 "any_fp_register_operand"))
    (set (match_operand:DF 2 "fp_register_operand")
 	(unspec:DF [(match_operand:DI 3 "memory_operand")]
 		   UNSPEC_FILD_ATOMIC))
@@ -340,13 +382,34 @@
 		   UNSPEC_FIST_ATOMIC))]
   "!TARGET_64BIT
    && peep2_reg_dead_p (3, operands[2])
-   && rtx_equal_p (operands[0], adjust_address_nv (operands[3], DFmode, 0))"
+   && rtx_equal_p (XEXP (operands[0], 0), XEXP (operands[3], 0))"
   [(set (match_dup 5) (match_dup 1))]
   "operands[5] = gen_lowpart (DFmode, operands[4]);")
 
 (define_peephole2
   [(set (match_operand:DF 0 "memory_operand")
-	(match_operand:DF 1 "fp_register_operand"))
+	(match_operand:DF 1 "any_fp_register_operand"))
+   (set (mem:BLK (scratch:SI))
+	(unspec:BLK [(mem:BLK (scratch:SI))] UNSPEC_MEMORY_BLOCKAGE))
+   (set (match_operand:DF 2 "fp_register_operand")
+	(unspec:DF [(match_operand:DI 3 "memory_operand")]
+		   UNSPEC_FILD_ATOMIC))
+   (set (match_operand:DI 4 "memory_operand")
+	(unspec:DI [(match_dup 2)]
+		   UNSPEC_FIST_ATOMIC))]
+  "!TARGET_64BIT
+   && peep2_reg_dead_p (4, operands[2])
+   && rtx_equal_p (XEXP (operands[0], 0), XEXP (operands[3], 0))"
+  [(const_int 0)]
+{
+  emit_insn (gen_memory_blockage ());
+  emit_move_insn (gen_lowpart (DFmode, operands[4]), operands[1]);
+  DONE;
+})
+
+(define_peephole2
+  [(set (match_operand:DF 0 "memory_operand")
+	(match_operand:DF 1 "any_fp_register_operand"))
    (set (match_operand:DF 2 "sse_reg_operand")
 	(unspec:DF [(match_operand:DI 3 "memory_operand")]
 		   UNSPEC_LDX_ATOMIC))
@@ -355,10 +418,31 @@
 		   UNSPEC_STX_ATOMIC))]
   "!TARGET_64BIT
    && peep2_reg_dead_p (3, operands[2])
-   && rtx_equal_p (operands[0], adjust_address_nv (operands[3], DFmode, 0))"
+   && rtx_equal_p (XEXP (operands[0], 0), XEXP (operands[3], 0))"
   [(set (match_dup 5) (match_dup 1))]
   "operands[5] = gen_lowpart (DFmode, operands[4]);")
 
+(define_peephole2
+  [(set (match_operand:DF 0 "memory_operand")
+	(match_operand:DF 1 "any_fp_register_operand"))
+   (set (mem:BLK (scratch:SI))
+	(unspec:BLK [(mem:BLK (scratch:SI))] UNSPEC_MEMORY_BLOCKAGE))
+   (set (match_operand:DF 2 "sse_reg_operand")
+	(unspec:DF [(match_operand:DI 3 "memory_operand")]
+		   UNSPEC_LDX_ATOMIC))
+   (set (match_operand:DI 4 "memory_operand")
+	(unspec:DI [(match_dup 2)]
+		   UNSPEC_STX_ATOMIC))]
+  "!TARGET_64BIT
+   && peep2_reg_dead_p (4, operands[2])
+   && rtx_equal_p (XEXP (operands[0], 0), XEXP (operands[3], 0))"
+  [(const_int 0)]
+{
+  emit_insn (gen_memory_blockage ());
+  emit_move_insn (gen_lowpart (DFmode, operands[4]), operands[1]);
+  DONE;
+})
+
 ;; ??? You'd think that we'd be able to perform this via FLOAT + FIX_TRUNC
 ;; operations.  But the fix_trunc patterns want way more setup than we want
 ;; to provide.  Note that the scratch is DFmode instead of XFmode in order
diff --git a/gcc/config/i386/t-cet b/gcc/config/i386/t-cet
new file mode 100644
index 00000000000..317f30dbb98
--- /dev/null
+++ b/gcc/config/i386/t-cet
@@ -0,0 +1,21 @@
+# Copyright (C) 2017 Free Software Foundation, Inc.
+#
+# This file is part of GCC.
+#
+# GCC is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GCC is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+cet.o: $(srcdir)/config/i386/cet.c
+	  $(COMPILE) $<
+	  $(POSTCOMPILE)
diff --git a/gcc/config/i386/winnt.c b/gcc/config/i386/winnt.c
index e690d2b907d..2f8518e1f1d 100644
--- a/gcc/config/i386/winnt.c
+++ b/gcc/config/i386/winnt.c
@@ -1217,8 +1217,7 @@ void
 i386_pe_start_function (FILE *f, const char *name, tree decl)
 {
   i386_pe_maybe_record_exported_symbol (decl, name, 0);
-  if (write_symbols != SDB_DEBUG)
-    i386_pe_declare_function_type (f, name, TREE_PUBLIC (decl));
+  i386_pe_declare_function_type (f, name, TREE_PUBLIC (decl));
   /* In case section was altered by debugging output.  */
   if (decl != NULL_TREE)
     switch_to_section (function_section (decl));
diff --git a/gcc/config/i386/x86-tune-costs.h b/gcc/config/i386/x86-tune-costs.h
index d27072c0901..c7ac70e8453 100644
--- a/gcc/config/i386/x86-tune-costs.h
+++ b/gcc/config/i386/x86-tune-costs.h
@@ -1,4 +1,26 @@
+/* Costs of operations of individual x86 CPUs.
+   Copyright (C) 1988-2017 Free Software Foundation, Inc.
 
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
 /* Processor costs (relative to an add) */
 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes.  */
 #define COSTS_N_BYTES(N) ((N) * 2)
@@ -33,6 +55,8 @@ struct processor_costs ix86_size_cost = {/* costs for tuning for size */
   COSTS_N_BYTES (3),			/* cost of movzx */
   0,					/* "large" insn */
   2,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2. */
   2,				     /* cost for loading QImode using movzbl */
   {2, 2, 2},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -48,12 +72,18 @@ struct processor_costs ix86_size_cost = {/* costs for tuning for size */
 					   in SImode and DImode */
   {3, 3},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  3,					/* cost of moving SSE register */
-  {3, 3, 3},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {3, 3, 3},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  3, 3, 3,				/* cost of moving XMM,YMM,ZMM register */
+  {3, 3, 3, 3, 3},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {3, 3, 3, 3, 3},			/* cost of unaligned SSE load
+					   in 128bit, 256bit and 512bit */
+  {3, 3, 3, 3, 3},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {3, 3, 3, 3, 3},				/* cost of unaligned SSE store
+					   in 128bit, 256bit and 512bit */
+  3, 3,					/* SSE->integer and integer->SSE moves */
+  5, 0,					/* Gather load static, per_elt.  */
+  5, 0,					/* Gather store static, per_elt.  */
   0,					/* size of l1 cache  */
   0,					/* size of l2 cache  */
   0,					/* size of prefetch block */
@@ -65,20 +95,22 @@ struct processor_costs ix86_size_cost = {/* costs for tuning for size */
   COSTS_N_BYTES (2),			/* cost of FABS instruction.  */
   COSTS_N_BYTES (2),			/* cost of FCHS instruction.  */
   COSTS_N_BYTES (2),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_BYTES (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_BYTES (2),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_BYTES (2),			/* cost of MULSS instruction.  */
+  COSTS_N_BYTES (2),			/* cost of MULSD instruction.  */
+  COSTS_N_BYTES (2),			/* cost of FMA SS instruction.  */
+  COSTS_N_BYTES (2),			/* cost of FMA SD instruction.  */
+  COSTS_N_BYTES (2),			/* cost of DIVSS instruction.  */
+  COSTS_N_BYTES (2),			/* cost of DIVSD instruction.  */
+  COSTS_N_BYTES (2),			/* cost of SQRTSS instruction.  */
+  COSTS_N_BYTES (2),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   ix86_size_memcpy,
   ix86_size_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  1,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  1,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_BYTES (1),			/* cond_taken_branch_cost.  */
+  COSTS_N_BYTES (1),			/* cond_not_taken_branch_cost.  */
 };
 
 /* Processor costs (relative to an add) */
@@ -110,6 +142,9 @@ struct processor_costs i386_cost = {	/* 386 specific costs */
   COSTS_N_INSNS (2),			/* cost of movzx */
   15,					/* "large" insn */
   3,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   4,				     /* cost for loading QImode using movzbl */
   {2, 4, 2},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -125,12 +160,16 @@ struct processor_costs i386_cost = {	/* 386 specific costs */
 					   in SImode and DImode */
   {4, 8},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 8, 16},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 8, 16},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 8, 16, 32, 64},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned loads.  */
+  {4, 8, 16, 32, 64},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned stores.  */
+  3, 3,					/* SSE->integer and integer->SSE moves */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   0,					/* size of l1 cache  */
   0,					/* size of l2 cache  */
   0,					/* size of prefetch block */
@@ -142,20 +181,22 @@ struct processor_costs i386_cost = {	/* 386 specific costs */
   COSTS_N_INSNS (22),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (24),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (122),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (23),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (27),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (27),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (27),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (27),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (88),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (88),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (122),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (122),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   i386_memcpy,
   i386_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs i486_memcpy[2] = {
@@ -186,6 +227,9 @@ struct processor_costs i486_cost = {	/* 486 specific costs */
   COSTS_N_INSNS (2),			/* cost of movzx */
   15,					/* "large" insn */
   3,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   4,				     /* cost for loading QImode using movzbl */
   {2, 4, 2},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -201,12 +245,16 @@ struct processor_costs i486_cost = {	/* 486 specific costs */
 					   in SImode and DImode */
   {4, 8},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 8, 16},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 8, 16},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 8, 16, 32, 64},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned loads.  */
+  {4, 8, 16, 32, 64},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned stores.  */
+  3, 3,					/* SSE->integer and integer->SSE moves */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   4,					/* size of l1 cache.  486 has 8kB cache
 					   shared for code and data, so 4kB is
 					   not really precise.  */
@@ -220,20 +268,22 @@ struct processor_costs i486_cost = {	/* 486 specific costs */
   COSTS_N_INSNS (3),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (3),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (83),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (8),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (16),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (16),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (16),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (16),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (73),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (74),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (83),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (83),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   i486_memcpy,
   i486_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs pentium_memcpy[2] = {
@@ -264,6 +314,9 @@ struct processor_costs pentium_cost = {
   COSTS_N_INSNS (2),			/* cost of movzx */
   8,					/* "large" insn */
   6,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   6,				     /* cost for loading QImode using movzbl */
   {2, 4, 2},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -279,12 +332,16 @@ struct processor_costs pentium_cost = {
 					   in SImode and DImode */
   {8, 8},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 8, 16},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 8, 16},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 8, 16, 32, 64},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned loads.  */
+  {4, 8, 16, 32, 64},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned stores.  */
+  3, 3,					/* SSE->integer and integer->SSE moves */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   8,					/* size of l1 cache.  */
   8,					/* size of l2 cache  */
   0,					/* size of prefetch block */
@@ -296,20 +353,22 @@ struct processor_costs pentium_cost = {
   COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (70),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (3),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (3),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (39),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (39),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (70),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (70),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   pentium_memcpy,
   pentium_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static const
@@ -333,6 +392,9 @@ struct processor_costs lakemont_cost = {
   COSTS_N_INSNS (2),			/* cost of movzx */
   8,					/* "large" insn */
   17,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   6,				     /* cost for loading QImode using movzbl */
   {2, 4, 2},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -348,12 +410,16 @@ struct processor_costs lakemont_cost = {
 					   in SImode and DImode */
   {8, 8},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 8, 16},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 8, 16},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 8, 16, 32, 64},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned loads.  */
+  {4, 8, 16, 32, 64},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned stores.  */
+  3, 3,					/* SSE->integer and integer->SSE moves */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   8,					/* size of l1 cache.  */
   8,					/* size of l2 cache  */
   0,					/* size of prefetch block */
@@ -365,20 +431,22 @@ struct processor_costs lakemont_cost = {
   COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (70),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (5),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (5),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (10),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (10),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (31),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (60),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (31),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (63),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   pentium_memcpy,
   pentium_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
@@ -417,6 +485,9 @@ struct processor_costs pentiumpro_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   6,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   2,				     /* cost for loading QImode using movzbl */
   {4, 4, 4},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -432,12 +503,16 @@ struct processor_costs pentiumpro_cost = {
 					   in SImode and DImode */
   {2, 2},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {2, 2, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {2, 2, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 8, 16, 32, 64},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned loads.  */
+  {4, 8, 16, 32, 64},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 8, 16, 32, 64},			/* cost of unaligned stores.  */
+  3, 3,					/* SSE->integer and integer->SSE moves */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   8,					/* size of l1 cache.  */
   256,					/* size of l2 cache  */
   32,					/* size of prefetch block */
@@ -449,20 +524,22 @@ struct processor_costs pentiumpro_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (56),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (7),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (7),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (18),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (18),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (31),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (31),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   pentiumpro_memcpy,
   pentiumpro_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs geode_memcpy[2] = {
@@ -492,13 +569,16 @@ struct processor_costs geode_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   4,					/* MOVE_RATIO */
-  1,				     /* cost for loading QImode using movzbl */
-  {1, 1, 1},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  2,				     /* cost for loading QImode using movzbl */
+  {2, 2, 2},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {1, 1, 1},				/* cost of storing integer registers */
-  1,					/* cost of reg,reg fld/fst */
-  {1, 1, 1},				/* cost of loading fp registers
+  {2, 2, 2},				/* cost of storing integer registers */
+  2,					/* cost of reg,reg fld/fst */
+  {2, 2, 2},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
   {4, 6, 6},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
@@ -508,12 +588,16 @@ struct processor_costs geode_cost = {
 					   in SImode and DImode */
   {2, 2},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {2, 2, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {2, 2, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {2, 2, 8, 16, 32},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {2, 2, 8, 16, 32},			/* cost of unaligned loads.  */
+  {2, 2, 8, 16, 32},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {2, 2, 8, 16, 32},			/* cost of unaligned stores.  */
+  6, 6,					/* SSE->integer and integer->SSE moves */
+  2, 2,					/* Gather load static, per_elt.  */
+  2, 2,					/* Gather store static, per_elt.  */
   64,					/* size of l1 cache.  */
   128,					/* size of l2 cache.  */
   32,					/* size of prefetch block */
@@ -525,20 +609,22 @@ struct processor_costs geode_cost = {
   COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (54),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (6),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (11),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (11),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (17),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (17),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (47),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (47),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (54),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (54),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   geode_memcpy,
   geode_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs k6_memcpy[2] = {
@@ -568,6 +654,9 @@ struct processor_costs k6_cost = {
   COSTS_N_INSNS (2),			/* cost of movzx */
   8,					/* "large" insn */
   4,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   3,				     /* cost for loading QImode using movzbl */
   {4, 5, 4},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -583,12 +672,16 @@ struct processor_costs k6_cost = {
 					   in SImode and DImode */
   {2, 2},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {2, 2, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {2, 2, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  6,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {2, 2, 8, 16, 32},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {2, 2, 8, 16, 32},			/* cost of unaligned loads.  */
+  {2, 2, 8, 16, 32},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {2, 2, 8, 16, 32},			/* cost of unaligned stores.  */
+  6, 6,					/* SSE->integer and integer->SSE moves */
+  2, 2,					/* Gather load static, per_elt.  */
+  2, 2,					/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   32,					/* size of l2 cache.  Some models
 					   have integrated l2 cache, but
@@ -603,20 +696,22 @@ struct processor_costs k6_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (56),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (2),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (2),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (2),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (4),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (56),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (56),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (56),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (56),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   k6_memcpy,
   k6_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 /* For some reason, Athlon deals better with REP prefix (relative to loops)
@@ -649,6 +744,9 @@ struct processor_costs athlon_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   4,				     /* cost for loading QImode using movzbl */
   {3, 4, 3},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -664,12 +762,16 @@ struct processor_costs athlon_cost = {
 					   in SImode and DImode */
   {4, 4},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 6},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 5},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  5,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 4, 6, 12, 24},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 4, 6, 12, 24},			/* cost of unaligned loads.  */
+  {4, 4, 5, 10, 20},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 4, 5, 10, 20},			/* cost of unaligned stores.  */
+  5, 5,					/* SSE->integer and integer->SSE moves */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   64,					/* size of l1 cache.  */
   256,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -681,20 +783,23 @@ struct processor_costs athlon_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (4),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (8),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (8),			/* cost of FMA SD instruction.  */
+  /* 11-16  */
+  COSTS_N_INSNS (16),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (24),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (19),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (19),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   athlon_memcpy,
   athlon_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 /* K8 has optimized REP instruction for medium sized blocks, but for very
@@ -731,6 +836,9 @@ struct processor_costs k8_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   4,				     /* cost for loading QImode using movzbl */
   {3, 4, 3},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -746,12 +854,16 @@ struct processor_costs k8_cost = {
 					   in SImode and DImode */
   {4, 4},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 3, 6},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 5},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  5,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 3, 6, 12, 24},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 3, 6, 12, 24},			/* cost of unaligned loads.  */
+  {4, 4, 5, 10, 20},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 4, 5, 10, 20},			/* cost of unaligned stores.  */
+  5, 5,					/* SSE->integer and integer->SSE moves */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   64,					/* size of l1 cache.  */
   512,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -768,20 +880,23 @@ struct processor_costs k8_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (4),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (8),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (8),			/* cost of FMA SD instruction.  */
+  /* 11-16  */
+  COSTS_N_INSNS (16),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (20),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (19),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (27),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   k8_memcpy,
   k8_memset,
-  4,					/* scalar_stmt_cost.  */
-  2,					/* scalar load_cost.  */
-  2,					/* scalar_store_cost.  */
-  5,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  2,					/* vec_align_load_cost.  */
-  3,					/* vec_unalign_load_cost.  */
-  3,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  2,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_not_taken_branch_cost.  */
 };
 
 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
@@ -817,6 +932,9 @@ struct processor_costs amdfam10_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   4,				     /* cost for loading QImode using movzbl */
   {3, 4, 3},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
@@ -832,12 +950,14 @@ struct processor_costs amdfam10_cost = {
 					   in SImode and DImode */
   {4, 4},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 3},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 5},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {4, 4, 3, 6, 12},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 4, 3, 7, 12},			/* cost of unaligned loads.  */
+  {4, 4, 5, 10, 20},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {4, 4, 5, 10, 20},			/* cost of unaligned stores.  */
+  3, 3,					/* SSE->integer and integer->SSE moves */
   					/* On K8:
   					    MOVD reg64, xmmreg Double FSTORE 4
 					    MOVD reg32, xmmreg Double FSTORE 4
@@ -846,6 +966,8 @@ struct processor_costs amdfam10_cost = {
 							       1/1  1/1
 					    MOVD reg32, xmmreg Double FADD 3
 							       1/1  1/1 */
+  4, 4,					/* Gather load static, per_elt.  */
+  4, 4,					/* Gather store static, per_elt.  */
   64,					/* size of l1 cache.  */
   512,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -862,20 +984,23 @@ struct processor_costs amdfam10_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (4),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (8),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (8),			/* cost of FMA SD instruction.  */
+  /* 11-16  */
+  COSTS_N_INSNS (16),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (20),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (19),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (27),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   amdfam10_memcpy,
   amdfam10_memset,
-  4,					/* scalar_stmt_cost.  */
-  2,					/* scalar load_cost.  */
-  2,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  2,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  2,					/* vec_store_cost.  */
-  2,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 /*  BDVER1 has optimized REP instruction for medium sized blocks, but for
@@ -912,35 +1037,34 @@ const struct processor_costs bdver1_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
-  4,				     /* cost for loading QImode using movzbl */
-  {5, 5, 4},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  8,				     /* cost for loading QImode using movzbl */
+  {8, 8, 8},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
-  2,					/* cost of reg,reg fld/fst */
-  {5, 5, 12},				/* cost of loading fp registers
+  {8, 8, 8},				/* cost of storing integer registers */
+  4,					/* cost of reg,reg fld/fst */
+  {12, 12, 28},				/* cost of loading fp registers
 		   			   in SFmode, DFmode and XFmode */
-  {4, 4, 8},				/* cost of storing fp registers
+  {10, 10, 18},				/* cost of storing fp registers
  		   			   in SFmode, DFmode and XFmode */
-  2,					/* cost of moving MMX register */
-  {4, 4},				/* cost of loading MMX registers
+  4,					/* cost of moving MMX register */
+  {12, 12},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {4, 4},				/* cost of storing MMX registers
+  {10, 10},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 4},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 4},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  2,					/* MMX or SSE register to integer */
-  					/* On K8:
-					    MOVD reg64, xmmreg Double FSTORE 4
-					    MOVD reg32, xmmreg Double FSTORE 4
-					   On AMDFAM10:
-					    MOVD reg64, xmmreg Double FADD 3
-							       1/1  1/1
-					    MOVD reg32, xmmreg Double FADD 3
-							       1/1  1/1 */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {12, 12, 10, 20, 30},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {12, 12, 10, 20, 30},			/* cost of unaligned loads.  */
+  {10, 10, 10, 20, 30},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 20, 30},			/* cost of unaligned stores.  */
+  16, 20,				/* SSE->integer and integer->SSE moves */
+  12, 12,				/* Gather load static, per_elt.  */
+  10, 10,				/* Gather store static, per_elt.  */
   16,					/* size of l1 cache.  */
   2048,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -957,20 +1081,24 @@ const struct processor_costs bdver1_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (6),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (6),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  /* 9-24  */
+  COSTS_N_INSNS (24),			/* cost of DIVSS instruction.  */
+  /* 9-27  */
+  COSTS_N_INSNS (27),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (15),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (26),			/* cost of SQRTSD instruction.  */
   1, 2, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   bdver1_memcpy,
   bdver1_memset,
-  6,					/* scalar_stmt_cost.  */
-  4,					/* scalar load_cost.  */
-  4,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  4,					/* vec_align_load_cost.  */
-  4,					/* vec_unalign_load_cost.  */
-  4,					/* vec_store_cost.  */
-  4,					/* cond_taken_branch_cost.  */
-  2,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (4),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_not_taken_branch_cost.  */
 };
 
 /*  BDVER2 has optimized REP instruction for medium sized blocks, but for
@@ -1008,35 +1136,34 @@ const struct processor_costs bdver2_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
-  4,				     /* cost for loading QImode using movzbl */
-  {5, 5, 4},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  8,				     /* cost for loading QImode using movzbl */
+  {8, 8, 8},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
-  2,					/* cost of reg,reg fld/fst */
-  {5, 5, 12},				/* cost of loading fp registers
+  {8, 8, 8},				/* cost of storing integer registers */
+  4,					/* cost of reg,reg fld/fst */
+  {12, 12, 28},				/* cost of loading fp registers
 		   			   in SFmode, DFmode and XFmode */
-  {4, 4, 8},				/* cost of storing fp registers
+  {10, 10, 18},				/* cost of storing fp registers
  		   			   in SFmode, DFmode and XFmode */
-  2,					/* cost of moving MMX register */
-  {4, 4},				/* cost of loading MMX registers
+  4,					/* cost of moving MMX register */
+  {12, 12},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {4, 4},				/* cost of storing MMX registers
+  {10, 10},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 4},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 4},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  2,					/* MMX or SSE register to integer */
-  					/* On K8:
-					    MOVD reg64, xmmreg Double FSTORE 4
-					    MOVD reg32, xmmreg Double FSTORE 4
-					   On AMDFAM10:
-					    MOVD reg64, xmmreg Double FADD 3
-							       1/1  1/1
-					    MOVD reg32, xmmreg Double FADD 3
-							       1/1  1/1 */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {12, 12, 10, 20, 30},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {12, 12, 10, 20, 30},			/* cost of unaligned loads.  */
+  {10, 10, 10, 20, 30},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 20, 30},			/* cost of unaligned stores.  */
+  16, 20,				/* SSE->integer and integer->SSE moves */
+  12, 12,				/* Gather load static, per_elt.  */
+  10, 10,				/* Gather store static, per_elt.  */
   16,					/* size of l1 cache.  */
   2048,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1053,20 +1180,24 @@ const struct processor_costs bdver2_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (6),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (6),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  /* 9-24  */
+  COSTS_N_INSNS (24),			/* cost of DIVSS instruction.  */
+  /* 9-27  */
+  COSTS_N_INSNS (27),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (15),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (26),			/* cost of SQRTSD instruction.  */
   1, 2, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   bdver2_memcpy,
   bdver2_memset,
-  6,					/* scalar_stmt_cost.  */
-  4,					/* scalar load_cost.  */
-  4,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  4,					/* vec_align_load_cost.  */
-  4,					/* vec_unalign_load_cost.  */
-  4,					/* vec_store_cost.  */
-  4,					/* cond_taken_branch_cost.  */
-  2,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (4),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_not_taken_branch_cost.  */
 };
 
 
@@ -1103,27 +1234,34 @@ struct processor_costs bdver3_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
-  4,				     /* cost for loading QImode using movzbl */
-  {5, 5, 4},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  8,				     /* cost for loading QImode using movzbl */
+  {8, 8, 8},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
-  2,					/* cost of reg,reg fld/fst */
-  {5, 5, 12},				/* cost of loading fp registers
+  {8, 8, 8},				/* cost of storing integer registers */
+  4,					/* cost of reg,reg fld/fst */
+  {12, 12, 28},				/* cost of loading fp registers
 		   			   in SFmode, DFmode and XFmode */
-  {4, 4, 8},				/* cost of storing fp registers
+  {10, 10, 18},				/* cost of storing fp registers
  		   			   in SFmode, DFmode and XFmode */
-  2,					/* cost of moving MMX register */
-  {4, 4},				/* cost of loading MMX registers
+  4,					/* cost of moving MMX register */
+  {12, 12},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {4, 4},				/* cost of storing MMX registers
+  {10, 10},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 4},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 4},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  2,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {12, 12, 10, 20, 30},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {12, 12, 10, 20, 30},			/* cost of unaligned loads.  */
+  {10, 10, 10, 20, 30},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 20, 30},			/* cost of unaligned stores.  */
+  16, 20,				/* SSE->integer and integer->SSE moves */
+  12, 12,				/* Gather load static, per_elt.  */
+  10, 10,				/* Gather store static, per_elt.  */
   16,					/* size of l1 cache.  */
   2048,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1140,20 +1278,24 @@ struct processor_costs bdver3_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (6),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (6),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  /* 9-24  */
+  COSTS_N_INSNS (24),			/* cost of DIVSS instruction.  */
+  /* 9-27  */
+  COSTS_N_INSNS (27),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (15),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (26),			/* cost of SQRTSD instruction.  */
   1, 2, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   bdver3_memcpy,
   bdver3_memset,
-  6,					/* scalar_stmt_cost.  */
-  4,					/* scalar load_cost.  */
-  4,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  4,					/* vec_align_load_cost.  */
-  4,					/* vec_unalign_load_cost.  */
-  4,					/* vec_store_cost.  */
-  4,					/* cond_taken_branch_cost.  */
-  2,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (4),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_not_taken_branch_cost.  */
 };
 
 /*  BDVER4 has optimized REP instruction for medium sized blocks, but for
@@ -1189,27 +1331,34 @@ struct processor_costs bdver4_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
-  4,				     /* cost for loading QImode using movzbl */
-  {5, 5, 4},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  8,				     /* cost for loading QImode using movzbl */
+  {8, 8, 8},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
-  2,					/* cost of reg,reg fld/fst */
-  {5, 5, 12},				/* cost of loading fp registers
+  {8, 8, 8},				/* cost of storing integer registers */
+  4,					/* cost of reg,reg fld/fst */
+  {12, 12, 28},				/* cost of loading fp registers
 		   			   in SFmode, DFmode and XFmode */
-  {4, 4, 8},				/* cost of storing fp registers
+  {10, 10, 18},				/* cost of storing fp registers
  		   			   in SFmode, DFmode and XFmode */
-  2,					/* cost of moving MMX register */
-  {4, 4},				/* cost of loading MMX registers
+  4,					/* cost of moving MMX register */
+  {12, 12},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {4, 4},				/* cost of storing MMX registers
+  {10, 10},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 4},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 4},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  2,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {12, 12, 10, 20, 30},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {12, 12, 10, 20, 30},			/* cost of unaligned loads.  */
+  {10, 10, 10, 20, 30},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 20, 30},			/* cost of unaligned stores.  */
+  16, 20,				/* SSE->integer and integer->SSE moves */
+  12, 12,				/* Gather load static, per_elt.  */
+  10, 10,				/* Gather store static, per_elt.  */
   16,					/* size of l1 cache.  */
   2048,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1226,20 +1375,24 @@ struct processor_costs bdver4_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (6),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (6),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  /* 9-24  */
+  COSTS_N_INSNS (24),			/* cost of DIVSS instruction.  */
+  /* 9-27  */
+  COSTS_N_INSNS (27),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (15),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (26),			/* cost of SQRTSD instruction.  */
   1, 2, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   bdver4_memcpy,
   bdver4_memset,
-  6,					/* scalar_stmt_cost.  */
-  4,					/* scalar load_cost.  */
-  4,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  4,					/* vec_align_load_cost.  */
-  4,					/* vec_unalign_load_cost.  */
-  4,					/* vec_store_cost.  */
-  4,					/* cond_taken_branch_cost.  */
-  2,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (4),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_not_taken_branch_cost.  */
 };
 
 
@@ -1264,42 +1417,59 @@ struct processor_costs znver1_cost = {
   {COSTS_N_INSNS (3),			/* cost of starting multiply for QI.  */
    COSTS_N_INSNS (3),			/*				 HI.  */
    COSTS_N_INSNS (3),			/*				 SI.  */
-   COSTS_N_INSNS (4),			/*				 DI.  */
-   COSTS_N_INSNS (4)},			/*			      other.  */
+   COSTS_N_INSNS (3),			/*				 DI.  */
+   COSTS_N_INSNS (3)},			/*			      other.  */
   0,					/* cost of multiply per each bit
 					    set.  */
-  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI.  */
-   COSTS_N_INSNS (35),			/*			    HI.  */
-   COSTS_N_INSNS (51),			/*			    SI.  */
-   COSTS_N_INSNS (83),			/*			    DI.  */
-   COSTS_N_INSNS (83)},			/*			    other.  */
+   /* Depending on parameters, idiv can get faster on ryzen.  This is upper
+      bound.  */
+  {COSTS_N_INSNS (16),			/* cost of a divide/mod for QI.  */
+   COSTS_N_INSNS (22),			/*			    HI.  */
+   COSTS_N_INSNS (30),			/*			    SI.  */
+   COSTS_N_INSNS (45),			/*			    DI.  */
+   COSTS_N_INSNS (45)},			/*			    other.  */
   COSTS_N_INSNS (1),			/* cost of movsx.  */
   COSTS_N_INSNS (1),			/* cost of movzx.  */
   8,					/* "large" insn.  */
   9,					/* MOVE_RATIO.  */
-  4,					/* cost for loading QImode using
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+
+  /* reg-reg moves are done by renaming and thus they are even cheaper than
+     1 cycle. Becuase reg-reg move cost is 2 and the following tables correspond
+     to doubles of latencies, we do not model this correctly.  It does not
+     seem to make practical difference to bump prices up even more.  */
+  6,					/* cost for loading QImode using
 					   movzbl.  */
-  {5, 5, 4},				/* cost of loading integer registers
+  {6, 6, 6},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer
+  {8, 8, 8},				/* cost of storing integer
 					   registers.  */
   2,					/* cost of reg,reg fld/fst.  */
-  {5, 5, 12},				/* cost of loading fp registers
+  {6, 6, 16},				/* cost of loading fp registers
 		   			   in SFmode, DFmode and XFmode.  */
-  {4, 4, 8},				/* cost of storing fp registers
+  {8, 8, 16},				/* cost of storing fp registers
  		   			   in SFmode, DFmode and XFmode.  */
   2,					/* cost of moving MMX register.  */
-  {4, 4},				/* cost of loading MMX registers
+  {6, 6},				/* cost of loading MMX registers
 					   in SImode and DImode.  */
-  {4, 4},				/* cost of storing MMX registers
+  {8, 8},				/* cost of storing MMX registers
 					   in SImode and DImode.  */
-  2,					/* cost of moving SSE register.  */
-  {4, 4, 4},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode.  */
-  {4, 4, 4},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode.  */
-  2,					/* MMX or SSE register to integer.  */
+  2, 3, 6,				/* cost of moving XMM,YMM,ZMM register.  */
+  {6, 6, 6, 10, 20},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit.  */
+  {6, 6, 6, 10, 20},			/* cost of unaligned loads.  */
+  {8, 8, 8, 8, 16},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit.  */
+  {8, 8, 8, 8, 16},			/* cost of unaligned stores.  */
+  6, 6,					/* SSE->integer and integer->SSE moves.  */
+  /* VGATHERDPD is 23 uops and throughput is 9, VGATHERDPD is 35 uops,
+     throughput 12.  Approx 9 uops do not depend on vector size and every load
+     is 7 uops.  */
+  18, 8,				/* Gather load static, per_elt.  */
+  18, 10,				/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   512,					/* size of l2 cache.  */
   64,					/* size of prefetch block.  */
@@ -1310,12 +1480,26 @@ struct processor_costs znver1_cost = {
      time).  */
   100,					/* number of parallel prefetches.  */
   3,					/* Branch cost.  */
-  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
-  COSTS_N_INSNS (6),			/* cost of FMUL instruction.  */
-  COSTS_N_INSNS (42),			/* cost of FDIV instruction.  */
-  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
-  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
-  COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FADD and FSUB insns.  */
+  COSTS_N_INSNS (5),			/* cost of FMUL instruction.  */
+  /* Latency of fdiv is 8-15.  */
+  COSTS_N_INSNS (15),			/* cost of FDIV instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
+  /* Latency of fsqrt is 4-10.  */
+  COSTS_N_INSNS (10),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (3),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (10),			/* cost of DIVSS instruction.  */
+  /* 9-13  */
+  COSTS_N_INSNS (13),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (10),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (15),			/* cost of SQRTSD instruction.  */
   /* Zen can execute 4 integer operations per cycle. FP operations take 3 cycles
      and it can execute 2 integer additions and 2 multiplications thus
      reassociation may make sense up to with of 6.  SPEC2k6 bencharks suggests
@@ -1327,17 +1511,8 @@ struct processor_costs znver1_cost = {
   4, 4, 3, 6,				/* reassoc int, fp, vec_int, vec_fp.  */
   znver1_memcpy,
   znver1_memset,
-  6,					/* scalar_stmt_cost.  */
-  4,					/* scalar load_cost.  */
-  4,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  4,					/* vec_align_load_cost.  */
-  4,					/* vec_unalign_load_cost.  */
-  4,					/* vec_store_cost.  */
-  4,					/* cond_taken_branch_cost.  */
-  2,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (4),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_not_taken_branch_cost.  */
 };
 
   /* BTVER1 has optimized REP instruction for medium sized blocks, but for
@@ -1373,35 +1548,34 @@ const struct processor_costs btver1_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
-  4,				     /* cost for loading QImode using movzbl */
-  {3, 4, 3},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  8,				     /* cost for loading QImode using movzbl */
+  {6, 8, 6},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {3, 4, 3},				/* cost of storing integer registers */
+  {6, 8, 6},				/* cost of storing integer registers */
   4,					/* cost of reg,reg fld/fst */
-  {4, 4, 12},				/* cost of loading fp registers
+  {12, 12, 28},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {6, 6, 8},				/* cost of storing fp registers
+  {12, 12, 38},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
-  2,					/* cost of moving MMX register */
-  {3, 3},				/* cost of loading MMX registers
+  4,					/* cost of moving MMX register */
+  {10, 10},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {4, 4},				/* cost of storing MMX registers
+  {12, 12},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 3},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 5},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
-					/* On K8:
-					   MOVD reg64, xmmreg Double FSTORE 4
-					   MOVD reg32, xmmreg Double FSTORE 4
-					   On AMDFAM10:
-					   MOVD reg64, xmmreg Double FADD 3
-							       1/1  1/1
-					    MOVD reg32, xmmreg Double FADD 3
-							       1/1  1/1 */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {10, 10, 12, 24, 48},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 12, 24, 48},			/* cost of unaligned loads.  */
+  {10, 10, 12, 24, 48},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 12, 24, 48},			/* cost of unaligned stores.  */
+  14, 14,				/* SSE->integer and integer->SSE moves */
+  10, 10,				/* Gather load static, per_elt.  */
+  10, 10,				/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   512,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1413,20 +1587,22 @@ const struct processor_costs btver1_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (2),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (13),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (17),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (14),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (48),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   btver1_memcpy,
   btver1_memset,
-  4,					/* scalar_stmt_cost.  */
-  2,					/* scalar load_cost.  */
-  2,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  2,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  2,					/* vec_store_cost.  */
-  2,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs btver2_memcpy[2] = {
@@ -1459,35 +1635,34 @@ const struct processor_costs btver2_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   9,					/* MOVE_RATIO */
-  4,				     /* cost for loading QImode using movzbl */
-  {3, 4, 3},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  8,				     /* cost for loading QImode using movzbl */
+  {8, 8, 6},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {3, 4, 3},				/* cost of storing integer registers */
+  {8, 8, 6},				/* cost of storing integer registers */
   4,					/* cost of reg,reg fld/fst */
-  {4, 4, 12},				/* cost of loading fp registers
+  {12, 12, 28},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {6, 6, 8},				/* cost of storing fp registers
+  {12, 12, 38},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
-  2,					/* cost of moving MMX register */
-  {3, 3},				/* cost of loading MMX registers
+  4,					/* cost of moving MMX register */
+  {10, 10},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {4, 4},				/* cost of storing MMX registers
+  {12, 12},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {4, 4, 3},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {4, 4, 5},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  3,					/* MMX or SSE register to integer */
-					/* On K8:
-					   MOVD reg64, xmmreg Double FSTORE 4
-					   MOVD reg32, xmmreg Double FSTORE 4
-					   On AMDFAM10:
-					   MOVD reg64, xmmreg Double FADD 3
-							       1/1  1/1
-					    MOVD reg32, xmmreg Double FADD 3
-							       1/1  1/1 */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {10, 10, 12, 24, 48},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 12, 24, 48},			/* cost of unaligned loads.  */
+  {10, 10, 12, 24, 48},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 12, 24, 48},			/* cost of unaligned stores.  */
+  14, 14,				/* SSE->integer and integer->SSE moves */
+  10, 10,				/* Gather load static, per_elt.  */
+  10, 10,				/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   2048,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1499,20 +1674,22 @@ const struct processor_costs btver2_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (2),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (4),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (13),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (19),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (16),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (21),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   btver2_memcpy,
   btver2_memset,
-  4,					/* scalar_stmt_cost.  */
-  2,					/* scalar load_cost.  */
-  2,					/* scalar_store_cost.  */
-  6,					/* vec_stmt_cost.  */
-  0,					/* vec_to_scalar_cost.  */
-  2,					/* scalar_to_vec_cost.  */
-  2,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  2,					/* vec_store_cost.  */
-  2,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs pentium4_memcpy[2] = {
@@ -1544,27 +1721,34 @@ struct processor_costs pentium4_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   16,					/* "large" insn */
   6,					/* MOVE_RATIO */
-  2,				     /* cost for loading QImode using movzbl */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  5,				     /* cost for loading QImode using movzbl */
   {4, 5, 4},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
   {2, 3, 2},				/* cost of storing integer registers */
-  2,					/* cost of reg,reg fld/fst */
-  {2, 2, 6},				/* cost of loading fp registers
+  12,					/* cost of reg,reg fld/fst */
+  {14, 14, 14},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {4, 4, 6},				/* cost of storing fp registers
+  {14, 14, 14},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
-  2,					/* cost of moving MMX register */
-  {2, 2},				/* cost of loading MMX registers
+  12,					/* cost of moving MMX register */
+  {16, 16},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {2, 2},				/* cost of storing MMX registers
+  {16, 16},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  12,					/* cost of moving SSE register */
-  {12, 12, 12},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {2, 2, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  10,					/* MMX or SSE register to integer */
+  12, 24, 48,				/* cost of moving XMM,YMM,ZMM register */
+  {16, 16, 16, 32, 64},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {32, 32, 32, 64, 128},		/* cost of unaligned loads.  */
+  {16, 16, 16, 32, 64},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {32, 32, 32, 64, 128},		/* cost of unaligned stores.  */
+  20, 12,				/* SSE->integer and integer->SSE moves */
+  16, 16,				/* Gather load static, per_elt.  */
+  16, 16,				/* Gather store static, per_elt.  */
   8,					/* size of l1 cache.  */
   256,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1576,20 +1760,22 @@ struct processor_costs pentium4_cost = {
   COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (43),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (4),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (6),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (23),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (38),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (23),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (38),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   pentium4_memcpy,
   pentium4_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs nocona_memcpy[2] = {
@@ -1624,27 +1810,34 @@ struct processor_costs nocona_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   16,					/* "large" insn */
   17,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   4,				     /* cost for loading QImode using movzbl */
   {4, 4, 4},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
   {4, 4, 4},				/* cost of storing integer registers */
-  3,					/* cost of reg,reg fld/fst */
-  {12, 12, 12},				/* cost of loading fp registers
+  12,					/* cost of reg,reg fld/fst */
+  {14, 14, 14},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {4, 4, 4},				/* cost of storing fp registers
+  {14, 14, 14},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
-  6,					/* cost of moving MMX register */
+  14,					/* cost of moving MMX register */
   {12, 12},				/* cost of loading MMX registers
 					   in SImode and DImode */
   {12, 12},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  6,					/* cost of moving SSE register */
-  {12, 12, 12},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {12, 12, 12},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  8,					/* MMX or SSE register to integer */
+  6, 12, 24,				/* cost of moving XMM,YMM,ZMM register */
+  {12, 12, 12, 24, 48},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {24, 24, 24, 48, 96},			/* cost of unaligned loads.  */
+  {12, 12, 12, 24, 48},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {24, 24, 24, 48, 96},			/* cost of unaligned stores.  */
+  20, 12,				/* SSE->integer and integer->SSE moves */
+  12, 12,				/* Gather load static, per_elt.  */
+  12, 12,				/* Gather store static, per_elt.  */
   8,					/* size of l1 cache.  */
   1024,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1656,20 +1849,22 @@ struct processor_costs nocona_cost = {
   COSTS_N_INSNS (3),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (3),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (44),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (2),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (5),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (7),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (7),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (7),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (7),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (32),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (40),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (32),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (41),			/* cost of SQRTSD instruction.  */
   1, 1, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   nocona_memcpy,
   nocona_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs atom_memcpy[2] = {
@@ -1702,27 +1897,34 @@ struct processor_costs atom_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   17,					/* MOVE_RATIO */
-  4,					/* cost for loading QImode using movzbl */
-  {4, 4, 4},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  6,					/* cost for loading QImode using movzbl */
+  {6, 6, 6},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
+  {6, 6, 6},				/* cost of storing integer registers */
   4,					/* cost of reg,reg fld/fst */
-  {12, 12, 12},				/* cost of loading fp registers
+  {6, 6, 18},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {6, 6, 8},				/* cost of storing fp registers
+  {14, 14, 24},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
   2,					/* cost of moving MMX register */
   {8, 8},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {8, 8},				/* cost of storing MMX registers
+  {10, 10},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {8, 8, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {8, 8, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  5,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {8, 8, 8, 16, 32},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {16, 16, 16, 32, 64},			/* cost of unaligned loads.  */
+  {8, 8, 8, 16, 32},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {16, 16, 16, 32, 64},			/* cost of unaligned stores.  */
+  8, 6,					/* SSE->integer and integer->SSE moves */
+  8, 8,					/* Gather load static, per_elt.  */
+  8, 8,					/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   256,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1734,20 +1936,22 @@ struct processor_costs atom_cost = {
   COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (5),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (31),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (60),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (31),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (63),			/* cost of SQRTSD instruction.  */
   2, 2, 2, 2,				/* reassoc int, fp, vec_int, vec_fp.  */
   atom_memcpy,
   atom_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs slm_memcpy[2] = {
@@ -1780,27 +1984,34 @@ struct processor_costs slm_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   17,					/* MOVE_RATIO */
-  4,					/* cost for loading QImode using movzbl */
-  {4, 4, 4},				/* cost of loading integer registers
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  8,					/* cost for loading QImode using movzbl */
+  {8, 8, 8},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
-  4,					/* cost of reg,reg fld/fst */
-  {12, 12, 12},				/* cost of loading fp registers
+  {6, 6, 6},				/* cost of storing integer registers */
+  2,					/* cost of reg,reg fld/fst */
+  {8, 8, 18},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {6, 6, 8},				/* cost of storing fp registers
+  {6, 6, 18},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
   2,					/* cost of moving MMX register */
   {8, 8},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {8, 8},				/* cost of storing MMX registers
+  {6, 6},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {8, 8, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {8, 8, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  5,					/* MMX or SSE register to integer */
+  2, 4, 8,				/* cost of moving XMM,YMM,ZMM register */
+  {8, 8, 8, 16, 32},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {16, 16, 16, 32, 64},			/* cost of unaligned loads.  */
+  {8, 8, 8, 16, 32},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {16, 16, 16, 32, 64},			/* cost of unaligned stores.  */
+  8, 6,					/* SSE->integer and integer->SSE moves */
+  8, 8,					/* Gather load static, per_elt.  */
+  8, 8,					/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   256,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1812,20 +2023,22 @@ struct processor_costs slm_cost = {
   COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (39),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (69),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (20),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (35),			/* cost of SQRTSD instruction.  */
   1, 2, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   slm_memcpy,
   slm_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  4,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 static stringop_algs intel_memcpy[2] = {
@@ -1858,27 +2071,34 @@ struct processor_costs intel_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   17,					/* MOVE_RATIO */
-  4,					/* cost for loading QImode using movzbl */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  6,				     /* cost for loading QImode using movzbl */
   {4, 4, 4},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
-  4,					/* cost of reg,reg fld/fst */
-  {12, 12, 12},				/* cost of loading fp registers
+  {6, 6, 6},				/* cost of storing integer registers */
+  2,					/* cost of reg,reg fld/fst */
+  {6, 6, 8},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {6, 6, 8},				/* cost of storing fp registers
+  {6, 6, 10},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
   2,					/* cost of moving MMX register */
-  {8, 8},				/* cost of loading MMX registers
+  {6, 6},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {8, 8},				/* cost of storing MMX registers
+  {6, 6},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {8, 8, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {8, 8, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  5,					/* MMX or SSE register to integer */
+  2, 2, 2,				/* cost of moving XMM,YMM,ZMM register */
+  {6, 6, 6, 6, 6},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 10, 10},			/* cost of unaligned loads.  */
+  {6, 6, 6, 6, 6},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 10, 10},			/* cost of unaligned loads.  */
+  4, 4,					/* SSE->integer and integer->SSE moves */
+  6, 6,					/* Gather load static, per_elt.  */
+  6, 6,					/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   256,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1890,20 +2110,22 @@ struct processor_costs intel_cost = {
   COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
   COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (8),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (8),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (8),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (8),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (6),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (20),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (20),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (40),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (40),			/* cost of SQRTSD instruction.  */
   1, 4, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   intel_memcpy,
   intel_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  4,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 /* Generic should produce code tuned for Core-i7 (and newer chips)
@@ -1922,8 +2144,7 @@ static stringop_algs generic_memset[2] = {
 static const
 struct processor_costs generic_cost = {
   COSTS_N_INSNS (1),			/* cost of an add instruction */
-  /* On all chips taken into consideration lea is 2 cycles and more.  With
-     this cost however our current implementation of synth_mult results in
+  /* Setting cost to 2 makes our current implementation of synth_mult result in
      use of unnecessary temporary registers causing regression on several
      SPECfp benchmarks.  */
   COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
@@ -1944,27 +2165,34 @@ struct processor_costs generic_cost = {
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   17,					/* MOVE_RATIO */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
   4,				     /* cost for loading QImode using movzbl */
   {4, 4, 4},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
+  {6, 6, 6},				/* cost of storing integer registers */
   4,					/* cost of reg,reg fld/fst */
-  {12, 12, 12},				/* cost of loading fp registers
+  {6, 6, 12},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {6, 6, 8},				/* cost of storing fp registers
+  {6, 6, 12},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
   2,					/* cost of moving MMX register */
-  {8, 8},				/* cost of loading MMX registers
+  {6, 6},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {8, 8},				/* cost of storing MMX registers
+  {6, 6},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {8, 8, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {8, 8, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  5,					/* MMX or SSE register to integer */
+  2, 3, 4,				/* cost of moving XMM,YMM,ZMM register */
+  {6, 6, 6, 10, 15},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 15, 20},			/* cost of unaligned loads.  */
+  {6, 6, 6, 10, 15},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {10, 10, 10, 15, 20},			/* cost of unaligned storess.  */
+  20, 20,				/* SSE->integer and integer->SSE moves */
+  6, 6,					/* Gather load static, per_elt.  */
+  6, 6,					/* Gather store static, per_elt.  */
   32,					/* size of l1 cache.  */
   512,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
@@ -1972,26 +2200,28 @@ struct processor_costs generic_cost = {
   /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
      value is increased to perhaps more appropriate value of 5.  */
   3,					/* Branch cost */
-  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
-  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
+  COSTS_N_INSNS (3),			/* cost of FADD and FSUB insns.  */
+  COSTS_N_INSNS (3),			/* cost of FMUL instruction.  */
   COSTS_N_INSNS (20),			/* cost of FDIV instruction.  */
-  COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
-  COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
   COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (18),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (32),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (30),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (58),			/* cost of SQRTSD instruction.  */
   1, 2, 1, 1,				/* reassoc int, fp, vec_int, vec_fp.  */
   generic_memcpy,
   generic_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
 /* core_cost should produce code tuned for Core familly of CPUs.  */
@@ -2021,63 +2251,78 @@ struct processor_costs core_cost = {
    COSTS_N_INSNS (4),			/*				 HI */
    COSTS_N_INSNS (3),			/*				 SI */
    COSTS_N_INSNS (4),			/*				 DI */
-   COSTS_N_INSNS (2)},			/*			      other */
+   COSTS_N_INSNS (4)},			/*			      other */
   0,					/* cost of multiply per each bit set */
-  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
-   COSTS_N_INSNS (26),			/*			    HI */
-   COSTS_N_INSNS (42),			/*			    SI */
-   COSTS_N_INSNS (74),			/*			    DI */
-   COSTS_N_INSNS (74)},			/*			    other */
+  {COSTS_N_INSNS (8),			/* cost of a divide/mod for QI */
+   COSTS_N_INSNS (8),			/*			    HI */
+   /* 8-11 */
+   COSTS_N_INSNS (11),			/*			    SI */
+   /* 24-81 */
+   COSTS_N_INSNS (81),			/*			    DI */
+   COSTS_N_INSNS (81)},			/*			    other */
   COSTS_N_INSNS (1),			/* cost of movsx */
   COSTS_N_INSNS (1),			/* cost of movzx */
   8,					/* "large" insn */
   17,					/* MOVE_RATIO */
-  4,				     /* cost for loading QImode using movzbl */
+
+  /* All move costs are relative to integer->integer move times 2 and thus
+     they are latency*2. */
+  6,				     /* cost for loading QImode using movzbl */
   {4, 4, 4},				/* cost of loading integer registers
 					   in QImode, HImode and SImode.
 					   Relative to reg-reg move (2).  */
-  {4, 4, 4},				/* cost of storing integer registers */
-  4,					/* cost of reg,reg fld/fst */
-  {12, 12, 12},				/* cost of loading fp registers
+  {6, 6, 6},				/* cost of storing integer registers */
+  2,					/* cost of reg,reg fld/fst */
+  {6, 6, 8},				/* cost of loading fp registers
 					   in SFmode, DFmode and XFmode */
-  {6, 6, 8},				/* cost of storing fp registers
+  {6, 6, 10},				/* cost of storing fp registers
 					   in SFmode, DFmode and XFmode */
   2,					/* cost of moving MMX register */
-  {8, 8},				/* cost of loading MMX registers
+  {6, 6},				/* cost of loading MMX registers
 					   in SImode and DImode */
-  {8, 8},				/* cost of storing MMX registers
+  {6, 6},				/* cost of storing MMX registers
 					   in SImode and DImode */
-  2,					/* cost of moving SSE register */
-  {8, 8, 8},				/* cost of loading SSE registers
-					   in SImode, DImode and TImode */
-  {8, 8, 8},				/* cost of storing SSE registers
-					   in SImode, DImode and TImode */
-  5,					/* MMX or SSE register to integer */
+  2, 2, 4,				/* cost of moving XMM,YMM,ZMM register */
+  {6, 6, 6, 6, 12},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {6, 6, 6, 6, 12},			/* cost of unaligned loads.  */
+  {6, 6, 6, 6, 12},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {6, 6, 6, 6, 12},			/* cost of unaligned stores.  */
+  2, 2,					/* SSE->integer and integer->SSE moves */
+  /* VGATHERDPD is 7 uops, rec throughput 5, while VGATHERDPD is 9 uops,
+     rec. throughput 6.
+     So 5 uops statically and one uops per load.  */
+  10, 6,				/* Gather load static, per_elt.  */
+  10, 6,				/* Gather store static, per_elt.  */
   64,					/* size of l1 cache.  */
   512,					/* size of l2 cache.  */
   64,					/* size of prefetch block */
   6,					/* number of parallel prefetches */
   /* FIXME perhaps more appropriate value is 5.  */
   3,					/* Branch cost */
-  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
-  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
-  COSTS_N_INSNS (20),			/* cost of FDIV instruction.  */
-  COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
-  COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
-  COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
+  COSTS_N_INSNS (3),			/* cost of FADD and FSUB insns.  */
+  COSTS_N_INSNS (5),			/* cost of FMUL instruction.  */
+  /* 10-24 */
+  COSTS_N_INSNS (24),			/* cost of FDIV instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
+  COSTS_N_INSNS (23),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (18),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (32),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (30),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (58),			/* cost of SQRTSD instruction.  */
   1, 4, 2, 2,				/* reassoc int, fp, vec_int, vec_fp.  */
   core_memcpy,
   core_memset,
-  1,					/* scalar_stmt_cost.  */
-  1,					/* scalar load_cost.  */
-  1,					/* scalar_store_cost.  */
-  1,					/* vec_stmt_cost.  */
-  1,					/* vec_to_scalar_cost.  */
-  1,					/* scalar_to_vec_cost.  */
-  1,					/* vec_align_load_cost.  */
-  2,					/* vec_unalign_load_cost.  */
-  1,					/* vec_store_cost.  */
-  3,					/* cond_taken_branch_cost.  */
-  1,					/* cond_not_taken_branch_cost.  */
+  COSTS_N_INSNS (3),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (1),			/* cond_not_taken_branch_cost.  */
 };
 
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 9d01761eff9..99282c88341 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -48,7 +48,8 @@ DEF_TUNE (X86_TUNE_SCHEDULE, "schedule",
    over partial stores.  For example preffer MOVZBL or MOVQ to load 8bit
    value over movb.  */
 DEF_TUNE (X86_TUNE_PARTIAL_REG_DEPENDENCY, "partial_reg_dependency",
-          m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_SILVERMONT | m_INTEL
+          m_P4_NOCONA | m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE
+	  | m_BONNELL | m_SILVERMONT | m_INTEL
 	  | m_KNL | m_KNM | m_AMD_MULTIPLE | m_GENERIC)
 
 /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: This knob promotes all store
@@ -84,8 +85,9 @@ DEF_TUNE (X86_TUNE_PARTIAL_FLAG_REG_STALL, "partial_flag_reg_stall",
 /* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid
    partial dependencies.  */
 DEF_TUNE (X86_TUNE_MOVX, "movx",
-          m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_SILVERMONT
-	  | m_KNL | m_KNM | m_INTEL | m_GEODE | m_AMD_MULTIPLE  | m_GENERIC)
+          m_PPRO | m_P4_NOCONA | m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE
+	  | m_BONNELL | m_SILVERMONT | m_KNL | m_KNM | m_INTEL
+	  | m_GEODE | m_AMD_MULTIPLE  | m_GENERIC)
 
 /* X86_TUNE_MEMORY_MISMATCH_STALL: Avoid partial stores that are followed by
    full sized loads.  */
@@ -218,10 +220,15 @@ DEF_TUNE (X86_TUNE_LCP_STALL, "lcp_stall", m_CORE_ALL | m_GENERIC)
    as "add mem, reg".  */
 DEF_TUNE (X86_TUNE_READ_MODIFY, "read_modify", ~(m_PENT | m_LAKEMONT | m_PPRO))
 
-/* X86_TUNE_USE_INCDEC: Enable use of inc/dec instructions.   */
+/* X86_TUNE_USE_INCDEC: Enable use of inc/dec instructions.
+
+   Core2 and nehalem has stall of 7 cycles for partial flag register stalls.
+   Sandy bridge and Ivy bridge generate extra uop.  On Haswell this extra uop
+   is output only when the values needs to be really merged, which is not
+   done by GCC generated code.  */
 DEF_TUNE (X86_TUNE_USE_INCDEC, "use_incdec",
-          ~(m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_SILVERMONT | m_INTEL
-	   |  m_KNL | m_KNM | m_GENERIC))
+          ~(m_P4_NOCONA | m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE
+	    | m_BONNELL | m_SILVERMONT | m_INTEL |  m_KNL | m_KNM | m_GENERIC))
 
 /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
    for DFmode copies */
@@ -364,7 +371,7 @@ DEF_TUNE (X86_TUNE_SSE_LOAD0_BY_PXOR, "sse_load0_by_pxor",
    to SSE registers.  If disabled, the moves will be done by storing
    the value to memory and reloading.  */
 DEF_TUNE (X86_TUNE_INTER_UNIT_MOVES_TO_VEC, "inter_unit_moves_to_vec",
-          ~(m_AMD_MULTIPLE | m_GENERIC))
+          ~(m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER | m_GENERIC))
 
 /* X86_TUNE_INTER_UNIT_MOVES_TO_VEC: Enable moves in from SSE
    to integer registers.  If disabled, the moves will be done by storing
diff --git a/gcc/config/ia64/ia64.h b/gcc/config/ia64/ia64.h
index e7073d1cf20..eceab5f23b6 100644
--- a/gcc/config/ia64/ia64.h
+++ b/gcc/config/ia64/ia64.h
@@ -1470,7 +1470,7 @@ do {									\
 /* Likewise.  */
 
 
-/* Macros for SDB and Dwarf Output.  */
+/* Macros for Dwarf Output.  */
 
 /* Define this macro if GCC should produce dwarf version 2 format debugging
    output in response to the `-g' option.  */
diff --git a/gcc/config/m68k/m68kelf.h b/gcc/config/m68k/m68kelf.h
index fb1a0a4d917..159223f64c7 100644
--- a/gcc/config/m68k/m68kelf.h
+++ b/gcc/config/m68k/m68kelf.h
@@ -97,7 +97,7 @@ do {								\
 
 /* Define how the m68k registers should be numbered for Dwarf output.
    The numbering provided here should be compatible with the native
-   SVR4 SDB debugger in the m68k/SVR4 reference port, where d0-d7
+   SVR4 debugger in the m68k/SVR4 reference port, where d0-d7
    are 0-7, a0-a8 are 8-15, and fp0-fp7 are 16-23.  */
 
 #undef DBX_REGISTER_NUMBER
diff --git a/gcc/config/mips/mips.h b/gcc/config/mips/mips.h
index 550d283158e..f5c28bf70e3 100644
--- a/gcc/config/mips/mips.h
+++ b/gcc/config/mips/mips.h
@@ -1314,9 +1314,7 @@ struct mips_cpu_info {
 %{g} %{g0} %{g1} %{g2} %{g3} \
 %{ggdb:-g} %{ggdb0:-g0} %{ggdb1:-g1} %{ggdb2:-g2} %{ggdb3:-g3} \
 %{gstabs:-g} %{gstabs0:-g0} %{gstabs1:-g1} %{gstabs2:-g2} %{gstabs3:-g3} \
-%{gstabs+:-g} %{gstabs+0:-g0} %{gstabs+1:-g1} %{gstabs+2:-g2} %{gstabs+3:-g3} \
-%{gcoff:-g} %{gcoff0:-g0} %{gcoff1:-g1} %{gcoff2:-g2} %{gcoff3:-g3} \
-%{gcoff*:-mdebug} %{!gcoff*:-no-mdebug}"
+%{gstabs+:-g} %{gstabs+0:-g0} %{gstabs+1:-g1} %{gstabs+2:-g2} %{gstabs+3:-g3}"
 #endif
 
 /* FP_ASM_SPEC represents the floating-point options that must be passed
diff --git a/gcc/config/mmix/mmix.h b/gcc/config/mmix/mmix.h
index 5dafe2dbf98..2ee3592f3c8 100644
--- a/gcc/config/mmix/mmix.h
+++ b/gcc/config/mmix/mmix.h
@@ -761,7 +761,7 @@ typedef struct { int regs; int lib; } CUMULATIVE_ARGS;
 /* (empty) */
 
 
-/* Node: SDB and DWARF */
+/* Node: DWARF */
 #define DWARF2_DEBUGGING_INFO 1
 #define DWARF2_ASM_LINE_DEBUG_INFO 1
 
diff --git a/gcc/config/msp430/msp430.c b/gcc/config/msp430/msp430.c
index 8b025c755bf..04fe58ed2fc 100644
--- a/gcc/config/msp430/msp430.c
+++ b/gcc/config/msp430/msp430.c
@@ -753,6 +753,10 @@ hwmult_name (unsigned int val)
 static void
 msp430_option_override (void)
 {
+  /* The MSP430 architecture can safely dereference a NULL pointer. In fact,
+  there are memory mapped registers there.  */
+  flag_delete_null_pointer_checks = 0;
+
   init_machine_status = msp430_init_machine_status;
 
   if (target_cpu)
diff --git a/gcc/config/nds32/nds32.c b/gcc/config/nds32/nds32.c
index add64ee4a80..6657b354a4b 100644
--- a/gcc/config/nds32/nds32.c
+++ b/gcc/config/nds32/nds32.c
@@ -3765,7 +3765,7 @@ nds32_target_alignment (rtx_insn *label)
 
 /* -- File Names in DBX Format.  */
 
-/* -- Macros for SDB and DWARF Output.  */
+/* -- Macros for DWARF Output.  */
 
 /* -- Macros for VMS Debug Format.  */
 
diff --git a/gcc/config/nios2/constraints.md b/gcc/config/nios2/constraints.md
index c6c539265ac..51f71cf742e 100644
--- a/gcc/config/nios2/constraints.md
+++ b/gcc/config/nios2/constraints.md
@@ -95,8 +95,8 @@
        (match_test "TARGET_ARCH_R2 && ANDCLEAR_INT (ival)")))
 
 (define_constraint "S"
-  "An immediate stored in small data, accessible by GP."
-  (match_test "gprel_constant_p (op)"))
+  "An immediate stored in small data, accessible by GP, or by offset from r0."
+  (match_test "gprel_constant_p (op) || r0rel_constant_p (op)"))
 
 (define_constraint "T"
   "A constant unspec offset representing a relocation."
diff --git a/gcc/config/nios2/nios2-protos.h b/gcc/config/nios2/nios2-protos.h
index 4478334970c..84d450bfe94 100644
--- a/gcc/config/nios2/nios2-protos.h
+++ b/gcc/config/nios2/nios2-protos.h
@@ -30,6 +30,11 @@ extern bool nios2_expand_return (void);
 extern void nios2_function_profiler (FILE *, int);
 
 #ifdef RTX_CODE
+extern bool nios2_large_constant_p (rtx);
+extern bool nios2_symbolic_memory_operand_p (rtx);
+
+extern rtx nios2_split_large_constant (rtx, rtx);
+extern rtx nios2_split_symbolic_memory_operand (rtx);
 extern bool nios2_emit_move_sequence (rtx *, machine_mode);
 extern void nios2_emit_expensive_div (rtx *, machine_mode);
 extern void nios2_adjust_call_address (rtx *, rtx);
@@ -47,6 +52,7 @@ extern const char * nios2_add_insn_asm (rtx_insn *, rtx *);
 
 extern bool nios2_legitimate_pic_operand_p (rtx);
 extern bool gprel_constant_p (rtx);
+extern bool r0rel_constant_p (rtx);
 extern bool nios2_regno_ok_for_base_p (int, bool);
 extern bool nios2_unspec_reloc_p (rtx);
 
diff --git a/gcc/config/nios2/nios2.c b/gcc/config/nios2/nios2.c
index 2a23886da48..f0bc668edd1 100644
--- a/gcc/config/nios2/nios2.c
+++ b/gcc/config/nios2/nios2.c
@@ -50,11 +50,14 @@
 #include "langhooks.h"
 #include "stor-layout.h"
 #include "builtins.h"
+#include "tree-pass.h"
+#include "xregex.h"
 
 /* This file should be included last.  */
 #include "target-def.h"
 
 /* Forward function declarations.  */
+static bool nios2_symbolic_constant_p (rtx);
 static bool prologue_saved_reg_p (unsigned);
 static void nios2_load_pic_register (void);
 static void nios2_register_custom_code (unsigned int, enum nios2_ccs_code, int);
@@ -62,6 +65,7 @@ static const char *nios2_unspec_reloc_name (int);
 static void nios2_register_builtin_fndecl (unsigned, tree);
 static rtx nios2_ldst_parallel (bool, bool, bool, rtx, int,
 				unsigned HOST_WIDE_INT, bool);
+static int nios2_address_cost (rtx, machine_mode, addr_space_t, bool);
 
 /* Threshold for data being put into the small data/bss area, instead
    of the normal data area (references to the small data/bss area take
@@ -102,6 +106,10 @@ static int custom_code_index[256];
 /* Set to true if any conflicts (re-use of a code between 0-255) are found.  */
 static bool custom_code_conflict = false;
 
+/* State for command-line options.  */
+regex_t nios2_gprel_sec_regex;
+regex_t nios2_r0rel_sec_regex;
+
 
 /* Definition of builtin function types for nios2.  */
 
@@ -1108,7 +1116,9 @@ nios2_initial_elimination_offset (int from, int to)
   switch (from)
     {
     case FRAME_POINTER_REGNUM:
-      offset = cfun->machine->args_size;
+      /* This is the high end of the local variable storage, not the
+	 hard frame pointer.  */
+      offset = cfun->machine->args_size + cfun->machine->var_size;
       break;
 
     case ARG_POINTER_REGNUM:
@@ -1370,6 +1380,31 @@ nios2_option_override (void)
 	nios2_gpopt_option = gpopt_local;
     }
 
+  /* GP-relative and r0-relative addressing don't make sense for PIC.  */
+  if (flag_pic)
+    {
+      if (nios2_gpopt_option != gpopt_none)
+	error ("-mgpopt not supported with PIC.");
+      if (nios2_gprel_sec)
+	error ("-mgprel-sec= not supported with PIC.");
+      if (nios2_r0rel_sec)
+	error ("-mr0rel-sec= not supported with PIC.");
+    }
+
+  /* Process -mgprel-sec= and -m0rel-sec=.  */
+  if (nios2_gprel_sec)
+    {
+      if (regcomp (&nios2_gprel_sec_regex, nios2_gprel_sec, 
+		   REG_EXTENDED | REG_NOSUB))
+	error ("-mgprel-sec= argument is not a valid regular expression.");
+    }
+  if (nios2_r0rel_sec)
+    {
+      if (regcomp (&nios2_r0rel_sec_regex, nios2_r0rel_sec, 
+		   REG_EXTENDED | REG_NOSUB))
+	error ("-mr0rel-sec= argument is not a valid regular expression.");
+    }
+
   /* If we don't have mul, we don't have mulx either!  */
   if (!TARGET_HAS_MUL && TARGET_HAS_MULX)
     target_flags &= ~MASK_HAS_MULX;
@@ -1430,29 +1465,25 @@ nios2_simple_const_p (const_rtx cst)
    cost has been computed, and false if subexpressions should be
    scanned.  In either case, *TOTAL contains the cost result.  */
 static bool
-nios2_rtx_costs (rtx x, machine_mode mode ATTRIBUTE_UNUSED,
-		 int outer_code ATTRIBUTE_UNUSED,
-		 int opno ATTRIBUTE_UNUSED,
-		 int *total, bool speed ATTRIBUTE_UNUSED)
+nios2_rtx_costs (rtx x, machine_mode mode,
+		 int outer_code,
+		 int opno,
+		 int *total, bool speed)
 {
   int code = GET_CODE (x);
 
   switch (code)
     {
       case CONST_INT:
-        if (INTVAL (x) == 0)
+        if (INTVAL (x) == 0 || nios2_simple_const_p (x))
           {
             *total = COSTS_N_INSNS (0);
             return true;
           }
-        else if (nios2_simple_const_p (x))
-          {
-            *total = COSTS_N_INSNS (2);
-            return true;
-          }
         else
           {
-            *total = COSTS_N_INSNS (4);
+	    /* High + lo_sum.  */
+            *total = COSTS_N_INSNS (1);
             return true;
           }
 
@@ -1460,10 +1491,30 @@ nios2_rtx_costs (rtx x, machine_mode mode ATTRIBUTE_UNUSED,
       case SYMBOL_REF:
       case CONST:
       case CONST_DOUBLE:
-        {
-          *total = COSTS_N_INSNS (4);
-          return true;
-        }
+	if (gprel_constant_p (x) || r0rel_constant_p (x))
+          {
+            *total = COSTS_N_INSNS (1);
+            return true;
+          }
+	else
+	  {
+	    /* High + lo_sum.  */
+	    *total = COSTS_N_INSNS (1);
+	    return true;
+	  }
+
+      case HIGH:
+	{
+	  /* This is essentially a constant.  */
+	  *total = COSTS_N_INSNS (0);
+	  return true;
+	}
+
+      case LO_SUM:
+	{
+	  *total = COSTS_N_INSNS (0);
+	  return true;
+	}
 
       case AND:
 	{
@@ -1477,29 +1528,83 @@ nios2_rtx_costs (rtx x, machine_mode mode ATTRIBUTE_UNUSED,
 	  return false;
 	}
 
+      /* For insns that have an execution latency (3 cycles), don't
+	 penalize by the full amount since we can often schedule
+	 to avoid it.  */
       case MULT:
         {
-          *total = COSTS_N_INSNS (1);
+	  if (!TARGET_HAS_MUL)
+	    *total = COSTS_N_INSNS (5);  /* Guess?  */
+	  else if (speed)
+	    *total = COSTS_N_INSNS (2);  /* Latency adjustment.  */
+	  else 
+	    *total = COSTS_N_INSNS (1);
           return false;
         }
-      case SIGN_EXTEND:
+
+      case DIV:
         {
-          *total = COSTS_N_INSNS (3);
+	  if (!TARGET_HAS_DIV)
+	    *total = COSTS_N_INSNS (5);  /* Guess?  */
+	  else if (speed)
+	    *total = COSTS_N_INSNS (2);  /* Latency adjustment.  */
+	  else 
+	    *total = COSTS_N_INSNS (1);
           return false;
         }
-      case ZERO_EXTEND:
+
+      case ASHIFT:
+      case ASHIFTRT:
+      case LSHIFTRT:
+      case ROTATE:
         {
-          *total = COSTS_N_INSNS (1);
+	  if (!speed)
+	    *total = COSTS_N_INSNS (1);
+	  else 
+	    *total = COSTS_N_INSNS (2);  /* Latency adjustment.  */
           return false;
         }
+	
+      case ZERO_EXTRACT:
+	if (TARGET_HAS_BMX)
+	  {
+	    *total = COSTS_N_INSNS (1);
+	    return true;
+	  }
+	return false;
 
-    case ZERO_EXTRACT:
-      if (TARGET_HAS_BMX)
+      case SIGN_EXTEND:
+        {
+	  if (MEM_P (XEXP (x, 0)))
+	    *total = COSTS_N_INSNS (1);
+	  else
+	    *total = COSTS_N_INSNS (3);
+	  return false;
+	}
+
+      case MEM:
 	{
-          *total = COSTS_N_INSNS (1);
-          return true;
+	  rtx addr = XEXP (x, 0);
+
+	  /* Account for cost of different addressing modes.  */
+	  *total = nios2_address_cost (addr, mode, ADDR_SPACE_GENERIC, speed);
+
+	  if (outer_code == SET && opno == 0)
+	    /* Stores execute in 1 cycle accounted for by
+	       the outer SET.  */
+	    ;
+	  else if (outer_code == SET || outer_code == SIGN_EXTEND
+		   || outer_code == ZERO_EXTEND)
+	    /* Latency adjustment.  */
+	    {
+	      if (speed)
+		*total += COSTS_N_INSNS (1);
+	    }
+	  else
+	    /* This is going to have to be split into a load.  */
+	    *total += COSTS_N_INSNS (speed ? 2 : 1);
+	  return true;
 	}
-      return false;
 
       default:
         return false;
@@ -1904,7 +2009,53 @@ nios2_validate_compare (machine_mode mode, rtx *cmp, rtx *op1, rtx *op2)
 }
 
 
-/* Addressing Modes.  */
+/* Addressing modes and constants.  */
+
+/* Symbolic constants are split into high/lo_sum pairs during the 
+   split1 pass.  After that, they are not considered legitimate addresses.
+   This function returns true if in a pre-split context where these
+   constants are allowed.  */
+static bool
+nios2_symbolic_constant_allowed (void)
+{
+  /* The reload_completed check is for the benefit of
+     nios2_asm_output_mi_thunk and perhaps other places that try to
+     emulate a post-reload pass.  */
+  return !(cfun->curr_properties & PROP_rtl_split_insns) && !reload_completed;
+}
+
+/* Return true if X is constant expression with a reference to an
+   "ordinary" symbol; not GOT-relative, not GP-relative, not TLS.  */
+static bool
+nios2_symbolic_constant_p (rtx x)
+{
+  rtx base, offset;
+
+  if (flag_pic)
+    return false;
+  if (GET_CODE (x) == LABEL_REF)
+    return true;
+  else if (CONSTANT_P (x))
+    {
+      split_const (x, &base, &offset);
+      return (SYMBOL_REF_P (base)
+		&& !SYMBOL_REF_TLS_MODEL (base)
+		&& !gprel_constant_p (base)
+		&& !r0rel_constant_p (base)
+		&& SMALL_INT (INTVAL (offset)));
+    }
+  return false;
+}
+
+/* Return true if X is an expression of the form 
+   (PLUS reg symbolic_constant).  */
+static bool
+nios2_plus_symbolic_constant_p (rtx x)
+{
+  return (GET_CODE (x) == PLUS
+	  && REG_P (XEXP (x, 0))
+	  && nios2_symbolic_constant_p (XEXP (x, 1)));
+}
 
 /* Implement TARGET_LEGITIMATE_CONSTANT_P.  */
 static bool
@@ -1973,6 +2124,8 @@ nios2_valid_addr_expr_p (rtx base, rtx offset, bool strict_p)
 	  && nios2_regno_ok_for_base_p (REGNO (base), strict_p)
 	  && (offset == NULL_RTX
 	      || nios2_valid_addr_offset_p (offset)
+	      || (nios2_symbolic_constant_allowed () 
+		  && nios2_symbolic_constant_p (offset))
 	      || nios2_unspec_reloc_p (offset)));
 }
 
@@ -1990,11 +2143,16 @@ nios2_legitimate_address_p (machine_mode mode ATTRIBUTE_UNUSED,
 
       /* Else, fall through.  */
     case CONST:
-      if (gprel_constant_p (operand))
+      if (gprel_constant_p (operand) || r0rel_constant_p (operand))
 	return true;
 
       /* Else, fall through.  */
     case LABEL_REF:
+      if (nios2_symbolic_constant_allowed () 
+	  && nios2_symbolic_constant_p (operand))
+	return true;
+
+      /* Else, fall through.  */
     case CONST_INT:
     case CONST_DOUBLE:
       return false;
@@ -2009,9 +2167,28 @@ nios2_legitimate_address_p (machine_mode mode ATTRIBUTE_UNUSED,
         rtx op0 = XEXP (operand, 0);
         rtx op1 = XEXP (operand, 1);
 
-	return (nios2_valid_addr_expr_p (op0, op1, strict_p)
-		|| nios2_valid_addr_expr_p (op1, op0, strict_p));
+	if (nios2_valid_addr_expr_p (op0, op1, strict_p) 
+	    || nios2_valid_addr_expr_p (op1, op0, strict_p))
+	  return true;
       }
+      break;
+
+      /* %lo(constant)(reg)
+	 This requires a 16-bit relocation and isn't valid with R2
+	 io-variant load/stores.  */
+    case LO_SUM:
+      if (TARGET_ARCH_R2 
+	  && (TARGET_BYPASS_CACHE || TARGET_BYPASS_CACHE_VOLATILE))
+	return false;
+      else
+	{
+	  rtx op0 = XEXP (operand, 0);
+	  rtx op1 = XEXP (operand, 1);
+
+	  return (REG_P (op0)
+		  && nios2_regno_ok_for_base_p (REGNO (op0), strict_p)
+		  && nios2_large_constant_p (op1));
+	}
 
     default:
       break;
@@ -2019,6 +2196,106 @@ nios2_legitimate_address_p (machine_mode mode ATTRIBUTE_UNUSED,
   return false;
 }
 
+/* Implement TARGET_ADDRESS_COST.
+   Experimentation has shown that we get better code by penalizing the
+   the (plus reg symbolic_constant) and (plus reg (const ...)) forms
+   but giving (plus reg symbol_ref) address modes the same cost as those
+   that don't require splitting.  Also, from a theoretical point of view:
+   - This is in line with the recommendation in the GCC internals 
+     documentation to make address forms involving multiple
+     registers more expensive than single-register forms.  
+   - OTOH it still encourages fwprop1 to propagate constants into 
+     address expressions more aggressively.
+   - We should discourage splitting (symbol + offset) into hi/lo pairs
+     to allow CSE'ing the symbol when it's used with more than one offset,
+     but not so heavily as to avoid this addressing mode at all.  */
+static int
+nios2_address_cost (rtx address, 
+		    machine_mode mode ATTRIBUTE_UNUSED,
+		    addr_space_t as ATTRIBUTE_UNUSED, 
+		    bool speed ATTRIBUTE_UNUSED)
+{
+  if (nios2_plus_symbolic_constant_p (address))
+    return COSTS_N_INSNS (1);
+  if (nios2_symbolic_constant_p (address))
+    {
+      if (GET_CODE (address) == CONST)
+	return COSTS_N_INSNS (1);
+      else
+	return COSTS_N_INSNS (0);
+    }
+  return COSTS_N_INSNS (0);
+}
+
+/* Return true if X is a MEM whose address expression involves a symbolic
+   constant.  */
+bool
+nios2_symbolic_memory_operand_p (rtx x)
+{
+  rtx addr;
+
+  if (GET_CODE (x) != MEM)
+    return false;
+  addr = XEXP (x, 0);
+
+  return (nios2_symbolic_constant_p (addr)
+	  || nios2_plus_symbolic_constant_p (addr));
+}
+
+
+/* Return true if X is something that needs to be split into a 
+   high/lo_sum pair.  */
+bool
+nios2_large_constant_p (rtx x)
+{
+  return (nios2_symbolic_constant_p (x)
+	  || nios2_large_unspec_reloc_p (x));
+}
+
+/* Given an RTX X that satisfies nios2_large_constant_p, split it into
+   high and lo_sum parts using TEMP as a scratch register.  Emit the high 
+   instruction and return the lo_sum expression.  */
+rtx
+nios2_split_large_constant (rtx x, rtx temp)
+{
+  emit_insn (gen_rtx_SET (temp, gen_rtx_HIGH (Pmode, copy_rtx (x))));
+  return gen_rtx_LO_SUM (Pmode, temp, copy_rtx (x));
+}
+
+/* Split an RTX of the form
+     (plus op0 op1)
+   where op1 is a large constant into
+     (set temp (high op1))
+     (set temp (plus op0 temp))
+     (lo_sum temp op1)
+   returning the lo_sum expression as the value.  */
+static rtx
+nios2_split_plus_large_constant (rtx op0, rtx op1)
+{
+  rtx temp = gen_reg_rtx (Pmode);
+  op0 = force_reg (Pmode, op0);
+
+  emit_insn (gen_rtx_SET (temp, gen_rtx_HIGH (Pmode, copy_rtx (op1))));
+  emit_insn (gen_rtx_SET (temp, gen_rtx_PLUS (Pmode, op0, temp)));
+  return gen_rtx_LO_SUM (Pmode, temp, copy_rtx (op1));
+}
+
+/* Given a MEM OP with an address that includes a splittable symbol,
+   emit some instructions to do the split and return a new MEM.  */
+rtx
+nios2_split_symbolic_memory_operand (rtx op)
+{
+  rtx addr = XEXP (op, 0);
+
+  if (nios2_symbolic_constant_p (addr))
+    addr = nios2_split_large_constant (addr, gen_reg_rtx (Pmode));
+  else if (nios2_plus_symbolic_constant_p (addr))
+    addr = nios2_split_plus_large_constant (XEXP (addr, 0), XEXP (addr, 1));
+  else
+    gcc_unreachable ();
+  return replace_equiv_address (op, addr, false);
+}
+
 /* Return true if SECTION is a small section name.  */
 static bool
 nios2_small_section_name_p (const char *section)
@@ -2026,7 +2303,17 @@ nios2_small_section_name_p (const char *section)
   return (strcmp (section, ".sbss") == 0
 	  || strncmp (section, ".sbss.", 6) == 0
 	  || strcmp (section, ".sdata") == 0
-	  || strncmp (section, ".sdata.", 7) == 0);
+	  || strncmp (section, ".sdata.", 7) == 0
+	  || (nios2_gprel_sec 
+	      && regexec (&nios2_gprel_sec_regex, section, 0, NULL, 0) == 0));
+}
+
+/* Return true if SECTION is a r0-relative section name.  */
+static bool
+nios2_r0rel_section_name_p (const char *section)
+{
+  return (nios2_r0rel_sec 
+	  && regexec (&nios2_r0rel_sec_regex, section, 0, NULL, 0) == 0);
 }
 
 /* Return true if EXP should be placed in the small data section.  */
@@ -2135,6 +2422,33 @@ nios2_symbol_ref_in_small_data_p (rtx sym)
     }
 }
 
+/* Likewise for r0-relative addressing.  */
+static bool
+nios2_symbol_ref_in_r0rel_data_p (rtx sym)
+{
+  tree decl;
+
+  gcc_assert (GET_CODE (sym) == SYMBOL_REF);
+  decl = SYMBOL_REF_DECL (sym);
+
+  /* TLS variables are not accessed through r0.  */
+  if (SYMBOL_REF_TLS_MODEL (sym) != 0)
+    return false;
+
+  /* On Nios II R2, there is no r0-relative relocation that can be
+     used with "io" instructions.  So, if we are implicitly generating
+     those instructions, we cannot emit r0-relative accesses.  */
+  if (TARGET_ARCH_R2
+      && (TARGET_BYPASS_CACHE || TARGET_BYPASS_CACHE_VOLATILE))
+    return false;
+
+  /* If the user has explicitly placed the symbol in a r0rel section
+     via an attribute, generate r0-relative addressing.  */
+  if (decl && DECL_SECTION_NAME (decl))
+    return nios2_r0rel_section_name_p (DECL_SECTION_NAME (decl));
+  return false;
+}
+
 /* Implement TARGET_SECTION_TYPE_FLAGS.  */
 
 static unsigned int
@@ -2221,6 +2535,9 @@ nios2_legitimize_constant_address (rtx addr)
     base = nios2_legitimize_tls_address (base);
   else if (flag_pic)
     base = nios2_load_pic_address (base, UNSPEC_PIC_SYM, NULL_RTX);
+  else if (!nios2_symbolic_constant_allowed () 
+	   && nios2_symbolic_constant_p (addr))
+    return nios2_split_large_constant (addr, gen_reg_rtx (Pmode));
   else
     return addr;
 
@@ -2241,9 +2558,35 @@ static rtx
 nios2_legitimize_address (rtx x, rtx oldx ATTRIBUTE_UNUSED,
 			  machine_mode mode ATTRIBUTE_UNUSED)
 {
+  rtx op0, op1;
+  
   if (CONSTANT_P (x))
     return nios2_legitimize_constant_address (x);
 
+  /* Remaining cases all involve something + a constant.  */
+  if (GET_CODE (x) != PLUS)
+    return x;
+
+  op0 = XEXP (x, 0);
+  op1 = XEXP (x, 1);
+
+  /* Target-independent code turns (exp + constant) into plain
+     register indirect.  Although subsequent optimization passes will
+     eventually sort that out, ivopts uses the unoptimized form for
+     computing its cost model, so we get better results by generating
+     the correct form from the start.  */
+  if (nios2_valid_addr_offset_p (op1))
+    return gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), copy_rtx (op1));
+
+  /* We may need to split symbolic constants now.  */
+  else if (nios2_symbolic_constant_p (op1))
+    {
+      if (nios2_symbolic_constant_allowed ())
+	return gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), copy_rtx (op1));
+      else
+	return nios2_split_plus_large_constant (op0, op1);
+    }
+
   /* For the TLS LE (Local Exec) model, the compiler may try to
      combine constant offsets with unspec relocs, creating address RTXs
      looking like this:
@@ -2266,20 +2609,19 @@ nios2_legitimize_address (rtx x, rtx oldx ATTRIBUTE_UNUSED,
                               (const_int 48 [0x30])))] UNSPEC_ADD_TLS_LE)))
 
      Which will be output as '%tls_le(var+48)(r23)' in assembly.  */
-  if (GET_CODE (x) == PLUS
-      && GET_CODE (XEXP (x, 1)) == CONST)
+  else if (GET_CODE (op1) == CONST)
     {
       rtx unspec, offset;
-      split_const (XEXP (x, 1), &unspec, &offset);
+      split_const (op1, &unspec, &offset);
       if (GET_CODE (unspec) == UNSPEC
 	  && !nios2_large_offset_p (XINT (unspec, 1))
 	  && offset != const0_rtx)
 	{
-	  rtx reg = force_reg (Pmode, XEXP (x, 0));
+	  rtx reg = force_reg (Pmode, op0);
 	  unspec = copy_rtx (unspec);
 	  XVECEXP (unspec, 0, 0)
 	    = plus_constant (Pmode, XVECEXP (unspec, 0, 0), INTVAL (offset));
-	  x = gen_rtx_PLUS (Pmode, reg, gen_rtx_CONST (Pmode, unspec));
+	  return gen_rtx_PLUS (Pmode, reg, gen_rtx_CONST (Pmode, unspec));
 	}
     }
 
@@ -2340,10 +2682,29 @@ nios2_emit_move_sequence (rtx *operands, machine_mode mode)
 	      return true;
 	    }
 	}
-      else if (!gprel_constant_p (from))
+      else if (gprel_constant_p (from) || r0rel_constant_p (from))
+	/* Handled directly by movsi_internal as gp + offset 
+	   or r0 + offset.  */
+	;
+      else if (nios2_large_constant_p (from))
+	/* This case covers either a regular symbol reference or an UNSPEC
+	   representing a 32-bit offset.  We split the former 
+	   only conditionally and the latter always.  */
 	{
-	  if (!nios2_large_unspec_reloc_p (from))
-	    from = nios2_legitimize_constant_address (from);
+	  if (!nios2_symbolic_constant_allowed () 
+	      || nios2_large_unspec_reloc_p (from))
+	    {
+	      rtx lo = nios2_split_large_constant (from, to);
+	      emit_insn (gen_rtx_SET (to, lo));
+	      set_unique_reg_note (get_last_insn (), REG_EQUAL,
+				   copy_rtx (operands[1]));
+	      return true;
+	    }
+	}
+      else 
+	/* This is a TLS or PIC symbol.  */
+	{
+	  from = nios2_legitimize_constant_address (from);
 	  if (CONSTANT_P (from))
 	    {
 	      emit_insn (gen_rtx_SET (to,
@@ -2654,6 +3015,7 @@ nios2_print_operand (FILE *file, rtx op, int letter)
       break;
     }
 
+  debug_rtx (op);
   output_operand_lossage ("Unsupported operand for code '%c'", letter);
   gcc_unreachable ();
 }
@@ -2672,6 +3034,20 @@ gprel_constant_p (rtx op)
   return false;
 }
 
+/* Likewise if this is a zero-relative accessible reference.  */
+bool
+r0rel_constant_p (rtx op)
+{
+  if (GET_CODE (op) == SYMBOL_REF
+      && nios2_symbol_ref_in_r0rel_data_p (op))
+    return true;
+  else if (GET_CODE (op) == CONST
+           && GET_CODE (XEXP (op, 0)) == PLUS)
+    return r0rel_constant_p (XEXP (XEXP (op, 0), 0));
+
+  return false;
+}
+
 /* Return the name string for a supported unspec reloc offset.  */
 static const char *
 nios2_unspec_reloc_name (int unspec)
@@ -2736,7 +3112,13 @@ nios2_print_operand_address (FILE *file, machine_mode mode, rtx op)
           fprintf (file, ")(%s)", reg_names[GP_REGNO]);
           return;
         }
-
+      else if (r0rel_constant_p (op))
+        {
+          fprintf (file, "%%lo(");
+          output_addr_const (file, op);
+          fprintf (file, ")(r0)");
+          return;
+        }
       break;
 
     case PLUS:
@@ -2759,6 +3141,20 @@ nios2_print_operand_address (FILE *file, machine_mode mode, rtx op)
       }
       break;
 
+    case LO_SUM:
+      {
+        rtx op0 = XEXP (op, 0);
+        rtx op1 = XEXP (op, 1);
+
+	if (REG_P (op0) && CONSTANT_P (op1))
+	  {
+	    nios2_print_operand (file, op1, 'L');
+	    fprintf (file, "(%s)", reg_names[REGNO (op0)]);
+	    return;
+	  }
+      }
+      break;
+
     case REG:
       fprintf (file, "0(%s)", reg_names[REGNO (op)]);
       return;
@@ -4328,9 +4724,12 @@ nios2_cdx_narrow_form_p (rtx_insn *insn)
 		|| TARGET_BYPASS_CACHE)
 	      return false;
 	    addr = XEXP (mem, 0);
-	    /* GP-based references are never narrow.  */
-	    if (gprel_constant_p (addr))
+	    /* GP-based and R0-based references are never narrow.  */
+	    if (gprel_constant_p (addr) || r0rel_constant_p (addr))
 		return false;
+	    /* %lo requires a 16-bit relocation and is never narrow.  */
+	    if (GET_CODE (addr) == LO_SUM)
+	      return false;
 	    ret = split_mem_address (addr, &rhs1, &rhs2);
 	    gcc_assert (ret);
 	  }
@@ -4372,8 +4771,11 @@ nios2_cdx_narrow_form_p (rtx_insn *insn)
 	      || TARGET_BYPASS_CACHE)
 	    return false;
 	  addr = XEXP (mem, 0);
-	  /* GP-based references are never narrow.  */
-	  if (gprel_constant_p (addr))
+	  /* GP-based and r0-based references are never narrow.  */
+	  if (gprel_constant_p (addr) || r0rel_constant_p (addr))
+	    return false;
+	  /* %lo requires a 16-bit relocation and is never narrow.  */
+	  if (GET_CODE (addr) == LO_SUM)
 	    return false;
 	  ret = split_mem_address (addr, &rhs1, &rhs2);
 	  gcc_assert (ret);
@@ -5054,15 +5456,15 @@ nios2_adjust_reg_alloc_order (void)
 #undef TARGET_LEGITIMATE_ADDRESS_P
 #define TARGET_LEGITIMATE_ADDRESS_P nios2_legitimate_address_p
 
-#undef TARGET_LRA_P
-#define TARGET_LRA_P hook_bool_void_false
-
 #undef TARGET_PREFERRED_RELOAD_CLASS
 #define TARGET_PREFERRED_RELOAD_CLASS nios2_preferred_reload_class
 
 #undef TARGET_RTX_COSTS
 #define TARGET_RTX_COSTS nios2_rtx_costs
 
+#undef TARGET_ADDRESS_COST
+#define TARGET_ADDRESS_COST nios2_address_cost
+
 #undef TARGET_HAVE_TLS
 #define TARGET_HAVE_TLS TARGET_LINUX_ABI
 
diff --git a/gcc/config/nios2/nios2.h b/gcc/config/nios2/nios2.h
index 420543e4f46..9fdff024cd8 100644
--- a/gcc/config/nios2/nios2.h
+++ b/gcc/config/nios2/nios2.h
@@ -252,6 +252,7 @@ enum reg_class
 
 /* Stack layout.  */
 #define STACK_GROWS_DOWNWARD 1
+#define FRAME_GROWS_DOWNWARD 1
 #define FIRST_PARM_OFFSET(FUNDECL) 0
 
 /* Before the prologue, RA lives in r31.  */
diff --git a/gcc/config/nios2/nios2.md b/gcc/config/nios2/nios2.md
index 206ebce1c46..ef2883f2516 100644
--- a/gcc/config/nios2/nios2.md
+++ b/gcc/config/nios2/nios2.md
@@ -201,7 +201,7 @@
   "addi\\t%0, %1, %L2"
   [(set_attr "type" "alu")])
 
-(define_insn "movqi_internal"
+(define_insn_and_split "movqi_internal"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=m, r,r")
         (match_operand:QI 1 "general_operand"       "rM,m,rI"))]
   "(register_operand (operands[0], QImode)
@@ -224,20 +224,47 @@
 	gcc_unreachable ();
       }
   }
+  "(nios2_symbolic_memory_operand_p (operands[0]) 
+   || nios2_symbolic_memory_operand_p (operands[1]))"
+  [(set (match_dup 0) (match_dup 1))]
+  {
+    if (nios2_symbolic_memory_operand_p (operands[0]))
+      operands[0] = nios2_split_symbolic_memory_operand (operands[0]);
+    else
+      operands[1] = nios2_split_symbolic_memory_operand (operands[1]);
+  }
   [(set_attr "type" "st,ld,mov")])
 
-(define_insn "movhi_internal"
+(define_insn_and_split "movhi_internal"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=m, r,r")
         (match_operand:HI 1 "general_operand"       "rM,m,rI"))]
   "(register_operand (operands[0], HImode)
     || reg_or_0_operand (operands[1], HImode))"
-  "@
-    sth%o0%.\\t%z1, %0
-    ldhu%o1%.\\t%0, %1
-    mov%i1%.\\t%0, %z1"
+  {
+    switch (which_alternative)
+      {
+      case 0:
+        return "sth%o0%.\\t%z1, %0";
+      case 1:
+        return "ldhu%o1%.\\t%0, %1";
+      case 2:
+        return "mov%i1%.\\t%0, %z1";
+      default:
+	gcc_unreachable ();
+      }
+  }
+  "(nios2_symbolic_memory_operand_p (operands[0]) 
+   || nios2_symbolic_memory_operand_p (operands[1]))"
+  [(set (match_dup 0) (match_dup 1))]
+  {
+    if (nios2_symbolic_memory_operand_p (operands[0]))
+      operands[0] = nios2_split_symbolic_memory_operand (operands[0]);
+    else
+      operands[1] = nios2_split_symbolic_memory_operand (operands[1]);
+  }
   [(set_attr "type" "st,ld,mov")])
 
-(define_insn "movsi_internal"
+(define_insn_and_split "movsi_internal"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=m, r,r,   r")
         (match_operand:SI 1 "general_operand"       "rM,m,rIJK,S"))]
   "(register_operand (operands[0], SImode)
@@ -269,6 +296,18 @@
 	gcc_unreachable ();
       }
   }
+  "(nios2_symbolic_memory_operand_p (operands[0]) 
+    || nios2_symbolic_memory_operand_p (operands[1])
+    || nios2_large_constant_p (operands[1]))"
+  [(set (match_dup 0) (match_dup 1))]
+  {
+    if (nios2_symbolic_memory_operand_p (operands[0]))
+      operands[0] = nios2_split_symbolic_memory_operand (operands[0]);
+    else if (nios2_symbolic_memory_operand_p (operands[1]))
+      operands[1] = nios2_split_symbolic_memory_operand (operands[1]);
+    else
+      operands[1] = nios2_split_large_constant (operands[1], operands[0]);
+  }
   [(set_attr "type" "st,ld,mov,alu")])
 
 (define_mode_iterator BH [QI HI])
@@ -318,42 +357,62 @@
 (define_mode_iterator QX [HI SI])
 
 ;; Zero extension patterns
-(define_insn "zero_extendhisi2"
+(define_insn_and_split "zero_extendhisi2"
   [(set (match_operand:SI 0 "register_operand" "=r,r")
         (zero_extend:SI (match_operand:HI 1 "nonimmediate_operand" "r,m")))]
   ""
   "@
     andi%.\\t%0, %1, 0xffff
     ldhu%o1%.\\t%0, %1"
+  "nios2_symbolic_memory_operand_p (operands[1])"
+  [(set (match_dup 0) (zero_extend:SI (match_dup 1)))]
+  {
+    operands[1] = nios2_split_symbolic_memory_operand (operands[1]);
+  }
   [(set_attr "type"     "and,ld")])
 
-(define_insn "zero_extendqi<mode>2"
+(define_insn_and_split "zero_extendqi<mode>2"
   [(set (match_operand:QX 0 "register_operand" "=r,r")
         (zero_extend:QX (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
   ""
   "@
     andi%.\\t%0, %1, 0xff
     ldbu%o1%.\\t%0, %1"
+  "nios2_symbolic_memory_operand_p (operands[1])"
+  [(set (match_dup 0) (zero_extend:QX (match_dup 1)))]
+  {
+    operands[1] = nios2_split_symbolic_memory_operand (operands[1]);
+  }
   [(set_attr "type"     "and,ld")])
 
 ;; Sign extension patterns
 
-(define_insn "extendhisi2"
+(define_insn_and_split "extendhisi2"
   [(set (match_operand:SI 0 "register_operand"                     "=r,r")
         (sign_extend:SI (match_operand:HI 1 "nonimmediate_operand"  "r,m")))]
   ""
   "@
    #
    ldh%o1%.\\t%0, %1"
+  "nios2_symbolic_memory_operand_p (operands[1])"
+  [(set (match_dup 0) (sign_extend:SI (match_dup 1)))]
+  {
+    operands[1] = nios2_split_symbolic_memory_operand (operands[1]);
+  }
   [(set_attr "type" "alu,ld")])
 
-(define_insn "extendqi<mode>2"
+(define_insn_and_split "extendqi<mode>2"
   [(set (match_operand:QX 0 "register_operand"                     "=r,r")
         (sign_extend:QX (match_operand:QI 1 "nonimmediate_operand"  "r,m")))]
   ""
   "@
    #
    ldb%o1%.\\t%0, %1"
+  "nios2_symbolic_memory_operand_p (operands[1])"
+  [(set (match_dup 0) (sign_extend:QX (match_dup 1)))]
+  {
+    operands[1] = nios2_split_symbolic_memory_operand (operands[1]);
+  }
   [(set_attr "type" "alu,ld")])
 
 ;; Split patterns for register alternative cases.
diff --git a/gcc/config/nios2/nios2.opt b/gcc/config/nios2/nios2.opt
index 08cb93541ee..a50dbee3fa7 100644
--- a/gcc/config/nios2/nios2.opt
+++ b/gcc/config/nios2/nios2.opt
@@ -586,3 +586,11 @@ Enable generation of R2 BMX instructions.
 mcdx
 Target Report Mask(HAS_CDX)
 Enable generation of R2 CDX instructions.
+
+mgprel-sec=
+Target RejectNegative Joined Var(nios2_gprel_sec) Init(NULL)
+Regular expression matching additional GP-addressible section names.
+
+mr0rel-sec=
+Target RejectNegative Joined Var(nios2_r0rel_sec) Init(NULL)
+Regular expression matching section names for r0-relative addressing.
diff --git a/gcc/config/pa/pa.h b/gcc/config/pa/pa.h
index 9ed929a301e..514de811f17 100644
--- a/gcc/config/pa/pa.h
+++ b/gcc/config/pa/pa.h
@@ -691,8 +691,8 @@ void hppa_profile_hook (int label_no);
 extern int may_call_alloca;
 
 #define EXIT_IGNORE_STACK	\
- (maybe_nonzero (get_frame_size ())	\
-  || cfun->calls_alloca || maybe_nonzero (crtl->outgoing_args_size))
+ (may_ne (get_frame_size (), 0)	\
+  || cfun->calls_alloca || may_ne (crtl->outgoing_args_size, 0))
 
 /* Length in units of the trampoline for entering a nested function.  */
 
diff --git a/gcc/config/powerpcspe/powerpcspe.c b/gcc/config/powerpcspe/powerpcspe.c
index 6da9f59148d..7a817fa3493 100644
--- a/gcc/config/powerpcspe/powerpcspe.c
+++ b/gcc/config/powerpcspe/powerpcspe.c
@@ -5860,6 +5860,7 @@ rs6000_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
         return 3;
 
       case unaligned_load:
+      case vector_gather_load:
 	if (TARGET_P9_VECTOR)
 	  return 3;
 
@@ -5901,6 +5902,7 @@ rs6000_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
         return 2;
 
       case unaligned_store:
+      case vector_scatter_store:
 	if (TARGET_EFFICIENT_UNALIGNED_VSX)
 	  return 1;
 
@@ -9539,6 +9541,8 @@ rs6000_delegitimize_address (rtx orig_x)
 static bool
 rs6000_const_not_ok_for_debug_p (rtx x)
 {
+  if (GET_CODE (x) == UNSPEC)
+    return true;
   if (GET_CODE (x) == SYMBOL_REF
       && CONSTANT_POOL_ADDRESS_P (x))
     {
diff --git a/gcc/config/riscv/pic.md b/gcc/config/riscv/pic.md
index 6a29ead32d3..03b8f9bc669 100644
--- a/gcc/config/riscv/pic.md
+++ b/gcc/config/riscv/pic.md
@@ -22,13 +22,20 @@
 ;; Simplify PIC loads to static variables.
 ;; These should go away once we figure out how to emit auipc discretely.
 
-(define_insn "*local_pic_load<mode>"
+(define_insn "*local_pic_load_s<mode>"
   [(set (match_operand:ANYI 0 "register_operand" "=r")
-	(mem:ANYI (match_operand 1 "absolute_symbolic_operand" "")))]
+	(sign_extend:ANYI (mem:ANYI (match_operand 1 "absolute_symbolic_operand" ""))))]
   "USE_LOAD_ADDRESS_MACRO (operands[1])"
   "<load>\t%0,%1"
   [(set (attr "length") (const_int 8))])
 
+(define_insn "*local_pic_load_u<mode>"
+  [(set (match_operand:ZERO_EXTEND_LOAD 0 "register_operand" "=r")
+	(zero_extend:ZERO_EXTEND_LOAD (mem:ZERO_EXTEND_LOAD (match_operand 1 "absolute_symbolic_operand" ""))))]
+  "USE_LOAD_ADDRESS_MACRO (operands[1])"
+  "<load>u\t%0,%1"
+  [(set (attr "length") (const_int 8))])
+
 (define_insn "*local_pic_load<mode>"
   [(set (match_operand:ANYF 0 "register_operand" "=f")
 	(mem:ANYF (match_operand 1 "absolute_symbolic_operand" "")))
diff --git a/gcc/config/riscv/riscv.c b/gcc/config/riscv/riscv.c
index 06106f22b8b..8ce93528b99 100644
--- a/gcc/config/riscv/riscv.c
+++ b/gcc/config/riscv/riscv.c
@@ -1334,6 +1334,22 @@ riscv_legitimize_move (machine_mode mode, rtx dest, rtx src)
       return true;
     }
 
+  /* RISC-V GCC may generate non-legitimate address due to we provide some
+     pattern for optimize access PIC local symbol and it's make GCC generate
+     unrecognizable instruction during optmizing.  */
+
+  if (MEM_P (dest) && !riscv_legitimate_address_p (mode, XEXP (dest, 0),
+						   reload_completed))
+    {
+      XEXP (dest, 0) = riscv_force_address (XEXP (dest, 0), mode);
+    }
+
+  if (MEM_P (src) && !riscv_legitimate_address_p (mode, XEXP (src, 0),
+						  reload_completed))
+    {
+      XEXP (src, 0) = riscv_force_address (XEXP (src, 0), mode);
+    }
+
   return false;
 }
 
diff --git a/gcc/config/riscv/riscv.md b/gcc/config/riscv/riscv.md
index fd9236c7c17..9f056bbcda4 100644
--- a/gcc/config/riscv/riscv.md
+++ b/gcc/config/riscv/riscv.md
@@ -259,6 +259,9 @@
 ;; Iterator for QImode extension patterns.
 (define_mode_iterator SUPERQI [HI SI (DI "TARGET_64BIT")])
 
+;; Iterator for extending loads.
+(define_mode_iterator ZERO_EXTEND_LOAD [QI HI (SI "TARGET_64BIT")])
+
 ;; Iterator for hardware integer modes narrower than XLEN.
 (define_mode_iterator SUBX [QI HI (SI "TARGET_64BIT")])
 
diff --git a/gcc/config/rl78/rl78-protos.h b/gcc/config/rl78/rl78-protos.h
index a155df61b99..976bffa61e7 100644
--- a/gcc/config/rl78/rl78-protos.h
+++ b/gcc/config/rl78/rl78-protos.h
@@ -54,3 +54,13 @@ void		rl78_output_aligned_common (FILE *, tree, const char *,
 					    int, int, int);
 
 int		rl78_one_far_p (rtx *operands, int num_operands);
+
+#ifdef RTX_CODE
+#ifdef HAVE_MACHINE_MODES
+
+rtx rl78_emit_libcall (const char*, enum rtx_code,
+                       enum machine_mode, enum machine_mode,
+                       int, rtx*);
+
+#endif
+#endif
diff --git a/gcc/config/rl78/rl78.c b/gcc/config/rl78/rl78.c
index ce66866ef84..d2baa8ccfae 100644
--- a/gcc/config/rl78/rl78.c
+++ b/gcc/config/rl78/rl78.c
@@ -362,6 +362,7 @@ rl78_option_override (void)
   if (TARGET_ES0
       && strcmp (lang_hooks.name, "GNU C")
       && strcmp (lang_hooks.name, "GNU C11")
+      && strcmp (lang_hooks.name, "GNU C17")
       && strcmp (lang_hooks.name, "GNU C89")
       && strcmp (lang_hooks.name, "GNU C99")
       /* Compiling with -flto results in a language of GNU GIMPLE being used... */
@@ -4793,6 +4794,45 @@ rl78_addsi3_internal (rtx * operands, unsigned int alternative)
     }
 }
 
+rtx
+rl78_emit_libcall (const char *name, enum rtx_code code,
+                   enum machine_mode dmode, enum machine_mode smode,
+                   int noperands, rtx *operands)
+{
+  rtx ret;
+  rtx_insn *insns;
+  rtx libcall;
+  rtx equiv;
+
+  start_sequence ();
+  libcall = gen_rtx_SYMBOL_REF (Pmode, name);
+
+  switch (noperands)
+    {
+    case 2:
+      ret = emit_library_call_value (libcall, NULL_RTX, LCT_CONST,
+                                     dmode, operands[1], smode);
+      equiv = gen_rtx_fmt_e (code, dmode, operands[1]);
+      break;
+
+    case 3:
+      ret = emit_library_call_value (libcall, NULL_RTX,
+                                     LCT_CONST, dmode,
+                                     operands[1], smode, operands[2],
+                                     smode);
+      equiv = gen_rtx_fmt_ee (code, dmode, operands[1], operands[2]);
+      break;
+
+    default:
+      gcc_unreachable ();
+    }
+
+  insns = get_insns ();
+  end_sequence ();
+  emit_libcall_block (insns, operands[0], ret, equiv);
+  return ret;
+}
+
 
 #undef  TARGET_PREFERRED_RELOAD_CLASS
 #define TARGET_PREFERRED_RELOAD_CLASS rl78_preferred_reload_class
diff --git a/gcc/config/rl78/rl78.md b/gcc/config/rl78/rl78.md
index 722d98439b2..c53ca0ff98d 100644
--- a/gcc/config/rl78/rl78.md
+++ b/gcc/config/rl78/rl78.md
@@ -224,6 +224,16 @@
    DONE;"
 )
 
+(define_expand "adddi3"
+  [(set (match_operand:DI          0 "nonimmediate_operand" "")
+	(plus:DI (match_operand:DI 1 "general_operand"      "")
+		 (match_operand:DI 2 "general_operand"      "")))
+   ]
+  ""
+  "rl78_emit_libcall (\"__adddi3\", PLUS, DImode, DImode, 3, operands);
+   DONE;"
+)
+
 (define_insn "addsi3_internal_virt"
   [(set (match_operand:SI          0 "nonimmediate_operand" "=v,&vm, vm")
 	(plus:SI (match_operand:SI 1 "general_operand"      "0, vim, vim")
@@ -258,6 +268,16 @@
   DONE;"
 )
 
+(define_expand "subdi3"
+ [(set (match_operand:DI          0 "nonimmediate_operand" "")
+    (minus:DI (match_operand:DI 1 "general_operand"      "")
+         (match_operand:DI    2 "general_operand"      "")))
+   ]
+  ""
+  "rl78_emit_libcall (\"__subdi3\", MINUS, DImode, DImode, 3, operands);
+   DONE;"
+)
+
 (define_insn "subsi3_internal_virt"
   [(set (match_operand:SI           0 "nonimmediate_operand" "=v,&vm, vm")
 	(minus:SI (match_operand:SI 1 "general_operand"      "0, vim, vim")
diff --git a/gcc/config/rs6000/aix.h b/gcc/config/rs6000/aix.h
index 36c4a522b4f..31fda583c2c 100644
--- a/gcc/config/rs6000/aix.h
+++ b/gcc/config/rs6000/aix.h
@@ -77,6 +77,9 @@
 #undef  TARGET_IEEEQUAD
 #define TARGET_IEEEQUAD 0
 
+#undef  TARGET_IEEEQUAD_DEFAULT
+#define TARGET_IEEEQUAD_DEFAULT 0
+
 /* The AIX linker will discard static constructors in object files before
    collect has a chance to see them, so scan the object files directly.  */
 #define COLLECT_EXPORT_LIST
diff --git a/gcc/config/rs6000/altivec.md b/gcc/config/rs6000/altivec.md
index 8f2631ccc7c..d0fcd1c3d8a 100644
--- a/gcc/config/rs6000/altivec.md
+++ b/gcc/config/rs6000/altivec.md
@@ -789,11 +789,12 @@
   rtx zero = gen_reg_rtx (V8HImode);
 
   emit_insn (gen_altivec_vspltish (zero, const0_rtx));
-  emit_insn (gen_altivec_vmladduhm(operands[0], operands[1], operands[2], zero));
+  emit_insn (gen_fmav8hi4 (operands[0], operands[1], operands[2], zero));
 
   DONE;
 })
 
+
 ;; Fused multiply subtract 
 (define_insn "*altivec_vnmsubfp"
   [(set (match_operand:V4SF 0 "register_operand" "=v")
@@ -930,7 +931,7 @@
   "vmhraddshs %0,%1,%2,%3"
   [(set_attr "type" "veccomplex")])
 
-(define_insn "altivec_vmladduhm"
+(define_insn "fmav8hi4"
   [(set (match_operand:V8HI 0 "register_operand" "=v")
         (plus:V8HI (mult:V8HI (match_operand:V8HI 1 "register_operand" "v")
 		   	      (match_operand:V8HI 2 "register_operand" "v"))
diff --git a/gcc/config/rs6000/darwin.h b/gcc/config/rs6000/darwin.h
index 2c3b04c613b..5f03fa5f206 100644
--- a/gcc/config/rs6000/darwin.h
+++ b/gcc/config/rs6000/darwin.h
@@ -272,6 +272,9 @@ extern int darwin_emit_branch_islands;
 #undef  TARGET_IEEEQUAD
 #define TARGET_IEEEQUAD 0
 
+#undef  TARGET_IEEEQUAD_DEFAULT
+#define TARGET_IEEEQUAD_DEFAULT 0
+
 /* Since Darwin doesn't do TOCs, stub this out.  */
 
 #define ASM_OUTPUT_SPECIAL_POOL_ENTRY_P(X, MODE)  ((void)X, (void)MODE, 0)
diff --git a/gcc/config/rs6000/emmintrin.h b/gcc/config/rs6000/emmintrin.h
new file mode 100644
index 00000000000..6a33ad9dd13
--- /dev/null
+++ b/gcc/config/rs6000/emmintrin.h
@@ -0,0 +1,2340 @@
+/* Copyright (C) 2003-2017 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Implemented from the specification included in the Intel C++ Compiler
+   User Guide and Reference, version 9.0.  */
+
+#ifndef NO_WARN_X86_INTRINSICS
+/* This header is distributed to simplify porting x86_64 code that
+   makes explicit use of Intel intrinsics to powerpc64le.
+   It is the user's responsibility to determine if the results are
+   acceptable and make additional changes as necessary.
+   Note that much code that uses Intel intrinsics can be rewritten in
+   standard C or GNU C extensions, which are more portable and better
+   optimized across multiple targets.
+
+   In the specific case of X86 SSE2 (__m128i, __m128d) intrinsics,
+   the PowerPC VMX/VSX ISA is a good match for vector double SIMD
+   operations.  However scalar double operations in vector (XMM)
+   registers require the POWER8 VSX ISA (2.07) level. Also there are
+   important differences for data format and placement of double
+   scalars in the vector register.
+
+   For PowerISA Scalar double is in FPRs (left most 64-bits of the
+   low 32 VSRs), while X86_64 SSE2 uses the right most 64-bits of
+   the XMM. These differences require extra steps on POWER to match
+   the SSE2 scalar double semantics.
+
+   Most SSE2 scalar double intrinsic operations can be performed more
+   efficiently as C language double scalar operations or optimized to
+   use vector SIMD operations.  We recommend this for new applications.
+
+   Another difference is the format and details of the X86_64 MXSCR vs
+   the PowerISA FPSCR / VSCR registers. We recommend applications
+   replace direct access to the MXSCR with the more portable <fenv.h>
+   Posix APIs. */
+#warning "Please read comment above.  Use -DNO_WARN_X86_INTRINSICS to disable this warning."
+#endif
+
+#ifndef EMMINTRIN_H_
+#define EMMINTRIN_H_
+
+#include <altivec.h>
+#include <assert.h>
+
+/* We need definitions from the SSE header files.  */
+#include <xmmintrin.h>
+
+/* SSE2 */
+typedef __vector double __v2df;
+typedef __vector long long __v2di;
+typedef __vector unsigned long long __v2du;
+typedef __vector int __v4si;
+typedef __vector unsigned int __v4su;
+typedef __vector short __v8hi;
+typedef __vector unsigned short __v8hu;
+typedef __vector signed char __v16qi;
+typedef __vector unsigned char __v16qu;
+
+/* The Intel API is flexible enough that we must allow aliasing with other
+   vector types, and their scalar components.  */
+typedef long long __m128i __attribute__ ((__vector_size__ (16), __may_alias__));
+typedef double __m128d __attribute__ ((__vector_size__ (16), __may_alias__));
+
+/* Unaligned version of the same types.  */
+typedef long long __m128i_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
+typedef double __m128d_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
+
+/* Create a vector with element 0 as F and the rest zero.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_sd (double __F)
+{
+  return __extension__ (__m128d){ __F, 0.0 };
+}
+
+/* Create a vector with both elements equal to F.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set1_pd (double __F)
+{
+  return __extension__ (__m128d){ __F, __F };
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_pd1 (double __F)
+{
+  return _mm_set1_pd (__F);
+}
+
+/* Create a vector with the lower value X and upper value W.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_pd (double __W, double __X)
+{
+  return __extension__ (__m128d){ __X, __W };
+}
+
+/* Create a vector with the lower value W and upper value X.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setr_pd (double __W, double __X)
+{
+  return __extension__ (__m128d){ __W, __X };
+}
+
+/* Create an undefined vector.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_undefined_pd (void)
+{
+  __m128d __Y = __Y;
+  return __Y;
+}
+
+/* Create a vector of zeros.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setzero_pd (void)
+{
+  return (__m128d) vec_splats (0);
+}
+
+/* Sets the low DPFP value of A from the low value of B.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_move_sd (__m128d __A, __m128d __B)
+{
+  __v2df result = (__v2df) __A;
+  result [0] = ((__v2df) __B)[0];
+  return (__m128d) result;
+}
+
+/* Load two DPFP values from P.  The address must be 16-byte aligned.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_load_pd (double const *__P)
+{
+  assert(((unsigned long)__P & 0xfUL) == 0UL);
+  return ((__m128d)vec_ld(0, (__v16qu*)__P));
+}
+
+/* Load two DPFP values from P.  The address need not be 16-byte aligned.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_loadu_pd (double const *__P)
+{
+  return (vec_vsx_ld(0, __P));
+}
+
+/* Create a vector with all two elements equal to *P.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_load1_pd (double const *__P)
+{
+  return (vec_splats (*__P));
+}
+
+/* Create a vector with element 0 as *P and the rest zero.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_load_sd (double const *__P)
+{
+  return _mm_set_sd (*__P);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_load_pd1 (double const *__P)
+{
+  return _mm_load1_pd (__P);
+}
+
+/* Load two DPFP values in reverse order.  The address must be aligned.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_loadr_pd (double const *__P)
+{
+  __v2df __tmp = _mm_load_pd (__P);
+  return (__m128d)vec_xxpermdi (__tmp, __tmp, 2);
+}
+
+/* Store two DPFP values.  The address must be 16-byte aligned.  */
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_store_pd (double *__P, __m128d __A)
+{
+  assert(((unsigned long)__P & 0xfUL) == 0UL);
+  vec_st((__v16qu)__A, 0, (__v16qu*)__P);
+}
+
+/* Store two DPFP values.  The address need not be 16-byte aligned.  */
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_storeu_pd (double *__P, __m128d __A)
+{
+  *(__m128d *)__P = __A;
+}
+
+/* Stores the lower DPFP value.  */
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_store_sd (double *__P, __m128d __A)
+{
+  *__P = ((__v2df)__A)[0];
+}
+
+extern __inline double __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsd_f64 (__m128d __A)
+{
+  return ((__v2df)__A)[0];
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_storel_pd (double *__P, __m128d __A)
+{
+  _mm_store_sd (__P, __A);
+}
+
+/* Stores the upper DPFP value.  */
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_storeh_pd (double *__P, __m128d __A)
+{
+  *__P = ((__v2df)__A)[1];
+}
+/* Store the lower DPFP value across two words.
+   The address must be 16-byte aligned.  */
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_store1_pd (double *__P, __m128d __A)
+{
+  _mm_store_pd (__P, vec_splat (__A, 0));
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_store_pd1 (double *__P, __m128d __A)
+{
+  _mm_store1_pd (__P, __A);
+}
+
+/* Store two DPFP values in reverse order.  The address must be aligned.  */
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_storer_pd (double *__P, __m128d __A)
+{
+  _mm_store_pd (__P, vec_xxpermdi (__A, __A, 2));
+}
+
+/* Intel intrinsic.  */
+extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi128_si64 (__m128i __A)
+{
+  return ((__v2di)__A)[0];
+}
+
+/* Microsoft intrinsic.  */
+extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi128_si64x (__m128i __A)
+{
+  return ((__v2di)__A)[0];
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_pd (__m128d __A, __m128d __B)
+{
+  return (__m128d) ((__v2df)__A + (__v2df)__B);
+}
+
+/* Add the lower double-precision (64-bit) floating-point element in
+   a and b, store the result in the lower element of dst, and copy
+   the upper element from a to the upper element of dst. */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_sd (__m128d __A, __m128d __B)
+{
+  __A[0] = __A[0] + __B[0];
+  return (__A);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sub_pd (__m128d __A, __m128d __B)
+{
+  return (__m128d) ((__v2df)__A - (__v2df)__B);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sub_sd (__m128d __A, __m128d __B)
+{
+  __A[0] = __A[0] - __B[0];
+  return (__A);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mul_pd (__m128d __A, __m128d __B)
+{
+  return (__m128d) ((__v2df)__A * (__v2df)__B);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mul_sd (__m128d __A, __m128d __B)
+{
+  __A[0] = __A[0] * __B[0];
+  return (__A);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_div_pd (__m128d __A, __m128d __B)
+{
+  return (__m128d) ((__v2df)__A / (__v2df)__B);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_div_sd (__m128d __A, __m128d __B)
+{
+  __A[0] = __A[0] / __B[0];
+  return (__A);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sqrt_pd (__m128d __A)
+{
+  return (vec_sqrt (__A));
+}
+
+/* Return pair {sqrt (B[0]), A[1]}.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sqrt_sd (__m128d __A, __m128d __B)
+{
+  __v2df c;
+  c = vec_sqrt ((__v2df) _mm_set1_pd (__B[0]));
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_min_pd (__m128d __A, __m128d __B)
+{
+  return (vec_min (__A, __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_min_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = vec_min (a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_max_pd (__m128d __A, __m128d __B)
+{
+  return (vec_max (__A, __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_max_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = vec_max (a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpeq_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmpeq ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmplt_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmplt ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmple_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmple ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpgt_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmpgt ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpge_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmpge ((__v2df) __A,(__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpneq_pd (__m128d __A, __m128d __B)
+{
+  __v2df temp = (__v2df) vec_cmpeq ((__v2df) __A, (__v2df)__B);
+  return ((__m128d)vec_nor (temp, temp));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpnlt_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmpge ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpnle_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmpgt ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpngt_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmple ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpnge_pd (__m128d __A, __m128d __B)
+{
+  return ((__m128d)vec_cmplt ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpord_pd (__m128d __A, __m128d __B)
+{
+#if _ARCH_PWR8
+  __v2du c, d;
+  /* Compare against self will return false (0's) if NAN.  */
+  c = (__v2du)vec_cmpeq (__A, __A);
+  d = (__v2du)vec_cmpeq (__B, __B);
+#else
+  __v2du a, b;
+  __v2du c, d;
+  const __v2du double_exp_mask  = {0x7ff0000000000000, 0x7ff0000000000000};
+  a = (__v2du)vec_abs ((__v2df)__A);
+  b = (__v2du)vec_abs ((__v2df)__B);
+  c = (__v2du)vec_cmpgt (double_exp_mask, a);
+  d = (__v2du)vec_cmpgt (double_exp_mask, b);
+#endif
+  /* A != NAN and B != NAN.  */
+  return ((__m128d)vec_and(c, d));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpunord_pd (__m128d __A, __m128d __B)
+{
+#if _ARCH_PWR8
+  __v2du c, d;
+  /* Compare against self will return false (0's) if NAN.  */
+  c = (__v2du)vec_cmpeq ((__v2df)__A, (__v2df)__A);
+  d = (__v2du)vec_cmpeq ((__v2df)__B, (__v2df)__B);
+  /* A == NAN OR B == NAN converts too:
+     NOT(A != NAN) OR NOT(B != NAN).  */
+  c = vec_nor (c, c);
+  return ((__m128d)vec_orc(c, d));
+#else
+  __v2du c, d;
+  /* Compare against self will return false (0's) if NAN.  */
+  c = (__v2du)vec_cmpeq ((__v2df)__A, (__v2df)__A);
+  d = (__v2du)vec_cmpeq ((__v2df)__B, (__v2df)__B);
+  /* Convert the true ('1's) is NAN.  */
+  c = vec_nor (c, c);
+  d = vec_nor (d, d);
+  return ((__m128d)vec_or(c, d));
+#endif
+}
+
+extern __inline  __m128d  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpeq_sd(__m128d  __A, __m128d  __B)
+{
+  __v2df a, b, c;
+  /* PowerISA VSX does not allow partial (for just lower double)
+     results. So to insure we don't generate spurious exceptions
+     (from the upper double values) we splat the lower double
+     before we do the operation. */
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = (__v2df) vec_cmpeq(a, b);
+  /* Then we merge the lower double result with the original upper
+     double from __A.  */
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmplt_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = (__v2df) vec_cmplt(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmple_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = (__v2df) vec_cmple(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpgt_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = (__v2df) vec_cmpgt(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpge_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = (__v2df) vec_cmpge(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpneq_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  c = (__v2df) vec_cmpeq(a, b);
+  c = vec_nor (c, c);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpnlt_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  /* Not less than is just greater than or equal.  */
+  c = (__v2df) vec_cmpge(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpnle_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  /* Not less than or equal is just greater than.  */
+  c = (__v2df) vec_cmpge(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpngt_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  /* Not greater than is just less than or equal.  */
+  c = (__v2df) vec_cmple(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpnge_sd (__m128d __A, __m128d __B)
+{
+  __v2df a, b, c;
+  a = vec_splats (__A[0]);
+  b = vec_splats (__B[0]);
+  /* Not greater than or equal is just less than.  */
+  c = (__v2df) vec_cmplt(a, b);
+  return (__m128d) _mm_setr_pd (c[0], __A[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpord_sd (__m128d __A, __m128d __B)
+{
+  __v2df r;
+  r = (__v2df)_mm_cmpord_pd (vec_splats (__A[0]), vec_splats (__B[0]));
+  return (__m128d) _mm_setr_pd (r[0], ((__v2df)__A)[1]);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpunord_sd (__m128d __A, __m128d __B)
+{
+  __v2df r;
+  r = _mm_cmpunord_pd (vec_splats (__A[0]), vec_splats (__B[0]));
+  return (__m128d) _mm_setr_pd (r[0], __A[1]);
+}
+
+/* FIXME
+   The __mm_comi??_sd and __mm_ucomi??_sd implementations below are
+   exactly the same because GCC for PowerPC only generates unordered
+   compares (scalar and vector).
+   Technically __mm_comieq_sp et all should be using the ordered
+   compare and signal for QNaNs.  The __mm_ucomieq_sd et all should
+   be OK.   */
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_comieq_sd (__m128d __A, __m128d __B)
+{
+  return (__A[0] == __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_comilt_sd (__m128d __A, __m128d __B)
+{
+  return (__A[0] < __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_comile_sd (__m128d __A, __m128d __B)
+{
+  return (__A[0] <= __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_comigt_sd (__m128d __A, __m128d __B)
+{
+  return (__A[0] > __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_comige_sd (__m128d __A, __m128d __B)
+{
+  return (__A[0] >= __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_comineq_sd (__m128d __A, __m128d __B)
+{
+  return (__A[0] != __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_ucomieq_sd (__m128d __A, __m128d __B)
+{
+	return (__A[0] == __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_ucomilt_sd (__m128d __A, __m128d __B)
+{
+	return (__A[0] < __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_ucomile_sd (__m128d __A, __m128d __B)
+{
+	return (__A[0] <= __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_ucomigt_sd (__m128d __A, __m128d __B)
+{
+	return (__A[0] > __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_ucomige_sd (__m128d __A, __m128d __B)
+{
+	return (__A[0] >= __B[0]);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_ucomineq_sd (__m128d __A, __m128d __B)
+{
+  return (__A[0] != __B[0]);
+}
+
+/* Create a vector of Qi, where i is the element number.  */
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_epi64x (long long __q1, long long __q0)
+{
+  return __extension__ (__m128i)(__v2di){ __q0, __q1 };
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_epi64 (__m64 __q1,  __m64 __q0)
+{
+  return _mm_set_epi64x ((long long)__q1, (long long)__q0);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_epi32 (int __q3, int __q2, int __q1, int __q0)
+{
+  return __extension__ (__m128i)(__v4si){ __q0, __q1, __q2, __q3 };
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_epi16 (short __q7, short __q6, short __q5, short __q4,
+	       short __q3, short __q2, short __q1, short __q0)
+{
+  return __extension__ (__m128i)(__v8hi){
+    __q0, __q1, __q2, __q3, __q4, __q5, __q6, __q7 };
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set_epi8 (char __q15, char __q14, char __q13, char __q12,
+	      char __q11, char __q10, char __q09, char __q08,
+	      char __q07, char __q06, char __q05, char __q04,
+	      char __q03, char __q02, char __q01, char __q00)
+{
+  return __extension__ (__m128i)(__v16qi){
+    __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07,
+    __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15
+  };
+}
+
+/* Set all of the elements of the vector to A.  */
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set1_epi64x (long long __A)
+{
+  return _mm_set_epi64x (__A, __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set1_epi64 (__m64 __A)
+{
+  return _mm_set_epi64 (__A, __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set1_epi32 (int __A)
+{
+  return _mm_set_epi32 (__A, __A, __A, __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set1_epi16 (short __A)
+{
+  return _mm_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_set1_epi8 (char __A)
+{
+  return _mm_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A,
+		       __A, __A, __A, __A, __A, __A, __A, __A);
+}
+
+/* Create a vector of Qi, where i is the element number.
+   The parameter order is reversed from the _mm_set_epi* functions.  */
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setr_epi64 (__m64 __q0, __m64 __q1)
+{
+  return _mm_set_epi64 (__q1, __q0);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setr_epi32 (int __q0, int __q1, int __q2, int __q3)
+{
+  return _mm_set_epi32 (__q3, __q2, __q1, __q0);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setr_epi16 (short __q0, short __q1, short __q2, short __q3,
+	        short __q4, short __q5, short __q6, short __q7)
+{
+  return _mm_set_epi16 (__q7, __q6, __q5, __q4, __q3, __q2, __q1, __q0);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setr_epi8 (char __q00, char __q01, char __q02, char __q03,
+	       char __q04, char __q05, char __q06, char __q07,
+	       char __q08, char __q09, char __q10, char __q11,
+	       char __q12, char __q13, char __q14, char __q15)
+{
+  return _mm_set_epi8 (__q15, __q14, __q13, __q12, __q11, __q10, __q09, __q08,
+		       __q07, __q06, __q05, __q04, __q03, __q02, __q01, __q00);
+}
+
+/* Create a vector with element 0 as *P and the rest zero.  */
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_load_si128 (__m128i const *__P)
+{
+  return *__P;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_loadu_si128 (__m128i_u const *__P)
+{
+  return (__m128i) (vec_vsx_ld(0, (signed int const *)__P));
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_loadl_epi64 (__m128i_u const *__P)
+{
+  return _mm_set_epi64 ((__m64)0LL, *(__m64 *)__P);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_store_si128 (__m128i *__P, __m128i __B)
+{
+  assert(((unsigned long )__P & 0xfUL) == 0UL);
+  vec_st ((__v16qu) __B, 0, (__v16qu*)__P);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_storeu_si128 (__m128i_u *__P, __m128i __B)
+{
+  *__P = __B;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_storel_epi64 (__m128i_u *__P, __m128i __B)
+{
+  *(long long *)__P = ((__v2di)__B)[0];
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_movepi64_pi64 (__m128i_u __B)
+{
+  return (__m64) ((__v2di)__B)[0];
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_movpi64_epi64 (__m64 __A)
+{
+  return _mm_set_epi64 ((__m64)0LL, __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_move_epi64 (__m128i __A)
+{
+  return _mm_set_epi64 ((__m64)0LL, (__m64)__A[0]);
+}
+
+/* Create an undefined vector.  */
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_undefined_si128 (void)
+{
+  __m128i __Y = __Y;
+  return __Y;
+}
+
+/* Create a vector of zeros.  */
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_setzero_si128 (void)
+{
+  return __extension__ (__m128i)(__v4si){ 0, 0, 0, 0 };
+}
+
+#ifdef _ARCH_PWR8
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtepi32_pd (__m128i __A)
+{
+  __v2di val;
+  /* For LE need to generate Vector Unpack Low Signed Word.
+     Which is generated from unpackh.  */
+  val = (__v2di)vec_unpackh ((__v4si)__A);
+
+  return (__m128d)vec_ctf (val, 0);
+}
+#endif
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtepi32_ps (__m128i __A)
+{
+  return ((__m128)vec_ctf((__v4si)__A, 0));
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtpd_epi32 (__m128d __A)
+{
+  __v2df rounded = vec_rint (__A);
+  __v4si result, temp;
+  const __v4si vzero =
+    { 0, 0, 0, 0 };
+
+  /* VSX Vector truncate Double-Precision to integer and Convert to
+   Signed Integer Word format with Saturate.  */
+  __asm__(
+      "xvcvdpsxws %x0,%x1"
+      : "=wa" (temp)
+      : "wa" (rounded)
+      : );
+
+#ifdef _ARCH_PWR8
+  temp = vec_mergeo (temp, temp);
+  result = (__v4si)vec_vpkudum ((vector long)temp, (vector long)vzero);
+#else
+  {
+    const __v16qu pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
+	0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f };
+    result = (__v4si) vec_perm ((__v16qu) temp, (__v16qu) vzero, pkperm);
+  }
+#endif
+  return (__m128i) result;
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtpd_pi32 (__m128d __A)
+{
+  __m128i result = _mm_cvtpd_epi32(__A);
+
+  return (__m64) result[0];
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtpd_ps (__m128d __A)
+{
+  __v4sf result;
+  __v4si temp;
+  const __v4si vzero = { 0, 0, 0, 0 };
+
+  __asm__(
+      "xvcvdpsp %x0,%x1"
+      : "=wa" (temp)
+      : "wa" (__A)
+      : );
+
+#ifdef _ARCH_PWR8
+  temp = vec_mergeo (temp, temp);
+  result = (__v4sf)vec_vpkudum ((vector long)temp, (vector long)vzero);
+#else
+  {
+    const __v16qu pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
+	0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f };
+    result = (__v4sf) vec_perm ((__v16qu) temp, (__v16qu) vzero, pkperm);
+  }
+#endif
+  return ((__m128)result);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvttpd_epi32 (__m128d __A)
+{
+  __v4si result;
+  __v4si temp;
+  const __v4si vzero = { 0, 0, 0, 0 };
+
+  /* VSX Vector truncate Double-Precision to integer and Convert to
+   Signed Integer Word format with Saturate.  */
+  __asm__(
+      "xvcvdpsxws %x0,%x1"
+      : "=wa" (temp)
+      : "wa" (__A)
+      : );
+
+#ifdef _ARCH_PWR8
+  temp = vec_mergeo (temp, temp);
+  result = (__v4si)vec_vpkudum ((vector long)temp, (vector long)vzero);
+#else
+  {
+    const __v16qu pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
+	0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f };
+    result = (__v4si) vec_perm ((__v16qu) temp, (__v16qu) vzero, pkperm);
+  }
+#endif
+
+  return ((__m128i) result);
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvttpd_pi32 (__m128d __A)
+{
+  __m128i result = _mm_cvttpd_epi32 (__A);
+
+  return (__m64) result[0];
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi128_si32 (__m128i __A)
+{
+  return ((__v4si)__A)[0];
+}
+
+#ifdef _ARCH_PWR8
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtpi32_pd (__m64 __A)
+{
+  __v4si temp;
+  __v2di tmp2;
+  __v2df result;
+
+  temp = (__v4si)vec_splats (__A);
+  tmp2 = (__v2di)vec_unpackl (temp);
+  result = vec_ctf ((__vector signed long)tmp2, 0);
+  return (__m128d)result;
+}
+#endif
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtps_epi32 (__m128 __A)
+{
+  __v4sf rounded;
+  __v4si result;
+
+  rounded = vec_rint((__v4sf) __A);
+  result = vec_cts (rounded, 0);
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvttps_epi32 (__m128 __A)
+{
+  __v4si result;
+
+  result = vec_cts ((__v4sf) __A, 0);
+  return (__m128i) result;
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtps_pd (__m128 __A)
+{
+  /* Check if vec_doubleh is defined by <altivec.h>. If so use that. */
+#ifdef vec_doubleh
+  return (__m128d) vec_doubleh ((__v4sf)__A);
+#else
+  /* Otherwise the compiler is not current and so need to generate the
+     equivalent code.  */
+  __v4sf a = (__v4sf)__A;
+  __v4sf temp;
+  __v2df result;
+#ifdef __LITTLE_ENDIAN__
+  /* The input float values are in elements {[0], [1]} but the convert
+     instruction needs them in elements {[1], [3]}, So we use two
+     shift left double vector word immediates to get the elements
+     lined up.  */
+  temp = __builtin_vsx_xxsldwi (a, a, 3);
+  temp = __builtin_vsx_xxsldwi (a, temp, 2);
+#elif __BIG_ENDIAN__
+  /* The input float values are in elements {[0], [1]} but the convert
+     instruction needs them in elements {[0], [2]}, So we use two
+     shift left double vector word immediates to get the elements
+     lined up.  */
+  temp = vec_vmrghw (a, a);
+#endif
+  __asm__(
+      " xvcvspdp %x0,%x1"
+      : "=wa" (result)
+      : "wa" (temp)
+      : );
+  return (__m128d) result;
+#endif
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsd_si32 (__m128d __A)
+{
+  __v2df rounded = vec_rint((__v2df) __A);
+  int result = ((__v2df)rounded)[0];
+
+  return result;
+}
+/* Intel intrinsic.  */
+extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsd_si64 (__m128d __A)
+{
+  __v2df rounded = vec_rint ((__v2df) __A );
+  long long result = ((__v2df) rounded)[0];
+
+  return result;
+}
+
+/* Microsoft intrinsic.  */
+extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsd_si64x (__m128d __A)
+{
+  return _mm_cvtsd_si64 ((__v2df)__A);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvttsd_si32 (__m128d __A)
+{
+  int result = ((__v2df)__A)[0];
+
+  return result;
+}
+
+/* Intel intrinsic.  */
+extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvttsd_si64 (__m128d __A)
+{
+  long long result = ((__v2df)__A)[0];
+
+  return result;
+}
+
+/* Microsoft intrinsic.  */
+extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvttsd_si64x (__m128d __A)
+{
+  return _mm_cvttsd_si64 (__A);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsd_ss (__m128 __A, __m128d __B)
+{
+  __v4sf result = (__v4sf)__A;
+
+#ifdef __LITTLE_ENDIAN__
+  __v4sf temp_s;
+  /* Copy double element[0] to element [1] for conversion.  */
+  __v2df temp_b = vec_splat((__v2df)__B, 0);
+
+  /* Pre-rotate __A left 3 (logically right 1) elements.  */
+  result = __builtin_vsx_xxsldwi (result, result, 3);
+  /* Convert double to single float scalar in a vector.  */
+  __asm__(
+      "xscvdpsp %x0,%x1"
+      : "=wa" (temp_s)
+      : "wa" (temp_b)
+      : );
+  /* Shift the resulting scalar into vector element [0].  */
+  result = __builtin_vsx_xxsldwi (result, temp_s, 1);
+#else
+  result [0] = ((__v2df)__B)[0];
+#endif
+  return (__m128) result;
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi32_sd (__m128d __A, int __B)
+{
+  __v2df result = (__v2df)__A;
+  double db = __B;
+  result [0] = db;
+  return (__m128d)result;
+}
+
+/* Intel intrinsic.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi64_sd (__m128d __A, long long __B)
+{
+  __v2df result = (__v2df)__A;
+  double db = __B;
+  result [0] = db;
+  return (__m128d)result;
+}
+
+/* Microsoft intrinsic.  */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi64x_sd (__m128d __A, long long __B)
+{
+  return _mm_cvtsi64_sd (__A, __B);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtss_sd (__m128d __A, __m128 __B)
+{
+#ifdef __LITTLE_ENDIAN__
+  /* Use splat to move element [0] into position for the convert. */
+  __v4sf temp = vec_splat ((__v4sf)__B, 0);
+  __v2df res;
+  /* Convert single float scalar to double in a vector.  */
+  __asm__(
+      "xscvspdp %x0,%x1"
+      : "=wa" (res)
+      : "wa" (temp)
+      : );
+  return (__m128d) vec_mergel (res, (__v2df)__A);
+#else
+  __v2df res = (__v2df)__A;
+  res [0] = ((__v4sf)__B) [0];
+  return (__m128d) res;
+#endif
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_shuffle_pd(__m128d __A, __m128d __B, const int __mask)
+{
+  __vector double result;
+  const int litmsk = __mask & 0x3;
+
+  if (litmsk == 0)
+    result = vec_mergeh (__A, __B);
+#if __GNUC__ < 6
+  else if (litmsk == 1)
+    result = vec_xxpermdi (__B, __A, 2);
+  else if (litmsk == 2)
+    result = vec_xxpermdi (__B, __A, 1);
+#else
+  else if (litmsk == 1)
+    result = vec_xxpermdi (__A, __B, 2);
+  else if (litmsk == 2)
+    result = vec_xxpermdi (__A, __B, 1);
+#endif
+  else
+    result = vec_mergel (__A, __B);
+
+  return result;
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpackhi_pd (__m128d __A, __m128d __B)
+{
+  return (__m128d) vec_mergel ((__v2df)__A, (__v2df)__B);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_pd (__m128d __A, __m128d __B)
+{
+  return (__m128d) vec_mergeh ((__v2df)__A, (__v2df)__B);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_loadh_pd (__m128d __A, double const *__B)
+{
+  __v2df result = (__v2df)__A;
+  result [1] = *__B;
+  return (__m128d)result;
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_loadl_pd (__m128d __A, double const *__B)
+{
+  __v2df result = (__v2df)__A;
+  result [0] = *__B;
+  return (__m128d)result;
+}
+
+#ifdef _ARCH_PWR8
+/* Intrinsic functions that require PowerISA 2.07 minimum.  */
+
+/* Creates a 2-bit mask from the most significant bits of the DPFP values.  */
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_movemask_pd (__m128d  __A)
+{
+  __vector __m64 result;
+  static const __vector unsigned int perm_mask =
+    {
+#ifdef __LITTLE_ENDIAN__
+	0x80800040, 0x80808080, 0x80808080, 0x80808080
+#elif __BIG_ENDIAN__
+      0x80808080, 0x80808080, 0x80808080, 0x80800040
+#endif
+    };
+
+  result = (__vector __m64) vec_vbpermq ((__vector unsigned char) __A,
+					 (__vector unsigned char) perm_mask);
+
+#ifdef __LITTLE_ENDIAN__
+  return result[1];
+#elif __BIG_ENDIAN__
+  return result[0];
+#endif
+}
+#endif /* _ARCH_PWR8 */
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_packs_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_packs ((__v8hi) __A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_packs_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_packs ((__v4si)__A, (__v4si)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_packus_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_packsu ((__v8hi) __A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpackhi_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergel ((__v16qu)__A, (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpackhi_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergel ((__v8hu)__A, (__v8hu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpackhi_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergel ((__v4su)__A, (__v4su)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpackhi_epi64 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergel ((__vector long)__A, (__vector long)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergeh ((__v16qu)__A, (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergeh ((__v8hi)__A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergeh ((__v4si)__A, (__v4si)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_unpacklo_epi64 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_mergeh ((__vector long)__A, (__vector long)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v16qu)__A + (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v8hu)__A + (__v8hu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v4su)__A + (__v4su)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_add_epi64 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v2du)__A + (__v2du)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_adds_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_adds ((__v16qi)__A, (__v16qi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_adds_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_adds ((__v8hi)__A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_adds_epu8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_adds ((__v16qu)__A, (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_adds_epu16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_adds ((__v8hu)__A, (__v8hu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sub_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v16qu)__A - (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sub_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v8hu)__A - (__v8hu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sub_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v4su)__A - (__v4su)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sub_epi64 (__m128i __A, __m128i __B)
+{
+  return (__m128i) ((__v2du)__A - (__v2du)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_subs_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_subs ((__v16qi)__A, (__v16qi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_subs_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_subs ((__v8hi)__A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_subs_epu8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_subs ((__v16qu)__A, (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_subs_epu16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_subs ((__v8hu)__A, (__v8hu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_madd_epi16 (__m128i __A, __m128i __B)
+{
+  __vector signed int zero = {0, 0, 0, 0};
+
+  return (__m128i) vec_vmsumshm ((__v8hi)__A, (__v8hi)__B, zero);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mulhi_epi16 (__m128i __A, __m128i __B)
+{
+  __vector signed int w0, w1;
+
+  __vector unsigned char xform1 = {
+#ifdef __LITTLE_ENDIAN__
+      0x02, 0x03, 0x12, 0x13,  0x06, 0x07, 0x16, 0x17,
+      0x0A, 0x0B, 0x1A, 0x1B,  0x0E, 0x0F, 0x1E, 0x1F
+#elif __BIG_ENDIAN__
+      0x00, 0x01, 0x10, 0x11,  0x04, 0x05, 0x14, 0x15,
+      0x08, 0x09, 0x18, 0x19,  0x0C, 0x0D, 0x1C, 0x1D
+#endif
+    };
+
+  w0 = vec_vmulesh ((__v8hi)__A, (__v8hi)__B);
+  w1 = vec_vmulosh ((__v8hi)__A, (__v8hi)__B);
+  return (__m128i) vec_perm (w0, w1, xform1);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mullo_epi16 (__m128i __A, __m128i __B)
+{
+    return (__m128i) ((__v8hi)__A * (__v8hi)__B);
+}
+
+extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mul_su32 (__m64 __A, __m64 __B)
+{
+  unsigned int a = __A;
+  unsigned int b = __B;
+
+  return ((__m64)a * (__m64)b);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mul_epu32 (__m128i __A, __m128i __B)
+{
+#if __GNUC__ < 8
+  __v2du result;
+
+#ifdef __LITTLE_ENDIAN__
+  /* VMX Vector Multiply Odd Unsigned Word.  */
+  __asm__(
+      "vmulouw %0,%1,%2"
+      : "=v" (result)
+      : "v" (__A), "v" (__B)
+      : );
+#elif __BIG_ENDIAN__
+  /* VMX Vector Multiply Even Unsigned Word.  */
+  __asm__(
+      "vmuleuw %0,%1,%2"
+      : "=v" (result)
+      : "v" (__A), "v" (__B)
+      : );
+#endif
+  return (__m128i) result;
+#else
+#ifdef __LITTLE_ENDIAN__
+  return (__m128i) vec_mule ((__v4su)__A, (__v4su)__B);
+#elif __BIG_ENDIAN__
+  return (__m128i) vec_mulo ((__v4su)__A, (__v4su)__B);
+#endif
+#endif
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_slli_epi16 (__m128i __A, int __B)
+{
+  __v8hu lshift;
+  __v8hi result = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+  if (__B < 16)
+    {
+      if (__builtin_constant_p(__B))
+	  lshift = (__v8hu) vec_splat_s16(__B);
+      else
+	  lshift = vec_splats ((unsigned short) __B);
+
+      result = vec_vslh ((__v8hi) __A, lshift);
+    }
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_slli_epi32 (__m128i __A, int __B)
+{
+  __v4su lshift;
+  __v4si result = { 0, 0, 0, 0 };
+
+  if (__B < 32)
+    {
+      if (__builtin_constant_p(__B))
+	lshift = (__v4su) vec_splat_s32(__B);
+      else
+	lshift = vec_splats ((unsigned int) __B);
+
+      result = vec_vslw ((__v4si) __A, lshift);
+    }
+
+  return (__m128i) result;
+}
+
+#ifdef _ARCH_PWR8
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_slli_epi64 (__m128i __A, int __B)
+{
+  __v2du lshift;
+  __v2di result = { 0, 0 };
+
+  if (__B < 64)
+    {
+      if (__builtin_constant_p(__B))
+	{
+	  if (__B < 32)
+	      lshift = (__v2du) vec_splat_s32(__B);
+	    else
+	      lshift = (__v2du) vec_splats((unsigned long long)__B);
+	}
+      else
+	  lshift = (__v2du) vec_splats ((unsigned int) __B);
+
+      result = vec_vsld ((__v2di) __A, lshift);
+    }
+
+  return (__m128i) result;
+}
+#endif
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srai_epi16 (__m128i __A, int __B)
+{
+  __v8hu rshift = { 15, 15, 15, 15, 15, 15, 15, 15 };
+  __v8hi result;
+
+  if (__B < 16)
+    {
+      if (__builtin_constant_p(__B))
+	rshift = (__v8hu) vec_splat_s16(__B);
+      else
+	rshift = vec_splats ((unsigned short) __B);
+    }
+  result = vec_vsrah ((__v8hi) __A, rshift);
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srai_epi32 (__m128i __A, int __B)
+{
+  __v4su rshift = { 31, 31, 31, 31 };
+  __v4si result;
+
+  if (__B < 32)
+    {
+      if (__builtin_constant_p(__B))
+	{
+	  if (__B < 16)
+	      rshift = (__v4su) vec_splat_s32(__B);
+	    else
+	      rshift = (__v4su) vec_splats((unsigned int)__B);
+	}
+      else
+	rshift = vec_splats ((unsigned int) __B);
+    }
+  result = vec_vsraw ((__v4si) __A, rshift);
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_bslli_si128 (__m128i __A, const int __N)
+{
+  __v16qu result;
+  const __v16qu zeros = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+  if (__N < 16)
+    result = vec_sld ((__v16qu) __A, zeros, __N);
+  else
+    result = zeros;
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_bsrli_si128 (__m128i __A, const int __N)
+{
+  __v16qu result;
+  const __v16qu zeros = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+  if (__N < 16)
+    if (__builtin_constant_p(__N))
+      /* Would like to use Vector Shift Left Double by Octet
+	 Immediate here to use the immediate form and avoid
+	 load of __N * 8 value into a separate VR.  */
+      result = vec_sld (zeros, (__v16qu) __A, (16 - __N));
+    else
+      {
+	__v16qu shift = vec_splats((unsigned char)(__N*8));
+	result = vec_sro ((__v16qu)__A, shift);
+      }
+  else
+    result = zeros;
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srli_si128 (__m128i __A, const int __N)
+{
+  return _mm_bsrli_si128 (__A, __N);
+}
+
+extern __inline  __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_slli_si128 (__m128i __A, const int _imm5)
+{
+  __v16qu result;
+  const __v16qu zeros = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+  if (_imm5 < 16)
+#ifdef __LITTLE_ENDIAN__
+    result = vec_sld ((__v16qu) __A, zeros, _imm5);
+#elif __BIG_ENDIAN__
+    result = vec_sld (zeros, (__v16qu) __A, (16 - _imm5));
+#endif
+  else
+    result = zeros;
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+
+_mm_srli_epi16 (__m128i  __A, int __B)
+{
+  __v8hu rshift;
+  __v8hi result = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+  if (__B < 16)
+    {
+      if (__builtin_constant_p(__B))
+	rshift = (__v8hu) vec_splat_s16(__B);
+      else
+	rshift = vec_splats ((unsigned short) __B);
+
+      result = vec_vsrh ((__v8hi) __A, rshift);
+    }
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srli_epi32 (__m128i __A, int __B)
+{
+  __v4su rshift;
+  __v4si result = { 0, 0, 0, 0 };
+
+  if (__B < 32)
+    {
+      if (__builtin_constant_p(__B))
+	{
+	  if (__B < 16)
+	      rshift = (__v4su) vec_splat_s32(__B);
+	    else
+	      rshift = (__v4su) vec_splats((unsigned int)__B);
+	}
+      else
+	rshift = vec_splats ((unsigned int) __B);
+
+      result = vec_vsrw ((__v4si) __A, rshift);
+    }
+
+  return (__m128i) result;
+}
+
+#ifdef _ARCH_PWR8
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srli_epi64 (__m128i __A, int __B)
+{
+  __v2du rshift;
+  __v2di result = { 0, 0 };
+
+  if (__B < 64)
+    {
+      if (__builtin_constant_p(__B))
+	{
+	  if (__B < 16)
+	      rshift = (__v2du) vec_splat_s32(__B);
+	    else
+	      rshift = (__v2du) vec_splats((unsigned long long)__B);
+	}
+      else
+	rshift = (__v2du) vec_splats ((unsigned int) __B);
+
+      result = vec_vsrd ((__v2di) __A, rshift);
+    }
+
+  return (__m128i) result;
+}
+#endif
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sll_epi16 (__m128i __A, __m128i __B)
+{
+  __v8hu lshift, shmask;
+  const __v8hu shmax = { 15, 15, 15, 15, 15, 15, 15, 15 };
+  __v8hu result;
+
+#ifdef __LITTLE_ENDIAN__
+  lshift = vec_splat ((__v8hu)__B, 0);
+#elif __BIG_ENDIAN__
+  lshift = vec_splat ((__v8hu)__B, 3);
+#endif
+  shmask = lshift <= shmax;
+  result = vec_vslh ((__v8hu) __A, lshift);
+  result = vec_sel (shmask, result, shmask);
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sll_epi32 (__m128i __A, __m128i __B)
+{
+  __v4su lshift, shmask;
+  const __v4su shmax = { 32, 32, 32, 32 };
+  __v4su result;
+#ifdef __LITTLE_ENDIAN__
+  lshift = vec_splat ((__v4su)__B, 0);
+#elif __BIG_ENDIAN__
+  lshift = vec_splat ((__v4su)__B, 1);
+#endif
+  shmask = lshift < shmax;
+  result = vec_vslw ((__v4su) __A, lshift);
+  result = vec_sel (shmask, result, shmask);
+
+  return (__m128i) result;
+}
+
+#ifdef _ARCH_PWR8
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sll_epi64 (__m128i __A, __m128i __B)
+{
+  __v2du lshift, shmask;
+  const __v2du shmax = { 64, 64 };
+  __v2du result;
+
+  lshift = (__v2du) vec_splat ((__v2du)__B, 0);
+  shmask = lshift < shmax;
+  result = vec_vsld ((__v2du) __A, lshift);
+  result = (__v2du) vec_sel ((__v2df) shmask, (__v2df) result,
+			      (__v2df) shmask);
+
+  return (__m128i) result;
+}
+#endif
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sra_epi16 (__m128i __A, __m128i __B)
+{
+  const __v8hu rshmax = { 15, 15, 15, 15, 15, 15, 15, 15 };
+  __v8hu rshift;
+  __v8hi result;
+
+#ifdef __LITTLE_ENDIAN__
+  rshift = vec_splat ((__v8hu)__B, 0);
+#elif __BIG_ENDIAN__
+  rshift = vec_splat ((__v8hu)__B, 3);
+#endif
+  rshift = vec_min (rshift, rshmax);
+  result = vec_vsrah ((__v8hi) __A, rshift);
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sra_epi32 (__m128i __A, __m128i __B)
+{
+  const __v4su rshmax = { 31, 31, 31, 31 };
+  __v4su rshift;
+  __v4si result;
+
+#ifdef __LITTLE_ENDIAN__
+  rshift = vec_splat ((__v4su)__B, 0);
+#elif __BIG_ENDIAN__
+  rshift = vec_splat ((__v4su)__B, 1);
+#endif
+  rshift = vec_min (rshift, rshmax);
+  result = vec_vsraw ((__v4si) __A, rshift);
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srl_epi16 (__m128i __A, __m128i __B)
+{
+  __v8hu rshift, shmask;
+  const __v8hu shmax = { 15, 15, 15, 15, 15, 15, 15, 15 };
+  __v8hu result;
+
+#ifdef __LITTLE_ENDIAN__
+  rshift = vec_splat ((__v8hu)__B, 0);
+#elif __BIG_ENDIAN__
+  rshift = vec_splat ((__v8hu)__B, 3);
+#endif
+  shmask = rshift <= shmax;
+  result = vec_vsrh ((__v8hu) __A, rshift);
+  result = vec_sel (shmask, result, shmask);
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srl_epi32 (__m128i __A, __m128i __B)
+{
+  __v4su rshift, shmask;
+  const __v4su shmax = { 32, 32, 32, 32 };
+  __v4su result;
+
+#ifdef __LITTLE_ENDIAN__
+  rshift = vec_splat ((__v4su)__B, 0);
+#elif __BIG_ENDIAN__
+  rshift = vec_splat ((__v4su)__B, 1);
+#endif
+  shmask = rshift < shmax;
+  result = vec_vsrw ((__v4su) __A, rshift);
+  result = vec_sel (shmask, result, shmask);
+
+  return (__m128i) result;
+}
+
+#ifdef _ARCH_PWR8
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_srl_epi64 (__m128i __A, __m128i __B)
+{
+  __v2du rshift, shmask;
+  const __v2du shmax = { 64, 64 };
+  __v2du result;
+
+  rshift = (__v2du) vec_splat ((__v2du)__B, 0);
+  shmask = rshift < shmax;
+  result = vec_vsrd ((__v2du) __A, rshift);
+  result = (__v2du)vec_sel ((__v2du)shmask, (__v2du)result, (__v2du)shmask);
+
+  return (__m128i) result;
+}
+#endif
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_and_pd (__m128d __A, __m128d __B)
+{
+  return (vec_and ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_andnot_pd (__m128d __A, __m128d __B)
+{
+  return (vec_andc ((__v2df) __B, (__v2df) __A));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_or_pd (__m128d __A, __m128d __B)
+{
+  return (vec_or ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_xor_pd (__m128d __A, __m128d __B)
+{
+  return (vec_xor ((__v2df) __A, (__v2df) __B));
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpeq_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmpeq ((__v16qi) __A, (__v16qi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpeq_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmpeq ((__v8hi) __A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpeq_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmpeq ((__v4si) __A, (__v4si)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmplt_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmplt ((__v16qi) __A, (__v16qi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmplt_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmplt ((__v8hi) __A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmplt_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmplt ((__v4si) __A, (__v4si)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpgt_epi8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmpgt ((__v16qi) __A, (__v16qi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpgt_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmpgt ((__v8hi) __A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmpgt_epi32 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_cmpgt ((__v4si) __A, (__v4si)__B);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_extract_epi16 (__m128i const __A, int const __N)
+{
+  return (unsigned short) ((__v8hi)__A)[__N & 7];
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_insert_epi16 (__m128i const __A, int const __D, int const __N)
+{
+  __v8hi result = (__v8hi)__A;
+
+  result [(__N & 7)] = __D;
+
+  return (__m128i) result;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_max_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_max ((__v8hi)__A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_max_epu8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_max ((__v16qu) __A, (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_min_epi16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_min ((__v8hi) __A, (__v8hi)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_min_epu8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_min ((__v16qu) __A, (__v16qu)__B);
+}
+
+
+#ifdef _ARCH_PWR8
+/* Intrinsic functions that require PowerISA 2.07 minimum.  */
+
+/* Creates a 4-bit mask from the most significant bits of the SPFP values.  */
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_movemask_epi8 (__m128i __A)
+{
+  __vector __m64 result;
+  static const __vector unsigned char perm_mask =
+    {
+#ifdef __LITTLE_ENDIAN__
+	0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40,
+	0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00
+#elif __BIG_ENDIAN__
+	0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38,
+	0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78
+#endif
+    };
+
+  result = (__vector __m64) vec_vbpermq ((__vector unsigned char) __A,
+					 (__vector unsigned char) perm_mask);
+
+#ifdef __LITTLE_ENDIAN__
+  return result[1];
+#elif __BIG_ENDIAN__
+  return result[0];
+#endif
+}
+#endif /* _ARCH_PWR8 */
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mulhi_epu16 (__m128i __A, __m128i __B)
+{
+  __v4su w0, w1;
+  __v16qu xform1 = {
+#ifdef __LITTLE_ENDIAN__
+      0x02, 0x03, 0x12, 0x13,  0x06, 0x07, 0x16, 0x17,
+      0x0A, 0x0B, 0x1A, 0x1B,  0x0E, 0x0F, 0x1E, 0x1F
+#elif __BIG_ENDIAN__
+      0x00, 0x01, 0x10, 0x11,  0x04, 0x05, 0x14, 0x15,
+      0x08, 0x09, 0x18, 0x19,  0x0C, 0x0D, 0x1C, 0x1D
+#endif
+    };
+
+  w0 = vec_vmuleuh ((__v8hu)__A, (__v8hu)__B);
+  w1 = vec_vmulouh ((__v8hu)__A, (__v8hu)__B);
+  return (__m128i) vec_perm (w0, w1, xform1);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_shufflehi_epi16 (__m128i __A, const int __mask)
+{
+  unsigned long element_selector_98 = __mask & 0x03;
+  unsigned long element_selector_BA = (__mask >> 2) & 0x03;
+  unsigned long element_selector_DC = (__mask >> 4) & 0x03;
+  unsigned long element_selector_FE = (__mask >> 6) & 0x03;
+  static const unsigned short permute_selectors[4] =
+    {
+#ifdef __LITTLE_ENDIAN__
+	      0x0908, 0x0B0A, 0x0D0C, 0x0F0E
+#elif __BIG_ENDIAN__
+	      0x0607, 0x0405, 0x0203, 0x0001
+#endif
+    };
+  __v2du pmask =
+#ifdef __LITTLE_ENDIAN__
+      { 0x1716151413121110UL,  0x1f1e1d1c1b1a1918UL};
+#elif __BIG_ENDIAN__
+      { 0x1011121314151617UL,  0x18191a1b1c1d1e1fUL};
+#endif
+  __m64_union t;
+  __v2du a, r;
+
+#ifdef __LITTLE_ENDIAN__
+  t.as_short[0] = permute_selectors[element_selector_98];
+  t.as_short[1] = permute_selectors[element_selector_BA];
+  t.as_short[2] = permute_selectors[element_selector_DC];
+  t.as_short[3] = permute_selectors[element_selector_FE];
+#elif __BIG_ENDIAN__
+  t.as_short[3] = permute_selectors[element_selector_98];
+  t.as_short[2] = permute_selectors[element_selector_BA];
+  t.as_short[1] = permute_selectors[element_selector_DC];
+  t.as_short[0] = permute_selectors[element_selector_FE];
+#endif
+#ifdef __LITTLE_ENDIAN__
+  pmask[1] = t.as_m64;
+#elif __BIG_ENDIAN__
+  pmask[0] = t.as_m64;
+#endif
+  a = (__v2du)__A;
+  r = vec_perm (a, a, (__vector unsigned char)pmask);
+  return (__m128i) r;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_shufflelo_epi16 (__m128i __A, const int __mask)
+{
+  unsigned long element_selector_10 = __mask & 0x03;
+  unsigned long element_selector_32 = (__mask >> 2) & 0x03;
+  unsigned long element_selector_54 = (__mask >> 4) & 0x03;
+  unsigned long element_selector_76 = (__mask >> 6) & 0x03;
+  static const unsigned short permute_selectors[4] =
+    {
+#ifdef __LITTLE_ENDIAN__
+	      0x0100, 0x0302, 0x0504, 0x0706
+#elif __BIG_ENDIAN__
+	      0x0e0f, 0x0c0d, 0x0a0b, 0x0809
+#endif
+    };
+  __v2du pmask = { 0x1011121314151617UL,  0x1f1e1d1c1b1a1918UL};
+  __m64_union t;
+  __v2du a, r;
+
+#ifdef __LITTLE_ENDIAN__
+  t.as_short[0] = permute_selectors[element_selector_10];
+  t.as_short[1] = permute_selectors[element_selector_32];
+  t.as_short[2] = permute_selectors[element_selector_54];
+  t.as_short[3] = permute_selectors[element_selector_76];
+#elif __BIG_ENDIAN__
+  t.as_short[3] = permute_selectors[element_selector_10];
+  t.as_short[2] = permute_selectors[element_selector_32];
+  t.as_short[1] = permute_selectors[element_selector_54];
+  t.as_short[0] = permute_selectors[element_selector_76];
+#endif
+#ifdef __LITTLE_ENDIAN__
+  pmask[0] = t.as_m64;
+#elif __BIG_ENDIAN__
+  pmask[1] = t.as_m64;
+#endif
+  a = (__v2du)__A;
+  r = vec_perm (a, a, (__vector unsigned char)pmask);
+  return (__m128i) r;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_shuffle_epi32 (__m128i __A, const int __mask)
+{
+  unsigned long element_selector_10 = __mask & 0x03;
+  unsigned long element_selector_32 = (__mask >> 2) & 0x03;
+  unsigned long element_selector_54 = (__mask >> 4) & 0x03;
+  unsigned long element_selector_76 = (__mask >> 6) & 0x03;
+  static const unsigned int permute_selectors[4] =
+    {
+#ifdef __LITTLE_ENDIAN__
+	0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
+#elif __BIG_ENDIAN__
+      0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203
+#endif
+    };
+  __v4su t;
+
+#ifdef __LITTLE_ENDIAN__
+  t[0] = permute_selectors[element_selector_10];
+  t[1] = permute_selectors[element_selector_32];
+  t[2] = permute_selectors[element_selector_54] + 0x10101010;
+  t[3] = permute_selectors[element_selector_76] + 0x10101010;
+#elif __BIG_ENDIAN__
+  t[3] = permute_selectors[element_selector_10] + 0x10101010;
+  t[2] = permute_selectors[element_selector_32] + 0x10101010;
+  t[1] = permute_selectors[element_selector_54];
+  t[0] = permute_selectors[element_selector_76];
+#endif
+  return (__m128i)vec_perm ((__v4si) __A, (__v4si)__A, (__vector unsigned char)t);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskmoveu_si128 (__m128i __A, __m128i __B, char *__C)
+{
+  __v2du hibit = { 0x7f7f7f7f7f7f7f7fUL, 0x7f7f7f7f7f7f7f7fUL};
+  __v16qu mask, tmp;
+  __m128i *p = (__m128i*)__C;
+
+  tmp = (__v16qu)_mm_loadu_si128(p);
+  mask = (__v16qu)vec_cmpgt ((__v16qu)__B, (__v16qu)hibit);
+  tmp = vec_sel (tmp, (__v16qu)__A, mask);
+  _mm_storeu_si128 (p, (__m128i)tmp);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_avg_epu8 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_avg ((__v16qu)__A, (__v16qu)__B);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_avg_epu16 (__m128i __A, __m128i __B)
+{
+  return (__m128i) vec_avg ((__v8hu)__A, (__v8hu)__B);
+}
+
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_sad_epu8 (__m128i __A, __m128i __B)
+{
+  __v16qu a, b;
+  __v16qu vmin, vmax, vabsdiff;
+  __v4si vsum;
+  const __v4su zero = { 0, 0, 0, 0 };
+  __v4si result;
+
+  a = (__v16qu) __A;
+  b = (__v16qu) __B;
+  vmin = vec_min (a, b);
+  vmax = vec_max (a, b);
+  vabsdiff = vec_sub (vmax, vmin);
+  /* Sum four groups of bytes into integers.  */
+  vsum = (__vector signed int) vec_sum4s (vabsdiff, zero);
+  /* Sum across four integers with two integer results.  */
+  result = vec_sum2s (vsum, (__vector signed int) zero);
+  /* Rotate the sums into the correct position.  */
+#ifdef __LITTLE_ENDIAN__
+  result = vec_sld (result, result, 4);
+#elif __BIG_ENDIAN__
+  result = vec_sld (result, result, 6);
+#endif
+  /* Rotate the sums into the correct position.  */
+  return (__m128i) result;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_stream_si32 (int *__A, int __B)
+{
+  /* Use the data cache block touch for store transient.  */
+  __asm__ (
+    "dcbtstt 0,%0"
+    :
+    : "b" (__A)
+    : "memory"
+  );
+  *__A = __B;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_stream_si64 (long long int *__A, long long int __B)
+{
+  /* Use the data cache block touch for store transient.  */
+  __asm__ (
+    "	dcbtstt	0,%0"
+    :
+    : "b" (__A)
+    : "memory"
+  );
+  *__A = __B;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_stream_si128 (__m128i *__A, __m128i __B)
+{
+  /* Use the data cache block touch for store transient.  */
+  __asm__ (
+    "dcbtstt 0,%0"
+    :
+    : "b" (__A)
+    : "memory"
+  );
+  *__A = __B;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_stream_pd (double *__A, __m128d __B)
+{
+  /* Use the data cache block touch for store transient.  */
+  __asm__ (
+    "dcbtstt 0,%0"
+    :
+    : "b" (__A)
+    : "memory"
+  );
+  *(__m128d*)__A = __B;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_clflush (void const *__A)
+{
+  /* Use the data cache block flush.  */
+  __asm__ (
+    "dcbf 0,%0"
+    :
+    : "b" (__A)
+    : "memory"
+  );
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_lfence (void)
+{
+  /* Use light weight sync for load to load ordering.  */
+  __atomic_thread_fence (__ATOMIC_RELEASE);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mfence (void)
+{
+  /* Use heavy weight sync for any to any ordering.  */
+  __atomic_thread_fence (__ATOMIC_SEQ_CST);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi32_si128 (int __A)
+{
+  return _mm_set_epi32 (0, 0, 0, __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi64_si128 (long long __A)
+{
+  return __extension__ (__m128i)(__v2di){ __A, 0LL };
+}
+
+/* Microsoft intrinsic.  */
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cvtsi64x_si128 (long long __A)
+{
+  return __extension__ (__m128i)(__v2di){ __A, 0LL };
+}
+
+/* Casts between various SP, DP, INT vector types.  Note that these do no
+   conversion of values, they just change the type.  */
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_castpd_ps(__m128d __A)
+{
+  return (__m128) __A;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_castpd_si128(__m128d __A)
+{
+  return (__m128i) __A;
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_castps_pd(__m128 __A)
+{
+  return (__m128d) __A;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_castps_si128(__m128 __A)
+{
+  return (__m128i) __A;
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_castsi128_ps(__m128i __A)
+{
+  return (__m128) __A;
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_castsi128_pd(__m128i __A)
+{
+  return (__m128d) __A;
+}
+
+#endif /* EMMINTRIN_H_ */
diff --git a/gcc/config/rs6000/rs6000-builtin.def b/gcc/config/rs6000/rs6000-builtin.def
index ac9ddae3ef0..c8a425cba7e 100644
--- a/gcc/config/rs6000/rs6000-builtin.def
+++ b/gcc/config/rs6000/rs6000-builtin.def
@@ -961,7 +961,7 @@ BU_SPECIAL_X (RS6000_BUILTIN_NONE, NULL, 0, RS6000_BTC_MISC)
 BU_ALTIVEC_3 (VMADDFP,        "vmaddfp",        FP,    	fmav4sf4)
 BU_ALTIVEC_3 (VMHADDSHS,      "vmhaddshs",      SAT,   	altivec_vmhaddshs)
 BU_ALTIVEC_3 (VMHRADDSHS,     "vmhraddshs",     SAT,   	altivec_vmhraddshs)
-BU_ALTIVEC_3 (VMLADDUHM,      "vmladduhm",      CONST, 	altivec_vmladduhm)
+BU_ALTIVEC_3 (VMLADDUHM,      "vmladduhm",      CONST, 	fmav8hi4)
 BU_ALTIVEC_3 (VMSUMUBM,       "vmsumubm",       CONST, 	altivec_vmsumubm)
 BU_ALTIVEC_3 (VMSUMMBM,       "vmsummbm",       CONST, 	altivec_vmsummbm)
 BU_ALTIVEC_3 (VMSUMUHM,       "vmsumuhm",       CONST, 	altivec_vmsumuhm)
@@ -2374,17 +2374,13 @@ BU_FLOAT128_1 (FABSQ,		"fabsq",       CONST, abskf2)
 BU_FLOAT128_2 (COPYSIGNQ,	"copysignq",   CONST, copysignkf3)
 
 /* 1, 2, and 3 argument IEEE 128-bit floating point functions that require ISA
-   3.0 hardware.  These functions use the new 'f128' suffix.  Eventually the
-   standard functions should be folded into the common built-in function
-   handling. */
-BU_FLOAT128_1_HW (SQRTF128,	 "sqrtf128",		   CONST, sqrtkf2)
+   3.0 hardware.  These functions use the new 'f128' suffix.  */
 BU_FLOAT128_1_HW (SQRTF128_ODD,	 "sqrtf128_round_to_odd",  CONST, sqrtkf2_odd)
 BU_FLOAT128_1_HW (TRUNCF128_ODD, "truncf128_round_to_odd", CONST, trunckfdf2_odd)
 BU_FLOAT128_2_HW (ADDF128_ODD,	 "addf128_round_to_odd",   CONST, addkf3_odd)
 BU_FLOAT128_2_HW (SUBF128_ODD,	 "subf128_round_to_odd",   CONST, subkf3_odd)
 BU_FLOAT128_2_HW (MULF128_ODD,	 "mulf128_round_to_odd",   CONST, mulkf3_odd)
 BU_FLOAT128_2_HW (DIVF128_ODD,	 "divf128_round_to_odd",   CONST, divkf3_odd)
-BU_FLOAT128_3_HW (FMAF128,	 "fmaf128",		   CONST, fmakf4_hw)
 BU_FLOAT128_3_HW (FMAF128_ODD,	 "fmaf128_round_to_odd",   CONST, fmakf4_odd)
 
 /* 1 argument crypto functions.  */
diff --git a/gcc/config/rs6000/rs6000-protos.h b/gcc/config/rs6000/rs6000-protos.h
index db0e692739c..721b906ee65 100644
--- a/gcc/config/rs6000/rs6000-protos.h
+++ b/gcc/config/rs6000/rs6000-protos.h
@@ -124,7 +124,6 @@ extern void print_operand_address (FILE *, rtx);
 extern enum rtx_code rs6000_reverse_condition (machine_mode,
 					       enum rtx_code);
 extern rtx rs6000_emit_eqne (machine_mode, rtx, rtx, rtx);
-extern void rs6000_emit_sISEL (machine_mode, rtx[]);
 extern void rs6000_emit_sCOND (machine_mode, rtx[]);
 extern void rs6000_emit_cbranch (machine_mode, rtx[]);
 extern char * output_cbranch (rtx, const char *, int, rtx_insn *);
@@ -132,6 +131,7 @@ extern const char * output_probe_stack_range (rtx, rtx, rtx);
 extern void rs6000_emit_dot_insn (rtx dst, rtx src, int dot, rtx ccreg);
 extern bool rs6000_emit_set_const (rtx, rtx);
 extern int rs6000_emit_cmove (rtx, rtx, rtx, rtx);
+extern int rs6000_emit_int_cmove (rtx, rtx, rtx, rtx);
 extern int rs6000_emit_vector_cond_expr (rtx, rtx, rtx, rtx, rtx, rtx);
 extern void rs6000_emit_minmax (rtx, enum rtx_code, rtx, rtx);
 extern void rs6000_split_signbit (rtx, rtx);
diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c
index e02b0863dbf..6402c0386a6 100644
--- a/gcc/config/rs6000/rs6000.c
+++ b/gcc/config/rs6000/rs6000.c
@@ -88,6 +88,20 @@
 #define TARGET_NO_PROTOTYPE 0
 #endif
 
+  /* Set -mabi=ieeelongdouble on some old targets.  In the future, power server
+     systems will also set long double to be IEEE 128-bit.  AIX and Darwin
+     explicitly redefine TARGET_IEEEQUAD and TARGET_IEEEQUAD_DEFAULT to 0, so
+     those systems will not pick up this default.  This needs to be after all
+     of the include files, so that POWERPC_LINUX and POWERPC_FREEBSD are
+     properly defined.  */
+#ifndef TARGET_IEEEQUAD_DEFAULT
+#if !defined (POWERPC_LINUX) && !defined (POWERPC_FREEBSD)
+#define TARGET_IEEEQUAD_DEFAULT 1
+#else
+#define TARGET_IEEEQUAD_DEFAULT 0
+#endif
+#endif
+
 #define min(A,B)	((A) < (B) ? (A) : (B))
 #define max(A,B)	((A) > (B) ? (A) : (B))
 
@@ -1345,7 +1359,6 @@ static void rs6000_common_init_builtins (void);
 static void paired_init_builtins (void);
 static rtx paired_expand_predicate_builtin (enum insn_code, tree, rtx);
 static void htm_init_builtins (void);
-static int rs6000_emit_int_cmove (rtx, rtx, rtx, rtx);
 static rs6000_stack_t *rs6000_stack_info (void);
 static void is_altivec_return_reg (rtx, void *);
 int easy_vector_constant (rtx, machine_mode);
@@ -2880,6 +2893,13 @@ rs6000_debug_reg_global (void)
   fprintf (stderr, DEBUG_FMT_D, "tls_size", rs6000_tls_size);
   fprintf (stderr, DEBUG_FMT_D, "long_double_size",
 	   rs6000_long_double_type_size);
+  if (rs6000_long_double_type_size == 128)
+    {
+      fprintf (stderr, DEBUG_FMT_S, "long double type",
+	       TARGET_IEEEQUAD ? "IEEE" : "IBM");
+      fprintf (stderr, DEBUG_FMT_S, "default long double type",
+	       TARGET_IEEEQUAD_DEFAULT ? "IEEE" : "IBM");
+    }
   fprintf (stderr, DEBUG_FMT_D, "sched_restricted_insns_priority",
 	   (int)rs6000_sched_restricted_insns_priority);
   fprintf (stderr, DEBUG_FMT_D, "Number of standard builtins",
@@ -4562,13 +4582,26 @@ rs6000_option_override_internal (bool global_init_p)
 	rs6000_long_double_type_size = RS6000_DEFAULT_LONG_DOUBLE_SIZE;
     }
 
-  /* Set -mabi=ieeelongdouble on some old targets.  Note, AIX and Darwin
-     explicitly redefine TARGET_IEEEQUAD to 0, so those systems will not
-     pick up this default.  */
-#if !defined (POWERPC_LINUX) && !defined (POWERPC_FREEBSD)
+  /* Set -mabi=ieeelongdouble on some old targets.  In the future, power server
+     systems will also set long double to be IEEE 128-bit.  AIX and Darwin
+     explicitly redefine TARGET_IEEEQUAD and TARGET_IEEEQUAD_DEFAULT to 0, so
+     those systems will not pick up this default.  Warn if the user changes the
+     default unless -Wno-psabi.  */
   if (!global_options_set.x_rs6000_ieeequad)
-    rs6000_ieeequad = 1;
-#endif
+    rs6000_ieeequad = TARGET_IEEEQUAD_DEFAULT;
+
+  else if (rs6000_ieeequad != TARGET_IEEEQUAD_DEFAULT && TARGET_LONG_DOUBLE_128)
+    {
+      static bool warned_change_long_double;
+      if (!warned_change_long_double)
+	{
+	  warned_change_long_double = true;
+	  if (TARGET_IEEEQUAD)
+	    warning (OPT_Wpsabi, "Using IEEE extended precision long double");
+	  else
+	    warning (OPT_Wpsabi, "Using IBM extended precision long double");
+	}
+    }
 
   /* Enable the default support for IEEE 128-bit floating point on Linux VSX
      sytems.  In GCC 7, we would enable the the IEEE 128-bit floating point
@@ -5424,9 +5457,7 @@ rs6000_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
         return 3;
 
       case unaligned_load:
-	if (TARGET_P9_VECTOR)
-	  return 3;
-
+      case vector_gather_load:
 	if (TARGET_EFFICIENT_UNALIGNED_VSX)
 	  return 1;
 
@@ -5465,6 +5496,7 @@ rs6000_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
         return 2;
 
       case unaligned_store:
+      case vector_scatter_store:
 	if (TARGET_EFFICIENT_UNALIGNED_VSX)
 	  return 1;
 
@@ -8991,6 +9023,8 @@ rs6000_delegitimize_address (rtx orig_x)
 static bool
 rs6000_const_not_ok_for_debug_p (rtx x)
 {
+  if (GET_CODE (x) == UNSPEC)
+    return true;
   if (GET_CODE (x) == SYMBOL_REF
       && CONSTANT_POOL_ADDRESS_P (x))
     {
@@ -16614,6 +16648,22 @@ rs6000_gimple_fold_builtin (gimple_stmt_iterator *gsi)
 	 gsi_replace (gsi, g, true);
 	 return true;
       }
+
+    /* Vector Fused multiply-add (fma).  */
+    case ALTIVEC_BUILTIN_VMADDFP:
+    case VSX_BUILTIN_XVMADDDP:
+    case ALTIVEC_BUILTIN_VMLADDUHM:
+      {
+       arg0 = gimple_call_arg (stmt, 0);
+       arg1 = gimple_call_arg (stmt, 1);
+       tree arg2 = gimple_call_arg (stmt, 2);
+       lhs = gimple_call_lhs (stmt);
+       gimple *g = gimple_build_assign (lhs, FMA_EXPR , arg0, arg1, arg2);
+       gimple_set_location (g, gimple_location (stmt));
+       gsi_replace (gsi, g, true);
+       return true;
+      }
+
     default:
 	if (TARGET_DEBUG_BUILTIN)
 	   fprintf (stderr, "gimple builtin intrinsic not matched:%d %s %s\n",
@@ -22429,14 +22479,6 @@ rs6000_expand_float128_convert (rtx dest, rtx src, bool unsigned_p)
 }
 
 
-/* Emit the RTL for an sISEL pattern.  */
-
-void
-rs6000_emit_sISEL (machine_mode mode ATTRIBUTE_UNUSED, rtx operands[])
-{
-  rs6000_emit_int_cmove (operands[0], operands[1], const1_rtx, const0_rtx);
-}
-
 /* Emit RTL that sets a register to zero if OP1 and OP2 are equal.  SCRATCH
    can be used as that dest register.  Return the dest register.  */
 
@@ -23212,7 +23254,7 @@ rs6000_emit_cmove (rtx dest, rtx op, rtx true_cond, rtx false_cond)
 
 /* Same as above, but for ints (isel).  */
 
-static int
+int
 rs6000_emit_int_cmove (rtx dest, rtx op, rtx true_cond, rtx false_cond)
 {
   rtx condition_rtx, cr;
diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
index aad382ced33..ed5ff397e07 100644
--- a/gcc/config/rs6000/rs6000.md
+++ b/gcc/config/rs6000/rs6000.md
@@ -3829,11 +3829,19 @@
 
 ; Special case for less-than-0.  We can do it with just one machine
 ; instruction, but the generic optimizers do not realise it is cheap.
-(define_insn "*lt0_disi"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-	(lt:DI (match_operand:SI 1 "gpc_reg_operand" "r")
-	       (const_int 0)))]
+(define_insn "*lt0_<mode>di"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+	(lt:GPR (match_operand:DI 1 "gpc_reg_operand" "r")
+		(const_int 0)))]
   "TARGET_POWERPC64"
+  "srdi %0,%1,63"
+  [(set_attr "type" "shift")])
+
+(define_insn "*lt0_<mode>si"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+	(lt:GPR (match_operand:SI 1 "gpc_reg_operand" "r")
+		(const_int 0)))]
+  ""
   "rlwinm %0,%1,1,31,31"
   [(set_attr "type" "shift")])
 
@@ -10333,6 +10341,9 @@
 	{
 	  rtx loop_lab, end_loop;
 	  bool rotated = CONST_INT_P (rounded_size);
+	  rtx update = GEN_INT (-probe_interval);
+	  if (probe_interval > 32768)
+	    update = force_reg (Pmode, update);
 
 	  emit_stack_clash_protection_probe_loop_start (&loop_lab, &end_loop,
 							last_addr, rotated);
@@ -10340,13 +10351,11 @@
 	  if (Pmode == SImode)
 	    emit_insn (gen_movsi_update_stack (stack_pointer_rtx,
 					       stack_pointer_rtx,
-					       GEN_INT (-probe_interval),
-					       chain));
+					       update, chain));
 	  else
 	    emit_insn (gen_movdi_di_update_stack (stack_pointer_rtx,
 					          stack_pointer_rtx,
-					          GEN_INT (-probe_interval),
-					          chain));
+					          update, chain));
 	  emit_stack_clash_protection_probe_loop_end (loop_lab, end_loop,
 						      last_addr, rotated);
 	}
@@ -11157,7 +11166,7 @@
   [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
 	 (match_operand 1 "" "g,g"))
    (use (match_operand:P 2 "memory_operand" "<ptrm>,<ptrm>"))
-   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (set (reg:P TOC_REGNUM) (unspec:P [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_AIX"
   "<ptrload> 2,%2\;b%T0l\;<ptrload> 2,%3(1)"
@@ -11169,7 +11178,7 @@
 	(call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
 	      (match_operand 2 "" "g,g")))
    (use (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
-   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 4 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (set (reg:P TOC_REGNUM) (unspec:P [(match_operand:P 4 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_AIX"
   "<ptrload> 2,%3\;b%T1l\;<ptrload> 2,%4(1)"
@@ -11183,7 +11192,7 @@
 (define_insn "*call_indirect_elfv2<mode>"
   [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
 	 (match_operand 1 "" "g,g"))
-   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 2 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (set (reg:P TOC_REGNUM) (unspec:P [(match_operand:P 2 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_ELFv2"
   "b%T0l\;<ptrload> 2,%2(1)"
@@ -11194,7 +11203,7 @@
   [(set (match_operand 0 "" "")
 	(call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
 	      (match_operand 2 "" "g,g")))
-   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (set (reg:P TOC_REGNUM) (unspec:P [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_ELFv2"
   "b%T1l\;<ptrload> 2,%3(1)"
@@ -11773,7 +11782,7 @@
 {
   /* Use ISEL if the user asked for it.  */
   if (TARGET_ISEL)
-    rs6000_emit_sISEL (<MODE>mode, operands);
+    rs6000_emit_int_cmove (operands[0], operands[1], const1_rtx, const0_rtx);
 
   /* Expanding EQ and NE directly to some machine instructions does not help
      but does hurt combine.  So don't.  */
@@ -12133,7 +12142,7 @@
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
 	(unspec:SI [(match_operand:CC 1 "cc_reg_operand" "y")]
 		   UNSPEC_MV_CR_OV))]
-  "TARGET_ISEL"
+  "TARGET_PAIRED_FLOAT"
   "mfcr %0\;rlwinm %0,%0,%t1,1"
   [(set_attr "type" "mfcr")
    (set_attr "length" "8")])
diff --git a/gcc/config/rs6000/rs6000.opt b/gcc/config/rs6000/rs6000.opt
index c42818fbc04..e7d0829495e 100644
--- a/gcc/config/rs6000/rs6000.opt
+++ b/gcc/config/rs6000/rs6000.opt
@@ -381,10 +381,10 @@ mabi=d32
 Target RejectNegative Undocumented Warn(using old darwin ABI) Var(rs6000_darwin64_abi, 0)
 
 mabi=ieeelongdouble
-Target RejectNegative Undocumented Warn(using IEEE extended precision long double) Var(rs6000_ieeequad) Save
+Target RejectNegative Var(rs6000_ieeequad) Save
 
 mabi=ibmlongdouble
-Target RejectNegative Undocumented Warn(using IBM extended precision long double) Var(rs6000_ieeequad, 0)
+Target RejectNegative Var(rs6000_ieeequad, 0)
 
 mcpu=
 Target RejectNegative Joined Var(rs6000_cpu_index) Init(-1) Enum(rs6000_cpu_opt_value) Save
diff --git a/gcc/config/rs6000/x86intrin.h b/gcc/config/rs6000/x86intrin.h
index 624e498a292..33e3176108b 100644
--- a/gcc/config/rs6000/x86intrin.h
+++ b/gcc/config/rs6000/x86intrin.h
@@ -39,6 +39,8 @@
 #include <mmintrin.h>
 
 #include <xmmintrin.h>
+
+#include <emmintrin.h>
 #endif /* __ALTIVEC__ */
 
 #include <bmiintrin.h>
diff --git a/gcc/config/s390/s390.c b/gcc/config/s390/s390.c
index 2258148c573..29d017ff148 100644
--- a/gcc/config/s390/s390.c
+++ b/gcc/config/s390/s390.c
@@ -85,6 +85,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "symbol-summary.h"
 #include "ipa-prop.h"
 #include "ipa-fnsummary.h"
+#include "sched-int.h"
 
 /* This file should be included last.  */
 #include "target-def.h"
@@ -357,6 +358,18 @@ static rtx_insn *last_scheduled_insn;
 #define MAX_SCHED_UNITS 3
 static int last_scheduled_unit_distance[MAX_SCHED_UNITS];
 
+#define NUM_SIDES 2
+static int current_side = 1;
+#define LONGRUNNING_THRESHOLD 5
+
+/* Estimate of number of cycles a long-running insn occupies an
+   execution unit.  */
+static unsigned fxu_longrunning[NUM_SIDES];
+static unsigned vfu_longrunning[NUM_SIDES];
+
+/* Factor to scale latencies by, determined by measurements.  */
+#define LATENCY_FACTOR 4
+
 /* The maximum score added for an instruction whose unit hasn't been
    in use for MAX_SCHED_MIX_DISTANCE steps.  Increase this value to
    give instruction mix scheduling more priority over instruction
@@ -3719,6 +3732,8 @@ s390_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
       case vector_stmt:
       case vector_load:
       case vector_store:
+      case vector_gather_load:
+      case vector_scatter_store:
       case vec_to_scalar:
       case scalar_to_vec:
       case cond_branch_not_taken:
@@ -14606,6 +14621,28 @@ s390_z10_prevent_earlyload_conflicts (rtx_insn **ready, int *nready_p)
   ready[0] = tmp;
 }
 
+/* Returns TRUE if BB is entered via a fallthru edge and all other
+   incoming edges are less than unlikely.  */
+static bool
+s390_bb_fallthru_entry_likely (basic_block bb)
+{
+  edge e, fallthru_edge;
+  edge_iterator ei;
+
+  if (!bb)
+    return false;
+
+  fallthru_edge = find_fallthru_edge (bb->preds);
+  if (!fallthru_edge)
+    return false;
+
+  FOR_EACH_EDGE (e, ei, bb->preds)
+    if (e != fallthru_edge
+	&& e->probability >= profile_probability::unlikely ())
+      return false;
+
+  return true;
+}
 
 /* The s390_sched_state variable tracks the state of the current or
    the last instruction group.
@@ -14614,7 +14651,7 @@ s390_z10_prevent_earlyload_conflicts (rtx_insn **ready, int *nready_p)
    3     the last group is complete - normal insns
    4     the last group was a cracked/expanded insn */
 
-static int s390_sched_state;
+static int s390_sched_state = 0;
 
 #define S390_SCHED_STATE_NORMAL  3
 #define S390_SCHED_STATE_CRACKED 4
@@ -14755,7 +14792,24 @@ s390_sched_score (rtx_insn *insn)
 	if (m & unit_mask)
 	  score += (last_scheduled_unit_distance[i] * MAX_SCHED_MIX_SCORE /
 		    MAX_SCHED_MIX_DISTANCE);
+
+      unsigned latency = insn_default_latency (insn);
+
+      int other_side = 1 - current_side;
+
+      /* Try to delay long-running insns when side is busy.  */
+      if (latency > LONGRUNNING_THRESHOLD)
+	{
+	  if (get_attr_z13_unit_fxu (insn) && fxu_longrunning[current_side]
+	      && fxu_longrunning[other_side] <= fxu_longrunning[current_side])
+	    score = MAX (0, score - 10);
+
+	  if (get_attr_z13_unit_vfu (insn) && vfu_longrunning[current_side]
+	      && vfu_longrunning[other_side] <= vfu_longrunning[current_side])
+	    score = MAX (0, score - 10);
+	}
     }
+
   return score;
 }
 
@@ -14874,6 +14928,8 @@ s390_sched_variable_issue (FILE *file, int verbose, rtx_insn *insn, int more)
 {
   last_scheduled_insn = insn;
 
+  bool starts_group = false;
+
   if (s390_tune >= PROCESSOR_2827_ZEC12
       && reload_completed
       && recog_memoized (insn) >= 0)
@@ -14881,6 +14937,11 @@ s390_sched_variable_issue (FILE *file, int verbose, rtx_insn *insn, int more)
       unsigned int mask = s390_get_sched_attrmask (insn);
 
       if ((mask & S390_SCHED_ATTR_MASK_CRACKED) != 0
+	  || (mask & S390_SCHED_ATTR_MASK_EXPANDED) != 0
+	  || (mask & S390_SCHED_ATTR_MASK_GROUPALONE) != 0)
+	starts_group = true;
+
+      if ((mask & S390_SCHED_ATTR_MASK_CRACKED) != 0
 	  || (mask & S390_SCHED_ATTR_MASK_EXPANDED) != 0)
 	s390_sched_state = S390_SCHED_STATE_CRACKED;
       else if ((mask & S390_SCHED_ATTR_MASK_ENDGROUP) != 0
@@ -14892,14 +14953,15 @@ s390_sched_variable_issue (FILE *file, int verbose, rtx_insn *insn, int more)
 	  switch (s390_sched_state)
 	    {
 	    case 0:
+	      starts_group = true;
+	      /* fallthrough */
 	    case 1:
 	    case 2:
+	      s390_sched_state++;
+	      break;
 	    case S390_SCHED_STATE_NORMAL:
-	      if (s390_sched_state == S390_SCHED_STATE_NORMAL)
-		s390_sched_state = 1;
-	      else
-		s390_sched_state++;
-
+	      starts_group = true;
+	      s390_sched_state = 1;
 	      break;
 	    case S390_SCHED_STATE_CRACKED:
 	      s390_sched_state = S390_SCHED_STATE_NORMAL;
@@ -14922,6 +14984,27 @@ s390_sched_variable_issue (FILE *file, int verbose, rtx_insn *insn, int more)
 	      last_scheduled_unit_distance[i]++;
 	}
 
+      /* If this insn started a new group, the side flipped.  */
+      if (starts_group)
+	current_side = current_side ? 0 : 1;
+
+      for (int i = 0; i < 2; i++)
+	{
+	  if (fxu_longrunning[i] >= 1)
+	    fxu_longrunning[i] -= 1;
+	  if (vfu_longrunning[i] >= 1)
+	    vfu_longrunning[i] -= 1;
+	}
+
+      unsigned latency = insn_default_latency (insn);
+      if (latency > LONGRUNNING_THRESHOLD)
+	{
+	  if (get_attr_z13_unit_fxu (insn))
+	    fxu_longrunning[current_side] = latency * LATENCY_FACTOR;
+	  else
+	    vfu_longrunning[current_side] = latency * LATENCY_FACTOR;
+	}
+
       if (verbose > 5)
 	{
 	  unsigned int sched_mask;
@@ -14978,7 +15061,21 @@ s390_sched_init (FILE *file ATTRIBUTE_UNUSED,
 {
   last_scheduled_insn = NULL;
   memset (last_scheduled_unit_distance, 0, MAX_SCHED_UNITS * sizeof (int));
-  s390_sched_state = 0;
+
+  /* If the next basic block is most likely entered via a fallthru edge
+     we keep the last sched state.  Otherwise we start a new group.
+     The scheduler traverses basic blocks in "instruction stream" ordering
+     so if we see a fallthru edge here, s390_sched_state will be of its
+     source block.
+
+     current_sched_info->prev_head is the insn before the first insn of the
+     block of insns to be scheduled.
+     */
+  rtx_insn *insn = current_sched_info->prev_head
+    ? NEXT_INSN (current_sched_info->prev_head) : NULL;
+  basic_block bb = insn ? BLOCK_FOR_INSN (insn) : NULL;
+  if (s390_tune < PROCESSOR_2964_Z13 || !s390_bb_fallthru_entry_likely (bb))
+    s390_sched_state = 0;
 }
 
 /* This target hook implementation for TARGET_LOOP_UNROLL_ADJUST calculates
diff --git a/gcc/config/spu/spu.c b/gcc/config/spu/spu.c
index e792650184b..606934bcfe7 100644
--- a/gcc/config/spu/spu.c
+++ b/gcc/config/spu/spu.c
@@ -6633,6 +6633,8 @@ spu_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
         return 2;
 
       case unaligned_load:
+      case vector_gather_load:
+      case vector_scatter_store:
         return 2;
 
       case cond_branch_taken:
diff --git a/gcc/config/stormy16/stormy16.h b/gcc/config/stormy16/stormy16.h
index 094a2f08e43..dfc659c2e98 100644
--- a/gcc/config/stormy16/stormy16.h
+++ b/gcc/config/stormy16/stormy16.h
@@ -446,7 +446,7 @@ enum reg_class
 #define PREFERRED_DEBUGGING_TYPE DWARF2_DEBUG
 
 
-/* Macros for SDB and Dwarf Output.  */
+/* Macros for Dwarf Output.  */
 
 /* Define this macro if addresses in Dwarf 2 debugging info should not
    be the same size as pointers on the target architecture.  The
diff --git a/gcc/config/visium/visium.c b/gcc/config/visium/visium.c
index e028dc479d3..6861f127867 100644
--- a/gcc/config/visium/visium.c
+++ b/gcc/config/visium/visium.c
@@ -2940,12 +2940,6 @@ visium_select_cc_mode (enum rtx_code code, rtx op0, rtx op1)
       /* This is a btst, the result is in C instead of Z.  */
       return CCCmode;
 
-    case CONST_INT:
-      /* This is a degenerate case, typically an uninitialized variable.  */
-      gcc_assert (op0 == constm1_rtx);
-
-      /* ... fall through ... */
-
     case REG:
     case AND:
     case IOR:
@@ -2962,6 +2956,17 @@ visium_select_cc_mode (enum rtx_code code, rtx op0, rtx op1)
 	 when applied to a comparison with zero.  */
       return CCmode;
 
+    /* ??? Cater to the junk RTXes sent by try_merge_compare.  */
+    case ASM_OPERANDS:
+    case CALL:
+    case CONST_INT:
+    case LO_SUM:
+    case HIGH:
+    case MEM:
+    case UNSPEC:
+    case ZERO_EXTEND:
+      return CCmode;
+
     default:
       gcc_unreachable ();
     }
diff --git a/gcc/config/visium/visium.h b/gcc/config/visium/visium.h
index 3b229f1a1e6..85735953968 100644
--- a/gcc/config/visium/visium.h
+++ b/gcc/config/visium/visium.h
@@ -1527,9 +1527,8 @@ do									\
    automatic variable having address X (an RTL expression).  The
    default computation assumes that X is based on the frame-pointer
    and gives the offset from the frame-pointer.  This is required for
-   targets that produce debugging output for DBX or COFF-style
-   debugging output for SDB and allow the frame-pointer to be
-   eliminated when the `-g' options is used. */
+   targets that produce debugging output for DBX and allow the frame-pointer
+   to be eliminated when the `-g' options is used. */
 #define DEBUGGER_AUTO_OFFSET(X) \
   (GET_CODE (X) == PLUS ? INTVAL (XEXP (X, 1)) : 0)
 
diff --git a/gcc/config/vx-common.h b/gcc/config/vx-common.h
index 5cc965cab78..d8f04eced4d 100644
--- a/gcc/config/vx-common.h
+++ b/gcc/config/vx-common.h
@@ -72,7 +72,6 @@ along with GCC; see the file COPYING3.  If not see
 /* None of these other formats is supported.  */
 #undef DWARF_DEBUGGING_INFO
 #undef DBX_DEBUGGING_INFO
-#undef SDB_DEBUGGING_INFO
 #undef XCOFF_DEBUGGING_INFO
 #undef VMS_DEBUGGING_INFO
 
diff --git a/gcc/configure b/gcc/configure
index 13f97cd3663..fb40ead9204 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -942,6 +942,7 @@ enable_fix_cortex_a53_843419
 with_glibc_version
 enable_gnu_unique_object
 enable_linker_build_id
+enable_libssp
 enable_default_ssp
 with_long_double_128
 with_gc
@@ -1682,6 +1683,7 @@ Optional Features:
                           extension on glibc systems
   --enable-linker-build-id
                           compiler will always pass --build-id to linker
+  --enable-libssp         enable linking against libssp
   --enable-default-ssp    enable Stack Smashing Protection as default
   --enable-maintainer-mode
                           enable make rules and dependencies not useful (and
@@ -4987,7 +4989,7 @@ acx_cv_cc_gcc_supports_ada=no
 # Other compilers, like HP Tru64 UNIX cc, exit successfully when
 # given a .adb file, but produce no object file.  So we must check
 # if an object file was really produced to guard against this.
-errors=`(${CC} -I"$srcdir"/ada -c conftest.adb) 2>&1 || echo failure`
+errors=`(${CC} -I"$srcdir"/ada/libgnat -c conftest.adb) 2>&1 || echo failure`
 if test x"$errors" = x && test -f conftest.$ac_objext; then
   acx_cv_cc_gcc_supports_ada=yes
 fi
@@ -7321,10 +7323,10 @@ fi
 if test "${enable_coverage+set}" = set; then :
   enableval=$enable_coverage; case "${enableval}" in
   yes|noopt)
-    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O0"
+    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O0 -fkeep-inline-functions -fkeep-static-functions"
     ;;
   opt)
-    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O2"
+    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O2 -fkeep-inline-functions -fkeep-static-functions"
     ;;
   no)
     # a.k.a. --disable-coverage
@@ -18440,7 +18442,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 18443 "configure"
+#line 18445 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -18546,7 +18548,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 18549 "configure"
+#line 18551 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -22783,15 +22785,25 @@ if test $in_tree_ld != yes ; then
   else
     case "${target}" in
       *-*-solaris2*)
-	# See acinclude.m4 (gcc_SUN_LD_VERSION) for the version number
-	# format.
+	# Solaris 2 ld -V output looks like this for a regular version:
 	#
-	# Don't reuse gcc_gv_sun_ld_vers_* in case a linker other than
-	# /usr/ccs/bin/ld has been configured.
+	# ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1699
+	#
+	# but test versions add stuff at the end:
+	#
+	# ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1701:onnv-ab196087-6931056-03/25/10
+	#
+	# In Solaris 11.4, this was changed to
+	#
+	# ld: Solaris ELF Utilities: 11.4-1.3123
+	#
+	# ld and ld.so.1 are guaranteed to be updated in lockstep, so ld version
+	# numbers can be used in ld.so.1 feature checks even if a different
+	# linker is configured.
 	ld_ver=`$gcc_cv_ld -V 2>&1`
-	if echo "$ld_ver" | grep 'Solaris Link Editors' > /dev/null; then
+	if echo "$ld_ver" | $EGREP 'Solaris Link Editors|Solaris ELF Utilities' > /dev/null; then
 	  ld_vers=`echo $ld_ver | sed -n \
-	    -e 's,^.*: 5\.[0-9][0-9]*-\([0-9]\.[0-9][0-9]*\).*$,\1,p'`
+	    -e 's,^.*: \(5\|1[0-9]\)\.[0-9][0-9]*-\([0-9]\.[0-9][0-9]*\).*$,\2,p'`
 	  ld_vers_major=`expr "$ld_vers" : '\([0-9]*\)'`
 	  ld_vers_minor=`expr "$ld_vers" : '[0-9]*\.\([0-9]*\)'`
 	fi
@@ -22915,29 +22927,6 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_ld_ro_rw_mix" >&5
 $as_echo "$gcc_cv_ld_ro_rw_mix" >&6; }
 
-if test "x${build}" = "x${target}" && test "x${build}" = "x${host}"; then
-  case "${target}" in
-    *-*-solaris2*)
-      #
-      # Solaris 2 ld -V output looks like this for a regular version:
-      #
-      # ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1699
-      #
-      # but test versions add stuff at the end:
-      #
-      # ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1701:onnv-ab196087-6931056-03/25/10
-      #
-      gcc_cv_sun_ld_ver=`/usr/ccs/bin/ld -V 2>&1`
-      if echo "$gcc_cv_sun_ld_ver" | grep 'Solaris Link Editors' > /dev/null; then
-	gcc_cv_sun_ld_vers=`echo $gcc_cv_sun_ld_ver | sed -n \
-	  -e 's,^.*: 5\.[0-9][0-9]*-\([0-9]\.[0-9][0-9]*\).*$,\1,p'`
-	gcc_cv_sun_ld_vers_major=`expr "$gcc_cv_sun_ld_vers" : '\([0-9]*\)'`
-	gcc_cv_sun_ld_vers_minor=`expr "$gcc_cv_sun_ld_vers" : '[0-9]*\.\([0-9]*\)'`
-      fi
-      ;;
-  esac
-fi
-
 # Check whether --enable-initfini-array was given.
 if test "${enable_initfini_array+set}" = set; then :
   enableval=$enable_initfini_array;
@@ -25552,6 +25541,38 @@ $as_echo "$as_me: WARNING: LTO for $target requires binutils >= 2.20.1, but vers
 	;;
     esac
 
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for -xbrace_comment" >&5
+$as_echo_n "checking assembler for -xbrace_comment... " >&6; }
+if test "${gcc_cv_as_ix86_xbrace_comment+set}" = set; then :
+  $as_echo_n "(cached) " >&6
+else
+  gcc_cv_as_ix86_xbrace_comment=no
+  if test x$gcc_cv_as != x; then
+    $as_echo '.text' > conftest.s
+    if { ac_try='$gcc_cv_as $gcc_cv_as_flags -xbrace_comment=no -o conftest.o conftest.s >&5'
+  { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; }
+    then
+	gcc_cv_as_ix86_xbrace_comment=yes
+    else
+      echo "configure: failed program was" >&5
+      cat conftest.s >&5
+    fi
+    rm -f conftest.o conftest.s
+  fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_as_ix86_xbrace_comment" >&5
+$as_echo "$gcc_cv_as_ix86_xbrace_comment" >&6; }
+if test $gcc_cv_as_ix86_xbrace_comment = yes; then
+
+$as_echo "#define HAVE_AS_XBRACE_COMMENT_OPTION 1" >>confdefs.h
+
+fi
+
+
     # Test if the assembler supports the section flag 'e' for specifying
     # an excluded section.
     { $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .section with e" >&5
@@ -29008,6 +29029,18 @@ $as_echo "#define HAVE_SOLARIS_CRTS 1" >>confdefs.h
 
 fi
 
+# Check whether --enable-libssp was given.
+if test "${enable_libssp+set}" = set; then :
+  enableval=$enable_libssp; case "${enableval}" in
+  yes|no)
+    ;;
+  *)
+    as_fn_error "unknown libssp setting $enableval" "$LINENO" 5
+    ;;
+esac
+fi
+
+
 # Test for stack protector support in target C library.
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking __stack_chk_fail in target C library" >&5
 $as_echo_n "checking __stack_chk_fail in target C library... " >&6; }
@@ -29015,6 +29048,11 @@ if test "${gcc_cv_libc_provides_ssp+set}" = set; then :
   $as_echo_n "(cached) " >&6
 else
   gcc_cv_libc_provides_ssp=no
+  if test "x$enable_libssp" = "xno"; then
+    gcc_cv_libc_provides_ssp=yes
+  elif test "x$enable_libssp" = "xyes"; then
+    gcc_cv_libc_provides_ssp=no
+  else
     case "$target" in
        *-*-musl*)
 	 # All versions of musl provide stack protector
@@ -29062,8 +29100,9 @@ else
 fi
 
         ;;
-  *) gcc_cv_libc_provides_ssp=no ;;
+       *) gcc_cv_libc_provides_ssp=no ;;
     esac
+  fi
 fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_libc_provides_ssp" >&5
 $as_echo "$gcc_cv_libc_provides_ssp" >&6; }
diff --git a/gcc/configure.ac b/gcc/configure.ac
index 82711389281..0e5167695a2 100644
--- a/gcc/configure.ac
+++ b/gcc/configure.ac
@@ -362,7 +362,7 @@ rm -f a.out a.exe b.out
 # Find the native compiler
 AC_PROG_CC
 AC_PROG_CXX
-ACX_PROG_GNAT([-I"$srcdir"/ada])
+ACX_PROG_GNAT([-I"$srcdir"/ada/libgnat])
 
 # Do configure tests with the C++ compiler, since that's what we build with.
 AC_LANG(C++)
@@ -728,10 +728,10 @@ AC_ARG_ENABLE(coverage,
 		 default is noopt])],
 [case "${enableval}" in
   yes|noopt)
-    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O0"
+    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O0 -fkeep-inline-functions -fkeep-static-functions"
     ;;
   opt)
-    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O2"
+    coverage_flags="-fprofile-arcs -ftest-coverage -frandom-seed=\$@ -O2 -fkeep-inline-functions -fkeep-static-functions"
     ;;
   no)
     # a.k.a. --disable-coverage
@@ -2587,15 +2587,25 @@ if test $in_tree_ld != yes ; then
   else
     case "${target}" in
       *-*-solaris2*)
-	# See acinclude.m4 (gcc_SUN_LD_VERSION) for the version number
-	# format.
+	# Solaris 2 ld -V output looks like this for a regular version:
 	#
-	# Don't reuse gcc_gv_sun_ld_vers_* in case a linker other than
-	# /usr/ccs/bin/ld has been configured.
+	# ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1699
+	#
+	# but test versions add stuff at the end:
+	#
+	# ld: Software Generation Utilities - Solaris Link Editors: 5.11-1.1701:onnv-ab196087-6931056-03/25/10
+	#
+	# In Solaris 11.4, this was changed to
+	#
+	# ld: Solaris ELF Utilities: 11.4-1.3123
+	#
+	# ld and ld.so.1 are guaranteed to be updated in lockstep, so ld version
+	# numbers can be used in ld.so.1 feature checks even if a different
+	# linker is configured.
 	ld_ver=`$gcc_cv_ld -V 2>&1`
-	if echo "$ld_ver" | grep 'Solaris Link Editors' > /dev/null; then
+	if echo "$ld_ver" | $EGREP 'Solaris Link Editors|Solaris ELF Utilities' > /dev/null; then
 	  ld_vers=`echo $ld_ver | sed -n \
-	    -e 's,^.*: 5\.[0-9][0-9]*-\([0-9]\.[0-9][0-9]*\).*$,\1,p'`
+	    -e 's,^.*: \(5\|1[0-9]\)\.[0-9][0-9]*-\([0-9]\.[0-9][0-9]*\).*$,\2,p'`
 	  ld_vers_major=`expr "$ld_vers" : '\([0-9]*\)'`
 	  ld_vers_minor=`expr "$ld_vers" : '[0-9]*\.\([0-9]*\)'`
 	fi
@@ -4103,6 +4113,11 @@ foo:	nop
 	;;
     esac
 
+    gcc_GAS_CHECK_FEATURE([-xbrace_comment], gcc_cv_as_ix86_xbrace_comment,,
+      [-xbrace_comment=no], [.text],,
+      [AC_DEFINE(HAVE_AS_XBRACE_COMMENT_OPTION, 1,
+		[Define if your assembler supports -xbrace_comment option.])])
+
     # Test if the assembler supports the section flag 'e' for specifying
     # an excluded section.
     gcc_GAS_CHECK_FEATURE([.section with e], gcc_cv_as_section_has_e,
@@ -5751,10 +5766,25 @@ if test x$gcc_cv_solaris_crts = xyes; then
   	    [Define if the system-provided CRTs are present on Solaris.])
 fi
 
+AC_ARG_ENABLE(libssp,
+[AS_HELP_STRING([--enable-libssp], [enable linking against libssp])],
+[case "${enableval}" in
+  yes|no)
+    ;;
+  *)
+    AC_MSG_ERROR([unknown libssp setting $enableval])
+    ;;
+esac], [])
+
 # Test for stack protector support in target C library.
 AC_CACHE_CHECK(__stack_chk_fail in target C library,
-      gcc_cv_libc_provides_ssp,
-      [gcc_cv_libc_provides_ssp=no
+  gcc_cv_libc_provides_ssp,
+  [gcc_cv_libc_provides_ssp=no
+  if test "x$enable_libssp" = "xno"; then
+    gcc_cv_libc_provides_ssp=yes
+  elif test "x$enable_libssp" = "xyes"; then
+    gcc_cv_libc_provides_ssp=no
+  else
     case "$target" in
        *-*-musl*)
 	 # All versions of musl provide stack protector
@@ -5791,8 +5821,9 @@ AC_CACHE_CHECK(__stack_chk_fail in target C library,
 	 AC_CHECK_FUNC(__stack_chk_fail,[gcc_cv_libc_provides_ssp=yes],
            [echo "no __stack_chk_fail on this target"])
         ;;
-  *) gcc_cv_libc_provides_ssp=no ;;
-    esac])
+       *) gcc_cv_libc_provides_ssp=no ;;
+    esac
+  fi])
 
 if test x$gcc_cv_libc_provides_ssp = xyes; then
   AC_DEFINE(TARGET_LIBC_PROVIDES_SSP, 1,
diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog
index 3abf79440cc..590e3221c8c 100644
--- a/gcc/cp/ChangeLog
+++ b/gcc/cp/ChangeLog
@@ -1,3 +1,363 @@
+2017-11-03  Nathan Sidwell  <nathan@acm.org>
+
+	PR c++/82710
+	* decl.c (grokdeclarator): Protect MAYBE_CLASS things from paren
+	warning too.
+
+2017-11-02  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/81957
+	* pt.c (make_pack_expansion): Add tsubst_flags_t parameter.
+	(expand_integer_pack, convert_template_argument, coerce_template_parms,
+	gen_elem_of_pack_expansion_instantiation, tsubst_pack_expansion,
+	unify): Adjust calls.
+	* tree.c (cp_build_qualified_type_real): Likewise.
+	* cp-tree.h (make_pack_expansion): Adjust declaration.
+
+2017-11-02  Nathan Sidwell  <nathan@acm.org>
+
+	* cp-tree.h (IDENTIFIER_NEWDEL_OP_P): Restore, adjust.
+	(IDENTIFIER_NEW_OP_P): New.
+	* decl.c (grokdeclarator): Restore IDENTIFIER_NEWDEL_OP_P use.
+	* pt.c (push_template_decl_real): Likewise.
+	* typeck.c (check_return_expr): Use IDENTIFIER_NEW_OP_P.
+
+	PR c++/82710
+	* decl.c (grokdeclarator): Don't warn when parens protect a return
+	type from a qualified name.
+
+2017-11-01  Nathan Sidwell  <nathan@acm.org>
+
+	* cp-tree.h (enum cp_identifier_kind): Delete cik_newdel_op.
+	Renumber and reserve udlit value.
+	(IDENTIFIER_NEWDEL_OP_P): Delete.
+	(IDENTIFIER_OVL_OP_P): New.
+	(IDENTIFIER_ASSIGN_OP_P): Adjust.
+	(IDENTIFIER_CONV_OP_P): Adjust.
+	(IDENTIFIER_OVL_OP_INFO): Adjust.
+	(IDENTIFIER_OVL_OP_FLAGS): New.
+	* decl.c (grokdeclarator): Use IDENTIFIER_OVL_OP_FLAGS.
+	* lex.c (get_identifier_kind_name): Adjust.
+	(init_operators): Don't special case new/delete ops.
+	* mangle.c (write_unqualified_id): Use IDENTIFIER_OVL_OP_P.
+	* pt.c (push_template_decl_real): Use IDENTIFIER_OVL_OP_FLAGS.
+	* typeck.c (check_return_expr): Likewise.
+
+	* cp-tree.h (assign_op_identifier, call_op_identifier): Use
+	compressed code.
+	(struct lang_decl_fn): Use compressed operator code.
+	(DECL_OVERLOADED_OPERATOR_CODE): Replace with ...
+	(DECL_OVERLOADED_OPERATOR_CODE_RAW): ... this.
+	(DECL_OVERLOADED_OPERATOR_CODE_IS): Use it.
+	* decl.c (duplicate_decls): Use DECL_OVERLOADED_OPERATOR_CODE_RAW.
+	(build_library_fn): Likewise.
+	(grok_op_properties): Likewise.
+	* mangle.c (write_unqualified_name): Likewise.
+	* method.c (implicitly_declare_fn): Likewise.
+	* typeck.c (check_return_expr): Use DECL_OVERLOADED_OPERATOR_IS.
+
+	* cp-tree.h (IDENTIFIER_CP_INDEX): Define.
+	(enum ovl_op_flags): Add OVL_OP_FLAG_AMBIARY.
+	(enum ovl_op_code): New.
+	(struct ovl_op_info): Add ovl_op_code field.
+	(ovl_op_info): Size by OVL_OP_MAX.
+	(ovl_op_mapping, ovl_op_alternate): Declare.
+	(OVL_OP_INFO): Adjust for mapping array.
+	(IDENTIFIER_OVL_OP_INFO): New.
+	* decl.c (ambi_op_p, unary_op_p): Delete.
+	(grok_op_properties): Use IDENTIFIER_OVL_OP_INFO and
+	ovl_op_alternate.
+	* lex.c (ovl_op_info): Adjust and static initialize.
+	(ovl_op_mappings, ovl_op_alternate): Define.
+	(init_operators): Iterate over ovl_op_info array and init mappings
+	& alternate arrays.
+	* mangle.c (write_unqualified_id): Use IDENTIFIER_OVL_OP_INFO.
+	* operators.def (DEF_OPERATOR): Remove KIND parm.
+	(DEF_SIMPLE_OPERATOR): Delete.
+	(OPERATOR_TRANSITION): Expand if defined.
+
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* pt.c (listify): Use %< and %> for description of #include.
+
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* class.c (explain_non_literal_class): Use UNKNOWN_LOCATION rather
+	than 0.
+	* name-lookup.c (suggest_alternatives_for): Update for renaming of
+	inform_at_rich_loc.
+	(maybe_suggest_missing_header): Likewise.
+	(suggest_alternative_in_explicit_scope): Likewise.
+	* parser.c (cp_parser_diagnose_invalid_type_name): Likewise for
+	renaming of error_at_rich_loc.
+	(cp_parser_string_literal): Likewise.
+	(cp_parser_nested_name_specifier_opt): Likewise.
+	(cp_parser_cast_expression): Likewise for renaming of
+	warning_at_rich_loc.
+	(cp_parser_decl_specifier_seq): Likewise for renaming of
+	error_at_rich_loc and warning_at_rich_loc.
+	(cp_parser_elaborated_type_specifier): Likewise for renaming of
+	pedwarn_at_rich_loc.
+	(cp_parser_cv_qualifier_seq_opt): Likewise for renaming of
+	error_at_rich_loc.
+	(cp_parser_virt_specifier_seq_opt): Likewise.
+	(cp_parser_class_specifier_1): Likewise.
+	(cp_parser_class_head): Likewise.
+	(cp_parser_member_declaration): Likewise for renaming of
+	pedwarn_at_rich_loc, warning_at_rich_loc, and error_at_rich_loc.
+	(cp_parser_enclosed_template_argument_list): Likewise for renaming
+	of error_at_rich_loc.
+	(set_and_check_decl_spec_loc): Likewise.
+	* pt.c (listify): Likewise.
+	* rtti.c (typeid_ok_p): Likewise.
+	* semantics.c (process_outer_var_ref): Use UNKNOWN_LOCATION rather
+	than 0.
+	* typeck.c (access_failure_info::maybe_suggest_accessor): Update
+	for renaming of inform_at_rich_loc.
+	(finish_class_member_access_expr): Likewise for renaming of
+	error_at_rich_loc.
+
+2017-10-31  Nathan Sidwell  <nathan@acm.org>
+
+	* cp-tree.h (struct operator_name_info_t): Rename to ...
+	(struct ovl_op_info_t): ... here.  Add tree_code field.
+	(operator_name_info, assignment_operator_name_info): Delete.
+	(ovl_op_info): Declare.
+	(OVL_OP_INFO): Adjust.
+	* decl.c (grok_op_properties): Use ovl_op_flags.
+	* lex.c (operator_name_info, assignment_operator_name_info):
+	Delete.
+	(ovl_op_info): Define.
+	(set_operator_ident): Adjust.
+	(init_operators): Set tree_code.
+	* mangle.c (write_unqualified_id): Adjust operator array scan.
+
+	* lex.c (init_operators): Allow NULL operator name.  Don't add
+	special cases.
+	* operators.def: Use NULL for mangling only operators.  Move to
+	after regular operators but move assignment operators last.
+
+	* cp-tree.h (enum ovl_op_flags): New.
+	(struct operator_name_info_t): Rename arity to flags.
+	* lex.c (set_operator_ident): New.
+	(init_operators): Use it.  Adjust for flags.
+	* mangle.c (write_unqualified_id): Adjust for flags.
+	* operators.def: Replace arity with flags.
+
+	* cp-tree.h (ovl_op_identifier): New.
+	(assign_op_identifier, call_op_identifier): Adjust.
+	(cp_operator_id, cp_assignment_operator_ide): Delete.
+	(SET_OVERLOADED_OPERATOR_CODE): Delete.
+	(OVL_OP_INFO): New.
+	* call.c (op_error): Use OVL_OP_INFO.
+	(build_conditional_expr_1): Use ovl_op_identifier.
+	(build_new_op_1): Use OVL_OP_INFO & ovl_op_identifier.
+	(build_op_delete_call): Likewise.
+	* class.c (type_requires_array_cookie): Use ovl_op_identifier.
+	* decl.c (duplicate_decls): Directly copy operator code.
+	(builtin_function_1): Do not set operator code.
+	(build_library_fn): Directly set operator code.
+	(push_cp_library_fn): Use ovl_op_identifier.
+	(grok_op_properties): Directly set operator code.
+	* decl2.c (maybe_warn_sized_delete): Use ovl_op_identifier.
+	* error.c (dump_expr): Use OVL_OP_INFO.
+	(op_to_string): Add assop arg. Use OVL_OP_INFO.
+	(assop_to_string): Delete.
+	(args_to_string): Adjust.
+	* init.c (build_new_1): Use ovl_op_identifier.
+	* mangle.c (write_unqualified_name): Use OVL_OP_INFO.
+	(write_expression): Likewise.
+	* method.c (synthesized_method_walk): Use ovl_op_identifier.
+	(implicitly_declare_fn): Use assign_op_identifier. Directly set
+	operator code.
+	* name-lookup.c (get_class_binding): Use assign_op_identifier.
+	* parser.c (cp_parser_operator): Use ovl_op_identifier.
+	(cp_parser_omp_clause_reduction): Likewise.
+	* semantics.c (omp_reduction_id): Likewise.
+	* typeck.c (cxx_sizeof_or_alignof_type): Use OVL_OP_INFO.
+
+	* cp-tree.h (assign_op_identifier, call_op_identifier): Define.
+	(LAMBDA_FUNCTION_P): Use DECL_OVERLOADED_OPERATOR_IS.
+	(DECL_OVERLOADED_OPERATOR_P): Just retuurn true/false.
+	(DECL_OVERLOADED_OPERATOR_CODE, DECL_OVERLOADED_OPERATOR_IS): Define.
+	* call.c (add_function_candidate): Use
+	DECL_OVERLOADED_OPERATOR_IS.
+	(build_op_call_1): Use call_op_identifier &
+	DECL_OVERLOADED_OPERATOR_IS.
+	(build_over_call): Likewise.
+	(has_trivial_copy_assign_p): Use assign_op_identifier.
+	(build_special_member_call): Likewise.
+	* class.c (dfs_declare_virt_assop_and_dtor): Likewise.
+	(vbase_has_user_provided_move_assign,
+	classtype_has_move_assign_or_move_ctor_p): Likewise.
+	* decl.c (duplicate_decls): Use DECL_OVERLOADED_OPERATOR_CODE.
+	(grok_special_member_properties): Use assign_op_identifier.
+	(start_preparsed_function): Use DECL_OVERLOADED_OPERATOR_IS.
+	* decl2.c (mark_used): Use DECL_CONV_FN_P.
+	* dump.c (dump_access): Delete prototype.
+	(dump_op): Delete.
+	(cp_dump_tree): Don't call it.
+	* lambda.c (lambda_function): Use call_op_identifier.
+	(maybe_add_lambda_conv_op): Not an overloaded operator.  Remove
+	unneeded braces.
+	* mangle.c (write_unqualified_name): Use DECL_OVERLOADED_OPERTOR_CODE.
+	* method.c (do_build_copy_assign): Use assign_op_identifier.
+	(synthesize_method): Use DECL_OVERLOADED_OPERATOR_IS.
+	(get_copy_assign): Use assign_op_identifier.
+	(synthesized_method_walk): Likewise.
+	(defaultable_fn_check): Use DECL_OVERLOADED_OPERATOR_IS.
+	* parser.c (cp_parser_lambda_declarator_opt): Use
+	call_op_identifier.
+	* semanitics.c (classtype_has_nothrow_assign_or_copy_p): Use
+	assign_op_identifier.
+	* tree.c (special_function_p):  Use DECL_OVERLOADED_OPERATOR_IS.
+	* typeck.c (check_return_expr): Use DECL_OVERLOADED_OPERATOR_CODE.
+	(check_return_expr): Use assign_op_identifier.
+
+2017-10-30  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82085
+	* pt.c (tsubst_copy_and_build, [INDIRECT_REF]): For a REFERENCE_REF_P,
+	unconditionally call convert_from_reference.
+
+2017-10-30  Nathan Sidwell  <nathan@acm.org>
+
+	* call.c (build_op_call_1): Test for FUNCTION_DECL in same manner
+	as a few lines earlier.
+	* cp-tree.h (PACK_EXPANSION_PATTERN): Fix white space.
+	* decl.c (grokfndecl): Fix indentation.
+	(compute_array_index_type): Use processing_template_decl_sentinel.
+	(grok_op_properties): Move warnings to end.  Reorder other checks
+	to group similar entities.  Tweak diagnostics.
+	* lex.c (unqualified_name_lookup_error): No need to check name is
+	not ERROR_MARK operator.
+	* parser.c (cp_parser_operator): Select operator code before
+	looking it up.
+	* typeck.c (check_return_expr): Fix indentation and line wrapping.
+
+2017-10-27  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	* pt.c (invalid_nontype_parm_type_p): Return a bool instead of an int.
+
+2017-10-26  Nathan Sidwell  <nathan@acm.org>
+
+	* decl.c (sort_labels): Restore function.
+	(pop_labels): Sort labels
+	(identify_goto): Add translation markup.
+
+2017-10-25  Nathan Sidwell  <nathan@acm.org>
+
+	Kill IDENTIFIER_LABEL_VALUE.
+	* cp-tree.h (lang_identifier): Delete label_value slot.
+	(IDENTIFIER_LABEL_VALUE, SET_IDENTIFIER_LABEL_VALUE): Delete.
+	(struct named_label_hasher): Rename to ...
+	(struct named_label_hash): ... here.  Reimplement.
+	(struct language_function): Adjust x_named_labels.
+	* name-lookup.h (struct cp_label_binding): Delete.
+	(struct cp_binding_level): Delete shadowed_labels slot.
+	* decl.c (struct named_label_entry): Add name and outer slots.
+	(pop_label): Rename to ...
+	(check_label_used): ... here.  Don't pop.
+	(note_label, sort_labels): Delete.
+	(pop_labels, pop_local_label): Reimplement.
+	(poplevel): Pop local labels as any other decl. Remove
+	shadowed_labels handling.
+	(named_label_hash::hash, named_label_hash::equal): New.
+	(make_label_decl): Absorb into ...
+	(lookup_label_1): ... here.  Add making_local_p arg, reimplement.
+	(lookup_label, declare_local_label): Adjust.
+	(check_goto, define_label): Adjust.
+	* lex.c (make_conv_op_name): Don't clear IDENTIFIER_LABEL_VALUE.
+	* ptree.c (cxx_print_identifier): Don't print identifier binding.
+
+	* decl.c (identifier_goto): Reduce duplication.
+	(check_previous_goto_1): Likewise.
+	(check_goto): Move var decls to initialization.
+	(check_omp_return, define_label_1, define_label): Likewise.
+
+2017-10-25  Jakub Jelinek  <jakub@redhat.com>
+
+	PR libstdc++/81706
+	* decl.c (duplicate_decls): Copy "omp declare simd" attributes from
+	newdecl to corresponding __builtin_ if any.
+
+2017-10-24  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82466
+	* decl.c (duplicate_decls): Warn for built-in functions declared as
+	non-function, use OPT_Wbuiltin_declaration_mismatch.
+
+	* decl.c (duplicate_decls): Avoid redundant '+' in warning_at.
+
+2017-10-24  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/80991
+	* pt.c (value_dependent_expression_p, [TRAIT_EXPR]): Handle
+	a TREE_LIST as TRAIT_EXPR_TYPE2.
+
+2017-10-24  Mukesh Kapoor  <mukesh.kapoor@oracle.com>
+	    Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82307
+	* cvt.c (type_promotes_to): Implement C++17, 7.6/4, about unscoped
+	enumeration type whose underlying type is fixed.
+
+2017-10-23  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/80449
+	* semantics.c (finish_compound_literal): Check do_auto_deduction
+	return value for error_mark_node.
+
+2017-10-23  Jason Merrill  <jason@redhat.com>
+
+	PR c++/77369 - wrong noexcept handling in C++14 and below
+	* tree.c (strip_typedefs): Canonicalize TYPE_RAISES_EXCEPTIONS.
+
+2017-10-20  Nathan Sidwell  <nathan@acm.org>
+
+	* class.c (layout_class_type): Cleanup as-base creation, determine
+	mode here.
+	(finish_struct_1): ... not here.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/82600
+	* typeck.c (check_return_expr): Don't call
+	maybe_warn_about_returning_address_of_local in templates.
+
+2017-10-17  Nathan Sidwell  <nathan@acm.org>
+
+	PR c++/82560
+	* call.c (build_over_call): Don't pass tf_no_cleanup to nested
+	calls.
+
+	PR middle-end/82546
+	* cp-objcp-common.c (cp_tree_size): Reformat.  Adjust returns size
+	of TYPE nodes.
+
+2017-10-13  Jason Merrill  <jason@redhat.com>
+
+	PR c++/82357 - bit-field in template
+	* tree.c (cp_stabilize_reference): Just return a NON_DEPENDENT_EXPR.
+
+2017-10-13  David Malcolm  <dmalcolm@redhat.com>
+
+	* cp-tree.h (maybe_show_extern_c_location): New decl.
+	* decl.c (grokfndecl): When complaining about literal operators
+	with C linkage, issue a note giving the location of the
+	extern "C".
+	* parser.c (cp_parser_new): Initialize new field
+	"innermost_linkage_specification_location".
+	(cp_parser_linkage_specification): Store the location
+	of the linkage specification within the cp_parser.
+	(cp_parser_explicit_specialization): When complaining about
+	template specializations with C linkage, issue a note giving the
+	location of the extern "C".
+	(cp_parser_explicit_template_declaration): Likewise for templates.
+	(maybe_show_extern_c_location): New function.
+	* parser.h (struct cp_parser): New field
+	"innermost_linkage_specification_location".
+
 2017-10-12  Nathan Sidwell  <nathan@acm.org>
 
 	* cp-tree.h (cp_expr): Add const operator * and operator->
diff --git a/gcc/cp/call.c b/gcc/cp/call.c
index 8794210be0a..49cda986f44 100644
--- a/gcc/cp/call.c
+++ b/gcc/cp/call.c
@@ -2082,7 +2082,7 @@ add_function_candidate (struct z_candidate **candidates,
       if (DECL_CONSTRUCTOR_P (fn))
 	i = 1;
       else if (DECL_ASSIGNMENT_OPERATOR_P (fn)
-	       && DECL_OVERLOADED_OPERATOR_P (fn) == NOP_EXPR)
+	       && DECL_OVERLOADED_OPERATOR_IS (fn, NOP_EXPR))
 	i = 2;
       else
 	i = 0;
@@ -4474,7 +4474,7 @@ build_op_call_1 (tree obj, vec<tree, va_gc> **args, tsubst_flags_t complain)
 
   if (TYPE_BINFO (type))
     {
-      fns = lookup_fnfields (TYPE_BINFO (type), cp_operator_id (CALL_EXPR), 1);
+      fns = lookup_fnfields (TYPE_BINFO (type), call_op_identifier, 1);
       if (fns == error_mark_node)
 	return error_mark_node;
     }
@@ -4557,19 +4557,20 @@ build_op_call_1 (tree obj, vec<tree, va_gc> **args, tsubst_flags_t complain)
             }
 	  result = error_mark_node;
 	}
-      /* Since cand->fn will be a type, not a function, for a conversion
-	 function, we must be careful not to unconditionally look at
-	 DECL_NAME here.  */
       else if (TREE_CODE (cand->fn) == FUNCTION_DECL
-	       && DECL_OVERLOADED_OPERATOR_P (cand->fn) == CALL_EXPR)
+	       && DECL_OVERLOADED_OPERATOR_P (cand->fn)
+	       && DECL_OVERLOADED_OPERATOR_IS (cand->fn, CALL_EXPR))
 	result = build_over_call (cand, LOOKUP_NORMAL, complain);
       else
 	{
-	  if (DECL_P (cand->fn))
+	  if (TREE_CODE (cand->fn) == FUNCTION_DECL)
 	    obj = convert_like_with_context (cand->convs[0], obj, cand->fn,
 					     -1, complain);
 	  else
-	    obj = convert_like (cand->convs[0], obj, complain);
+	    {
+	      gcc_checking_assert (TYPE_P (cand->fn));
+	      obj = convert_like (cand->convs[0], obj, complain);
+	    }
 	  obj = convert_from_reference (obj);
 	  result = cp_build_function_call_vec (obj, args, complain);
 	}
@@ -4619,12 +4620,8 @@ static void
 op_error (location_t loc, enum tree_code code, enum tree_code code2,
 	  tree arg1, tree arg2, tree arg3, bool match)
 {
-  const char *opname;
-
-  if (code == MODIFY_EXPR)
-    opname = assignment_operator_name_info[code2].name;
-  else
-    opname = operator_name_info[code].name;
+  bool assop = code == MODIFY_EXPR;
+  const char *opname = OVL_OP_INFO (assop, assop ? code2 : code)->name;
 
   switch (code)
     {
@@ -5183,7 +5180,7 @@ build_conditional_expr_1 (location_t loc, tree arg1, tree arg2, tree arg3,
       add_builtin_candidates (&candidates,
 			      COND_EXPR,
 			      NOP_EXPR,
-			      cp_operator_id (COND_EXPR),
+			      ovl_op_identifier (false, COND_EXPR),
 			      args,
 			      LOOKUP_NORMAL, complain);
 
@@ -5573,7 +5570,6 @@ build_new_op_1 (location_t loc, enum tree_code code, int flags, tree arg1,
 {
   struct z_candidate *candidates = 0, *cand;
   vec<tree, va_gc> *arglist;
-  tree fnname;
   tree args[3];
   tree result = NULL_TREE;
   bool result_valid_p = false;
@@ -5590,14 +5586,13 @@ build_new_op_1 (location_t loc, enum tree_code code, int flags, tree arg1,
       || error_operand_p (arg3))
     return error_mark_node;
 
-  if (code == MODIFY_EXPR)
+  bool ismodop = code == MODIFY_EXPR;
+  if (ismodop)
     {
       code2 = TREE_CODE (arg3);
       arg3 = NULL_TREE;
-      fnname = cp_assignment_operator_id (code2);
     }
-  else
-    fnname = cp_operator_id (code);
+  tree fnname = ovl_op_identifier (ismodop, ismodop ? code2 : code);
 
   arg1 = prep_operand (arg1);
 
@@ -5792,7 +5787,7 @@ build_new_op_1 (location_t loc, enum tree_code code, int flags, tree arg1,
 		? G_("no %<%D(int)%> declared for postfix %qs,"
 		     " trying prefix operator instead")
 		: G_("no %<%D(int)%> declared for postfix %qs");
-	      permerror (loc, msg, fnname, operator_name_info[code].name);
+	      permerror (loc, msg, fnname, OVL_OP_INFO (false, code)->name);
 	    }
 
 	  if (!flag_permissive)
@@ -6204,7 +6199,7 @@ build_op_delete_call (enum tree_code code, tree addr, tree size,
 
   type = strip_array_types (TREE_TYPE (TREE_TYPE (addr)));
 
-  fnname = cp_operator_id (code);
+  fnname = ovl_op_identifier (false, code);
 
   if (CLASS_TYPE_P (type)
       && COMPLETE_TYPE_P (complete_type (type))
@@ -6433,7 +6428,7 @@ build_op_delete_call (enum tree_code code, tree addr, tree size,
 
   if (complain & tf_error)
     error ("no suitable %<operator %s%> for %qT",
-	   operator_name_info[(int)code].name, type);
+	   OVL_OP_INFO (false, code)->name, type);
   return error_mark_node;
 }
 
@@ -7717,8 +7712,11 @@ build_over_call (struct z_candidate *cand, int flags, tsubst_flags_t complain)
     }
 
   /* N3276 magic doesn't apply to nested calls.  */
-  int decltype_flag = (complain & tf_decltype);
+  tsubst_flags_t decltype_flag = (complain & tf_decltype);
   complain &= ~tf_decltype;
+  /* No-Cleanup doesn't apply to nested calls either.  */
+  tsubst_flags_t no_cleanup_complain = complain;
+  complain &= ~tf_no_cleanup;
 
   /* Find maximum size of vector to hold converted arguments.  */
   parmlen = list_length (parm);
@@ -7916,7 +7914,7 @@ build_over_call (struct z_candidate *cand, int flags, tsubst_flags_t complain)
       if (flags & LOOKUP_NO_CONVERSION)
 	conv->user_conv_p = true;
 
-      tsubst_flags_t arg_complain = complain & (~tf_no_cleanup);
+      tsubst_flags_t arg_complain = complain;
       if (!conversion_warning)
 	arg_complain &= ~tf_warning;
 
@@ -8110,7 +8108,8 @@ build_over_call (struct z_candidate *cand, int flags, tsubst_flags_t complain)
 	  return val;
 	}
     }
-  else if (DECL_OVERLOADED_OPERATOR_P (fn) == NOP_EXPR
+  else if (DECL_ASSIGNMENT_OPERATOR_P (fn)
+	   && DECL_OVERLOADED_OPERATOR_IS (fn, NOP_EXPR)
 	   && trivial_fn_p (fn)
 	   && !DECL_DELETED_FN (fn))
     {
@@ -8164,7 +8163,8 @@ build_over_call (struct z_candidate *cand, int flags, tsubst_flags_t complain)
       else if (default_ctor_p (fn))
 	{
 	  if (is_dummy_object (argarray[0]))
-	    return force_target_expr (DECL_CONTEXT (fn), void_node, complain);
+	    return force_target_expr (DECL_CONTEXT (fn), void_node,
+				      no_cleanup_complain);
 	  else
 	    return cp_build_indirect_ref (argarray[0], RO_NULL, complain);
 	}
@@ -8277,7 +8277,7 @@ first_non_public_field (tree type)
 static bool
 has_trivial_copy_assign_p (tree type, bool access, bool *hasassign)
 {
-  tree fns = get_class_binding (type, cp_assignment_operator_id (NOP_EXPR));
+  tree fns = get_class_binding (type, assign_op_identifier);
   bool all_trivial = true;
 
   /* Iterate over overloads of the assignment operator, checking
@@ -8777,8 +8777,7 @@ build_special_member_call (tree instance, tree name, vec<tree, va_gc> **args,
   vec<tree, va_gc> *allocated = NULL;
   tree ret;
 
-  gcc_assert (IDENTIFIER_CDTOR_P (name)
-	      || name == cp_assignment_operator_id (NOP_EXPR));
+  gcc_assert (IDENTIFIER_CDTOR_P (name) || name == assign_op_identifier);
   if (TYPE_P (binfo))
     {
       /* Resolve the name.  */
@@ -8804,7 +8803,7 @@ build_special_member_call (tree instance, tree name, vec<tree, va_gc> **args,
       if (!same_type_ignoring_top_level_qualifiers_p
 	  (TREE_TYPE (instance), BINFO_TYPE (binfo)))
 	{
-	  if (name != cp_assignment_operator_id (NOP_EXPR))
+	  if (IDENTIFIER_CDTOR_P (name))
 	    /* For constructors and destructors, either the base is
 	       non-virtual, or it is virtual but we are doing the
 	       conversion from a constructor or destructor for the
@@ -8812,10 +8811,13 @@ build_special_member_call (tree instance, tree name, vec<tree, va_gc> **args,
 	       statically.  */
 	    instance = convert_to_base_statically (instance, binfo);
 	  else
-	    /* However, for assignment operators, we must convert
-	       dynamically if the base is virtual.  */
-	    instance = build_base_path (PLUS_EXPR, instance,
-					binfo, /*nonnull=*/1, complain);
+	    {
+	      /* However, for assignment operators, we must convert
+		 dynamically if the base is virtual.  */
+	      gcc_checking_assert (name == assign_op_identifier);
+	      instance = build_base_path (PLUS_EXPR, instance,
+					  binfo, /*nonnull=*/1, complain);
+	    }
 	}
     }
 
@@ -9062,7 +9064,6 @@ build_new_method_call_1 (tree instance, tree fns, vec<tree, va_gc> **args,
      static member function.  */
   instance = mark_type_use (instance);
 
-
   /* Figure out whether to skip the first argument for the error
      message we will display to users if an error occurs.  We don't
      want to display any compiler-generated arguments.  The "this"
diff --git a/gcc/cp/class.c b/gcc/cp/class.c
index a90b85f2a5c..98e62c6ad45 100644
--- a/gcc/cp/class.c
+++ b/gcc/cp/class.c
@@ -3011,7 +3011,7 @@ static tree
 dfs_declare_virt_assop_and_dtor (tree binfo, void *data)
 {
   tree bv, fn, t = (tree)data;
-  tree opname = cp_assignment_operator_id (NOP_EXPR);
+  tree opname = assign_op_identifier;
 
   gcc_assert (t && CLASS_TYPE_P (t));
   gcc_assert (binfo && TREE_CODE (binfo) == TREE_BINFO);
@@ -5038,7 +5038,7 @@ vbase_has_user_provided_move_assign (tree type)
   /* Does the type itself have a user-provided move assignment operator?  */
   if (!CLASSTYPE_LAZY_MOVE_ASSIGN (type))
     for (ovl_iterator iter (get_class_binding_direct
-			    (type, cp_assignment_operator_id (NOP_EXPR)));
+			    (type, assign_op_identifier));
 	 iter; ++iter)
       if (!DECL_ARTIFICIAL (*iter) && move_fn_p (*iter))
 	return true;
@@ -5186,7 +5186,7 @@ classtype_has_move_assign_or_move_ctor_p (tree t, bool user_p)
 
   if (!CLASSTYPE_LAZY_MOVE_ASSIGN (t))
     for (ovl_iterator iter (get_class_binding_direct
-			    (t, cp_assignment_operator_id (NOP_EXPR)));
+			    (t, assign_op_identifier));
 	 iter; ++iter)
       if ((!user_p || !DECL_ARTIFICIAL (*iter)) && move_fn_p (*iter))
 	return true;
@@ -5304,7 +5304,7 @@ type_requires_array_cookie (tree type)
      the array to the deallocation function, so we will need to store
      a cookie.  */
   fns = lookup_fnfields (TYPE_BINFO (type),
-			 cp_operator_id (VEC_DELETE_EXPR),
+			 ovl_op_identifier (false, VEC_DELETE_EXPR),
 			 /*protect=*/0);
   /* If there are no `operator []' members, or the lookup is
      ambiguous, then we don't need a cookie.  */
@@ -5394,18 +5394,20 @@ explain_non_literal_class (tree t)
     /* Already explained.  */
     return;
 
-  inform (0, "%q+T is not literal because:", t);
+  inform (UNKNOWN_LOCATION, "%q+T is not literal because:", t);
   if (cxx_dialect < cxx17 && LAMBDA_TYPE_P (t))
-    inform (0, "  %qT is a closure type, which is only literal in "
+    inform (UNKNOWN_LOCATION,
+	    "  %qT is a closure type, which is only literal in "
 	    "C++17 and later", t);
   else if (TYPE_HAS_NONTRIVIAL_DESTRUCTOR (t))
-    inform (0, "  %q+T has a non-trivial destructor", t);
+    inform (UNKNOWN_LOCATION, "  %q+T has a non-trivial destructor", t);
   else if (CLASSTYPE_NON_AGGREGATE (t)
 	   && !TYPE_HAS_TRIVIAL_DFLT (t)
 	   && !LAMBDA_TYPE_P (t)
 	   && !TYPE_HAS_CONSTEXPR_CTOR (t))
     {
-      inform (0, "  %q+T is not an aggregate, does not have a trivial "
+      inform (UNKNOWN_LOCATION,
+	      "  %q+T is not an aggregate, does not have a trivial "
 	      "default constructor, and has no constexpr constructor that "
 	      "is not a copy or move constructor", t);
       if (type_has_non_user_provided_default_constructor (t))
@@ -5437,7 +5439,8 @@ explain_non_literal_class (tree t)
 	  tree basetype = TREE_TYPE (base_binfo);
 	  if (!CLASSTYPE_LITERAL_P (basetype))
 	    {
-	      inform (0, "  base class %qT of %q+T is non-literal",
+	      inform (UNKNOWN_LOCATION,
+		      "  base class %qT of %q+T is non-literal",
 		      basetype, t);
 	      explain_non_literal_class (basetype);
 	      return;
@@ -5992,8 +5995,6 @@ layout_class_type (tree t, tree *virtuals_p)
   bool last_field_was_bitfield = false;
   /* The location at which the next field should be inserted.  */
   tree *next_field;
-  /* T, as a base class.  */
-  tree base_t;
 
   /* Keep track of the first non-static data member.  */
   non_static_data_members = TYPE_FIELDS (t);
@@ -6218,15 +6219,11 @@ layout_class_type (tree t, tree *virtuals_p)
      that the type is laid out they are no longer important.  */
   remove_zero_width_bit_fields (t);
 
-  /* Create the version of T used for virtual bases.  We do not use
-     make_class_type for this version; this is an artificial type.  For
-     a POD type, we just reuse T.  */
   if (CLASSTYPE_NON_LAYOUT_POD_P (t) || CLASSTYPE_EMPTY_P (t))
     {
-      base_t = make_node (TREE_CODE (t));
-
-      /* Set the size and alignment for the new type.  */
-      tree eoc;
+      /* T needs a different layout as a base (eliding virtual bases
+	 or whatever).  Create that version.  */
+      tree base_t = make_node (TREE_CODE (t));
 
       /* If the ABI version is not at least two, and the last
 	 field was a bit-field, RLI may not be on a byte
@@ -6235,7 +6232,7 @@ layout_class_type (tree t, tree *virtuals_p)
 	 indicates the total number of bits used.  Therefore,
 	 rli_size_so_far, rather than rli_size_unit_so_far, is
 	 used to compute TYPE_SIZE_UNIT.  */
-      eoc = end_of_class (t, /*include_virtuals_p=*/0);
+      tree eoc = end_of_class (t, /*include_virtuals_p=*/0);
       TYPE_SIZE_UNIT (base_t)
 	= size_binop (MAX_EXPR,
 		      fold_convert (sizetype,
@@ -6252,7 +6249,8 @@ layout_class_type (tree t, tree *virtuals_p)
       SET_TYPE_ALIGN (base_t, rli->record_align);
       TYPE_USER_ALIGN (base_t) = TYPE_USER_ALIGN (t);
 
-      /* Copy the fields from T.  */
+      /* Copy the non-static data members of T. This will include its
+	 direct non-virtual bases & vtable.  */
       next_field = &TYPE_FIELDS (base_t);
       for (field = TYPE_FIELDS (t); field; field = DECL_CHAIN (field))
 	if (TREE_CODE (field) == FIELD_DECL)
@@ -6263,9 +6261,14 @@ layout_class_type (tree t, tree *virtuals_p)
 	  }
       *next_field = NULL_TREE;
 
+      /* We use the base type for trivial assignments, and hence it
+	 needs a mode.  */
+      compute_record_mode (base_t);
+
+      TYPE_CONTEXT (base_t) = t;
+
       /* Record the base version of the type.  */
       CLASSTYPE_AS_BASE (t) = base_t;
-      TYPE_CONTEXT (base_t) = t;
     }
   else
     CLASSTYPE_AS_BASE (t) = t;
@@ -6822,11 +6825,6 @@ finish_struct_1 (tree t)
 
   set_class_bindings (t);
 
-  if (CLASSTYPE_AS_BASE (t) != t)
-    /* We use the base type for trivial assignments, and hence it
-       needs a mode.  */
-    compute_record_mode (CLASSTYPE_AS_BASE (t));
-
   /* With the layout complete, check for flexible array members and
      zero-length arrays that might overlap other members in the final
      layout.  */
diff --git a/gcc/cp/constexpr.c b/gcc/cp/constexpr.c
index 1aa529eb8dc..fdc296908af 100644
--- a/gcc/cp/constexpr.c
+++ b/gcc/cp/constexpr.c
@@ -1269,8 +1269,7 @@ cxx_bind_parameters_in_call (const constexpr_ctx *ctx, tree t,
 	{
 	  x = ctx->object;
 	  /* We don't use cp_build_addr_expr here because we don't want to
-	     capture the object argument until we've chosen a non-static member
-	     function.  */
+	     capture the object argument during constexpr evaluation.  */
 	  x = build_address (x);
 	}
       bool lval = false;
diff --git a/gcc/cp/cp-objcp-common.c b/gcc/cp/cp-objcp-common.c
index f251b05775b..e051d66b67b 100644
--- a/gcc/cp/cp-objcp-common.c
+++ b/gcc/cp/cp-objcp-common.c
@@ -61,43 +61,34 @@ cxx_warn_unused_global_decl (const_tree decl)
 size_t
 cp_tree_size (enum tree_code code)
 {
+  gcc_checking_assert (code >= NUM_TREE_CODES);
   switch (code)
     {
-    case PTRMEM_CST:		return sizeof (struct ptrmem_cst);
-    case BASELINK:		return sizeof (struct tree_baselink);
+    case PTRMEM_CST:		return sizeof (ptrmem_cst);
+    case BASELINK:		return sizeof (tree_baselink);
     case TEMPLATE_PARM_INDEX:	return sizeof (template_parm_index);
-    case DEFAULT_ARG:		return sizeof (struct tree_default_arg);
-    case DEFERRED_NOEXCEPT:	return sizeof (struct tree_deferred_noexcept);
-    case OVERLOAD:		return sizeof (struct tree_overload);
-    case STATIC_ASSERT:         return sizeof (struct tree_static_assert);
+    case DEFAULT_ARG:		return sizeof (tree_default_arg);
+    case DEFERRED_NOEXCEPT:	return sizeof (tree_deferred_noexcept);
+    case OVERLOAD:		return sizeof (tree_overload);
+    case STATIC_ASSERT:         return sizeof (tree_static_assert);
     case TYPE_ARGUMENT_PACK:
-    case TYPE_PACK_EXPANSION:
-      return sizeof (struct tree_common);
-
+    case TYPE_PACK_EXPANSION:	return sizeof (tree_type_non_common);
     case NONTYPE_ARGUMENT_PACK:
-    case EXPR_PACK_EXPANSION:
-      return sizeof (struct tree_exp);
-
-    case ARGUMENT_PACK_SELECT:
-      return sizeof (struct tree_argument_pack_select);
-
-    case TRAIT_EXPR:
-      return sizeof (struct tree_trait_expr);
-
-    case LAMBDA_EXPR:           return sizeof (struct tree_lambda_expr);
-
-    case TEMPLATE_INFO:         return sizeof (struct tree_template_info);
-
-    case CONSTRAINT_INFO:       return sizeof (struct tree_constraint_info);
-
-    case USERDEF_LITERAL:	return sizeof (struct tree_userdef_literal);
-
-    case TEMPLATE_DECL:		return sizeof (struct tree_template_decl);
-
+    case EXPR_PACK_EXPANSION:	return sizeof (tree_exp);
+    case ARGUMENT_PACK_SELECT:	return sizeof (tree_argument_pack_select);
+    case TRAIT_EXPR:		return sizeof (tree_trait_expr);
+    case LAMBDA_EXPR:           return sizeof (tree_lambda_expr);
+    case TEMPLATE_INFO:         return sizeof (tree_template_info);
+    case CONSTRAINT_INFO:       return sizeof (tree_constraint_info);
+    case USERDEF_LITERAL:	return sizeof (tree_userdef_literal);
+    case TEMPLATE_DECL:		return sizeof (tree_template_decl);
     default:
-      if (TREE_CODE_CLASS (code) == tcc_declaration)
-	return sizeof (struct tree_decl_non_common);
-      gcc_unreachable ();
+      switch (TREE_CODE_CLASS (code))
+	{
+	case tcc_declaration:	return sizeof (tree_decl_non_common);
+	case tcc_type:		return sizeof (tree_type_non_common);
+	default: gcc_unreachable ();
+	}
     }
   /* NOTREACHED */
 }
diff --git a/gcc/cp/cp-tree.h b/gcc/cp/cp-tree.h
index dc98dd881c5..874cbcbd2bd 100644
--- a/gcc/cp/cp-tree.h
+++ b/gcc/cp/cp-tree.h
@@ -217,7 +217,7 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
    things) to iterate over their overloads defined by/for a type.  For
    example:
 
-     tree ovlid = cp_assignment_operator_id (NOP_EXPR);
+     tree ovlid = assign_op_identifier;
      tree overloads = get_class_binding (type, ovlid);
      for (ovl_iterator it (overloads); it; ++it) { ... }
 
@@ -244,22 +244,15 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
 /* The name of a destructor that destroys virtual base classes, and
    then deletes the entire object.  */
 #define deleting_dtor_identifier	cp_global_trees[CPTI_DELETING_DTOR_IDENTIFIER]
+
+#define ovl_op_identifier(ISASS, CODE)  (OVL_OP_INFO(ISASS, CODE)->identifier)
+#define assign_op_identifier (ovl_op_info[true][OVL_OP_NOP_EXPR].identifier)
+#define call_op_identifier (ovl_op_info[false][OVL_OP_CALL_EXPR].identifier)
 /* The name used for conversion operators -- but note that actual
    conversion functions use special identifiers outside the identifier
    table.  */
 #define conv_op_identifier		cp_global_trees[CPTI_CONV_OP_IDENTIFIER]
 
-/* The name of the identifier used internally to represent operator CODE.  */
-#define cp_operator_id(CODE) \
-  (operator_name_info[(int) (CODE)].identifier)
-
-/* The name of the identifier used to represent assignment operator CODE,
-   both simple (i.e., operator= with CODE == NOP_EXPR) and compound (e.g.,
-   operator+= with CODE == PLUS_EXPR).  Includes copy and move assignment.
-   Use copy_fn_p() to test specifically for copy assignment.  */
-#define cp_assignment_operator_id(CODE)				\
-  (assignment_operator_name_info[(int) (CODE)].identifier)
-
 #define delta_identifier		cp_global_trees[CPTI_DELTA_IDENTIFIER]
 #define in_charge_identifier		cp_global_trees[CPTI_IN_CHARGE_IDENTIFIER]
 /* The name of the parameter that contains a pointer to the VTT to use
@@ -561,7 +554,6 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
 struct GTY(()) lang_identifier {
   struct c_common_identifier c_common;
   cxx_binding *bindings;
-  tree label_value;
 };
 
 /* Return a typed pointer version of T if it designates a
@@ -996,11 +988,6 @@ enum GTY(()) abstract_class_use {
 #define SET_IDENTIFIER_TYPE_VALUE(NODE,TYPE) (TREE_TYPE (NODE) = (TYPE))
 #define IDENTIFIER_HAS_TYPE_VALUE(NODE) (IDENTIFIER_TYPE_VALUE (NODE) ? 1 : 0)
 
-#define IDENTIFIER_LABEL_VALUE(NODE) \
-  (LANG_IDENTIFIER_CAST (NODE)->label_value)
-#define SET_IDENTIFIER_LABEL_VALUE(NODE, VALUE)   \
-  IDENTIFIER_LABEL_VALUE (NODE) = (VALUE)
-
 /* Kinds of identifiers.  Values are carefully chosen.  */
 enum cp_identifier_kind {
   cik_normal = 0,	/* Not a special identifier.  */
@@ -1009,9 +996,9 @@ enum cp_identifier_kind {
   cik_dtor = 3,		/* Destructor (in-chg, deleting, complete or
 			   base).  */
   cik_simple_op = 4,	/* Non-assignment operator name.  */
-  cik_newdel_op = 5,	/* New or delete operator name.  */
-  cik_assign_op = 6,	/* An assignment operator name.  */
-  cik_conv_op = 7,	/* Conversion operator name.  */
+  cik_assign_op = 5,	/* An assignment operator name.  */
+  cik_conv_op = 6,	/* Conversion operator name.  */
+  cik_reserved_for_udlit = 7,	/* Not yet in use  */
   cik_max
 };
 
@@ -1066,24 +1053,38 @@ enum cp_identifier_kind {
 #define IDENTIFIER_ANY_OP_P(NODE)		\
   (IDENTIFIER_KIND_BIT_2 (NODE))
 
-/* True if this identifier is for new or delete operator.  Value 5.  */
-#define IDENTIFIER_NEWDEL_OP_P(NODE)		\
-  (IDENTIFIER_KIND_BIT_2 (NODE)			\
-   & (!IDENTIFIER_KIND_BIT_1 (NODE))		\
-   & IDENTIFIER_KIND_BIT_0 (NODE))
+/* True if this identifier is for an overloaded operator. Values 4, 5.  */
+#define IDENTIFIER_OVL_OP_P(NODE)		\
+  (IDENTIFIER_ANY_OP_P (NODE)			\
+   & (!IDENTIFIER_KIND_BIT_1 (NODE)))
 
-/* True if this identifier is for any assignment. Values 6.  */
+/* True if this identifier is for any assignment. Values 5.  */
 #define IDENTIFIER_ASSIGN_OP_P(NODE)		\
-  (IDENTIFIER_KIND_BIT_2 (NODE)			\
-   & IDENTIFIER_KIND_BIT_1 (NODE)		\
-   & (!IDENTIFIER_KIND_BIT_0 (NODE)))
+  (IDENTIFIER_OVL_OP_P (NODE)			\
+   & IDENTIFIER_KIND_BIT_0 (NODE))
 
 /* True if this identifier is the name of a type-conversion
    operator.  Value 7.  */
 #define IDENTIFIER_CONV_OP_P(NODE)		\
-  (IDENTIFIER_KIND_BIT_2 (NODE)			\
+  (IDENTIFIER_ANY_OP_P (NODE)			\
    & IDENTIFIER_KIND_BIT_1 (NODE)		\
-   & IDENTIFIER_KIND_BIT_0 (NODE))
+   & (!IDENTIFIER_KIND_BIT_0 (NODE)))
+
+/* True if this identifier is a new or delete operator.  */
+#define IDENTIFIER_NEWDEL_OP_P(NODE)		\
+  (IDENTIFIER_OVL_OP_P (NODE)			\
+   && IDENTIFIER_OVL_OP_FLAGS (NODE) & OVL_OP_FLAG_ALLOC)
+
+/* True if this identifier is a new operator.  */
+#define IDENTIFIER_NEW_OP_P(NODE)					\
+  (IDENTIFIER_OVL_OP_P (NODE)						\
+   && (IDENTIFIER_OVL_OP_FLAGS (NODE)					\
+       & (OVL_OP_FLAG_ALLOC | OVL_OP_FLAG_DELETE)) == OVL_OP_FLAG_ALLOC)
+
+/* Access a C++-specific index for identifier NODE.
+   Used to optimize operator mappings etc.  */
+#define IDENTIFIER_CP_INDEX(NODE)		\
+  (IDENTIFIER_NODE_CHECK(NODE)->base.u.bits.address_space)
 
 /* In a RECORD_TYPE or UNION_TYPE, nonzero if any component is read-only.  */
 #define C_TYPE_FIELDS_READONLY(TYPE) \
@@ -1209,9 +1210,10 @@ struct GTY (()) tree_trait_expr {
   (CLASS_TYPE_P (NODE) && CLASSTYPE_LAMBDA_EXPR (NODE))
 
 /* Test if FUNCTION_DECL is a lambda function.  */
-#define LAMBDA_FUNCTION_P(FNDECL)			\
-  (DECL_DECLARES_FUNCTION_P (FNDECL)			\
-   && DECL_OVERLOADED_OPERATOR_P (FNDECL) == CALL_EXPR	\
+#define LAMBDA_FUNCTION_P(FNDECL)				\
+  (DECL_DECLARES_FUNCTION_P (FNDECL)				\
+   && DECL_OVERLOADED_OPERATOR_P (FNDECL)			\
+   && DECL_OVERLOADED_OPERATOR_IS (FNDECL, CALL_EXPR)		\
    && LAMBDA_TYPE_P (CP_DECL_CONTEXT (FNDECL)))
 
 enum cp_lambda_default_capture_mode_type {
@@ -1662,12 +1664,22 @@ struct cxx_int_tree_map_hasher : ggc_ptr_hash<cxx_int_tree_map>
   static bool equal (cxx_int_tree_map *, cxx_int_tree_map *);
 };
 
-struct named_label_entry;
+struct named_label_entry; /* Defined in decl.c.  */
 
-struct named_label_hasher : ggc_ptr_hash<named_label_entry>
+struct named_label_hash : ggc_remove <named_label_entry *>
 {
-  static hashval_t hash (named_label_entry *);
-  static bool equal (named_label_entry *, named_label_entry *);
+  typedef named_label_entry *value_type;
+  typedef tree compare_type; /* An identifier.  */
+
+  inline static hashval_t hash (value_type);
+  inline static bool equal (const value_type, compare_type);
+
+  inline static void mark_empty (value_type &p) {p = NULL;}
+  inline static bool is_empty (value_type p) {return !p;}
+
+  /* Nothing is deletable.  Everything is insertable.  */
+  inline static bool is_deleted (value_type) { return false; }
+  inline static void mark_deleted (value_type) { gcc_unreachable (); }
 };
 
 /* Global state pertinent to the current function.  */
@@ -1696,7 +1708,8 @@ struct GTY(()) language_function {
 
   BOOL_BITFIELD invalid_constexpr : 1;
 
-  hash_table<named_label_hasher> *x_named_labels;
+  hash_table<named_label_hash> *x_named_labels;
+
   cp_binding_level *bindings;
   vec<tree, va_gc> *x_local_names;
   /* Tracking possibly infinite loops.  This is a vec<tree> only because
@@ -2475,26 +2488,24 @@ struct GTY(()) lang_decl_min {
 struct GTY(()) lang_decl_fn {
   struct lang_decl_min min;
 
-  /* In an overloaded operator, this is the value of
-     DECL_OVERLOADED_OPERATOR_P.
-     FIXME: We should really do better in compressing this.  */
-  ENUM_BITFIELD (tree_code) operator_code : 16;
-
+  /* In a overloaded operator, this is the compressed operator code.  */
+  unsigned ovl_op_code : 6;
   unsigned global_ctor_p : 1;
   unsigned global_dtor_p : 1;
+
   unsigned static_function : 1;
   unsigned pure_virtual : 1;
   unsigned defaulted_p : 1;
   unsigned has_in_charge_parm_p : 1;
   unsigned has_vtt_parm_p : 1;
   unsigned pending_inline_p : 1;
-
   unsigned nonconverting : 1;
   unsigned thunk_p : 1;
+
   unsigned this_thunk_p : 1;
   unsigned hidden_friend_p : 1;
   unsigned omp_declare_reduction_p : 1;
-  /* 3 spare bits.  */
+  unsigned spare : 13;
 
   /* 32-bits padding on 64-bit host.  */
 
@@ -2801,23 +2812,24 @@ struct GTY(()) lang_decl {
 #define SET_VAR_HAD_UNKNOWN_BOUND(NODE) \
   (DECL_LANG_SPECIFIC (VAR_DECL_CHECK (NODE))->u.base.unknown_bound_p = true)
 
-/* Set the overloaded operator code for NODE to CODE.  */
-#define SET_OVERLOADED_OPERATOR_CODE(NODE, CODE) \
-  (LANG_DECL_FN_CHECK (NODE)->operator_code = (CODE))
-
-/* If NODE is an overloaded operator, then this returns the TREE_CODE
-   associated with the overloaded operator.  If NODE is not an
-   overloaded operator, ERROR_MARK is returned.  Since the numerical
-   value of ERROR_MARK is zero, this macro can be used as a predicate
-   to test whether or not NODE is an overloaded operator.  */
+/* True iff decl NODE is for an overloaded operator.  */
 #define DECL_OVERLOADED_OPERATOR_P(NODE)		\
-  (IDENTIFIER_ANY_OP_P (DECL_NAME (NODE))		\
-   ? LANG_DECL_FN_CHECK (NODE)->operator_code : ERROR_MARK)
+  IDENTIFIER_ANY_OP_P (DECL_NAME (NODE))
 
 /* Nonzero if NODE is an assignment operator (including += and such).  */
-#define DECL_ASSIGNMENT_OPERATOR_P(NODE) \
+#define DECL_ASSIGNMENT_OPERATOR_P(NODE)		 \
   IDENTIFIER_ASSIGN_OP_P (DECL_NAME (NODE))
 
+/* NODE is a function_decl for an overloaded operator.  Return its
+   compressed (raw) operator code.  Note that this is not a TREE_CODE.  */
+#define DECL_OVERLOADED_OPERATOR_CODE_RAW(NODE)		\
+  (LANG_DECL_FN_CHECK (NODE)->ovl_op_code)
+
+/* DECL is an overloaded operator.  Test whether it is for TREE_CODE
+   (a literal constant).  */
+#define DECL_OVERLOADED_OPERATOR_IS(DECL, CODE)			\
+  (DECL_OVERLOADED_OPERATOR_CODE_RAW (DECL) == OVL_OP_##CODE)
+
 /* For FUNCTION_DECLs: nonzero means that this function is a
    constructor or a destructor with an extra in-charge parameter to
    control whether or not virtual bases are constructed.  */
@@ -3455,7 +3467,7 @@ extern void decl_shadowed_for_var_insert (tree, tree);
 /* Extracts the type or expression pattern from a TYPE_PACK_EXPANSION or
    EXPR_PACK_EXPANSION.  */
 #define PACK_EXPANSION_PATTERN(NODE)                            \
-  (TREE_CODE (NODE) == TYPE_PACK_EXPANSION? TREE_TYPE (NODE)    \
+  (TREE_CODE (NODE) == TYPE_PACK_EXPANSION ? TREE_TYPE (NODE)    \
    : TREE_OPERAND (NODE, 0))
 
 /* Sets the type or expression pattern for a TYPE_PACK_EXPANSION or
@@ -5474,23 +5486,63 @@ enum auto_deduction_context
 
 extern void init_reswords (void);
 
-typedef struct GTY(()) operator_name_info_t {
+/* Various flags for the overloaded operator information.  */
+enum ovl_op_flags
+  {
+    OVL_OP_FLAG_NONE = 0,	/* Don't care.  */
+    OVL_OP_FLAG_UNARY = 1,	/* Is unary.  */
+    OVL_OP_FLAG_BINARY = 2,	/* Is binary.  */
+    OVL_OP_FLAG_AMBIARY = 3,	/* May be unary or binary.  */
+    OVL_OP_FLAG_ALLOC = 4,  	/* operator new or delete.  */
+    OVL_OP_FLAG_DELETE = 1,	/* operator delete.  */
+    OVL_OP_FLAG_VEC = 2		/* vector new or delete.  */
+  };
+
+/* Compressed operator codes.  Order is determined by operators.def
+   and does not match that of tree_codes.  */
+enum ovl_op_code
+  {
+    OVL_OP_ERROR_MARK,
+    OVL_OP_NOP_EXPR,
+#define DEF_OPERATOR(NAME, CODE, MANGLING, FLAGS) OVL_OP_##CODE,
+#define DEF_ASSN_OPERATOR(NAME, CODE, MANGLING) /* NOTHING */
+#include "operators.def"
+    OVL_OP_MAX
+  };
+
+struct GTY(()) ovl_op_info_t {
   /* The IDENTIFIER_NODE for the operator.  */
   tree identifier;
   /* The name of the operator.  */
   const char *name;
   /* The mangled name of the operator.  */
   const char *mangled_name;
-  /* The arity of the operator.  */
-  int arity;
-} operator_name_info_t;
+  /* The (regular) tree code.  */
+  enum tree_code tree_code : 16;
+  /* The (compressed) operator code.  */
+  enum ovl_op_code ovl_op_code : 8;
+  /* The ovl_op_flags of the operator */
+  unsigned flags : 8;
+};
 
-/* A mapping from tree codes to operator name information.  */
-extern GTY(()) operator_name_info_t operator_name_info
-  [(int) MAX_TREE_CODES];
-/* Similar, but for assignment operators.  */
-extern GTY(()) operator_name_info_t assignment_operator_name_info
-  [(int) MAX_TREE_CODES];
+/* Overloaded operator info indexed by ass_op_p & ovl_op_code.  */
+extern GTY(()) ovl_op_info_t ovl_op_info[2][OVL_OP_MAX];
+/* Mapping from tree_codes to ovl_op_codes.  */
+extern GTY(()) unsigned char ovl_op_mapping[MAX_TREE_CODES];
+/* Mapping for ambi-ary operators from the binary to the unary.  */
+extern GTY(()) unsigned char ovl_op_alternate[OVL_OP_MAX];
+
+/* Given an ass_op_p boolean and a tree code, return a pointer to its
+   overloaded operator info.  Tree codes for non-overloaded operators
+   map to the error-operator.  */
+#define OVL_OP_INFO(IS_ASS_P, TREE_CODE)			\
+  (&ovl_op_info[(IS_ASS_P) != 0][ovl_op_mapping[(TREE_CODE)]])
+/* Overloaded operator info for an identifier for which
+   IDENTIFIER_OVL_OP_P is true.  */
+#define IDENTIFIER_OVL_OP_INFO(NODE) \
+  (&ovl_op_info[IDENTIFIER_KIND_BIT_0 (NODE)][IDENTIFIER_CP_INDEX (NODE)])
+#define IDENTIFIER_OVL_OP_FLAGS(NODE) \
+  (IDENTIFIER_OVL_OP_INFO (NODE)->flags)
 
 /* A type-qualifier, or bitmask therefore, using the TYPE_QUAL
    constants.  */
@@ -6356,6 +6408,7 @@ extern bool parsing_nsdmi (void);
 extern bool parsing_default_capturing_generic_lambda_in_template (void);
 extern void inject_this_parameter (tree, cp_cv_quals);
 extern location_t defarg_location (tree);
+extern void maybe_show_extern_c_location (void);
 
 /* in pt.c */
 extern bool check_template_shadow		(tree);
@@ -6429,7 +6482,7 @@ extern bool uses_parameter_packs                (tree);
 extern bool template_parameter_pack_p           (const_tree);
 extern bool function_parameter_pack_p		(const_tree);
 extern bool function_parameter_expanded_from_pack_p (tree, tree);
-extern tree make_pack_expansion                 (tree);
+extern tree make_pack_expansion                 (tree, tsubst_flags_t = tf_warning_or_error);
 extern bool check_for_bare_parameter_packs      (tree);
 extern tree build_template_info			(tree, tree);
 extern tree get_template_info			(const_tree);
diff --git a/gcc/cp/cvt.c b/gcc/cp/cvt.c
index c0d0a600562..9ce094eb2a5 100644
--- a/gcc/cp/cvt.c
+++ b/gcc/cp/cvt.c
@@ -1834,12 +1834,27 @@ type_promotes_to (tree type)
 	   || type == char32_type_node
 	   || type == wchar_type_node)
     {
+      tree prom = type;
+
+      if (TREE_CODE (type) == ENUMERAL_TYPE)
+	{
+	  prom = ENUM_UNDERLYING_TYPE (prom);
+	  if (!ENUM_IS_SCOPED (type)
+	      && ENUM_FIXED_UNDERLYING_TYPE_P (type))
+	    {
+	      /* ISO C++17, 7.6/4.  A prvalue of an unscoped enumeration type
+		 whose underlying type is fixed (10.2) can be converted to a
+		 prvalue of its underlying type. Moreover, if integral promotion
+		 can be applied to its underlying type, a prvalue of an unscoped
+		 enumeration type whose underlying type is fixed can also be 
+		 converted to a prvalue of the promoted underlying type.  */
+	      return type_promotes_to (prom);
+	    }
+	}
+
       int precision = MAX (TYPE_PRECISION (type),
 			   TYPE_PRECISION (integer_type_node));
       tree totype = c_common_type_for_size (precision, 0);
-      tree prom = type;
-      if (TREE_CODE (prom) == ENUMERAL_TYPE)
-	prom = ENUM_UNDERLYING_TYPE (prom);
       if (TYPE_UNSIGNED (prom)
 	  && ! int_fits_type_p (TYPE_MAX_VALUE (prom), totype))
 	prom = c_common_type_for_size (precision, 1);
diff --git a/gcc/cp/decl.c b/gcc/cp/decl.c
index 5d3f39e1f59..49b871564d6 100644
--- a/gcc/cp/decl.c
+++ b/gcc/cp/decl.c
@@ -65,8 +65,6 @@ static const char *redeclaration_error_message (tree, tree);
 
 static int decl_jump_unsafe (tree);
 static void require_complete_types_for_parms (tree);
-static bool ambi_op_p (enum tree_code);
-static bool unary_op_p (enum tree_code);
 static void push_local_name (tree);
 static tree grok_reference_init (tree, tree, tree, int);
 static tree grokvardecl (tree, tree, tree, const cp_decl_specifier_seq *,
@@ -189,27 +187,33 @@ struct GTY((chain_next ("%h.next"))) named_label_use_entry {
    function, and so we can check the validity of jumps to these labels.  */
 
 struct GTY((for_user)) named_label_entry {
-  /* The decl itself.  */
-  tree label_decl;
+
+  tree name;  /* Name of decl. */
+
+  tree label_decl; /* LABEL_DECL, unless deleted local label. */
+
+  named_label_entry *outer; /* Outer shadowed chain.  */
 
   /* The binding level to which the label is *currently* attached.
      This is initially set to the binding level in which the label
      is defined, but is modified as scopes are closed.  */
   cp_binding_level *binding_level;
+
   /* The head of the names list that was current when the label was
      defined, or the inner scope popped.  These are the decls that will
      be skipped when jumping to the label.  */
   tree names_in_scope;
+
   /* A vector of all decls from all binding levels that would be
      crossed by a backward branch to the label.  */
   vec<tree, va_gc> *bad_decls;
 
   /* A list of uses of the label, before the label is defined.  */
-  struct named_label_use_entry *uses;
+  named_label_use_entry *uses;
 
   /* The following bits are set after the label is defined, and are
-     updated as scopes are popped.  They indicate that a backward jump
-     to the label will illegally enter a scope of the given flavor.  */
+     updated as scopes are popped.  They indicate that a jump to the
+     label will illegally enter a scope of the given flavor.  */
   bool in_try_scope;
   bool in_catch_scope;
   bool in_omp_scope;
@@ -347,7 +351,7 @@ finish_scope (void)
    in a valid manner, and issue any appropriate warnings or errors.  */
 
 static void
-pop_label (tree label, tree old_value)
+check_label_used (tree label)
 {
   if (!processing_template_decl)
     {
@@ -364,18 +368,6 @@ pop_label (tree label, tree old_value)
       else 
 	warn_for_unused_label (label);
     }
-
-  SET_IDENTIFIER_LABEL_VALUE (DECL_NAME (label), old_value);
-}
-
-/* Push all named labels into a vector, so that we can sort it on DECL_UID
-   to avoid code generation differences.  */
-
-int
-note_label (named_label_entry **slot, vec<named_label_entry **> &labels)
-{
-  labels.quick_push (slot);
-  return 1;
 }
 
 /* Helper function to sort named label entries in a vector by DECL_UID.  */
@@ -383,13 +375,11 @@ note_label (named_label_entry **slot, vec<named_label_entry **> &labels)
 static int
 sort_labels (const void *a, const void *b)
 {
-  named_label_entry **slot1 = *(named_label_entry **const *) a;
-  named_label_entry **slot2 = *(named_label_entry **const *) b;
-  if (DECL_UID ((*slot1)->label_decl) < DECL_UID ((*slot2)->label_decl))
-    return -1;
-  if (DECL_UID ((*slot1)->label_decl) > DECL_UID ((*slot2)->label_decl))
-    return 1;
-  return 0;
+  tree label1 = *(tree const *) a;
+  tree label2 = *(tree const *) b;
+
+  /* DECL_UIDs can never be equal.  */
+  return DECL_UID (label1) > DECL_UID (label2) ? -1 : +1;
 }
 
 /* At the end of a function, all labels declared within the function
@@ -399,46 +389,58 @@ sort_labels (const void *a, const void *b)
 static void
 pop_labels (tree block)
 {
-  if (named_labels)
+  if (!named_labels)
+    return;
+
+  /* We need to add the labels to the block chain, so debug
+     information is emitted.  But, we want the order to be stable so
+     need to sort them first.  Otherwise the debug output could be
+     randomly ordered.  I guess it's mostly stable, unless the hash
+     table implementation changes.  */
+  auto_vec<tree, 32> labels (named_labels->elements ());
+  hash_table<named_label_hash>::iterator end (named_labels->end ());
+  for (hash_table<named_label_hash>::iterator iter
+	 (named_labels->begin ()); iter != end; ++iter)
     {
-      auto_vec<named_label_entry **, 32> labels;
-      named_label_entry **slot;
-      unsigned int i;
+      named_label_entry *ent = *iter;
 
-      /* Push all the labels into a vector and sort them by DECL_UID,
-	 so that gaps between DECL_UIDs don't affect code generation.  */
-      labels.reserve_exact (named_labels->elements ());
-      named_labels->traverse<vec<named_label_entry **> &, note_label> (labels);
-      labels.qsort (sort_labels);
-      FOR_EACH_VEC_ELT (labels, i, slot)
-	{
-	  struct named_label_entry *ent = *slot;
+      gcc_checking_assert (!ent->outer);
+      if (ent->label_decl)
+	labels.quick_push (ent->label_decl);
+      ggc_free (ent);
+    }
+  named_labels = NULL;
+  labels.qsort (sort_labels);
 
-	  pop_label (ent->label_decl, NULL_TREE);
+  while (labels.length ())
+    {
+      tree label = labels.pop ();
 
-	  /* Put the labels into the "variables" of the top-level block,
-	     so debugger can see them.  */
-	  DECL_CHAIN (ent->label_decl) = BLOCK_VARS (block);
-	  BLOCK_VARS (block) = ent->label_decl;
+      DECL_CHAIN (label) = BLOCK_VARS (block);
+      BLOCK_VARS (block) = label;
 
-	  named_labels->clear_slot (slot);
-	}
-      named_labels = NULL;
+      check_label_used (label);
     }
 }
 
 /* At the end of a block with local labels, restore the outer definition.  */
 
 static void
-pop_local_label (tree label, tree old_value)
+pop_local_label (tree id, tree label)
 {
-  struct named_label_entry dummy;
-
-  pop_label (label, old_value);
+  check_label_used (label);
+  named_label_entry **slot = named_labels->find_slot_with_hash
+    (id, IDENTIFIER_HASH_VALUE (id), NO_INSERT);
+  named_label_entry *ent = *slot;
 
-  dummy.label_decl = label;
-  named_label_entry **slot = named_labels->find_slot (&dummy, NO_INSERT);
-  named_labels->clear_slot (slot);
+  if (ent->outer)
+    ent = ent->outer;
+  else
+    {
+      ent = ggc_cleared_alloc<named_label_entry> ();
+      ent->name = id;
+    }
+  *slot = ent;
 }
 
 /* The following two routines are used to interface to Objective-C++.
@@ -579,7 +581,6 @@ poplevel (int keep, int reverse, int functionbody)
   int leaving_for_scope;
   scope_kind kind;
   unsigned ix;
-  cp_label_binding *label_bind;
 
   bool subtime = timevar_cond_start (TV_NAME_LOOKUP);
  restart:
@@ -613,11 +614,12 @@ poplevel (int keep, int reverse, int functionbody)
      Usually current_binding_level->names is in reverse order.
      But parameter decls were previously put in forward order.  */
 
+  decls = current_binding_level->names;
   if (reverse)
-    current_binding_level->names
-      = decls = nreverse (current_binding_level->names);
-  else
-    decls = current_binding_level->names;
+    {
+      decls = nreverse (decls);
+      current_binding_level->names = decls;
+    }
 
   /* If there were any declarations or structure tags in that level,
      or if this level is a function body,
@@ -770,7 +772,10 @@ poplevel (int keep, int reverse, int functionbody)
 	    }
 	}
       /* Remove the binding.  */
-      pop_local_binding (name, decl);
+      if (TREE_CODE (decl) == LABEL_DECL)
+	pop_local_label (name, decl);
+      else
+	pop_local_binding (name, decl);
     }
 
   /* Remove declarations for any `for' variables from inner scopes
@@ -784,11 +789,6 @@ poplevel (int keep, int reverse, int functionbody)
        link; link = TREE_CHAIN (link))
     SET_IDENTIFIER_TYPE_VALUE (TREE_PURPOSE (link), TREE_VALUE (link));
 
-  /* Restore the IDENTIFIER_LABEL_VALUEs for local labels.  */
-  FOR_EACH_VEC_SAFE_ELT_REVERSE (current_binding_level->shadowed_labels,
-			         ix, label_bind)
-    pop_local_label (label_bind->label, label_bind->prev_value);
-
   /* There may be OVERLOADs (wrapped in TREE_LISTs) on the BLOCK_VARs
      list if a `using' declaration put them there.  The debugging
      back ends won't understand OVERLOAD, so we remove them here.
@@ -1431,7 +1431,15 @@ duplicate_decls (tree newdecl, tree olddecl, bool newdecl_is_friend)
 	  /* Avoid warnings redeclaring built-ins which have not been
 	     explicitly declared.  */
 	  if (DECL_ANTICIPATED (olddecl))
-	    return NULL_TREE;
+	    {
+	      if (TREE_PUBLIC (newdecl)
+		  && CP_DECL_CONTEXT (newdecl) == global_namespace)
+		warning_at (DECL_SOURCE_LOCATION (newdecl),
+			    OPT_Wbuiltin_declaration_mismatch,
+			    "built-in function %qD declared as non-function",
+			    newdecl);
+	      return NULL_TREE;
+	    }
 
 	  /* If you declare a built-in or predefined function name as static,
 	     the old definition is overridden, but optionally warn this was a
@@ -1522,7 +1530,7 @@ next_arg:;
 
 	      warning_at (DECL_SOURCE_LOCATION (newdecl),
 			  OPT_Wbuiltin_declaration_mismatch,
-			  "declaration of %q+#D conflicts with built-in "
+			  "declaration of %q#D conflicts with built-in "
 			  "declaration %q#D", newdecl, olddecl);
 	    }
 	  else if ((DECL_EXTERN_C_P (newdecl)
@@ -1911,9 +1919,9 @@ next_arg:;
       DECL_FINAL_P (newdecl) |= DECL_FINAL_P (olddecl);
       DECL_OVERRIDE_P (newdecl) |= DECL_OVERRIDE_P (olddecl);
       DECL_THIS_STATIC (newdecl) |= DECL_THIS_STATIC (olddecl);
-      if (DECL_OVERLOADED_OPERATOR_P (olddecl) != ERROR_MARK)
-	SET_OVERLOADED_OPERATOR_CODE
-	  (newdecl, DECL_OVERLOADED_OPERATOR_P (olddecl));
+      if (DECL_OVERLOADED_OPERATOR_P (olddecl))
+	DECL_OVERLOADED_OPERATOR_CODE_RAW (newdecl)
+	  = DECL_OVERLOADED_OPERATOR_CODE_RAW (olddecl);
       new_defines_function = DECL_INITIAL (newdecl) != NULL_TREE;
 
       /* Optionally warn about more than one declaration for the same
@@ -2470,6 +2478,8 @@ next_arg:;
 		  break;
 		}
 	    }
+
+	  copy_attributes_to_builtin (newdecl);
 	}
       if (new_defines_function)
 	/* If defining a function declared with other language
@@ -2939,81 +2949,83 @@ redeclaration_error_message (tree newdecl, tree olddecl)
     }
 }
 
+
 /* Hash and equality functions for the named_label table.  */
 
 hashval_t
-named_label_hasher::hash (named_label_entry *ent)
+named_label_hash::hash (const value_type entry)
 {
-  return DECL_UID (ent->label_decl);
+  return IDENTIFIER_HASH_VALUE (entry->name);
 }
 
 bool
-named_label_hasher::equal (named_label_entry *a, named_label_entry *b)
+named_label_hash::equal (const value_type entry, compare_type name)
 {
-  return a->label_decl == b->label_decl;
+  return name == entry->name;
 }
 
-/* Create a new label, named ID.  */
+/* Look for a label named ID in the current function.  If one cannot
+   be found, create one.  Return the named_label_entry, or NULL on
+   failure.  */
 
-static tree
-make_label_decl (tree id, int local_p)
+static named_label_entry *
+lookup_label_1 (tree id, bool making_local_p)
 {
-  struct named_label_entry *ent;
-  tree decl;
-
-  decl = build_decl (input_location, LABEL_DECL, id, void_type_node);
-
-  DECL_CONTEXT (decl) = current_function_decl;
-  SET_DECL_MODE (decl, VOIDmode);
-  C_DECLARED_LABEL_FLAG (decl) = local_p;
-
-  /* Say where one reference is to the label, for the sake of the
-     error if it is not defined.  */
-  DECL_SOURCE_LOCATION (decl) = input_location;
-
-  /* Record the fact that this identifier is bound to this label.  */
-  SET_IDENTIFIER_LABEL_VALUE (id, decl);
+  /* You can't use labels at global scope.  */
+  if (current_function_decl == NULL_TREE)
+    {
+      error ("label %qE referenced outside of any function", id);
+      return NULL;
+    }
 
-  /* Create the label htab for the function on demand.  */
   if (!named_labels)
-    named_labels = hash_table<named_label_hasher>::create_ggc (13);
+    named_labels = hash_table<named_label_hash>::create_ggc (13);
 
-  /* Record this label on the list of labels used in this function.
-     We do this before calling make_label_decl so that we get the
-     IDENTIFIER_LABEL_VALUE before the new label is declared.  */
-  ent = ggc_cleared_alloc<named_label_entry> ();
-  ent->label_decl = decl;
-
-  named_label_entry **slot = named_labels->find_slot (ent, INSERT);
-  gcc_assert (*slot == NULL);
-  *slot = ent;
+  hashval_t hash = IDENTIFIER_HASH_VALUE (id);
+  named_label_entry **slot
+    = named_labels->find_slot_with_hash (id, hash, INSERT);
+  named_label_entry *old = *slot;
+  
+  if (old && old->label_decl)
+    {
+      if (!making_local_p)
+	return old;
 
-  return decl;
-}
+      if (old->binding_level == current_binding_level)
+	{
+	  error ("local label %qE conflicts with existing label", id);
+	  inform (DECL_SOURCE_LOCATION (old->label_decl), "previous label");
+	  return NULL;
+	}
+    }
 
-/* Look for a label named ID in the current function.  If one cannot
-   be found, create one.  (We keep track of used, but undefined,
-   labels, and complain about them at the end of a function.)  */
+  /* We are making a new decl, create or reuse the named_label_entry  */
+  named_label_entry *ent = NULL;
+  if (old && !old->label_decl)
+    ent = old;
+  else
+    {
+      ent = ggc_cleared_alloc<named_label_entry> ();
+      ent->name = id;
+      ent->outer = old;
+      *slot = ent;
+    }
 
-static tree
-lookup_label_1 (tree id)
-{
-  tree decl;
+  /* Now create the LABEL_DECL.  */
+  tree decl = build_decl (input_location, LABEL_DECL, id, void_type_node);
 
-  /* You can't use labels at global scope.  */
-  if (current_function_decl == NULL_TREE)
+  DECL_CONTEXT (decl) = current_function_decl;
+  SET_DECL_MODE (decl, VOIDmode);
+  if (making_local_p)
     {
-      error ("label %qE referenced outside of any function", id);
-      return NULL_TREE;
+      C_DECLARED_LABEL_FLAG (decl) = true;
+      DECL_CHAIN (decl) = current_binding_level->names;
+      current_binding_level->names = decl;
     }
 
-  /* See if we've already got this label.  */
-  decl = IDENTIFIER_LABEL_VALUE (id);
-  if (decl != NULL_TREE && DECL_CONTEXT (decl) == current_function_decl)
-    return decl;
+  ent->label_decl = decl;
 
-  decl = make_label_decl (id, /*local_p=*/0);
-  return decl;
+  return ent;
 }
 
 /* Wrapper for lookup_label_1.  */
@@ -3021,30 +3033,19 @@ lookup_label_1 (tree id)
 tree
 lookup_label (tree id)
 {
-  tree ret;
   bool subtime = timevar_cond_start (TV_NAME_LOOKUP);
-  ret = lookup_label_1 (id);
+  named_label_entry *ent = lookup_label_1 (id, false);
   timevar_cond_stop (TV_NAME_LOOKUP, subtime);
-  return ret;
+  return ent ? ent->label_decl : NULL_TREE;
 }
 
-/* Declare a local label named ID.  */
-
 tree
 declare_local_label (tree id)
 {
-  tree decl;
-  cp_label_binding bind;
-
-  /* Add a new entry to the SHADOWED_LABELS list so that when we leave
-     this scope we can restore the old value of IDENTIFIER_TYPE_VALUE.  */
-  bind.prev_value = IDENTIFIER_LABEL_VALUE (id);
-
-  decl = make_label_decl (id, /*local_p=*/1);
-  bind.label = decl;
-  vec_safe_push (current_binding_level->shadowed_labels, bind);
-
-  return decl;
+  bool subtime = timevar_cond_start (TV_NAME_LOOKUP);
+  named_label_entry *ent = lookup_label_1 (id, true);
+  timevar_cond_stop (TV_NAME_LOOKUP, subtime);
+  return ent ? ent->label_decl : NULL_TREE;
 }
 
 /* Returns nonzero if it is ill-formed to jump past the declaration of
@@ -3083,8 +3084,9 @@ identify_goto (tree decl, location_t loc, const location_t *locus,
 	       diagnostic_t diag_kind)
 {
   bool complained
-    = (decl ? emit_diagnostic (diag_kind, loc, 0, "jump to label %qD", decl)
-	    : emit_diagnostic (diag_kind, loc, 0, "jump to case label"));
+    = emit_diagnostic (diag_kind, loc, 0,
+		       decl ? N_("jump to label %qD")
+		       : N_("jump to case label"), decl);
   if (complained && locus)
     inform (*locus, "  from here");
   return complained;
@@ -3139,68 +3141,62 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
 			"  crosses initialization of %q#D", new_decls);
 	      else
 		inform (DECL_SOURCE_LOCATION (new_decls),
-			"  enters scope of %q#D which has "
+			"  enters scope of %q#D, which has "
 			"non-trivial destructor", new_decls);
 	    }
 	}
 
       if (b == level)
 	break;
-      if ((b->kind == sk_try || b->kind == sk_catch) && !saw_eh)
+
+      const char *inf = NULL;
+      location_t loc = input_location;
+      switch (b->kind)
 	{
-	  if (identified < 2)
-	    {
-	      complained = identify_goto (decl, input_location, locus,
-					  DK_ERROR);
-	      identified = 2;
-	    }
-	  if (complained)
-	    {
-	      if (b->kind == sk_try)
-		inform (input_location, "  enters try block");
-	      else
-		inform (input_location, "  enters catch block");
-	    }
+	case sk_try:
+	  if (!saw_eh)
+	    inf = N_("enters try block");
 	  saw_eh = true;
-	}
-      if (b->kind == sk_omp && !saw_omp)
-	{
-	  if (identified < 2)
-	    {
-	      complained = identify_goto (decl, input_location, locus,
-					  DK_ERROR);
-	      identified = 2;
-	    }
-	  if (complained)
-	    inform (input_location, "  enters OpenMP structured block");
+	  break;
+
+	case sk_catch:
+	  if (!saw_eh)
+	    inf = N_("enters catch block");
+	  saw_eh = true;
+	  break;
+
+	case sk_omp:
+	  if (!saw_omp)
+	    inf = N_("enters OpenMP structured block");
 	  saw_omp = true;
-	}
-      if (b->kind == sk_transaction && !saw_tm)
-	{
-	  if (identified < 2)
+	  break;
+
+	case sk_transaction:
+	  if (!saw_tm)
+	    inf = N_("enters synchronized or atomic statement");
+	  saw_tm = true;
+	  break;
+
+	case sk_block:
+	  if (!saw_cxif && level_for_constexpr_if (b->level_chain))
 	    {
-	      complained = identify_goto (decl, input_location, locus,
-					  DK_ERROR);
-	      identified = 2;
+	      inf = N_("enters constexpr if statement");
+	      loc = EXPR_LOCATION (b->level_chain->this_entity);
+	      saw_cxif = true;
 	    }
-	  if (complained)
-	    inform (input_location,
-		    "  enters synchronized or atomic statement");
-	  saw_tm = true;
+	  break;
+
+	default:
+	  break;
 	}
-      if (!saw_cxif && b->kind == sk_block
-	  && level_for_constexpr_if (b->level_chain))
+
+      if (inf)
 	{
 	  if (identified < 2)
-	    {
-	      complained = identify_goto (decl, input_location, locus,
-					  DK_ERROR);
-	      identified = 2;
-	    }
+	    complained = identify_goto (decl, input_location, locus, DK_ERROR);
+	  identified = 2;
 	  if (complained)
-	    inform (EXPR_LOCATION (b->level_chain->this_entity),
-		    "  enters constexpr if statement");
-	  saw_cxif = true;
+	    inform (loc, "  %s", inf);
 	}
     }
 
@@ -3227,12 +3223,6 @@ check_switch_goto (cp_binding_level* level)
 void
 check_goto (tree decl)
 {
-  struct named_label_entry *ent, dummy;
-  bool saw_catch = false, complained = false;
-  int identified = 0;
-  tree bad;
-  unsigned ix;
-
   /* We can't know where a computed goto is jumping.
      So we assume that it's OK.  */
   if (TREE_CODE (decl) != LABEL_DECL)
@@ -3243,22 +3233,22 @@ check_goto (tree decl)
   if (decl == cdtor_label)
     return;
 
-  dummy.label_decl = decl;
-  ent = named_labels->find (&dummy);
-  gcc_assert (ent != NULL);
+  hashval_t hash = IDENTIFIER_HASH_VALUE (DECL_NAME (decl));
+  named_label_entry **slot
+    = named_labels->find_slot_with_hash (DECL_NAME (decl), hash, NO_INSERT);
+  named_label_entry *ent = *slot;
 
   /* If the label hasn't been defined yet, defer checking.  */
   if (! DECL_INITIAL (decl))
     {
-      struct named_label_use_entry *new_use;
-
       /* Don't bother creating another use if the last goto had the
 	 same data, and will therefore create the same set of errors.  */
       if (ent->uses
 	  && ent->uses->names_in_scope == current_binding_level->names)
 	return;
 
-      new_use = ggc_alloc<named_label_use_entry> ();
+      named_label_use_entry *new_use
+	= ggc_alloc<named_label_use_entry> ();
       new_use->binding_level = current_binding_level;
       new_use->names_in_scope = current_binding_level->names;
       new_use->o_goto_locus = input_location;
@@ -3269,6 +3259,11 @@ check_goto (tree decl)
       return;
     }
 
+  bool saw_catch = false, complained = false;
+  int identified = 0;
+  tree bad;
+  unsigned ix;
+
   if (ent->in_try_scope || ent->in_catch_scope || ent->in_transaction_scope
       || ent->in_constexpr_if
       || ent->in_omp_scope || !vec_safe_is_empty (ent->bad_decls))
@@ -3329,27 +3324,24 @@ check_goto (tree decl)
 	inform (input_location, "  enters OpenMP structured block");
     }
   else if (flag_openmp)
-    {
-      cp_binding_level *b;
-      for (b = current_binding_level; b ; b = b->level_chain)
-	{
-	  if (b == ent->binding_level)
+    for (cp_binding_level *b = current_binding_level; b ; b = b->level_chain)
+      {
+	if (b == ent->binding_level)
+	  break;
+	if (b->kind == sk_omp)
+	  {
+	    if (identified < 2)
+	      {
+		complained = identify_goto (decl,
+					    DECL_SOURCE_LOCATION (decl),
+					    &input_location, DK_ERROR);
+		identified = 2;
+	      }
+	    if (complained)
+	      inform (input_location, "  exits OpenMP structured block");
 	    break;
-	  if (b->kind == sk_omp)
-	    {
-	      if (identified < 2)
-		{
-		  complained = identify_goto (decl,
-					      DECL_SOURCE_LOCATION (decl),
-					      &input_location, DK_ERROR);
-		  identified = 2;
-		}
-	      if (complained)
-		inform (input_location, "  exits OpenMP structured block");
-	      break;
-	    }
-	}
-    }
+	  }
+      }
 }
 
 /* Check that a return is ok wrt OpenMP structured blocks.
@@ -3358,8 +3350,7 @@ check_goto (tree decl)
 bool
 check_omp_return (void)
 {
-  cp_binding_level *b;
-  for (b = current_binding_level; b ; b = b->level_chain)
+  for (cp_binding_level *b = current_binding_level; b ; b = b->level_chain)
     if (b->kind == sk_omp)
       {
 	error ("invalid exit from OpenMP structured block");
@@ -3376,25 +3367,15 @@ check_omp_return (void)
 static tree
 define_label_1 (location_t location, tree name)
 {
-  struct named_label_entry *ent, dummy;
-  cp_binding_level *p;
-  tree decl;
-
-  decl = lookup_label (name);
-
-  dummy.label_decl = decl;
-  ent = named_labels->find (&dummy);
-  gcc_assert (ent != NULL);
-
   /* After labels, make any new cleanups in the function go into their
      own new (temporary) binding contour.  */
-  for (p = current_binding_level;
+  for (cp_binding_level *p = current_binding_level;
        p->kind != sk_function_parms;
        p = p->level_chain)
     p->more_cleanups_ok = 0;
 
-  if (name == get_identifier ("wchar_t"))
-    permerror (input_location, "label named wchar_t");
+  named_label_entry *ent = lookup_label_1 (name, false);
+  tree decl = ent->label_decl;
 
   if (DECL_INITIAL (decl) != NULL_TREE)
     {
@@ -3403,8 +3384,6 @@ define_label_1 (location_t location, tree name)
     }
   else
     {
-      struct named_label_use_entry *use;
-
       /* Mark label as having been defined.  */
       DECL_INITIAL (decl) = error_mark_node;
       /* Say where in the source.  */
@@ -3413,7 +3392,7 @@ define_label_1 (location_t location, tree name)
       ent->binding_level = current_binding_level;
       ent->names_in_scope = current_binding_level->names;
 
-      for (use = ent->uses; use ; use = use->next)
+      for (named_label_use_entry *use = ent->uses; use; use = use->next)
 	check_previous_goto (decl, use);
       ent->uses = NULL;
     }
@@ -3426,9 +3405,8 @@ define_label_1 (location_t location, tree name)
 tree
 define_label (location_t location, tree name)
 {
-  tree ret;
   bool running = timevar_cond_start (TV_NAME_LOOKUP);
-  ret = define_label_1 (location, name);
+  tree ret = define_label_1 (location, name);
   timevar_cond_stop (TV_NAME_LOOKUP, running);
   return ret;
 }
@@ -4379,7 +4357,6 @@ builtin_function_1 (tree decl, tree context, bool is_global)
   retrofit_lang_decl (decl);
 
   DECL_ARTIFICIAL (decl) = 1;
-  SET_OVERLOADED_OPERATOR_CODE (decl, ERROR_MARK);
   SET_DECL_LANGUAGE (decl, lang_c);
   /* Runtime library routines are, by definition, available in an
      external shared object.  */
@@ -4467,7 +4444,8 @@ build_library_fn (tree name, enum tree_code operator_code, tree type,
   DECL_EXTERNAL (fn) = 1;
   TREE_PUBLIC (fn) = 1;
   DECL_ARTIFICIAL (fn) = 1;
-  SET_OVERLOADED_OPERATOR_CODE (fn, operator_code);
+  DECL_OVERLOADED_OPERATOR_CODE_RAW (fn)
+    = OVL_OP_INFO (false, operator_code)->ovl_op_code;
   SET_DECL_LANGUAGE (fn, lang_c);
   /* Runtime library routines are, by definition, available in an
      external shared object.  */
@@ -4532,9 +4510,8 @@ static tree
 push_cp_library_fn (enum tree_code operator_code, tree type,
 		    int ecf_flags)
 {
-  tree fn = build_cp_library_fn (cp_operator_id (operator_code),
-				 operator_code,
-				 type, ecf_flags);
+  tree fn = build_cp_library_fn (ovl_op_identifier (false, operator_code),
+				 operator_code, type, ecf_flags);
   pushdecl (fn);
   if (flag_tm)
     apply_tm_attr (fn, get_identifier ("transaction_safe"));
@@ -8722,7 +8699,7 @@ grokfndecl (tree ctype,
 		  "deduction guide %qD must not have a function body", decl);
     }
   else if (IDENTIFIER_ANY_OP_P (DECL_NAME (decl))
-      && !grok_op_properties (decl, /*complain=*/true))
+	   && !grok_op_properties (decl, /*complain=*/true))
     return NULL_TREE;
   else if (UDLIT_OPER_P (DECL_NAME (decl)))
     {
@@ -8733,6 +8710,7 @@ grokfndecl (tree ctype,
       if (DECL_LANGUAGE (decl) == lang_c)
 	{
 	  error ("literal operator with C linkage");
+	  maybe_show_extern_c_location ();
 	  return NULL_TREE;
 	}
 
@@ -9495,22 +9473,20 @@ compute_array_index_type (tree name, tree size, tsubst_flags_t complain)
     itype = build_min (MINUS_EXPR, sizetype, size, integer_one_node);
   else
     {
-      HOST_WIDE_INT saved_processing_template_decl;
-
       /* Compute the index of the largest element in the array.  It is
 	 one less than the number of elements in the array.  We save
 	 and restore PROCESSING_TEMPLATE_DECL so that computations in
 	 cp_build_binary_op will be appropriately folded.  */
-      saved_processing_template_decl = processing_template_decl;
-      processing_template_decl = 0;
-      itype = cp_build_binary_op (input_location,
-				  MINUS_EXPR,
-				  cp_convert (ssizetype, size, complain),
-				  cp_convert (ssizetype, integer_one_node,
-					      complain),
-				  complain);
-      itype = maybe_constant_value (itype);
-      processing_template_decl = saved_processing_template_decl;
+      {
+	processing_template_decl_sentinel s;
+	itype = cp_build_binary_op (input_location,
+				    MINUS_EXPR,
+				    cp_convert (ssizetype, size, complain),
+				    cp_convert (ssizetype, integer_one_node,
+						complain),
+				    complain);
+	itype = maybe_constant_value (itype);
+      }
 
       if (!TREE_CONSTANT (itype))
 	{
@@ -10816,18 +10792,27 @@ grokdeclarator (const cp_declarator *declarator,
 					    attr_flags);
 	}
 
+      inner_declarator = declarator->declarator;
+
       /* We don't want to warn in parmeter context because we don't
 	 yet know if the parse will succeed, and this might turn out
 	 to be a constructor call.  */
       if (decl_context != PARM
-	  && declarator->parenthesized != UNKNOWN_LOCATION)
+	  && declarator->parenthesized != UNKNOWN_LOCATION
+	  /* If the type is class-like and the inner name used a
+	     global namespace qualifier, we need the parens.
+	     Unfortunately all we can tell is whether a qualified name
+	     was used or not.  */
+	  && !(inner_declarator
+	       && inner_declarator->kind == cdk_id
+	       && inner_declarator->u.id.qualifying_scope
+	       && (MAYBE_CLASS_TYPE_P (type)
+		   || TREE_CODE (type) == ENUMERAL_TYPE)))
 	warning_at (declarator->parenthesized, OPT_Wparentheses,
 		    "unnecessary parentheses in declaration of %qs", name);
       if (declarator->kind == cdk_id || declarator->kind == cdk_decomp)
 	break;
 
-      inner_declarator = declarator->declarator;
-
       switch (declarator->kind)
 	{
 	case cdk_array:
@@ -12841,7 +12826,7 @@ grok_special_member_properties (tree decl)
 	  && !ctor && !move_fn_p (decl))
 	TYPE_HAS_CONSTEXPR_CTOR (class_type) = 1;
     }
-  else if (DECL_NAME (decl) == cp_assignment_operator_id (NOP_EXPR))
+  else if (DECL_NAME (decl) == assign_op_identifier)
     {
       /* [class.copy]
 
@@ -12901,114 +12886,72 @@ grok_ctor_properties (const_tree ctype, const_tree decl)
   return true;
 }
 
-/* An operator with this code is unary, but can also be binary.  */
-
-static bool
-ambi_op_p (enum tree_code code)
-{
-  return (code == INDIRECT_REF
-	  || code == ADDR_EXPR
-	  || code == UNARY_PLUS_EXPR
-	  || code == NEGATE_EXPR
-	  || code == PREINCREMENT_EXPR
-	  || code == PREDECREMENT_EXPR);
-}
-
-/* An operator with this name can only be unary.  */
-
-static bool
-unary_op_p (enum tree_code code)
-{
-  return (code == TRUTH_NOT_EXPR
-	  || code == BIT_NOT_EXPR
-	  || code == COMPONENT_REF
-	  || code == TYPE_EXPR);
-}
-
-/* DECL is a declaration for an overloaded operator.  If COMPLAIN is true,
-   errors are issued for invalid declarations.  */
+/* DECL is a declaration for an overloaded or conversion operator.  If
+   COMPLAIN is true, errors are issued for invalid declarations.  */
 
 bool
 grok_op_properties (tree decl, bool complain)
 {
   tree argtypes = TYPE_ARG_TYPES (TREE_TYPE (decl));
-  tree argtype;
-  int methodp = (TREE_CODE (TREE_TYPE (decl)) == METHOD_TYPE);
+  bool methodp = TREE_CODE (TREE_TYPE (decl)) == METHOD_TYPE;
   tree name = DECL_NAME (decl);
-  enum tree_code operator_code;
-  int arity;
-  bool ellipsis_p;
-  tree class_type;
 
-  /* Count the number of arguments and check for ellipsis.  */
-  for (argtype = argtypes, arity = 0;
-       argtype && argtype != void_list_node;
-       argtype = TREE_CHAIN (argtype))
-    ++arity;
-  ellipsis_p = !argtype;
-
-  class_type = DECL_CONTEXT (decl);
+  tree class_type = DECL_CONTEXT (decl);
   if (class_type && !CLASS_TYPE_P (class_type))
     class_type = NULL_TREE;
 
+  tree_code operator_code;
+  unsigned op_flags;
   if (IDENTIFIER_CONV_OP_P (name))
-    operator_code = TYPE_EXPR;
+    {
+      /* Conversion operators are TYPE_EXPR for the purposes of this
+	 function.  */
+      operator_code = TYPE_EXPR;
+      op_flags = OVL_OP_FLAG_UNARY;
+    }
   else
     {
-      /* It'd be nice to hang something else of the identifier to
-	 find CODE more directly.  */
-      bool assign_op = IDENTIFIER_ASSIGN_OP_P (name);
-      const operator_name_info_t *oni
-	= (assign_op ? assignment_operator_name_info : operator_name_info);
+      const ovl_op_info_t *ovl_op = IDENTIFIER_OVL_OP_INFO (name);
 
-      if (false)
-	;
-#define DEF_OPERATOR(NAME, CODE, MANGLING, ARITY, KIND)		\
-      else if (assign_op == (KIND == cik_assign_op)		\
-	       && oni[int (CODE)].identifier == name)		\
-	operator_code = (CODE);
-#include "operators.def"
-#undef DEF_OPERATOR
-      else
-	gcc_unreachable ();
-      }
-    while (0);
-  gcc_assert (operator_code != MAX_TREE_CODES);
-  SET_OVERLOADED_OPERATOR_CODE (decl, operator_code);
+      operator_code = ovl_op->tree_code;
+      op_flags = ovl_op->flags;
+      gcc_checking_assert (operator_code != ERROR_MARK);
+      DECL_OVERLOADED_OPERATOR_CODE_RAW (decl) = ovl_op->ovl_op_code;
+    }
 
-  if (class_type)
-    switch (operator_code)
-      {
-      case NEW_EXPR:
-	TYPE_HAS_NEW_OPERATOR (class_type) = 1;
-	break;
+  if (op_flags & OVL_OP_FLAG_ALLOC)
+    {
+      /* operator new and operator delete are quite special.  */
+      if (class_type)
+	switch (op_flags)
+	  {
+	  case OVL_OP_FLAG_ALLOC:
+	    TYPE_HAS_NEW_OPERATOR (class_type) = 1;
+	    break;
 
-      case DELETE_EXPR:
-	TYPE_GETS_DELETE (class_type) |= 1;
-	break;
+	  case OVL_OP_FLAG_ALLOC | OVL_OP_FLAG_DELETE:
+	    TYPE_GETS_DELETE (class_type) |= 1;
+	    break;
 
-      case VEC_NEW_EXPR:
-	TYPE_HAS_ARRAY_NEW_OPERATOR (class_type) = 1;
-	break;
+	  case OVL_OP_FLAG_ALLOC | OVL_OP_FLAG_VEC:
+	    TYPE_HAS_ARRAY_NEW_OPERATOR (class_type) = 1;
+	    break;
 
-      case VEC_DELETE_EXPR:
-	TYPE_GETS_DELETE (class_type) |= 2;
-	break;
+	  case OVL_OP_FLAG_ALLOC | OVL_OP_FLAG_DELETE | OVL_OP_FLAG_VEC:
+	    TYPE_GETS_DELETE (class_type) |= 2;
+	    break;
 
-      default:
-	break;
-      }
+	  default:
+	    gcc_unreachable ();
+	  }
 
-    /* [basic.std.dynamic.allocation]/1:
+      /* [basic.std.dynamic.allocation]/1:
 
-       A program is ill-formed if an allocation function is declared
-       in a namespace scope other than global scope or declared static
-       in global scope.
+	 A program is ill-formed if an allocation function is declared
+	 in a namespace scope other than global scope or declared
+	 static in global scope.
 
-       The same also holds true for deallocation functions.  */
-  if (operator_code == NEW_EXPR || operator_code == VEC_NEW_EXPR
-      || operator_code == DELETE_EXPR || operator_code == VEC_DELETE_EXPR)
-    {
+	 The same also holds true for deallocation functions.  */
       if (DECL_NAMESPACE_SCOPE_P (decl))
 	{
 	  if (CP_DECL_CONTEXT (decl) != global_namespace)
@@ -13016,287 +12959,269 @@ grok_op_properties (tree decl, bool complain)
 	      error ("%qD may not be declared within a namespace", decl);
 	      return false;
 	    }
-	  else if (!TREE_PUBLIC (decl))
+
+	  if (!TREE_PUBLIC (decl))
 	    {
 	      error ("%qD may not be declared as static", decl);
 	      return false;
 	    }
 	}
-    }
 
-  if (operator_code == NEW_EXPR || operator_code == VEC_NEW_EXPR)
-    {
-      TREE_TYPE (decl) = coerce_new_type (TREE_TYPE (decl));
-      DECL_IS_OPERATOR_NEW (decl) = 1;
+      if (op_flags & OVL_OP_FLAG_DELETE)
+	TREE_TYPE (decl) = coerce_delete_type (TREE_TYPE (decl));
+      else
+	{
+	  DECL_IS_OPERATOR_NEW (decl) = 1;
+	  TREE_TYPE (decl) = coerce_new_type (TREE_TYPE (decl));
+	}
+
+      return true;
     }
-  else if (operator_code == DELETE_EXPR || operator_code == VEC_DELETE_EXPR)
-    TREE_TYPE (decl) = coerce_delete_type (TREE_TYPE (decl));
-  else
+
+  /* An operator function must either be a non-static member function
+     or have at least one parameter of a class, a reference to a class,
+     an enumeration, or a reference to an enumeration.  13.4.0.6 */
+  if (! methodp || DECL_STATIC_FUNCTION_P (decl))
     {
-      /* An operator function must either be a non-static member function
-	 or have at least one parameter of a class, a reference to a class,
-	 an enumeration, or a reference to an enumeration.  13.4.0.6 */
-      if (! methodp || DECL_STATIC_FUNCTION_P (decl))
+      if (operator_code == TYPE_EXPR
+	  || operator_code == CALL_EXPR
+	  || operator_code == COMPONENT_REF
+	  || operator_code == ARRAY_REF
+	  || operator_code == NOP_EXPR)
+	{
+	  error ("%qD must be a nonstatic member function", decl);
+	  return false;
+	}
+
+      if (DECL_STATIC_FUNCTION_P (decl))
 	{
-	  if (operator_code == TYPE_EXPR
-	      || operator_code == CALL_EXPR
-	      || operator_code == COMPONENT_REF
-	      || operator_code == ARRAY_REF
-	      || operator_code == NOP_EXPR)
+	  error ("%qD must be either a non-static member "
+		 "function or a non-member function", decl);
+	  return false;
+	}
+
+      for (tree arg = argtypes; ; arg = TREE_CHAIN (arg))
+	{
+	  if (!arg || arg == void_list_node)
 	    {
-	      error ("%qD must be a nonstatic member function", decl);
+	      if (complain)
+		error ("%qD must have an argument of class or "
+		       "enumerated type", decl);
 	      return false;
 	    }
-	  else
-	    {
-	      tree p;
+      
+	  tree type = non_reference (TREE_VALUE (arg));
+	  if (type == error_mark_node)
+	    return false;
+	  
+	  /* MAYBE_CLASS_TYPE_P, rather than CLASS_TYPE_P, is used
+	     because these checks are performed even on template
+	     functions.  */
+	  if (MAYBE_CLASS_TYPE_P (type)
+	      || TREE_CODE (type) == ENUMERAL_TYPE)
+	    break;
+	}
+    }
 
-	      if (DECL_STATIC_FUNCTION_P (decl))
-		{
-		  error ("%qD must be either a non-static member "
-			 "function or a non-member function", decl);
-		  return false;
-		}
+  if (operator_code == CALL_EXPR)
+    /* There are no further restrictions on the arguments to an overloaded
+       "operator ()".  */
+    return true;
 
-	      for (p = argtypes; p && p != void_list_node; p = TREE_CHAIN (p))
-		{
-		  tree arg = non_reference (TREE_VALUE (p));
-		  if (arg == error_mark_node)
-		    return false;
-
-		  /* MAYBE_CLASS_TYPE_P, rather than CLASS_TYPE_P, is used
-		     because these checks are performed even on
-		     template functions.  */
-		  if (MAYBE_CLASS_TYPE_P (arg)
-		      || TREE_CODE (arg) == ENUMERAL_TYPE)
-		    break;
-		}
+  if (operator_code == COND_EXPR)
+    {
+      /* 13.4.0.3 */
+      error ("ISO C++ prohibits overloading operator ?:");
+      return false;
+    }
 
-	      if (!p || p == void_list_node)
-		{
-		  if (complain)
-		    error ("%qD must have an argument of class or "
-			   "enumerated type", decl);
-		  return false;
-		}
-	    }
+  /* Count the number of arguments and check for ellipsis.  */
+  int arity = 0;
+  for (tree arg = argtypes; arg != void_list_node; arg = TREE_CHAIN (arg))
+    {
+      if (!arg)
+	{
+	  /* Variadic.  */
+	  error ("%qD must not have variable number of arguments", decl);
+	  return false;
 	}
+      ++arity;
+    }
 
-      /* There are no restrictions on the arguments to an overloaded
-	 "operator ()".  */
-      if (operator_code == CALL_EXPR)
-	return true;
-
-      /* Warn about conversion operators that will never be used.  */
-      if (IDENTIFIER_CONV_OP_P (name)
-	  && ! DECL_TEMPLATE_INFO (decl)
-	  && warn_conversion
-	  /* Warn only declaring the function; there is no need to
-	     warn again about out-of-class definitions.  */
-	  && class_type == current_class_type)
+  /* Verify correct number of arguments.  */
+  switch (op_flags)
+    {
+    case OVL_OP_FLAG_AMBIARY:
+      if (arity == 1)
 	{
-	  tree t = TREE_TYPE (name);
-	  int ref = (TREE_CODE (t) == REFERENCE_TYPE);
-
-	  if (ref)
-	    t = TYPE_MAIN_VARIANT (TREE_TYPE (t));
-
-	  if (VOID_TYPE_P (t))
-            warning (OPT_Wconversion,
-                     ref
-                     ? G_("conversion to a reference to void "
-                          "will never use a type conversion operator")
-                     : G_("conversion to void "
-                          "will never use a type conversion operator"));
-	  else if (class_type)
-	    {
-	      if (t == class_type)
-                warning (OPT_Wconversion,
-                     ref
-                     ? G_("conversion to a reference to the same type "
-                          "will never use a type conversion operator")
-                     : G_("conversion to the same type "
-                          "will never use a type conversion operator"));		
-	      /* Don't force t to be complete here.  */
-	      else if (MAYBE_CLASS_TYPE_P (t)
-		       && COMPLETE_TYPE_P (t)
-		       && DERIVED_FROM_P (t, class_type))
-                 warning (OPT_Wconversion,
-                          ref
-                          ? G_("conversion to a reference to a base class "
-                               "will never use a type conversion operator")
-                          : G_("conversion to a base class "
-                               "will never use a type conversion operator"));		
-	    }
-
+	  /* We have a unary instance of an ambi-ary op.  Remap to the
+	     unary one.  */
+	  unsigned alt = ovl_op_alternate[ovl_op_mapping [operator_code]];
+	  const ovl_op_info_t *ovl_op = &ovl_op_info[false][alt];
+	  gcc_checking_assert (ovl_op->flags == OVL_OP_FLAG_UNARY);
+	  operator_code = ovl_op->tree_code;
+	  DECL_OVERLOADED_OPERATOR_CODE_RAW (decl) = ovl_op->ovl_op_code;
 	}
-
-      if (operator_code == COND_EXPR)
+      else if (arity != 2)
 	{
-	  /* 13.4.0.3 */
-	  error ("ISO C++ prohibits overloading operator ?:");
+	  /* This was an ambiguous operator but is invalid. */
+	  error (methodp
+		 ? G_("%qD must have either zero or one argument")
+		 : G_("%qD must have either one or two arguments"), decl);
 	  return false;
 	}
-      else if (ellipsis_p)
+      else if ((operator_code == POSTINCREMENT_EXPR
+		|| operator_code == POSTDECREMENT_EXPR)
+	       && ! processing_template_decl
+	       /* x++ and x--'s second argument must be an int.  */
+	       && ! same_type_p (TREE_VALUE (TREE_CHAIN (argtypes)),
+				 integer_type_node))
+	{
+	  error (methodp
+		 ? G_("postfix %qD must have %<int%> as its argument")
+		 : G_("postfix %qD must have %<int%> as its second argument"),
+		 decl);
+	  return false;
+	}
+      break;
+
+    case OVL_OP_FLAG_UNARY:
+      if (arity != 1)
 	{
-	  error ("%qD must not have variable number of arguments", decl);
+	  error (methodp
+		 ? G_("%qD must have no arguments")
+		 : G_("%qD must have exactly one argument"), decl);
 	  return false;
 	}
-      else if (ambi_op_p (operator_code))
+      break;
+
+    case OVL_OP_FLAG_BINARY:
+      if (arity != 2)
 	{
-	  if (arity == 1)
-	    /* We pick the one-argument operator codes by default, so
-	       we don't have to change anything.  */
-	    ;
-	  else if (arity == 2)
-	    {
-	      /* If we thought this was a unary operator, we now know
-		 it to be a binary operator.  */
-	      switch (operator_code)
-		{
-		case INDIRECT_REF:
-		  operator_code = MULT_EXPR;
-		  break;
+	  error (methodp
+		 ? G_("%qD must have exactly one argument")
+		 : G_("%qD must have exactly two arguments"), decl);
+	  return false;
+	}
+      break;
 
-		case ADDR_EXPR:
-		  operator_code = BIT_AND_EXPR;
-		  break;
+    default:
+      gcc_unreachable ();
+    }
 
-		case UNARY_PLUS_EXPR:
-		  operator_code = PLUS_EXPR;
-		  break;
+  /* There can be no default arguments.  */
+  for (tree arg = argtypes; arg != void_list_node; arg = TREE_CHAIN (arg))
+    if (TREE_PURPOSE (arg))
+      {
+	TREE_PURPOSE (arg) = NULL_TREE;
+	if (operator_code == POSTINCREMENT_EXPR
+	    || operator_code == POSTDECREMENT_EXPR)
+	  pedwarn (input_location, OPT_Wpedantic,
+		   "%qD cannot have default arguments", decl);
+	else
+	  {
+	    error ("%qD cannot have default arguments", decl);
+	    return false;
+	  }
+      }
 
-		case NEGATE_EXPR:
-		  operator_code = MINUS_EXPR;
-		  break;
+  /* At this point the declaration is well-formed.  It may not be
+     sensible though.  */
 
-		case PREINCREMENT_EXPR:
-		  operator_code = POSTINCREMENT_EXPR;
-		  break;
+  /* Check member function warnings only on the in-class declaration.
+     There's no point warning on an out-of-class definition.  */
+  if (class_type && class_type != current_class_type)
+    return true;
 
-		case PREDECREMENT_EXPR:
-		  operator_code = POSTDECREMENT_EXPR;
-		  break;
+  /* Warn about conversion operators that will never be used.  */
+  if (IDENTIFIER_CONV_OP_P (name)
+      && ! DECL_TEMPLATE_INFO (decl)
+      && warn_conversion)
+    {
+      tree t = TREE_TYPE (name);
+      int ref = (TREE_CODE (t) == REFERENCE_TYPE);
 
-		default:
-		  gcc_unreachable ();
-		}
+      if (ref)
+	t = TYPE_MAIN_VARIANT (TREE_TYPE (t));
 
-	      SET_OVERLOADED_OPERATOR_CODE (decl, operator_code);
+      if (VOID_TYPE_P (t))
+	warning (OPT_Wconversion,
+		 ref
+		 ? G_("conversion to a reference to void "
+		      "will never use a type conversion operator")
+		 : G_("conversion to void "
+		      "will never use a type conversion operator"));
+      else if (class_type)
+	{
+	  if (t == class_type)
+	    warning (OPT_Wconversion,
+                     ref
+                     ? G_("conversion to a reference to the same type "
+                          "will never use a type conversion operator")
+                     : G_("conversion to the same type "
+                          "will never use a type conversion operator"));
+	  /* Don't force t to be complete here.  */
+	  else if (MAYBE_CLASS_TYPE_P (t)
+		   && COMPLETE_TYPE_P (t)
+		   && DERIVED_FROM_P (t, class_type))
+	    warning (OPT_Wconversion,
+		     ref
+		     ? G_("conversion to a reference to a base class "
+			  "will never use a type conversion operator")
+		     : G_("conversion to a base class "
+			  "will never use a type conversion operator"));
+	}
+    }
 
-	      if ((operator_code == POSTINCREMENT_EXPR
-		   || operator_code == POSTDECREMENT_EXPR)
-		  && ! processing_template_decl
-		  && ! same_type_p (TREE_VALUE (TREE_CHAIN (argtypes)), integer_type_node))
-		{
-		  if (methodp)
-		    error ("postfix %qD must take %<int%> as its argument",
-			   decl);
-		  else
-		    error ("postfix %qD must take %<int%> as its second "
-			   "argument", decl);
-		  return false;
-		}
-	    }
-	  else
-	    {
-	      if (methodp)
-		error ("%qD must take either zero or one argument", decl);
-	      else
-		error ("%qD must take either one or two arguments", decl);
-	      return false;
-	    }
+  if (!warn_ecpp)
+    return true;
 
-	  /* More Effective C++ rule 6.  */
-	  if (warn_ecpp
-	      && (operator_code == POSTINCREMENT_EXPR
-		  || operator_code == POSTDECREMENT_EXPR
-		  || operator_code == PREINCREMENT_EXPR
-		  || operator_code == PREDECREMENT_EXPR))
-	    {
-	      tree arg = TREE_VALUE (argtypes);
-	      tree ret = TREE_TYPE (TREE_TYPE (decl));
-	      if (methodp || TREE_CODE (arg) == REFERENCE_TYPE)
-		arg = TREE_TYPE (arg);
-	      arg = TYPE_MAIN_VARIANT (arg);
-	      if (operator_code == PREINCREMENT_EXPR
-		  || operator_code == PREDECREMENT_EXPR)
-		{
-		  if (TREE_CODE (ret) != REFERENCE_TYPE
-		      || !same_type_p (TYPE_MAIN_VARIANT (TREE_TYPE (ret)),
-				       arg))
-		    warning (OPT_Weffc__, "prefix %qD should return %qT", decl,
-			     build_reference_type (arg));
-		}
-	      else
-		{
-		  if (!same_type_p (TYPE_MAIN_VARIANT (ret), arg))
-		    warning (OPT_Weffc__, "postfix %qD should return %qT", decl, arg);
-		}
-	    }
+  /* Effective C++ rules below.  */
+
+  /* More Effective C++ rule 7.  */
+  if (operator_code == TRUTH_ANDIF_EXPR
+      || operator_code == TRUTH_ORIF_EXPR
+      || operator_code == COMPOUND_EXPR)
+    warning (OPT_Weffc__,
+	     "user-defined %qD always evaluates both arguments", decl);
+  
+  /* More Effective C++ rule 6.  */
+  if (operator_code == POSTINCREMENT_EXPR
+      || operator_code == POSTDECREMENT_EXPR
+      || operator_code == PREINCREMENT_EXPR
+      || operator_code == PREDECREMENT_EXPR)
+    {
+      tree arg = TREE_VALUE (argtypes);
+      tree ret = TREE_TYPE (TREE_TYPE (decl));
+      if (methodp || TREE_CODE (arg) == REFERENCE_TYPE)
+	arg = TREE_TYPE (arg);
+      arg = TYPE_MAIN_VARIANT (arg);
+
+      if (operator_code == PREINCREMENT_EXPR
+	  || operator_code == PREDECREMENT_EXPR)
+	{
+	  if (TREE_CODE (ret) != REFERENCE_TYPE
+	      || !same_type_p (TYPE_MAIN_VARIANT (TREE_TYPE (ret)), arg))
+	    warning (OPT_Weffc__, "prefix %qD should return %qT", decl,
+		     build_reference_type (arg));
 	}
-      else if (unary_op_p (operator_code))
+      else
 	{
-	  if (arity != 1)
-	    {
-	      if (methodp)
-		error ("%qD must take %<void%>", decl);
-	      else
-		error ("%qD must take exactly one argument", decl);
-	      return false;
-	    }
+	  if (!same_type_p (TYPE_MAIN_VARIANT (ret), arg))
+	    warning (OPT_Weffc__, "postfix %qD should return %qT", decl, arg);
 	}
-      else /* if (binary_op_p (operator_code)) */
-	{
-	  if (arity != 2)
-	    {
-	      if (methodp)
-		error ("%qD must take exactly one argument", decl);
-	      else
-		error ("%qD must take exactly two arguments", decl);
-	      return false;
-	    }
+    }
 
-	  /* More Effective C++ rule 7.  */
-	  if (warn_ecpp
-	      && (operator_code == TRUTH_ANDIF_EXPR
-		  || operator_code == TRUTH_ORIF_EXPR
-		  || operator_code == COMPOUND_EXPR))
-	    warning (OPT_Weffc__, "user-defined %qD always evaluates both arguments",
-		     decl);
-	}
+  /* Effective C++ rule 23.  */
+  if (!DECL_ASSIGNMENT_OPERATOR_P (decl)
+      && (operator_code == PLUS_EXPR
+	  || operator_code == MINUS_EXPR
+	  || operator_code == TRUNC_DIV_EXPR
+	  || operator_code == MULT_EXPR
+	  || operator_code == TRUNC_MOD_EXPR)
+      && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == REFERENCE_TYPE)
+    warning (OPT_Weffc__, "%qD should return by value", decl);
 
-      /* Effective C++ rule 23.  */
-      if (warn_ecpp
-	  && arity == 2
-	  && !DECL_ASSIGNMENT_OPERATOR_P (decl)
-	  && (operator_code == PLUS_EXPR
-	      || operator_code == MINUS_EXPR
-	      || operator_code == TRUNC_DIV_EXPR
-	      || operator_code == MULT_EXPR
-	      || operator_code == TRUNC_MOD_EXPR)
-	  && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == REFERENCE_TYPE)
-	warning (OPT_Weffc__, "%qD should return by value", decl);
-
-      /* [over.oper]/8 */
-      for (; argtypes && argtypes != void_list_node;
-	  argtypes = TREE_CHAIN (argtypes))
-	if (TREE_PURPOSE (argtypes))
-	  {
-	    TREE_PURPOSE (argtypes) = NULL_TREE;
-	    if (operator_code == POSTINCREMENT_EXPR
-		|| operator_code == POSTDECREMENT_EXPR)
-	      {
-		pedwarn (input_location, OPT_Wpedantic, "%qD cannot have default arguments", 
-			 decl);
-	      }
-	    else
-	      {
-		error ("%qD cannot have default arguments", decl);
-		return false;
-	      }
-	  }
-    }
   return true;
 }
 
@@ -14799,9 +14724,11 @@ start_preparsed_function (tree decl1, tree attrs, int flags)
 
   /* Effective C++ rule 15.  */
   if (warn_ecpp
-      && DECL_OVERLOADED_OPERATOR_P (decl1) == NOP_EXPR
+      && DECL_ASSIGNMENT_OPERATOR_P (decl1)
+      && DECL_OVERLOADED_OPERATOR_IS (decl1, NOP_EXPR)
       && VOID_TYPE_P (TREE_TYPE (fntype)))
-    warning (OPT_Weffc__, "%<operator=%> should return a reference to %<*this%>");
+    warning (OPT_Weffc__,
+	     "%<operator=%> should return a reference to %<*this%>");
 
   /* Make the init_value nonzero so pushdecl knows this is not tentative.
      error_mark_node is replaced below (in poplevel) with the BLOCK.  */
diff --git a/gcc/cp/decl2.c b/gcc/cp/decl2.c
index bc509623b36..a23b96c53e7 100644
--- a/gcc/cp/decl2.c
+++ b/gcc/cp/decl2.c
@@ -4455,7 +4455,7 @@ maybe_warn_sized_delete (enum tree_code code)
   tree sized = NULL_TREE;
   tree unsized = NULL_TREE;
 
-  for (ovl_iterator iter (get_global_binding (cp_operator_id (code)));
+  for (ovl_iterator iter (get_global_binding (ovl_op_identifier (false, code)));
        iter; ++iter)
     {
       tree fn = *iter;
@@ -5101,11 +5101,11 @@ mark_used (tree decl, tsubst_flags_t complain)
       && DECL_DELETED_FN (decl))
     {
       if (DECL_ARTIFICIAL (decl)
-	  && DECL_OVERLOADED_OPERATOR_P (decl) == TYPE_EXPR
+	  && DECL_CONV_FN_P (decl)
 	  && LAMBDA_TYPE_P (DECL_CONTEXT (decl)))
 	/* We mark a lambda conversion op as deleted if we can't
 	   generate it properly; see maybe_add_lambda_conv_op.  */
-	sorry ("converting lambda which uses %<...%> to function pointer");
+	sorry ("converting lambda that uses %<...%> to function pointer");
       else if (complain & tf_error)
 	{
 	  error ("use of deleted function %qD", decl);
diff --git a/gcc/cp/dump.c b/gcc/cp/dump.c
index 6fafa5b792e..2e4740f71eb 100644
--- a/gcc/cp/dump.c
+++ b/gcc/cp/dump.c
@@ -24,10 +24,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "cp-tree.h"
 #include "tree-dump.h"
 
-static void dump_access (dump_info_p, tree);
-
-static void dump_op (dump_info_p, tree);
-
 /* Dump a representation of the accessibility information associated
    with T.  */
 
@@ -42,163 +38,6 @@ dump_access (dump_info_p di, tree t)
     dump_string_field (di, "accs", "pub");
 }
 
-/* Dump a representation of the specific operator for an overloaded
-   operator associated with node t.  */
-
-static void
-dump_op (dump_info_p di, tree t)
-{
-  switch (DECL_OVERLOADED_OPERATOR_P (t)) {
-    case NEW_EXPR:
-      dump_string (di, "new");
-      break;
-    case VEC_NEW_EXPR:
-      dump_string (di, "vecnew");
-      break;
-    case DELETE_EXPR:
-      dump_string (di, "delete");
-      break;
-    case VEC_DELETE_EXPR:
-      dump_string (di, "vecdelete");
-      break;
-    case UNARY_PLUS_EXPR:
-      dump_string (di, "pos");
-      break;
-    case NEGATE_EXPR:
-      dump_string (di, "neg");
-      break;
-    case ADDR_EXPR:
-      dump_string (di, "addr");
-      break;
-    case INDIRECT_REF:
-      dump_string(di, "deref");
-      break;
-    case BIT_NOT_EXPR:
-      dump_string(di, "not");
-      break;
-    case TRUTH_NOT_EXPR:
-      dump_string(di, "lnot");
-      break;
-    case PREINCREMENT_EXPR:
-      dump_string(di, "preinc");
-      break;
-    case PREDECREMENT_EXPR:
-      dump_string(di, "predec");
-      break;
-    case PLUS_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "plusassign");
-      else
-	dump_string(di, "plus");
-      break;
-    case MINUS_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "minusassign");
-      else
-	dump_string(di, "minus");
-      break;
-    case MULT_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "multassign");
-      else
-	dump_string (di, "mult");
-      break;
-    case TRUNC_DIV_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "divassign");
-      else
-	dump_string (di, "div");
-      break;
-    case TRUNC_MOD_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	 dump_string (di, "modassign");
-      else
-	dump_string (di, "mod");
-      break;
-    case BIT_AND_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "andassign");
-      else
-	dump_string (di, "and");
-      break;
-    case BIT_IOR_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "orassign");
-      else
-	dump_string (di, "or");
-      break;
-    case BIT_XOR_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "xorassign");
-      else
-	dump_string (di, "xor");
-      break;
-    case LSHIFT_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "lshiftassign");
-      else
-	dump_string (di, "lshift");
-      break;
-    case RSHIFT_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "rshiftassign");
-      else
-	dump_string (di, "rshift");
-      break;
-    case EQ_EXPR:
-      dump_string (di, "eq");
-      break;
-    case NE_EXPR:
-      dump_string (di, "ne");
-      break;
-    case LT_EXPR:
-      dump_string (di, "lt");
-      break;
-    case GT_EXPR:
-      dump_string (di, "gt");
-      break;
-    case LE_EXPR:
-      dump_string (di, "le");
-      break;
-    case GE_EXPR:
-      dump_string (di, "ge");
-      break;
-    case TRUTH_ANDIF_EXPR:
-      dump_string (di, "land");
-      break;
-    case TRUTH_ORIF_EXPR:
-      dump_string (di, "lor");
-      break;
-    case COMPOUND_EXPR:
-      dump_string (di, "compound");
-      break;
-    case MEMBER_REF:
-      dump_string (di, "memref");
-      break;
-    case COMPONENT_REF:
-      dump_string (di, "ref");
-      break;
-    case ARRAY_REF:
-      dump_string (di, "subs");
-      break;
-    case POSTINCREMENT_EXPR:
-      dump_string (di, "postinc");
-      break;
-    case POSTDECREMENT_EXPR:
-      dump_string (di, "postdec");
-      break;
-    case CALL_EXPR:
-      dump_string (di, "call");
-      break;
-    case NOP_EXPR:
-      if (DECL_ASSIGNMENT_OPERATOR_P (t))
-	dump_string (di, "assign");
-      break;
-    default:
-      break;
-  }
-}
-
 /* Dump information common to statements from STMT.  */
 
 static void
@@ -303,10 +142,8 @@ cp_dump_tree (void* dump_info, tree t)
     case FUNCTION_DECL:
       if (!DECL_THUNK_P (t))
 	{
-	  if (DECL_OVERLOADED_OPERATOR_P (t)) {
+	  if (DECL_OVERLOADED_OPERATOR_P (t))
 	    dump_string_field (di, "note", "operator");
-	    dump_op (di, t);
-	  }
 	  if (DECL_FUNCTION_MEMBER_P (t))
 	    {
 	      dump_string_field (di, "note", "member");
diff --git a/gcc/cp/error.c b/gcc/cp/error.c
index 7a98d2e3594..2537713b5c9 100644
--- a/gcc/cp/error.c
+++ b/gcc/cp/error.c
@@ -50,13 +50,12 @@ static cxx_pretty_printer * const cxx_pp = &actual_pretty_printer;
 # define NEXT_CODE(T) (TREE_CODE (TREE_TYPE (T)))
 
 static const char *args_to_string (tree, int);
-static const char *assop_to_string (enum tree_code);
 static const char *code_to_string (enum tree_code);
 static const char *cv_to_string (tree, int);
 static const char *decl_to_string (tree, int);
 static const char *expr_to_string (tree);
 static const char *fndecl_to_string (tree, int);
-static const char *op_to_string	(enum tree_code);
+static const char *op_to_string	(bool, enum tree_code);
 static const char *parm_to_string (int);
 static const char *type_to_string (tree, int);
 
@@ -2230,8 +2229,7 @@ dump_expr (cxx_pretty_printer *pp, tree t, int flags)
 
     case INIT_EXPR:
     case MODIFY_EXPR:
-      dump_binary_op (pp, assignment_operator_name_info[NOP_EXPR].name,
-		      t, flags);
+      dump_binary_op (pp, OVL_OP_INFO (true, NOP_EXPR)->name, t, flags);
       break;
 
     case PLUS_EXPR:
@@ -2255,7 +2253,7 @@ dump_expr (cxx_pretty_printer *pp, tree t, int flags)
     case EQ_EXPR:
     case NE_EXPR:
     case EXACT_DIV_EXPR:
-      dump_binary_op (pp, operator_name_info[TREE_CODE (t)].name, t, flags);
+      dump_binary_op (pp, OVL_OP_INFO (false, TREE_CODE (t))->name, t, flags);
       break;
 
     case CEIL_DIV_EXPR:
@@ -2386,14 +2384,14 @@ dump_expr (cxx_pretty_printer *pp, tree t, int flags)
     case TRUTH_NOT_EXPR:
     case PREDECREMENT_EXPR:
     case PREINCREMENT_EXPR:
-      dump_unary_op (pp, operator_name_info [TREE_CODE (t)].name, t, flags);
+      dump_unary_op (pp, OVL_OP_INFO (false, TREE_CODE (t))->name, t, flags);
       break;
 
     case POSTDECREMENT_EXPR:
     case POSTINCREMENT_EXPR:
       pp_cxx_left_paren (pp);
       dump_expr (pp, TREE_OPERAND (t, 0), flags | TFF_EXPR_IN_PARENS);
-      pp_cxx_ws_string (pp, operator_name_info[TREE_CODE (t)].name);
+      pp_cxx_ws_string (pp, OVL_OP_INFO (false, TREE_CODE (t))->name);
       pp_cxx_right_paren (pp);
       break;
 
@@ -2656,7 +2654,7 @@ dump_expr (cxx_pretty_printer *pp, tree t, int flags)
 
     case REALPART_EXPR:
     case IMAGPART_EXPR:
-      pp_cxx_ws_string (pp, operator_name_info[TREE_CODE (t)].name);
+      pp_cxx_ws_string (pp, OVL_OP_INFO (false, TREE_CODE (t))->name);
       pp_cxx_whitespace (pp);
       dump_expr (pp, TREE_OPERAND (t, 0), flags);
       break;
@@ -3136,9 +3134,9 @@ parm_to_string (int p)
 }
 
 static const char *
-op_to_string (enum tree_code p)
+op_to_string (bool assop, enum tree_code p)
 {
-  tree id = operator_name_info[p].identifier;
+  tree id = ovl_op_identifier (assop, p);
   return id ? IDENTIFIER_POINTER (id) : M_("<unknown>");
 }
 
@@ -3180,13 +3178,6 @@ type_to_string (tree typ, int verbose)
 }
 
 static const char *
-assop_to_string (enum tree_code p)
-{
-  tree id = assignment_operator_name_info[(int) p].identifier;
-  return id ? IDENTIFIER_POINTER (id) : M_("{unknown}");
-}
-
-static const char *
 args_to_string (tree p, int verbose)
 {
   int flags = 0;
@@ -4044,9 +4035,9 @@ cp_printer (pretty_printer *pp, text_info *text, const char *spec,
     case 'E': result = expr_to_string (next_tree);		break;
     case 'F': result = fndecl_to_string (next_tree, verbose);	break;
     case 'L': result = language_to_string (next_lang);		break;
-    case 'O': result = op_to_string (next_tcode);		break;
+    case 'O': result = op_to_string (false, next_tcode);	break;
     case 'P': result = parm_to_string (next_int);		break;
-    case 'Q': result = assop_to_string (next_tcode);		break;
+    case 'Q': result = op_to_string (true, next_tcode);		break;
     case 'S': result = subst_to_string (next_tree);		break;
     case 'T': result = type_to_string (next_tree, verbose);	break;
     case 'V': result = cv_to_string (next_tree, verbose);	break;
diff --git a/gcc/cp/init.c b/gcc/cp/init.c
index ad30f247cf6..9e6e3aff779 100644
--- a/gcc/cp/init.c
+++ b/gcc/cp/init.c
@@ -3055,7 +3055,7 @@ build_new_1 (vec<tree, va_gc> **placement, tree type, tree nelts,
   tree fnname;
   tree fns;
 
-  fnname = cp_operator_id (array_p ? VEC_NEW_EXPR : NEW_EXPR);
+  fnname = ovl_op_identifier (false, array_p ? VEC_NEW_EXPR : NEW_EXPR);
 
   member_new_p = !globally_qualified_p
 		 && CLASS_TYPE_P (elt_type)
diff --git a/gcc/cp/lambda.c b/gcc/cp/lambda.c
index 76f2f29578f..bb6c68a100a 100644
--- a/gcc/cp/lambda.c
+++ b/gcc/cp/lambda.c
@@ -201,7 +201,7 @@ lambda_function (tree lambda)
   if (CLASSTYPE_TEMPLATE_INSTANTIATION (type)
       && !COMPLETE_OR_OPEN_TYPE_P (type))
     return NULL_TREE;
-  lambda = lookup_member (type, cp_operator_id (CALL_EXPR),
+  lambda = lookup_member (type, call_op_identifier,
 			  /*protect=*/0, /*want_type=*/false,
 			  tf_warning_or_error);
   if (lambda)
@@ -1258,7 +1258,6 @@ maybe_add_lambda_conv_op (tree type)
   tree fn = convfn;
   DECL_SOURCE_LOCATION (fn) = DECL_SOURCE_LOCATION (callop);
   SET_DECL_ALIGN (fn, MINIMUM_METHOD_BOUNDARY);
-  SET_OVERLOADED_OPERATOR_CODE (fn, TYPE_EXPR);
   grokclassfn (type, fn, NO_SPECIAL);
   set_linkage_according_to_type (type, fn);
   rest_of_decl_compilation (fn, namespace_bindings_p (), at_eof);
@@ -1312,11 +1311,9 @@ maybe_add_lambda_conv_op (tree type)
     fn = add_inherited_template_parms (fn, DECL_TI_TEMPLATE (callop));
 
   if (flag_sanitize & SANITIZE_NULL)
-    {
-      /* Don't UBsan this function; we're deliberately calling op() with a null
-	 object argument.  */
-      add_no_sanitize_value (fn, SANITIZE_UNDEFINED);
-    }
+    /* Don't UBsan this function; we're deliberately calling op() with a null
+       object argument.  */
+    add_no_sanitize_value (fn, SANITIZE_UNDEFINED);
 
   add_method (type, fn, false);
 
diff --git a/gcc/cp/lex.c b/gcc/cp/lex.c
index fd93401f9a6..c097f4b54cf 100644
--- a/gcc/cp/lex.c
+++ b/gcc/cp/lex.c
@@ -77,17 +77,20 @@ cxx_finish (void)
   c_common_finish ();
 }
 
-/* A mapping from tree codes to operator name information.  */
-operator_name_info_t operator_name_info[(int) MAX_TREE_CODES];
-/* Similar, but for assignment operators.  */
-operator_name_info_t assignment_operator_name_info[(int) MAX_TREE_CODES];
-
-/* Initialize data structures that keep track of operator names.  */
-
-#define DEF_OPERATOR(NAME, C, M, AR, AP) \
- CONSTRAINT (C, sizeof "operator " + sizeof NAME <= 256);
+ovl_op_info_t ovl_op_info[2][OVL_OP_MAX] = 
+  {
+    {
+      {NULL_TREE, NULL, NULL, ERROR_MARK, OVL_OP_ERROR_MARK, 0},
+      {NULL_TREE, NULL, NULL, NOP_EXPR, OVL_OP_NOP_EXPR, 0},
+#define DEF_OPERATOR(NAME, CODE, MANGLING, FLAGS) \
+      {NULL_TREE, NAME, MANGLING, CODE, OVL_OP_##CODE, FLAGS},
+#define OPERATOR_TRANSITION }, {			\
+      {NULL_TREE, NULL, NULL, ERROR_MARK, OVL_OP_ERROR_MARK, 0},
 #include "operators.def"
-#undef DEF_OPERATOR
+    }
+  };
+unsigned char ovl_op_mapping[MAX_TREE_CODES];
+unsigned char ovl_op_alternate[OVL_OP_MAX];
 
 /* Get the name of the kind of identifier T.  */
 
@@ -97,7 +100,8 @@ get_identifier_kind_name (tree id)
   /* Keep in sync with cp_id_kind enumeration.  */
   static const char *const names[cik_max] = {
     "normal", "keyword", "constructor", "destructor",
-    "assign-op", "op-assign-op", "simple-op", "conv-op", };
+    "simple-op", "assign-op", "conv-op", "<reserved>udlit-op"
+  };
 
   unsigned kind = 0;
   kind |= IDENTIFIER_KIND_BIT_2 (id) << 2;
@@ -120,66 +124,94 @@ set_identifier_kind (tree id, cp_identifier_kind kind)
   IDENTIFIER_KIND_BIT_0 (id) |= (kind >> 0) & 1;
 }
 
+/* Create and tag the internal operator name for the overloaded
+   operator PTR describes.  */
+
+static tree
+set_operator_ident (ovl_op_info_t *ptr)
+{
+  char buffer[32];
+  size_t len = snprintf (buffer, sizeof (buffer), "operator%s%s",
+			 &" "[ptr->name[0] && ptr->name[0] != '_'
+			      && !ISALPHA (ptr->name[0])],
+			 ptr->name);
+  gcc_checking_assert (len < sizeof (buffer));
+
+  tree ident = get_identifier_with_length (buffer, len);
+  ptr->identifier = ident;
+
+  return ident;
+}
+
+/* Initialize data structures that keep track of operator names.  */
+
 static void
 init_operators (void)
 {
-  tree identifier;
-  char buffer[256];
-  struct operator_name_info_t *oni;
-
-#define DEF_OPERATOR(NAME, CODE, MANGLING, ARITY, KIND)			\
-  sprintf (buffer, "operator%s%s", !NAME[0]				\
-	   || NAME[0] == '_' || ISALPHA (NAME[0]) ? " " : "", NAME);	\
-  identifier = get_identifier (buffer);					\
-									\
-  if (KIND != cik_simple_op || !IDENTIFIER_ANY_OP_P (identifier))	\
-    set_identifier_kind (identifier, KIND);				\
-  									\
-  oni = (KIND == cik_assign_op						\
-	 ? &assignment_operator_name_info[(int) CODE]			\
-	 : &operator_name_info[(int) CODE]);				\
-  oni->identifier = identifier;						\
-  oni->name = NAME;							\
-  oni->mangled_name = MANGLING;						\
-  oni->arity = ARITY;
+  /* We rely on both these being zero.  */
+  gcc_checking_assert (!OVL_OP_ERROR_MARK && !ERROR_MARK);
 
-#include "operators.def"
-#undef DEF_OPERATOR
-
-  operator_name_info[(int) TYPE_EXPR] = operator_name_info[(int) CAST_EXPR];
-  operator_name_info[(int) ERROR_MARK].identifier
-    = get_identifier ("<invalid operator>");
-
-  /* Handle some special cases.  These operators are not defined in
-     the language, but can be produced internally.  We may need them
-     for error-reporting.  (Eventually, we should ensure that this
-     does not happen.  Error messages involving these operators will
-     be confusing to users.)  */
-
-  operator_name_info [(int) INIT_EXPR].name
-    = operator_name_info [(int) MODIFY_EXPR].name;
-
-  operator_name_info [(int) EXACT_DIV_EXPR].name = "(ceiling /)";
-  operator_name_info [(int) CEIL_DIV_EXPR].name = "(ceiling /)";
-  operator_name_info [(int) FLOOR_DIV_EXPR].name = "(floor /)";
-  operator_name_info [(int) ROUND_DIV_EXPR].name = "(round /)";
-  operator_name_info [(int) CEIL_MOD_EXPR].name = "(ceiling %)";
-  operator_name_info [(int) FLOOR_MOD_EXPR].name = "(floor %)";
-  operator_name_info [(int) ROUND_MOD_EXPR].name = "(round %)";
-
-  operator_name_info [(int) ABS_EXPR].name = "abs";
-  operator_name_info [(int) TRUTH_AND_EXPR].name = "strict &&";
-  operator_name_info [(int) TRUTH_OR_EXPR].name = "strict ||";
-  operator_name_info [(int) RANGE_EXPR].name = "...";
-  operator_name_info [(int) UNARY_PLUS_EXPR].name = "+";
-
-  assignment_operator_name_info [(int) EXACT_DIV_EXPR].name = "(exact /=)";
-  assignment_operator_name_info [(int) CEIL_DIV_EXPR].name = "(ceiling /=)";
-  assignment_operator_name_info [(int) FLOOR_DIV_EXPR].name = "(floor /=)";
-  assignment_operator_name_info [(int) ROUND_DIV_EXPR].name = "(round /=)";
-  assignment_operator_name_info [(int) CEIL_MOD_EXPR].name = "(ceiling %=)";
-  assignment_operator_name_info [(int) FLOOR_MOD_EXPR].name = "(floor %=)";
-  assignment_operator_name_info [(int) ROUND_MOD_EXPR].name = "(round %=)";
+  /* This loop iterates backwards because we need to move the
+     assignment operators down to their correct slots.  I.e. morally
+     equivalent to an overlapping memmove where dest > src.  Slot
+     zero is for error_mark, so hae no operator. */
+  for (unsigned ix = OVL_OP_MAX; --ix;)
+    {
+      ovl_op_info_t *op_ptr = &ovl_op_info[false][ix];
+
+      if (op_ptr->name)
+	{
+	  /* Make sure it fits in lang_decl_fn::operator_code. */
+	  gcc_checking_assert (op_ptr->ovl_op_code < (1 << 6));
+	  tree ident = set_operator_ident (op_ptr);
+	  if (unsigned index = IDENTIFIER_CP_INDEX (ident))
+	    {
+	      ovl_op_info_t *bin_ptr = &ovl_op_info[false][index];
+
+	      /* They should only differ in unary/binary ness.  */
+	      gcc_checking_assert ((op_ptr->flags ^ bin_ptr->flags)
+				   == OVL_OP_FLAG_AMBIARY);
+	      bin_ptr->flags |= op_ptr->flags;
+	      ovl_op_alternate[index] = ix;
+	    }
+	  else
+	    {
+	      IDENTIFIER_CP_INDEX (ident) = ix;
+	      set_identifier_kind (ident, cik_simple_op);
+	    }
+	}
+      if (op_ptr->tree_code)
+	{
+	  gcc_checking_assert (op_ptr->ovl_op_code == ix
+			       && !ovl_op_mapping[op_ptr->tree_code]);
+	  ovl_op_mapping[op_ptr->tree_code] = op_ptr->ovl_op_code;
+	}
+
+      ovl_op_info_t *as_ptr = &ovl_op_info[true][ix];
+      if (as_ptr->name)
+	{
+	  /* These will be placed at the start of the array, move to
+	     the correct slot and initialize.  */
+	  if (as_ptr->ovl_op_code != ix)
+	    {
+	      ovl_op_info_t *dst_ptr = &ovl_op_info[true][as_ptr->ovl_op_code];
+	      gcc_assert (as_ptr->ovl_op_code > ix && !dst_ptr->tree_code);
+	      memcpy (dst_ptr, as_ptr, sizeof (*dst_ptr));
+	      memset (as_ptr, 0, sizeof (*as_ptr));
+	      as_ptr = dst_ptr;
+	    }
+
+	  tree ident = set_operator_ident (as_ptr);
+	  gcc_checking_assert (!IDENTIFIER_CP_INDEX (ident));
+	  IDENTIFIER_CP_INDEX (ident) = as_ptr->ovl_op_code;
+	  set_identifier_kind (ident, cik_assign_op);
+
+	  gcc_checking_assert (!ovl_op_mapping[as_ptr->tree_code]
+			       || (ovl_op_mapping[as_ptr->tree_code]
+				   == as_ptr->ovl_op_code));
+	  ovl_op_mapping[as_ptr->tree_code] = as_ptr->ovl_op_code;
+	}
+    }
 }
 
 /* Initialize the reserved words.  */
@@ -462,10 +494,7 @@ unqualified_name_lookup_error (tree name, location_t loc)
     loc = EXPR_LOC_OR_LOC (name, input_location);
 
   if (IDENTIFIER_ANY_OP_P (name))
-    {
-      if (name != cp_operator_id (ERROR_MARK))
-	error_at (loc, "%qD not defined", name);
-    }
+    error_at (loc, "%qD not defined", name);
   else
     {
       if (!objc_diagnose_private_ivar (name))
@@ -585,7 +614,6 @@ make_conv_op_name (tree type)
 
       /* Just in case something managed to bind.  */
       IDENTIFIER_BINDING (identifier) = NULL;
-      IDENTIFIER_LABEL_VALUE (identifier) = NULL_TREE;
 
       /* Hang TYPE off the identifier so it can be found easily later
 	 when performing conversions.  */
diff --git a/gcc/cp/mangle.c b/gcc/cp/mangle.c
index 19e7c62e483..ea7f55ca293 100644
--- a/gcc/cp/mangle.c
+++ b/gcc/cp/mangle.c
@@ -1263,35 +1263,10 @@ write_unqualified_id (tree identifier)
 {
   if (IDENTIFIER_CONV_OP_P (identifier))
     write_conversion_operator_name (TREE_TYPE (identifier));
-  else if (IDENTIFIER_ANY_OP_P (identifier))
+  else if (IDENTIFIER_OVL_OP_P (identifier))
     {
-      int i;
-      const char *mangled_name = NULL;
-
-      /* Unfortunately, there is no easy way to go from the
-	 name of the operator back to the corresponding tree
-	 code.  */
-      for (i = 0; i < MAX_TREE_CODES; ++i)
-	if (operator_name_info[i].identifier == identifier)
-	  {
-	    /* The ABI says that we prefer binary operator
-	       names to unary operator names.  */
-	    if (operator_name_info[i].arity == 2)
-	      {
-		mangled_name = operator_name_info[i].mangled_name;
-		break;
-	      }
-	    else if (!mangled_name)
-	      mangled_name = operator_name_info[i].mangled_name;
-	  }
-	else if (assignment_operator_name_info[i].identifier
-		 == identifier)
-	  {
-	    mangled_name
-	      = assignment_operator_name_info[i].mangled_name;
-	    break;
-	  }
-      write_string (mangled_name);
+      const ovl_op_info_t *ovl_op = IDENTIFIER_OVL_OP_INFO (identifier);
+      write_string (ovl_op->mangled_name);
     }
   else if (UDLIT_OPER_P (identifier))
     write_literal_operator_name (identifier);
@@ -1345,13 +1320,10 @@ write_unqualified_name (tree decl)
 	}
       else if (DECL_OVERLOADED_OPERATOR_P (decl))
 	{
-	  operator_name_info_t *oni;
-	  if (DECL_ASSIGNMENT_OPERATOR_P (decl))
-	    oni = assignment_operator_name_info;
-	  else
-	    oni = operator_name_info;
-
-	  write_string (oni[DECL_OVERLOADED_OPERATOR_P (decl)].mangled_name);
+	  const char *mangled_name
+	    = (ovl_op_info[DECL_ASSIGNMENT_OPERATOR_P (decl)]
+	       [DECL_OVERLOADED_OPERATOR_CODE_RAW (decl)].mangled_name);
+	  write_string (mangled_name);
 	}
       else if (UDLIT_OPER_P (DECL_NAME (decl)))
 	write_literal_operator_name (DECL_NAME (decl));
@@ -3065,8 +3037,8 @@ write_expression (tree expr)
   else if (TREE_CODE (expr) == MODOP_EXPR)
     {
       enum tree_code subop = TREE_CODE (TREE_OPERAND (expr, 1));
-      const char *name = (assignment_operator_name_info[(int) subop]
-			  .mangled_name);
+      const char *name = OVL_OP_INFO (true, subop)->mangled_name;
+
       write_string (name);
       write_expression (TREE_OPERAND (expr, 0));
       write_expression (TREE_OPERAND (expr, 2));
@@ -3091,7 +3063,7 @@ write_expression (tree expr)
       if (NEW_EXPR_USE_GLOBAL (expr))
 	write_string ("gs");
 
-      write_string (operator_name_info[(int) code].mangled_name);
+      write_string (OVL_OP_INFO (false, code)->mangled_name);
 
       for (t = placement; t; t = TREE_CHAIN (t))
 	write_expression (TREE_VALUE (t));
@@ -3131,7 +3103,7 @@ write_expression (tree expr)
       if (DELETE_EXPR_USE_GLOBAL (expr))
 	write_string ("gs");
 
-      write_string (operator_name_info[(int) code].mangled_name);
+      write_string (OVL_OP_INFO (false, code)->mangled_name);
 
       write_expression (TREE_OPERAND (expr, 0));
     }
@@ -3196,7 +3168,7 @@ write_expression (tree expr)
 
 	  if (TREE_CODE (ob) == ARROW_EXPR)
 	    {
-	      write_string (operator_name_info[(int)code].mangled_name);
+	      write_string (OVL_OP_INFO (false, code)->mangled_name);
 	      ob = TREE_OPERAND (ob, 0);
 	      write_expression (ob);
 	    }
@@ -3213,7 +3185,7 @@ write_expression (tree expr)
 	}
 
       /* If it wasn't any of those, recursively expand the expression.  */
-      name = operator_name_info[(int) code].mangled_name;
+      name = OVL_OP_INFO (false, code)->mangled_name;
 
       /* We used to mangle const_cast and static_cast like a C cast.  */
       if (code == CONST_CAST_EXPR
@@ -3222,7 +3194,7 @@ write_expression (tree expr)
 	  if (abi_warn_or_compat_version_crosses (6))
 	    G.need_abi_warning = 1;
 	  if (!abi_version_at_least (6))
-	    name = operator_name_info[CAST_EXPR].mangled_name;
+	    name = OVL_OP_INFO (false, CAST_EXPR)->mangled_name;
 	}
 
       if (name == NULL)
@@ -3323,7 +3295,7 @@ write_expression (tree expr)
 		  if (i == 0)
 		    {
 		      int fcode = TREE_INT_CST_LOW (operand);
-		      write_string (operator_name_info[fcode].mangled_name);
+		      write_string (OVL_OP_INFO (false, fcode)->mangled_name);
 		      continue;
 		    }
 		  else if (code == BINARY_LEFT_FOLD_EXPR)
diff --git a/gcc/cp/method.c b/gcc/cp/method.c
index 4e56874ae26..714b5087991 100644
--- a/gcc/cp/method.c
+++ b/gcc/cp/method.c
@@ -815,7 +815,7 @@ do_build_copy_assign (tree fndecl)
 	  parmvec = make_tree_vector_single (converted_parm);
 	  finish_expr_stmt
 	    (build_special_member_call (current_class_ref,
-					cp_assignment_operator_id (NOP_EXPR),
+					assign_op_identifier,
 					&parmvec,
 					base_binfo,
 					flags,
@@ -929,7 +929,8 @@ synthesize_method (tree fndecl)
   start_preparsed_function (fndecl, NULL_TREE, SF_DEFAULT | SF_PRE_PARSED);
   stmt = begin_function_body ();
 
-  if (DECL_OVERLOADED_OPERATOR_P (fndecl) == NOP_EXPR)
+  if (DECL_ASSIGNMENT_OPERATOR_P (fndecl)
+      && DECL_OVERLOADED_OPERATOR_IS (fndecl, NOP_EXPR))
     {
       do_build_copy_assign (fndecl);
       need_body = false;
@@ -1108,7 +1109,7 @@ get_copy_assign (tree type)
   int quals = (TYPE_HAS_CONST_COPY_ASSIGN (type)
 	       ? TYPE_QUAL_CONST : TYPE_UNQUALIFIED);
   tree argtype = build_stub_type (type, quals, false);
-  tree fn = locate_fn_flags (type, cp_assignment_operator_id (NOP_EXPR), argtype,
+  tree fn = locate_fn_flags (type, assign_op_identifier, argtype,
 			     LOOKUP_NORMAL, tf_warning_or_error);
   if (fn == error_mark_node)
     return NULL_TREE;
@@ -1565,7 +1566,7 @@ synthesized_method_walk (tree ctype, special_function_kind sfk, bool const_p,
     case sfk_move_assignment:
     case sfk_copy_assignment:
       assign_p = true;
-      fnname = cp_assignment_operator_id (NOP_EXPR);
+      fnname = assign_op_identifier;
       break;
 
     case sfk_destructor:
@@ -1702,12 +1703,12 @@ synthesized_method_walk (tree ctype, special_function_kind sfk, bool const_p,
 	{
 	  /* Unlike for base ctor/op=/dtor, for operator delete it's fine
 	     to have a null fn (no class-specific op delete).  */
-	  fn = locate_fn_flags (ctype, cp_operator_id (DELETE_EXPR),
+	  fn = locate_fn_flags (ctype, ovl_op_identifier (false, DELETE_EXPR),
 				ptr_type_node, flags, tf_none);
 	  if (fn && fn == error_mark_node)
 	    {
 	      if (complain & tf_error)
-		locate_fn_flags (ctype, cp_operator_id (DELETE_EXPR),
+		locate_fn_flags (ctype, ovl_op_identifier (false, DELETE_EXPR),
 				 ptr_type_node, flags, complain);
 	      if (deleted_p)
 		*deleted_p = true;
@@ -2007,7 +2008,7 @@ implicitly_declare_fn (special_function_kind kind, tree type,
 	  || kind == sfk_move_assignment)
 	{
 	  return_type = build_reference_type (type);
-	  name = cp_assignment_operator_id (NOP_EXPR);
+	  name = assign_op_identifier;
 	}
       else
 	name = ctor_identifier;
@@ -2077,7 +2078,7 @@ implicitly_declare_fn (special_function_kind kind, tree type,
 
   if (!IDENTIFIER_CDTOR_P (name))
     /* Assignment operator.  */
-    SET_OVERLOADED_OPERATOR_CODE (fn, NOP_EXPR);
+    DECL_OVERLOADED_OPERATOR_CODE_RAW (fn) = OVL_OP_NOP_EXPR;
   else if (IDENTIFIER_CTOR_P (name))
     DECL_CXX_CONSTRUCTOR_P (fn) = true;
   else
@@ -2318,7 +2319,7 @@ defaultable_fn_check (tree fn)
   else if (DECL_DESTRUCTOR_P (fn))
     kind = sfk_destructor;
   else if (DECL_ASSIGNMENT_OPERATOR_P (fn)
-	   && DECL_OVERLOADED_OPERATOR_P (fn) == NOP_EXPR)
+	   && DECL_OVERLOADED_OPERATOR_IS (fn, NOP_EXPR))
     {
       if (copy_fn_p (fn))
 	kind = sfk_copy_assignment;
diff --git a/gcc/cp/name-lookup.c b/gcc/cp/name-lookup.c
index b1b4ebbb7de..b4976d8b7cc 100644
--- a/gcc/cp/name-lookup.c
+++ b/gcc/cp/name-lookup.c
@@ -1299,7 +1299,7 @@ get_class_binding (tree klass, tree name, int type_or_fns)
 	  if (CLASSTYPE_LAZY_DESTRUCTOR (klass))
 	    lazily_declare_fn (sfk_destructor, klass);
 	}
-      else if (name == cp_assignment_operator_id (NOP_EXPR))
+      else if (name == assign_op_identifier)
 	{
 	  if (CLASSTYPE_LAZY_COPY_ASSIGN (klass))
 	    lazily_declare_fn (sfk_copy_assignment, klass);
@@ -5377,7 +5377,7 @@ suggest_alternatives_for (location_t location, tree name,
       gcc_rich_location richloc (location);
 
       richloc.add_fixit_replace (fuzzy);
-      inform_at_rich_loc (&richloc, "suggested alternative: %qs", fuzzy);
+      inform (&richloc, "suggested alternative: %qs", fuzzy);
     }
 }
 
@@ -5485,10 +5485,10 @@ maybe_suggest_missing_header (location_t location, tree name, tree scope)
 
   gcc_rich_location richloc (location);
   maybe_add_include_fixit (&richloc, header_hint);
-  inform_at_rich_loc (&richloc,
-		      "%<std::%s%> is defined in header %qs;"
-		      " did you forget to %<#include %s%>?",
-		      name_str, header_hint, header_hint);
+  inform (&richloc,
+	  "%<std::%s%> is defined in header %qs;"
+	  " did you forget to %<#include %s%>?",
+	  name_str, header_hint, header_hint);
   return true;
 }
 
@@ -5518,8 +5518,8 @@ suggest_alternative_in_explicit_scope (location_t location, tree name,
     {
       gcc_rich_location richloc (location);
       richloc.add_fixit_replace (fuzzy_name);
-      inform_at_rich_loc (&richloc, "suggested alternative: %qs",
-			  fuzzy_name);
+      inform (&richloc, "suggested alternative: %qs",
+	      fuzzy_name);
       return true;
     }
 
diff --git a/gcc/cp/name-lookup.h b/gcc/cp/name-lookup.h
index bf0bf85cd53..1fc128070d3 100644
--- a/gcc/cp/name-lookup.h
+++ b/gcc/cp/name-lookup.h
@@ -148,15 +148,6 @@ struct GTY(()) cp_class_binding {
   tree identifier;
 };
 
-
-struct GTY(()) cp_label_binding {
-  /* The bound LABEL_DECL.  */
-  tree label;
-  /* The previous IDENTIFIER_LABEL_VALUE.  */
-  tree prev_value;
-};
-
-
 /* For each binding contour we allocate a binding_level structure
    which records the names defined in that contour.
    Contours include:
@@ -202,10 +193,6 @@ struct GTY(()) cp_binding_level {
       the class.  */
   tree type_shadowed;
 
-  /* Similar to class_shadowed, but for IDENTIFIER_LABEL_VALUE, and
-      used for all binding levels.  */
-  vec<cp_label_binding, va_gc> *shadowed_labels;
-
   /* For each level (except not the global one),
       a chain of BLOCK nodes for all the levels
       that were entered and exited one level down.  */
diff --git a/gcc/cp/operators.def b/gcc/cp/operators.def
index 7dfdd227241..119529ccddd 100644
--- a/gcc/cp/operators.def
+++ b/gcc/cp/operators.def
@@ -45,116 +45,114 @@ along with GCC; see the file COPYING3.  If not see
      mangled under the new ABI.  For `operator +', for example, this
      would be "pl".
 
-   ARITY
+   FLAGS
 
-     The arity of the operator, or -1 if any arity is allowed.  (As
-     for `operator ()'.)  Postincrement and postdecrement operators
-     are marked as binary.
-
-   ASSN_P
-
-     A boolean value.  If nonzero, this is an assignment operator.
+     ovl_op_flags bits.  Postincrement and postdecrement operators are
+     marked as binary.
 
    Before including this file, you should define DEF_OPERATOR
    to take these arguments.
 
    There is code (such as in grok_op_properties) that depends on the
-   order the operators are presented in this file.  In particular,
-   unary operators must precede binary operators.  */
-
-/* Use DEF_SIMPLE_OPERATOR to define a non-assignment operator.  Its
-   arguments are as for DEF_OPERATOR, but there is no need to provide
-   an ASSIGNMENT_P argument; it is always zero.  */
-
-#define DEF_SIMPLE_OPERATOR(NAME, CODE, MANGLING, ARITY) \
-  DEF_OPERATOR(NAME, CODE, MANGLING, ARITY, cik_simple_op)
+   order the operators are presented in this file.  Unary_ops must
+   preceed a matching binary op (i.e. '+').  Assignment operators must
+   be last, after OPERATOR_TRANSITION.  */
 
 /* Use DEF_ASSN_OPERATOR to define an assignment operator.  Its
    arguments are as for DEF_OPERATOR, but there is no need to provide
-   an ASSIGNMENT_P argument; it is always one.  */
-
-#define DEF_ASSN_OPERATOR(NAME, CODE, MANGLING, ARITY) \
-  DEF_OPERATOR(NAME, CODE, MANGLING, ARITY, cik_assign_op)
-
-/* Memory allocation operators.  */
-DEF_OPERATOR ("new", NEW_EXPR, "nw", -1, cik_newdel_op)
-DEF_OPERATOR ("new []", VEC_NEW_EXPR, "na", -1, cik_newdel_op)
-DEF_OPERATOR ("delete", DELETE_EXPR, "dl", -1, cik_newdel_op)
-DEF_OPERATOR ("delete []", VEC_DELETE_EXPR, "da", -1, cik_newdel_op)
+   FLAGS (OVL_OP_FLAG_BINARY).  */
+
+#ifndef DEF_ASSN_OPERATOR
+#define DEF_ASSN_OPERATOR(NAME, CODE, MANGLING) \
+  DEF_OPERATOR(NAME, CODE, MANGLING, OVL_OP_FLAG_BINARY)
+#endif
+
+/* Memory allocation operators.  ARITY has special meaning. */
+DEF_OPERATOR ("new", NEW_EXPR, "nw", OVL_OP_FLAG_ALLOC)
+DEF_OPERATOR ("new []", VEC_NEW_EXPR, "na",
+	      OVL_OP_FLAG_ALLOC | OVL_OP_FLAG_VEC)
+DEF_OPERATOR ("delete", DELETE_EXPR, "dl",
+	      OVL_OP_FLAG_ALLOC | OVL_OP_FLAG_DELETE)
+DEF_OPERATOR ("delete []", VEC_DELETE_EXPR, "da",
+	      OVL_OP_FLAG_ALLOC | OVL_OP_FLAG_DELETE | OVL_OP_FLAG_VEC)
 
 /* Unary operators.  */
-DEF_SIMPLE_OPERATOR ("+", UNARY_PLUS_EXPR, "ps", 1)
-DEF_SIMPLE_OPERATOR ("-", NEGATE_EXPR, "ng", 1)
-DEF_SIMPLE_OPERATOR ("&", ADDR_EXPR, "ad", 1)
-DEF_SIMPLE_OPERATOR ("*", INDIRECT_REF, "de", 1)
-DEF_SIMPLE_OPERATOR ("~", BIT_NOT_EXPR, "co", 1)
-DEF_SIMPLE_OPERATOR ("!", TRUTH_NOT_EXPR, "nt", 1)
-DEF_SIMPLE_OPERATOR ("++", PREINCREMENT_EXPR, "pp", 1)
-DEF_SIMPLE_OPERATOR ("--", PREDECREMENT_EXPR, "mm", 1)
-DEF_SIMPLE_OPERATOR ("sizeof", SIZEOF_EXPR, "sz", 1)
-/* These are extensions.  */
-DEF_SIMPLE_OPERATOR ("alignof", ALIGNOF_EXPR, "az", 1)
-DEF_SIMPLE_OPERATOR ("__imag__", IMAGPART_EXPR, "v18__imag__", 1)
-DEF_SIMPLE_OPERATOR ("__real__", REALPART_EXPR, "v18__real__", 1)
+DEF_OPERATOR ("+", UNARY_PLUS_EXPR, "ps", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("-", NEGATE_EXPR, "ng", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("&", ADDR_EXPR, "ad", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("*", INDIRECT_REF, "de", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("~", BIT_NOT_EXPR, "co", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("!", TRUTH_NOT_EXPR, "nt", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("++", PREINCREMENT_EXPR, "pp", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("--", PREDECREMENT_EXPR, "mm", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("->", COMPONENT_REF, "pt", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("sizeof", SIZEOF_EXPR, "sz", OVL_OP_FLAG_UNARY)
 
-/* The cast operators.  */
-DEF_SIMPLE_OPERATOR ("", CAST_EXPR, "cv", 1)
-DEF_SIMPLE_OPERATOR ("dynamic_cast", DYNAMIC_CAST_EXPR, "dc", 1)
-DEF_SIMPLE_OPERATOR ("reinterpret_cast", REINTERPRET_CAST_EXPR, "rc", 1)
-DEF_SIMPLE_OPERATOR ("const_cast", CONST_CAST_EXPR, "cc", 1)
-DEF_SIMPLE_OPERATOR ("static_cast", STATIC_CAST_EXPR, "sc", 1)
+/* These are extensions.  */
+DEF_OPERATOR ("alignof", ALIGNOF_EXPR, "az", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("__imag__", IMAGPART_EXPR, "v18__imag__", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR ("__real__", REALPART_EXPR, "v18__real__", OVL_OP_FLAG_UNARY)
 
 /* Binary operators.  */
-DEF_SIMPLE_OPERATOR ("+", PLUS_EXPR, "pl", 2)
-DEF_SIMPLE_OPERATOR ("-", MINUS_EXPR, "mi", 2)
-DEF_SIMPLE_OPERATOR ("*", MULT_EXPR, "ml", 2)
-DEF_SIMPLE_OPERATOR ("/", TRUNC_DIV_EXPR, "dv", 2)
-DEF_SIMPLE_OPERATOR ("%", TRUNC_MOD_EXPR, "rm", 2)
-DEF_SIMPLE_OPERATOR ("&", BIT_AND_EXPR, "an", 2)
-DEF_SIMPLE_OPERATOR ("|", BIT_IOR_EXPR, "or", 2)
-DEF_SIMPLE_OPERATOR ("^", BIT_XOR_EXPR, "eo", 2)
-DEF_SIMPLE_OPERATOR ("<<", LSHIFT_EXPR, "ls", 2)
-DEF_SIMPLE_OPERATOR (">>", RSHIFT_EXPR, "rs", 2)
-DEF_SIMPLE_OPERATOR ("==", EQ_EXPR, "eq", 2)
-DEF_SIMPLE_OPERATOR ("!=", NE_EXPR, "ne", 2)
-DEF_SIMPLE_OPERATOR ("<", LT_EXPR, "lt", 2)
-DEF_SIMPLE_OPERATOR (">", GT_EXPR, "gt", 2)
-DEF_SIMPLE_OPERATOR ("<=", LE_EXPR, "le", 2)
-DEF_SIMPLE_OPERATOR (">=", GE_EXPR, "ge", 2)
-DEF_SIMPLE_OPERATOR ("&&", TRUTH_ANDIF_EXPR, "aa", 2)
-DEF_SIMPLE_OPERATOR ("||", TRUTH_ORIF_EXPR, "oo", 2)
-DEF_SIMPLE_OPERATOR (",", COMPOUND_EXPR, "cm", 2)
-DEF_SIMPLE_OPERATOR ("->*", MEMBER_REF, "pm", 2)
-DEF_SIMPLE_OPERATOR (".*", DOTSTAR_EXPR, "ds", 2)
-DEF_SIMPLE_OPERATOR ("->", COMPONENT_REF, "pt", 2)
-DEF_SIMPLE_OPERATOR ("[]", ARRAY_REF, "ix", 2)
-DEF_SIMPLE_OPERATOR ("++", POSTINCREMENT_EXPR, "pp", 2)
-DEF_SIMPLE_OPERATOR ("--", POSTDECREMENT_EXPR, "mm", 2)
-/* This one is needed for mangling.  */
-DEF_SIMPLE_OPERATOR ("::", SCOPE_REF, "sr", 2)
-
-/* Assignment operators.  */
-DEF_ASSN_OPERATOR ("=", NOP_EXPR, "aS", 2)
-DEF_ASSN_OPERATOR ("+=", PLUS_EXPR, "pL", 2)
-DEF_ASSN_OPERATOR ("-=", MINUS_EXPR, "mI", 2)
-DEF_ASSN_OPERATOR ("*=", MULT_EXPR, "mL", 2)
-DEF_ASSN_OPERATOR ("/=", TRUNC_DIV_EXPR, "dV", 2)
-DEF_ASSN_OPERATOR ("%=", TRUNC_MOD_EXPR, "rM", 2)
-DEF_ASSN_OPERATOR ("&=", BIT_AND_EXPR, "aN", 2)
-DEF_ASSN_OPERATOR ("|=", BIT_IOR_EXPR, "oR", 2)
-DEF_ASSN_OPERATOR ("^=", BIT_XOR_EXPR, "eO", 2)
-DEF_ASSN_OPERATOR ("<<=", LSHIFT_EXPR, "lS", 2)
-DEF_ASSN_OPERATOR (">>=", RSHIFT_EXPR, "rS", 2)
-
-/* Ternary operators.  */
-DEF_SIMPLE_OPERATOR ("?:", COND_EXPR, "qu", 3)
+DEF_OPERATOR ("+", PLUS_EXPR, "pl", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("-", MINUS_EXPR, "mi", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("*", MULT_EXPR, "ml", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("/", TRUNC_DIV_EXPR, "dv", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("%", TRUNC_MOD_EXPR, "rm", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("&", BIT_AND_EXPR, "an", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("|", BIT_IOR_EXPR, "or", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("^", BIT_XOR_EXPR, "eo", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("<<", LSHIFT_EXPR, "ls", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR (">>", RSHIFT_EXPR, "rs", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("==", EQ_EXPR, "eq", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("!=", NE_EXPR, "ne", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("<", LT_EXPR, "lt", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR (">", GT_EXPR, "gt", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("<=", LE_EXPR, "le", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR (">=", GE_EXPR, "ge", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("&&", TRUTH_ANDIF_EXPR, "aa", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("||", TRUTH_ORIF_EXPR, "oo", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR (",", COMPOUND_EXPR, "cm", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("->*", MEMBER_REF, "pm", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR (".*", DOTSTAR_EXPR, "ds", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("[]", ARRAY_REF, "ix", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("++", POSTINCREMENT_EXPR, "pp", OVL_OP_FLAG_BINARY)
+DEF_OPERATOR ("--", POSTDECREMENT_EXPR, "mm", OVL_OP_FLAG_BINARY)
 
 /* Miscellaneous.  */
-DEF_SIMPLE_OPERATOR ("()", CALL_EXPR, "cl", -1)
-
-/* Variadic templates extension. */
-DEF_SIMPLE_OPERATOR ("...", EXPR_PACK_EXPANSION, "sp", 1)
-DEF_SIMPLE_OPERATOR ("... +", UNARY_LEFT_FOLD_EXPR, "fl", 2)
-DEF_SIMPLE_OPERATOR ("+ ...", UNARY_RIGHT_FOLD_EXPR, "fr", 2)
-DEF_SIMPLE_OPERATOR ("+ ... +", BINARY_LEFT_FOLD_EXPR, "fL", 3)
-DEF_SIMPLE_OPERATOR ("+ ... +", BINARY_RIGHT_FOLD_EXPR, "fR", 3)
+DEF_OPERATOR ("?:", COND_EXPR, "qu", OVL_OP_FLAG_NONE)
+DEF_OPERATOR ("()", CALL_EXPR, "cl", OVL_OP_FLAG_NONE)
+
+/* Operators needed for mangling.  */
+DEF_OPERATOR (NULL, CAST_EXPR, "cv", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR (NULL, DYNAMIC_CAST_EXPR, "dc", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR (NULL, REINTERPRET_CAST_EXPR, "rc", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR (NULL, CONST_CAST_EXPR, "cc", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR (NULL, STATIC_CAST_EXPR, "sc", OVL_OP_FLAG_UNARY)
+DEF_OPERATOR (NULL, SCOPE_REF, "sr", OVL_OP_FLAG_NONE)
+DEF_OPERATOR (NULL, EXPR_PACK_EXPANSION, "sp", OVL_OP_FLAG_NONE)
+DEF_OPERATOR (NULL, UNARY_LEFT_FOLD_EXPR, "fl", OVL_OP_FLAG_NONE)
+DEF_OPERATOR (NULL, UNARY_RIGHT_FOLD_EXPR, "fr", OVL_OP_FLAG_NONE)
+DEF_OPERATOR (NULL, BINARY_LEFT_FOLD_EXPR, "fL", OVL_OP_FLAG_NONE)
+DEF_OPERATOR (NULL, BINARY_RIGHT_FOLD_EXPR, "fR", OVL_OP_FLAG_NONE)
+
+#ifdef OPERATOR_TRANSITION
+OPERATOR_TRANSITION
+#undef OPERATOR_TRANSITION
+#endif
+
+/* Assignment operators.  */
+DEF_ASSN_OPERATOR ("=", NOP_EXPR, "aS")
+DEF_ASSN_OPERATOR ("+=", PLUS_EXPR, "pL")
+DEF_ASSN_OPERATOR ("-=", MINUS_EXPR, "mI")
+DEF_ASSN_OPERATOR ("*=", MULT_EXPR, "mL")
+DEF_ASSN_OPERATOR ("/=", TRUNC_DIV_EXPR, "dV")
+DEF_ASSN_OPERATOR ("%=", TRUNC_MOD_EXPR, "rM")
+DEF_ASSN_OPERATOR ("&=", BIT_AND_EXPR, "aN")
+DEF_ASSN_OPERATOR ("|=", BIT_IOR_EXPR, "oR")
+DEF_ASSN_OPERATOR ("^=", BIT_XOR_EXPR, "eO")
+DEF_ASSN_OPERATOR ("<<=", LSHIFT_EXPR, "lS")
+DEF_ASSN_OPERATOR (">>=", RSHIFT_EXPR, "rS")
+
+#undef DEF_ASSN_OPERATOR
+#undef DEF_OPERATOR
diff --git a/gcc/cp/parser.c b/gcc/cp/parser.c
index 810e2b7f72e..77b96376e13 100644
--- a/gcc/cp/parser.c
+++ b/gcc/cp/parser.c
@@ -3294,9 +3294,9 @@ cp_parser_diagnose_invalid_type_name (cp_parser *parser, tree id,
 	{
 	  gcc_rich_location richloc (location);
 	  richloc.add_fixit_replace (suggestion);
-	  error_at_rich_loc (&richloc,
-			     "%qE does not name a type; did you mean %qs?",
-			     id, suggestion);
+	  error_at (&richloc,
+		    "%qE does not name a type; did you mean %qs?",
+		    id, suggestion);
 	}
       else
 	error_at (location, "%qE does not name a type", id);
@@ -3937,6 +3937,9 @@ cp_parser_new (void)
   /* Allow constrained-type-specifiers. */
   parser->prevent_constrained_type_specifiers = 0;
 
+  /* We haven't yet seen an 'extern "C"'.  */
+  parser->innermost_linkage_specification_location = UNKNOWN_LOCATION;
+
   return parser;
 }
 
@@ -4104,9 +4107,9 @@ cp_parser_string_literal (cp_parser *parser, bool translate, bool wide_ok,
 		{
 		  rich_location rich_loc (line_table, tok->location);
 		  rich_loc.add_range (last_tok_loc, false);
-		  error_at_rich_loc (&rich_loc,
-				     "unsupported non-standard concatenation "
-				     "of string literals");
+		  error_at (&rich_loc,
+			    "unsupported non-standard concatenation "
+			    "of string literals");
 		}
 	    }
 
@@ -6160,9 +6163,9 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
 	    {
 	      gcc_rich_location richloc (token->location);
 	      richloc.add_fixit_replace ("::");
-	      error_at_rich_loc (&richloc,
-				 "found %<:%> in nested-name-specifier, "
-				 "expected %<::%>");
+	      error_at (&richloc,
+			"found %<:%> in nested-name-specifier, "
+			"expected %<::%>");
 	      token->type = CPP_SCOPE;
 	    }
 
@@ -9094,8 +9097,8 @@ cp_parser_cast_expression (cp_parser *parser, bool address_p, bool cast_p,
 		  gcc_rich_location rich_loc (input_location);
 		  maybe_add_cast_fixit (&rich_loc, open_paren_loc, close_paren_loc,
 					expr, type);
-		  warning_at_rich_loc (&rich_loc, OPT_Wold_style_cast,
-				       "use of old-style cast to %q#T", type);
+		  warning_at (&rich_loc, OPT_Wold_style_cast,
+			      "use of old-style cast to %q#T", type);
 		}
 
 	      /* Only type conversions to integral or enumeration types
@@ -10611,8 +10614,7 @@ cp_parser_lambda_declarator_opt (cp_parser* parser, tree lambda_expr)
 
     p = obstack_alloc (&declarator_obstack, 0);
 
-    declarator = make_id_declarator (NULL_TREE, cp_operator_id (CALL_EXPR),
-				     sfk_none);
+    declarator = make_id_declarator (NULL_TREE, call_op_identifier, sfk_none);
 
     quals = (LAMBDA_EXPR_MUTABLE_P (lambda_expr)
 	     ? TYPE_UNQUALIFIED : TYPE_QUAL_CONST);
@@ -13547,7 +13549,7 @@ cp_parser_decl_specifier_seq (cp_parser* parser,
 	    {
 	      gcc_rich_location richloc (token->location);
 	      richloc.add_fixit_remove ();
-	      error_at_rich_loc (&richloc, "%<friend%> used outside of class");
+	      error_at (&richloc, "%<friend%> used outside of class");
 	      cp_lexer_purge_token (parser->lexer);
 	    }
 	  else
@@ -13613,9 +13615,9 @@ cp_parser_decl_specifier_seq (cp_parser* parser,
 		 we're complaining about C++0x compatibility.  */
 	      gcc_rich_location richloc (token->location);
 	      richloc.add_fixit_remove ();
-	      warning_at_rich_loc (&richloc, OPT_Wc__11_compat,
-				   "%<auto%> changes meaning in C++11; "
-				   "please remove it");
+	      warning_at (&richloc, OPT_Wc__11_compat,
+			  "%<auto%> changes meaning in C++11; "
+			  "please remove it");
 
               /* Set the storage class anyway.  */
               cp_parser_set_storage_class (parser, decl_specs, RID_AUTO,
@@ -13848,9 +13850,11 @@ cp_parser_linkage_specification (cp_parser* parser)
   tree linkage;
 
   /* Look for the `extern' keyword.  */
-  cp_parser_require_keyword (parser, RID_EXTERN, RT_EXTERN);
+  cp_token *extern_token
+    = cp_parser_require_keyword (parser, RID_EXTERN, RT_EXTERN);
 
   /* Look for the string-literal.  */
+  cp_token *string_token = cp_lexer_peek_token (parser->lexer);
   linkage = cp_parser_string_literal (parser, false, false);
 
   /* Transform the literal into an identifier.  If the literal is a
@@ -13869,6 +13873,20 @@ cp_parser_linkage_specification (cp_parser* parser)
   /* We're now using the new linkage.  */
   push_lang_context (linkage);
 
+  /* Preserve the location of the the innermost linkage specification,
+     tracking the locations of nested specifications via a local.  */
+  location_t saved_location
+    = parser->innermost_linkage_specification_location;
+  /* Construct a location ranging from the start of the "extern" to
+     the end of the string-literal, with the caret at the start, e.g.:
+       extern "C" {
+       ^~~~~~~~~~
+  */
+  parser->innermost_linkage_specification_location
+    = make_location (extern_token->location,
+		     extern_token->location,
+		     get_finish (string_token->location));
+
   /* If the next token is a `{', then we're using the first
      production.  */
   if (cp_lexer_next_token_is (parser->lexer, CPP_OPEN_BRACE))
@@ -13899,6 +13917,9 @@ cp_parser_linkage_specification (cp_parser* parser)
 
   /* We're done with the linkage-specification.  */
   pop_lang_context ();
+
+  /* Restore location of parent linkage specification, if any.  */
+  parser->innermost_linkage_specification_location = saved_location;
 }
 
 /* Parse a static_assert-declaration.
@@ -14693,12 +14714,13 @@ cp_parser_operator (cp_parser* parser)
   location_t start_loc = token->location;
 
   /* Figure out which operator we have.  */
+  enum tree_code op = ERROR_MARK;
+  bool assop = false;
+  bool consumed = false;
   switch (token->type)
     {
     case CPP_KEYWORD:
       {
-	enum tree_code op;
-
 	/* The keyword should be either `new' or `delete'.  */
 	if (token->keyword == RID_NEW)
 	  op = NEW_EXPR;
@@ -14722,160 +14744,166 @@ cp_parser_operator (cp_parser* parser)
 	    if (cp_token *close_token
 		= cp_parser_require (parser, CPP_CLOSE_SQUARE, RT_CLOSE_SQUARE))
 	      end_loc = close_token->location;
-	    id = cp_operator_id (op == NEW_EXPR
-			      ? VEC_NEW_EXPR : VEC_DELETE_EXPR);
+	    op = op == NEW_EXPR ? VEC_NEW_EXPR : VEC_DELETE_EXPR;
 	  }
-	/* Otherwise, we have the non-array variant.  */
-	else
-	  id = cp_operator_id (op);
-
-	location_t loc = make_location (start_loc, start_loc, end_loc);
-
-	return cp_expr (id, loc);
+	start_loc = make_location (start_loc, start_loc, end_loc);
+	consumed = true;
+	break;
       }
 
     case CPP_PLUS:
-      id = cp_operator_id (PLUS_EXPR);
+      op = PLUS_EXPR;
       break;
 
     case CPP_MINUS:
-      id = cp_operator_id (MINUS_EXPR);
+      op = MINUS_EXPR;
       break;
 
     case CPP_MULT:
-      id = cp_operator_id (MULT_EXPR);
+      op = MULT_EXPR;
       break;
 
     case CPP_DIV:
-      id = cp_operator_id (TRUNC_DIV_EXPR);
+      op = TRUNC_DIV_EXPR;
       break;
 
     case CPP_MOD:
-      id = cp_operator_id (TRUNC_MOD_EXPR);
+      op = TRUNC_MOD_EXPR;
       break;
 
     case CPP_XOR:
-      id = cp_operator_id (BIT_XOR_EXPR);
+      op = BIT_XOR_EXPR;
       break;
 
     case CPP_AND:
-      id = cp_operator_id (BIT_AND_EXPR);
+      op = BIT_AND_EXPR;
       break;
 
     case CPP_OR:
-      id = cp_operator_id (BIT_IOR_EXPR);
+      op = BIT_IOR_EXPR;
       break;
 
     case CPP_COMPL:
-      id = cp_operator_id (BIT_NOT_EXPR);
+      op = BIT_NOT_EXPR;
       break;
 
     case CPP_NOT:
-      id = cp_operator_id (TRUTH_NOT_EXPR);
+      op = TRUTH_NOT_EXPR;
       break;
 
     case CPP_EQ:
-      id = cp_assignment_operator_id (NOP_EXPR);
+      assop = true;
+      op = NOP_EXPR;
       break;
 
     case CPP_LESS:
-      id = cp_operator_id (LT_EXPR);
+      op = LT_EXPR;
       break;
 
     case CPP_GREATER:
-      id = cp_operator_id (GT_EXPR);
+      op = GT_EXPR;
       break;
 
     case CPP_PLUS_EQ:
-      id = cp_assignment_operator_id (PLUS_EXPR);
+      assop = true;
+      op = PLUS_EXPR;
       break;
 
     case CPP_MINUS_EQ:
-      id = cp_assignment_operator_id (MINUS_EXPR);
+      assop = true;
+      op = MINUS_EXPR;
       break;
 
     case CPP_MULT_EQ:
-      id = cp_assignment_operator_id (MULT_EXPR);
+      assop = true;
+      op = MULT_EXPR;
       break;
 
     case CPP_DIV_EQ:
-      id = cp_assignment_operator_id (TRUNC_DIV_EXPR);
+      assop = true;
+      op = TRUNC_DIV_EXPR;
       break;
 
     case CPP_MOD_EQ:
-      id = cp_assignment_operator_id (TRUNC_MOD_EXPR);
+      assop = true;
+      op = TRUNC_MOD_EXPR;
       break;
 
     case CPP_XOR_EQ:
-      id = cp_assignment_operator_id (BIT_XOR_EXPR);
+      assop = true;
+      op = BIT_XOR_EXPR;
       break;
 
     case CPP_AND_EQ:
-      id = cp_assignment_operator_id (BIT_AND_EXPR);
+      assop = true;
+      op = BIT_AND_EXPR;
       break;
 
     case CPP_OR_EQ:
-      id = cp_assignment_operator_id (BIT_IOR_EXPR);
+      assop = true;
+      op = BIT_IOR_EXPR;
       break;
 
     case CPP_LSHIFT:
-      id = cp_operator_id (LSHIFT_EXPR);
+      op = LSHIFT_EXPR;
       break;
 
     case CPP_RSHIFT:
-      id = cp_operator_id (RSHIFT_EXPR);
+      op = RSHIFT_EXPR;
       break;
 
     case CPP_LSHIFT_EQ:
-      id = cp_assignment_operator_id (LSHIFT_EXPR);
+      assop = true;
+      op = LSHIFT_EXPR;
       break;
 
     case CPP_RSHIFT_EQ:
-      id = cp_assignment_operator_id (RSHIFT_EXPR);
+      assop = true;
+      op = RSHIFT_EXPR;
       break;
 
     case CPP_EQ_EQ:
-      id = cp_operator_id (EQ_EXPR);
+      op = EQ_EXPR;
       break;
 
     case CPP_NOT_EQ:
-      id = cp_operator_id (NE_EXPR);
+      op = NE_EXPR;
       break;
 
     case CPP_LESS_EQ:
-      id = cp_operator_id (LE_EXPR);
+      op = LE_EXPR;
       break;
 
     case CPP_GREATER_EQ:
-      id = cp_operator_id (GE_EXPR);
+      op = GE_EXPR;
       break;
 
     case CPP_AND_AND:
-      id = cp_operator_id (TRUTH_ANDIF_EXPR);
+      op = TRUTH_ANDIF_EXPR;
       break;
 
     case CPP_OR_OR:
-      id = cp_operator_id (TRUTH_ORIF_EXPR);
+      op = TRUTH_ORIF_EXPR;
       break;
 
     case CPP_PLUS_PLUS:
-      id = cp_operator_id (POSTINCREMENT_EXPR);
+      op = POSTINCREMENT_EXPR;
       break;
 
     case CPP_MINUS_MINUS:
-      id = cp_operator_id (PREDECREMENT_EXPR);
+      op = PREDECREMENT_EXPR;
       break;
 
     case CPP_COMMA:
-      id = cp_operator_id (COMPOUND_EXPR);
+      op = COMPOUND_EXPR;
       break;
 
     case CPP_DEREF_STAR:
-      id = cp_operator_id (MEMBER_REF);
+      op = MEMBER_REF;
       break;
 
     case CPP_DEREF:
-      id = cp_operator_id (COMPONENT_REF);
+      op = COMPONENT_REF;
       break;
 
     case CPP_OPEN_PAREN:
@@ -14885,7 +14913,9 @@ cp_parser_operator (cp_parser* parser)
         parens.consume_open (parser);
         /* Look for the matching `)'.  */
         parens.require_close (parser);
-        return cp_operator_id (CALL_EXPR);
+	op = CALL_EXPR;
+	consumed = true;
+	break;
       }
 
     case CPP_OPEN_SQUARE:
@@ -14893,7 +14923,9 @@ cp_parser_operator (cp_parser* parser)
       cp_lexer_consume_token (parser->lexer);
       /* Look for the matching `]'.  */
       cp_parser_require (parser, CPP_CLOSE_SQUARE, RT_CLOSE_SQUARE);
-      return cp_operator_id (ARRAY_REF);
+      op = ARRAY_REF;
+      consumed = true;
+      break;
 
     case CPP_UTF8STRING:
     case CPP_UTF8STRING_USERDEF:
@@ -14972,8 +15004,12 @@ cp_parser_operator (cp_parser* parser)
 
   /* If we have selected an identifier, we need to consume the
      operator token.  */
-  if (id)
-    cp_lexer_consume_token (parser->lexer);
+  if (op != ERROR_MARK)
+    {
+      id = ovl_op_identifier (assop, op);
+      if (!consumed)
+	cp_lexer_consume_token (parser->lexer);
+    }
   /* Otherwise, no valid operator name was present.  */
   else
     {
@@ -16643,6 +16679,7 @@ cp_parser_explicit_specialization (cp_parser* parser)
   if (current_lang_name == lang_name_c)
     {
       error_at (token->location, "template specialization with C linkage");
+      maybe_show_extern_c_location ();
       /* Give it C++ linkage to avoid confusing other parts of the
 	 front end.  */
       push_lang_context (lang_name_cplusplus);
@@ -17624,9 +17661,9 @@ cp_parser_elaborated_type_specifier (cp_parser* parser,
 	  gcc_rich_location richloc (token->location);
 	  richloc.add_range (input_location, false);
 	  richloc.add_fixit_remove ();
-	  pedwarn_at_rich_loc (&richloc, 0, "elaborated-type-specifier for "
-			       "a scoped enum must not use the %qD keyword",
-			       token->u.value);
+	  pedwarn (&richloc, 0, "elaborated-type-specifier for "
+		   "a scoped enum must not use the %qD keyword",
+		   token->u.value);
 	  /* Consume the `struct' or `class' and parse it anyway.  */
 	  cp_lexer_consume_token (parser->lexer);
 	}
@@ -20622,7 +20659,7 @@ cp_parser_cv_qualifier_seq_opt (cp_parser* parser)
 	{
 	  gcc_rich_location richloc (token->location);
 	  richloc.add_fixit_remove ();
-	  error_at_rich_loc (&richloc, "duplicate cv-qualifier");
+	  error_at (&richloc, "duplicate cv-qualifier");
 	  cp_lexer_purge_token (parser->lexer);
 	}
       else
@@ -20771,7 +20808,7 @@ cp_parser_virt_specifier_seq_opt (cp_parser* parser)
 	{
 	  gcc_rich_location richloc (token->location);
 	  richloc.add_fixit_remove ();
-	  error_at_rich_loc (&richloc, "duplicate virt-specifier");
+	  error_at (&richloc, "duplicate virt-specifier");
 	  cp_lexer_purge_token (parser->lexer);
 	}
       else
@@ -22569,14 +22606,14 @@ cp_parser_class_specifier_1 (cp_parser* parser)
 	  richloc.add_fixit_insert_before (next_loc, ";");
 
 	if (CLASSTYPE_DECLARED_CLASS (type))
-	  error_at_rich_loc (&richloc,
-			     "expected %<;%> after class definition");
+	  error_at (&richloc,
+		    "expected %<;%> after class definition");
 	else if (TREE_CODE (type) == RECORD_TYPE)
-	  error_at_rich_loc (&richloc,
-			     "expected %<;%> after struct definition");
+	  error_at (&richloc,
+		    "expected %<;%> after struct definition");
 	else if (TREE_CODE (type) == UNION_TYPE)
-	  error_at_rich_loc (&richloc,
-			     "expected %<;%> after union definition");
+	  error_at (&richloc,
+		    "expected %<;%> after union definition");
 	else
 	  gcc_unreachable ();
 
@@ -23023,9 +23060,9 @@ cp_parser_class_head (cp_parser* parser,
       rich_location richloc (line_table, reported_loc);
       richloc.add_fixit_insert_before (class_head_start_location,
                                        "template <> ");
-      error_at_rich_loc
-        (&richloc,
-         "an explicit specialization must be preceded by %<template <>%>");
+      error_at (&richloc,
+		"an explicit specialization must be preceded by"
+		" %<template <>%>");
       invalid_explicit_specialization_p = true;
       /* Take the same action that would have been taken by
 	 cp_parser_explicit_specialization.  */
@@ -23493,7 +23530,7 @@ cp_parser_member_declaration (cp_parser* parser)
 	    {
 	      gcc_rich_location richloc (token->location);
 	      richloc.add_fixit_remove ();
-	      pedwarn_at_rich_loc (&richloc, OPT_Wpedantic, "extra %<;%>");
+	      pedwarn (&richloc, OPT_Wpedantic, "extra %<;%>");
 	    }
 	}
       else
@@ -23836,9 +23873,9 @@ cp_parser_member_declaration (cp_parser* parser)
 			= cp_lexer_consume_token (parser->lexer)->location;
 		      gcc_rich_location richloc (semicolon_loc);
 		      richloc.add_fixit_remove ();
-		      warning_at_rich_loc (&richloc, OPT_Wextra_semi,
-					   "extra %<;%> after in-class "
-					   "function definition");
+		      warning_at (&richloc, OPT_Wextra_semi,
+				  "extra %<;%> after in-class "
+				  "function definition");
 		    }
 		  goto out;
 		}
@@ -23881,8 +23918,8 @@ cp_parser_member_declaration (cp_parser* parser)
 		  cp_token *token = cp_lexer_previous_token (parser->lexer);
 		  gcc_rich_location richloc (token->location);
 		  richloc.add_fixit_remove ();
-		  error_at_rich_loc (&richloc, "stray %<,%> at end of "
-				     "member declaration");
+		  error_at (&richloc, "stray %<,%> at end of "
+			    "member declaration");
 		}
 	    }
 	  /* If the next token isn't a `;', then we have a parse error.  */
@@ -23895,8 +23932,8 @@ cp_parser_member_declaration (cp_parser* parser)
 	      cp_token *token = cp_lexer_previous_token (parser->lexer);
 	      gcc_rich_location richloc (token->location);
 	      richloc.add_fixit_insert_after (";");
-	      error_at_rich_loc (&richloc, "expected %<;%> at end of "
-				 "member declaration");
+	      error_at (&richloc, "expected %<;%> at end of "
+			"member declaration");
 
 	      /* Assume that the user meant to provide a semicolon.  If
 		 we were to cp_parser_skip_to_end_of_statement, we might
@@ -26979,6 +27016,7 @@ cp_parser_explicit_template_declaration (cp_parser* parser, bool member_p)
   if (current_lang_name == lang_name_c)
     {
       error_at (location, "template with C linkage");
+      maybe_show_extern_c_location ();
       /* Give it C++ linkage to avoid confusing other parts of the
          front end.  */
       push_lang_context (lang_name_cplusplus);
@@ -27506,8 +27544,8 @@ cp_parser_enclosed_template_argument_list (cp_parser* parser)
 	  cp_token *token = cp_lexer_peek_token (parser->lexer);
 	  gcc_rich_location richloc (token->location);
 	  richloc.add_fixit_replace ("> >");
-	  error_at_rich_loc (&richloc, "%<>>%> should be %<> >%> "
-			     "within a nested template argument list");
+	  error_at (&richloc, "%<>>%> should be %<> >%> "
+		    "within a nested template argument list");
 
 	  token->type = CPP_GREATER;
 	}
@@ -28136,7 +28174,7 @@ set_and_check_decl_spec_loc (cp_decl_specifier_seq *decl_specs,
 	    {
 	      gcc_rich_location richloc (location);
 	      richloc.add_fixit_remove ();
-	      error_at_rich_loc (&richloc, "duplicate %qD", token->u.value);
+	      error_at (&richloc, "duplicate %qD", token->u.value);
 	    }
 	}
       else
@@ -28160,7 +28198,7 @@ set_and_check_decl_spec_loc (cp_decl_specifier_seq *decl_specs,
 	  };
 	  gcc_rich_location richloc (location);
 	  richloc.add_fixit_remove ();
-	  error_at_rich_loc (&richloc, "duplicate %qs", decl_spec_names[ds]);
+	  error_at (&richloc, "duplicate %qs", decl_spec_names[ds]);
 	}
     }
 }
@@ -32550,21 +32588,21 @@ cp_parser_omp_clause_reduction (cp_parser *parser, tree list)
 	    code = MIN_EXPR;
 	  else if (strcmp (p, "max") == 0)
 	    code = MAX_EXPR;
-	  else if (id == cp_operator_id (PLUS_EXPR))
+	  else if (id == ovl_op_identifier (false, PLUS_EXPR))
 	    code = PLUS_EXPR;
-	  else if (id == cp_operator_id (MULT_EXPR))
+	  else if (id == ovl_op_identifier (false, MULT_EXPR))
 	    code = MULT_EXPR;
-	  else if (id == cp_operator_id (MINUS_EXPR))
+	  else if (id == ovl_op_identifier (false, MINUS_EXPR))
 	    code = MINUS_EXPR;
-	  else if (id == cp_operator_id (BIT_AND_EXPR))
+	  else if (id == ovl_op_identifier (false, BIT_AND_EXPR))
 	    code = BIT_AND_EXPR;
-	  else if (id == cp_operator_id (BIT_IOR_EXPR))
+	  else if (id == ovl_op_identifier (false, BIT_IOR_EXPR))
 	    code = BIT_IOR_EXPR;
-	  else if (id == cp_operator_id (BIT_XOR_EXPR))
+	  else if (id == ovl_op_identifier (false, BIT_XOR_EXPR))
 	    code = BIT_XOR_EXPR;
-	  else if (id == cp_operator_id (TRUTH_ANDIF_EXPR))
+	  else if (id == ovl_op_identifier (false, TRUTH_ANDIF_EXPR))
 	    code = TRUTH_ANDIF_EXPR;
-	  else if (id == cp_operator_id (TRUTH_ORIF_EXPR))
+	  else if (id == ovl_op_identifier (false, TRUTH_ORIF_EXPR))
 	    code = TRUTH_ORIF_EXPR;
 	  id = omp_reduction_id (code, id, NULL_TREE);
 	  tree scope = parser->scope;
@@ -39552,4 +39590,17 @@ finish_fully_implicit_template (cp_parser *parser, tree member_decl_opt)
   return member_decl_opt;
 }
 
+/* Helper function for diagnostics that have complained about things
+   being used with 'extern "C"' linkage.
+
+   Attempt to issue a note showing where the 'extern "C"' linkage began.  */
+
+void
+maybe_show_extern_c_location (void)
+{
+  if (the_parser->innermost_linkage_specification_location != UNKNOWN_LOCATION)
+    inform (the_parser->innermost_linkage_specification_location,
+	    "%<extern \"C\"%> linkage started here");
+}
+
 #include "gt-cp-parser.h"
diff --git a/gcc/cp/parser.h b/gcc/cp/parser.h
index 0994e1e7f4f..f4f4a010964 100644
--- a/gcc/cp/parser.h
+++ b/gcc/cp/parser.h
@@ -412,6 +412,10 @@ struct GTY(()) cp_parser {
      context e.g., because they could never be deduced.  */
   int prevent_constrained_type_specifiers;
 
+  /* Location of the string-literal token within the current linkage
+     specification, if any, or UNKNOWN_LOCATION otherwise.  */
+  location_t innermost_linkage_specification_location;
+
 };
 
 /* In parser.c  */
diff --git a/gcc/cp/pt.c b/gcc/cp/pt.c
index ba52f3b57a6..710333ddaba 100644
--- a/gcc/cp/pt.c
+++ b/gcc/cp/pt.c
@@ -203,7 +203,7 @@ static void tsubst_default_arguments (tree, tsubst_flags_t);
 static tree for_each_template_parm_r (tree *, int *, void *);
 static tree copy_default_args_to_explicit_spec_1 (tree, tree);
 static void copy_default_args_to_explicit_spec (tree);
-static int invalid_nontype_parm_type_p (tree, tsubst_flags_t);
+static bool invalid_nontype_parm_type_p (tree, tsubst_flags_t);
 static bool dependent_template_arg_p (tree);
 static bool any_template_arguments_need_structural_equality_p (tree);
 static bool dependent_type_p_r (tree);
@@ -3435,7 +3435,7 @@ expand_integer_pack (tree call, tree args, tsubst_flags_t complain,
 	  call = copy_node (call);
 	  CALL_EXPR_ARG (call, 0) = hi;
 	}
-      tree ex = make_pack_expansion (call);
+      tree ex = make_pack_expansion (call, complain);
       tree vec = make_tree_vec (1);
       TREE_VEC_ELT (vec, 0) = ex;
       return vec;
@@ -3724,7 +3724,7 @@ uses_parameter_packs (tree t)
    EXPR_PACK_EXPANSION, TYPE_PACK_EXPANSION, or TREE_LIST,
    respectively.  */
 tree 
-make_pack_expansion (tree arg)
+make_pack_expansion (tree arg, tsubst_flags_t complain)
 {
   tree result;
   tree parameter_packs = NULL_TREE;
@@ -3770,7 +3770,9 @@ make_pack_expansion (tree arg)
 
       if (parameter_packs == NULL_TREE)
         {
-          error ("base initializer expansion %qT contains no parameter packs", arg);
+	  if (complain & tf_error)
+	    error ("base initializer expansion %qT contains no parameter packs",
+		   arg);
           delete ppd.visited;
           return error_mark_node;
         }
@@ -3834,10 +3836,13 @@ make_pack_expansion (tree arg)
   /* Make sure we found some parameter packs.  */
   if (parameter_packs == NULL_TREE)
     {
-      if (TYPE_P (arg))
-        error ("expansion pattern %qT contains no argument packs", arg);
-      else
-        error ("expansion pattern %qE contains no argument packs", arg);
+      if (complain & tf_error)
+	{
+	  if (TYPE_P (arg))
+	    error ("expansion pattern %qT contains no argument packs", arg);
+	  else
+	    error ("expansion pattern %qE contains no argument packs", arg);
+	}
       return error_mark_node;
     }
   PACK_EXPANSION_PARAMETER_PACKS (result) = parameter_packs;
@@ -5564,7 +5569,7 @@ push_template_decl_real (tree decl, bool is_friend)
 	  (TI_ARGS (tinfo),
 	   TI_ARGS (get_template_info (DECL_TEMPLATE_RESULT (tmpl)))))
 	{
-	  error ("template arguments to %qD do not match original"
+	  error ("template arguments to %qD do not match original "
 		 "template %qD", decl, DECL_TEMPLATE_RESULT (tmpl));
 	  if (!uses_template_parms (TI_ARGS (tinfo)))
 	    inform (input_location, "use %<template<>%> for"
@@ -7694,7 +7699,7 @@ convert_template_argument (tree parm,
                       if (DECL_TEMPLATE_TEMPLATE_PARM_P (val))
                         val = TREE_TYPE (val);
 		      if (TREE_CODE (orig_arg) == TYPE_PACK_EXPANSION)
-			val = make_pack_expansion (val);
+			val = make_pack_expansion (val, complain);
                     }
 		}
 	      else
@@ -8188,7 +8193,7 @@ coerce_template_parms (tree parms,
 	      else if (TYPE_P (conv) && !TYPE_P (pattern))
 		/* Recover from missing typename.  */
 		TREE_VEC_ELT (inner_args, arg_idx)
-		  = make_pack_expansion (conv);
+		  = make_pack_expansion (conv, complain);
 
               /* We don't know how many args we have yet, just
                  use the unconverted ones for now.  */
@@ -11161,7 +11166,7 @@ gen_elem_of_pack_expansion_instantiation (tree pattern,
       the Ith element resulting from the substituting is going to
       be a pack expansion as well.  */
   if (ith_elem_is_expansion)
-    t = make_pack_expansion (t);
+    t = make_pack_expansion (t, complain);
 
   return t;
 }
@@ -11573,7 +11578,7 @@ tsubst_pack_expansion (tree t, tree args, tsubst_flags_t complain,
       /* We got some full packs, but we can't substitute them in until we
 	 have values for all the packs.  So remember these until then.  */
 
-      t = make_pack_expansion (pattern);
+      t = make_pack_expansion (pattern, complain);
       PACK_EXPANSION_EXTRA_ARGS (t) = args;
       return t;
     }
@@ -11588,7 +11593,7 @@ tsubst_pack_expansion (tree t, tree args, tsubst_flags_t complain,
 			 /*integral_constant_expression_p=*/false);
       else
 	t = tsubst (pattern, args, complain, in_decl);
-      t = make_pack_expansion (t);
+      t = make_pack_expansion (t, complain);
       return t;
     }
 
@@ -12195,7 +12200,7 @@ tsubst_function_decl (tree t, tree args, tsubst_flags_t complain,
 	 We also deal with the peculiar case:
 
 	 template <class T> struct S {
-	 template <class U> friend void f();
+	   template <class U> friend void f();
 	 };
 	 template <class U> void f() {}
 	 template S<int>;
@@ -17079,8 +17084,7 @@ tsubst_copy_and_build (tree t,
 	    /* A type conversion to reference type will be enclosed in
 	       such an indirect ref, but the substitution of the cast
 	       will have also added such an indirect ref.  */
-	    if (TREE_CODE (TREE_TYPE (r)) == REFERENCE_TYPE)
-	      r = convert_from_reference (r);
+	    r = convert_from_reference (r);
 	  }
 	else
 	  r = build_x_indirect_ref (input_location, r, RO_UNARY_STAR,
@@ -21324,7 +21328,7 @@ unify (tree tparms, tree targs, tree parm, tree arg, int strict,
 	  if (REFERENCE_REF_P (arg))
 	    arg = TREE_OPERAND (arg, 0);
 	  if (pexp)
-	    arg = make_pack_expansion (arg);
+	    arg = make_pack_expansion (arg, complain);
 	  return unify (tparms, targs, TREE_OPERAND (parm, 0), arg,
 			strict, explain_p);
 	}
@@ -23618,31 +23622,31 @@ instantiating_current_function_p (void)
 }
 
 /* [temp.param] Check that template non-type parm TYPE is of an allowable
-   type. Return zero for ok, nonzero for disallowed. Issue error and
-   warning messages under control of COMPLAIN.  */
+   type.  Return false for ok, true for disallowed.  Issue error and
+   inform messages under control of COMPLAIN.  */
 
-static int
+static bool
 invalid_nontype_parm_type_p (tree type, tsubst_flags_t complain)
 {
   if (INTEGRAL_OR_ENUMERATION_TYPE_P (type))
-    return 0;
+    return false;
   else if (POINTER_TYPE_P (type))
-    return 0;
+    return false;
   else if (TYPE_PTRMEM_P (type))
-    return 0;
+    return false;
   else if (TREE_CODE (type) == TEMPLATE_TYPE_PARM)
-    return 0;
+    return false;
   else if (TREE_CODE (type) == TYPENAME_TYPE)
-    return 0;
+    return false;
   else if (TREE_CODE (type) == DECLTYPE_TYPE)
-    return 0;
+    return false;
   else if (TREE_CODE (type) == NULLPTR_TYPE)
-    return 0;
+    return false;
   /* A bound template template parm could later be instantiated to have a valid
      nontype parm type via an alias template.  */
   else if (cxx_dialect >= cxx11
 	   && TREE_CODE (type) == BOUND_TEMPLATE_TEMPLATE_PARM)
-    return 0;
+    return false;
 
   if (complain & tf_error)
     {
@@ -23652,7 +23656,7 @@ invalid_nontype_parm_type_p (tree type, tsubst_flags_t complain)
 	error ("%q#T is not a valid type for a template non-type parameter",
 	       type);
     }
-  return 1;
+  return true;
 }
 
 /* Returns TRUE if TYPE is dependent, in the sense of [temp.dep.type].
@@ -24019,8 +24023,21 @@ value_dependent_expression_p (tree expression)
     case TRAIT_EXPR:
       {
 	tree type2 = TRAIT_EXPR_TYPE2 (expression);
-	return (dependent_type_p (TRAIT_EXPR_TYPE1 (expression))
-		|| (type2 ? dependent_type_p (type2) : false));
+
+	if (dependent_type_p (TRAIT_EXPR_TYPE1 (expression)))
+	  return true;
+
+	if (!type2)
+	  return false;
+
+	if (TREE_CODE (type2) != TREE_LIST)
+	  return dependent_type_p (type2);
+
+	for (; type2; type2 = TREE_CHAIN (type2))
+	  if (dependent_type_p (TREE_VALUE (type2)))
+	    return true;
+
+	return false;
       }
 
     case MODOP_EXPR:
@@ -25118,9 +25135,9 @@ listify (tree arg)
     {    
       gcc_rich_location richloc (input_location);
       maybe_add_include_fixit (&richloc, "<initializer_list>");
-      error_at_rich_loc (&richloc,
-                         "deducing from brace-enclosed initializer list"
-                         " requires #include <initializer_list>");
+      error_at (&richloc,
+		"deducing from brace-enclosed initializer list"
+		" requires %<#include <initializer_list>%>");
 
       return error_mark_node;
     }
diff --git a/gcc/cp/ptree.c b/gcc/cp/ptree.c
index 50c717e286e..90bae2a7039 100644
--- a/gcc/cp/ptree.c
+++ b/gcc/cp/ptree.c
@@ -177,7 +177,6 @@ cxx_print_identifier (FILE *file, tree node, int indent)
     indent_to (file, indent + 4);
   fprintf (file, "%s local bindings <%p>", get_identifier_kind_name (node),
 	   (void *) IDENTIFIER_BINDING (node));
-  print_node (file, "label", IDENTIFIER_LABEL_VALUE (node), indent + 4);
 }
 
 void
diff --git a/gcc/cp/rtti.c b/gcc/cp/rtti.c
index 5b2326cbbb6..10ecbfd9589 100644
--- a/gcc/cp/rtti.c
+++ b/gcc/cp/rtti.c
@@ -319,9 +319,9 @@ typeid_ok_p (void)
     {
       gcc_rich_location richloc (input_location);
       maybe_add_include_fixit (&richloc, "<typeinfo>");
-      error_at_rich_loc (&richloc,
-			 "must %<#include <typeinfo>%> before using"
-			 " %<typeid%>");
+      error_at (&richloc,
+		"must %<#include <typeinfo>%> before using"
+		" %<typeid%>");
 
       return false;
     }
diff --git a/gcc/cp/semantics.c b/gcc/cp/semantics.c
index a512664e396..664952e749c 100644
--- a/gcc/cp/semantics.c
+++ b/gcc/cp/semantics.c
@@ -2711,8 +2711,12 @@ finish_compound_literal (tree type, tree compound_literal,
 
   if (tree anode = type_uses_auto (type))
     if (CLASS_PLACEHOLDER_TEMPLATE (anode))
-      type = do_auto_deduction (type, compound_literal, anode, complain,
-				adc_variable_type);
+      {
+	type = do_auto_deduction (type, compound_literal, anode, complain,
+				  adc_variable_type);
+	if (type == error_mark_node)
+	  return error_mark_node;
+      }
 
   if (processing_template_decl)
     {
@@ -3395,7 +3399,7 @@ process_outer_var_ref (tree decl, tsubst_flags_t complain, bool force_use)
 	    inform (location_of (closure),
 		    "the lambda has no capture-default");
 	  else if (TYPE_CLASS_SCOPE_P (closure))
-	    inform (0, "lambda in local class %q+T cannot "
+	    inform (UNKNOWN_LOCATION, "lambda in local class %q+T cannot "
 		    "capture variables from the enclosing context",
 		    TYPE_CONTEXT (closure));
 	  inform (DECL_SOURCE_LOCATION (decl), "%q#D declared here", decl);
@@ -5095,7 +5099,7 @@ omp_reduction_id (enum tree_code reduction_code, tree reduction_id, tree type)
     case BIT_IOR_EXPR:
     case TRUTH_ANDIF_EXPR:
     case TRUTH_ORIF_EXPR:
-      reduction_id = cp_operator_id (reduction_code);
+      reduction_id = ovl_op_identifier (false, reduction_code);
       break;
     case MIN_EXPR:
       p = "min";
@@ -9016,8 +9020,7 @@ classtype_has_nothrow_assign_or_copy_p (tree type, bool assign_p)
   tree fns = NULL_TREE;
 
   if (assign_p || TYPE_HAS_COPY_CTOR (type))
-    fns = get_class_binding (type,
-			     assign_p ? cp_assignment_operator_id (NOP_EXPR)
+    fns = get_class_binding (type, assign_p ? assign_op_identifier
 			     : ctor_identifier);
 
   bool saw_copy = false;
diff --git a/gcc/cp/tree.c b/gcc/cp/tree.c
index e21ff6a1572..b63f2ae4c5d 100644
--- a/gcc/cp/tree.c
+++ b/gcc/cp/tree.c
@@ -333,6 +333,10 @@ cp_stabilize_reference (tree ref)
 {
   switch (TREE_CODE (ref))
     {
+    case NON_DEPENDENT_EXPR:
+      /* We aren't actually evaluating this.  */
+      return ref;
+
     /* We need to treat specially anything stabilize_reference doesn't
        handle specifically.  */
     case VAR_DECL:
@@ -1204,7 +1208,7 @@ cp_build_qualified_type_real (tree type,
       tree t = PACK_EXPANSION_PATTERN (type);
 
       t = cp_build_qualified_type_real (t, type_quals, complain);
-      return make_pack_expansion (t);
+      return make_pack_expansion (t, complain);
     }
 
   /* A reference or method type shall not be cv-qualified.
@@ -1435,7 +1439,11 @@ strip_typedefs (tree t, bool *remove_attributes)
 	  is_variant = true;
 
 	type = strip_typedefs (TREE_TYPE (t), remove_attributes);
-	changed = type != TREE_TYPE (t) || is_variant;
+	tree canon_spec = (flag_noexcept_type
+			   ? canonical_eh_spec (TYPE_RAISES_EXCEPTIONS (t))
+			   : NULL_TREE);
+	changed = (type != TREE_TYPE (t) || is_variant
+		   || TYPE_RAISES_EXCEPTIONS (t) != canon_spec);
 
 	for (arg_node = TYPE_ARG_TYPES (t);
 	     arg_node;
@@ -1494,9 +1502,8 @@ strip_typedefs (tree t, bool *remove_attributes)
 					type_memfn_rqual (t));
 	  }
 
-	if (TYPE_RAISES_EXCEPTIONS (t))
-	  result = build_exception_variant (result,
-					    TYPE_RAISES_EXCEPTIONS (t));
+	if (canon_spec)
+	  result = build_exception_variant (result, canon_spec);
 	if (TYPE_HAS_LATE_RETURN_TYPE (t))
 	  TYPE_HAS_LATE_RETURN_TYPE (result) = 1;
       }
@@ -4840,7 +4847,8 @@ special_function_p (const_tree decl)
     return sfk_move_constructor;
   if (DECL_CONSTRUCTOR_P (decl))
     return sfk_constructor;
-  if (DECL_OVERLOADED_OPERATOR_P (decl) == NOP_EXPR)
+  if (DECL_ASSIGNMENT_OPERATOR_P (decl)
+      && DECL_OVERLOADED_OPERATOR_IS (decl, NOP_EXPR))
     {
       if (copy_fn_p (decl))
 	return sfk_copy_assignment;
diff --git a/gcc/cp/typeck.c b/gcc/cp/typeck.c
index d87ee62ad1a..7db8719d50d 100644
--- a/gcc/cp/typeck.c
+++ b/gcc/cp/typeck.c
@@ -1562,7 +1562,7 @@ cxx_sizeof_or_alignof_type (tree type, enum tree_code op, bool complain)
       if (complain)
 	pedwarn (input_location, OPT_Wpointer_arith, 
 		 "invalid application of %qs to a member function", 
-		 operator_name_info[(int) op].name);
+		 OVL_OP_INFO (false, op)->name);
       else
 	return error_mark_node;
       value = size_one_node;
@@ -2677,8 +2677,8 @@ access_failure_info::maybe_suggest_accessor (bool const_p) const
   pretty_printer pp;
   pp_printf (&pp, "%s()", IDENTIFIER_POINTER (DECL_NAME (accessor)));
   richloc.add_fixit_replace (pp_formatted_text (&pp));
-  inform_at_rich_loc (&richloc, "field %q#D can be accessed via %q#D",
-		      m_field_decl, accessor);
+  inform (&richloc, "field %q#D can be accessed via %q#D",
+	  m_field_decl, accessor);
 }
 
 /* This function is called by the parser to process a class member
@@ -2883,12 +2883,12 @@ finish_class_member_access_expr (cp_expr object, tree name, bool template_p,
 		      gcc_rich_location rich_loc (bogus_component_loc);
 		      rich_loc.add_fixit_misspelled_id (bogus_component_loc,
 							guessed_id);
-		      error_at_rich_loc
-			(&rich_loc,
-			 "%q#T has no member named %qE; did you mean %qE?",
-			 TREE_CODE (access_path) == TREE_BINFO
-			 ? TREE_TYPE (access_path) : object_type, name,
-			 guessed_id);
+		      error_at (&rich_loc,
+				"%q#T has no member named %qE;"
+				" did you mean %qE?",
+				TREE_CODE (access_path) == TREE_BINFO
+				? TREE_TYPE (access_path) : object_type,
+				name, guessed_id);
 		    }
 		  else
 		    error ("%q#T has no member named %qE",
@@ -9048,10 +9048,11 @@ check_return_expr (tree retval, bool *no_warning)
 	/* You can return a `void' value from a function of `void'
 	   type.  In that case, we have to evaluate the expression for
 	   its side-effects.  */
-	  finish_expr_stmt (retval);
+	finish_expr_stmt (retval);
       else
-	permerror (input_location, "return-statement with a value, in function "
-		   "returning 'void'");
+	permerror (input_location,
+		   "return-statement with a value, in function "
+		   "returning %qT", valtype);
       current_function_returns_null = 1;
 
       /* There's really no value to return, after all.  */
@@ -9075,8 +9076,7 @@ check_return_expr (tree retval, bool *no_warning)
     }
 
   /* Only operator new(...) throw(), can return NULL [expr.new/13].  */
-  if ((DECL_OVERLOADED_OPERATOR_P (current_function_decl) == NEW_EXPR
-       || DECL_OVERLOADED_OPERATOR_P (current_function_decl) == VEC_NEW_EXPR)
+  if (IDENTIFIER_NEW_OP_P (DECL_NAME (current_function_decl))
       && !TYPE_NOTHROW_P (TREE_TYPE (current_function_decl))
       && ! flag_check_new
       && retval && null_ptr_cst_p (retval))
@@ -9085,7 +9085,7 @@ check_return_expr (tree retval, bool *no_warning)
 
   /* Effective C++ rule 15.  See also start_function.  */
   if (warn_ecpp
-      && DECL_NAME (current_function_decl) == cp_assignment_operator_id (NOP_EXPR))
+      && DECL_NAME (current_function_decl) == assign_op_identifier)
     {
       bool warn = true;
 
@@ -9231,7 +9231,8 @@ check_return_expr (tree retval, bool *no_warning)
 	       && TREE_CODE (TREE_OPERAND (retval, 1)) == AGGR_INIT_EXPR)
 	retval = build2 (COMPOUND_EXPR, TREE_TYPE (retval), retval,
 			 TREE_OPERAND (retval, 0));
-      else if (maybe_warn_about_returning_address_of_local (retval))
+      else if (!processing_template_decl
+	       && maybe_warn_about_returning_address_of_local (retval))
 	retval = build2 (COMPOUND_EXPR, TREE_TYPE (retval), retval,
 			 build_zero_cst (TREE_TYPE (retval)));
     }
diff --git a/gcc/dbxout.c b/gcc/dbxout.c
index 18e16658227..5a2bbfaedbc 100644
--- a/gcc/dbxout.c
+++ b/gcc/dbxout.c
@@ -2486,7 +2486,7 @@ dbxout_expand_expr (tree expr)
 	      return NULL;
 	    x = adjust_address_nv (x, mode, tree_to_shwi (offset));
 	  }
-	if (maybe_nonzero (bitpos))
+	if (may_ne (bitpos, 0))
 	  x = adjust_address_nv (x, mode, bits_to_bytes_round_down (bitpos));
 
 	return x;
diff --git a/gcc/debug.h b/gcc/debug.h
index 915420baded..19b27848ca8 100644
--- a/gcc/debug.h
+++ b/gcc/debug.h
@@ -228,7 +228,6 @@ extern void debug_nothing_tree_charstar_uhwi (tree, const char *,
 /* Hooks for various debug formats.  */
 extern const struct gcc_debug_hooks do_nothing_debug_hooks;
 extern const struct gcc_debug_hooks dbx_debug_hooks;
-extern const struct gcc_debug_hooks sdb_debug_hooks;
 extern const struct gcc_debug_hooks xcoff_debug_hooks;
 extern const struct gcc_debug_hooks dwarf2_debug_hooks;
 extern const struct gcc_debug_hooks dwarf2_lineno_debug_hooks;
diff --git a/gcc/defaults.h b/gcc/defaults.h
index 99cd9db5191..768c9879df9 100644
--- a/gcc/defaults.h
+++ b/gcc/defaults.h
@@ -894,14 +894,10 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define DEFAULT_GDB_EXTENSIONS 1
 #endif
 
-#ifndef SDB_DEBUGGING_INFO
-#define SDB_DEBUGGING_INFO 0
-#endif
-
 /* If more than one debugging type is supported, you must define
    PREFERRED_DEBUGGING_TYPE to choose the default.  */
 
-#if 1 < (defined (DBX_DEBUGGING_INFO) + (SDB_DEBUGGING_INFO) \
+#if 1 < (defined (DBX_DEBUGGING_INFO) \
          + defined (DWARF2_DEBUGGING_INFO) + defined (XCOFF_DEBUGGING_INFO) \
          + defined (VMS_DEBUGGING_INFO))
 #ifndef PREFERRED_DEBUGGING_TYPE
@@ -913,9 +909,6 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #elif defined DBX_DEBUGGING_INFO
 #define PREFERRED_DEBUGGING_TYPE DBX_DEBUG
 
-#elif SDB_DEBUGGING_INFO
-#define PREFERRED_DEBUGGING_TYPE SDB_DEBUG
-
 #elif defined DWARF2_DEBUGGING_INFO || defined DWARF2_LINENO_DEBUGGING_INFO
 #define PREFERRED_DEBUGGING_TYPE DWARF2_DEBUG
 
diff --git a/gcc/diagnostic-color.c b/gcc/diagnostic-color.c
index b8cf6f2c045..ccbae4ba223 100644
--- a/gcc/diagnostic-color.c
+++ b/gcc/diagnostic-color.c
@@ -24,90 +24,7 @@
 #  include <windows.h>
 #endif
 
-/* Select Graphic Rendition (SGR, "\33[...m") strings.  */
-/* Also Erase in Line (EL) to Right ("\33[K") by default.  */
-/*    Why have EL to Right after SGR?
-	 -- The behavior of line-wrapping when at the bottom of the
-	    terminal screen and at the end of the current line is often
-	    such that a new line is introduced, entirely cleared with
-	    the current background color which may be different from the
-	    default one (see the boolean back_color_erase terminfo(5)
-	    capability), thus scrolling the display by one line.
-	    The end of this new line will stay in this background color
-	    even after reverting to the default background color with
-	    "\33[m', unless it is explicitly cleared again with "\33[K"
-	    (which is the behavior the user would instinctively expect
-	    from the whole thing).  There may be some unavoidable
-	    background-color flicker at the end of this new line because
-	    of this (when timing with the monitor's redraw is just right).
-	 -- The behavior of HT (tab, "\t") is usually the same as that of
-	    Cursor Forward Tabulation (CHT) with a default parameter
-	    of 1 ("\33[I"), i.e., it performs pure movement to the next
-	    tab stop, without any clearing of either content or screen
-	    attributes (including background color); try
-	       printf 'asdfqwerzxcv\rASDF\tZXCV\n'
-	    in a bash(1) shell to demonstrate this.  This is not what the
-	    user would instinctively expect of HT (but is ok for CHT).
-	    The instinctive behavior would include clearing the terminal
-	    cells that are skipped over by HT with blank cells in the
-	    current screen attributes, including background color;
-	    the boolean dest_tabs_magic_smso terminfo(5) capability
-	    indicates this saner behavior for HT, but only some rare
-	    terminals have it (although it also indicates a special
-	    glitch with standout mode in the Teleray terminal for which
-	    it was initially introduced).  The remedy is to add "\33K"
-	    after each SGR sequence, be it START (to fix the behavior
-	    of any HT after that before another SGR) or END (to fix the
-	    behavior of an HT in default background color that would
-	    follow a line-wrapping at the bottom of the screen in another
-	    background color, and to complement doing it after START).
-	    Piping GCC's output through a pager such as less(1) avoids
-	    any HT problems since the pager performs tab expansion.
-
-      Generic disadvantages of this remedy are:
-	 -- Some very rare terminals might support SGR but not EL (nobody
-	    will use "gcc -fdiagnostics-color" on a terminal that does not
-	    support SGR in the first place).
-	 -- Having these extra control sequences might somewhat complicate
-	    the task of any program trying to parse "gcc -fdiagnostics-color"
-	    output in order to extract structuring information from it.
-      A specific disadvantage to doing it after SGR START is:
-	 -- Even more possible background color flicker (when timing
-	    with the monitor's redraw is just right), even when not at the
-	    bottom of the screen.
-      There are no additional disadvantages specific to doing it after
-      SGR END.
-
-      It would be impractical for GCC to become a full-fledged
-      terminal program linked against ncurses or the like, so it will
-      not detect terminfo(5) capabilities.  */
-#define COLOR_SEPARATOR		";"
-#define COLOR_NONE		"00"
-#define COLOR_BOLD		"01"
-#define COLOR_UNDERSCORE	"04"
-#define COLOR_BLINK		"05"
-#define COLOR_REVERSE		"07"
-#define COLOR_FG_BLACK		"30"
-#define COLOR_FG_RED		"31"
-#define COLOR_FG_GREEN		"32"
-#define COLOR_FG_YELLOW		"33"
-#define COLOR_FG_BLUE		"34"
-#define COLOR_FG_MAGENTA	"35"
-#define COLOR_FG_CYAN		"36"
-#define COLOR_FG_WHITE		"37"
-#define COLOR_BG_BLACK		"40"
-#define COLOR_BG_RED		"41"
-#define COLOR_BG_GREEN		"42"
-#define COLOR_BG_YELLOW		"43"
-#define COLOR_BG_BLUE		"44"
-#define COLOR_BG_MAGENTA	"45"
-#define COLOR_BG_CYAN		"46"
-#define COLOR_BG_WHITE		"47"
-#define SGR_START		"\33["
-#define SGR_END			"m\33[K"
-#define SGR_SEQ(str)		SGR_START str SGR_END
-#define SGR_RESET		SGR_SEQ("")
-
+#include "color-macros.h"
 
 /* The context and logic for choosing default --color screen attributes
    (foreground and background colors, etc.) are the following.
diff --git a/gcc/diagnostic-core.h b/gcc/diagnostic-core.h
index 1fa28027b5b..24025f1ef80 100644
--- a/gcc/diagnostic-core.h
+++ b/gcc/diagnostic-core.h
@@ -61,33 +61,32 @@ extern void internal_error_no_backtrace (const char *, ...)
 extern bool warning (int, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern bool warning_n (location_t, int, int, const char *, const char *, ...)
     ATTRIBUTE_GCC_DIAG(4,6) ATTRIBUTE_GCC_DIAG(5,6);
+extern bool warning_n (rich_location *, int, int, const char *,
+		       const char *, ...)
+    ATTRIBUTE_GCC_DIAG(4, 6) ATTRIBUTE_GCC_DIAG(5, 6);
 extern bool warning_at (location_t, int, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool warning_at_rich_loc (rich_location *, int, const char *, ...)
+extern bool warning_at (rich_location *, int, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool warning_at_rich_loc_n (rich_location *, int, int, const char *,
-				  const char *, ...)
-    ATTRIBUTE_GCC_DIAG(4, 6) ATTRIBUTE_GCC_DIAG(5, 6);
 extern void error (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
 extern void error_n (location_t, int, const char *, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,5) ATTRIBUTE_GCC_DIAG(4,5);
 extern void error_at (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
-extern void error_at_rich_loc (rich_location *, const char *, ...)
+extern void error_at (rich_location *, const char *, ...)
   ATTRIBUTE_GCC_DIAG(2,3);
 extern void fatal_error (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3)
      ATTRIBUTE_NORETURN;
 /* Pass one of the OPT_W* from options.h as the second parameter.  */
 extern bool pedwarn (location_t, int, const char *, ...)
      ATTRIBUTE_GCC_DIAG(3,4);
-extern bool pedwarn_at_rich_loc (rich_location *, int, const char *, ...)
+extern bool pedwarn (rich_location *, int, const char *, ...)
      ATTRIBUTE_GCC_DIAG(3,4);
 extern bool permerror (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
-extern bool permerror_at_rich_loc (rich_location *, const char *,
+extern bool permerror (rich_location *, const char *,
 				   ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern void sorry (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
 extern void inform (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
-extern void inform_at_rich_loc (rich_location *, const char *,
-				...) ATTRIBUTE_GCC_DIAG(2,3);
+extern void inform (rich_location *, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern void inform_n (location_t, int, const char *, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,5) ATTRIBUTE_GCC_DIAG(4,5);
 extern void verbatim (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
diff --git a/gcc/diagnostic-show-locus.c b/gcc/diagnostic-show-locus.c
index 35121117f49..a1ce682403b 100644
--- a/gcc/diagnostic-show-locus.c
+++ b/gcc/diagnostic-show-locus.c
@@ -29,6 +29,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "diagnostic-color.h"
 #include "gcc-rich-location.h"
 #include "selftest.h"
+#include "selftest-diagnostic.h"
 
 #ifdef HAVE_TERMIOS_H
 # include <termios.h>
@@ -1987,34 +1988,6 @@ namespace selftest {
 
 /* Selftests for diagnostic_show_locus.  */
 
-/* Convenience subclass of diagnostic_context for testing
-   diagnostic_show_locus.  */
-
-class test_diagnostic_context : public diagnostic_context
-{
- public:
-  test_diagnostic_context ()
-  {
-    diagnostic_initialize (this, 0);
-    show_caret = true;
-    show_column = true;
-    start_span = start_span_cb;
-  }
-  ~test_diagnostic_context ()
-  {
-    diagnostic_finish (this);
-  }
-
-  /* Implementation of diagnostic_start_span_fn, hiding the
-     real filename (to avoid printing the names of tempfiles).  */
-  static void
-  start_span_cb (diagnostic_context *context, expanded_location exploc)
-  {
-    exploc.file = "FILENAME";
-    default_diagnostic_start_span_fn (context, exploc);
-  }
-};
-
 /* Verify that diagnostic_show_locus works sanely on UNKNOWN_LOCATION.  */
 
 static void
diff --git a/gcc/diagnostic.c b/gcc/diagnostic.c
index a98bf4a3333..813bca6f65d 100644
--- a/gcc/diagnostic.c
+++ b/gcc/diagnostic.c
@@ -33,6 +33,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "diagnostic-color.h"
 #include "edit-context.h"
 #include "selftest.h"
+#include "selftest-diagnostic.h"
 
 #ifdef HAVE_TERMIOS_H
 # include <termios.h>
@@ -50,12 +51,9 @@ along with GCC; see the file COPYING3.  If not see
 /* Prototypes.  */
 static bool diagnostic_impl (rich_location *, int, const char *,
 			     va_list *, diagnostic_t) ATTRIBUTE_GCC_DIAG(3,0);
-static bool diagnostic_n_impl (location_t, int, int, const char *,
+static bool diagnostic_n_impl (rich_location *, int, int, const char *,
 			       const char *, va_list *,
 			       diagnostic_t) ATTRIBUTE_GCC_DIAG(5,0);
-static bool diagnostic_n_impl_richloc (rich_location *, int, int, const char *,
-				       const char *, va_list *,
-				       diagnostic_t) ATTRIBUTE_GCC_DIAG(5,0);
 
 static void error_recursion (diagnostic_context *) ATTRIBUTE_NORETURN;
 static void real_abort (void) ATTRIBUTE_NORETURN;
@@ -1074,10 +1072,9 @@ diagnostic_append_note (diagnostic_context *context,
   va_end (ap);
 }
 
-/* Implement emit_diagnostic, inform, inform_at_rich_loc, warning, warning_at,
-   warning_at_rich_loc, pedwarn, permerror, permerror_at_rich_loc, error,
-   error_at, error_at_rich_loc, sorry, fatal_error, internal_error, and
-   internal_error_no_backtrace, as documented and defined below.  */
+/* Implement emit_diagnostic, inform, warning, warning_at, pedwarn,
+   permerror, error, error_at, error_at, sorry, fatal_error, internal_error,
+   and internal_error_no_backtrace, as documented and defined below.  */
 static bool
 diagnostic_impl (rich_location *richloc, int opt,
 		 const char *gmsgid,
@@ -1099,12 +1096,13 @@ diagnostic_impl (rich_location *richloc, int opt,
   return diagnostic_report_diagnostic (global_dc, &diagnostic);
 }
 
-/* Same as diagonostic_n_impl taking rich_location instead of location_t.  */
+/* Implement inform_n, warning_n, and error_n, as documented and
+   defined below.  */
 static bool
-diagnostic_n_impl_richloc (rich_location *richloc, int opt, int n,
-			   const char *singular_gmsgid,
-			   const char *plural_gmsgid,
-			   va_list *ap, diagnostic_t kind)
+diagnostic_n_impl (rich_location *richloc, int opt, int n,
+		   const char *singular_gmsgid,
+		   const char *plural_gmsgid,
+		   va_list *ap, diagnostic_t kind)
 {
   diagnostic_info diagnostic;
   diagnostic_set_info_translated (&diagnostic,
@@ -1113,19 +1111,6 @@ diagnostic_n_impl_richloc (rich_location *richloc, int opt, int n,
   if (kind == DK_WARNING)
     diagnostic.option_index = opt;
   return diagnostic_report_diagnostic (global_dc, &diagnostic);
-} 
-
-/* Implement inform_n, warning_n, and error_n, as documented and
-   defined below.  */
-static bool
-diagnostic_n_impl (location_t location, int opt, int n,
-		   const char *singular_gmsgid,
-		   const char *plural_gmsgid,
-		   va_list *ap, diagnostic_t kind)
-{
-  rich_location richloc (line_table, location);
-  return diagnostic_n_impl_richloc (&richloc, opt, n,
-				    singular_gmsgid, plural_gmsgid, ap, kind);
 }
 
 /* Wrapper around diagnostic_impl taking a variable argument list.  */
@@ -1164,10 +1149,12 @@ inform (location_t location, const char *gmsgid, ...)
   va_end (ap);
 }
 
-/* Same as "inform", but at RICHLOC.  */
+/* Same as "inform" above, but at RICHLOC.  */
 void
-inform_at_rich_loc (rich_location *richloc, const char *gmsgid, ...)
+inform (rich_location *richloc, const char *gmsgid, ...)
 {
+  gcc_assert (richloc);
+
   va_list ap;
   va_start (ap, gmsgid);
   diagnostic_impl (richloc, -1, gmsgid, &ap, DK_NOTE);
@@ -1182,7 +1169,8 @@ inform_n (location_t location, int n, const char *singular_gmsgid,
 {
   va_list ap;
   va_start (ap, plural_gmsgid);
-  diagnostic_n_impl (location, -1, n, singular_gmsgid, plural_gmsgid,
+  rich_location richloc (line_table, location);
+  diagnostic_n_impl (&richloc, -1, n, singular_gmsgid, plural_gmsgid,
 		     &ap, DK_NOTE);
   va_end (ap);
 }
@@ -1216,11 +1204,13 @@ warning_at (location_t location, int opt, const char *gmsgid, ...)
   return ret;
 }
 
-/* Same as warning at, but using RICHLOC.  */
+/* Same as "warning at" above, but using RICHLOC.  */
 
 bool
-warning_at_rich_loc (rich_location *richloc, int opt, const char *gmsgid, ...)
+warning_at (rich_location *richloc, int opt, const char *gmsgid, ...)
 {
+  gcc_assert (richloc);
+
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = diagnostic_impl (richloc, opt, gmsgid, &ap, DK_WARNING);
@@ -1228,17 +1218,19 @@ warning_at_rich_loc (rich_location *richloc, int opt, const char *gmsgid, ...)
   return ret;
 }
 
-/* Same as warning_at_rich_loc but for plural variant.  */
+/* Same as warning_n plural variant below, but using RICHLOC.  */
 
 bool
-warning_at_rich_loc_n (rich_location *richloc, int opt, int n,
-		       const char *singular_gmsgid, const char *plural_gmsgid, ...)
+warning_n (rich_location *richloc, int opt, int n,
+	   const char *singular_gmsgid, const char *plural_gmsgid, ...)
 {
+  gcc_assert (richloc);
+
   va_list ap;
   va_start (ap, plural_gmsgid);
-  bool ret = diagnostic_n_impl_richloc (richloc, opt, n,
-					singular_gmsgid, plural_gmsgid,
-					&ap, DK_WARNING);
+  bool ret = diagnostic_n_impl (richloc, opt, n,
+				singular_gmsgid, plural_gmsgid,
+				&ap, DK_WARNING);
   va_end (ap);
   return ret;
 }
@@ -1253,7 +1245,8 @@ warning_n (location_t location, int opt, int n, const char *singular_gmsgid,
 {
   va_list ap;
   va_start (ap, plural_gmsgid);
-  bool ret = diagnostic_n_impl (location, opt, n,
+  rich_location richloc (line_table, location);
+  bool ret = diagnostic_n_impl (&richloc, opt, n,
 				singular_gmsgid, plural_gmsgid,
 				&ap, DK_WARNING);
   va_end (ap);
@@ -1284,11 +1277,13 @@ pedwarn (location_t location, int opt, const char *gmsgid, ...)
   return ret;
 }
 
-/* Same as pedwarn, but using RICHLOC.  */
+/* Same as pedwarn above, but using RICHLOC.  */
 
 bool
-pedwarn_at_rich_loc (rich_location *richloc, int opt, const char *gmsgid, ...)
+pedwarn (rich_location *richloc, int opt, const char *gmsgid, ...)
 {
+  gcc_assert (richloc);
+
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = diagnostic_impl (richloc, opt, gmsgid, &ap, DK_PEDWARN);
@@ -1314,11 +1309,13 @@ permerror (location_t location, const char *gmsgid, ...)
   return ret;
 }
 
-/* Same as "permerror", but at RICHLOC.  */
+/* Same as "permerror" above, but at RICHLOC.  */
 
 bool
-permerror_at_rich_loc (rich_location *richloc, const char *gmsgid, ...)
+permerror (rich_location *richloc, const char *gmsgid, ...)
 {
+  gcc_assert (richloc);
+
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = diagnostic_impl (richloc, -1, gmsgid, &ap, DK_PERMERROR);
@@ -1346,7 +1343,8 @@ error_n (location_t location, int n, const char *singular_gmsgid,
 {
   va_list ap;
   va_start (ap, plural_gmsgid);
-  diagnostic_n_impl (location, -1, n, singular_gmsgid, plural_gmsgid,
+  rich_location richloc (line_table, location);
+  diagnostic_n_impl (&richloc, -1, n, singular_gmsgid, plural_gmsgid,
 		     &ap, DK_ERROR);
   va_end (ap);
 }
@@ -1365,8 +1363,10 @@ error_at (location_t loc, const char *gmsgid, ...)
 /* Same as above, but use RICH_LOC.  */
 
 void
-error_at_rich_loc (rich_location *richloc, const char *gmsgid, ...)
+error_at (rich_location *richloc, const char *gmsgid, ...)
 {
+  gcc_assert (richloc);
+
   va_list ap;
   va_start (ap, gmsgid);
   diagnostic_impl (richloc, -1, gmsgid, &ap, DK_ERROR);
@@ -1628,6 +1628,45 @@ test_print_parseable_fixits_replace ()
 		pp_formatted_text (&pp));
 }
 
+/* Verify that
+     diagnostic_get_location_text (..., SHOW_COLUMN)
+   generates EXPECTED_LOC_TEXT, given FILENAME, LINE, COLUMN, with
+   colorization disabled.  */
+
+static void
+assert_location_text (const char *expected_loc_text,
+		      const char *filename, int line, int column,
+		      bool show_column)
+{
+  test_diagnostic_context dc;
+  dc.show_column = show_column;
+
+  expanded_location xloc;
+  xloc.file = filename;
+  xloc.line = line;
+  xloc.column = column;
+  xloc.data = NULL;
+  xloc.sysp = false;
+
+  char *actual_loc_text = diagnostic_get_location_text (&dc, xloc);
+  ASSERT_STREQ (expected_loc_text, actual_loc_text);
+  free (actual_loc_text);
+}
+
+/* Verify that diagnostic_get_location_text works as expected.  */
+
+static void
+test_diagnostic_get_location_text ()
+{
+  const char *old_progname = progname;
+  progname = "PROGNAME";
+  assert_location_text ("PROGNAME:", NULL, 0, 0, true);
+  assert_location_text ("<built-in>:", "<built-in>", 42, 10, true);
+  assert_location_text ("foo.c:42:10:", "foo.c", 42, 10, true);
+  assert_location_text ("foo.c:42:", "foo.c", 42, 10, false);
+  progname = old_progname;
+}
+
 /* Run all of the selftests within this file.  */
 
 void
@@ -1638,6 +1677,7 @@ diagnostic_c_tests ()
   test_print_parseable_fixits_insert ();
   test_print_parseable_fixits_remove ();
   test_print_parseable_fixits_replace ();
+  test_diagnostic_get_location_text ();
 }
 
 } // namespace selftest
diff --git a/gcc/doc/cpp.texi b/gcc/doc/cpp.texi
index 52f2606eadc..8cafb6554f8 100644
--- a/gcc/doc/cpp.texi
+++ b/gcc/doc/cpp.texi
@@ -211,8 +211,8 @@ Standard C@.  In its default mode, the GNU C preprocessor does not do a
 few things required by the standard.  These are features which are
 rarely, if ever, used, and may cause surprising changes to the meaning
 of a program which does not expect them.  To get strict ISO Standard C,
-you should use the @option{-std=c90}, @option{-std=c99} or
-@option{-std=c11} options, depending
+you should use the @option{-std=c90}, @option{-std=c99},
+@option{-std=c11} or @option{-std=c17} options, depending
 on which version of the standard you want.  To get all the mandatory
 diagnostics, you must also use @option{-pedantic}.  @xref{Invocation}.
 
@@ -1857,8 +1857,11 @@ implementation, unless GNU CPP is being used with GCC@.
 
 The value @code{199409L} signifies the 1989 C standard as amended in
 1994, which is the current default; the value @code{199901L} signifies
-the 1999 revision of the C standard.  Support for the 1999 revision is
-not yet complete.
+the 1999 revision of the C standard; the value @code{201112L}
+signifies the 2011 revision of the C standard; the value
+@code{201710L} signifies the 2017 revision of the C standard (which is
+otherwise identical to the 2011 version apart from correction of
+defects).
 
 This macro is not defined if the @option{-traditional-cpp} option is
 used, nor when compiling C++ or Objective-C@.
@@ -2366,6 +2369,21 @@ the include file @file{math.h} can define the macros
 @code{FP_FAST_FMA}, @code{FP_FAST_FMAF}, and @code{FP_FAST_FMAL}
 for compatibility with the 1999 C standard.
 
+@item __FP_FAST_FMAF16
+@itemx __FP_FAST_FMAF32
+@itemx __FP_FAST_FMAF64
+@itemx __FP_FAST_FMAF128
+@itemx __FP_FAST_FMAF32X
+@itemx __FP_FAST_FMAF64X
+@itemx __FP_FAST_FMAF128X
+These macros are defined with the value 1 if the backend supports the
+@code{fma} functions using the additional @code{_Float@var{n}} and
+@code{_Float@var{n}x} types that are defined in ISO/IEC TS
+18661-3:2015.  The include file @file{math.h} can define the
+@code{FP_FAST_FMAF@var{n}} and @code{FP_FAST_FMAF@var{n}x} macros if
+the user defined @code{__STDC_WANT_IEC_60559_TYPES_EXT__} before
+including @file{math.h}.
+
 @item __GCC_IEC_559
 This macro is defined to indicate the intended level of support for
 IEEE 754 (IEC 60559) floating-point arithmetic.  It expands to a
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index d9b7a540cbd..8aa443f87fb 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -5690,6 +5690,58 @@ Specify which floating-point unit to use.  You must specify the
 @code{target("fpmath=sse,387")} option as
 @code{target("fpmath=sse+387")} because the comma would separate
 different options.
+
+@item nocf_check
+@cindex @code{nocf_check} function attribute
+The @code{nocf_check} attribute on a function is used to inform the
+compiler that the function's prologue should not be instrumented when
+compiled with the @option{-fcf-protection=branch} option.  The
+compiler assumes that the function's address is a valid target for a
+control-flow transfer.
+
+The @code{nocf_check} attribute on a type of pointer to function is
+used to inform the compiler that a call through the pointer should
+not be instrumented when compiled with the
+@option{-fcf-protection=branch} option.  The compiler assumes
+that the function's address from the pointer is a valid target for
+a control-flow transfer.  A direct function call through a function
+name is assumed to be a safe call thus direct calls are not
+instrumented by the compiler.
+
+The @code{nocf_check} attribute is applied to an object's type.
+In case of assignment of a function address or a function pointer to
+another pointer, the attribute is not carried over from the right-hand
+object's type; the type of left-hand object stays unchanged.  The
+compiler checks for @code{nocf_check} attribute mismatch and reports
+a warning in case of mismatch.
+
+@smallexample
+@{
+int foo (void) __attribute__(nocf_check);
+void (*foo1)(void) __attribute__(nocf_check);
+void (*foo2)(void);
+
+int
+foo (void) /* The function's address is assumed to be valid.  */
+
+  /* This call site is not checked for control-flow validity.  */
+  (*foo1)();
+
+  foo1 = foo2; /* A warning is printed about attribute mismatch.  */
+  /* This call site is still not checked for control-flow validity.  */
+  (*foo1)();
+
+  /* This call site is checked for control-flow validity.  */
+  (*foo2)();
+
+  foo2 = foo1; /* A warning is printed about attribute mismatch.  */
+  /* This call site is still checked for control-flow validity.  */
+  (*foo2)();
+
+  return 0;
+@}
+@end smallexample
+
 @end table
 
 On the x86, the inliner does not inline a
@@ -7723,8 +7775,8 @@ GCC implements three different semantics of declaring a function
 inline.  One is available with @option{-std=gnu89} or
 @option{-fgnu89-inline} or when @code{gnu_inline} attribute is present
 on all inline declarations, another when
-@option{-std=c99}, @option{-std=c11},
-@option{-std=gnu99} or @option{-std=gnu11}
+@option{-std=c99},
+@option{-std=gnu99} or an option for a later C version is used
 (without @option{-fgnu89-inline}), and the third
 is used when compiling C++.
 
@@ -10869,6 +10921,7 @@ in the Cilk Plus language manual which can be found at
 @cindex built-in functions
 @findex __builtin_alloca
 @findex __builtin_alloca_with_align
+@findex __builtin_alloca_with_align_and_max
 @findex __builtin_call_with_static_chain
 @findex __builtin_fpclassify
 @findex __builtin_isfinite
@@ -11516,6 +11569,16 @@ an extension.  @xref{Variable Length}, for details.
 
 @end deftypefn
 
+@deftypefn {Built-in Function} void *__builtin_alloca_with_align_and_max (size_t size, size_t alignment, size_t max_size)
+Similar to @code{__builtin_alloca_with_align} but takes an extra argument
+specifying an upper bound for @var{size} in case its value cannot be computed
+at compile time, for use by @option{-fstack-usage}, @option{-Wstack-usage}
+and @option{-Walloca-larger-than}.  @var{max_size} must be a constant integer
+expression, it has no effect on code generation and no attempt is made to
+check its compatibility with @var{size}.
+
+@end deftypefn
+
 @deftypefn {Built-in Function} int __builtin_types_compatible_p (@var{type1}, @var{type2})
 
 You can use the built-in function @code{__builtin_types_compatible_p} to
@@ -21456,6 +21519,25 @@ void __builtin_ia32_wrpkru (unsigned int)
 unsigned int __builtin_ia32_rdpkru ()
 @end smallexample
 
+The following built-in functions are available when @option{-mcet} is used.
+They are used to support Intel Control-flow Enforcment Technology (CET).
+Each built-in function generates the  machine instruction that is part of the
+function's name.
+@smallexample
+unsigned int __builtin_ia32_rdsspd (unsigned int)
+unsigned long long __builtin_ia32_rdsspq (unsigned long long)
+void __builtin_ia32_incsspd (unsigned int)
+void __builtin_ia32_incsspq (unsigned long long)
+void __builtin_ia32_saveprevssp(void);
+void __builtin_ia32_rstorssp(void *);
+void __builtin_ia32_wrssd(unsigned int, void *);
+void __builtin_ia32_wrssq(unsigned long long, void *);
+void __builtin_ia32_wrussd(unsigned int, void *);
+void __builtin_ia32_wrussq(unsigned long long, void *);
+void __builtin_ia32_setssbsy(void);
+void __builtin_ia32_clrssbsy(void *);
+@end smallexample
+
 @node x86 transactional memory intrinsics
 @subsection x86 Transactional Memory Intrinsics
 
diff --git a/gcc/doc/gcov.texi b/gcc/doc/gcov.texi
index 706aa6cf0b0..5c4ba8a51a7 100644
--- a/gcc/doc/gcov.texi
+++ b/gcc/doc/gcov.texi
@@ -125,6 +125,8 @@ gcov [@option{-v}|@option{--version}] [@option{-h}|@option{--help}]
      [@option{-d}|@option{--display-progress}]
      [@option{-f}|@option{--function-summaries}]
      [@option{-i}|@option{--intermediate-format}]
+     [@option{-j}|@option{--human-readable}]
+     [@option{-k}|@option{--use-colors}]
      [@option{-l}|@option{--long-file-names}]
      [@option{-m}|@option{--demangled-names}]
      [@option{-n}|@option{--no-output}]
@@ -185,10 +187,14 @@ be used by @command{lcov} or other tools. The output is a single
 The format of the intermediate @file{.gcov} file is plain text with
 one entry per line
 
+@item -j
+@itemx --human-readable
+Write counts in human readable format (like 24k).
+
 @smallexample
 file:@var{source_file_name}
 function:@var{line_number},@var{execution_count},@var{function_name}
-lcount:@var{line number},@var{execution_count}
+lcount:@var{line number},@var{execution_count},@var{has_unexecuted_block}
 branch:@var{line_number},@var{branch_coverage_type}
 
 Where the @var{branch_coverage_type} is
@@ -207,14 +213,22 @@ Here is a sample when @option{-i} is used in conjunction with @option{-b} option
 file:array.cc
 function:11,1,_Z3sumRKSt6vectorIPiSaIS0_EE
 function:22,1,main
-lcount:11,1
-lcount:12,1
-lcount:14,1
+lcount:11,1,0
+lcount:12,1,0
+lcount:14,1,0
 branch:14,taken
-lcount:26,1
+lcount:26,1,0
 branch:28,nottaken
 @end smallexample
 
+@item -k
+@itemx --use-colors
+
+Use colors for lines of code that have zero coverage.  We use red color for
+non-exceptional lines and cyan for exceptional.  Same colors are used for
+basic blocks with @option{-a} option.
+
+
 @item -l
 @itemx --long-file-names
 Create long file names for included source files.  For example, if the
@@ -327,6 +341,16 @@ non-exceptional paths or only exceptional paths such as C++ exception
 handlers, respectively. Given @samp{-a} option, unexecuted blocks are
 marked @samp{$$$$$} or @samp{%%%%%}, depending on whether a basic block
 is reachable via non-exceptional or exceptional paths.
+Executed basic blocks having a statement with zero @var{execution_count}
+end with @samp{*} character and are colored with magenta color with @option{-k}
+option.
+
+Note that GCC can completely remove the bodies of functions that are
+not needed -- for instance if they are inlined everywhere.  Such functions
+are marked with @samp{-}, which can be confusing.
+Use the @option{-fkeep-inline-functions} and @option{-fkeep-static-functions}
+options to retain these functions and
+allow gcov to properly show their @var{execution_count}.
 
 Some lines of information at the start have @var{line_number} of zero.
 These preamble lines are of the form
diff --git a/gcc/doc/generic.texi b/gcc/doc/generic.texi
index 5c39f453e7f..bb42269ef53 100644
--- a/gcc/doc/generic.texi
+++ b/gcc/doc/generic.texi
@@ -1107,8 +1107,10 @@ constant @var{base} and @var{step}.  The value of @var{base} is
 given by @code{VEC_SERIES_CST_BASE} and the value of @var{step} is
 given by @code{VEC_SERIES_CST_STEP}.
 
-These nodes are restricted to integral types, in order to avoid
-specifying the rounding behavior for floating-point types.
+At present only variable-length vectors use @code{VEC_SERIES_CST};
+constant-length vectors use @code{VECTOR_CST} instead.  The nodes
+are also restricted to integral types, in order to avoid specifying
+the rounding behavior for floating-point types.
 
 @item STRING_CST
 These nodes represent string-constants.  The @code{TREE_STRING_LENGTH}
diff --git a/gcc/doc/gimple.texi b/gcc/doc/gimple.texi
index 635abd39b6e..fa98800a675 100644
--- a/gcc/doc/gimple.texi
+++ b/gcc/doc/gimple.texi
@@ -1310,11 +1310,13 @@ operand is validated with @code{is_gimple_operand}).
 @end deftypefn
 
 
-@deftypefn {GIMPLE function} gcall *gimple_build_call_from_tree (tree call_expr)
-Build a @code{GIMPLE_CALL} from a @code{CALL_EXPR} node.  The arguments and the
-function are taken from the expression directly.  This routine
-assumes that @code{call_expr} is already in GIMPLE form.  That is, its
-operands are GIMPLE values and the function call needs no further
+@deftypefn {GIMPLE function} gcall *gimple_build_call_from_tree (tree call_expr, @
+tree fnptrtype)
+Build a @code{GIMPLE_CALL} from a @code{CALL_EXPR} node.  The arguments
+and the function are taken from the expression directly.  The type of the
+@code{GIMPLE_CALL} is set from the second parameter passed by a caller.
+This routine assumes that @code{call_expr} is already in GIMPLE form.
+That is, its operands are GIMPLE values and the function call needs no further
 simplification.  All the call flags in @code{call_expr} are copied over
 to the new @code{GIMPLE_CALL}.
 @end deftypefn
diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index da360da1c50..b10c94af5ca 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -1007,31 +1007,6 @@ Specify that stabs debugging
 information should be used instead of whatever format the host normally
 uses.  Normally GCC uses the same debug format as the host system.
 
-On MIPS based systems and on Alphas, you must specify whether you want
-GCC to create the normal ECOFF debugging format, or to use BSD-style
-stabs passed through the ECOFF symbol table.  The normal ECOFF debug
-format cannot fully handle languages other than C@.  BSD stabs format can
-handle other languages, but it only works with the GNU debugger GDB@.
-
-Normally, GCC uses the ECOFF debugging format by default; if you
-prefer BSD stabs, specify @option{--with-stabs} when you configure GCC@.
-
-No matter which default you choose when you configure GCC, the user
-can use the @option{-gcoff} and @option{-gstabs+} options to specify explicitly
-the debug format for a particular compilation.
-
-@option{--with-stabs} is meaningful on the ISC system on the 386, also, if
-@option{--with-gas} is used.  It selects use of stabs debugging
-information embedded in COFF output.  This kind of debugging information
-supports C++ well; ordinary COFF debugging information does not.
-
-@option{--with-stabs} is also meaningful on 386 systems running SVR4.  It
-selects use of stabs debugging information embedded in ELF output.  The
-C++ compiler currently (2.6.0) does not support the DWARF debugging
-information normally used on 386 SVR4 platforms; stabs provide a
-workable alternative.  This requires gas and gdb, as the normal SVR4
-tools can not generate or interpret stabs.
-
 @item --with-tls=@var{dialect}
 Specify the default TLS dialect, for systems were there is a choice.
 For ARM targets, possible values for @var{dialect} are @code{gnu} or
@@ -1665,7 +1640,8 @@ not be built.
 
 @item --disable-libssp
 Specify that the run-time libraries for stack smashing protection
-should not be built.
+should not be built or linked against.  On many targets library support
+is provided by the C library instead.
 
 @item --disable-libquadmath
 Specify that the GCC quad-precision math library should not be built.
@@ -2492,6 +2468,13 @@ useful to verify the full @option{-fcompare-debug} testing coverage.  It
 must be used along with @code{bootstrap-debug-lean} and
 @code{bootstrap-debug-lib}.
 
+@item @samp{bootstrap-cet}
+This option enables Intel CET for host tools during bootstrapping.
+@samp{BUILD_CONFIG=bootstrap-cet} is equivalent to adding
+@option{-fcf-protection -mcet} to @samp{BOOT_CFLAGS}.  This option
+assumes that the host supports Intel CET (e.g. GNU assembler version
+2.30 or later).
+
 @item @samp{bootstrap-time}
 Arranges for the run time of each program started by the GCC driver,
 built in any stage, to be logged to @file{time.log}, in the top level of
@@ -3161,8 +3144,6 @@ information have to.
 @item
 @uref{#alpha-x-x,,alpha*-*-*}
 @item
-@uref{#alpha-dec-osf51,,alpha*-dec-osf5.1}
-@item
 @uref{#amd64-x-solaris210,,amd64-*-solaris2.10}
 @item
 @uref{#arm-x-eabi,,arm-*-eabi}
@@ -3213,10 +3194,6 @@ information have to.
 @item
 @uref{#mips-x-x,,mips-*-*}
 @item
-@uref{#mips-sgi-irix5,,mips-sgi-irix5}
-@item
-@uref{#mips-sgi-irix6,,mips-sgi-irix6}
-@item
 @uref{#nds32le-x-elf,,nds32le-*-elf}
 @item
 @uref{#nds32be-x-elf,,nds32be-*-elf}
@@ -3346,8 +3323,7 @@ The workaround is disabled by default if neither of
 @anchor{alpha-x-x}
 @heading alpha*-*-*
 This section contains general configuration information for all
-alpha-based platforms using ELF (in particular, ignore this section for
-DEC OSF/1, Digital UNIX and Tru64 UNIX)@.  In addition to reading this
+Alpha-based platforms using ELF@.  In addition to reading this
 section, please read all other sections that match your target.
 
 We require binutils 2.11.2 or newer.
@@ -3358,20 +3334,6 @@ shared libraries.
 @html
 <hr />
 @end html
-@anchor{alpha-dec-osf51}
-@heading alpha*-dec-osf5.1
-Systems using processors that implement the DEC Alpha architecture and
-are running the DEC/Compaq/HP Unix (DEC OSF/1, Digital UNIX, or Compaq/HP
-Tru64 UNIX) operating system, for example the DEC Alpha AXP systems.
-
-Support for Tru64 UNIX V5.1 has been removed in GCC 4.8.  As of GCC 4.6,
-support for Tru64 UNIX V4.0 and V5.0 has been removed.  As of GCC 3.2,
-versions before @code{alpha*-dec-osf4} are no longer supported.  (These
-are the versions which identify themselves as DEC OSF/1.)
-
-@html
-<hr />
-@end html
 @anchor{amd64-x-solaris210}
 @heading amd64-*-solaris2.1[0-9]*
 This is a synonym for @samp{x86_64-*-solaris2.1[0-9]*}.
@@ -3799,12 +3761,10 @@ It is recommended that you configure GCC to use the GNU assembler.  The
 versions included in Solaris 10, from GNU binutils 2.15 (in
 @file{/usr/sfw/bin/gas}), and Solaris 11, from GNU binutils 2.19 or
 newer (also available as @file{/usr/bin/gas} and
-@file{/usr/gnu/bin/as}), work fine.  Please note that the current
-version, from GNU binutils 2.26, only works on Solaris 12 when using the
-Solaris linker.  On Solaris 10 and 11, you either have to wait for GNU
-binutils 2.26.1 or newer, or stay with GNU binutils 2.25.1.  Recent
-versions of the Solaris assembler in @file{/usr/ccs/bin/as} work almost
-as well, though.
+@file{/usr/gnu/bin/as}), work fine.  The current version, from GNU
+binutils 2.29, is known to work, but the version from GNU binutils 2.26
+must be avoided.  Recent versions of the Solaris assembler in
+@file{/usr/ccs/bin/as} work almost as well, though.
 @c FIXME: as patch requirements?
 
 For linking, the Solaris linker, is preferred.  If you want to use the GNU
@@ -3812,7 +3772,7 @@ linker instead, note that due to a packaging bug the version in Solaris
 10, from GNU binutils 2.15 (in @file{/usr/sfw/bin/gld}), cannot be used,
 while the version in Solaris 11, from GNU binutils 2.19 or newer (also
 in @file{/usr/gnu/bin/ld} and @file{/usr/bin/gld}), works, as does the
-latest version, from GNU binutils 2.26.
+latest version, from GNU binutils 2.29.
 
 To use GNU @command{as}, configure with the options
 @option{--with-gnu-as --with-as=@//usr/@/sfw/@/bin/@/gas}.  It may be necessary
@@ -4160,22 +4120,6 @@ use traps on systems that support them.
 @html
 <hr />
 @end html
-@anchor{mips-sgi-irix5}
-@heading mips-sgi-irix5
-Support for IRIX 5 has been removed in GCC 4.6.
-
-@html
-<hr />
-@end html
-@anchor{mips-sgi-irix6}
-@heading mips-sgi-irix6
-Support for IRIX 6.5 has been removed in GCC 4.8.  Support for IRIX 6
-releases before 6.5 has been removed in GCC 4.6, as well as support for
-the O32 ABI.
-
-@html
-<hr />
-@end html
 @anchor{moxie-x-elf}
 @heading moxie-*-elf
 The moxie processor.
@@ -4449,9 +4393,8 @@ versions included in Solaris 10, from GNU binutils 2.15 (in
 @file{/usr/sfw/bin/gas}), and Solaris 11,
 from GNU binutils 2.19 or newer (also in @file{/usr/bin/gas} and
 @file{/usr/gnu/bin/as}), are known to work.
-Current versions of GNU binutils (2.26)
-are known to work as well, with the caveat mentioned in
-@uref{#ix86-x-solaris210,,i?86-*-solaris2.10} .  Note that your mileage may vary
+The current version, from GNU binutils 2.29,
+is known to work as well.  Note that your mileage may vary
 if you use a combination of the GNU tools and the Solaris tools: while the
 combination GNU @command{as} + Sun @command{ld} should reasonably work,
 the reverse combination Sun @command{as} + GNU @command{ld} may fail to
@@ -4459,7 +4402,7 @@ build or cause memory corruption at runtime in some cases for C++ programs.
 @c FIXME: still?
 GNU @command{ld} usually works as well, although the version included in
 Solaris 10 cannot be used due to several bugs.  Again, the current
-version (2.26) is known to work, but generally lacks platform specific
+version (2.29) is known to work, but generally lacks platform specific
 features, so better stay with Solaris @command{ld}.  To use the LTO linker
 plugin (@option{-fuse-linker-plugin}) with GNU @command{ld}, GNU
 binutils @emph{must} be configured with @option{--enable-largefile}.
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 9bf1a17ebfb..4e96a3942c2 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -45,7 +45,7 @@ remainder.  @command{g++} accepts mostly the same options as @command{gcc}.
 @c man end
 @c man begin SEEALSO
 gpl(7), gfdl(7), fsf-funding(7),
-cpp(1), gcov(1), as(1), ld(1), gdb(1), adb(1), dbx(1), sdb(1)
+cpp(1), gcov(1), as(1), ld(1), gdb(1), dbx(1)
 and the Info entries for @file{gcc}, @file{cpp}, @file{as},
 @file{ld}, @file{binutils} and @file{gdb}.
 @c man end
@@ -315,7 +315,7 @@ Objective-C and Objective-C++ Dialects}.
 -Wstack-protector  -Wstack-usage=@var{len}  -Wstrict-aliasing @gol
 -Wstrict-aliasing=n  -Wstrict-overflow  -Wstrict-overflow=@var{n} @gol
 -Wstringop-overflow=@var{n} @gol
--Wsuggest-attribute=@r{[}pure@r{|}const@r{|}noreturn@r{|}format@r{]} @gol
+-Wsuggest-attribute=@r{[}pure@r{|}const@r{|}noreturn@r{|}format@r{|}malloc@r{]} @gol
 -Wsuggest-final-types @gol  -Wsuggest-final-methods  -Wsuggest-override @gol
 -Wmissing-format-attribute  -Wsubobject-linkage @gol
 -Wswitch  -Wswitch-bool  -Wswitch-default  -Wswitch-enum @gol
@@ -342,7 +342,7 @@ Objective-C and Objective-C++ Dialects}.
 
 @item Debugging Options
 @xref{Debugging Options,,Options for Debugging Your Program}.
-@gccoptlist{-g  -g@var{level}  -gcoff  -gdwarf  -gdwarf-@var{version} @gol
+@gccoptlist{-g  -g@var{level}  -gdwarf  -gdwarf-@var{version} @gol
 -ggdb  -grecord-gcc-switches  -gno-record-gcc-switches @gol
 -gstabs  -gstabs+  -gstrict-dwarf  -gno-strict-dwarf @gol
 -gcolumn-info  -gno-column-info @gol
@@ -461,6 +461,7 @@ Objective-C and Objective-C++ Dialects}.
 -fchkp-check-read  -fchkp-check-write  -fchkp-store-bounds @gol
 -fchkp-instrument-calls  -fchkp-instrument-marked-only @gol
 -fchkp-use-wrappers  -fchkp-flexible-struct-trailing-arrays@gol
+-fcf-protection==@r{[}full@r{|}branch@r{|}return@r{|}none@r{]} @gol
 -fstack-protector  -fstack-protector-all  -fstack-protector-strong @gol
 -fstack-protector-explicit  -fstack-check @gol
 -fstack-limit-register=@var{reg}  -fstack-limit-symbol=@var{sym} @gol
@@ -742,7 +743,7 @@ Objective-C and Objective-C++ Dialects}.
 @gccoptlist{-msmall-model  -mno-lsim}
 
 @emph{FT32 Options}
-@gccoptlist{-msim  -mlra  -mnodiv}
+@gccoptlist{-msim  -mlra  -mnodiv  -mft32b  -mcompress  -mnopm}
 
 @emph{FRV Options}
 @gccoptlist{-mgpr-32  -mgpr-64  -mfpr-32  -mfpr-64 @gol
@@ -947,6 +948,7 @@ Objective-C and Objective-C++ Dialects}.
 
 @emph{Nios II Options}
 @gccoptlist{-G @var{num}  -mgpopt=@var{option}  -mgpopt  -mno-gpopt @gol
+-mgprel-sec=@var{regexp} -mr0rel-sec=@var{regexp} @gol
 -mel  -meb @gol
 -mno-bypass-cache  -mbypass-cache @gol
 -mno-cache-volatile  -mcache-volatile @gol
@@ -987,7 +989,7 @@ See RS/6000 and PowerPC Options.
 -msmall-data-limit=@var{N-bytes} @gol
 -msave-restore  -mno-save-restore @gol
 -mstrict-align -mno-strict-align @gol
--mcmodel=@var{code-model} @gol
+-mcmodel=medlow -mcmodel=medany @gol
 -mexplicit-relocs  -mno-explicit-relocs @gol}
 
 @emph{RL78 Options}
@@ -1203,6 +1205,7 @@ See RS/6000 and PowerPC Options.
 -msse4a  -m3dnow  -m3dnowa  -mpopcnt  -mabm  -mbmi  -mtbm  -mfma4  -mxop @gol
 -mlzcnt  -mbmi2  -mfxsr  -mxsave  -mxsaveopt  -mrtm  -mlwp  -mmpx  @gol
 -mmwaitx  -mclzero  -mpku  -mthreads @gol
+-mcet -mibt -mshstk @gol
 -mms-bitfields  -mno-align-stringops  -minline-all-stringops @gol
 -minline-stringops-dynamically  -mstringop-strategy=@var{alg} @gol
 -mmemcpy-strategy=@var{strategy}  -mmemset-strategy=@var{strategy} @gol
@@ -1828,6 +1831,13 @@ substantially completely supported, modulo bugs, floating-point issues
 Annexes F and G) and the optional Annexes K (Bounds-checking
 interfaces) and L (Analyzability).  The name @samp{c1x} is deprecated.
 
+@item c17
+@itemx iso9899:2017
+ISO C17, the 2017 revision of the ISO C standard.  This standard is
+same as C11 except for corrections of defects (all of which are also
+applied with @option{-std=c11}) and a new value of
+@code{__STDC_VERSION__}, and so is supported to the same extent as C11.
+
 @item gnu90
 @itemx gnu89
 GNU dialect of ISO C90 (including some C99 features).
@@ -1838,9 +1848,12 @@ GNU dialect of ISO C99.  The name @samp{gnu9x} is deprecated.
 
 @item gnu11
 @itemx gnu1x
-GNU dialect of ISO C11.  This is the default for C code.
+GNU dialect of ISO C11.
 The name @samp{gnu1x} is deprecated.
 
+@item gnu17
+GNU dialect of ISO C17.  This is the default for C code.
+
 @item c++98
 @itemx c++03
 The 1998 ISO C++ standard plus the 2003 technical corrigendum and some
@@ -5201,7 +5214,7 @@ whether to issue a warning.  Similarly to @option{-Wstringop-overflow=3} this
 setting of the option may result in warnings for benign code.
 @end table
 
-@item -Wsuggest-attribute=@r{[}pure@r{|}const@r{|}noreturn@r{|}format@r{|}cold@r{]}
+@item -Wsuggest-attribute=@r{[}pure@r{|}const@r{|}noreturn@r{|}format@r{|}cold@r{|}malloc@r{]}
 @opindex Wsuggest-attribute=
 @opindex Wno-suggest-attribute=
 Warn for cases where adding an attribute may be beneficial. The
@@ -5211,21 +5224,25 @@ attributes currently supported are listed below.
 @item -Wsuggest-attribute=pure
 @itemx -Wsuggest-attribute=const
 @itemx -Wsuggest-attribute=noreturn
+@itemx -Wsuggest-attribute=malloc
 @opindex Wsuggest-attribute=pure
 @opindex Wno-suggest-attribute=pure
 @opindex Wsuggest-attribute=const
 @opindex Wno-suggest-attribute=const
 @opindex Wsuggest-attribute=noreturn
 @opindex Wno-suggest-attribute=noreturn
+@opindex Wsuggest-attribute=malloc
+@opindex Wno-suggest-attribute=malloc
 
 Warn about functions that might be candidates for attributes
-@code{pure}, @code{const} or @code{noreturn}.  The compiler only warns for
-functions visible in other compilation units or (in the case of @code{pure} and
-@code{const}) if it cannot prove that the function returns normally. A function
-returns normally if it doesn't contain an infinite loop or return abnormally
-by throwing, calling @code{abort} or trapping.  This analysis requires option
-@option{-fipa-pure-const}, which is enabled by default at @option{-O} and
-higher.  Higher optimization levels improve the accuracy of the analysis.
+@code{pure}, @code{const} or @code{noreturn} or @code{malloc}. The compiler
+only warns for functions visible in other compilation units or (in the case of
+@code{pure} and @code{const}) if it cannot prove that the function returns
+normally. A function returns normally if it doesn't contain an infinite loop or
+return abnormally by throwing, calling @code{abort} or trapping.  This analysis
+requires option @option{-fipa-pure-const}, which is enabled by default at
+@option{-O} and higher.  Higher optimization levels improve the accuracy
+of the analysis.
 
 @item -Wsuggest-attribute=format
 @itemx -Wmissing-format-attribute
@@ -6318,7 +6335,8 @@ attributes.
 @item -Wno-builtin-declaration-mismatch
 @opindex Wno-builtin-declaration-mismatch
 @opindex Wbuiltin-declaration-mismatch
-Warn if a built-in function is declared with the wrong signature.
+Warn if a built-in function is declared with the wrong signature or 
+as non-function.
 This warning is enabled by default.
 
 @item -Wno-builtin-macro-redefined
@@ -6893,7 +6911,7 @@ in their names, but apply to all currently-supported versions of DWARF.
 Produce debugging information in stabs format (if that is supported),
 without GDB extensions.  This is the format used by DBX on most BSD
 systems.  On MIPS, Alpha and System V Release 4 systems this option
-produces stabs debugging output that is not understood by DBX or SDB@.
+produces stabs debugging output that is not understood by DBX@.
 On System V Release 4 systems this option requires the GNU assembler.
 
 @item -gstabs+
@@ -6903,12 +6921,6 @@ using GNU extensions understood only by the GNU debugger (GDB)@.  The
 use of these extensions is likely to make other debuggers crash or
 refuse to read the program.
 
-@item -gcoff
-@opindex gcoff
-Produce debugging information in COFF format (if that is supported).
-This is the format used by SDB on most System V systems prior to
-System V Release 4.
-
 @item -gxcoff
 @opindex gxcoff
 Produce debugging information in XCOFF format (if that is supported).
@@ -6930,7 +6942,6 @@ supported).  This is the format used by DEBUG on Alpha/VMS systems.
 @item -g@var{level}
 @itemx -ggdb@var{level}
 @itemx -gstabs@var{level}
-@itemx -gcoff@var{level}
 @itemx -gxcoff@var{level}
 @itemx -gvms@var{level}
 Request debugging information and also use @var{level} to specify how
@@ -6979,7 +6990,12 @@ link processing time.  Merging is enabled by default.
 @item -fdebug-prefix-map=@var{old}=@var{new}
 @opindex fdebug-prefix-map
 When compiling files in directory @file{@var{old}}, record debugging
-information describing them as in @file{@var{new}} instead.
+information describing them as in @file{@var{new}} instead.  This can be
+used to replace a build-time path with an install-time path in the debug info.
+It can also be used to change an absolute path to a relative path by using
+@file{.} for @var{new}.  This can give more reproducible builds, which are
+location independent, but may require an extra command to tell GDB where to
+find the source files.
 
 @item -fvar-tracking
 @opindex fvar-tracking
@@ -7063,7 +7079,7 @@ Allow using extensions of later DWARF standard version than selected with
 @opindex gno-column-info
 Emit location column information into DWARF debugging information, rather
 than just file and line.
-This option is disabled by default.
+This option is enabled by default.
 
 @item -gz@r{[}=@var{type}@r{]}
 @opindex gz
@@ -7837,7 +7853,7 @@ Use @option{-fno-delete-null-pointer-checks} to disable this optimization
 for programs that depend on that behavior.
 
 This option is enabled by default on most targets.  On Nios II ELF, it
-defaults to off.  On AVR and CR16, this option is completely disabled.  
+defaults to off.  On AVR, CR16, and MSP430, this option is completely disabled.
 
 Passes that use the dataflow information
 are enabled independently at different optimization levels.
@@ -9712,18 +9728,26 @@ file if the target supports arbitrary sections.  The name of the
 function or the name of the data item determines the section's name
 in the output file.
 
-Use these options on systems where the linker can perform optimizations
-to improve locality of reference in the instruction space.  Most systems
-using the ELF object format and SPARC processors running Solaris 2 have
-linkers with such optimizations.  AIX may have these optimizations in
-the future.
-
-Only use these options when there are significant benefits from doing
-so.  When you specify these options, the assembler and linker
-create larger object and executable files and are also slower.
-You cannot use @command{gprof} on all systems if you
-specify this option, and you may have problems with debugging if
-you specify both this option and @option{-g}.
+Use these options on systems where the linker can perform optimizations to
+improve locality of reference in the instruction space.  Most systems using the
+ELF object format have linkers with such optimizations.  On AIX, the linker
+rearranges sections (CSECTs) based on the call graph.  The performance impact
+varies.
+
+Together with a linker garbage collection (linker @option{--gc-sections}
+option) these options may lead to smaller statically-linked executables (after
+stripping).
+
+On ELF/DWARF systems these options do not degenerate the quality of the debug
+information.  There could be issues with other object files/debug info formats.
+
+Only use these options when there are significant benefits from doing so.  When
+you specify these options, the assembler and linker create larger object and
+executable files and are also slower.  These options affect code generation.
+They prevent optimizations by the compiler and assembler using relative
+locations inside a translation unit since the locations are unknown until
+link time.  An example of such an optimization is relaxing calls to short call
+instructions.
 
 @item -fbranch-target-load-optimize
 @opindex fbranch-target-load-optimize
@@ -10851,9 +10875,9 @@ Link your object files with @option{-lgcov} or @option{-fprofile-arcs}
 Run the program on a representative workload to generate the arc profile
 information.  This may be repeated any number of times.  You can run
 concurrent instances of your program, and provided that the file system
-supports locking, the data files will be correctly updated.  Also
-@code{fork} calls are detected and correctly handled (double counting
-will not happen).
+supports locking, the data files will be correctly updated.  Unless
+a strict ISO C dialect option is in effect, @code{fork} calls are
+detected and correctly handled without double counting.
 
 @item
 For profile-directed optimizations, compile the source files again with
@@ -11141,6 +11165,15 @@ to verify the referenced object has the correct dynamic type.
 This option enables instrumentation of pointer arithmetics.  If the pointer
 arithmetics overflows, a run-time error is issued.
 
+@item -fsanitize=builtin
+@opindex fsanitize=builtin
+
+This option enables instrumentation of arguments to selected builtin
+functions.  If an invalid value is passed to such arguments, a run-time
+error is issued.  E.g.@ passing 0 as the argument to @code{__builtin_ctz}
+or @code{__builtin_clz} invokes undefined behavior and is diagnosed
+by this option.
+
 @end table
 
 While @option{-ftrapv} causes traps for signed overflows to be emitted,
@@ -11401,6 +11434,33 @@ is used to link a program, the GCC driver automatically links
 against @file{libmpxwrappers}.  See also @option{-static-libmpxwrappers}.
 Enabled by default.
 
+@item -fcf-protection==@r{[}full@r{|}branch@r{|}return@r{|}none@r{]}
+@opindex fcf-protection
+Enable code instrumentation of control-flow transfers to increase
+program security by checking that target addresses of control-flow
+transfer instructions (such as indirect function call, function return,
+indirect jump) are valid.  This prevents diverting the flow of control
+to an unexpected target.  This is intended to protect against such
+threats as Return-oriented Programming (ROP), and similarly
+call/jmp-oriented programming (COP/JOP).
+
+The value @code{branch} tells the compiler to implement checking of
+validity of control-flow transfer at the point of indirect branch
+instructions, i.e. call/jmp instructions.  The value @code{return}
+implements checking of validity at the point of returning from a
+function.  The value @code{full} is an alias for specifying both
+@code{branch} and @code{return}. The value @code{none} turns off
+instrumentation.
+
+You can also use the @code{nocf_check} attribute to identify
+which functions and calls should be skipped from instrumentation
+(@pxref{Function Attributes}).
+
+Currently the x86 GNU/Linux target provides an implementation based
+on Intel Control-flow Enforcement Technology (CET).  Instrumentation
+for x86 is controlled by target-specific options @option{-mcet},
+@option{-mibt} and @option{-mshstk} (@pxref{x86 Options}).
+
 @item -fstack-protector
 @opindex fstack-protector
 Emit extra code to check for buffer overflows, such as stack smashing
@@ -14279,7 +14339,7 @@ Specify the name of the target processor for which GCC should tune the
 performance of the code.  Permissible values for this option are:
 @samp{generic}, @samp{cortex-a35}, @samp{cortex-a53}, @samp{cortex-a55},
 @samp{cortex-a57}, @samp{cortex-a72}, @samp{cortex-a73}, @samp{cortex-a75},
-@samp{exynos-m1}, @samp{falkor}, @samp{qdf24xx},
+@samp{exynos-m1}, @samp{falkor}, @samp{qdf24xx}, @samp{saphira},
 @samp{xgene1}, @samp{vulcan}, @samp{thunderx},
 @samp{thunderxt88}, @samp{thunderxt88p1}, @samp{thunderxt81},
 @samp{thunderxt83}, @samp{thunderx2t99}, @samp{cortex-a57.cortex-a53},
@@ -14361,6 +14421,9 @@ replacing it with a number selects vector-length specific output.
 The possible lengths in the latter case are: 128, 256, 512, 1024
 and 2048.  @samp{scalable} is the default.
 
+At present, @samp{-msve-vector-bits=128} produces the same output
+as @samp{-msve-vector-bits=scalable}.
+
 @end table
 
 @subsubsection @option{-march} and @option{-mcpu} Feature Modifiers
@@ -14399,6 +14462,8 @@ Enable FP16 extension.  This also enables floating-point instructions.
 Enable the RcPc extension.  This does not change code generation from GCC,
 but is passed on to the assembler, enabling inline asm statements to use
 instructions from the RcPc extension.
+@item dotprod
+Enable the Dot Product extension.  This also enables Advanced SIMD instructions.
 
 @end table
 
@@ -15639,6 +15704,9 @@ The ARMv8.1 Advanced SIMD and floating-point instructions.
 The cryptographic instructions.  This also enables the Advanced SIMD and
 floating-point instructions.
 
+@item +dotprod
+Enable the Dot Product extension.  This also enables Advanced SIMD instructions.
+
 @item +nocrypto
 Disable the cryptographic extension.
 
@@ -15825,6 +15893,9 @@ Permissible names for this option are the same as those for
 The following extension options are common to the listed CPUs:
 
 @table @samp
+@item +nodsp
+Disable the DSP instructions on @samp{cortex-m33}.
+
 @item  +nofp
 Disables the floating-point instructions on @samp{arm9e},
 @samp{arm946e-s}, @samp{arm966e-s}, @samp{arm968e-s}, @samp{arm10e},
@@ -17717,6 +17788,18 @@ so by default the compiler uses standard reload.
 @opindex mnodiv
 Do not use div and mod instructions.
 
+@item -mft32b
+@opindex mft32b
+Enable use of the extended instructions of the FT32B processor.
+
+@item -mcompress
+@opindex mcompress
+Compress all code using the Ft32B code compression scheme.
+
+@item -mnopm
+@opindex  mnopm
+Do not generate code that reads program memory.
+
 @end table
 
 @node FRV Options
@@ -21128,6 +21211,32 @@ GOT data sections.  In this case, the 16-bit offset for GP-relative
 addressing may not be large enough to allow access to the entire 
 small data section.
 
+@item -mgprel-sec=@var{regexp}
+@opindex mgprel-sec
+This option specifies additional section names that can be accessed via
+GP-relative addressing.  It is most useful in conjunction with 
+@code{section} attributes on variable declarations 
+(@pxref{Common Variable Attributes}) and a custom linker script.  
+The @var{regexp} is a POSIX Extended Regular Expression.
+
+This option does not affect the behavior of the @option{-G} option, and 
+and the specified sections are in addition to the standard @code{.sdata} 
+and @code{.sbss} small-data sections that are recognized by @option{-mgpopt}.
+
+@item -mr0rel-sec=@var{regexp}
+@opindex mr0rel-sec
+This option specifies names of sections that can be accessed via a 
+16-bit offset from @code{r0}; that is, in the low 32K or high 32K 
+of the 32-bit address space.  It is most useful in conjunction with 
+@code{section} attributes on variable declarations 
+(@pxref{Common Variable Attributes}) and a custom linker script.  
+The @var{regexp} is a POSIX Extended Regular Expression.
+
+In contrast to the use of GP-relative addressing for small data, 
+zero-based addressing is never generated by default and there are no 
+conventional section names used in standard linker scripts for sections
+in the low or high areas of memory.
+
 @item -mel
 @itemx -meb
 @opindex mel
@@ -21631,9 +21740,26 @@ When generating PIC code, allow the use of PLTs. Ignored for non-PIC.
 
 @item -mabi=@var{ABI-string}
 @opindex mabi
-Specify integer and floating-point calling convention.  This defaults to the
-natural calling convention: e.g.@ LP64 for RV64I, ILP32 for RV32I, LP64D for
-RV64G.
+@item -mabi=@var{ABI-string}
+@opindex mabi
+Specify integer and floating-point calling convention.  @var{ABI-string}
+contains two parts: the size of integer types and the registers used for
+floating-point types.  For example @samp{-march=rv64ifd -mabi=lp64d} means that
+@samp{long} and pointers are 64-bit (implicitly defining @samp{int} to be
+32-bit), and that floating-point values up to 64 bits wide are passed in F
+registers.  Contrast this with @samp{-march=rv64ifd -mabi=lp64f}, which still
+allows the compiler to generate code that uses the F and D extensions but only
+allows floating-point values up to 32 bits long to be passed in registers; or
+@samp{-march=rv64ifd -mabi=lp64}, in which no floating-point arguments will be
+passed in registers.
+
+The default for this argument is system dependent, users who want a specific
+calling convention should specify one explicitly.  The valid calling
+conventions are: @samp{ilp32}, @samp{ilp32f}, @samp{ilp32d}, @samp{lp64},
+@samp{lp64f}, and @samp{lp64d}.  Some calling conventions are impossible to
+implement on some ISAs: for example, @samp{-march=rv32if -mabi=ilp32d} is
+invalid because the ABI requires 64-bit values be passed in F registers, but F
+registers are only 32 bits wide.
 
 @item -mfdiv
 @itemx -mno-fdiv
@@ -21671,9 +21797,18 @@ Use smaller but slower prologue and epilogue code.
 @opindex mstrict-align
 Do not generate unaligned memory accesses.
 
-@item -mcmodel=@var{code-model}
-@opindex mcmodel
-Specify the code model.
+@item -mcmodel=medlow
+@opindex mcmodel=medlow
+Generate code for the medium-low code model. The program and its statically
+defined symbols must lie within a single 2 GiB address range and must lie
+between absolute addresses @minus{}2 GiB and +2 GiB. Programs can be
+statically or dynamically linked. This is the default code model.
+
+@item -mcmodel=medany
+@opindex mcmodel=medany
+Generate code for the medium-any code model. The program and its statically
+defined symbols must be within any single 2 GiB address range. Programs can be
+statically or dynamically linked.
 
 @end table
 
@@ -22576,12 +22711,18 @@ Disable Book-E SPE ABI extensions for the current ABI@.
 @item -mabi=ibmlongdouble
 @opindex mabi=ibmlongdouble
 Change the current ABI to use IBM extended-precision long double.
-This is a PowerPC 32-bit SYSV ABI option.
+This is not likely to work if your system defaults to using IEEE
+extended-precision long double.  If you change the long double type
+from IEEE extended-precision, the compiler will issue a warning unless
+you use the @option{-Wno-psabi} option.
 
 @item -mabi=ieeelongdouble
 @opindex mabi=ieeelongdouble
 Change the current ABI to use IEEE extended-precision long double.
-This is a PowerPC 32-bit Linux ABI option.
+This is not likely to work if your system defaults to using IBM
+extended-precision long double.  If you change the long double type
+from IBM extended-precision, the compiler will issue a warning unless
+you use the @option{-Wno-psabi} option.
 
 @item -mabi=elfv1
 @opindex mabi=elfv1
@@ -25821,15 +25962,19 @@ preferred alignment to @option{-mpreferred-stack-boundary=2}.
 @need 200
 @itemx -mclzero
 @opindex mclzero
+@need 200
 @itemx -mpku
 @opindex mpku
+@need 200
+@itemx -mcet
+@opindex mcet
 These switches enable the use of instructions in the MMX, SSE,
 SSE2, SSE3, SSSE3, SSE4.1, AVX, AVX2, AVX512F, AVX512PF, AVX512ER, AVX512CD,
 SHA, AES, PCLMUL, FSGSBASE, RDRND, F16C, FMA, SSE4A, FMA4, XOP, LWP, ABM,
 AVX512VL, AVX512BW, AVX512DQ, AVX512IFMA AVX512VBMI, BMI, BMI2, FXSR,
-XSAVE, XSAVEOPT, LZCNT, RTM, MPX, MWAITX, PKU, 3DNow!@: or enhanced 3DNow!@:
-extended instruction sets.  Each has a corresponding @option{-mno-} option
-to disable use of these instructions.
+XSAVE, XSAVEOPT, LZCNT, RTM, MPX, MWAITX, PKU, IBT, SHSTK,
+3DNow!@: or enhanced 3DNow!@: extended instruction sets.  Each has a
+corresponding @option{-mno-} option to disable use of these instructions.
 
 These extensions are also available as built-in functions: see
 @ref{x86 Built-in Functions}, for details of the functions enabled and
@@ -25849,6 +25994,13 @@ supported architecture, using the appropriate flags.  In particular,
 the file containing the CPU detection code should be compiled without
 these options.
 
+The @option{-mcet} option turns on the @option{-mibt} and @option{-mshstk}
+options.  The @option{-mibt} option enables indirect branch tracking support
+and the @option{-mshstk} option enables shadow stack support from
+Intel Control-flow Enforcement Technology (CET).  The compiler also provides
+a number of built-in functions for fine-grained control in a CET-based
+application.  See @xref{x86 Built-in Functions}, for more information.
+
 @item -mdump-tune-features
 @opindex mdump-tune-features
 This option instructs GCC to dump the names of the x86 performance 
@@ -25927,6 +26079,24 @@ see @ref{Other Builtins} for details.
 This option enables use of the @code{movbe} instruction to implement
 @code{__builtin_bswap32} and @code{__builtin_bswap64}.
 
+@item -mibt
+@opindex mibt
+This option tells the compiler to use indirect branch tracking support
+(for indirect calls and jumps) from x86 Control-flow Enforcement
+Technology (CET).  The option has effect only if the
+@option{-fcf-protection=full} or @option{-fcf-protection=branch} option
+is specified. The option @option{-mibt} is on by default when the
+@code{-mcet} option is specified.
+
+@item -mshstk
+@opindex mshstk
+This option tells the compiler to use shadow stack support (return
+address tracking) from x86 Control-flow Enforcement Technology (CET).
+The option has effect only if the @option{-fcf-protection=full} or
+@option{-fcf-protection=return} option is specified.  The option
+@option{-mshstk} is on by default when the @option{-mcet} option is
+specified.
+
 @item -mcrc32
 @opindex mcrc32
 This option enables built-in functions @code{__builtin_ia32_crc32qi},
diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi
index 0aa8ec8812c..e4fed29a95b 100644
--- a/gcc/doc/md.texi
+++ b/gcc/doc/md.texi
@@ -6896,6 +6896,15 @@ scheduler and other passes from moving instructions and using register
 equivalences across the boundary defined by the blockage insn.
 This needs to be an UNSPEC_VOLATILE pattern or a volatile ASM.
 
+@cindex @code{memory_blockage} instruction pattern
+@item @samp{memory_blockage}
+This pattern, if defined, represents a compiler memory barrier, and will be
+placed at points across which RTL passes may not propagate memory accesses.
+This instruction needs to read and write volatile BLKmode memory.  It does
+not need to generate any machine instruction.  If this pattern is not defined,
+the compiler falls back to emitting an instruction corresponding
+to @code{asm volatile ("" ::: "memory")}.
+
 @cindex @code{memory_barrier} instruction pattern
 @item @samp{memory_barrier}
 If the target memory model is not fully synchronous, then this pattern
diff --git a/gcc/doc/passes.texi b/gcc/doc/passes.texi
index 93e70ee6406..2bf786c2568 100644
--- a/gcc/doc/passes.texi
+++ b/gcc/doc/passes.texi
@@ -981,11 +981,10 @@ these files.
 
 This is run after final because it must output the stack slot offsets
 for pseudo registers that did not get hard registers.  Source files
-are @file{dbxout.c} for DBX symbol table format, @file{sdbout.c} for
-SDB symbol table format, @file{dwarfout.c} for DWARF symbol table
-format, files @file{dwarf2out.c} and @file{dwarf2asm.c} for DWARF2
-symbol table format, and @file{vmsdbgout.c} for VMS debug symbol table
-format.
+are @file{dbxout.c} for DBX symbol table format, @file{dwarfout.c} for
+DWARF symbol table format, files @file{dwarf2out.c} and @file{dwarf2asm.c}
+for DWARF2 symbol table format, and @file{vmsdbgout.c} for VMS debug
+symbol table format.
 
 @end itemize
 
diff --git a/gcc/doc/poly-int.texi b/gcc/doc/poly-int.texi
index 851a86a431c..9519b620274 100644
--- a/gcc/doc/poly-int.texi
+++ b/gcc/doc/poly-int.texi
@@ -256,20 +256,6 @@ must_gt (@var{a}, @var{b}) == !may_le (@var{a}, @var{b})
 must_ne (@var{a}, @var{b}) == !may_eq (@var{a}, @var{b})
 @end example
 
-There are also helper functions for certain common operations:
-
-@example
-known_zero (@var{a}) == must_eq (@var{a}, 0)
-maybe_zero (@var{a}) == may_eq (@var{a}, 0)
-known_nonzero (@var{a}) == must_ne (@var{a}, 0)
-maybe_nonzero (@var{a}) == may_ne (@var{a}, 0)
-known_one (@var{a}) == must_eq (@var{a}, 1)
-known_all_ones (@var{a}) == must_eq (@var{a}, -1)
-@end example
-
-Using these helper functions removes the need to distinguish between
-signed and unsigned constants, such as @samp{0} and @samp{0U}.
-
 @node Properties of the @code{poly_int} comparisons
 @subsection Properties of the @code{poly_int} comparisons
 
diff --git a/gcc/doc/rtl.texi b/gcc/doc/rtl.texi
index 2156daf64a8..f583940b944 100644
--- a/gcc/doc/rtl.texi
+++ b/gcc/doc/rtl.texi
@@ -1599,7 +1599,12 @@ compute integer values.
 
 @findex MAX_BITSIZE_MODE_ANY_MODE
 @item MAX_BITSIZE_MODE_ANY_MODE
-The bitsize of the largest mode on the target.   
+The bitsize of the largest mode on the target.  The default value is
+the largest mode size given in the mode definition file, which is
+always correct for targets whose modes have a fixed size.  Targets
+that might increase the size of a mode beyond this default should define
+@code{MAX_BITSIZE_MODE_ANY_MODE} to the actual upper limit in
+@file{@var{machine}-modes.def}.
 @end table
 
 @findex byte_mode
@@ -4189,6 +4194,22 @@ is used in place of the actual insn pattern.  This is done in cases where
 the pattern is either complex or misleading.
 @end table
 
+The note @code{REG_CALL_NOCF_CHECK} is used in conjunction with the
+@option{-fcf-protection=branch} option.  The note is set if a
+@code{nocf_check} attribute is specified for a function type or a
+pointer to function type.  The note is stored in the @code{REG_NOTES}
+field of an insn.
+
+@table @code
+@findex REG_CALL_NOCF_CHECK
+@item REG_CALL_NOCF_CHECK
+Users have control through the @code{nocf_check} attribute to identify
+which calls to a function should be skipped from control-flow instrumentation
+when the option @option{-fcf-protection=branch} is specified.  The compiler
+puts a @code{REG_CALL_NOCF_CHECK} note on each @code{CALL_INSN} instruction
+that has a function type marked with a @code{nocf_check} attribute.
+@end table
+
 For convenience, the machine mode in an @code{insn_list} or
 @code{expr_list} is printed using these symbolic codes in debugging dumps.
 
@@ -4308,6 +4329,20 @@ There is only one @code{pc} expression.
 @item
 There is only one @code{cc0} expression.
 
+@cindex @code{const}, RTL sharing
+@item
+There is only one instance of the following structures for a given
+@var{m}, @var{x} and @var{y}:
+@example
+(const:@var{m} (vec_duplicate:@var{m} @var{x}))
+(const:@var{m} (vec_series:@var{m} @var{x} @var{y}))
+@end example
+This means, for example, that for a given @var{n} there is only ever a
+single instance of an expression like:
+@example
+(const:V@var{n}DI (vec_duplicate:V@var{n}DI (const_int 0)))
+@end example
+
 @cindex @code{const_double}, RTL sharing
 @item
 There is only one @code{const_double} expression with value 0 for
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index ff4ba5751d7..390bfcacccd 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -1373,6 +1373,10 @@ Target supports Fortran @code{real} kinds larger than @code{real(8)}.
 @subsubsection Vector-specific attributes
 
 @table @code
+@item vect_align_stack_vars
+The target's ABI allows stack variables to be aligned to the preferred
+vector alignment.
+
 @item vect_condition
 Target supports vector conditional operations.
 
@@ -1383,6 +1387,10 @@ have different type from the value operands.
 @item vect_double
 Target supports hardware vectors of @code{double}.
 
+@item vect_element_align_preferred
+The target's preferred vector alignment is the same as the element
+alignment.
+
 @item vect_float
 Target supports hardware vectors of @code{float}.
 
@@ -1395,6 +1403,9 @@ Target supports hardware vectors of @code{long}.
 @item vect_long_long
 Target supports hardware vectors of @code{long long}.
 
+@item vect_masked_store
+Target supports vector masked stores.
+
 @item vect_aligned_arrays
 Target aligns arrays to vector alignment boundary.
 
@@ -1448,9 +1459,43 @@ element types.
 @item vect_perm
 Target supports vector permutation.
 
+@item vect_perm_byte
+Target supports permutation of vectors with 8-bit elements.
+
+@item vect_perm_short
+Target supports permutation of vectors with 16-bit elements.
+
+@item vect_perm3_byte
+Target supports permutation of vectors with 8-bit elements, and for the
+default vector length it is possible to permute:
+@example
+@{ a0, a1, a2, b0, b1, b2, @dots{} @}
+@end example
+to:
+@example
+@{ a0, a0, a0, b0, b0, b0, @dots{} @}
+@{ a1, a1, a1, b1, b1, b1, @dots{} @}
+@{ a2, a2, a2, b2, b2, b2, @dots{} @}
+@end example
+using only two-vector permutes, regardless of how long the sequence is.
+
+@item vect_perm3_int
+Like @code{vect_perm3_byte}, but for 32-bit elements.
+
+@item vect_perm3_short
+Like @code{vect_perm3_byte}, but for 16-bit elements.
+
 @item vect_shift
 Target supports a hardware vector shift operation.
 
+@item vect_unaligned_possible
+Target prefers vectors to have an alignment greater than element
+alignment, but also allows unaligned vector accesses in some
+circumstances.
+
+@item vect_variable_length
+Target has variable-length vectors.
+
 @item vect_widen_sum_hi_to_si
 Target supports a vector widening summation of @code{short} operands
 into @code{int} results, or can promote (unpack) from @code{short}
@@ -1705,6 +1750,17 @@ ARM target supports executing instructions from ARMv8.2 with the FP16
 extension.  Some multilibs may be incompatible with these options.
 Implies arm_v8_2a_fp16_neon_ok and arm_v8_2a_fp16_scalar_hw.
 
+@item arm_v8_2a_dotprod_neon_ok
+@anchor{arm_v8_2a_dotprod_neon_ok}
+ARM target supports options to generate instructions from ARMv8.2 with
+the Dot Product extension. Some multilibs may be incompatible with these
+options.
+
+@item arm_v8_2a_dotprod_neon_hw
+ARM target supports executing instructions from ARMv8.2 with the Dot
+Product extension. Some multilibs may be incompatible with these options.
+Implies arm_v8_2a_dotprod_neon_ok.
+
 @item arm_prefer_ldrd_strd
 ARM target prefers @code{LDRD} and @code{STRD} instructions over
 @code{LDM} and @code{STM} instructions.
@@ -2311,6 +2367,11 @@ supported by the target; see the
 @ref{arm_v8_2a_fp16_neon_ok,,arm_v8_2a_fp16_neon_ok} effective target
 keyword.
 
+@item arm_v8_2a_dotprod_neon
+Add options for ARMv8.2 with Adv.SIMD Dot Product support, if this is
+supported by the target; see the
+@ref{arm_v8_2a_dotprod_neon_ok} effective target keyword.
+
 @item bind_pic_locally
 Add the target-specific flags needed to enable functions to bind
 locally when using pic/PIC passes in the testsuite.
@@ -2361,6 +2422,9 @@ Skip the test if the target does not support the @code{-fstack-check}
 option.  If @var{check} is @code{""}, support for @code{-fstack-check}
 is checked, for @code{-fstack-check=("@var{check}")} otherwise.
 
+@item dg-require-stack-size @var{size}
+Skip the test if the target does not support a stack size of @var{size}.
+
 @item dg-require-visibility @var{vis}
 Skip the test if the target does not support the @code{visibility} attribute.
 If @var{vis} is @code{""}, support for @code{visibility("hidden")} is
diff --git a/gcc/doc/standards.texi b/gcc/doc/standards.texi
index d4112b37863..a40899dba85 100644
--- a/gcc/doc/standards.texi
+++ b/gcc/doc/standards.texi
@@ -36,6 +36,8 @@ with some exceptions, and possibly with some extensions.
 @cindex C11
 @cindex ISO C1X
 @cindex C1X
+@cindex ISO C17
+@cindex C17
 @cindex Technical Corrigenda
 @cindex TC1
 @cindex Technical Corrigendum 1
@@ -100,7 +102,11 @@ in 2011 as ISO/IEC 9899:2011.  (While in development, drafts of this
 standard version were referred to as @dfn{C1X}.)
 GCC has substantially complete support
 for this standard, enabled with @option{-std=c11} or
-@option{-std=iso9899:2011}.  
+@option{-std=iso9899:2011}.  A version with corrections integrated is
+known as @dfn{C17} and is supported with @option{-std=c17} or
+@option{-std=iso9899:2017}; the corrections are also applied with
+@option{-std=c11}, and the only difference between the options is the
+value of @code{__STDC_VERSION__}.
 
 By default, GCC provides some extensions to the C language that, on
 rare occasions conflict with the C standard.  @xref{C
diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi
index a6bdb8ff277..acadc73667e 100644
--- a/gcc/doc/tm.texi
+++ b/gcc/doc/tm.texi
@@ -4313,6 +4313,17 @@ ISO/IEC TS 18661-3:2015; that is, @var{n} is one of 32, 64, 128, or,
 if @var{extended} is false, 16 or greater than 128 and a multiple of 32.
 @end deftypefn
 
+@deftypefn {Target Hook} bool TARGET_FLOATN_BUILTIN_P (int @var{func})
+Define this to return true if the @code{_Float@var{n}} and
+@code{_Float@var{n}x} built-in functions should implicitly enable the
+built-in function without the @code{__builtin_} prefix in addition to the
+normal built-in function with the @code{__builtin_} prefix.  The default is
+to only enable built-in functions without the @code{__builtin_} prefix for
+the GNU C langauge.  In strict ANSI/ISO mode, the built-in function without
+the @code{__builtin_} prefix is not enabled.  The argument @code{FUNC} is the
+@code{enum built_in_function} id of the function to be enabled.
+@end deftypefn
+
 @deftypefn {Target Hook} bool TARGET_SMALL_REGISTER_CLASSES_FOR_MODE_P (machine_mode @var{mode})
 Define this to return nonzero for machine modes for which the port has
 small register classes.  If this target hook returns nonzero for a given
@@ -7744,7 +7755,7 @@ for the file format in use is appropriate.
 @end defmac
 
 @deftypefn {Target Hook} void TARGET_ASM_OUTPUT_SOURCE_FILENAME (FILE *@var{file}, const char *@var{name})
-Output COFF information or DWARF debugging information which indicates that filename @var{name} is the current source file to the stdio stream @var{file}.
+Output DWARF debugging information which indicates that filename @var{name} is the current source file to the stdio stream @var{file}.
  
  This target hook need not be defined if the standard form of output for the file format in use is appropriate.
 @end deftypefn
@@ -9392,7 +9403,7 @@ This macro need only be defined if the target might save registers in the
 function prologue at an offset to the stack pointer that is not aligned to
 @code{UNITS_PER_WORD}.  The definition should be the negative minimum
 alignment if @code{STACK_GROWS_DOWNWARD} is true, and the positive
-minimum alignment otherwise.  @xref{SDB and DWARF}.  Only applicable if
+minimum alignment otherwise.  @xref{DWARF}.  Only applicable if
 the target supports DWARF 2 frame unwind information.
 @end defmac
 
@@ -9566,7 +9577,7 @@ This describes how to specify debugging information.
 * DBX Options::        Macros enabling specific options in DBX format.
 * DBX Hooks::          Hook macros for varying DBX format.
 * File Names and DBX:: Macros controlling output of file names in DBX format.
-* SDB and DWARF::      Macros for SDB (COFF) and DWARF formats.
+* DWARF::              Macros for DWARF format.
 * VMS Debug::          Macros for VMS debug format.
 @end menu
 
@@ -9600,9 +9611,8 @@ A C expression that returns the integer offset value for an automatic
 variable having address @var{x} (an RTL expression).  The default
 computation assumes that @var{x} is based on the frame-pointer and
 gives the offset from the frame-pointer.  This is required for targets
-that produce debugging output for DBX or COFF-style debugging output
-for SDB and allow the frame-pointer to be eliminated when the
-@option{-g} options is used.
+that produce debugging output for DBX and allow the frame-pointer to be
+eliminated when the @option{-g} option is used.
 @end defmac
 
 @defmac DEBUGGER_ARG_OFFSET (@var{offset}, @var{x})
@@ -9616,7 +9626,7 @@ A C expression that returns the type of debugging output GCC should
 produce when the user specifies just @option{-g}.  Define
 this if you have arranged for GCC to support more than one format of
 debugging output.  Currently, the allowable values are @code{DBX_DEBUG},
-@code{SDB_DEBUG}, @code{DWARF_DEBUG}, @code{DWARF2_DEBUG},
+@code{DWARF_DEBUG}, @code{DWARF2_DEBUG},
 @code{XCOFF_DEBUG}, @code{VMS_DEBUG}, and @code{VMS_AND_DWARF2_DEBUG}.
 
 When the user specifies @option{-ggdb}, GCC normally also uses the
@@ -9627,7 +9637,7 @@ defined, GCC uses @code{DBX_DEBUG}.
 
 The value of this macro only affects the default debugging output; the
 user can always get a specific type of output by using @option{-gstabs},
-@option{-gcoff}, @option{-gdwarf-2}, @option{-gxcoff}, or @option{-gvms}.
+@option{-gdwarf-2}, @option{-gxcoff}, or @option{-gvms}.
 @end defmac
 
 @node DBX Options
@@ -9845,16 +9855,11 @@ whose value is the highest absolute text address in the file.
 @end defmac
 
 @need 2000
-@node SDB and DWARF
-@subsection Macros for SDB and DWARF Output
+@node DWARF
+@subsection Macros for DWARF Output
 
 @c prevent bad page break with this line
-Here are macros for SDB and DWARF output.
-
-@defmac SDB_DEBUGGING_INFO
-Define this macro to 1 if GCC should produce COFF-style debugging output
-for SDB in response to the @option{-g} option.
-@end defmac
+Here are macros for DWARF output.
 
 @defmac DWARF2_DEBUGGING_INFO
 Define this macro if GCC should produce dwarf version 2 format
@@ -9960,40 +9965,6 @@ If defined, this target hook is a function which outputs a DTP-relative
 reference to the given TLS symbol of the specified size.
 @end deftypefn
 
-@defmac PUT_SDB_@dots{}
-Define these macros to override the assembler syntax for the special
-SDB assembler directives.  See @file{sdbout.c} for a list of these
-macros and their arguments.  If the standard syntax is used, you need
-not define them yourself.
-@end defmac
-
-@defmac SDB_DELIM
-Some assemblers do not support a semicolon as a delimiter, even between
-SDB assembler directives.  In that case, define this macro to be the
-delimiter to use (usually @samp{\n}).  It is not necessary to define
-a new set of @code{PUT_SDB_@var{op}} macros if this is the only change
-required.
-@end defmac
-
-@defmac SDB_ALLOW_UNKNOWN_REFERENCES
-Define this macro to allow references to unknown structure,
-union, or enumeration tags to be emitted.  Standard COFF does not
-allow handling of unknown references, MIPS ECOFF has support for
-it.
-@end defmac
-
-@defmac SDB_ALLOW_FORWARD_REFERENCES
-Define this macro to allow references to structure, union, or
-enumeration tags that have not yet been seen to be handled.  Some
-assemblers choke if forward tags are used, while some require it.
-@end defmac
-
-@defmac SDB_OUTPUT_SOURCE_LINE (@var{stream}, @var{line})
-A C statement to output SDB debugging information before code for line
-number @var{line} of the current source file to the stdio stream
-@var{stream}.  The default is to emit an @code{.ln} directive.
-@end defmac
-
 @need 2000
 @node VMS Debug
 @subsection Macros for VMS Debug Format
diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in
index b5e2771a831..7cbce20b877 100644
--- a/gcc/doc/tm.texi.in
+++ b/gcc/doc/tm.texi.in
@@ -3330,6 +3330,8 @@ stack.
 
 @hook TARGET_FLOATN_MODE
 
+@hook TARGET_FLOATN_BUILTIN_P
+
 @hook TARGET_SMALL_REGISTER_CLASSES_FOR_MODE_P
 
 @node Scalar Return
@@ -6458,7 +6460,7 @@ This macro need only be defined if the target might save registers in the
 function prologue at an offset to the stack pointer that is not aligned to
 @code{UNITS_PER_WORD}.  The definition should be the negative minimum
 alignment if @code{STACK_GROWS_DOWNWARD} is true, and the positive
-minimum alignment otherwise.  @xref{SDB and DWARF}.  Only applicable if
+minimum alignment otherwise.  @xref{DWARF}.  Only applicable if
 the target supports DWARF 2 frame unwind information.
 @end defmac
 
@@ -6582,7 +6584,7 @@ This describes how to specify debugging information.
 * DBX Options::        Macros enabling specific options in DBX format.
 * DBX Hooks::          Hook macros for varying DBX format.
 * File Names and DBX:: Macros controlling output of file names in DBX format.
-* SDB and DWARF::      Macros for SDB (COFF) and DWARF formats.
+* DWARF::              Macros for DWARF format.
 * VMS Debug::          Macros for VMS debug format.
 @end menu
 
@@ -6616,9 +6618,8 @@ A C expression that returns the integer offset value for an automatic
 variable having address @var{x} (an RTL expression).  The default
 computation assumes that @var{x} is based on the frame-pointer and
 gives the offset from the frame-pointer.  This is required for targets
-that produce debugging output for DBX or COFF-style debugging output
-for SDB and allow the frame-pointer to be eliminated when the
-@option{-g} options is used.
+that produce debugging output for DBX and allow the frame-pointer to be
+eliminated when the @option{-g} option is used.
 @end defmac
 
 @defmac DEBUGGER_ARG_OFFSET (@var{offset}, @var{x})
@@ -6632,7 +6633,7 @@ A C expression that returns the type of debugging output GCC should
 produce when the user specifies just @option{-g}.  Define
 this if you have arranged for GCC to support more than one format of
 debugging output.  Currently, the allowable values are @code{DBX_DEBUG},
-@code{SDB_DEBUG}, @code{DWARF_DEBUG}, @code{DWARF2_DEBUG},
+@code{DWARF_DEBUG}, @code{DWARF2_DEBUG},
 @code{XCOFF_DEBUG}, @code{VMS_DEBUG}, and @code{VMS_AND_DWARF2_DEBUG}.
 
 When the user specifies @option{-ggdb}, GCC normally also uses the
@@ -6643,7 +6644,7 @@ defined, GCC uses @code{DBX_DEBUG}.
 
 The value of this macro only affects the default debugging output; the
 user can always get a specific type of output by using @option{-gstabs},
-@option{-gcoff}, @option{-gdwarf-2}, @option{-gxcoff}, or @option{-gvms}.
+@option{-gdwarf-2}, @option{-gxcoff}, or @option{-gvms}.
 @end defmac
 
 @node DBX Options
@@ -6861,16 +6862,11 @@ whose value is the highest absolute text address in the file.
 @end defmac
 
 @need 2000
-@node SDB and DWARF
-@subsection Macros for SDB and DWARF Output
+@node DWARF
+@subsection Macros for DWARF Output
 
 @c prevent bad page break with this line
-Here are macros for SDB and DWARF output.
-
-@defmac SDB_DEBUGGING_INFO
-Define this macro to 1 if GCC should produce COFF-style debugging output
-for SDB in response to the @option{-g} option.
-@end defmac
+Here are macros for DWARF output.
 
 @defmac DWARF2_DEBUGGING_INFO
 Define this macro if GCC should produce dwarf version 2 format
@@ -6946,40 +6942,6 @@ is referenced by a function.
 
 @hook TARGET_ASM_OUTPUT_DWARF_DTPREL
 
-@defmac PUT_SDB_@dots{}
-Define these macros to override the assembler syntax for the special
-SDB assembler directives.  See @file{sdbout.c} for a list of these
-macros and their arguments.  If the standard syntax is used, you need
-not define them yourself.
-@end defmac
-
-@defmac SDB_DELIM
-Some assemblers do not support a semicolon as a delimiter, even between
-SDB assembler directives.  In that case, define this macro to be the
-delimiter to use (usually @samp{\n}).  It is not necessary to define
-a new set of @code{PUT_SDB_@var{op}} macros if this is the only change
-required.
-@end defmac
-
-@defmac SDB_ALLOW_UNKNOWN_REFERENCES
-Define this macro to allow references to unknown structure,
-union, or enumeration tags to be emitted.  Standard COFF does not
-allow handling of unknown references, MIPS ECOFF has support for
-it.
-@end defmac
-
-@defmac SDB_ALLOW_FORWARD_REFERENCES
-Define this macro to allow references to structure, union, or
-enumeration tags that have not yet been seen to be handled.  Some
-assemblers choke if forward tags are used, while some require it.
-@end defmac
-
-@defmac SDB_OUTPUT_SOURCE_LINE (@var{stream}, @var{line})
-A C statement to output SDB debugging information before code for line
-number @var{line} of the current source file to the stdio stream
-@var{stream}.  The default is to emit an @code{.ln} directive.
-@end defmac
-
 @need 2000
 @node VMS Debug
 @subsection Macros for VMS Debug Format
diff --git a/gcc/dojump.c b/gcc/dojump.c
index 97c4100b663..9efc4a465ba 100644
--- a/gcc/dojump.c
+++ b/gcc/dojump.c
@@ -89,7 +89,7 @@ do_pending_stack_adjust (void)
 {
   if (inhibit_defer_pop == 0)
     {
-      if (maybe_nonzero (pending_stack_adjust))
+      if (may_ne (pending_stack_adjust, 0))
 	adjust_stack (gen_int_mode (pending_stack_adjust, Pmode));
       pending_stack_adjust = 0;
     }
diff --git a/gcc/dse.c b/gcc/dse.c
index 90ec76a36f7..8ff2137876f 100644
--- a/gcc/dse.c
+++ b/gcc/dse.c
@@ -1492,7 +1492,7 @@ record_store (rtx body, bb_info_t bb_info)
       group_info *group = rtx_group_vec[group_id];
       mem_addr = group->canon_base_addr;
     }
-  if (maybe_nonzero (offset))
+  if (may_ne (offset, 0))
     mem_addr = plus_constant (get_address_mode (mem), mem_addr, offset);
 
   while (ptr)
@@ -1827,7 +1827,7 @@ get_stored_val (store_info *store_info, machine_mode read_mode,
   else
     gap = read_offset - store_info->offset;
 
-  if (maybe_nonzero (gap))
+  if (may_ne (gap, 0))
     {
       poly_int64 shift = gap * BITS_PER_UNIT;
       poly_int64 access_size = GET_MODE_SIZE (read_mode) + gap;
@@ -2099,7 +2099,7 @@ check_mem_read_rtx (rtx *loc, bb_info_t bb_info)
       group_info *group = rtx_group_vec[group_id];
       mem_addr = group->canon_base_addr;
     }
-  if (maybe_nonzero (offset))
+  if (may_ne (offset, 0))
     mem_addr = plus_constant (get_address_mode (mem), mem_addr, offset);
 
   if (group_id >= 0)
diff --git a/gcc/dwarf2cfi.c b/gcc/dwarf2cfi.c
index 82e6571964b..ce1f1a21124 100644
--- a/gcc/dwarf2cfi.c
+++ b/gcc/dwarf2cfi.c
@@ -793,14 +793,17 @@ def_cfa_0 (dw_cfa_location *old_cfa, dw_cfa_location *new_cfa)
 	cfi->dw_cfi_opc = DW_CFA_def_cfa_offset;
       cfi->dw_cfi_oprnd1.dw_cfi_offset = const_offset;
     }
-  else if (must_eq (new_cfa->offset, old_cfa->offset)
+  else if (new_cfa->offset.is_constant ()
+	   && must_eq (new_cfa->offset, old_cfa->offset)
 	   && old_cfa->reg != INVALID_REGNUM
 	   && !new_cfa->indirect
 	   && !old_cfa->indirect)
     {
       /* Construct a "DW_CFA_def_cfa_register <register>" instruction,
 	 indicating the CFA register has changed to <register> but the
-	 offset has not changed.  */
+	 offset has not changed.  This requires the old CFA to have
+	 been set as a register plus offset rather than a general
+	 DW_CFA_def_cfa_expression.  */
       cfi->dw_cfi_opc = DW_CFA_def_cfa_register;
       cfi->dw_cfi_oprnd1.dw_cfi_reg_num = new_cfa->reg;
     }
@@ -937,7 +940,7 @@ notice_args_size (rtx_insn *insn)
 
   args_size = get_args_size (note);
   delta = args_size - cur_trace->end_true_args_size;
-  if (known_zero (delta))
+  if (must_eq (delta, 0))
     return;
 
   cur_trace->end_true_args_size = args_size;
@@ -1946,7 +1949,7 @@ dwarf2out_frame_debug_expr (rtx expr)
 	{
 	  /* We're storing the current CFA reg into the stack.  */
 
-	  if (known_zero (cur_cfa->offset))
+	  if (must_eq (cur_cfa->offset, 0))
 	    {
               /* Rule 19 */
               /* If stack is aligned, putting CFA reg into stack means
@@ -2358,7 +2361,7 @@ maybe_record_trace_start_abnormal (rtx_insn *start, rtx_insn *origin)
   dw_cfa_location save_cfa;
 
   save_args_size = cur_trace->end_true_args_size;
-  if (known_zero (save_args_size))
+  if (must_eq (save_args_size, 0))
     {
       maybe_record_trace_start (start, origin);
       return;
diff --git a/gcc/dwarf2out.c b/gcc/dwarf2out.c
index 82b46169a18..c0f93d763f5 100644
--- a/gcc/dwarf2out.c
+++ b/gcc/dwarf2out.c
@@ -1489,7 +1489,7 @@ loc_descr_plus_const (dw_loc_descr_ref *list_head, poly_int64 poly_offset)
 
   gcc_assert (*list_head != NULL);
 
-  if (known_zero (poly_offset))
+  if (must_eq (poly_offset, 0))
     return;
 
   /* Find the end of the chain.  */
@@ -5381,6 +5381,16 @@ splice_child_die (dw_die_ref parent, dw_die_ref child)
   reparent_child (child, parent);
 }
 
+/* Create and return a new die with TAG_VALUE as tag.  */
+ 
+static inline dw_die_ref
+new_die_raw (enum dwarf_tag tag_value)
+{
+  dw_die_ref die = ggc_cleared_alloc<die_node> ();
+  die->die_tag = tag_value;
+  return die;
+}
+
 /* Create and return a new die with a parent of PARENT_DIE.  If
    PARENT_DIE is NULL, the new DIE is placed in limbo and an
    associated tree T must be supplied to determine parenthood
@@ -5389,9 +5399,7 @@ splice_child_die (dw_die_ref parent, dw_die_ref child)
 static inline dw_die_ref
 new_die (enum dwarf_tag tag_value, dw_die_ref parent_die, tree t)
 {
-  dw_die_ref die = ggc_cleared_alloc<die_node> ();
-
-  die->die_tag = tag_value;
+  dw_die_ref die = new_die_raw (tag_value);
 
   if (parent_die != NULL)
     add_child_die (parent_die, die);
@@ -5585,8 +5593,7 @@ add_AT_external_die_ref (dw_die_ref die, enum dwarf_attribute attr_kind,
 {
   /* Create a fake DIE that contains the reference.  Don't use
      new_die because we don't want to end up in the limbo list.  */
-  dw_die_ref ref = ggc_cleared_alloc<die_node> ();
-  ref->die_tag = die->die_tag;
+  dw_die_ref ref = new_die_raw (die->die_tag);
   ref->die_id.die_symbol = IDENTIFIER_POINTER (get_identifier (symbol));
   ref->die_offset = offset;
   ref->with_offset = 1;
@@ -7727,13 +7734,10 @@ should_move_die_to_comdat (dw_die_ref die)
 static dw_die_ref
 clone_die (dw_die_ref die)
 {
-  dw_die_ref clone;
+  dw_die_ref clone = new_die_raw (die->die_tag);
   dw_attr_node *a;
   unsigned ix;
 
-  clone = ggc_cleared_alloc<die_node> ();
-  clone->die_tag = die->die_tag;
-
   FOR_EACH_VEC_SAFE_ELT (die->die_attr, ix, a)
     add_dwarf_attr (clone, a);
 
@@ -7777,8 +7781,7 @@ clone_as_declaration (dw_die_ref die)
       return clone;
     }
 
-  clone = ggc_cleared_alloc<die_node> ();
-  clone->die_tag = die->die_tag;
+  clone = new_die_raw (die->die_tag);
 
   FOR_EACH_VEC_SAFE_ELT (die->die_attr, ix, a)
     {
@@ -12105,9 +12108,6 @@ base_type_die (tree type, bool reverse)
   struct fixed_point_type_info fpt_info;
   tree type_bias = NULL_TREE;
 
-  if (TREE_CODE (type) == ERROR_MARK || TREE_CODE (type) == VOID_TYPE)
-    return 0;
-
   /* If this is a subtype that should not be emitted as a subrange type,
      use the base type.  See subrange_type_for_debug_p.  */
   if (TREE_CODE (type) == INTEGER_TYPE && TREE_TYPE (type) != NULL_TREE)
@@ -12200,7 +12200,7 @@ base_type_die (tree type, bool reverse)
       gcc_unreachable ();
     }
 
-  base_type_result = new_die (DW_TAG_base_type, comp_unit_die (), type);
+  base_type_result = new_die_raw (DW_TAG_base_type);
 
   add_AT_unsigned (base_type_result, DW_AT_byte_size,
 		   int_size_in_bytes (type));
@@ -12256,8 +12256,6 @@ base_type_die (tree type, bool reverse)
 		     | dw_scalar_form_reference,
 		     NULL);
 
-  add_pubtype (type, base_type_result);
-
   return base_type_result;
 }
 
@@ -12285,8 +12283,6 @@ is_base_type (tree type)
 {
   switch (TREE_CODE (type))
     {
-    case ERROR_MARK:
-    case VOID_TYPE:
     case INTEGER_TYPE:
     case REAL_TYPE:
     case FIXED_POINT_TYPE:
@@ -12295,6 +12291,7 @@ is_base_type (tree type)
     case POINTER_BOUNDS_TYPE:
       return 1;
 
+    case VOID_TYPE:
     case ARRAY_TYPE:
     case RECORD_TYPE:
     case UNION_TYPE:
@@ -12500,6 +12497,8 @@ modified_type_die (tree type, int cv_quals, bool reverse,
   /* Only these cv-qualifiers are currently handled.  */
   const int cv_qual_mask = (TYPE_QUAL_CONST | TYPE_QUAL_VOLATILE
 			    | TYPE_QUAL_RESTRICT | TYPE_QUAL_ATOMIC);
+  const bool reverse_base_type
+    = need_endianity_attribute_p (reverse) && is_base_type (type);
 
   if (code == ERROR_MARK)
     return NULL;
@@ -12550,29 +12549,33 @@ modified_type_die (tree type, int cv_quals, bool reverse,
 	qualified_type = size_type_node;
     }
 
-
   /* If we do, then we can just use its DIE, if it exists.  */
   if (qualified_type)
     {
       mod_type_die = lookup_type_die (qualified_type);
 
-      /* DW_AT_endianity doesn't come from a qualifier on the type.  */
+      /* DW_AT_endianity doesn't come from a qualifier on the type, so it is
+	 dealt with specially: the DIE with the attribute, if it exists, is
+	 placed immediately after the regular DIE for the same base type.  */
       if (mod_type_die
-	  && (!need_endianity_attribute_p (reverse)
-	      || !is_base_type (type)
-	      || get_AT_unsigned (mod_type_die, DW_AT_endianity)))
+	  && (!reverse_base_type
+	      || ((mod_type_die = mod_type_die->die_sib) != NULL
+		  && get_AT_unsigned (mod_type_die, DW_AT_endianity))))
 	return mod_type_die;
     }
 
   name = qualified_type ? TYPE_NAME (qualified_type) : NULL;
 
   /* Handle C typedef types.  */
-  if (name && TREE_CODE (name) == TYPE_DECL && DECL_ORIGINAL_TYPE (name)
+  if (name
+      && TREE_CODE (name) == TYPE_DECL
+      && DECL_ORIGINAL_TYPE (name)
       && !DECL_ARTIFICIAL (name))
     {
       tree dtype = TREE_TYPE (name);
 
-      if (qualified_type == dtype)
+      /* Skip the typedef for base types with DW_AT_endianity, no big deal.  */
+      if (qualified_type == dtype && !reverse_base_type)
 	{
 	  tree origin = decl_ultimate_origin (name);
 
@@ -12685,8 +12688,7 @@ modified_type_die (tree type, int cv_quals, bool reverse,
 	      }
 	    if (first)
 	      {
-		d = ggc_cleared_alloc<die_node> ();
-		d->die_tag = dwarf_qual_info[i].t;
+		d = new_die_raw (dwarf_qual_info[i].t);
 		add_child_die_after (mod_scope, d, last);
 		last = d;
 	      }
@@ -12744,7 +12746,21 @@ modified_type_die (tree type, int cv_quals, bool reverse,
       item_type = TREE_TYPE (type);
     }
   else if (is_base_type (type))
-    mod_type_die = base_type_die (type, reverse);
+    {
+      mod_type_die = base_type_die (type, reverse);
+
+      /* The DIE with DW_AT_endianity is placed right after the naked DIE.  */
+      if (reverse_base_type)
+	{
+	  dw_die_ref after_die
+	    = modified_type_die (type, cv_quals, false, context_die);
+	  add_child_die_after (comp_unit_die (), mod_type_die, after_die);
+	}
+      else
+	add_child_die (comp_unit_die (), mod_type_die);
+
+      add_pubtype (type, mod_type_die);
+    }
   else
     {
       gen_type_die (type, context_die);
@@ -12806,7 +12822,7 @@ modified_type_die (tree type, int cv_quals, bool reverse,
 			  name ? IDENTIFIER_POINTER (name) : "__unknown__");
     }
 
-  if (qualified_type)
+  if (qualified_type && !reverse_base_type)
     equate_type_number_to_die (qualified_type, mod_type_die);
 
   if (item_type)
@@ -13760,7 +13776,7 @@ tls_mem_loc_descriptor (rtx mem)
   if (loc_result == NULL)
     return NULL;
 
-  if (maybe_nonzero (MEM_OFFSET (mem)))
+  if (may_ne (MEM_OFFSET (mem), 0))
     loc_descr_plus_const (&loc_result, MEM_OFFSET (mem));
 
   return loc_result;
@@ -13791,9 +13807,17 @@ expansion_failed (tree expr, rtx rtl, char const *reason)
 static bool
 const_ok_for_output_1 (rtx rtl)
 {
-  if (GET_CODE (rtl) == UNSPEC)
+  if (targetm.const_not_ok_for_debug_p (rtl))
     {
-      /* If delegitimize_address couldn't do anything with the UNSPEC, assume
+      if (GET_CODE (rtl) != UNSPEC)
+	{
+	  expansion_failed (NULL_TREE, rtl,
+			    "Expression rejected for debug by the backend.\n");
+	  return false;
+	}
+
+      /* If delegitimize_address couldn't do anything with the UNSPEC, and
+	 the target hook doesn't explicitly allow it in debug info, assume
 	 we can't express it in the debug info.  */
       /* Don't complain about TLS UNSPECs, those are just too hard to
 	 delegitimize.  Note this could be a non-decl SYMBOL_REF such as
@@ -16668,7 +16692,7 @@ loc_list_for_address_of_addr_expr_of_indirect_ref (tree loc, bool toplev,
 			NULL_RTX, "no indirect ref in inner refrence");
       return 0;
     }
-  if (!offset && known_zero (bitpos))
+  if (!offset && must_eq (bitpos, 0))
     list_ret = loc_list_from_tree (TREE_OPERAND (obj, 0), toplev ? 2 : 1,
 				   context);
   else if (toplev
@@ -16694,7 +16718,7 @@ loc_list_for_address_of_addr_expr_of_indirect_ref (tree loc, bool toplev,
       if (bytepos.is_constant (&value) && value > 0)
 	add_loc_descr_to_each (list_ret,
 			       new_loc_descr (DW_OP_plus_uconst, value, 0));
-      else if (maybe_nonzero (bytepos))
+      else if (may_ne (bytepos, 0))
 	loc_list_plus_const (list_ret, bytepos);
       add_loc_descr_to_each (list_ret,
 			     new_loc_descr (DW_OP_stack_value, 0, 0));
@@ -17675,7 +17699,7 @@ loc_list_from_tree_1 (tree loc, int want_address,
 
 	list_ret = loc_list_from_tree_1 (obj,
 					 want_address == 2
-					 && known_zero (bitpos)
+					 && must_eq (bitpos, 0)
 					 && !offset ? 2 : 1,
 					 context);
 	/* TODO: We can extract value of the small expression via shifting even
@@ -17706,7 +17730,7 @@ loc_list_from_tree_1 (tree loc, int want_address,
 	if (bytepos.is_constant (&value) && value > 0)
 	  add_loc_descr_to_each (list_ret, new_loc_descr (DW_OP_plus_uconst,
 							  value, 0));
-	else if (maybe_nonzero (bytepos))
+	else if (may_ne (bytepos, 0))
 	  loc_list_plus_const (list_ret, bytepos);
 
 	have_address = 1;
@@ -19188,7 +19212,7 @@ rtl_for_decl_location (tree decl)
 	 fact we are not.  We need to adjust the offset of the
 	 storage location to reflect the actual value's bytes,
 	 else gdb will not be able to display it.  */
-      if (maybe_nonzero (offset))
+      if (may_ne (offset, 0))
 	rtl = gen_rtx_MEM (TYPE_MODE (TREE_TYPE (decl)),
 			   plus_constant (addr_mode, XEXP (rtl, 0), offset));
     }
@@ -20599,8 +20623,7 @@ dwarf2out_vms_debug_main_pointer (void)
   dw_die_ref die;
 
   /* Allocate the VMS debug main subprogram die.  */
-  die = ggc_cleared_alloc<die_node> ();
-  die->die_tag = DW_TAG_subprogram;
+  die = new_die_raw (DW_TAG_subprogram);
   add_name_attribute (die, VMS_DEBUG_MAIN_POINTER);
   ASM_GENERATE_INTERNAL_LABEL (label, PROLOGUE_END_LABEL,
 			       current_function_funcdef_no);
@@ -21022,10 +21045,10 @@ gen_array_type_die (tree type, dw_die_ref context_die)
     add_AT_unsigned (array_die, DW_AT_ordering, DW_ORD_col_major);
 
 #if 0
-  /* We default the array ordering.  SDB will probably do
-     the right things even if DW_AT_ordering is not present.  It's not even
-     an issue until we start to get into multidimensional arrays anyway.  If
-     SDB is ever caught doing the Wrong Thing for multi-dimensional arrays,
+  /* We default the array ordering.  Debuggers will probably do the right
+     things even if DW_AT_ordering is not present.  It's not even an issue
+     until we start to get into multidimensional arrays anyway.  If a debugger
+     is ever caught doing the Wrong Thing for multi-dimensional arrays,
      then we'll have to put the DW_AT_ordering attribute back in.  (But if
      and when we find out that we need to put these in, we will only do so
      for multidimensional arrays.  */
@@ -23517,6 +23540,8 @@ highest_c_language (const char *lang1, const char *lang2)
   if (strcmp ("GNU C++98", lang1) == 0 || strcmp ("GNU C++98", lang2) == 0)
     return "GNU C++98";
 
+  if (strcmp ("GNU C17", lang1) == 0 || strcmp ("GNU C17", lang2) == 0)
+    return "GNU C17";
   if (strcmp ("GNU C11", lang1) == 0 || strcmp ("GNU C11", lang2) == 0)
     return "GNU C11";
   if (strcmp ("GNU C99", lang1) == 0 || strcmp ("GNU C99", lang2) == 0)
@@ -23593,7 +23618,8 @@ gen_compile_unit_die (const char *filename)
 	    language = DW_LANG_C99;
 
 	  if (dwarf_version >= 5 /* || !dwarf_strict */)
-	    if (strcmp (language_string, "GNU C11") == 0)
+	    if (strcmp (language_string, "GNU C11") == 0
+		|| strcmp (language_string, "GNU C17") == 0)
 	      language = DW_LANG_C11;
 	}
     }
diff --git a/gcc/early-remat.c b/gcc/early-remat.c
index a6e5b797dee..94e87d96ffe 100644
--- a/gcc/early-remat.c
+++ b/gcc/early-remat.c
@@ -672,10 +672,11 @@ early_remat::dump_block_info (basic_block bb)
   fprintf (dump_file, "\n;;%*s:", width, "successors");
   dump_edge_list (bb, true);
 
-  fprintf (dump_file, "\n;;%*s: %d", width, "Frequency", bb->frequency);
+  fprintf (dump_file, "\n;;%*s: %d", width, "frequency",
+	   bb->count.to_frequency (m_fn));
 
   if (info->last_call)
-    fprintf (dump_file, "\n;;%*s: %d", width, "Last call",
+    fprintf (dump_file, "\n;;%*s: %d", width, "last call",
 	     INSN_UID (info->last_call));
 
   if (!empty_p (info->rd_in))
@@ -2218,7 +2219,7 @@ early_remat::local_remat_cheaper_p (unsigned int query_bb_index)
 	  }
 	else if (m_block_info[e->src->index].last_call)
 	  /* We'll rematerialize after the call.  */
-	  frequency += e->src->frequency;
+	  frequency += e->src->count.to_frequency (m_fn);
 	else if (m_block_info[e->src->index].remat_frequency_valid_p)
 	  /* Add the cost of rematerializing at the head of E->src
 	     or in its predecessors (whichever is cheaper).  */
@@ -2234,10 +2235,11 @@ early_remat::local_remat_cheaper_p (unsigned int query_bb_index)
 
       /* If rematerializing in and before the block have equal cost, prefer
 	 rematerializing in the block.  This should shorten the live range.  */
-      if (!can_move_p || frequency >= bb->frequency)
+      int bb_frequency = bb->count.to_frequency (m_fn);
+      if (!can_move_p || frequency >= bb_frequency)
 	{
 	  info->local_remat_cheaper_p = true;
-	  info->remat_frequency = bb->frequency;
+	  info->remat_frequency = bb_frequency;
 	}
       else
 	info->remat_frequency = frequency;
@@ -2252,7 +2254,7 @@ early_remat::local_remat_cheaper_p (unsigned int query_bb_index)
 	    {
 	      fprintf (dump_file, ";; Block %d has frequency %d,"
 		       " rematerializing in predecessors has frequency %d",
-		       bb->index, bb->frequency, frequency);
+		       bb->index, bb_frequency, frequency);
 	      if (info->local_remat_cheaper_p)
 		fprintf (dump_file, "; prefer to rematerialize"
 			 " in the block\n");
diff --git a/gcc/emit-rtl.c b/gcc/emit-rtl.c
index 28e6dd85e0d..af4a038d75a 100644
--- a/gcc/emit-rtl.c
+++ b/gcc/emit-rtl.c
@@ -450,7 +450,7 @@ get_reg_attrs (tree decl, poly_int64 offset)
   reg_attrs attrs;
 
   /* If everything is the default, we can just return zero.  */
-  if (decl == 0 && known_zero (offset))
+  if (decl == 0 && must_eq (offset, 0))
     return 0;
 
   attrs.decl = decl;
@@ -982,7 +982,7 @@ validate_subreg (machine_mode omode, machine_mode imode,
 
   /* Paradoxical subregs must have offset zero.  */
   if (may_gt (osize, isize))
-    return known_zero (offset);
+    return must_eq (offset, 0U);
 
   /* This is a normal subreg.  Verify that the offset is representable.  */
 
@@ -1028,7 +1028,7 @@ validate_subreg (machine_mode omode, machine_mode imode,
       if (!can_div_trunc_p (offset, block_size, &start_reg, &offset_within_reg)
 	  || (BYTES_BIG_ENDIAN
 	      ? may_ne (offset_within_reg, block_size - osize)
-	      : maybe_nonzero (offset_within_reg)))
+	      : may_ne (offset_within_reg, 0U)))
 	return false;
     }
   return true;
@@ -1156,7 +1156,7 @@ subreg_memory_offset (machine_mode outer_mode, machine_mode inner_mode,
 {
   if (paradoxical_subreg_p (outer_mode, inner_mode))
     {
-      gcc_assert (known_zero (offset));
+      gcc_assert (must_eq (offset, 0U));
       return -subreg_lowpart_offset (inner_mode, outer_mode);
     }
   return offset;
@@ -2178,7 +2178,7 @@ set_mem_attributes_minus_bitpos (rtx ref, tree t, int objectp,
   /* If we modified OFFSET based on T, then subtract the outstanding
      bit position offset.  Similarly, increase the size of the accessed
      object to contain the negative offset.  */
-  if (maybe_nonzero (apply_bitpos))
+  if (may_ne (apply_bitpos, 0))
     {
       gcc_assert (attrs.offset_known_p);
       poly_int64 bytepos = bits_to_bytes_round_down (apply_bitpos);
@@ -2402,8 +2402,8 @@ adjust_address_1 (rtx memref, machine_mode mode, poly_int64 offset,
 
   /* If there are no changes, just return the original memory reference.  */
   if (mode == GET_MODE (memref)
-      && known_zero (offset)
-      && (known_zero (size)
+      && must_eq (offset, 0)
+      && (must_eq (size, 0)
 	  || (attrs.size_known_p && must_eq (attrs.size, size)))
       && (!validate || memory_address_addr_space_p (mode, addr,
 						    attrs.addrspace)))
@@ -2451,7 +2451,7 @@ adjust_address_1 (rtx memref, machine_mode mode, poly_int64 offset,
 
   /* If the address is a REG, change_address_1 rightfully returns memref,
      but this would destroy memref's MEM_ATTRS.  */
-  if (new_rtx == memref && maybe_nonzero (offset))
+  if (new_rtx == memref && may_ne (offset, 0))
     new_rtx = copy_rtx (new_rtx);
 
   /* Conservatively drop the object if we don't know where we start from.  */
@@ -2478,13 +2478,13 @@ adjust_address_1 (rtx memref, machine_mode mode, poly_int64 offset,
   /* Compute the new alignment by taking the MIN of the alignment and the
      lowest-order set bit in OFFSET, but don't change the alignment if OFFSET
      if zero.  */
-  if (maybe_nonzero (offset))
+  if (may_ne (offset, 0))
     {
       max_align = known_alignment (offset) * BITS_PER_UNIT;
       attrs.align = MIN (attrs.align, max_align);
     }
 
-  if (maybe_nonzero (size))
+  if (may_ne (size, 0))
     {
       /* Drop the object if the new right end is not within its bounds.  */
       if (adjust_object && may_gt (offset + size, attrs.size))
@@ -3935,6 +3935,7 @@ try_split (rtx pat, rtx_insn *trial, int last)
 	case REG_NORETURN:
 	case REG_SETJMP:
 	case REG_TM:
+	case REG_CALL_NOCF_CHECK:
 	  for (insn = insn_last; insn != NULL_RTX; insn = PREV_INSN (insn))
 	    {
 	      if (CALL_P (insn))
diff --git a/gcc/except.c b/gcc/except.c
index 041f89a55e5..30f303a7bf8 100644
--- a/gcc/except.c
+++ b/gcc/except.c
@@ -1003,7 +1003,6 @@ dw2_build_landing_pads (void)
 
       bb = emit_to_new_bb_before (seq, label_rtx (lp->post_landing_pad));
       bb->count = bb->next_bb->count;
-      bb->frequency = bb->next_bb->frequency;
       make_single_succ_edge (bb, bb->next_bb, e_flags);
       if (current_loops)
 	{
diff --git a/gcc/explow.c b/gcc/explow.c
index 5d738c1c422..4c99d4e2871 100644
--- a/gcc/explow.c
+++ b/gcc/explow.c
@@ -102,7 +102,7 @@ plus_constant (machine_mode mode, rtx x, poly_int64 c, bool inplace)
 
   gcc_assert (GET_MODE (x) == VOIDmode || GET_MODE (x) == mode);
 
-  if (known_zero (c))
+  if (must_eq (c, 0))
     return x;
 
  restart:
@@ -195,7 +195,7 @@ plus_constant (machine_mode mode, rtx x, poly_int64 c, bool inplace)
       break;
     }
 
-  if (maybe_nonzero (c))
+  if (may_ne (c, 0))
     x = gen_rtx_PLUS (mode, x, gen_int_mode (c, mode));
 
   if (GET_CODE (x) == SYMBOL_REF || GET_CODE (x) == LABEL_REF)
@@ -1334,6 +1334,9 @@ get_stack_check_protect (void)
    REQUIRED_ALIGN is the alignment (in bits) required for the region
    of memory.
 
+   MAX_SIZE is an upper bound for SIZE, if SIZE is not constant, or -1 if
+   no such upper bound is known.
+
    If CANNOT_ACCUMULATE is set to TRUE, the caller guarantees that the
    stack space allocated by the generated code cannot be added with itself
    in the course of the execution of the function.  It is always safe to
@@ -1343,7 +1346,9 @@ get_stack_check_protect (void)
 
 rtx
 allocate_dynamic_stack_space (rtx size, unsigned size_align,
-			      unsigned required_align, bool cannot_accumulate)
+			      unsigned required_align,
+			      HOST_WIDE_INT max_size,
+			      bool cannot_accumulate)
 {
   HOST_WIDE_INT stack_usage_size = -1;
   rtx_code_label *final_label;
@@ -1382,8 +1387,12 @@ allocate_dynamic_stack_space (rtx size, unsigned size_align,
 	    }
 	}
 
-      /* If the size is not constant, we can't say anything.  */
-      if (stack_usage_size == -1)
+      /* If the size is not constant, try the maximum size.  */
+      if (stack_usage_size < 0)
+	stack_usage_size = max_size;
+
+      /* If the size is still not constant, we can't say anything.  */
+      if (stack_usage_size < 0)
 	{
 	  current_function_has_unbounded_dynamic_stack_size = 1;
 	  stack_usage_size = 0;
diff --git a/gcc/explow.h b/gcc/explow.h
index c79065113b5..e981a656254 100644
--- a/gcc/explow.h
+++ b/gcc/explow.h
@@ -94,7 +94,8 @@ extern void update_nonlocal_goto_save_area (void);
 extern void record_new_stack_level (void);
 
 /* Allocate some space on the stack dynamically and return its address.  */
-extern rtx allocate_dynamic_stack_space (rtx, unsigned, unsigned, bool);
+extern rtx allocate_dynamic_stack_space (rtx, unsigned, unsigned,
+					 HOST_WIDE_INT, bool);
 
 /* Calculate the necessary size of a constant dynamic stack allocation from the
    size of the variable area.  */
diff --git a/gcc/expmed.c b/gcc/expmed.c
index e99ff2e3dde..4565bcce99e 100644
--- a/gcc/expmed.c
+++ b/gcc/expmed.c
@@ -561,7 +561,7 @@ strict_volatile_bitfield_p (rtx op0, unsigned HOST_WIDE_INT bitsize,
     return false;
 
   /* Check for cases where the C++ memory model applies.  */
-  if (maybe_nonzero (bitregion_end)
+  if (may_ne (bitregion_end, 0U)
       && (may_lt (bitnum - bitnum % modesize, bitregion_start)
 	  || may_gt (bitnum - bitnum % modesize + modesize - 1,
 		     bitregion_end)))
@@ -779,7 +779,7 @@ store_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
       rtx sub;
       HOST_WIDE_INT regnum;
       poly_uint64 regsize = REGMODE_NATURAL_SIZE (GET_MODE (op0));
-      if (known_zero (bitnum)
+      if (must_eq (bitnum, 0U)
 	  && must_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
 	{
 	  sub = simplify_gen_subreg (GET_MODE (op0), value, fieldmode, 0);
@@ -1129,7 +1129,7 @@ store_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
   /* Under the C++0x memory model, we must not touch bits outside the
      bit region.  Adjust the address to start at the beginning of the
      bit region.  */
-  if (MEM_P (str_rtx) && maybe_nonzero (bitregion_start))
+  if (MEM_P (str_rtx) && may_ne (bitregion_start, 0U))
     {
       scalar_int_mode best_mode;
       machine_mode addr_mode = VOIDmode;
@@ -1370,7 +1370,7 @@ store_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
 	 UNIT close to the end of the region as needed.  If op0 is a REG
 	 or SUBREG of REG, don't do this, as there can't be data races
 	 on a register and we can expand shorter code in some cases.  */
-      if (maybe_nonzero (bitregion_end)
+      if (may_ne (bitregion_end, 0U)
 	  && unit > BITS_PER_UNIT
 	  && may_gt (bitpos + bitsdone - thispos + unit, bitregion_end + 1)
 	  && !REG_P (op0)
@@ -1617,7 +1617,7 @@ extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
 
   if (REG_P (op0)
       && mode == GET_MODE (op0)
-      && known_zero (bitnum)
+      && must_eq (bitnum, 0U)
       && must_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
     {
       if (reverse)
@@ -1631,7 +1631,7 @@ extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
       && !MEM_P (op0)
       && VECTOR_MODE_P (tmode)
       && must_eq (bitsize, GET_MODE_SIZE (tmode))
-      && may_gt (GET_MODE (op0), GET_MODE_SIZE (tmode)))
+      && may_gt (GET_MODE_SIZE (GET_MODE (op0)), GET_MODE_SIZE (tmode)))
     {
       machine_mode new_mode = GET_MODE (op0);
       if (GET_MODE_INNER (new_mode) != GET_MODE_INNER (tmode))
@@ -1708,13 +1708,14 @@ extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
     }
 
   /* Use vec_extract patterns for extracting parts of vectors whenever
-     available.  */
+     available.  If that fails, see whether the current modes and bitregion
+     give a natural subreg.  */
   machine_mode outermode = GET_MODE (op0);
   if (VECTOR_MODE_P (outermode) && !MEM_P (op0))
     {
       scalar_mode innermode = GET_MODE_INNER (outermode);
-      enum insn_code icode = convert_optab_handler (vec_extract_optab,
-						    outermode, innermode);
+      enum insn_code icode
+	= convert_optab_handler (vec_extract_optab, outermode, innermode);
       poly_uint64 pos;
       if (icode != CODE_FOR_nothing
 	  && must_eq (bitsize, GET_MODE_BITSIZE (innermode))
@@ -1736,6 +1737,15 @@ extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
 	      return target;
 	    }
 	}
+      /* Using subregs is useful if we're extracting the least-significant
+	 vector element, or if we're extracting one register vector from
+	 a multi-register vector.  extract_bit_field_as_subreg checks
+	 for valid bitsize and bitnum, so we don't need to do that here.
+
+	 The mode check makes sure that we're extracting either
+	 a single element or a subvector with the same element type.
+	 If the modes aren't such a natural fit, fall through and
+	 bitcast to integers first.  */
       if (GET_MODE_INNER (mode) == innermode)
 	{
 	  rtx sub = extract_bit_field_as_subreg (mode, op0, bitsize, bitnum);
@@ -2051,9 +2061,9 @@ extract_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
   machine_mode mode1;
 
   /* Handle -fstrict-volatile-bitfields in the cases where it applies.  */
-  if (maybe_nonzero (GET_MODE_BITSIZE (GET_MODE (str_rtx))))
+  if (may_ne (GET_MODE_BITSIZE (GET_MODE (str_rtx)), 0))
     mode1 = GET_MODE (str_rtx);
-  else if (target && maybe_nonzero (GET_MODE_BITSIZE (GET_MODE (target))))
+  else if (target && may_ne (GET_MODE_BITSIZE (GET_MODE (target)), 0))
     mode1 = GET_MODE (target);
   else
     mode1 = tmode;
diff --git a/gcc/expr.c b/gcc/expr.c
index 8e290dedda7..31ff5e188e1 100644
--- a/gcc/expr.c
+++ b/gcc/expr.c
@@ -2210,7 +2210,7 @@ emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
 		 can be used to determine the object and the bit field
 		 to be extracted.  */
 	      tmps[i] = XEXP (src, elt);
-	      if (maybe_nonzero (subpos)
+	      if (may_ne (subpos, 0)
 		  || may_ne (subpos + bytelen, slen0)
 		  || (!CONSTANT_P (tmps[i])
 		      && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode)))
@@ -2223,7 +2223,7 @@ emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
 	    {
 	      rtx mem;
 
-	      gcc_assert (known_zero (bytepos));
+	      gcc_assert (must_eq (bytepos, 0));
 	      mem = assign_stack_temp (GET_MODE (src), slen);
 	      emit_move_insn (mem, src);
 	      tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
@@ -2271,7 +2271,7 @@ emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
 				     bytepos * BITS_PER_UNIT, 1, NULL_RTX,
 				     mode, mode, false, NULL);
 
-      if (maybe_nonzero (shift))
+      if (may_ne (shift, 0))
 	tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
 				shift, tmps[i], 0);
     }
@@ -2540,7 +2540,7 @@ emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED,
 	      machine_mode dest_mode = GET_MODE (dest);
 	      machine_mode tmp_mode = GET_MODE (tmps[i]);
 
-	      gcc_assert (known_zero (bytepos) && XVECLEN (src, 0));
+	      gcc_assert (must_eq (bytepos, 0) && XVECLEN (src, 0));
 
 	      if (GET_MODE_ALIGNMENT (dest_mode)
 		  >= GET_MODE_ALIGNMENT (tmp_mode))
@@ -3884,12 +3884,12 @@ push_block (rtx size, poly_int64 extra, int below)
   size = convert_modes (Pmode, ptr_mode, size, 1);
   if (CONSTANT_P (size))
     anti_adjust_stack (plus_constant (Pmode, size, extra));
-  else if (REG_P (size) && known_zero (extra))
+  else if (REG_P (size) && must_eq (extra, 0))
     anti_adjust_stack (size);
   else
     {
       temp = copy_to_mode_reg (Pmode, size);
-      if (maybe_nonzero (extra))
+      if (may_ne (extra, 0))
 	temp = expand_binop (Pmode, add_optab, temp,
 			     gen_int_mode (extra, Pmode),
 			     temp, 0, OPTAB_LIB_WIDEN);
@@ -3899,7 +3899,7 @@ push_block (rtx size, poly_int64 extra, int below)
   if (STACK_GROWS_DOWNWARD)
     {
       temp = virtual_outgoing_args_rtx;
-      if (maybe_nonzero (extra) && below)
+      if (may_ne (extra, 0) && below)
 	temp = plus_constant (Pmode, temp, extra);
     }
   else
@@ -3908,7 +3908,7 @@ push_block (rtx size, poly_int64 extra, int below)
       if (poly_int_rtx_p (size, &csize))
 	temp = plus_constant (Pmode, virtual_outgoing_args_rtx,
 			      -csize - (below ? 0 : extra));
-      else if (maybe_nonzero (extra) && !below)
+      else if (may_ne (extra, 0) && !below)
 	temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
 			     negate_rtx (Pmode, plus_constant (Pmode, size,
 							       extra)));
@@ -4091,7 +4091,7 @@ fixup_args_size_notes (rtx_insn *prev, rtx_insn *last,
 	continue;
 
       poly_int64 this_delta = find_args_size_adjust (insn);
-      if (known_zero (this_delta))
+      if (must_eq (this_delta, 0))
 	{
 	  if (!CALL_P (insn)
 	      || ACCUMULATE_OUTGOING_ARGS
@@ -4363,7 +4363,7 @@ emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
 	  /* Push padding now if padding above and stack grows down,
 	     or if padding below and stack grows up.
 	     But if space already allocated, this has already been done.  */
-	  if (maybe_nonzero (extra)
+	  if (may_ne (extra, 0)
 	      && args_addr == 0
 	      && where_pad != PAD_NONE
 	      && where_pad != stack_direction)
@@ -4490,7 +4490,7 @@ emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
       /* Push padding now if padding above and stack grows down,
 	 or if padding below and stack grows up.
 	 But if space already allocated, this has already been done.  */
-      if (maybe_nonzero (extra)
+      if (may_ne (extra, 0)
 	  && args_addr == 0
 	  && where_pad != PAD_NONE
 	  && where_pad != stack_direction)
@@ -4543,7 +4543,7 @@ emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
       /* Push padding now if padding above and stack grows down,
 	 or if padding below and stack grows up.
 	 But if space already allocated, this has already been done.  */
-      if (maybe_nonzero (extra)
+      if (may_ne (extra, 0)
 	  && args_addr == 0
 	  && where_pad != PAD_NONE
 	  && where_pad != stack_direction)
@@ -4592,7 +4592,7 @@ emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
 	}
     }
 
-  if (maybe_nonzero (extra) && args_addr == 0 && where_pad == stack_direction)
+  if (may_ne (extra, 0) && args_addr == 0 && where_pad == stack_direction)
     anti_adjust_stack (gen_int_mode (extra, Pmode));
 
   if (alignment_pad && args_addr == 0)
@@ -5087,7 +5087,7 @@ expand_assignment (tree to, tree from, bool nontemporal)
 	     be expected to result in single move instructions.  */
 	  poly_int64 bytepos;
 	  if (mode1 != VOIDmode
-	      && maybe_nonzero (bitpos)
+	      && may_ne (bitpos, 0)
 	      && may_gt (bitsize, 0)
 	      && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
 	      && multiple_p (bitpos, bitsize)
@@ -5124,13 +5124,13 @@ expand_assignment (tree to, tree from, bool nontemporal)
 	  poly_int64 mode_bitsize = GET_MODE_BITSIZE (to_mode);
 	  unsigned short inner_bitsize = GET_MODE_UNIT_BITSIZE (to_mode);
 	  if (COMPLEX_MODE_P (TYPE_MODE (TREE_TYPE (from)))
-	      && known_zero (bitpos)
+	      && must_eq (bitpos, 0)
 	      && must_eq (bitsize, mode_bitsize))
 	    result = store_expr (from, to_rtx, false, nontemporal, reversep);
 	  else if (must_eq (bitsize, inner_bitsize)
-		   && (known_zero (bitpos)
+		   && (must_eq (bitpos, 0)
 		       || must_eq (bitpos, inner_bitsize)))
-	    result = store_expr (from, XEXP (to_rtx, maybe_nonzero (bitpos)),
+	    result = store_expr (from, XEXP (to_rtx, may_ne (bitpos, 0)),
 				 false, nontemporal, reversep);
 	  else if (must_le (bitpos + bitsize, inner_bitsize))
 	    result = store_field (XEXP (to_rtx, 0), bitsize, bitpos,
@@ -5143,8 +5143,7 @@ expand_assignment (tree to, tree from, bool nontemporal)
 				  bitregion_start, bitregion_end,
 				  mode1, from, get_alias_set (to),
 				  nontemporal, reversep);
-	  else if (known_zero (bitpos)
-		   && must_eq (bitsize, mode_bitsize))
+	  else if (must_eq (bitpos, 0) && must_eq (bitsize, mode_bitsize))
 	    {
 	      rtx from_rtx;
 	      result = expand_normal (from);
@@ -6118,12 +6117,12 @@ store_constructor_field (rtx target, poly_uint64 bitsize, poly_int64 bitpos,
       /* We can only call store_constructor recursively if the size and
 	 bit position are on a byte boundary.  */
       && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
-      && maybe_nonzero (bitsize)
+      && may_ne (bitsize, 0U)
       && multiple_p (bitsize, BITS_PER_UNIT, &bytesize)
       /* If we have a nonzero bitpos for a register target, then we just
 	 let store_field do the bitfield handling.  This is unlikely to
 	 generate unnecessary clear instructions anyways.  */
-      && (known_zero (bitpos) || MEM_P (target)))
+      && (must_eq (bitpos, 0) || MEM_P (target)))
     {
       if (MEM_P (target))
 	{
@@ -6197,7 +6196,7 @@ store_constructor (tree exp, rtx target, int cleared, poly_int64 size,
 	reverse = TYPE_REVERSE_STORAGE_ORDER (type);
 
 	/* If size is zero or the target is already cleared, do nothing.  */
-	if (known_zero (size) || cleared)
+	if (must_eq (size, 0) || cleared)
 	  cleared = 1;
 	/* We either clear the aggregate or indicate the value is dead.  */
 	else if ((TREE_CODE (type) == UNION_TYPE
@@ -6805,7 +6804,7 @@ store_field (rtx target, poly_int64 bitsize, poly_int64 bitpos,
   /* If we have nothing to store, do nothing unless the expression has
      side-effects.  Don't do that for zero sized addressable lhs of
      calls.  */
-  if (known_zero (bitsize)
+  if (must_eq (bitsize, 0)
       && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
 	  || TREE_CODE (exp) != CALL_EXPR))
     return expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
@@ -6814,7 +6813,7 @@ store_field (rtx target, poly_int64 bitsize, poly_int64 bitpos,
     {
       /* We're storing into a struct containing a single __complex.  */
 
-      gcc_assert (known_zero (bitpos));
+      gcc_assert (must_eq (bitpos, 0));
       return store_expr (exp, target, 0, nontemporal, reverse);
     }
 
@@ -7901,7 +7900,7 @@ expand_expr_addr_expr_1 (tree exp, rtx target, scalar_int_mode tmode,
   /* We must have made progress.  */
   gcc_assert (inner != exp);
 
-  subtarget = offset || maybe_nonzero (bitpos) ? NULL_RTX : target;
+  subtarget = offset || may_ne (bitpos, 0) ? NULL_RTX : target;
   /* For VIEW_CONVERT_EXPR, where the outer alignment is bigger than
      inner alignment, force the inner to be sufficiently aligned.  */
   if (CONSTANT_CLASS_P (inner)
@@ -7936,13 +7935,13 @@ expand_expr_addr_expr_1 (tree exp, rtx target, scalar_int_mode tmode,
 	result = simplify_gen_binary (PLUS, tmode, result, tmp);
       else
 	{
-	  subtarget = maybe_nonzero (bitpos) ? NULL_RTX : target;
+	  subtarget = may_ne (bitpos, 0) ? NULL_RTX : target;
 	  result = expand_simple_binop (tmode, PLUS, result, tmp, subtarget,
 					1, OPTAB_LIB_WIDEN);
 	}
     }
 
-  if (maybe_nonzero (bitpos))
+  if (may_ne (bitpos, 0))
     {
       /* Someone beforehand should have rejected taking the address
 	 of an object that isn't byte-aligned.  */
@@ -9969,43 +9968,24 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 	  && GET_MODE (decl_rtl) != dmode)
 	{
 	  machine_mode pmode;
-	  bool always_initialized_rtx;
 
 	  /* Get the signedness to be used for this variable.  Ensure we get
 	     the same mode we got when the variable was declared.  */
 	  if (code != SSA_NAME)
-	    {
-	      pmode = promote_decl_mode (exp, &unsignedp);
-	      always_initialized_rtx = true;
-	    }
+	    pmode = promote_decl_mode (exp, &unsignedp);
 	  else if ((g = SSA_NAME_DEF_STMT (ssa_name))
 		   && gimple_code (g) == GIMPLE_CALL
 		   && !gimple_call_internal_p (g))
-	    {
-	      pmode = promote_function_mode (type, mode, &unsignedp,
-					    gimple_call_fntype (g), 2);
-	      always_initialized_rtx
-		= always_initialized_rtx_for_ssa_name_p (ssa_name);
-	    }
+	    pmode = promote_function_mode (type, mode, &unsignedp,
+					   gimple_call_fntype (g),
+					   2);
 	  else
-	    {
-	      pmode = promote_ssa_mode (ssa_name, &unsignedp);
-	      always_initialized_rtx
-		= always_initialized_rtx_for_ssa_name_p (ssa_name);
-	    }
-
+	    pmode = promote_ssa_mode (ssa_name, &unsignedp);
 	  gcc_assert (GET_MODE (decl_rtl) == pmode);
 
 	  temp = gen_lowpart_SUBREG (mode, decl_rtl);
-
-	  /* We cannot assume anything about an existing extension if the
-	     register may contain uninitialized bits.  */
-	  if (always_initialized_rtx)
-	    {
-	      SUBREG_PROMOTED_VAR_P (temp) = 1;
-	      SUBREG_PROMOTED_SET (temp, unsignedp);
-	    }
-
+	  SUBREG_PROMOTED_VAR_P (temp) = 1;
+	  SUBREG_PROMOTED_SET (temp, unsignedp);
 	  return temp;
 	}
 
@@ -10242,7 +10222,7 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 	    poly_int64 offset = mem_ref_offset (exp).force_shwi ();
 	    base = TREE_OPERAND (base, 0);
 	    poly_uint64 type_size;
-	    if (known_zero (offset)
+	    if (must_eq (offset, 0)
 	        && !reverse
 		&& poly_int_tree_p (TYPE_SIZE (type), &type_size)
 		&& must_eq (GET_MODE_BITSIZE (DECL_MODE (base)), type_size))
@@ -10582,7 +10562,7 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 	/* Handle CONCAT first.  */
 	if (GET_CODE (op0) == CONCAT && !must_force_mem)
 	  {
-	    if (known_zero (bitpos)
+	    if (must_eq (bitpos, 0)
 		&& must_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
 		&& COMPLEX_MODE_P (mode1)
 		&& COMPLEX_MODE_P (GET_MODE (op0))
@@ -10615,10 +10595,10 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 		  }
 		return op0;
 	      }
-	    if (known_zero (bitpos)
+	    if (must_eq (bitpos, 0)
 		&& must_eq (bitsize,
 			    GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
-		&& maybe_nonzero (bitsize))
+		&& may_ne (bitsize, 0))
 	      {
 		op0 = XEXP (op0, 0);
 		mode2 = GET_MODE (op0);
@@ -10627,8 +10607,8 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 			      GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
 		     && must_eq (bitsize,
 				 GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 1))))
-		     && maybe_nonzero (bitpos)
-		     && maybe_nonzero (bitsize))
+		     && may_ne (bitpos, 0)
+		     && may_ne (bitsize, 0))
 	      {
 		op0 = XEXP (op0, 1);
 		bitpos = 0;
@@ -10683,7 +10663,7 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 
 	    /* See the comment in expand_assignment for the rationale.  */
 	    if (mode1 != VOIDmode
-		&& maybe_nonzero (bitpos)
+		&& may_ne (bitpos, 0)
 		&& may_gt (bitsize, 0)
 		&& multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
 		&& multiple_p (bitpos, bitsize)
@@ -10701,7 +10681,7 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 	/* If OFFSET is making OP0 more aligned than BIGGEST_ALIGNMENT,
 	   record its alignment as BIGGEST_ALIGNMENT.  */
 	if (MEM_P (op0)
-	    && known_zero (bitpos)
+	    && must_eq (bitpos, 0)
 	    && offset != 0
 	    && is_aligning_offset (offset, tem))
 	  set_mem_align (op0, BIGGEST_ALIGNMENT);
@@ -10775,7 +10755,7 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 
 		/* ??? Unlike the similar test a few lines below, this one is
 		   very likely obsolete.  */
-		if (known_zero (bitsize))
+		if (must_eq (bitsize, 0))
 		  return target;
 
 		/* In this case, BITPOS must start at a byte boundary and
@@ -10798,7 +10778,7 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 	       with SHIFT_COUNT_TRUNCATED == 0 and garbage otherwise.  Always
 	       return 0 for the sake of consistency, as reading a zero-sized
 	       bitfield is valid in Ada and the value is fully specified.  */
-	    if (known_zero (bitsize))
+	    if (must_eq (bitsize, 0))
 	      return const0_rtx;
 
 	    op0 = validize_mem (op0);
diff --git a/gcc/file-find.c b/gcc/file-find.c
index b072a4993d7..b5a1fe8494e 100644
--- a/gcc/file-find.c
+++ b/gcc/file-find.c
@@ -208,38 +208,3 @@ prefix_from_string (const char *p, struct path_prefix *pprefix)
     }
   free (nstore);
 }
-
-void
-remove_prefix (const char *prefix, struct path_prefix *pprefix)
-{
-  struct prefix_list *remove, **prev, **remove_prev = NULL;
-  int max_len = 0;
-
-  if (pprefix->plist)
-    {
-      prev = &pprefix->plist;
-      for (struct prefix_list *pl = pprefix->plist; pl->next; pl = pl->next)
-	{
-	  if (strcmp (prefix, pl->prefix) == 0)
-	    {
-	      remove = pl;
-	      remove_prev = prev;
-	      continue;
-	    }
-
-	  int l = strlen (pl->prefix);
-	  if (l > max_len)
-	    max_len = l;
-
-	  prev = &pl;
-	}
-
-      if (remove_prev)
-	{
-	  *remove_prev = remove->next;
-	  free (remove);
-	}
-
-      pprefix->max_len = max_len;
-    }
-}
diff --git a/gcc/file-find.h b/gcc/file-find.h
index 8f49a3af273..407feba26e7 100644
--- a/gcc/file-find.h
+++ b/gcc/file-find.h
@@ -41,7 +41,6 @@ extern void find_file_set_debug (bool);
 extern char *find_a_file (struct path_prefix *, const char *, int);
 extern void add_prefix (struct path_prefix *, const char *);
 extern void add_prefix_begin (struct path_prefix *, const char *);
-extern void remove_prefix (const char *prefix, struct path_prefix *);
 extern void prefix_from_env (const char *, struct path_prefix *);
 extern void prefix_from_string (const char *, struct path_prefix *);
 
diff --git a/gcc/final.c b/gcc/final.c
index 8dfaa2ead1b..3a127d9e7e7 100644
--- a/gcc/final.c
+++ b/gcc/final.c
@@ -92,8 +92,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "dbxout.h"
 #endif
 
-#include "sdbout.h"
-
 /* Most ports that aren't using cc0 don't need to define CC_STATUS_INIT.
    So define a null default for it to save conditionalization later.  */
 #ifndef CC_STATUS_INIT
@@ -696,8 +694,8 @@ compute_alignments (void)
     }
   loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
   FOR_EACH_BB_FN (bb, cfun)
-    if (bb->frequency > freq_max)
-      freq_max = bb->frequency;
+    if (bb->count.to_frequency (cfun) > freq_max)
+      freq_max = bb->count.to_frequency (cfun);
   freq_threshold = freq_max / PARAM_VALUE (PARAM_ALIGN_THRESHOLD);
 
   if (dump_file)
@@ -715,7 +713,8 @@ compute_alignments (void)
 	  if (dump_file)
 	    fprintf (dump_file,
 		     "BB %4i freq %4i loop %2i loop_depth %2i skipped.\n",
-		     bb->index, bb->frequency, bb->loop_father->num,
+		     bb->index, bb->count.to_frequency (cfun),
+		     bb->loop_father->num,
 		     bb_loop_depth (bb));
 	  continue;
 	}
@@ -733,7 +732,7 @@ compute_alignments (void)
 	{
 	  fprintf (dump_file, "BB %4i freq %4i loop %2i loop_depth"
 		   " %2i fall %4i branch %4i",
-		   bb->index, bb->frequency, bb->loop_father->num,
+		   bb->index, bb->count.to_frequency (cfun), bb->loop_father->num,
 		   bb_loop_depth (bb),
 		   fallthru_frequency, branch_frequency);
 	  if (!bb->loop_father->inner && bb->loop_father->num)
@@ -755,9 +754,10 @@ compute_alignments (void)
 
       if (!has_fallthru
 	  && (branch_frequency > freq_threshold
-	      || (bb->frequency > bb->prev_bb->frequency * 10
-		  && (bb->prev_bb->frequency
-		      <= ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency / 2))))
+	      || (bb->count.to_frequency (cfun) 
+			> bb->prev_bb->count.to_frequency (cfun) * 10
+		  && (bb->prev_bb->count.to_frequency (cfun)
+		      <= ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun) / 2))))
 	{
 	  log = JUMP_ALIGN (label);
 	  if (dump_file)
@@ -1945,8 +1945,6 @@ dump_basic_block_info (FILE *file, rtx_insn *insn, basic_block *start_to_bb,
       edge_iterator ei;
 
       fprintf (file, "%s BLOCK %d", ASM_COMMENT_START, bb->index);
-      if (bb->frequency)
-        fprintf (file, " freq:%d", bb->frequency);
       if (bb->count.initialized_p ())
 	{
           fprintf (file, ", count:");
@@ -2329,8 +2327,7 @@ final_scan_insn (rtx_insn *insn, FILE *file, int optimize_p ATTRIBUTE_UNUSED,
 	      TREE_ASM_WRITTEN (NOTE_BLOCK (insn)) = 1;
 	      BLOCK_IN_COLD_SECTION_P (NOTE_BLOCK (insn)) = in_cold_section_p;
 	    }
-	  if (write_symbols == DBX_DEBUG
-	      || write_symbols == SDB_DEBUG)
+	  if (write_symbols == DBX_DEBUG)
 	    {
 	      location_t *locus_ptr
 		= block_nonartificial_location (NOTE_BLOCK (insn));
@@ -2364,8 +2361,7 @@ final_scan_insn (rtx_insn *insn, FILE *file, int optimize_p ATTRIBUTE_UNUSED,
 	      gcc_assert (BLOCK_IN_COLD_SECTION_P (NOTE_BLOCK (insn))
 			  == in_cold_section_p);
 	    }
-	  if (write_symbols == DBX_DEBUG
-	      || write_symbols == SDB_DEBUG)
+	  if (write_symbols == DBX_DEBUG)
 	    {
 	      tree outer_block = BLOCK_SUPERCONTEXT (NOTE_BLOCK (insn));
 	      location_t *locus_ptr
@@ -4686,12 +4682,6 @@ rest_of_clean_state (void)
 	}
     }
 
-  /* In case the function was not output,
-     don't leave any temporary anonymous types
-     queued up for sdb output.  */
-  if (SDB_DEBUGGING_INFO && write_symbols == SDB_DEBUG)
-    sdbout_types (NULL_TREE);
-
   flag_rerun_cse_after_global_opts = 0;
   reload_completed = 0;
   epilogue_completed = 0;
diff --git a/gcc/flag-types.h b/gcc/flag-types.h
index 1f439d35b07..591b74457cd 100644
--- a/gcc/flag-types.h
+++ b/gcc/flag-types.h
@@ -24,7 +24,6 @@ enum debug_info_type
 {
   NO_DEBUG,	    /* Write no debug info.  */
   DBX_DEBUG,	    /* Write BSD .stabs for DBX (using dbxout.c).  */
-  SDB_DEBUG,	    /* Write COFF for (old) SDB (using sdbout.c).  */
   DWARF2_DEBUG,	    /* Write Dwarf v2 debug info (using dwarf2out.c).  */
   XCOFF_DEBUG,	    /* Write IBM/Xcoff debug info (using dbxout.c).  */
   VMS_DEBUG,        /* Write VMS debug info (using vmsdbgout.c).  */
@@ -246,6 +245,7 @@ enum sanitize_code {
   SANITIZE_VPTR = 1UL << 22,
   SANITIZE_BOUNDS_STRICT = 1UL << 23,
   SANITIZE_POINTER_OVERFLOW = 1UL << 24,
+  SANITIZE_BUILTIN = 1UL << 25,
   SANITIZE_SHIFT = SANITIZE_SHIFT_BASE | SANITIZE_SHIFT_EXPONENT,
   SANITIZE_UNDEFINED = SANITIZE_SHIFT | SANITIZE_DIVIDE | SANITIZE_UNREACHABLE
 		       | SANITIZE_VLA | SANITIZE_NULL | SANITIZE_RETURN
@@ -254,7 +254,7 @@ enum sanitize_code {
 		       | SANITIZE_NONNULL_ATTRIBUTE
 		       | SANITIZE_RETURNS_NONNULL_ATTRIBUTE
 		       | SANITIZE_OBJECT_SIZE | SANITIZE_VPTR
-		       | SANITIZE_POINTER_OVERFLOW,
+		       | SANITIZE_POINTER_OVERFLOW | SANITIZE_BUILTIN,
   SANITIZE_UNDEFINED_NONDEFAULT = SANITIZE_FLOAT_DIVIDE | SANITIZE_FLOAT_CAST
 				  | SANITIZE_BOUNDS_STRICT
 };
@@ -325,4 +325,13 @@ enum gfc_convert
 };
 
 
+/* Control-Flow Protection values.  */
+enum cf_protection_level
+{
+  CF_NONE = 0,
+  CF_BRANCH = 1 << 0,
+  CF_RETURN = 1 << 1,
+  CF_FULL = CF_BRANCH | CF_RETURN,
+  CF_SET = 1 << 2
+};
 #endif /* ! GCC_FLAG_TYPES_H */
diff --git a/gcc/fold-const-call.c b/gcc/fold-const-call.c
index 7b5e6819381..5d88a356d3b 100644
--- a/gcc/fold-const-call.c
+++ b/gcc/fold-const-call.c
@@ -596,6 +596,7 @@ fold_const_call_ss (real_value *result, combined_fn fn,
   switch (fn)
     {
     CASE_CFN_SQRT:
+    CASE_CFN_SQRT_FN:
       return (real_compare (GE_EXPR, arg, &dconst0)
 	      && do_mpfr_arg1 (result, mpfr_sqrt, arg, format));
 
@@ -1179,14 +1180,17 @@ fold_const_call_sss (real_value *result, combined_fn fn,
       return do_mpfr_arg2 (result, mpfr_hypot, arg0, arg1, format);
 
     CASE_CFN_COPYSIGN:
+    CASE_CFN_COPYSIGN_FN:
       *result = *arg0;
       real_copysign (result, arg1);
       return true;
 
     CASE_CFN_FMIN:
+    CASE_CFN_FMIN_FN:
       return do_mpfr_arg2 (result, mpfr_min, arg0, arg1, format);
 
     CASE_CFN_FMAX:
+    CASE_CFN_FMAX_FN:
       return do_mpfr_arg2 (result, mpfr_max, arg0, arg1, format);
 
     CASE_CFN_POW:
@@ -1473,6 +1477,7 @@ fold_const_call_ssss (real_value *result, combined_fn fn,
   switch (fn)
     {
     CASE_CFN_FMA:
+    CASE_CFN_FMA_FN:
       return do_mpfr_arg3 (result, mpfr_fma, arg0, arg1, arg2, format);
 
     case CFN_FMS:
diff --git a/gcc/fold-const.c b/gcc/fold-const.c
index 72e4f2d4c96..e92b5efed4a 100644
--- a/gcc/fold-const.c
+++ b/gcc/fold-const.c
@@ -3593,7 +3593,8 @@ operand_equal_p (const_tree arg0, const_tree arg1, unsigned int flags)
 #undef OP_SAME_WITH_NULL
 }
 
-/* Similar to operand_equal_p, but strip nops first.  */
+/* Similar to operand_equal_p, but see if ARG0 might be a variant of ARG1
+   with a different signedness or a narrower precision.  */
 
 static bool
 operand_equal_for_comparison_p (tree arg0, tree arg1)
@@ -3608,9 +3609,20 @@ operand_equal_for_comparison_p (tree arg0, tree arg1)
   /* Discard any conversions that don't change the modes of ARG0 and ARG1
      and see if the inner values are the same.  This removes any
      signedness comparison, which doesn't matter here.  */
-  STRIP_NOPS (arg0);
-  STRIP_NOPS (arg1);
-  if (operand_equal_p (arg0, arg1, 0))
+  tree op0 = arg0;
+  tree op1 = arg1;
+  STRIP_NOPS (op0);
+  STRIP_NOPS (op1);
+  if (operand_equal_p (op0, op1, 0))
+    return true;
+
+  /* Discard a single widening conversion from ARG1 and see if the inner
+     value is the same as ARG0.  */
+  if (CONVERT_EXPR_P (arg1)
+      && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg1, 0)))
+      && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg1, 0)))
+         < TYPE_PRECISION (TREE_TYPE (arg1))
+      && operand_equal_p (arg0, TREE_OPERAND (arg1, 0), 0))
     return true;
 
   return false;
@@ -4023,47 +4035,6 @@ invert_truthvalue_loc (location_t loc, tree arg)
 			       : TRUTH_NOT_EXPR,
 			  type, arg);
 }
-
-/* Knowing that ARG0 and ARG1 are both RDIV_EXPRs, simplify a binary operation
-   with code CODE.  This optimization is unsafe.  */
-static tree
-distribute_real_division (location_t loc, enum tree_code code, tree type,
-			  tree arg0, tree arg1)
-{
-  bool mul0 = TREE_CODE (arg0) == MULT_EXPR;
-  bool mul1 = TREE_CODE (arg1) == MULT_EXPR;
-
-  /* (A / C) +- (B / C) -> (A +- B) / C.  */
-  if (mul0 == mul1
-      && operand_equal_p (TREE_OPERAND (arg0, 1),
-		       TREE_OPERAND (arg1, 1), 0))
-    return fold_build2_loc (loc, mul0 ? MULT_EXPR : RDIV_EXPR, type,
-			fold_build2_loc (loc, code, type,
-				     TREE_OPERAND (arg0, 0),
-				     TREE_OPERAND (arg1, 0)),
-			TREE_OPERAND (arg0, 1));
-
-  /* (A / C1) +- (A / C2) -> A * (1 / C1 +- 1 / C2).  */
-  if (operand_equal_p (TREE_OPERAND (arg0, 0),
-		       TREE_OPERAND (arg1, 0), 0)
-      && TREE_CODE (TREE_OPERAND (arg0, 1)) == REAL_CST
-      && TREE_CODE (TREE_OPERAND (arg1, 1)) == REAL_CST)
-    {
-      REAL_VALUE_TYPE r0, r1;
-      r0 = TREE_REAL_CST (TREE_OPERAND (arg0, 1));
-      r1 = TREE_REAL_CST (TREE_OPERAND (arg1, 1));
-      if (!mul0)
-	real_arithmetic (&r0, RDIV_EXPR, &dconst1, &r0);
-      if (!mul1)
-        real_arithmetic (&r1, RDIV_EXPR, &dconst1, &r1);
-      real_arithmetic (&r0, code, &r0, &r1);
-      return fold_build2_loc (loc, MULT_EXPR, type,
-			  TREE_OPERAND (arg0, 0),
-			  build_real (type, r0));
-    }
-
-  return NULL_TREE;
-}
 
 /* Return a BIT_FIELD_REF of type TYPE to refer to BITSIZE bits of INNER
    starting at BITPOS.  The field is unsigned if UNSIGNEDP is nonzero
@@ -4107,7 +4078,7 @@ make_bit_field_ref (location_t loc, tree inner, tree orig_inner, tree type,
 			 build_fold_addr_expr (inner),
 			 build_int_cst (ptr_type_node, 0));
 
-  if (known_zero (bitpos) && !reversep)
+  if (must_eq (bitpos, 0) && !reversep)
     {
       tree size = TYPE_SIZE (TREE_TYPE (inner));
       if ((INTEGRAL_TYPE_P (TREE_TYPE (inner))
@@ -4250,21 +4221,20 @@ optimize_bit_field_compare (location_t loc, enum tree_code code,
 		      size_int (nbitsize - lbitsize - lbitpos));
 
   if (! const_p)
-    /* If not comparing with constant, just rework the comparison
-       and return.  */
-    return fold_build2_loc (loc, code, compare_type,
-			fold_build2_loc (loc, BIT_AND_EXPR, unsigned_type,
-				     make_bit_field_ref (loc, linner, lhs,
-							 unsigned_type,
-							 nbitsize, nbitpos,
-							 1, lreversep),
-				     mask),
-			fold_build2_loc (loc, BIT_AND_EXPR, unsigned_type,
-				     make_bit_field_ref (loc, rinner, rhs,
-							 unsigned_type,
-							 nbitsize, nbitpos,
-							 1, rreversep),
-				     mask));
+    {
+      if (nbitpos < 0)
+	return 0;
+
+      /* If not comparing with constant, just rework the comparison
+	 and return.  */
+      tree t1 = make_bit_field_ref (loc, linner, lhs, unsigned_type,
+				    nbitsize, nbitpos, 1, lreversep);
+      t1 = fold_build2_loc (loc, BIT_AND_EXPR, unsigned_type, t1, mask);
+      tree t2 = make_bit_field_ref (loc, rinner, rhs, unsigned_type,
+				    nbitsize, nbitpos, 1, rreversep);
+      t2 = fold_build2_loc (loc, BIT_AND_EXPR, unsigned_type, t2, mask);
+      return fold_build2_loc (loc, code, compare_type, t1, t2);
+    }
 
   /* Otherwise, we are handling the constant case.  See if the constant is too
      big for the field.  Warn and return a tree for 0 (false) if so.  We do
@@ -4295,6 +4265,9 @@ optimize_bit_field_compare (location_t loc, enum tree_code code,
 	}
     }
 
+  if (nbitpos < 0)
+    return 0;
+
   /* Single-bit compares should always be against zero.  */
   if (lbitsize == 1 && ! integer_zerop (rhs))
     {
@@ -6117,7 +6090,10 @@ fold_truth_andor_1 (location_t loc, enum tree_code code, tree truth_type,
 	 results.  */
       ll_mask = const_binop (BIT_IOR_EXPR, ll_mask, rl_mask);
       lr_mask = const_binop (BIT_IOR_EXPR, lr_mask, rr_mask);
-      if (lnbitsize == rnbitsize && xll_bitpos == xlr_bitpos)
+      if (lnbitsize == rnbitsize
+	  && xll_bitpos == xlr_bitpos
+	  && lnbitpos >= 0
+	  && rnbitpos >= 0)
 	{
 	  lhs = make_bit_field_ref (loc, ll_inner, ll_arg,
 				    lntype, lnbitsize, lnbitpos,
@@ -6141,10 +6117,14 @@ fold_truth_andor_1 (location_t loc, enum tree_code code, tree truth_type,
 	 Note that we still must mask the lhs/rhs expressions.  Furthermore,
 	 the mask must be shifted to account for the shift done by
 	 make_bit_field_ref.  */
-      if ((ll_bitsize + ll_bitpos == rl_bitpos
-	   && lr_bitsize + lr_bitpos == rr_bitpos)
-	  || (ll_bitpos == rl_bitpos + rl_bitsize
-	      && lr_bitpos == rr_bitpos + rr_bitsize))
+      if (((ll_bitsize + ll_bitpos == rl_bitpos
+	    && lr_bitsize + lr_bitpos == rr_bitpos)
+	   || (ll_bitpos == rl_bitpos + rl_bitsize
+	       && lr_bitpos == rr_bitpos + rr_bitsize))
+	  && ll_bitpos >= 0
+	  && rl_bitpos >= 0
+	  && lr_bitpos >= 0
+	  && rr_bitpos >= 0)
 	{
 	  tree type;
 
@@ -6213,6 +6193,9 @@ fold_truth_andor_1 (location_t loc, enum tree_code code, tree truth_type,
 	}
     }
 
+  if (lnbitpos < 0)
+    return 0;
+
   /* Construct the expression we will return.  First get the component
      reference we will make.  Unless the mask is all ones the width of
      that field, perform the mask operation.  Then compare with the
@@ -7977,7 +7960,7 @@ fold_unary_loc (location_t loc, enum tree_code code, tree type, tree op0)
 	     the address of the base if it has the same base type
 	     as the result type and the pointer type is unqualified.  */
 	  if (!offset
-	      && known_zero (bitpos)
+	      && must_eq (bitpos, 0)
 	      && (TYPE_MAIN_VARIANT (TREE_TYPE (type))
 		  == TYPE_MAIN_VARIANT (TREE_TYPE (base)))
 	      && TYPE_QUALS (type) == TYPE_UNQUALIFIED)
@@ -8501,7 +8484,7 @@ pointer_may_wrap_p (tree base, tree offset, poly_int64 bitpos)
   if (!total.to_uhwi (&total_hwi)
       || !poly_int_tree_p (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (base))),
 			   &size)
-      || known_zero (size))
+      || must_eq (size, 0U))
     return true;
 
   if (must_le (total_hwi, size))
@@ -8512,7 +8495,7 @@ pointer_may_wrap_p (tree base, tree offset, poly_int64 bitpos)
   if (TREE_CODE (base) == ADDR_EXPR
       && poly_int_tree_p (TYPE_SIZE_UNIT (TREE_TYPE (TREE_OPERAND (base, 0))),
 			  &size)
-      && maybe_nonzero (size)
+      && may_ne (size, 0U)
       && must_le (total_hwi, size))
     return false;
 
@@ -8794,7 +8777,7 @@ fold_comparison (location_t loc, enum tree_code code, tree type,
 		    eliminated.  When ptr is null, although the -> expression
 		    is strictly speaking invalid, GCC retains it as a matter
 		    of QoI.  See PR c/44555. */
-		 && (offset0 == NULL_TREE && known_nonzero (bitpos0)))
+		 && (offset0 == NULL_TREE && must_ne (bitpos0, 0)))
 		|| CONSTANT_CLASS_P (base0))
 	       && indirect_base0
 	       /* The caller guarantees that when one of the arguments is
@@ -9674,12 +9657,6 @@ fold_binary_loc (location_t loc,
 		}
 	    }
 
-	  if (flag_unsafe_math_optimizations
-	      && (TREE_CODE (arg0) == RDIV_EXPR || TREE_CODE (arg0) == MULT_EXPR)
-	      && (TREE_CODE (arg1) == RDIV_EXPR || TREE_CODE (arg1) == MULT_EXPR)
-	      && (tem = distribute_real_division (loc, code, type, arg0, arg1)))
-	    return tem;
-
           /* Convert a + (b*c + d*e) into (a + b*c) + d*e.
              We associate floats only if the user has specified
              -fassociative-math.  */
@@ -10079,13 +10056,6 @@ fold_binary_loc (location_t loc,
 	    return tem;
 	}
 
-      if (FLOAT_TYPE_P (type)
-	  && flag_unsafe_math_optimizations
-	  && (TREE_CODE (arg0) == RDIV_EXPR || TREE_CODE (arg0) == MULT_EXPR)
-	  && (TREE_CODE (arg1) == RDIV_EXPR || TREE_CODE (arg1) == MULT_EXPR)
-	  && (tem = distribute_real_division (loc, code, type, arg0, arg1)))
-	return tem;
-
       /* Handle (A1 * C1) - (A2 * C2) with A1, A2 or C1, C2 being the same or
 	 one.  Make sure the type is not saturating and has the signedness of
 	 the stripped operands, as fold_plusminus_mult_expr will re-associate.
@@ -11502,8 +11472,8 @@ fold_ternary_loc (location_t loc, enum tree_code code, tree type,
 
          Also try swapping the arguments and inverting the conditional.  */
       if (COMPARISON_CLASS_P (arg0)
-	  && operand_equal_for_comparison_p (TREE_OPERAND (arg0, 0), arg1)
-	  && !HONOR_SIGNED_ZEROS (element_mode (arg1)))
+	  && operand_equal_for_comparison_p (TREE_OPERAND (arg0, 0), op1)
+	  && !HONOR_SIGNED_ZEROS (element_mode (op1)))
 	{
 	  tem = fold_cond_expr_with_comparison (loc, type, arg0, op1, op2);
 	  if (tem)
@@ -13097,6 +13067,7 @@ tree_call_nonnegative_warnv_p (tree type, combined_fn fn, tree arg0, tree arg1,
       return true;
 
     CASE_CFN_SQRT:
+    CASE_CFN_SQRT_FN:
       /* sqrt(-0.0) is -0.0.  */
       if (!HONOR_SIGNED_ZEROS (element_mode (type)))
 	return true;
@@ -13141,14 +13112,17 @@ tree_call_nonnegative_warnv_p (tree type, combined_fn fn, tree arg0, tree arg1,
       return RECURSE (arg0);
 
     CASE_CFN_FMAX:
+    CASE_CFN_FMAX_FN:
       /* True if the 1st OR 2nd arguments are nonnegative.  */
       return RECURSE (arg0) || RECURSE (arg1);
 
     CASE_CFN_FMIN:
+    CASE_CFN_FMIN_FN:
       /* True if the 1st AND 2nd arguments are nonnegative.  */
       return RECURSE (arg0) && RECURSE (arg1);
 
     CASE_CFN_COPYSIGN:
+    CASE_CFN_COPYSIGN_FN:
       /* True if the 2nd argument is nonnegative.  */
       return RECURSE (arg1);
 
@@ -13647,7 +13621,9 @@ integer_valued_real_call_p (combined_fn fn, tree arg0, tree arg1, int depth)
       return true;
 
     CASE_CFN_FMIN:
+    CASE_CFN_FMIN_FN:
     CASE_CFN_FMAX:
+    CASE_CFN_FMAX_FN:
       return RECURSE (arg0) && RECURSE (arg1);
 
     default:
@@ -14732,9 +14708,14 @@ test_vector_folding ()
 static void
 test_vec_duplicate_folding ()
 {
-  tree type = build_vector_type (ssizetype, 4);
-  tree dup5 = build_vec_duplicate_cst (type, ssize_int (5));
-  tree dup3 = build_vec_duplicate_cst (type, ssize_int (3));
+  scalar_int_mode int_mode = SCALAR_INT_TYPE_MODE (ssizetype);
+  machine_mode vec_mode = targetm.vectorize.preferred_simd_mode (int_mode);
+  /* This will be 1 if VEC_MODE isn't a vector mode.  */
+  poly_uint64 nunits = GET_MODE_NUNITS (vec_mode);
+
+  tree type = build_vector_type (ssizetype, nunits);
+  tree dup5 = build_vector_from_val (type, ssize_int (5));
+  tree dup3 = build_vector_from_val (type, ssize_int (3));
 
   tree neg_dup5 = fold_unary (NEGATE_EXPR, type, dup5);
   ASSERT_EQ (uniform_vector_p (neg_dup5), ssize_int (-5));
@@ -14748,7 +14729,7 @@ test_vec_duplicate_folding ()
   tree dup5_lsl_2 = fold_binary (LSHIFT_EXPR, type, dup5, ssize_int (2));
   ASSERT_EQ (uniform_vector_p (dup5_lsl_2), ssize_int (20));
 
-  tree size_vector = build_vector_type (sizetype, 4);
+  tree size_vector = build_vector_type (sizetype, nunits);
   tree size_dup5 = fold_convert (size_vector, dup5);
   ASSERT_EQ (uniform_vector_p (size_dup5), size_int (5));
 
@@ -14757,6 +14738,54 @@ test_vec_duplicate_folding ()
   ASSERT_TRUE (operand_equal_p (dup5_expr, dup5_cst, 0));
 }
 
+/* Verify folding of VEC_SERIES_CSTs and VEC_SERIES_EXPRs.  */
+
+static void
+test_vec_series_folding ()
+{
+  scalar_int_mode int_mode = SCALAR_INT_TYPE_MODE (ssizetype);
+  machine_mode vec_mode = targetm.vectorize.preferred_simd_mode (int_mode);
+  poly_uint64 nunits = GET_MODE_NUNITS (vec_mode);
+  if (must_eq (nunits, 1U))
+    nunits = 4;
+
+  tree type = build_vector_type (ssizetype, nunits);
+  tree s5_4 = build_vec_series (type, ssize_int (5), ssize_int (4));
+  tree s3_9 = build_vec_series (type, ssize_int (3), ssize_int (9));
+
+  tree neg_s5_4_a = fold_unary (NEGATE_EXPR, type, s5_4);
+  tree neg_s5_4_b = build_vec_series (type, ssize_int (-5), ssize_int (-4));
+  ASSERT_TRUE (operand_equal_p (neg_s5_4_a, neg_s5_4_b, 0));
+
+  tree s8_s13_a = fold_binary (PLUS_EXPR, type, s5_4, s3_9);
+  tree s8_s13_b = build_vec_series (type, ssize_int (8), ssize_int (13));
+  ASSERT_TRUE (operand_equal_p (s8_s13_a, s8_s13_b, 0));
+
+  tree s2_m5_a = fold_binary (MINUS_EXPR, type, s5_4, s3_9);
+  tree s2_m5_b = build_vec_series (type, ssize_int (2), ssize_int (-5));
+  ASSERT_TRUE (operand_equal_p (s2_m5_a, s2_m5_b, 0));
+
+  tree s11 = build_vector_from_val (type, ssize_int (11));
+  tree s16_4_a = fold_binary (PLUS_EXPR, type, s5_4, s11);
+  tree s16_4_b = fold_binary (PLUS_EXPR, type, s11, s5_4);
+  tree s16_4_c = build_vec_series (type, ssize_int (16), ssize_int (4));
+  ASSERT_TRUE (operand_equal_p (s16_4_a, s16_4_c, 0));
+  ASSERT_TRUE (operand_equal_p (s16_4_b, s16_4_c, 0));
+
+  tree sm6_4_a = fold_binary (MINUS_EXPR, type, s5_4, s11);
+  tree sm6_4_b = build_vec_series (type, ssize_int (-6), ssize_int (4));
+  ASSERT_TRUE (operand_equal_p (sm6_4_a, sm6_4_b, 0));
+
+  tree s6_m4_a = fold_binary (MINUS_EXPR, type, s11, s5_4);
+  tree s6_m4_b = build_vec_series (type, ssize_int (6), ssize_int (-4));
+  ASSERT_TRUE (operand_equal_p (s6_m4_a, s6_m4_b, 0));
+
+  tree s5_4_expr = fold_binary (VEC_SERIES_EXPR, type,
+				ssize_int (5), ssize_int (4));
+  ASSERT_TRUE (operand_equal_p (s5_4_expr, s5_4, 0));
+  ASSERT_FALSE (operand_equal_p (s5_4_expr, s3_9, 0));
+}
+
 /* Run all of the selftests within this file.  */
 
 void
@@ -14765,6 +14794,7 @@ fold_const_c_tests ()
   test_arithmetic_folding ();
   test_vector_folding ();
   test_vec_duplicate_folding ();
+  test_vec_series_folding ();
 }
 
 } // namespace selftest
diff --git a/gcc/fortran/ChangeLog b/gcc/fortran/ChangeLog
index a37d16e51fa..aa43ff4ebff 100644
--- a/gcc/fortran/ChangeLog
+++ b/gcc/fortran/ChangeLog
@@ -1,3 +1,175 @@
+2017-11-04  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/81735
+	* trans-decl.c (gfc_trans_deferred_vars): Do a better job of a
+	case where 'tmp' was used unititialized and remove TODO.
+
+2017-11-03  Steven G. Kargl  <kargl@gcc.gnu.org>
+
+	PR fortran/82796
+	* resolve.c (resolve_equivalence): An entity in a common block within
+ 	a module cannot appear in an equivalence statement if the entity is
+	with a pure procedure.
+
+2017-10-31  Jim Wilson  <wilson@tuliptree.org>
+
+	* parse.c (unexpected_eof): Call gcc_unreachable before return.
+
+2017-10-30  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/80850
+	* trans_expr.c (gfc_conv_procedure_call): When passing a class
+	argument to an unlimited polymorphic dummy, it is wrong to cast
+	the passed expression as unlimited, unless it is unlimited. The
+	correct way is to assign to each of the fields and set the _len
+	field to zero.
+
+2017-10-30  Steven G. Kargl   <kargl@gcc.gnu.org>
+
+	* resolve.c (resolve_transfer): Set derived to correct symbol for
+	BT_CLASS.
+
+2017-10-29  Jim Wilson  <wilson@tuliptree.org>
+
+	* invoke.texi: Delete adb and sdb references.
+
+2017-10-28  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	* check.c (gfc_check_co_reduce): Clarify error message.
+
+2017-10-28  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/81758
+	* trans-expr.c (trans_class_vptr_len_assignment): 'vptr_expr'
+	must only be set if the right hand side expression is of type
+	class.
+
+2017-10-27  Steven G. Kargl  <kargl@gcc.gnu.org>
+
+	PR fortran/82620
+	* match.c (gfc_match_allocate): Exit early on syntax error.
+
+2017-10-27  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/56342
+	* simplify.c (is_constant_array_expr): If the expression is
+	a parameter array, call gfc_simplify_expr.
+
+2017-10-25  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* match.c (gfc_match_type_is): Fix typo in error message.
+
+2017-10-21  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/82586
+	* decl.c (gfc_get_pdt_instance): Remove the error message that
+	the parameter does not have a corresponding component since
+	this is now taken care of when the derived type is resolved. Go
+	straight to error return instead.
+	(gfc_match_formal_arglist): Make the PDT relevant errors
+	immediate so that parsing of the derived type can continue.
+	(gfc_match_derived_decl): Do not check the match status on
+	return from gfc_match_formal_arglist for the same reason.
+	* resolve.c (resolve_fl_derived0): Check that each type
+	parameter has a corresponding component.
+
+	PR fortran/82587
+	* resolve.c (resolve_generic_f): Check that the derived type
+	can be used before resolving the struture constructor.
+
+	PR fortran/82589
+	* symbol.c (check_conflict): Add the conflicts involving PDT
+	KIND and LEN attributes.
+
+2017-10-19  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* interface.c (check_sym_interfaces, check_uop_interfaces,
+	gfc_check_interfaces): Base interface_name buffer off
+	GFC_MAX_SYMBOL_LEN.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	PR fortran/82568
+	* gfortran.h (gfc_resolve_do_iterator): Add a bool arg.
+	(gfc_resolve_omp_local_vars): New declaration.
+	* openmp.c (omp_current_ctx): Make static.
+	(gfc_resolve_omp_parallel_blocks): Handle EXEC_OMP_TASKLOOP
+	and EXEC_OMP_TASKLOOP_SIMD.
+	(gfc_resolve_do_iterator): Add ADD_CLAUSE argument, if false,
+	don't actually add any clause.  Move omp_current_ctx test
+	earlier.
+	(handle_local_var, gfc_resolve_omp_local_vars): New functions.
+	* resolve.c (gfc_resolve_code): Call gfc_resolve_omp_parallel_blocks
+	instead of just gfc_resolve_omp_do_blocks for EXEC_OMP_TASKLOOP
+	and EXEC_OMP_TASKLOOP_SIMD.
+	(gfc_resolve_code): Adjust gfc_resolve_do_iterator caller.
+	(resolve_codes): Call gfc_resolve_omp_local_vars.
+
+2017-10-19  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* gfortran.h (gfc_lookup_function_fuzzy): New declaration.
+	(gfc_closest_fuzzy_match): New declaration.
+	(vec_push): New definition.
+	* misc.c (gfc_closest_fuzzy_match): New definition.
+	* resolve.c: Include spellcheck.h.
+	(lookup_function_fuzzy_find_candidates): New static function.
+	(lookup_uop_fuzzy_find_candidates): Likewise.
+	(lookup_uop_fuzzy): Likewise.
+	(resolve_operator) <INTRINSIC_USER>: Call lookup_uop_fuzzy.
+	(gfc_lookup_function_fuzzy): New definition.
+	(resolve_unknown_f): Call gfc_lookup_function_fuzzy.
+	* interface.c (check_interface0): Likewise.
+	(lookup_arg_fuzzy_find_candidates): New static function.
+	(lookup_arg_fuzzy ): Likewise.
+	(compare_actual_formal): Call lookup_arg_fuzzy.
+	* symbol.c: Include spellcheck.h.
+	(lookup_symbol_fuzzy_find_candidates): New static function.
+	(lookup_symbol_fuzzy): Likewise.
+	(gfc_set_default_type): Call lookup_symbol_fuzzy.
+	(lookup_component_fuzzy_find_candidates): New static function.
+	(lookup_component_fuzzy): Likewise.
+	(gfc_find_component): Call lookup_component_fuzzy.
+
+2017-10-18  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/82567
+	* frontend-passes.c (combine_array_constructor): If an array
+	constructor is all constants and has more elements than a small
+	constant, don't convert a*[b,c] to [a*b,a*c] to reduce compilation
+	times.
+
+2017-10-18  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/79795
+	* resolve.c (resovle_symbol): Change gcc_assert to
+	sensible error message.
+
+2017-10-18  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/82550
+	* trans_decl.c (gfc_get_symbol_decl): Procedure symbols that
+	have the 'used_in_submodule' attribute should be processed by
+	'gfc_get_extern_function_decl'.
+
+2017-10-16  Fritz Reese <fritzoreese@gmail.com>
+
+	PR fortran/82511
+	* trans-io.c (transfer_expr): Treat BT_UNION as BT_DERIVED.
+
+2017-10-15  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/82372
+	* fortran/scanner.c (last_error_char):  New global variable.
+	(gfc_scanner_init_1): Set last_error_char to NULL.
+	(gfc_gobble_whitespace): If a character not printable or
+	not newline, issue an error.
+
+2017-10-13  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/81048
+	* resolve.c (resolve_symbol): Ensure that derived type array
+	results get default initialization.
+
 2017-10-11  Nathan Sidwell  <nathan@acm.org>
 
 	* cpp.c (gfc_cpp_add_include_path): Update incpath_e names.
diff --git a/gcc/fortran/check.c b/gcc/fortran/check.c
index 681950e782f..759c15adaec 100644
--- a/gcc/fortran/check.c
+++ b/gcc/fortran/check.c
@@ -1731,7 +1731,7 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
 
   if (!gfc_compare_types (&a->ts, &sym->result->ts))
     {
-      gfc_error ("A argument at %L has type %s but the function passed as "
+      gfc_error ("The A argument at %L has type %s but the function passed as "
 		 "OPERATOR at %L returns %s",
 		 &a->where, gfc_typename (&a->ts), &op->where,
 		 gfc_typename (&sym->result->ts));
diff --git a/gcc/fortran/decl.c b/gcc/fortran/decl.c
index 5bf56c4d4b0..1a2d8f004ca 100644
--- a/gcc/fortran/decl.c
+++ b/gcc/fortran/decl.c
@@ -3242,13 +3242,10 @@ gfc_get_pdt_instance (gfc_actual_arglist *param_list, gfc_symbol **sym,
       param = type_param_name_list->sym;
 
       c1 = gfc_find_component (pdt, param->name, false, true, NULL);
+      /* An error should already have been thrown in resolve.c
+	 (resolve_fl_derived0).  */
       if (!pdt->attr.use_assoc && !c1)
-	{
-	  gfc_error ("The type parameter name list at %L contains a parameter "
-		     "'%qs' , which is not declared as a component of the type",
-		     &pdt->declared_at, param->name);
-	  goto error_return;
-	}
+	goto error_return;
 
       kind_expr = NULL;
       if (!name_seen)
@@ -5984,7 +5981,7 @@ gfc_match_formal_arglist (gfc_symbol *progname, int st_flag,
       /* The name of a program unit can be in a different namespace,
 	 so check for it explicitly.  After the statement is accepted,
 	 the name is checked for especially in gfc_get_symbol().  */
-      if (gfc_new_block != NULL && sym != NULL
+      if (gfc_new_block != NULL && sym != NULL && !typeparam
 	  && strcmp (sym->name, gfc_new_block->name) == 0)
 	{
 	  gfc_error ("Name %qs at %C is the name of the procedure",
@@ -5999,7 +5996,11 @@ gfc_match_formal_arglist (gfc_symbol *progname, int st_flag,
       m = gfc_match_char (',');
       if (m != MATCH_YES)
 	{
-	  gfc_error ("Unexpected junk in formal argument list at %C");
+	  if (typeparam)
+	    gfc_error_now ("Expected parameter list in type declaration "
+			   "at %C");
+	  else
+	    gfc_error ("Unexpected junk in formal argument list at %C");
 	  goto cleanup;
 	}
     }
@@ -6016,8 +6017,12 @@ ok:
 	  for (q = p->next; q; q = q->next)
 	    if (p->sym == q->sym)
 	      {
-		gfc_error ("Duplicate symbol %qs in formal argument list "
-			   "at %C", p->sym->name);
+		if (typeparam)
+		  gfc_error_now ("Duplicate name %qs in parameter "
+				 "list at %C", p->sym->name);
+		else
+		  gfc_error ("Duplicate symbol %qs in formal argument "
+			     "list at %C", p->sym->name);
 
 		m = MATCH_ERROR;
 		goto cleanup;
@@ -9814,9 +9819,9 @@ gfc_match_derived_decl (void)
 
   if (parameterized_type)
     {
-      m = gfc_match_formal_arglist (sym, 0, 0, true);
-      if (m != MATCH_YES)
-	return m;
+      /* Ignore error or mismatches to avoid the component declarations
+	 causing problems later.  */
+      gfc_match_formal_arglist (sym, 0, 0, true);
       m = gfc_match_eos ();
       if (m != MATCH_YES)
 	return m;
diff --git a/gcc/fortran/frontend-passes.c b/gcc/fortran/frontend-passes.c
index ae4fba63b3c..fcfaf9508c2 100644
--- a/gcc/fortran/frontend-passes.c
+++ b/gcc/fortran/frontend-passes.c
@@ -1635,6 +1635,8 @@ combine_array_constructor (gfc_expr *e)
   gfc_constructor *c, *new_c;
   gfc_constructor_base oldbase, newbase;
   bool scalar_first;
+  int n_elem;
+  bool all_const;
 
   /* Array constructors have rank one.  */
   if (e->rank != 1)
@@ -1674,12 +1676,38 @@ combine_array_constructor (gfc_expr *e)
   if (op2->ts.type == BT_CHARACTER)
     return false;
 
-  scalar = create_var (gfc_copy_expr (op2), "constr");
+  /* This might be an expanded constructor with very many constant values. If
+     we perform the operation here, we might end up with a long compile time
+     and actually longer execution time, so a length bound is in order here.
+     If the constructor constains something which is not a constant, it did
+     not come from an expansion, so leave it alone.  */
+
+#define CONSTR_LEN_MAX 4
 
   oldbase = op1->value.constructor;
+
+  n_elem = 0;
+  all_const = true;
+  for (c = gfc_constructor_first (oldbase); c; c = gfc_constructor_next(c))
+    {
+      if (c->expr->expr_type != EXPR_CONSTANT)
+	{
+	  all_const = false;
+	  break;
+	}
+      n_elem += 1;
+    }
+
+  if (all_const && n_elem > CONSTR_LEN_MAX)
+    return false;
+
+#undef CONSTR_LEN_MAX
+
   newbase = NULL;
   e->expr_type = EXPR_ARRAY;
 
+  scalar = create_var (gfc_copy_expr (op2), "constr");
+
   for (c = gfc_constructor_first (oldbase); c;
        c = gfc_constructor_next (c))
     {
diff --git a/gcc/fortran/gfortran.h b/gcc/fortran/gfortran.h
index b5fc1452747..2c2fc636708 100644
--- a/gcc/fortran/gfortran.h
+++ b/gcc/fortran/gfortran.h
@@ -2796,6 +2796,17 @@ void gfc_done_2 (void);
 
 int get_c_kind (const char *, CInteropKind_t *);
 
+const char *gfc_closest_fuzzy_match (const char *, char **);
+static inline void
+vec_push (char **&optr, size_t &osz, const char *elt)
+{
+  /* {auto,}vec.safe_push () replacement.  Don't ask..  */
+  // if (strlen (elt) < 4) return; premature optimization: eliminated by cutoff
+  optr = XRESIZEVEC (char *, optr, osz + 2);
+  optr[osz] = CONST_CAST (char *, elt);
+  optr[++osz] = NULL;
+}
+
 /* options.c */
 unsigned int gfc_option_lang_mask (void);
 void gfc_init_options_struct (struct gcc_options *);
@@ -3103,7 +3114,8 @@ void gfc_free_omp_declare_simd_list (gfc_omp_declare_simd *);
 void gfc_free_omp_udr (gfc_omp_udr *);
 gfc_omp_udr *gfc_omp_udr_find (gfc_symtree *, gfc_typespec *);
 void gfc_resolve_omp_directive (gfc_code *, gfc_namespace *);
-void gfc_resolve_do_iterator (gfc_code *, gfc_symbol *);
+void gfc_resolve_do_iterator (gfc_code *, gfc_symbol *, bool);
+void gfc_resolve_omp_local_vars (gfc_namespace *);
 void gfc_resolve_omp_parallel_blocks (gfc_code *, gfc_namespace *);
 void gfc_resolve_omp_do_blocks (gfc_code *, gfc_namespace *);
 void gfc_resolve_omp_declare_simd (gfc_namespace *);
@@ -3228,6 +3240,7 @@ bool gfc_type_is_extensible (gfc_symbol *);
 bool gfc_resolve_intrinsic (gfc_symbol *, locus *);
 bool gfc_explicit_interface_required (gfc_symbol *, char *, int);
 extern int gfc_do_concurrent_flag;
+const char* gfc_lookup_function_fuzzy (const char *, gfc_symtree *);
 
 
 /* array.c */
diff --git a/gcc/fortran/interface.c b/gcc/fortran/interface.c
index f8ef33fc778..9f0fcc82f24 100644
--- a/gcc/fortran/interface.c
+++ b/gcc/fortran/interface.c
@@ -1793,13 +1793,27 @@ check_interface0 (gfc_interface *p, const char *interface_name)
 	   || !p->sym->attr.if_source)
 	  && !gfc_fl_struct (p->sym->attr.flavor))
 	{
+	  const char *guessed
+	    = gfc_lookup_function_fuzzy (p->sym->name, p->sym->ns->sym_root);
+
 	  if (p->sym->attr.external)
-	    gfc_error ("Procedure %qs in %s at %L has no explicit interface",
-		       p->sym->name, interface_name, &p->sym->declared_at);
+	    if (guessed)
+	      gfc_error ("Procedure %qs in %s at %L has no explicit interface"
+			 "; did you mean %qs?",
+			 p->sym->name, interface_name, &p->sym->declared_at,
+			 guessed);
+	    else
+	      gfc_error ("Procedure %qs in %s at %L has no explicit interface",
+			 p->sym->name, interface_name, &p->sym->declared_at);
 	  else
-	    gfc_error ("Procedure %qs in %s at %L is neither function nor "
-		       "subroutine", p->sym->name, interface_name,
-		      &p->sym->declared_at);
+	    if (guessed)
+	      gfc_error ("Procedure %qs in %s at %L is neither function nor "
+			 "subroutine; did you mean %qs?", p->sym->name,
+			interface_name, &p->sym->declared_at, guessed);
+	    else
+	      gfc_error ("Procedure %qs in %s at %L is neither function nor "
+			 "subroutine", p->sym->name, interface_name,
+			&p->sym->declared_at);
 	  return true;
 	}
 
@@ -1904,7 +1918,7 @@ check_interface1 (gfc_interface *p, gfc_interface *q0,
 static void
 check_sym_interfaces (gfc_symbol *sym)
 {
-  char interface_name[100];
+  char interface_name[GFC_MAX_SYMBOL_LEN + sizeof("generic interface ''")];
   gfc_interface *p;
 
   if (sym->ns != gfc_current_ns)
@@ -1941,7 +1955,7 @@ check_sym_interfaces (gfc_symbol *sym)
 static void
 check_uop_interfaces (gfc_user_op *uop)
 {
-  char interface_name[100];
+  char interface_name[GFC_MAX_SYMBOL_LEN + sizeof("operator interface ''")];
   gfc_user_op *uop2;
   gfc_namespace *ns;
 
@@ -2018,7 +2032,7 @@ void
 gfc_check_interfaces (gfc_namespace *ns)
 {
   gfc_namespace *old_ns, *ns2;
-  char interface_name[100];
+  char interface_name[GFC_MAX_SYMBOL_LEN + sizeof("intrinsic '' operator")];
   int i;
 
   old_ns = gfc_current_ns;
@@ -2778,6 +2792,31 @@ is_procptr_result (gfc_expr *expr)
 }
 
 
+/* Recursively append candidate argument ARG to CANDIDATES.  Store the
+   number of total candidates in CANDIDATES_LEN.  */
+
+static void
+lookup_arg_fuzzy_find_candidates (gfc_formal_arglist *arg,
+				  char **&candidates,
+				  size_t &candidates_len)
+{
+  for (gfc_formal_arglist *p = arg; p && p->sym; p = p->next)
+    vec_push (candidates, candidates_len, p->sym->name);
+}
+
+
+/* Lookup argument ARG fuzzily, taking names in ARGUMENTS into account.  */
+
+static const char*
+lookup_arg_fuzzy (const char *arg, gfc_formal_arglist *arguments)
+{
+  char **candidates = NULL;
+  size_t candidates_len = 0;
+  lookup_arg_fuzzy_find_candidates (arguments, candidates, candidates_len);
+  return gfc_closest_fuzzy_match (arg, candidates);
+}
+
+
 /* Given formal and actual argument lists, see if they are compatible.
    If they are compatible, the actual argument list is sorted to
    correspond with the formal list, and elements for missing optional
@@ -2831,8 +2870,16 @@ compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (f == NULL)
 	    {
 	      if (where)
-		gfc_error ("Keyword argument %qs at %L is not in "
-			   "the procedure", a->name, &a->expr->where);
+		{
+		  const char *guessed = lookup_arg_fuzzy (a->name, formal);
+		  if (guessed)
+		    gfc_error ("Keyword argument %qs at %L is not in "
+			       "the procedure; did you mean %qs?",
+			       a->name, &a->expr->where, guessed);
+		  else
+		    gfc_error ("Keyword argument %qs at %L is not in "
+			       "the procedure", a->name, &a->expr->where);
+		}
 	      return false;
 	    }
 
@@ -3552,8 +3599,15 @@ gfc_procedure_use (gfc_symbol *sym, gfc_actual_arglist **ap, locus *where)
     {
       if (sym->ns->has_implicit_none_export && sym->attr.proc == PROC_UNKNOWN)
 	{
-	  gfc_error ("Procedure %qs called at %L is not explicitly declared",
-		     sym->name, where);
+	  const char *guessed
+	    = gfc_lookup_function_fuzzy (sym->name, sym->ns->sym_root);
+	  if (guessed)
+	    gfc_error ("Procedure %qs called at %L is not explicitly declared"
+		       "; did you mean %qs?",
+		       sym->name, where, guessed);
+	  else
+	    gfc_error ("Procedure %qs called at %L is not explicitly declared",
+		       sym->name, where);
 	  return false;
 	}
       if (warn_implicit_interface)
diff --git a/gcc/fortran/invoke.texi b/gcc/fortran/invoke.texi
index 8892d501d58..261f2535bb5 100644
--- a/gcc/fortran/invoke.texi
+++ b/gcc/fortran/invoke.texi
@@ -41,7 +41,7 @@ remainder.
 @c man end
 @c man begin SEEALSO
 gpl(7), gfdl(7), fsf-funding(7),
-cpp(1), gcov(1), gcc(1), as(1), ld(1), gdb(1), adb(1), dbx(1), sdb(1)
+cpp(1), gcov(1), gcc(1), as(1), ld(1), gdb(1), dbx(1)
 and the Info entries for @file{gcc}, @file{cpp}, @file{gfortran}, @file{as},
 @file{ld}, @file{binutils} and @file{gdb}.
 @c man end
diff --git a/gcc/fortran/match.c b/gcc/fortran/match.c
index 4d657e0bc34..dcabe269e61 100644
--- a/gcc/fortran/match.c
+++ b/gcc/fortran/match.c
@@ -3968,7 +3968,10 @@ gfc_match_allocate (void)
   saw_stat = saw_errmsg = saw_source = saw_mold = saw_deferred = false;
 
   if (gfc_match_char ('(') != MATCH_YES)
-    goto syntax;
+    {
+      gfc_syntax_error (ST_ALLOCATE);
+      return MATCH_ERROR;
+    }
 
   /* Match an optional type-spec.  */
   old_locus = gfc_current_locus;
@@ -6204,7 +6207,7 @@ gfc_match_type_is (void)
   return MATCH_YES;
 
 syntax:
-  gfc_error ("Ssyntax error in TYPE IS specification at %C");
+  gfc_error ("Syntax error in TYPE IS specification at %C");
 
 cleanup:
   if (c != NULL)
diff --git a/gcc/fortran/misc.c b/gcc/fortran/misc.c
index a2c199efb56..f47d111ba47 100644
--- a/gcc/fortran/misc.c
+++ b/gcc/fortran/misc.c
@@ -22,6 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "gfortran.h"
+#include "spellcheck.h"
 
 
 /* Initialize a typespec to unknown.  */
@@ -280,3 +281,43 @@ get_c_kind(const char *c_kind_name, CInteropKind_t kinds_table[])
 
   return ISOCBINDING_INVALID;
 }
+
+
+/* For a given name TYPO, determine the best candidate from CANDIDATES
+   perusing Levenshtein distance.  Frees CANDIDATES before returning.  */
+
+const char *
+gfc_closest_fuzzy_match (const char *typo, char **candidates)
+{
+  /* Determine closest match.  */
+  const char *best = NULL;
+  char **cand = candidates;
+  edit_distance_t best_distance = MAX_EDIT_DISTANCE;
+  const size_t tl = strlen (typo);
+
+  while (cand && *cand)
+    {
+      edit_distance_t dist = levenshtein_distance (typo, tl, *cand,
+	  strlen (*cand));
+      if (dist < best_distance)
+	{
+	   best_distance = dist;
+	   best = *cand;
+	}
+      cand++;
+    }
+  /* If more than half of the letters were misspelled, the suggestion is
+     likely to be meaningless.  */
+  if (best)
+    {
+      unsigned int cutoff = MAX (tl, strlen (best)) / 2;
+
+      if (best_distance > cutoff)
+	{
+	  XDELETEVEC (candidates);
+	  return NULL;
+	}
+      XDELETEVEC (candidates);
+    }
+  return best;
+}
diff --git a/gcc/fortran/openmp.c b/gcc/fortran/openmp.c
index c5e00888bbe..2606323d42a 100644
--- a/gcc/fortran/openmp.c
+++ b/gcc/fortran/openmp.c
@@ -5262,7 +5262,7 @@ resolve_omp_atomic (gfc_code *code)
 }
 
 
-struct fortran_omp_context
+static struct fortran_omp_context
 {
   gfc_code *code;
   hash_set<gfc_symbol *> *sharing_clauses;
@@ -5345,6 +5345,8 @@ gfc_resolve_omp_parallel_blocks (gfc_code *code, gfc_namespace *ns)
     case EXEC_OMP_TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO:
     case EXEC_OMP_TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD:
     case EXEC_OMP_TARGET_TEAMS_DISTRIBUTE_SIMD:
+    case EXEC_OMP_TASKLOOP:
+    case EXEC_OMP_TASKLOOP_SIMD:
     case EXEC_OMP_TEAMS_DISTRIBUTE:
     case EXEC_OMP_TEAMS_DISTRIBUTE_PARALLEL_DO:
     case EXEC_OMP_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD:
@@ -5390,8 +5392,11 @@ gfc_omp_restore_state (struct gfc_omp_saved_state *state)
    construct, where they are predetermined private.  */
 
 void
-gfc_resolve_do_iterator (gfc_code *code, gfc_symbol *sym)
+gfc_resolve_do_iterator (gfc_code *code, gfc_symbol *sym, bool add_clause)
 {
+  if (omp_current_ctx == NULL)
+    return;
+
   int i = omp_current_do_collapse;
   gfc_code *c = omp_current_do_code;
 
@@ -5410,9 +5415,6 @@ gfc_resolve_do_iterator (gfc_code *code, gfc_symbol *sym)
       c = c->block->next;
     }
 
-  if (omp_current_ctx == NULL)
-    return;
-
   /* An openacc context may represent a data clause.  Abort if so.  */
   if (!omp_current_ctx->is_openmp && !oacc_is_loop (omp_current_ctx->code))
     return;
@@ -5421,7 +5423,7 @@ gfc_resolve_do_iterator (gfc_code *code, gfc_symbol *sym)
       && omp_current_ctx->sharing_clauses->contains (sym))
     return;
 
-  if (! omp_current_ctx->private_iterators->add (sym))
+  if (! omp_current_ctx->private_iterators->add (sym) && add_clause)
     {
       gfc_omp_clauses *omp_clauses = omp_current_ctx->code->ext.omp_clauses;
       gfc_omp_namelist *p;
@@ -5433,6 +5435,22 @@ gfc_resolve_do_iterator (gfc_code *code, gfc_symbol *sym)
     }
 }
 
+static void
+handle_local_var (gfc_symbol *sym)
+{
+  if (sym->attr.flavor != FL_VARIABLE
+      || sym->as != NULL
+      || (sym->ts.type != BT_INTEGER && sym->ts.type != BT_REAL))
+    return;
+  gfc_resolve_do_iterator (sym->ns->code, sym, false);
+}
+
+void
+gfc_resolve_omp_local_vars (gfc_namespace *ns)
+{
+  if (omp_current_ctx)
+    gfc_traverse_ns (ns, handle_local_var);
+}
 
 static void
 resolve_omp_do (gfc_code *code)
diff --git a/gcc/fortran/parse.c b/gcc/fortran/parse.c
index eb0f92e734b..e4deff9c79e 100644
--- a/gcc/fortran/parse.c
+++ b/gcc/fortran/parse.c
@@ -2737,6 +2737,9 @@ unexpected_eof (void)
   gfc_done_2 ();
 
   longjmp (eof_buf, 1);
+
+  /* Avoids build error on systems where longjmp is not declared noreturn.  */
+  gcc_unreachable ();
 }
 
 
diff --git a/gcc/fortran/resolve.c b/gcc/fortran/resolve.c
index bd316344813..40c1cd3c96f 100644
--- a/gcc/fortran/resolve.c
+++ b/gcc/fortran/resolve.c
@@ -2694,6 +2694,8 @@ generic:
       if (!gfc_convert_to_structure_constructor (expr, intr->sym, NULL,
 						 NULL, false))
 	return false;
+      if (!gfc_use_derived (expr->ts.u.derived))
+	return false;
       return resolve_structure_cons (expr, 0);
     }
 
@@ -2801,6 +2803,43 @@ resolve_specific_f (gfc_expr *expr)
   return true;
 }
 
+/* Recursively append candidate SYM to CANDIDATES.  Store the number of
+   candidates in CANDIDATES_LEN.  */
+
+static void
+lookup_function_fuzzy_find_candidates (gfc_symtree *sym,
+				       char **&candidates,
+				       size_t &candidates_len)
+{
+  gfc_symtree *p;
+
+  if (sym == NULL)
+    return;
+  if ((sym->n.sym->ts.type != BT_UNKNOWN || sym->n.sym->attr.external)
+      && sym->n.sym->attr.flavor == FL_PROCEDURE)
+    vec_push (candidates, candidates_len, sym->name);
+
+  p = sym->left;
+  if (p)
+    lookup_function_fuzzy_find_candidates (p, candidates, candidates_len);
+
+  p = sym->right;
+  if (p)
+    lookup_function_fuzzy_find_candidates (p, candidates, candidates_len);
+}
+
+
+/* Lookup function FN fuzzily, taking names in SYMROOT into account.  */
+
+const char*
+gfc_lookup_function_fuzzy (const char *fn, gfc_symtree *symroot)
+{
+  char **candidates = NULL;
+  size_t candidates_len = 0;
+  lookup_function_fuzzy_find_candidates (symroot, candidates, candidates_len);
+  return gfc_closest_fuzzy_match (fn, candidates);
+}
+
 
 /* Resolve a procedure call not known to be generic nor specific.  */
 
@@ -2851,8 +2890,15 @@ set_type:
 
       if (ts->type == BT_UNKNOWN)
 	{
-	  gfc_error ("Function %qs at %L has no IMPLICIT type",
-		     sym->name, &expr->where);
+	  const char *guessed
+	    = gfc_lookup_function_fuzzy (sym->name, sym->ns->sym_root);
+	  if (guessed)
+	    gfc_error ("Function %qs at %L has no IMPLICIT type"
+		       "; did you mean %qs?",
+		       sym->name, &expr->where, guessed);
+	  else
+	    gfc_error ("Function %qs at %L has no IMPLICIT type",
+		       sym->name, &expr->where);
 	  return false;
 	}
       else
@@ -3713,6 +3759,46 @@ logical_to_bitwise (gfc_expr *e)
   return e;
 }
 
+/* Recursively append candidate UOP to CANDIDATES.  Store the number of
+   candidates in CANDIDATES_LEN.  */
+static void
+lookup_uop_fuzzy_find_candidates (gfc_symtree *uop,
+				  char **&candidates,
+				  size_t &candidates_len)
+{
+  gfc_symtree *p;
+
+  if (uop == NULL)
+    return;
+
+  /* Not sure how to properly filter here.  Use all for a start.
+     n.uop.op is NULL for empty interface operators (is that legal?) disregard
+     these as i suppose they don't make terribly sense.  */
+
+  if (uop->n.uop->op != NULL)
+    vec_push (candidates, candidates_len, uop->name);
+
+  p = uop->left;
+  if (p)
+    lookup_uop_fuzzy_find_candidates (p, candidates, candidates_len);
+
+  p = uop->right;
+  if (p)
+    lookup_uop_fuzzy_find_candidates (p, candidates, candidates_len);
+}
+
+/* Lookup user-operator OP fuzzily, taking names in UOP into account.  */
+
+static const char*
+lookup_uop_fuzzy (const char *op, gfc_symtree *uop)
+{
+  char **candidates = NULL;
+  size_t candidates_len = 0;
+  lookup_uop_fuzzy_find_candidates (uop, candidates, candidates_len);
+  return gfc_closest_fuzzy_match (op, candidates);
+}
+
+
 /* Resolve an operator expression node.  This can involve replacing the
    operation with a user defined function call.  */
 
@@ -3935,8 +4021,16 @@ resolve_operator (gfc_expr *e)
 
     case INTRINSIC_USER:
       if (e->value.op.uop->op == NULL)
-	sprintf (msg, _("Unknown operator %%<%s%%> at %%L"),
-		 e->value.op.uop->name);
+	{
+	  const char *name = e->value.op.uop->name;
+	  const char *guessed;
+	  guessed = lookup_uop_fuzzy (name, e->value.op.uop->ns->uop_root);
+	  if (guessed)
+	    sprintf (msg, _("Unknown operator %%<%s%%> at %%L; did you mean '%s'?"),
+		name, guessed);
+	  else
+	    sprintf (msg, _("Unknown operator %%<%s%%> at %%L"), name);
+	}
       else if (op2 == NULL)
 	sprintf (msg, _("Operand of user operator %%<%s%%> at %%L is %s"),
 		 e->value.op.uop->name, gfc_typename (&op1->ts));
@@ -9087,7 +9181,7 @@ resolve_transfer (gfc_code *code)
   if (dt && dt->dt_io_kind->value.iokind != M_INQUIRE
       && (ts->type == BT_DERIVED || ts->type == BT_CLASS))
     {
-      if (ts->type == BT_DERIVED)
+      if (ts->type == BT_DERIVED || ts->type == BT_CLASS)
 	derived = ts->u.derived;
       else
 	derived = ts->u.derived->components->ts.u.derived;
@@ -10916,6 +11010,8 @@ gfc_resolve_code (gfc_code *code, gfc_namespace *ns)
 	    case EXEC_OMP_TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD:
 	    case EXEC_OMP_TARGET_TEAMS_DISTRIBUTE_SIMD:
 	    case EXEC_OMP_TASK:
+	    case EXEC_OMP_TASKLOOP:
+	    case EXEC_OMP_TASKLOOP_SIMD:
 	    case EXEC_OMP_TEAMS:
 	    case EXEC_OMP_TEAMS_DISTRIBUTE:
 	    case EXEC_OMP_TEAMS_DISTRIBUTE_PARALLEL_DO:
@@ -10931,8 +11027,6 @@ gfc_resolve_code (gfc_code *code, gfc_namespace *ns)
 	    case EXEC_OMP_DO_SIMD:
 	    case EXEC_OMP_SIMD:
 	    case EXEC_OMP_TARGET_SIMD:
-	    case EXEC_OMP_TASKLOOP:
-	    case EXEC_OMP_TASKLOOP_SIMD:
 	      gfc_resolve_omp_do_blocks (code, ns);
 	      break;
 	    case EXEC_SELECT_TYPE:
@@ -11193,7 +11287,8 @@ start:
 	    {
 	      gfc_iterator *iter = code->ext.iterator;
 	      if (gfc_resolve_iterator (iter, true, false))
-		gfc_resolve_do_iterator (code, iter->var->symtree->n.sym);
+		gfc_resolve_do_iterator (code, iter->var->symtree->n.sym,
+					 true);
 	    }
 	  break;
 
@@ -13844,6 +13939,7 @@ resolve_fl_derived0 (gfc_symbol *sym)
 {
   gfc_symbol* super_type;
   gfc_component *c;
+  gfc_formal_arglist *f;
   bool success;
 
   if (sym->attr.unlimited_polymorphic)
@@ -13896,6 +13992,22 @@ resolve_fl_derived0 (gfc_symbol *sym)
       && !ensure_not_abstract (sym, super_type))
     return false;
 
+  /* Check that there is a component for every PDT parameter.  */
+  if (sym->attr.pdt_template)
+    {
+      for (f = sym->formal; f; f = f->next)
+	{
+	  c = gfc_find_component (sym, f->sym->name, true, true, NULL);
+	  if (c == NULL)
+	    {
+	      gfc_error ("Parameterized type %qs does not have a component "
+			 "corresponding to parameter %qs at %L", sym->name,
+			 f->sym->name, &sym->declared_at);
+	      break;
+	    }
+	}
+    }
+
   /* Add derived type to the derived type list.  */
   add_dt_to_dt_list (sym);
 
@@ -14403,7 +14515,23 @@ resolve_symbol (gfc_symbol *sym)
 
   if (as)
     {
-      gcc_assert (as->type != AS_IMPLIED_SHAPE);
+      /* If AS_IMPLIED_SHAPE makes it to here, it must be a bad
+	 specification expression.  */
+      if (as->type == AS_IMPLIED_SHAPE)
+	{
+	  int i;
+	  for (i=0; i<as->rank; i++)
+	    {
+	      if (as->lower[i] != NULL && as->upper[i] == NULL)
+		{
+		  gfc_error ("Bad specification for assumed size array at %L",
+			     &as->lower[i]->where);
+		  return;
+		}
+	    }
+	  gcc_unreachable();
+	}
+
       if (((as->type == AS_ASSUMED_SIZE && !as->cp_was_assumed)
 	   || as->type == AS_ASSUMED_SHAPE)
 	  && !sym->attr.dummy && !sym->attr.select_type_temporary)
@@ -14967,7 +15095,12 @@ resolve_symbol (gfc_symbol *sym)
 
       if ((!a->save && !a->dummy && !a->pointer
 	   && !a->in_common && !a->use_assoc
-	   && !a->result && !a->function)
+	   && a->referenced
+	   && !((a->function || a->result)
+		&& (!a->dimension
+		    || sym->ts.u.derived->attr.alloc_comp
+		    || sym->ts.u.derived->attr.pointer_comp))
+	   && !(a->function && sym != sym->result))
 	  || (a->dummy && a->intent == INTENT_OUT && !a->pointer))
 	apply_default_init (sym);
       else if (a->function && sym->result && a->access != ACCESS_PRIVATE
@@ -15803,9 +15936,22 @@ resolve_equivalence (gfc_equiv *eq)
 	  && sym->ns->proc_name->attr.pure
 	  && sym->attr.in_common)
 	{
-	  gfc_error ("Common block member %qs at %L cannot be an EQUIVALENCE "
-		     "object in the pure procedure %qs",
-		     sym->name, &e->where, sym->ns->proc_name->name);
+	  /* Need to check for symbols that may have entered the pure
+	     procedure via a USE statement.  */
+	  bool saw_sym = false;
+	  if (sym->ns->use_stmts)
+	    {
+	      gfc_use_rename *r;
+	      for (r = sym->ns->use_stmts->rename; r; r = r->next)
+		if (strcmp(r->use_name, sym->name) == 0) saw_sym = true; 
+	    }
+	  else
+	    saw_sym = true;
+
+	  if (saw_sym)
+	    gfc_error ("COMMON block member %qs at %L cannot be an "
+		       "EQUIVALENCE object in the pure procedure %qs",
+		       sym->name, &e->where, sym->ns->proc_name->name);
 	  break;
 	}
 
@@ -16239,6 +16385,7 @@ resolve_codes (gfc_namespace *ns)
   bitmap_obstack_initialize (&labels_obstack);
 
   gfc_resolve_oacc_declare (ns);
+  gfc_resolve_omp_local_vars (ns);
   gfc_resolve_code (ns->code, ns);
 
   bitmap_obstack_release (&labels_obstack);
diff --git a/gcc/fortran/scanner.c b/gcc/fortran/scanner.c
index 82f431da527..49decfac52a 100644
--- a/gcc/fortran/scanner.c
+++ b/gcc/fortran/scanner.c
@@ -80,6 +80,7 @@ static struct gfc_file_change
 size_t file_changes_cur, file_changes_count;
 size_t file_changes_allocated;
 
+static gfc_char_t *last_error_char;
 
 /* Functions dealing with our wide characters (gfc_char_t) and
    sequences of such characters.  */
@@ -269,6 +270,7 @@ gfc_scanner_init_1 (void)
   continue_line = 0;
 
   end_flag = 0;
+  last_error_char = NULL;
 }
 
 
@@ -1700,6 +1702,14 @@ gfc_gobble_whitespace (void)
     }
   while (gfc_is_whitespace (c));
 
+  if (!ISPRINT(c) && c != '\n' && last_error_char != gfc_current_locus.nextc)
+    {
+      char buf[20];
+      last_error_char = gfc_current_locus.nextc;
+      snprintf (buf, 20, "%2.2X", c);
+      gfc_error_now ("Invalid character 0x%s at %C", buf);
+    }
+
   gfc_current_locus = old_loc;
 }
 
diff --git a/gcc/fortran/simplify.c b/gcc/fortran/simplify.c
index 169aef1d892..ba010a0aebf 100644
--- a/gcc/fortran/simplify.c
+++ b/gcc/fortran/simplify.c
@@ -227,7 +227,8 @@ convert_boz (gfc_expr *x, int kind)
 }
 
 
-/* Test that the expression is an constant array.  */
+/* Test that the expression is an constant array, simplifying if
+   we are dealing with a parameter array.  */
 
 static bool
 is_constant_array_expr (gfc_expr *e)
@@ -237,6 +238,10 @@ is_constant_array_expr (gfc_expr *e)
   if (e == NULL)
     return true;
 
+  if (e->expr_type == EXPR_VARIABLE && e->rank > 0
+      && e->symtree->n.sym->attr.flavor == FL_PARAMETER)
+    gfc_simplify_expr (e, 1);
+
   if (e->expr_type != EXPR_ARRAY || !gfc_is_constant_expr (e))
     return false;
 
diff --git a/gcc/fortran/symbol.c b/gcc/fortran/symbol.c
index 4c109fdfbad..11b6f600103 100644
--- a/gcc/fortran/symbol.c
+++ b/gcc/fortran/symbol.c
@@ -245,6 +245,44 @@ gfc_get_default_type (const char *name, gfc_namespace *ns)
 }
 
 
+/* Recursively append candidate SYM to CANDIDATES.  Store the number of
+   candidates in CANDIDATES_LEN.  */
+
+static void
+lookup_symbol_fuzzy_find_candidates (gfc_symtree *sym,
+				     char **&candidates,
+				     size_t &candidates_len)
+{
+  gfc_symtree *p;
+
+  if (sym == NULL)
+    return;
+
+  if (sym->n.sym->ts.type != BT_UNKNOWN && sym->n.sym->ts.type != BT_PROCEDURE)
+    vec_push (candidates, candidates_len, sym->name);
+  p = sym->left;
+  if (p)
+    lookup_symbol_fuzzy_find_candidates (p, candidates, candidates_len);
+
+  p = sym->right;
+  if (p)
+    lookup_symbol_fuzzy_find_candidates (p, candidates, candidates_len);
+}
+
+
+/* Lookup symbol SYM_NAME fuzzily, taking names in SYMBOL into account.  */
+
+static const char*
+lookup_symbol_fuzzy (const char *sym_name, gfc_symbol *symbol)
+{
+  char **candidates = NULL;
+  size_t candidates_len = 0;
+  lookup_symbol_fuzzy_find_candidates (symbol->ns->sym_root, candidates,
+				       candidates_len);
+  return gfc_closest_fuzzy_match (sym_name, candidates);
+}
+
+
 /* Given a pointer to a symbol, set its type according to the first
    letter of its name.  Fails if the letter in question has no default
    type.  */
@@ -263,8 +301,14 @@ gfc_set_default_type (gfc_symbol *sym, int error_flag, gfc_namespace *ns)
     {
       if (error_flag && !sym->attr.untyped)
 	{
-	  gfc_error ("Symbol %qs at %L has no IMPLICIT type",
-		     sym->name, &sym->declared_at);
+	  const char *guessed = lookup_symbol_fuzzy (sym->name, sym);
+	  if (guessed)
+	    gfc_error ("Symbol %qs at %L has no IMPLICIT type"
+		       "; did you mean %qs?",
+		       sym->name, &sym->declared_at, guessed);
+	  else
+	    gfc_error ("Symbol %qs at %L has no IMPLICIT type",
+		       sym->name, &sym->declared_at);
 	  sym->attr.untyped = 1; /* Ensure we only give an error once.  */
 	}
 
@@ -382,7 +426,8 @@ check_conflict (symbol_attribute *attr, const char *name, locus *where)
     *is_bind_c = "BIND(C)", *procedure = "PROCEDURE",
     *proc_pointer = "PROCEDURE POINTER", *abstract = "ABSTRACT",
     *asynchronous = "ASYNCHRONOUS", *codimension = "CODIMENSION",
-    *contiguous = "CONTIGUOUS", *generic = "GENERIC", *automatic = "AUTOMATIC";
+    *contiguous = "CONTIGUOUS", *generic = "GENERIC", *automatic = "AUTOMATIC",
+    *pdt_len = "LEN", *pdt_kind = "KIND";
   static const char *threadprivate = "THREADPRIVATE";
   static const char *omp_declare_target = "OMP DECLARE TARGET";
   static const char *omp_declare_target_link = "OMP DECLARE TARGET LINK";
@@ -663,6 +708,23 @@ check_conflict (symbol_attribute *attr, const char *name, locus *where)
   conf (entry, oacc_declare_deviceptr)
   conf (entry, oacc_declare_device_resident)
 
+  conf (pdt_kind, allocatable)
+  conf (pdt_kind, pointer)
+  conf (pdt_kind, dimension)
+  conf (pdt_kind, codimension)
+
+  conf (pdt_len, allocatable)
+  conf (pdt_len, pointer)
+  conf (pdt_len, dimension)
+  conf (pdt_len, codimension)
+
+  if (attr->access == ACCESS_PRIVATE)
+    {
+      a1 = privat;
+      conf2 (pdt_kind);
+      conf2 (pdt_len);
+    }
+
   a1 = gfc_code2string (flavors, attr->flavor);
 
   if (attr->in_namelist
@@ -2336,6 +2398,32 @@ find_union_component (gfc_symbol *un, const char *name,
 }
 
 
+/* Recursively append candidate COMPONENT structures to CANDIDATES.  Store
+   the number of total candidates in CANDIDATES_LEN.  */
+
+static void
+lookup_component_fuzzy_find_candidates (gfc_component *component,
+					char **&candidates,
+					size_t &candidates_len)
+{
+  for (gfc_component *p = component; p; p = p->next)
+    vec_push (candidates, candidates_len, p->name);
+}
+
+
+/* Lookup component MEMBER fuzzily, taking names in COMPONENT into account.  */
+
+static const char*
+lookup_component_fuzzy (const char *member, gfc_component *component)
+{
+  char **candidates = NULL;
+  size_t candidates_len = 0;
+  lookup_component_fuzzy_find_candidates (component, candidates,
+					  candidates_len);
+  return gfc_closest_fuzzy_match (member, candidates);
+}
+
+
 /* Given a derived type node and a component name, try to locate the
    component structure.  Returns the NULL pointer if the component is
    not found or the components are private.  If noaccess is set, no access
@@ -2433,8 +2521,16 @@ gfc_find_component (gfc_symbol *sym, const char *name,
     }
 
   if (p == NULL && !silent)
-    gfc_error ("%qs at %C is not a member of the %qs structure",
-	       name, sym->name);
+    {
+      const char *guessed = lookup_component_fuzzy (name, sym->components);
+      if (guessed)
+	gfc_error ("%qs at %C is not a member of the %qs structure"
+		   "; did you mean %qs?",
+		   name, sym->name, guessed);
+      else
+	gfc_error ("%qs at %C is not a member of the %qs structure",
+		   name, sym->name);
+    }
 
   /* Component was found; build the ultimate component reference. */
   if (p != NULL && ref)
diff --git a/gcc/fortran/trans-decl.c b/gcc/fortran/trans-decl.c
index 019b8035b6f..45d5119236a 100644
--- a/gcc/fortran/trans-decl.c
+++ b/gcc/fortran/trans-decl.c
@@ -1670,7 +1670,9 @@ gfc_get_symbol_decl (gfc_symbol * sym)
     {
       /* Catch functions. Only used for actual parameters,
 	 procedure pointers and procptr initialization targets.  */
-      if (sym->attr.use_assoc || sym->attr.intrinsic
+      if (sym->attr.use_assoc
+	  || sym->attr.used_in_submodule
+	  || sym->attr.intrinsic
 	  || sym->attr.if_source != IFSRC_DECL)
 	{
 	  decl = gfc_get_extern_function_decl (sym);
@@ -4582,7 +4584,10 @@ gfc_trans_deferred_vars (gfc_symbol * proc_sym, gfc_wrapped_block * block)
 		    && sym->ts.u.cl->passed_length)
 		tmp = gfc_null_and_pass_deferred_len (sym, &init, &loc);
 	      else
-		gfc_restore_backend_locus (&loc);
+		{
+		  gfc_restore_backend_locus (&loc);
+		  tmp = NULL_TREE;
+		}
 
 	      /* Deallocate when leaving the scope. Nullifying is not
 		 needed.  */
@@ -4634,10 +4639,6 @@ gfc_trans_deferred_vars (gfc_symbol * proc_sym, gfc_wrapped_block * block)
 		}
 
 	      gfc_add_init_cleanup (block, gfc_finish_block (&init), tmp);
-	      /* TODO find out why this is necessary to stop double calls to
-		 free.  Somebody is reusing the expression in 'tmp' because
-		 it is being used unititialized.  */
-	      tmp = NULL_TREE;
 	    }
 	}
       else if (sym->ts.type == BT_CHARACTER && sym->ts.deferred)
diff --git a/gcc/fortran/trans-expr.c b/gcc/fortran/trans-expr.c
index 4e8bfc5d6f9..1a3e3d45e4c 100644
--- a/gcc/fortran/trans-expr.c
+++ b/gcc/fortran/trans-expr.c
@@ -5173,10 +5173,39 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 			}
 		      else
 			{
-			  gfc_add_modify (&parmse.pre, var,
-					  fold_build1_loc (input_location,
-							   VIEW_CONVERT_EXPR,
-							   type, parmse.expr));
+			  /* Since the internal representation of unlimited
+			     polymorphic expressions includes an extra field
+			     that other class objects do not, a cast to the
+			     formal type does not work.  */
+			  if (!UNLIMITED_POLY (e) && UNLIMITED_POLY (fsym))
+			    {
+			      tree efield;
+
+			      /* Set the _data field.  */
+			      tmp = gfc_class_data_get (var);
+			      efield = fold_convert (TREE_TYPE (tmp),
+					gfc_class_data_get (parmse.expr));
+			      gfc_add_modify (&parmse.pre, tmp, efield);
+
+			      /* Set the _vptr field.  */
+			      tmp = gfc_class_vptr_get (var);
+			      efield = fold_convert (TREE_TYPE (tmp),
+					gfc_class_vptr_get (parmse.expr));
+			      gfc_add_modify (&parmse.pre, tmp, efield);
+
+			      /* Set the _len field.  */
+			      tmp = gfc_class_len_get (var);
+			      gfc_add_modify (&parmse.pre, tmp,
+					      build_int_cst (TREE_TYPE (tmp), 0));
+			    }
+			  else
+			    {
+			      tmp = fold_build1_loc (input_location,
+						     VIEW_CONVERT_EXPR,
+						     type, parmse.expr);
+			      gfc_add_modify (&parmse.pre, var, tmp);
+					      ;
+			    }
 			  parmse.expr = gfc_build_addr_expr (NULL_TREE, var);
 			}
 		    }
@@ -8053,7 +8082,7 @@ trans_class_vptr_len_assignment (stmtblock_t *block, gfc_expr * le,
     {
       /* Get the vptr from the rhs expression only, when it is variable.
 	 Functions are expected to be assigned to a temporary beforehand.  */
-      vptr_expr = re->expr_type == EXPR_VARIABLE
+      vptr_expr = (re->expr_type == EXPR_VARIABLE && re->ts.type == BT_CLASS)
 	  ? gfc_find_and_cut_at_last_class_ref (re)
 	  : NULL;
       if (vptr_expr != NULL && vptr_expr->ts.type == BT_CLASS)
diff --git a/gcc/fortran/trans-io.c b/gcc/fortran/trans-io.c
index 026f9a993d2..f3e1f3e4d09 100644
--- a/gcc/fortran/trans-io.c
+++ b/gcc/fortran/trans-io.c
@@ -2404,7 +2404,7 @@ transfer_expr (gfc_se * se, gfc_typespec * ts, tree addr_expr,
     case BT_CLASS:
       if (ts->u.derived->components == NULL)
 	return;
-      if (ts->type == BT_DERIVED || ts->type == BT_CLASS)
+      if (gfc_bt_struct (ts->type) || ts->type == BT_CLASS)
 	{
 	  gfc_symbol *derived;
 	  gfc_symbol *dtio_sub = NULL;
@@ -2438,7 +2438,7 @@ transfer_expr (gfc_se * se, gfc_typespec * ts, tree addr_expr,
 	      function = iocall[IOCALL_X_DERIVED];
 	      break;
 	    }
-	  else if (ts->type == BT_DERIVED)
+	  else if (gfc_bt_struct (ts->type))
 	    {
 	      /* Recurse into the elements of the derived type.  */
 	      expr = gfc_evaluate_now (addr_expr, &se->pre);
diff --git a/gcc/function.c b/gcc/function.c
index 1c94329c063..3846555f963 100644
--- a/gcc/function.c
+++ b/gcc/function.c
@@ -415,7 +415,7 @@ assign_stack_local_1 (machine_mode mode, poly_int64 size,
 		     requested size is 0 or the estimated stack
 		     alignment >= mode alignment.  */
 		  gcc_assert ((kind & ASLK_REDUCE_ALIGN)
-			      || known_zero (size)
+			      || must_eq (size, 0)
 			      || (crtl->stack_alignment_estimated
 				  >= GET_MODE_ALIGNMENT (mode)));
 		  alignment_in_bits = crtl->stack_alignment_estimated;
@@ -430,7 +430,7 @@ assign_stack_local_1 (machine_mode mode, poly_int64 size,
   if (crtl->max_used_stack_slot_alignment < alignment_in_bits)
     crtl->max_used_stack_slot_alignment = alignment_in_bits;
 
-  if (mode != BLKmode || maybe_nonzero (size))
+  if (mode != BLKmode || may_ne (size, 0))
     {
       if (kind & ASLK_RECORD_PAD)
 	{
@@ -976,25 +976,26 @@ assign_temp (tree type_or_decl, int memory_required,
 
   if (mode == BLKmode || memory_required)
     {
-      HOST_WIDE_INT size = int_size_in_bytes (type);
+      poly_int64 size;
       rtx tmp;
 
-      /* Zero sized arrays are GNU C extension.  Set size to 1 to avoid
-	 problems with allocating the stack space.  */
-      if (size == 0)
-	size = 1;
-
       /* Unfortunately, we don't yet know how to allocate variable-sized
 	 temporaries.  However, sometimes we can find a fixed upper limit on
 	 the size, so try that instead.  */
-      else if (size == -1)
+      if (!poly_int_tree_p (TYPE_SIZE_UNIT (type), &size))
 	size = max_int_size_in_bytes (type);
 
+      /* Zero sized arrays are GNU C extension.  Set size to 1 to avoid
+	 problems with allocating the stack space.  */
+      if (must_eq (size, 0))
+	size = 1;
+
       /* The size of the temporary may be too large to fit into an integer.  */
       /* ??? Not sure this should happen except for user silliness, so limit
 	 this to things that aren't compiler-generated temporaries.  The
 	 rest of the time we'll die in assign_stack_temp_for_type.  */
-      if (decl && size == -1
+      if (decl
+	  && !known_size_p (size)
 	  && TREE_CODE (TYPE_SIZE_UNIT (type)) == INTEGER_CST)
 	{
 	  error ("size of variable %q+D is too large", decl);
@@ -1573,7 +1574,7 @@ instantiate_virtual_regs_in_insn (rtx_insn *insn)
 	 move insn in the initial rtl stream.  */
       new_rtx = instantiate_new_reg (SET_SRC (set), &offset);
       if (new_rtx
-	  && maybe_nonzero (offset)
+	  && may_ne (offset, 0)
 	  && REG_P (SET_DEST (set))
 	  && REGNO (SET_DEST (set)) > LAST_VIRTUAL_REGISTER)
 	{
@@ -1610,7 +1611,7 @@ instantiate_virtual_regs_in_insn (rtx_insn *insn)
 	  offset += delta;
 
 	  /* If the sum is zero, then replace with a plain move.  */
-	  if (known_zero (offset)
+	  if (must_eq (offset, 0)
 	      && REG_P (SET_DEST (set))
 	      && REGNO (SET_DEST (set)) > LAST_VIRTUAL_REGISTER)
 	    {
@@ -1688,7 +1689,7 @@ instantiate_virtual_regs_in_insn (rtx_insn *insn)
 	  new_rtx = instantiate_new_reg (x, &offset);
 	  if (new_rtx == NULL)
 	    continue;
-	  if (known_zero (offset))
+	  if (must_eq (offset, 0))
 	    x = new_rtx;
 	  else
 	    {
@@ -1713,7 +1714,7 @@ instantiate_virtual_regs_in_insn (rtx_insn *insn)
 	  new_rtx = instantiate_new_reg (SUBREG_REG (x), &offset);
 	  if (new_rtx == NULL)
 	    continue;
-	  if (maybe_nonzero (offset))
+	  if (may_ne (offset, 0))
 	    {
 	      start_sequence ();
 	      new_rtx = expand_simple_binop
@@ -2707,7 +2708,7 @@ assign_parm_find_stack_rtl (tree parm, struct assign_parm_data_one *data)
 	    {
 	      poly_int64 offset = subreg_lowpart_offset (DECL_MODE (parm),
 							 data->promoted_mode);
-	      if (maybe_nonzero (offset))
+	      if (may_ne (offset, 0))
 		set_mem_offset (stack_parm, MEM_OFFSET (stack_parm) - offset);
 	    }
 	}
@@ -3440,7 +3441,7 @@ assign_parm_setup_stack (struct assign_parm_data_all *all, tree parm,
 	  /* ??? This may need a big-endian conversion on sparc64.  */
 	  data->stack_parm
 	    = adjust_address (data->stack_parm, data->nominal_mode, 0);
-	  if (maybe_nonzero (offset) && MEM_OFFSET_KNOWN_P (data->stack_parm))
+	  if (may_ne (offset, 0) && MEM_OFFSET_KNOWN_P (data->stack_parm))
 	    set_mem_offset (data->stack_parm,
 			    MEM_OFFSET (data->stack_parm) + offset);
 	}
@@ -4061,10 +4062,9 @@ gimplify_parameters (void)
 		  DECL_IGNORED_P (addr) = 0;
 		  local = build_fold_indirect_ref (addr);
 
-		  t = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
-		  t = build_call_expr (t, 2, DECL_SIZE_UNIT (parm),
-				       size_int (DECL_ALIGN (parm)));
-
+		  t = build_alloca_call_expr (DECL_SIZE_UNIT (parm),
+					      DECL_ALIGN (parm),
+					      max_int_size_in_bytes (type));
 		  /* The call has been built for a variable-sized object.  */
 		  CALL_ALLOCA_FOR_VAR_P (t) = 1;
 		  t = fold_convert (ptr_type, t);
@@ -4731,11 +4731,11 @@ number_blocks (tree fn)
   int n_blocks;
   tree *block_vector;
 
-  /* For SDB and XCOFF debugging output, we start numbering the blocks
+  /* For XCOFF debugging output, we start numbering the blocks
      from 1 within each function, rather than keeping a running
      count.  */
-#if SDB_DEBUGGING_INFO || defined (XCOFF_DEBUGGING_INFO)
-  if (write_symbols == SDB_DEBUG || write_symbols == XCOFF_DEBUG)
+#if defined (XCOFF_DEBUGGING_INFO)
+  if (write_symbols == XCOFF_DEBUG)
     next_block_index = 1;
 #endif
 
@@ -5270,7 +5270,7 @@ expand_function_start (tree subr)
     }
 
   /* The following was moved from init_function_start.
-     The move is supposed to make sdb output more accurate.  */
+     The move was supposed to make sdb output more accurate.  */
   /* Indicate the beginning of the function body,
      as opposed to parm setup.  */
   emit_note (NOTE_INSN_FUNCTION_BEG);
@@ -5461,7 +5461,7 @@ expand_function_end (void)
   do_pending_stack_adjust ();
 
   /* Output a linenumber for the end of the function.
-     SDB depends on this.  */
+     SDB depended on this.  */
   set_curr_insn_location (input_location);
 
   /* Before the return label (if any), clobber the return
diff --git a/gcc/gcc-ar.c b/gcc/gcc-ar.c
index 78d2fc1ad30..d5d80e042e5 100644
--- a/gcc/gcc-ar.c
+++ b/gcc/gcc-ar.c
@@ -194,14 +194,6 @@ main (int ac, char **av)
 #ifdef CROSS_DIRECTORY_STRUCTURE
       real_exe_name = concat (target_machine, "-", PERSONALITY, NULL);
 #endif
-      /* Do not search original location in the same folder.  */
-      char *exe_folder = lrealpath (av[0]);
-      exe_folder[strlen (exe_folder) - strlen (lbasename (exe_folder))] = '\0';
-      char *location = concat (exe_folder, PERSONALITY, NULL);
-
-      if (access (location, X_OK) == 0)
-	remove_prefix (exe_folder, &path);
-
       exe_name = find_a_file (&path, real_exe_name, X_OK);
       if (!exe_name)
 	{
diff --git a/gcc/gcc.c b/gcc/gcc.c
index cec3ed5be5f..43e6d590c25 100644
--- a/gcc/gcc.c
+++ b/gcc/gcc.c
@@ -170,9 +170,10 @@ env_manager::restore ()
 
 
 /* By default there is no special suffix for target executables.  */
-/* FIXME: when autoconf is fixed, remove the host check - dj */
-#if defined(TARGET_EXECUTABLE_SUFFIX) && defined(HOST_EXECUTABLE_SUFFIX)
+#ifdef TARGET_EXECUTABLE_SUFFIX
 #define HAVE_TARGET_EXECUTABLE_SUFFIX
+#else
+#define TARGET_EXECUTABLE_SUFFIX ""
 #endif
 
 /* By default there is no special suffix for host executables.  */
@@ -1117,7 +1118,7 @@ static const char *cpp_unique_options =
  %{MMD:-MMD %{!o:%b.d}%{o*:%.d%*}}\
  %{M} %{MM} %{MF*} %{MG} %{MP} %{MQ*} %{MT*}\
  %{!E:%{!M:%{!MM:%{!MT:%{!MQ:%{MD|MMD:%{o*:-MQ %*}}}}}}}\
- %{remap} %{g3|ggdb3|gstabs3|gcoff3|gxcoff3|gvms3:-dD}\
+ %{remap} %{g3|ggdb3|gstabs3|gxcoff3|gvms3:-dD}\
  %{!iplugindir*:%{fplugin*:%:find-plugindir()}}\
  %{H} %C %{D*&U*&A*} %{i*} %Z %i\
  %{E|M|MM:%W{o*}}";
@@ -5314,7 +5315,7 @@ do_spec_1 (const char *spec, int inswitch, const char *soft_matched_part)
 	      buf = (char *) alloca (p - q + 1);
 	      strncpy (buf, q, p - q);
 	      buf[p - q] = 0;
-	      inform (0, "%s", _(buf));
+	      inform (UNKNOWN_LOCATION, "%s", _(buf));
 	      if (*p)
 		p++;
 	    }
@@ -8191,7 +8192,8 @@ driver::do_spec_on_infiles () const
 	      else if (compare_debug && debug_check_temp_file[0])
 		{
 		  if (verbose_flag)
-		    inform (0, "recompiling with -fcompare-debug");
+		    inform (UNKNOWN_LOCATION,
+			    "recompiling with -fcompare-debug");
 
 		  compare_debug = -compare_debug;
 		  n_switches = n_switches_debug_check[1];
@@ -8216,7 +8218,7 @@ driver::do_spec_on_infiles () const
 					       debug_check_temp_file[1]));
 
 		  if (verbose_flag)
-		    inform (0, "comparing final insns dumps");
+		    inform (UNKNOWN_LOCATION, "comparing final insns dumps");
 
 		  if (compare_files (debug_check_temp_file))
 		    this_file_error = 1;
diff --git a/gcc/gcov.c b/gcc/gcov.c
index c56bac20278..48bcdc0d4c3 100644
--- a/gcc/gcov.c
+++ b/gcc/gcov.c
@@ -33,6 +33,7 @@ along with Gcov; see the file COPYING3.  If not see
 #include "config.h"
 #define INCLUDE_ALGORITHM
 #define INCLUDE_VECTOR
+#define INCLUDE_STRING
 #include "system.h"
 #include "coretypes.h"
 #include "tm.h"
@@ -40,6 +41,7 @@ along with Gcov; see the file COPYING3.  If not see
 #include "diagnostic.h"
 #include "version.h"
 #include "demangle.h"
+#include "color-macros.h"
 
 #include <getopt.h>
 
@@ -106,9 +108,6 @@ typedef struct arc_info
   /* Loop making arc.  */
   unsigned int cycle : 1;
 
-  /* Next branch on line.  */
-  struct arc_info *line_next;
-
   /* Links to next arc on src and dst lists.  */
   struct arc_info *succ_next;
   struct arc_info *pred_next;
@@ -243,67 +242,109 @@ typedef struct coverage_info
 /* Describes a single line of source. Contains a chain of basic blocks
    with code on it.  */
 
-typedef struct line_info
+struct line_info
 {
+  /* Default constructor.  */
+  line_info ();
+
   /* Return true when NEEDLE is one of basic blocks the line belongs to.  */
   bool has_block (block_t *needle);
 
-  gcov_type count;	   /* execution count */
-  arc_t *branches;	   /* branches from blocks that end on this line.  */
-  block_t *blocks;	   /* blocks which start on this line.
-			      Used in all-blocks mode.  */
+  /* Execution count.  */
+  gcov_type count;
+
+  /* Branches from blocks that end on this line.  */
+  vector<arc_t *> branches;
+
+  /* blocks which start on this line.  Used in all-blocks mode.  */
+  vector<block_t *> blocks;
+
   unsigned exists : 1;
   unsigned unexceptional : 1;
-} line_t;
+  unsigned has_unexecuted_block : 1;
+};
 
-bool
-line_t::has_block (block_t *needle)
+line_info::line_info (): count (0), branches (), blocks (), exists (false),
+  unexceptional (0), has_unexecuted_block (0)
 {
-  for (block_t *n = blocks; n; n = n->chain)
-    if (n == needle)
-      return true;
+}
 
-  return false;
+bool
+line_info::has_block (block_t *needle)
+{
+  return std::find (blocks.begin (), blocks.end (), needle) != blocks.end ();
 }
 
 /* Describes a file mentioned in the block graph.  Contains an array
    of line info.  */
 
-typedef struct source_info
+struct source_info
 {
+  /* Default constructor.  */
+  source_info ();
+
   /* Canonical name of source file.  */
   char *name;
   time_t file_time;
 
-  /* Array of line information.  */
-  line_t *lines;
-  unsigned num_lines;
+  /* Vector of line information.  */
+  vector<line_info> lines;
 
   coverage_t coverage;
 
   /* Functions in this source file.  These are in ascending line
      number order.  */
   function_t *functions;
-} source_t;
+};
+
+source_info::source_info (): name (NULL), file_time (), lines (),
+  coverage (), functions (NULL)
+{
+}
 
-typedef struct name_map
+class name_map
 {
-  char *name;  /* Source file name */
+public:
+  name_map ()
+  {
+  }
+
+  name_map (char *_name, unsigned _src): name (_name), src (_src)
+  {
+  }
+
+  bool operator== (const name_map &rhs) const
+  {
+#if HAVE_DOS_BASED_FILE_SYSTEM
+    return strcasecmp (this->name, rhs.name) == 0;
+#else
+    return strcmp (this->name, rhs.name) == 0;
+#endif
+  }
+
+  bool operator< (const name_map &rhs) const
+  {
+#if HAVE_DOS_BASED_FILE_SYSTEM
+    return strcasecmp (this->name, rhs.name) < 0;
+#else
+    return strcmp (this->name, rhs.name) < 0;
+#endif
+  }
+
+  const char *name;  /* Source file name */
   unsigned src;  /* Source file */
-} name_map_t;
+};
 
 /* Holds a list of function basic block graphs.  */
 
 static function_t *functions;
 static function_t **fn_end = &functions;
 
-static source_t *sources;   /* Array of source files  */
-static unsigned n_sources;  /* Number of sources */
-static unsigned a_sources;  /* Allocated sources */
+/* Vector of source files.  */
+static vector<source_info> sources;
 
-static name_map_t *names;   /* Mapping of file names to sources */
-static unsigned n_names;    /* Number of names */
-static unsigned a_names;    /* Allocated names */
+/* Mapping of file names to sources */
+static vector<name_map> names;
 
 /* This holds data summary information.  */
 
@@ -381,11 +422,19 @@ static int flag_hash_filenames = 0;
 
 static int flag_verbose = 0;
 
+/* Print colored output.  */
+
+static int flag_use_colors = 0;
+
 /* Output count information for every basic block, not merely those
    that contain line number information.  */
 
 static int flag_all_blocks = 0;
 
+/* Output human readable numbers.  */
+
+static int flag_human_readable_numbers = 0;
+
 /* Output summary info for each function.  */
 
 static int flag_function_summary = 0;
@@ -424,8 +473,6 @@ static void print_version (void) ATTRIBUTE_NORETURN;
 static void process_file (const char *);
 static void generate_results (const char *);
 static void create_file_names (const char *);
-static int name_search (const void *, const void *);
-static int name_sort (const void *, const void *);
 static char *canonicalize_name (const char *);
 static unsigned find_source (const char *);
 static function_t *read_graph_file (void);
@@ -437,10 +484,10 @@ static void add_line_counts (coverage_t *, function_t *);
 static void executed_summary (unsigned, unsigned);
 static void function_summary (const coverage_t *, const char *);
 static const char *format_gcov (gcov_type, gcov_type, int);
-static void accumulate_line_counts (source_t *);
-static void output_gcov_file (const char *, source_t *);
+static void accumulate_line_counts (source_info *);
+static void output_gcov_file (const char *, source_info *);
 static int output_branch_count (FILE *, int, const arc_t *);
-static void output_lines (FILE *, const source_t *);
+static void output_lines (FILE *, const source_info *);
 static char *make_gcov_file_name (const char *, const char *);
 static char *mangle_name (const char *, char *);
 static void release_structures (void);
@@ -556,7 +603,7 @@ unblock (const block_t *u, block_vector_t &blocked,
 static loop_type
 circuit (block_t *v, arc_vector_t &path, block_t *start,
 	 block_vector_t &blocked, vector<block_vector_t> &block_lists,
-	 line_t &linfo, int64_t &count)
+	 line_info &linfo, int64_t &count)
 {
   loop_type result = NO_LOOP;
 
@@ -605,7 +652,7 @@ circuit (block_t *v, arc_vector_t &path, block_t *start,
    contains a negative loop, then perform the same function once again.  */
 
 static gcov_type
-get_cycles_count (line_t &linfo, bool handle_negative_cycles = true)
+get_cycles_count (line_info &linfo, bool handle_negative_cycles = true)
 {
   /* Note that this algorithm works even if blocks aren't in sorted order.
      Each iteration of the circuit detection is completely independent
@@ -615,12 +662,13 @@ get_cycles_count (line_t &linfo, bool handle_negative_cycles = true)
 
   loop_type result = NO_LOOP;
   gcov_type count = 0;
-  for (block_t *block = linfo.blocks; block; block = block->chain)
+  for (vector<block_t *>::iterator it = linfo.blocks.begin ();
+       it != linfo.blocks.end (); it++)
     {
       arc_vector_t path;
       block_vector_t blocked;
       vector<block_vector_t > block_lists;
-      result |= circuit (block, path, block, blocked, block_lists, linfo,
+      result |= circuit (*it, path, *it, blocked, block_lists, linfo,
 			 count);
     }
 
@@ -655,11 +703,6 @@ main (int argc, char **argv)
   /* Handle response files.  */
   expandargv (&argc, &argv);
 
-  a_names = 10;
-  names = XNEWVEC (name_map_t, a_names);
-  a_sources = 10;
-  sources = XNEWVEC (source_t, a_sources);
-
   argno = process_args (argc, argv);
   if (optind == argc)
     print_usage (true);
@@ -703,6 +746,8 @@ print_usage (int error_p)
   fnotice (file, "  -f, --function-summaries        Output summaries for each function\n");
   fnotice (file, "  -h, --help                      Print this help, then exit\n");
   fnotice (file, "  -i, --intermediate-format       Output .gcov file in intermediate text format\n");
+  fnotice (file, "  -j, --human-readable            Output human readable numbers\n");
+  fnotice (file, "  -k, --use-colors                Emit colored output\n");
   fnotice (file, "  -l, --long-file-names           Use long output file names for included\n\
                                     source files\n");
   fnotice (file, "  -m, --demangled-names           Output demangled function names\n");
@@ -744,6 +789,7 @@ static const struct option options[] =
   { "branch-probabilities", no_argument,       NULL, 'b' },
   { "branch-counts",        no_argument,       NULL, 'c' },
   { "intermediate-format",  no_argument,       NULL, 'i' },
+  { "human-readable",	    no_argument,       NULL, 'j' },
   { "no-output",            no_argument,       NULL, 'n' },
   { "long-file-names",      no_argument,       NULL, 'l' },
   { "function-summaries",   no_argument,       NULL, 'f' },
@@ -756,6 +802,7 @@ static const struct option options[] =
   { "unconditional-branches", no_argument,     NULL, 'u' },
   { "display-progress",     no_argument,       NULL, 'd' },
   { "hash-filenames",	    no_argument,       NULL, 'x' },
+  { "use-colors",	    no_argument,       NULL, 'k' },
   { 0, 0, 0, 0 }
 };
 
@@ -766,7 +813,7 @@ process_args (int argc, char **argv)
 {
   int opt;
 
-  const char *opts = "abcdfhilmno:prs:uvwx";
+  const char *opts = "abcdfhijklmno:prs:uvwx";
   while ((opt = getopt_long (argc, argv, opts, options, NULL)) != -1)
     {
       switch (opt)
@@ -789,6 +836,12 @@ process_args (int argc, char **argv)
 	case 'l':
 	  flag_long_names = 1;
 	  break;
+	case 'j':
+	  flag_human_readable_numbers = 1;
+	  break;
+	case 'k':
+	  flag_use_colors = 1;
+	  break;
 	case 'm':
 	  flag_demangled_names = 1;
 	  break;
@@ -839,28 +892,7 @@ process_args (int argc, char **argv)
 /* Output the result in intermediate format used by 'lcov'.
 
 The intermediate format contains a single file named 'foo.cc.gcov',
-with no source code included. A sample output is
-
-file:foo.cc
-function:5,1,_Z3foov
-function:13,1,main
-function:19,1,_GLOBAL__sub_I__Z3foov
-function:19,1,_Z41__static_initialization_and_destruction_0ii
-lcount:5,1
-lcount:7,9
-lcount:9,8
-lcount:11,1
-file:/.../iostream
-lcount:74,1
-file:/.../basic_ios.h
-file:/.../ostream
-file:/.../ios_base.h
-function:157,0,_ZStorSt12_Ios_IostateS_
-lcount:157,0
-file:/.../char_traits.h
-function:258,0,_ZNSt11char_traitsIcE6lengthEPKc
-lcount:258,0
-...
+with no source code included.
 
 The default gcov outputs multiple files: 'foo.cc.gcov',
 'iostream.gcov', 'ios_base.h.gcov', etc. with source code
@@ -868,10 +900,10 @@ included. Instead the intermediate format here outputs only a single
 file 'foo.cc.gcov' similar to the above example. */
 
 static void
-output_intermediate_file (FILE *gcov_file, source_t *src)
+output_intermediate_file (FILE *gcov_file, source_info *src)
 {
   unsigned line_num;    /* current line number.  */
-  const line_t *line;   /* current line info ptr.  */
+  const line_info *line;   /* current line info ptr.  */
   function_t *fn;       /* current function info ptr. */
 
   fprintf (gcov_file, "file:%s\n", src->name);    /* source file name */
@@ -885,32 +917,32 @@ output_intermediate_file (FILE *gcov_file, source_t *src)
     }
 
   for (line_num = 1, line = &src->lines[line_num];
-       line_num < src->num_lines;
+       line_num < src->lines.size ();
        line_num++, line++)
     {
-      arc_t *arc;
       if (line->exists)
-	fprintf (gcov_file, "lcount:%u,%s\n", line_num,
-		 format_gcov (line->count, 0, -1));
+	fprintf (gcov_file, "lcount:%u,%s,%d\n", line_num,
+		 format_gcov (line->count, 0, -1), line->has_unexecuted_block);
       if (flag_branches)
-	for (arc = line->branches; arc; arc = arc->line_next)
-          {
-            if (!arc->is_unconditional && !arc->is_call_non_return)
-              {
-                const char *branch_type;
-                /* branch:<line_num>,<branch_coverage_type>
-                   branch_coverage_type
-                     : notexec (Branch not executed)
-                     : taken (Branch executed and taken)
-                     : nottaken (Branch executed, but not taken)
-                */
-                if (arc->src->count)
-                  branch_type = (arc->count > 0) ? "taken" : "nottaken";
-                else
-                  branch_type = "notexec";
-                fprintf (gcov_file, "branch:%d,%s\n", line_num, branch_type);
-              }
-          }
+	for (vector<arc_t *>::const_iterator it = line->branches.begin ();
+	     it != line->branches.end (); it++)
+	  {
+	    if (!(*it)->is_unconditional && !(*it)->is_call_non_return)
+	      {
+		const char *branch_type;
+		/* branch:<line_num>,<branch_coverage_type>
+		   branch_coverage_type
+		     : notexec (Branch not executed)
+		     : taken (Branch executed and taken)
+		     : nottaken (Branch executed, but not taken)
+		*/
+		if ((*it)->src->count)
+		  branch_type = ((*it)->count > 0) ? "taken" : "nottaken";
+		else
+		  branch_type = "notexec";
+		fprintf (gcov_file, "branch:%d,%s\n", line_num, branch_type);
+	      }
+	  }
     }
 }
 
@@ -939,7 +971,7 @@ process_file (const char *file_name)
 	  unsigned line = fn->line;
 	  unsigned block_no;
 	  function_t *probe, **prev;
-	  
+
 	  /* Now insert it into the source file's list of
 	     functions. Normally functions will be encountered in
 	     ascending order, so a simple scan is quick.  Note we're
@@ -967,8 +999,8 @@ process_file (const char *file_name)
 		    {
 		      unsigned last_line
 			= block->locations[i].lines.back () + 1;
-		      if (last_line > sources[s].num_lines)
-			sources[s].num_lines = last_line;
+		      if (last_line > sources[s].lines.size ())
+			sources[s].lines.resize (last_line);
 		    }
 		}
 	    }
@@ -987,7 +1019,7 @@ process_file (const char *file_name)
 }
 
 static void
-output_gcov_file (const char *file_name, source_t *src)
+output_gcov_file (const char *file_name, source_info *src)
 {
   char *gcov_file_name = make_gcov_file_name (file_name, src->coverage.name);
 
@@ -1020,14 +1052,8 @@ output_gcov_file (const char *file_name, source_t *src)
 static void
 generate_results (const char *file_name)
 {
-  unsigned ix;
-  source_t *src;
   function_t *fn;
 
-  for (ix = n_sources, src = sources; ix--; src++)
-    if (src->num_lines)
-      src->lines = XCNEWVEC (line_t, src->num_lines);
-
   for (fn = functions; fn; fn = fn->next)
     {
       coverage_t coverage;
@@ -1042,18 +1068,23 @@ generate_results (const char *file_name)
 	}
     }
 
+  name_map needle;
+
   if (file_name)
     {
-      name_map_t *name_map = (name_map_t *)bsearch
-	(file_name, names, n_names, sizeof (*names), name_search);
-      if (name_map)
-	file_name = sources[name_map->src].coverage.name;
+      needle.name = file_name;
+      vector<name_map>::iterator it = std::find (names.begin (), names.end (),
+						 needle);
+      if (it != names.end ())
+	file_name = sources[it->src].coverage.name;
       else
 	file_name = canonicalize_name (file_name);
     }
 
-  for (ix = n_sources, src = sources; ix--; src++)
+  for (vector<source_info>::iterator it = sources.begin ();
+       it != sources.end (); it++)
     {
+      source_info *src = &(*it);
       if (flag_relative_only)
 	{
 	  /* Ignore this source, if it is an absolute path (after
@@ -1088,17 +1119,8 @@ generate_results (const char *file_name)
 static void
 release_structures (void)
 {
-  unsigned ix;
   function_t *fn;
 
-  for (ix = n_sources; ix--;)
-    free (sources[ix].lines);
-  free (sources);
-
-  for (ix = n_names; ix--;)
-    free (names[ix].name);
-  free (names);
-
   while ((fn = functions))
     {
       functions = fn->next;
@@ -1174,90 +1196,45 @@ create_file_names (const char *file_name)
   return;
 }
 
-/* A is a string and B is a pointer to name_map_t.  Compare for file
-   name orderability.  */
-
-static int
-name_search (const void *a_, const void *b_)
-{
-  const char *a = (const char *)a_;
-  const name_map_t *b = (const name_map_t *)b_;
-
-#if HAVE_DOS_BASED_FILE_SYSTEM
-  return strcasecmp (a, b->name);
-#else
-  return strcmp (a, b->name);
-#endif
-}
-
-/* A and B are a pointer to name_map_t.  Compare for file name
-   orderability.  */
-
-static int
-name_sort (const void *a_, const void *b_)
-{
-  const name_map_t *a = (const name_map_t *)a_;
-  return name_search (a->name, b_);
-}
-
 /* Find or create a source file structure for FILE_NAME. Copies
    FILE_NAME on creation */
 
 static unsigned
 find_source (const char *file_name)
 {
-  name_map_t *name_map;
   char *canon;
   unsigned idx;
   struct stat status;
 
   if (!file_name)
     file_name = "<unknown>";
-  name_map = (name_map_t *)bsearch
-    (file_name, names, n_names, sizeof (*names), name_search);
-  if (name_map)
-    {
-      idx = name_map->src;
-      goto check_date;
-    }
 
-  if (n_names + 2 > a_names)
+  name_map needle;
+  needle.name = file_name;
+
+  vector<name_map>::iterator it = std::find (names.begin (), names.end (),
+					     needle);
+  if (it != names.end ())
     {
-      /* Extend the name map array -- we'll be inserting one or two
-	 entries.  */
-      a_names *= 2;
-      name_map = XNEWVEC (name_map_t, a_names);
-      memcpy (name_map, names, n_names * sizeof (*names));
-      free (names);
-      names = name_map;
+      idx = it->src;
+      goto check_date;
     }
 
   /* Not found, try the canonical name. */
   canon = canonicalize_name (file_name);
-  name_map = (name_map_t *) bsearch (canon, names, n_names, sizeof (*names),
-				     name_search);
-  if (!name_map)
+  needle.name = canon;
+  it = std::find (names.begin (), names.end (), needle);
+  if (it == names.end ())
     {
       /* Not found with canonical name, create a new source.  */
-      source_t *src;
-
-      if (n_sources == a_sources)
-	{
-	  a_sources *= 2;
-	  src = XNEWVEC (source_t, a_sources);
-	  memcpy (src, sources, n_sources * sizeof (*sources));
-	  free (sources);
-	  sources = src;
-	}
-
-      idx = n_sources;
+      source_info *src;
 
-      name_map = &names[n_names++];
-      name_map->name = canon;
-      name_map->src = idx;
+      idx = sources.size ();
+      needle = name_map (canon, idx);
+      names.push_back (needle);
 
-      src = &sources[n_sources++];
-      memset (src, 0, sizeof (*src));
+      sources.push_back (source_info ());
+      src = &sources.back ();
       src->name = canon;
       src->coverage.name = src->name;
       if (source_length
@@ -1274,18 +1251,17 @@ find_source (const char *file_name)
 	src->file_time = status.st_mtime;
     }
   else
-    idx = name_map->src;
+    idx = it->src;
 
-  if (name_search (file_name, name_map))
+  needle.name = file_name;
+  if (std::find (names.begin (), names.end (), needle) == names.end ())
     {
       /* Append the non-canonical name.  */
-      name_map = &names[n_names++];
-      name_map->name = xstrdup (file_name);
-      name_map->src = idx;
+      names.push_back (name_map (xstrdup (file_name), idx));
     }
 
   /* Resort the name map.  */
-  qsort (names, n_names, sizeof (*names), name_sort);
+  std::sort (names.begin (), names.end ());
 
  check_date:
   if (sources[idx].file_time > bbg_file_time)
@@ -1947,6 +1923,33 @@ add_branch_counts (coverage_t *coverage, const arc_t *arc)
     }
 }
 
+/* Format COUNT, if flag_human_readable_numbers is set, return it human
+   readable format.  */
+
+static char const *
+format_count (gcov_type count)
+{
+  static char buffer[64];
+  const char *units = " kMGTPEZY";
+
+  if (count < 1000 || !flag_human_readable_numbers)
+    {
+      sprintf (buffer, "%" PRId64, count);
+      return buffer;
+    }
+
+  unsigned i;
+  gcov_type divisor = 1;
+  for (i = 0; units[i+1]; i++, divisor *= 1000)
+    {
+      if (count + divisor / 2 < 1000 * divisor)
+	break;
+    }
+  gcov_type r  = (count + divisor / 2) / divisor;
+  sprintf (buffer, "%" PRId64 "%c", r, units[i]);
+  return buffer;
+}
+
 /* Format a GCOV_TYPE integer as either a percent ratio, or absolute
    count.  If dp >= 0, format TOP/BOTTOM * 100 to DP decimal places.
    If DP is zero, no decimal point is printed. Only print 100% when
@@ -1994,7 +1997,7 @@ format_gcov (gcov_type top, gcov_type bottom, int dp)
 	}
     }
   else
-    sprintf (buffer, "%" PRId64, (int64_t)top);
+    return format_count (top);
 
   return buffer;
 }
@@ -2257,13 +2260,13 @@ add_line_counts (coverage_t *coverage, function_t *fn)
   /* Scan each basic block.  */
   for (unsigned ix = 0; ix != fn->blocks.size (); ix++)
     {
-      line_t *line = NULL;
+      line_info *line = NULL;
       block_t *block = &fn->blocks[ix];
       if (block->count && ix && ix + 1 != fn->blocks.size ())
 	fn->blocks_executed++;
       for (unsigned i = 0; i < block->locations.size (); i++)
 	{
-	  const source_t *src = &sources[block->locations[i].source_file_idx];
+	  source_info *src = &sources[block->locations[i].source_file_idx];
 
 	  vector<unsigned> &lines = block->locations[i].lines;
 	  for (unsigned j = 0; j < lines.size (); j++)
@@ -2278,7 +2281,11 @@ add_line_counts (coverage_t *coverage, function_t *fn)
 		}
 	      line->exists = 1;
 	      if (!block->exceptional)
-		line->unexceptional = 1;
+		{
+		  line->unexceptional = 1;
+		  if (block->count == 0)
+		    line->has_unexecuted_block = 1;
+		}
 	      line->count += block->count;
 	    }
 	}
@@ -2290,8 +2297,7 @@ add_line_counts (coverage_t *coverage, function_t *fn)
 	/* Entry or exit block */;
       else if (line != NULL)
 	{
-	  block->chain = line->blocks;
-	  line->blocks = block;
+	  line->blocks.push_back (block);
 
 	  if (flag_branches)
 	    {
@@ -2299,8 +2305,7 @@ add_line_counts (coverage_t *coverage, function_t *fn)
 
 	      for (arc = block->succ; arc; arc = arc->succ_next)
 		{
-		  arc->line_next = line->branches;
-		  line->branches = arc;
+		  line->branches.push_back (arc);
 		  if (coverage && !arc->is_unconditional)
 		    add_branch_counts (coverage, arc);
 		}
@@ -2315,11 +2320,10 @@ add_line_counts (coverage_t *coverage, function_t *fn)
 /* Accumulate the line counts of a file.  */
 
 static void
-accumulate_line_counts (source_t *src)
+accumulate_line_counts (source_info *src)
 {
-  line_t *line;
   function_t *fn, *fn_p, *fn_n;
-  unsigned ix;
+  unsigned ix = 0;
 
   /* Reverse the function order.  */
   for (fn = src->functions, fn_p = NULL; fn; fn_p = fn, fn = fn_n)
@@ -2329,9 +2333,11 @@ accumulate_line_counts (source_t *src)
     }
   src->functions = fn_p;
 
-  for (ix = src->num_lines, line = src->lines; ix--; line++)
+  for (vector<line_info>::reverse_iterator it = src->lines.rbegin ();
+       it != src->lines.rend (); it++)
     {
-      if (line->blocks)
+      line_info *line = &(*it);
+      if (!line->blocks.empty ())
 	{
 	  /* The user expects the line count to be the number of times
 	     a line has been executed. Simply summing the block count
@@ -2339,36 +2345,27 @@ accumulate_line_counts (source_t *src)
 	     is to sum the entry counts to the graph of blocks on this
 	     line, then find the elementary cycles of the local graph
 	     and add the transition counts of those cycles.  */
-	  block_t *block, *block_p, *block_n;
 	  gcov_type count = 0;
 
-	  /* Reverse the block information.  */
-	  for (block = line->blocks, block_p = NULL; block;
-	       block_p = block, block = block_n)
-	    {
-	      block_n = block->chain;
-	      block->chain = block_p;
-	      block->cycle.ident = ix;
-	    }
-	  line->blocks = block_p;
-
 	  /* Sum the entry arcs.  */
-	  for (block = line->blocks; block; block = block->chain)
+	  for (vector<block_t *>::iterator it = line->blocks.begin ();
+	       it != line->blocks.end (); it++)
 	    {
 	      arc_t *arc;
 
-	      for (arc = block->pred; arc; arc = arc->pred_next)
+	      for (arc = (*it)->pred; arc; arc = arc->pred_next)
 		if (flag_branches)
 		  add_branch_counts (&src->coverage, arc);
 	    }
 
 	  /* Cycle detection.  */
-	  for (block = line->blocks; block; block = block->chain)
+	  for (vector<block_t *>::iterator it = line->blocks.begin ();
+	       it != line->blocks.end (); it++)
 	    {
-	      for (arc_t *arc = block->pred; arc; arc = arc->pred_next)
+	      for (arc_t *arc = (*it)->pred; arc; arc = arc->pred_next)
 		if (!line->has_block (arc->src))
 		  count += arc->count;
-	      for (arc_t *arc = block->succ; arc; arc = arc->succ_next)
+	      for (arc_t *arc = (*it)->succ; arc; arc = arc->succ_next)
 		arc->cs_count = arc->count;
 	    }
 
@@ -2383,6 +2380,8 @@ accumulate_line_counts (source_t *src)
 	  if (line->count)
 	    src->coverage.lines_executed++;
 	}
+
+      ix++;
     }
 }
 
@@ -2468,28 +2467,101 @@ read_line (FILE *file)
   return pos ? string : NULL;
 }
 
+/* Pad string S with spaces from left to have total width equal to 9.  */
+
+static void
+pad_count_string (string &s)
+{
+  if (s.size () < 9)
+    s.insert (0, 9 - s.size (), ' ');
+}
+
+/* Print GCOV line beginning to F stream.  If EXISTS is set to true, the
+   line exists in source file.  UNEXCEPTIONAL indicated that it's not in
+   an exceptional statement.  The output is printed for LINE_NUM of given
+   COUNT of executions.  EXCEPTIONAL_STRING and UNEXCEPTIONAL_STRING are
+   used to indicate non-executed blocks.  */
+
+static void
+output_line_beginning (FILE *f, bool exists, bool unexceptional,
+		       bool has_unexecuted_block,
+		       gcov_type count, unsigned line_num,
+		       const char *exceptional_string,
+		       const char *unexceptional_string)
+{
+  string s;
+  if (exists)
+    {
+      if (count > 0)
+	{
+	  s = format_gcov (count, 0, -1);
+	  if (has_unexecuted_block)
+	    {
+	      if (flag_use_colors)
+		{
+		  pad_count_string (s);
+		  s = SGR_SEQ (COLOR_BG_MAGENTA COLOR_SEPARATOR COLOR_FG_WHITE);
+		  s += SGR_RESET;
+		}
+	      else
+		s += "*";
+	    }
+	  pad_count_string (s);
+	}
+      else
+	{
+	  if (flag_use_colors)
+	    {
+	      s = "0";
+	      pad_count_string (s);
+	      if (unexceptional)
+		s.insert (0, SGR_SEQ (COLOR_BG_RED
+				      COLOR_SEPARATOR COLOR_FG_WHITE));
+	      else
+		s.insert (0, SGR_SEQ (COLOR_BG_CYAN
+				      COLOR_SEPARATOR COLOR_FG_WHITE));
+	      s += SGR_RESET;
+	    }
+	  else
+	    {
+	      s = unexceptional ? unexceptional_string : exceptional_string;
+	      pad_count_string (s);
+	    }
+	}
+    }
+  else
+    {
+      s = "-";
+      pad_count_string (s);
+    }
+
+  fprintf (f, "%s:%5u", s.c_str (), line_num);
+}
+
 /* Read in the source file one line at a time, and output that line to
    the gcov file preceded by its execution count and other
    information.  */
 
 static void
-output_lines (FILE *gcov_file, const source_t *src)
+output_lines (FILE *gcov_file, const source_info *src)
 {
+#define  DEFAULT_LINE_START "        -:    0:"
+
   FILE *source_file;
   unsigned line_num;	/* current line number.  */
-  const line_t *line;           /* current line info ptr.  */
+  const line_info *line;  /* current line info ptr.  */
   const char *retval = "";	/* status of source file reading.  */
   function_t *fn = NULL;
 
-  fprintf (gcov_file, "%9s:%5d:Source:%s\n", "-", 0, src->coverage.name);
+  fprintf (gcov_file, DEFAULT_LINE_START "Source:%s\n", src->coverage.name);
   if (!multiple_files)
     {
-      fprintf (gcov_file, "%9s:%5d:Graph:%s\n", "-", 0, bbg_file_name);
-      fprintf (gcov_file, "%9s:%5d:Data:%s\n", "-", 0,
+      fprintf (gcov_file, DEFAULT_LINE_START "Graph:%s\n", bbg_file_name);
+      fprintf (gcov_file, DEFAULT_LINE_START "Data:%s\n",
 	       no_data_file ? "-" : da_file_name);
-      fprintf (gcov_file, "%9s:%5d:Runs:%u\n", "-", 0, object_runs);
+      fprintf (gcov_file, DEFAULT_LINE_START "Runs:%u\n", object_runs);
     }
-  fprintf (gcov_file, "%9s:%5d:Programs:%u\n", "-", 0, program_count);
+  fprintf (gcov_file, DEFAULT_LINE_START "Programs:%u\n", program_count);
 
   source_file = fopen (src->name, "r");
   if (!source_file)
@@ -2498,13 +2570,13 @@ output_lines (FILE *gcov_file, const source_t *src)
       retval = NULL;
     }
   else if (src->file_time == 0)
-    fprintf (gcov_file, "%9s:%5d:Source is newer than graph\n", "-", 0);
+    fprintf (gcov_file, DEFAULT_LINE_START "Source is newer than graph\n");
 
   if (flag_branches)
     fn = src->functions;
 
   for (line_num = 1, line = &src->lines[line_num];
-       line_num < src->num_lines; line_num++, line++)
+       line_num < src->lines.size (); line_num++, line++)
     {
       for (; fn && fn->line == line_num; fn = fn->next_file_fn)
 	{
@@ -2537,44 +2609,44 @@ output_lines (FILE *gcov_file, const source_t *src)
 	 Otherwise, print the execution count before the source line.
 	 There are 16 spaces of indentation added before the source
 	 line so that tabs won't be messed up.  */
-      fprintf (gcov_file, "%9s:%5u:%s\n",
-	       !line->exists ? "-" : line->count
-	       ? format_gcov (line->count, 0, -1)
-	       : line->unexceptional ? "#####" : "=====", line_num,
-	       retval ? retval : "/*EOF*/");
+      output_line_beginning (gcov_file, line->exists, line->unexceptional,
+			     line->has_unexecuted_block, line->count, line_num,
+			     "=====", "#####");
+      fprintf (gcov_file, ":%s\n", retval ? retval : "/*EOF*/");
 
       if (flag_all_blocks)
 	{
-	  block_t *block;
 	  arc_t *arc;
 	  int ix, jx;
 
-	  for (ix = jx = 0, block = line->blocks; block;
-	       block = block->chain)
+	  ix = jx = 0;
+	  for (vector<block_t *>::const_iterator it = line->blocks.begin ();
+	       it != line->blocks.end (); it++)
 	    {
-	      if (!block->is_call_return)
+	      if (!(*it)->is_call_return)
 		{
-		  fprintf (gcov_file, "%9s:%5u-block %2d",
-			   !line->exists ? "-" : block->count
-			   ? format_gcov (block->count, 0, -1)
-			   : block->exceptional ? "%%%%%" : "$$$$$",
-			   line_num, ix++);
+		  output_line_beginning (gcov_file, line->exists,
+					 (*it)->exceptional, false,
+					 (*it)->count, line_num,
+					 "%%%%%", "$$$$$");
+		  fprintf (gcov_file, "-block %2d", ix++);
 		  if (flag_verbose)
-		    fprintf (gcov_file, " (BB %u)", block->id);
+		    fprintf (gcov_file, " (BB %u)", (*it)->id);
 		  fprintf (gcov_file, "\n");
 		}
 	      if (flag_branches)
-		for (arc = block->succ; arc; arc = arc->succ_next)
+		for (arc = (*it)->succ; arc; arc = arc->succ_next)
 		  jx += output_branch_count (gcov_file, jx, arc);
 	    }
 	}
       else if (flag_branches)
 	{
 	  int ix;
-	  arc_t *arc;
 
-	  for (ix = 0, arc = line->branches; arc; arc = arc->line_next)
-	    ix += output_branch_count (gcov_file, ix, arc);
+	  ix = 0;
+	  for (vector<arc_t *>::const_iterator it = line->branches.begin ();
+	       it != line->branches.end (); it++)
+	    ix += output_branch_count (gcov_file, ix, (*it));
 	}
     }
 
diff --git a/gcc/gdbinit.in b/gcc/gdbinit.in
index be56b0ee25b..3e1279e5b2a 100644
--- a/gcc/gdbinit.in
+++ b/gcc/gdbinit.in
@@ -252,6 +252,9 @@ skip file is-a.h
 # And line-map.h.
 skip file line-map.h
 
+# And timevar.h.
+skip file timevar.h
+
 # Likewise, skip various inline functions in rtl.h.
 skip rtx_expr_list::next
 skip rtx_expr_list::element
diff --git a/gcc/gencfn-macros.c b/gcc/gencfn-macros.c
index 269429fabfc..5b38ac20a4d 100644
--- a/gcc/gencfn-macros.c
+++ b/gcc/gencfn-macros.c
@@ -98,11 +98,12 @@ is_group (string_set *builtins, const char *name, const char *const *suffixes)
 
 static void
 print_case_cfn (const char *name, bool internal_p,
-		const char *const *suffixes)
+		const char *const *suffixes, bool floatn_p)
 {
-  printf ("#define CASE_CFN_%s", name);
+  const char *floatn = (floatn_p) ? "_FN" : "";
+  printf ("#define CASE_CFN_%s%s", name, floatn);
   if (internal_p)
-    printf (" \\\n  case CFN_%s", name);
+    printf (" \\\n  case CFN_%s%s", name, floatn);
   for (unsigned int i = 0; suffixes[i]; ++i)
     printf ("%s \\\n  case CFN_BUILT_IN_%s%s",
 	    internal_p || i > 0 ? ":" : "", name, suffixes[i]);
@@ -115,9 +116,10 @@ print_case_cfn (const char *name, bool internal_p,
 
 static void
 print_define_operator_list (const char *name, bool internal_p,
-			    const char *const *suffixes)
+			    const char *const *suffixes, bool floatn_p)
 {
-  printf ("(define_operator_list %s\n", name);
+  const char *floatn = (floatn_p) ? "_FN" : "";
+  printf ("(define_operator_list %s%s\n", name, floatn);
   for (unsigned int i = 0; suffixes[i]; ++i)
     printf ("    BUILT_IN_%s%s\n", name, suffixes[i]);
   if (internal_p)
@@ -148,6 +150,8 @@ const char *const internal_fn_int_names[] = {
 };
 
 static const char *const flt_suffixes[] = { "F", "", "L", NULL };
+static const char *const fltfn_suffixes[] = { "F16", "F32", "F64", "F128",
+					      "F32X", "F64X", "F128X", NULL };
 static const char *const int_suffixes[] = { "", "L", "LL", "IMAX", NULL };
 
 static const char *const *const suffix_lists[] = {
@@ -200,15 +204,33 @@ main (int argc, char **argv)
 	{
 	  const char *root = name + 9;
 	  for (unsigned int j = 0; suffix_lists[j]; ++j)
-	    if (is_group (&builtins, root, suffix_lists[j]))
-	      {
-		bool internal_p = internal_fns.contains (root);
-		if (type == 'c')
-		  print_case_cfn (root, internal_p, suffix_lists[j]);
-		else
-		  print_define_operator_list (root, internal_p,
-					      suffix_lists[j]);
-	      }
+	    {
+	      const char *const *const suffix = suffix_lists[j];
+
+	      if (is_group (&builtins, root, suffix))
+		{
+		  bool internal_p = internal_fns.contains (root);
+
+		  if (type == 'c')
+		    print_case_cfn (root, internal_p, suffix, false);
+		  else
+		    print_define_operator_list (root, internal_p,
+						suffix, false);
+
+		      /* Support the _Float<N> and _Float<N>X math functions if
+			 they exist.  We put these out as a separate CFN macro,
+			 so code can add support or not as needed.  */
+		  if (suffix == flt_suffixes
+		      && is_group (&builtins, root, fltfn_suffixes))
+		    {
+		      if (type == 'c')
+			print_case_cfn (root, false, fltfn_suffixes, true);
+		      else
+			print_define_operator_list (root, false, fltfn_suffixes,
+						    true);
+		    }
+		}
+	    }
 	}
     }
 
diff --git a/gcc/genmodes.c b/gcc/genmodes.c
index 9a5ed03b6ec..64074629ea1 100644
--- a/gcc/genmodes.c
+++ b/gcc/genmodes.c
@@ -45,10 +45,6 @@ static const char *const mode_class_names[MAX_MODE_CLASS] =
 # define EXTRA_MODES_FILE ""
 #endif
 
-static int bits_per_unit;
-static int max_bitsize_mode_any_int;
-static int max_bitsize_mode_any_mode;
-
 /* Data structure for building up what we know about a mode.
    They're clustered by mode class.  */
 struct mode_data
@@ -383,9 +379,7 @@ complete_mode (struct mode_data *m)
       break;
 
     case MODE_VECTOR_BOOL:
-      validate_mode (m, UNSET, UNSET, SET, SET, UNSET);
-      m->precision = m->ncomponents;
-      m->bytesize = (m->ncomponents + bits_per_unit - 1) / bits_per_unit;
+      validate_mode (m, UNSET, SET, SET, SET, UNSET);
       break;
 
     case MODE_VECTOR_INT:
@@ -539,14 +533,12 @@ make_vector_modes (enum mode_class cl, unsigned int width,
     }
 }
 
-/* Create a vector of booleans with the given number of elements.
-   Each element has BImode and by default the vector is packed,
-   with element 0 being the lsb of the first byte in memory.
-   The target can create an upacked representation by changing
-   the size of the vector.  */
-#define VECTOR_BOOL_MODE(BITS) make_vector_bool_mode (BITS, __FILE__, __LINE__)
+/* Create a vector of booleans with COUNT elements and BYTESIZE bytes
+   in total.  */
+#define VECTOR_BOOL_MODE(COUNT, BYTESIZE) \
+  make_vector_bool_mode (COUNT, BYTESIZE, __FILE__, __LINE__)
 static void ATTRIBUTE_UNUSED
-make_vector_bool_mode (unsigned int bits,
+make_vector_bool_mode (unsigned int count, unsigned int bytesize,
 		       const char *file, unsigned int line)
 {
   struct mode_data *m = find_mode ("BI");
@@ -557,7 +549,7 @@ make_vector_bool_mode (unsigned int bits,
     }
 
   char buf[8];
-  if ((size_t) snprintf (buf, sizeof buf, "V%uBI", bits) >= sizeof buf)
+  if ((size_t) snprintf (buf, sizeof buf, "V%uBI", count) >= sizeof buf)
     {
       error ("%s:%d: number of vector elements is too high",
 	     file, line);
@@ -567,7 +559,8 @@ make_vector_bool_mode (unsigned int bits,
   struct mode_data *v = new_mode (MODE_VECTOR_BOOL,
 				  xstrdup (buf), file, line);
   v->component = m;
-  v->ncomponents = bits;
+  v->ncomponents = count;
+  v->bytesize = bytesize;
 }
 
 /* Input.  */
@@ -803,6 +796,10 @@ make_vector_mode (enum mode_class bclass,
 #define ADJUST_IBIT(M, X)  _ADD_ADJUST (ibit, M, X, ACCUM, UACCUM)
 #define ADJUST_FBIT(M, X)  _ADD_ADJUST (fbit, M, X, FRACT, UACCUM)
 
+static int bits_per_unit;
+static int max_bitsize_mode_any_int;
+static int max_bitsize_mode_any_mode;
+
 static void
 create_modes (void)
 {
@@ -969,9 +966,9 @@ calc_wider_mode (void)
 #define print_decl(TYPE, NAME, ASIZE) \
   puts ("\nconst " TYPE " " NAME "[" ASIZE "] =\n{");
 
-#define print_maybe_const_decl(TYPE, NAME, ASIZE, CATEGORY)	\
+#define print_maybe_const_decl(TYPE, NAME, ASIZE, NEEDS_ADJ)	\
   printf ("\n" TYPE " " NAME "[" ASIZE "] = \n{\n",		\
-	  CATEGORY ? "" : "const ")
+	  NEEDS_ADJ ? "" : "const ")
 
 #define print_closer() puts ("};")
 
@@ -1034,6 +1031,9 @@ emit_mode_size_inline (void)
       for (m = a->mode->contained; m; m = m->next_cont)
 	m->need_bytesize_adj = true;
     }
+
+  /* Changing the number of units by a factor of X also changes the size
+     by a factor of X.  */
   for (mode_adjust *a = adj_nunits; a; a = a->next)
     a->mode->need_bytesize_adj = true;
 
@@ -1049,7 +1049,7 @@ mode_size_inline (machine_mode mode)\n\
   extern %spoly_uint16_pod mode_size[NUM_MACHINE_MODES];\n\
   gcc_assert (mode >= 0 && mode < NUM_MACHINE_MODES);\n\
   switch (mode)\n\
-    {\n", adj_bytesize ? "" : "const ");
+    {\n", adj_nunits || adj_bytesize ? "" : "const ");
 
   for_all_modes (c, m)
     if (!m->need_bytesize_adj)
@@ -1305,7 +1305,8 @@ enum machine_mode\n{");
   /* I can't think of a better idea, can you?  */
   printf ("#define CONST_MODE_NUNITS%s\n", adj_nunits ? "" : " const");
   printf ("#define CONST_MODE_PRECISION%s\n", adj_nunits ? "" : " const");
-  printf ("#define CONST_MODE_SIZE%s\n", adj_bytesize ? "" : " const");
+  printf ("#define CONST_MODE_SIZE%s\n",
+	  adj_bytesize || adj_nunits ? "" : " const");
   printf ("#define CONST_MODE_UNIT_SIZE%s\n", adj_bytesize ? "" : " const");
   printf ("#define CONST_MODE_BASE_ALIGN%s\n", adj_alignment ? "" : " const");
 #if 0 /* disabled for backward compatibility, temporary */
@@ -1718,15 +1719,18 @@ emit_mode_adjustments (void)
   for (a = adj_nunits; a; a = a->next)
     {
       m = a->mode;
-      printf ("\n  /* %s:%d */\n  ps = %s;\n",
+      printf ("\n"
+	      "  {\n"
+	      "    /* %s:%d */\n  ps = %s;\n",
 	      a->file, a->line, a->adjustment);
-      printf ("  mode_nunits[E_%smode] = ps;\n", m->name);
-      printf ("  mode_size[E_%smode] = mode_unit_size[E_%smode] * ps;\n",
+      printf ("    int old_factor = vector_element_size"
+	      " (mode_precision[E_%smode], mode_nunits[E_%smode]);\n",
 	      m->name, m->name);
-      if (m->precision == (unsigned int) -1)
-	printf ("  mode_precision[E_%smode]"
-		" = poly_uint16 (mode_size[E_%smode])"
-		" * BITS_PER_UNIT;", m->name, m->name);
+      printf ("    mode_precision[E_%smode] = ps * old_factor;\n",  m->name);
+      printf ("    mode_size[E_%smode] = exact_div (mode_precision[E_%smode],"
+	      " BITS_PER_UNIT);\n", m->name, m->name);
+      printf ("    mode_nunits[E_%smode] = ps;\n", m->name);
+      printf ("  }\n");
     }
 
   /* Size adjustments must be propagated to all containing modes.
diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c
index 0a460a693c7..ed474edb2fc 100644
--- a/gcc/gimple-fold.c
+++ b/gcc/gimple-fold.c
@@ -6369,7 +6369,7 @@ fold_ctor_reference (tree type, tree ctor, poly_uint64 poly_offset,
 
   /* We found the field with exact match.  */
   if (useless_type_conversion_p (type, TREE_TYPE (ctor))
-      && known_zero (poly_offset))
+      && must_eq (poly_offset, 0U))
     return canonicalize_constructor_val (unshare_expr (ctor), from_decl);
 
   /* The remaining optimizations need a constant size and offset.  */
@@ -6584,7 +6584,6 @@ gimple_get_virt_method_for_vtable (HOST_WIDE_INT token,
   gcc_assert (init);
   if (init == error_mark_node)
     {
-      gcc_assert (in_lto_p);
       /* Pass down that we lost track of the target.  */
       if (can_refer)
 	*can_refer = false;
diff --git a/gcc/gimple-laddress.c b/gcc/gimple-laddress.c
index 45fa7c3cfd4..85a2b5932e0 100644
--- a/gcc/gimple-laddress.c
+++ b/gcc/gimple-laddress.c
@@ -111,7 +111,7 @@ pass_laddress::execute (function *fun)
 	  poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
 	  if (offset != NULL_TREE)
 	    {
-	      if (maybe_nonzero (bytepos))
+	      if (may_ne (bytepos, 0))
 		offset = size_binop (PLUS_EXPR, offset, size_int (bytepos));
 	      offset = force_gimple_operand_gsi (&gsi, offset, true, NULL,
 						 true, GSI_SAME_STMT);
diff --git a/gcc/gimple-pretty-print.c b/gcc/gimple-pretty-print.c
index bc0558fa818..8ee84a7cc0d 100644
--- a/gcc/gimple-pretty-print.c
+++ b/gcc/gimple-pretty-print.c
@@ -82,21 +82,17 @@ debug_gimple_stmt (gimple *gs)
    by xstrdup_for_dump.  */
 
 static const char *
-dump_profile (int frequency, profile_count &count)
+dump_profile (profile_count &count)
 {
-  float minimum = 0.01f;
-
-  gcc_assert (0 <= frequency && frequency <= REG_BR_PROB_BASE);
-  float fvalue = frequency * 100.0f / REG_BR_PROB_BASE;
-  if (fvalue < minimum && frequency > 0)
-    return "[0.01%]";
-
   char *buf;
-  if (count.initialized_p ())
-    buf = xasprintf ("[%.2f%%] [count: %" PRId64 "]", fvalue,
+  if (!count.initialized_p ())
+    return NULL;
+  if (count.ipa_p ())
+    buf = xasprintf ("[count: %" PRId64 "]",
+		     count.to_gcov_type ());
+  else if (count.initialized_p ())
+    buf = xasprintf ("[local count: %" PRId64 "]",
 		     count.to_gcov_type ());
-  else
-    buf = xasprintf ("[%.2f%%] [count: INV]", fvalue);
 
   const char *ret = xstrdup_for_dump (buf);
   free (buf);
@@ -109,7 +105,7 @@ dump_profile (int frequency, profile_count &count)
    by xstrdup_for_dump.  */
 
 static const char *
-dump_probability (profile_probability probability, profile_count &count)
+dump_probability (profile_probability probability)
 {
   float minimum = 0.01f;
   float fvalue = -1;
@@ -122,13 +118,10 @@ dump_probability (profile_probability probability, profile_count &count)
     }
 
   char *buf;
-  if (count.initialized_p ())
-    buf = xasprintf ("[%.2f%%] [count: %" PRId64 "]", fvalue,
-		     count.to_gcov_type ());
-  else if (probability.initialized_p ())
-    buf = xasprintf ("[%.2f%%] [count: INV]", fvalue);
+  if (probability.initialized_p ())
+    buf = xasprintf ("[%.2f%%]", fvalue);
   else
-    buf = xasprintf ("[INV] [count: INV]");
+    buf = xasprintf ("[INV]");
 
   const char *ret = xstrdup_for_dump (buf);
   free (buf);
@@ -141,7 +134,7 @@ dump_probability (profile_probability probability, profile_count &count)
 static void
 dump_edge_probability (pretty_printer *buffer, edge e)
 {
-  pp_scalar (buffer, " %s", dump_probability (e->probability, e->count));
+  pp_scalar (buffer, " %s", dump_probability (e->probability));
 }
 
 /* Print GIMPLE statement G to FILE using SPC indentation spaces and
@@ -2678,8 +2671,7 @@ dump_gimple_bb_header (FILE *outf, basic_block bb, int indent,
 	fprintf (outf, "%*sbb_%d:\n", indent, "", bb->index);
       else
 	fprintf (outf, "%*s<bb %d> %s:\n",
-		 indent, "", bb->index, dump_profile (bb->frequency,
-						      bb->count));
+		 indent, "", bb->index, dump_profile (bb->count));
     }
 }
 
diff --git a/gcc/gimple-ssa-backprop.c b/gcc/gimple-ssa-backprop.c
index 93365ae9c62..16363003115 100644
--- a/gcc/gimple-ssa-backprop.c
+++ b/gcc/gimple-ssa-backprop.c
@@ -354,6 +354,7 @@ backprop::process_builtin_call_use (gcall *call, tree rhs, usage_info *info)
       break;
 
     CASE_CFN_COPYSIGN:
+    CASE_CFN_COPYSIGN_FN:
       /* The sign of the first input is ignored.  */
       if (rhs != gimple_call_arg (call, 1))
 	info->flags.ignore_sign = true;
@@ -373,6 +374,7 @@ backprop::process_builtin_call_use (gcall *call, tree rhs, usage_info *info)
       }
 
     CASE_CFN_FMA:
+    CASE_CFN_FMA_FN:
     case CFN_FMS:
     case CFN_FNMA:
     case CFN_FNMS:
@@ -683,6 +685,7 @@ strip_sign_op_1 (tree rhs)
     switch (gimple_call_combined_fn (call))
       {
       CASE_CFN_COPYSIGN:
+      CASE_CFN_COPYSIGN_FN:
 	return gimple_call_arg (call, 0);
 
       default:
diff --git a/gcc/gimple-ssa-isolate-paths.c b/gcc/gimple-ssa-isolate-paths.c
index 807e0032410..9a010a6b395 100644
--- a/gcc/gimple-ssa-isolate-paths.c
+++ b/gcc/gimple-ssa-isolate-paths.c
@@ -154,7 +154,6 @@ isolate_path (basic_block bb, basic_block duplicate,
   if (!duplicate)
     {
       duplicate = duplicate_block (bb, NULL, NULL);
-      bb->frequency = 0;
       bb->count = profile_count::zero ();
       if (!ret_zero)
 	for (ei = ei_start (duplicate->succs); (e2 = ei_safe_edge (ei)); )
@@ -168,8 +167,7 @@ isolate_path (basic_block bb, basic_block duplicate,
       flush_pending_stmts (e2);
 
       /* Update profile only when redirection is really processed.  */
-      bb->frequency += EDGE_FREQUENCY (e);
-      bb->count += e->count;
+      bb->count += e->count ();
     }
 
   /* There may be more than one statement in DUPLICATE which exhibits
diff --git a/gcc/gimple-ssa-sprintf.c b/gcc/gimple-ssa-sprintf.c
index 7899e09195f..35ceb2cfb75 100644
--- a/gcc/gimple-ssa-sprintf.c
+++ b/gcc/gimple-ssa-sprintf.c
@@ -79,6 +79,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "toplev.h"
 #include "substring-locations.h"
 #include "diagnostic.h"
+#include "domwalk.h"
 
 /* The likely worst case value of MB_LEN_MAX for the target, large enough
    for UTF-8.  Ideally, this would be obtained by a target hook if it were
@@ -113,6 +114,19 @@ static int warn_level;
 
 struct format_result;
 
+class sprintf_dom_walker : public dom_walker
+{
+ public:
+  sprintf_dom_walker () : dom_walker (CDI_DOMINATORS) {}
+  ~sprintf_dom_walker () {}
+
+  edge before_dom_children (basic_block) FINAL OVERRIDE;
+  bool handle_gimple_call (gimple_stmt_iterator *);
+
+  struct call_info;
+  bool compute_format_length (call_info &, format_result *);
+};
+
 class pass_sprintf_length : public gimple_opt_pass
 {
   bool fold_return_value;
@@ -135,10 +149,6 @@ public:
       fold_return_value = param;
     }
 
-  bool handle_gimple_call (gimple_stmt_iterator *);
-
-  struct call_info;
-  bool compute_format_length (call_info &, format_result *);
 };
 
 bool
@@ -583,7 +593,7 @@ get_format_string (tree format, location_t *ploc)
 /* For convenience and brevity.  */
 
 static bool
-  (* const fmtwarn) (const substring_loc &, const source_range *,
+  (* const fmtwarn) (const substring_loc &, location_t,
 		     const char *, int, const char *, ...)
   = format_warning_at_substring;
 
@@ -976,7 +986,7 @@ bytes_remaining (unsigned HOST_WIDE_INT navail, const format_result &res)
 
 /* Description of a call to a formatted function.  */
 
-struct pass_sprintf_length::call_info
+struct sprintf_dom_walker::call_info
 {
   /* Function call statement.  */
   gimple *callstmt;
@@ -2348,7 +2358,7 @@ format_plain (const directive &dir, tree)
    should be diagnosed given the AVAILable space in the destination.  */
 
 static bool
-should_warn_p (const pass_sprintf_length::call_info &info,
+should_warn_p (const sprintf_dom_walker::call_info &info,
 	       const result_range &avail, const result_range &result)
 {
   if (result.max <= avail.min)
@@ -2418,8 +2428,8 @@ should_warn_p (const pass_sprintf_length::call_info &info,
    Return true if a warning has been issued.  */
 
 static bool
-maybe_warn (substring_loc &dirloc, source_range *pargrange,
-	    const pass_sprintf_length::call_info &info,
+maybe_warn (substring_loc &dirloc, location_t argloc,
+	    const sprintf_dom_walker::call_info &info,
 	    const result_range &avail_range, const result_range &res,
 	    const directive &dir)
 {
@@ -2476,8 +2486,8 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		  : G_("%qE writing a terminating nul past the end "
 		       "of the destination")));
 
-	  return fmtwarn (dirloc, NULL, NULL, info.warnopt (), fmtstr,
-			  info.func);
+	  return fmtwarn (dirloc, UNKNOWN_LOCATION, NULL, info.warnopt (),
+			  fmtstr, info.func);
 	}
 
       if (res.min == res.max)
@@ -2500,7 +2510,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 			  "%wu bytes into a region of size %wu"))
 		  : G_("%<%.*s%> directive writing %wu bytes "
 		       "into a region of size %wu")));
-	  return fmtwarn (dirloc, pargrange, NULL,
+	  return fmtwarn (dirloc, argloc, NULL,
 			  info.warnopt (), fmtstr, dir.len,
 			  target_to_host (hostdir, sizeof hostdir, dir.beg),
 			  res.min, navail);
@@ -2517,7 +2527,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		       "up to %wu bytes into a region of size %wu"))
 	       : G_("%<%.*s%> directive writing up to %wu bytes "
 		    "into a region of size %wu"));
-	  return fmtwarn (dirloc, pargrange, NULL,
+	  return fmtwarn (dirloc, argloc, NULL,
 			  info.warnopt (), fmtstr, dir.len,
 			  target_to_host (hostdir, sizeof hostdir, dir.beg),
 			  res.max, navail);
@@ -2537,7 +2547,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		       "likely %wu or more bytes into a region of size %wu"))
 	       : G_("%<%.*s%> directive writing likely %wu or more bytes "
 		    "into a region of size %wu"));
-	  return fmtwarn (dirloc, pargrange, NULL,
+	  return fmtwarn (dirloc, argloc, NULL,
 			  info.warnopt (), fmtstr, dir.len,
 			  target_to_host (hostdir, sizeof hostdir, dir.beg),
 			  res.likely, navail);
@@ -2554,7 +2564,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		       "between %wu and %wu bytes into a region of size %wu"))
 	       : G_("%<%.*s%> directive writing between %wu and "
 		    "%wu bytes into a region of size %wu"));
-	  return fmtwarn (dirloc, pargrange, NULL,
+	  return fmtwarn (dirloc, argloc, NULL,
 			  info.warnopt (), fmtstr, dir.len,
 			  target_to_host (hostdir, sizeof hostdir, dir.beg),
 			  res.min, res.max, navail);
@@ -2569,7 +2579,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		   "%wu or more bytes into a region of size %wu"))
 	   : G_("%<%.*s%> directive writing %wu or more bytes "
 		"into a region of size %wu"));
-      return fmtwarn (dirloc, pargrange, NULL,
+      return fmtwarn (dirloc, argloc, NULL,
 		      info.warnopt (), fmtstr, dir.len,
 		      target_to_host (hostdir, sizeof hostdir, dir.beg),
 		      res.min, navail);
@@ -2603,7 +2613,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 	      : G_("%qE writing a terminating nul past the end "
 		   "of the destination")));
 
-      return fmtwarn (dirloc, NULL, NULL, info.warnopt (), fmtstr,
+      return fmtwarn (dirloc, UNKNOWN_LOCATION, NULL, info.warnopt (), fmtstr,
 		      info.func);
     }
 
@@ -2628,7 +2638,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 	      : G_("%<%.*s%> directive writing %wu bytes "
 		   "into a region of size between %wu and %wu")));
 
-      return fmtwarn (dirloc, pargrange, NULL,
+      return fmtwarn (dirloc, argloc, NULL,
 		      info.warnopt (), fmtstr, dir.len,
 		      target_to_host (hostdir, sizeof hostdir, dir.beg),
 		      res.min, avail_range.min, avail_range.max);
@@ -2647,7 +2657,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		   "%wu and %wu"))
 	   : G_("%<%.*s%> directive writing up to %wu bytes "
 		"into a region of size between %wu and %wu"));
-      return fmtwarn (dirloc, pargrange, NULL,
+      return fmtwarn (dirloc, argloc, NULL,
 		      info.warnopt (), fmtstr, dir.len,
 		      target_to_host (hostdir, sizeof hostdir, dir.beg),
 		      res.max, avail_range.min, avail_range.max);
@@ -2669,7 +2679,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		   "%wu and %wu"))
 	   : G_("%<%.*s%> directive writing likely %wu or more bytes "
 		"into a region of size between %wu and %wu"));
-      return fmtwarn (dirloc, pargrange, NULL,
+      return fmtwarn (dirloc, argloc, NULL,
 		      info.warnopt (), fmtstr, dir.len,
 		      target_to_host (hostdir, sizeof hostdir, dir.beg),
 		      res.likely, avail_range.min, avail_range.max);
@@ -2688,7 +2698,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 		   "between %wu and %wu"))
 	   : G_("%<%.*s%> directive writing between %wu and "
 		"%wu bytes into a region of size between %wu and %wu"));
-      return fmtwarn (dirloc, pargrange, NULL,
+      return fmtwarn (dirloc, argloc, NULL,
 		      info.warnopt (), fmtstr, dir.len,
 		      target_to_host (hostdir, sizeof hostdir, dir.beg),
 		      res.min, res.max, avail_range.min, avail_range.max);
@@ -2705,7 +2715,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
 	       "%wu and %wu"))
        : G_("%<%.*s%> directive writing %wu or more bytes "
 	    "into a region of size between %wu and %wu"));
-  return fmtwarn (dirloc, pargrange, NULL,
+  return fmtwarn (dirloc, argloc, NULL,
 		  info.warnopt (), fmtstr, dir.len,
 		  target_to_host (hostdir, sizeof hostdir, dir.beg),
 		  res.min, avail_range.min, avail_range.max);
@@ -2716,7 +2726,7 @@ maybe_warn (substring_loc &dirloc, source_range *pargrange,
    in *RES.  Return true if the directive has been handled.  */
 
 static bool
-format_directive (const pass_sprintf_length::call_info &info,
+format_directive (const sprintf_dom_walker::call_info &info,
 		  format_result *res, const directive &dir)
 {
   /* Offset of the beginning of the directive from the beginning
@@ -2730,17 +2740,11 @@ format_directive (const pass_sprintf_length::call_info &info,
   substring_loc dirloc (info.fmtloc, TREE_TYPE (info.format),
 			offset, start, length);
 
-  /* Also create a location range for the argument if possible.
+  /* Also get the location of the argument if possible.
      This doesn't work for integer literals or function calls.  */
-  source_range argrange;
-  source_range *pargrange;
-  if (dir.arg && CAN_HAVE_LOCATION_P (dir.arg))
-    {
-      argrange = EXPR_LOCATION_RANGE (dir.arg);
-      pargrange = &argrange;
-    }
-  else
-    pargrange = NULL;
+  location_t argloc = UNKNOWN_LOCATION;
+  if (dir.arg)
+    argloc = EXPR_LOCATION (dir.arg);
 
   /* Bail when there is no function to compute the output length,
      or when minimum length checking has been disabled.   */
@@ -2797,7 +2801,7 @@ format_directive (const pass_sprintf_length::call_info &info,
 
   if (fmtres.nullp)
     {
-      fmtwarn (dirloc, pargrange, NULL, info.warnopt (),
+      fmtwarn (dirloc, argloc, NULL, info.warnopt (),
 	       "%<%.*s%> directive argument is null",
 	       dirlen, target_to_host (hostdir, sizeof hostdir, dir.beg));
 
@@ -2816,7 +2820,7 @@ format_directive (const pass_sprintf_length::call_info &info,
   bool warned = res->warned;
 
   if (!warned)
-    warned = maybe_warn (dirloc, pargrange, info, avail_range,
+    warned = maybe_warn (dirloc, argloc, info, avail_range,
 			 fmtres.range, dir);
 
   /* Bump up the total maximum if it isn't too big.  */
@@ -2862,7 +2866,7 @@ format_directive (const pass_sprintf_length::call_info &info,
 	 (like Glibc does under some conditions).  */
 
       if (fmtres.range.min == fmtres.range.max)
-	warned = fmtwarn (dirloc, pargrange, NULL,
+	warned = fmtwarn (dirloc, argloc, NULL,
 			  info.warnopt (),
 			  "%<%.*s%> directive output of %wu bytes exceeds "
 			  "minimum required size of 4095",
@@ -2878,7 +2882,7 @@ format_directive (const pass_sprintf_length::call_info &info,
 	       : G_("%<%.*s%> directive output between %wu and %wu "
 		    "bytes exceeds minimum required size of 4095"));
 
-	  warned = fmtwarn (dirloc, pargrange, NULL,
+	  warned = fmtwarn (dirloc, argloc, NULL,
 			    info.warnopt (), fmtstr, dirlen,
 			    target_to_host (hostdir, sizeof hostdir, dir.beg),
 			    fmtres.range.min, fmtres.range.max);
@@ -2906,7 +2910,7 @@ format_directive (const pass_sprintf_length::call_info &info,
 	 to exceed INT_MAX bytes.  */
 
       if (fmtres.range.min == fmtres.range.max)
-	warned = fmtwarn (dirloc, pargrange, NULL, info.warnopt (),
+	warned = fmtwarn (dirloc, argloc, NULL, info.warnopt (),
 			  "%<%.*s%> directive output of %wu bytes causes "
 			  "result to exceed %<INT_MAX%>",
 			  dirlen,
@@ -2920,7 +2924,7 @@ format_directive (const pass_sprintf_length::call_info &info,
 		     "bytes causes result to exceed %<INT_MAX%>")
 	       : G_ ("%<%.*s%> directive output between %wu and %wu "
 		     "bytes may cause result to exceed %<INT_MAX%>"));
-	  warned = fmtwarn (dirloc, pargrange, NULL,
+	  warned = fmtwarn (dirloc, argloc, NULL,
 			    info.warnopt (), fmtstr, dirlen,
 			    target_to_host (hostdir, sizeof hostdir, dir.beg),
 			    fmtres.range.min, fmtres.range.max);
@@ -3010,7 +3014,7 @@ format_directive (const pass_sprintf_length::call_info &info,
    the directive.  */
 
 static size_t
-parse_directive (pass_sprintf_length::call_info &info,
+parse_directive (sprintf_dom_walker::call_info &info,
 		 directive &dir, format_result *res,
 		 const char *str, unsigned *argno)
 {
@@ -3351,7 +3355,7 @@ parse_directive (pass_sprintf_length::call_info &info,
 	  substring_loc dirloc (info.fmtloc, TREE_TYPE (info.format),
 				caret, begin, end);
 
-	  fmtwarn (dirloc, NULL, NULL,
+	  fmtwarn (dirloc, UNKNOWN_LOCATION, NULL,
 		   info.warnopt (), "%<%.*s%> directive width out of range",
 		   dir.len, target_to_host (hostdir, sizeof hostdir, dir.beg));
 	}
@@ -3385,7 +3389,7 @@ parse_directive (pass_sprintf_length::call_info &info,
 	  substring_loc dirloc (info.fmtloc, TREE_TYPE (info.format),
 				caret, begin, end);
 
-	  fmtwarn (dirloc, NULL, NULL,
+	  fmtwarn (dirloc, UNKNOWN_LOCATION, NULL,
 		   info.warnopt (), "%<%.*s%> directive precision out of range",
 		   dir.len, target_to_host (hostdir, sizeof hostdir, dir.beg));
 	}
@@ -3437,7 +3441,7 @@ parse_directive (pass_sprintf_length::call_info &info,
    that caused the processing to be terminated early).  */
 
 bool
-pass_sprintf_length::compute_format_length (call_info &info,
+sprintf_dom_walker::compute_format_length (call_info &info,
 					    format_result *res)
 {
   if (dump_file)
@@ -3520,7 +3524,7 @@ get_destination_size (tree dest)
    of its return values.  */
 
 static bool
-is_call_safe (const pass_sprintf_length::call_info &info,
+is_call_safe (const sprintf_dom_walker::call_info &info,
 	      const format_result &res, bool under4k,
 	      unsigned HOST_WIDE_INT retval[2])
 {
@@ -3579,7 +3583,7 @@ is_call_safe (const pass_sprintf_length::call_info &info,
 
 static bool
 try_substitute_return_value (gimple_stmt_iterator *gsi,
-			     const pass_sprintf_length::call_info &info,
+			     const sprintf_dom_walker::call_info &info,
 			     const format_result &res)
 {
   tree lhs = gimple_get_lhs (info.callstmt);
@@ -3696,7 +3700,7 @@ try_substitute_return_value (gimple_stmt_iterator *gsi,
 
 static bool
 try_simplify_call (gimple_stmt_iterator *gsi,
-		   const pass_sprintf_length::call_info &info,
+		   const sprintf_dom_walker::call_info &info,
 		   const format_result &res)
 {
   unsigned HOST_WIDE_INT dummy[2];
@@ -3723,7 +3727,7 @@ try_simplify_call (gimple_stmt_iterator *gsi,
    and gsi_next should not be performed in the caller.  */
 
 bool
-pass_sprintf_length::handle_gimple_call (gimple_stmt_iterator *gsi)
+sprintf_dom_walker::handle_gimple_call (gimple_stmt_iterator *gsi)
 {
   call_info info = call_info ();
 
@@ -3988,6 +3992,24 @@ pass_sprintf_length::handle_gimple_call (gimple_stmt_iterator *gsi)
   return call_removed;
 }
 
+edge
+sprintf_dom_walker::before_dom_children (basic_block bb)
+{
+  for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si); )
+    {
+      /* Iterate over statements, looking for function calls.  */
+      gimple *stmt = gsi_stmt (si);
+
+      if (is_gimple_call (stmt) && handle_gimple_call (&si))
+	/* If handle_gimple_call returns true, the iterator is
+	   already pointing to the next statement.  */
+	continue;
+
+      gsi_next (&si);
+    }
+  return NULL;
+}
+
 /* Execute the pass for function FUN.  */
 
 unsigned int
@@ -3995,26 +4017,13 @@ pass_sprintf_length::execute (function *fun)
 {
   init_target_to_host_charmap ();
 
-  basic_block bb;
-  FOR_EACH_BB_FN (bb, fun)
-    {
-      for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si); )
-	{
-	  /* Iterate over statements, looking for function calls.  */
-	  gimple *stmt = gsi_stmt (si);
+  calculate_dominance_info (CDI_DOMINATORS);
 
-	  if (is_gimple_call (stmt) && handle_gimple_call (&si))
-	    /* If handle_gimple_call returns true, the iterator is
-	       already pointing to the next statement.  */
-	    continue;
-
-	  gsi_next (&si);
-	}
-    }
+  sprintf_dom_walker sprintf_dom_walker;
+  sprintf_dom_walker.walk (ENTRY_BLOCK_PTR_FOR_FN (fun));
 
   /* Clean up object size info.  */
   fini_object_sizes ();
-
   return 0;
 }
 
diff --git a/gcc/gimple-ssa-store-merging.c b/gcc/gimple-ssa-store-merging.c
index 0a15ffdd98b..c5f01d774f5 100644
--- a/gcc/gimple-ssa-store-merging.c
+++ b/gcc/gimple-ssa-store-merging.c
@@ -19,7 +19,8 @@
    <http://www.gnu.org/licenses/>.  */
 
 /* The purpose of this pass is to combine multiple memory stores of
-   constant values to consecutive memory locations into fewer wider stores.
+   constant values, values loaded from memory or bitwise operations
+   on those to consecutive memory locations into fewer wider stores.
    For example, if we have a sequence peforming four byte stores to
    consecutive memory locations:
    [p     ] := imm1;
@@ -29,21 +30,49 @@
    we can transform this into a single 4-byte store if the target supports it:
   [p] := imm1:imm2:imm3:imm4 //concatenated immediates according to endianness.
 
+   Or:
+   [p     ] := [q     ];
+   [p + 1B] := [q + 1B];
+   [p + 2B] := [q + 2B];
+   [p + 3B] := [q + 3B];
+   if there is no overlap can be transformed into a single 4-byte
+   load followed by single 4-byte store.
+
+   Or:
+   [p     ] := [q     ] ^ imm1;
+   [p + 1B] := [q + 1B] ^ imm2;
+   [p + 2B] := [q + 2B] ^ imm3;
+   [p + 3B] := [q + 3B] ^ imm4;
+   if there is no overlap can be transformed into a single 4-byte
+   load, xored with imm1:imm2:imm3:imm4 and stored using a single 4-byte store.
+
    The algorithm is applied to each basic block in three phases:
 
-   1) Scan through the basic block recording constant assignments to
+   1) Scan through the basic block recording assignments to
    destinations that can be expressed as a store to memory of a certain size
-   at a certain bit offset.  Record store chains to different bases in a
-   hash_map (m_stores) and make sure to terminate such chains when appropriate
-   (for example when when the stored values get used subsequently).
+   at a certain bit offset from expressions we can handle.  For bit-fields
+   we also note the surrounding bit region, bits that could be stored in
+   a read-modify-write operation when storing the bit-field.  Record store
+   chains to different bases in a hash_map (m_stores) and make sure to
+   terminate such chains when appropriate (for example when when the stored
+   values get used subsequently).
    These stores can be a result of structure element initializers, array stores
    etc.  A store_immediate_info object is recorded for every such store.
    Record as many such assignments to a single base as possible until a
    statement that interferes with the store sequence is encountered.
+   Each store has up to 2 operands, which can be an immediate constant
+   or a memory load, from which the value to be stored can be computed.
+   At most one of the operands can be a constant.  The operands are recorded
+   in store_operand_info struct.
 
    2) Analyze the chain of stores recorded in phase 1) (i.e. the vector of
    store_immediate_info objects) and coalesce contiguous stores into
-   merged_store_group objects.
+   merged_store_group objects.  For bit-fields stores, we don't need to
+   require the stores to be contiguous, just their surrounding bit regions
+   have to be contiguous.  If the expression being stored is different
+   between adjacent stores, such as one store storing a constant and
+   following storing a value loaded from memory, or if the loaded memory
+   objects are not adjacent, a new merged_store_group is created as well.
 
    For example, given the stores:
    [p     ] := 0;
@@ -126,14 +155,43 @@
 #include "tree-eh.h"
 #include "target.h"
 #include "gimplify-me.h"
+#include "rtl.h"
+#include "expr.h"	/* For get_bit_range.  */
 #include "selftest.h"
 
 /* The maximum size (in bits) of the stores this pass should generate.  */
 #define MAX_STORE_BITSIZE (BITS_PER_WORD)
 #define MAX_STORE_BYTES (MAX_STORE_BITSIZE / BITS_PER_UNIT)
 
+/* Limit to bound the number of aliasing checks for loads with the same
+   vuse as the corresponding store.  */
+#define MAX_STORE_ALIAS_CHECKS 64
+
 namespace {
 
+/* Struct recording one operand for the store, which is either a constant,
+   then VAL represents the constant and all the other fields are zero,
+   or a memory load, then VAL represents the reference, BASE_ADDR is non-NULL
+   and the other fields also reflect the memory load.  */
+
+struct store_operand_info
+{
+  tree val;
+  tree base_addr;
+  unsigned HOST_WIDE_INT bitsize;
+  unsigned HOST_WIDE_INT bitpos;
+  unsigned HOST_WIDE_INT bitregion_start;
+  unsigned HOST_WIDE_INT bitregion_end;
+  gimple *stmt;
+  store_operand_info ();
+};
+
+store_operand_info::store_operand_info ()
+  : val (NULL_TREE), base_addr (NULL_TREE), bitsize (0), bitpos (0),
+    bitregion_start (0), bitregion_end (0), stmt (NULL)
+{
+}
+
 /* Struct recording the information about a single store of an immediate
    to memory.  These are created in the first phase and coalesced into
    merged_store_group objects in the second phase.  */
@@ -142,19 +200,45 @@ struct store_immediate_info
 {
   unsigned HOST_WIDE_INT bitsize;
   unsigned HOST_WIDE_INT bitpos;
+  unsigned HOST_WIDE_INT bitregion_start;
+  /* This is one past the last bit of the bit region.  */
+  unsigned HOST_WIDE_INT bitregion_end;
   gimple *stmt;
   unsigned int order;
+  /* INTEGER_CST for constant stores, MEM_REF for memory copy or
+     BIT_*_EXPR for logical bitwise operation.  */
+  enum tree_code rhs_code;
+  /* Operands.  For BIT_*_EXPR rhs_code both operands are used, otherwise
+     just the first one.  */
+  store_operand_info ops[2];
   store_immediate_info (unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT,
-			gimple *, unsigned int);
+			unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT,
+			gimple *, unsigned int, enum tree_code,
+			const store_operand_info &,
+			const store_operand_info &);
 };
 
 store_immediate_info::store_immediate_info (unsigned HOST_WIDE_INT bs,
 					    unsigned HOST_WIDE_INT bp,
+					    unsigned HOST_WIDE_INT brs,
+					    unsigned HOST_WIDE_INT bre,
 					    gimple *st,
-					    unsigned int ord)
-  : bitsize (bs), bitpos (bp), stmt (st), order (ord)
+					    unsigned int ord,
+					    enum tree_code rhscode,
+					    const store_operand_info &op0r,
+					    const store_operand_info &op1r)
+  : bitsize (bs), bitpos (bp), bitregion_start (brs), bitregion_end (bre),
+    stmt (st), order (ord), rhs_code (rhscode)
+#if __cplusplus >= 201103L
+    , ops { op0r, op1r }
+{
+}
+#else
 {
+  ops[0] = op0r;
+  ops[1] = op1r;
 }
+#endif
 
 /* Struct representing a group of stores to contiguous memory locations.
    These are produced by the second phase (coalescing) and consumed in the
@@ -164,26 +248,34 @@ struct merged_store_group
 {
   unsigned HOST_WIDE_INT start;
   unsigned HOST_WIDE_INT width;
-  /* The size of the allocated memory for val.  */
+  unsigned HOST_WIDE_INT bitregion_start;
+  unsigned HOST_WIDE_INT bitregion_end;
+  /* The size of the allocated memory for val and mask.  */
   unsigned HOST_WIDE_INT buf_size;
+  unsigned HOST_WIDE_INT align_base;
+  unsigned HOST_WIDE_INT load_align_base[2];
 
   unsigned int align;
+  unsigned int load_align[2];
   unsigned int first_order;
   unsigned int last_order;
 
-  auto_vec<struct store_immediate_info *> stores;
+  auto_vec<store_immediate_info *> stores;
   /* We record the first and last original statements in the sequence because
      we'll need their vuse/vdef and replacement position.  It's easier to keep
      track of them separately as 'stores' is reordered by apply_stores.  */
   gimple *last_stmt;
   gimple *first_stmt;
   unsigned char *val;
+  unsigned char *mask;
 
   merged_store_group (store_immediate_info *);
   ~merged_store_group ();
   void merge_into (store_immediate_info *);
   void merge_overlapping (store_immediate_info *);
   bool apply_stores ();
+private:
+  void do_merge (store_immediate_info *);
 };
 
 /* Debug helper.  Dump LEN elements of byte array PTR to FD in hex.  */
@@ -287,8 +379,7 @@ clear_bit_region_be (unsigned char *ptr, unsigned int start,
 	   && len > BITS_PER_UNIT)
     {
       unsigned int nbytes = len / BITS_PER_UNIT;
-      for (unsigned int i = 0; i < nbytes; i++)
-	ptr[i] = 0U;
+      memset (ptr, 0, nbytes);
       if (len % BITS_PER_UNIT != 0)
 	clear_bit_region_be (ptr + nbytes, BITS_PER_UNIT - 1,
 			     len % BITS_PER_UNIT);
@@ -552,10 +643,30 @@ merged_store_group::merged_store_group (store_immediate_info *info)
 {
   start = info->bitpos;
   width = info->bitsize;
+  bitregion_start = info->bitregion_start;
+  bitregion_end = info->bitregion_end;
   /* VAL has memory allocated for it in apply_stores once the group
      width has been finalized.  */
   val = NULL;
-  align = get_object_alignment (gimple_assign_lhs (info->stmt));
+  mask = NULL;
+  unsigned HOST_WIDE_INT align_bitpos = 0;
+  get_object_alignment_1 (gimple_assign_lhs (info->stmt),
+			  &align, &align_bitpos);
+  align_base = start - align_bitpos;
+  for (int i = 0; i < 2; ++i)
+    {
+      store_operand_info &op = info->ops[i];
+      if (op.base_addr == NULL_TREE)
+	{
+	  load_align[i] = 0;
+	  load_align_base[i] = 0;
+	}
+      else
+	{
+	  get_object_alignment_1 (op.val, &load_align[i], &align_bitpos);
+	  load_align_base[i] = op.bitpos - align_bitpos;
+	}
+    }
   stores.create (1);
   stores.safe_push (info);
   last_stmt = info->stmt;
@@ -571,18 +682,37 @@ merged_store_group::~merged_store_group ()
     XDELETEVEC (val);
 }
 
-/* Merge a store recorded by INFO into this merged store.
-   The store is not overlapping with the existing recorded
-   stores.  */
-
+/* Helper method for merge_into and merge_overlapping to do
+   the common part.  */
 void
-merged_store_group::merge_into (store_immediate_info *info)
+merged_store_group::do_merge (store_immediate_info *info)
 {
-  unsigned HOST_WIDE_INT wid = info->bitsize;
-  /* Make sure we're inserting in the position we think we're inserting.  */
-  gcc_assert (info->bitpos == start + width);
+  bitregion_start = MIN (bitregion_start, info->bitregion_start);
+  bitregion_end = MAX (bitregion_end, info->bitregion_end);
+
+  unsigned int this_align;
+  unsigned HOST_WIDE_INT align_bitpos = 0;
+  get_object_alignment_1 (gimple_assign_lhs (info->stmt),
+			  &this_align, &align_bitpos);
+  if (this_align > align)
+    {
+      align = this_align;
+      align_base = info->bitpos - align_bitpos;
+    }
+  for (int i = 0; i < 2; ++i)
+    {
+      store_operand_info &op = info->ops[i];
+      if (!op.base_addr)
+	continue;
+
+      get_object_alignment_1 (op.val, &this_align, &align_bitpos);
+      if (this_align > load_align[i])
+	{
+	  load_align[i] = this_align;
+	  load_align_base[i] = op.bitpos - align_bitpos;
+	}
+    }
 
-  width += wid;
   gimple *stmt = info->stmt;
   stores.safe_push (info);
   if (info->order > last_order)
@@ -597,6 +727,22 @@ merged_store_group::merge_into (store_immediate_info *info)
     }
 }
 
+/* Merge a store recorded by INFO into this merged store.
+   The store is not overlapping with the existing recorded
+   stores.  */
+
+void
+merged_store_group::merge_into (store_immediate_info *info)
+{
+  unsigned HOST_WIDE_INT wid = info->bitsize;
+  /* Make sure we're inserting in the position we think we're inserting.  */
+  gcc_assert (info->bitpos >= start + width
+	      && info->bitregion_start <= bitregion_end);
+
+  width += wid;
+  do_merge (info);
+}
+
 /* Merge a store described by INFO into this merged store.
    INFO overlaps in some way with the current store (i.e. it's not contiguous
    which is handled by merged_store_group::merge_into).  */
@@ -604,23 +750,11 @@ merged_store_group::merge_into (store_immediate_info *info)
 void
 merged_store_group::merge_overlapping (store_immediate_info *info)
 {
-  gimple *stmt = info->stmt;
-  stores.safe_push (info);
-
   /* If the store extends the size of the group, extend the width.  */
-  if ((info->bitpos + info->bitsize) > (start + width))
+  if (info->bitpos + info->bitsize > start + width)
     width += info->bitpos + info->bitsize - (start + width);
 
-  if (info->order > last_order)
-    {
-      last_order = info->order;
-      last_stmt = stmt;
-    }
-  else if (info->order < first_order)
-    {
-      first_order = info->order;
-      first_stmt = stmt;
-    }
+  do_merge (info);
 }
 
 /* Go through all the recorded stores in this group in program order and
@@ -630,37 +764,43 @@ merged_store_group::merge_overlapping (store_immediate_info *info)
 bool
 merged_store_group::apply_stores ()
 {
-  /* The total width of the stores must add up to a whole number of bytes
-     and start at a byte boundary.  We don't support emitting bitfield
-     references for now.  Also, make sure we have more than one store
-     in the group, otherwise we cannot merge anything.  */
-  if (width % BITS_PER_UNIT != 0
-      || start % BITS_PER_UNIT != 0
+  /* Make sure we have more than one store in the group, otherwise we cannot
+     merge anything.  */
+  if (bitregion_start % BITS_PER_UNIT != 0
+      || bitregion_end % BITS_PER_UNIT != 0
       || stores.length () == 1)
     return false;
 
   stores.qsort (sort_by_order);
-  struct store_immediate_info *info;
+  store_immediate_info *info;
   unsigned int i;
   /* Create a buffer of a size that is 2 times the number of bytes we're
      storing.  That way native_encode_expr can write power-of-2-sized
      chunks without overrunning.  */
-  buf_size = 2 * (ROUND_UP (width, BITS_PER_UNIT) / BITS_PER_UNIT);
-  val = XCNEWVEC (unsigned char, buf_size);
+  buf_size = 2 * ((bitregion_end - bitregion_start) / BITS_PER_UNIT);
+  val = XNEWVEC (unsigned char, 2 * buf_size);
+  mask = val + buf_size;
+  memset (val, 0, buf_size);
+  memset (mask, ~0U, buf_size);
 
   FOR_EACH_VEC_ELT (stores, i, info)
     {
-      unsigned int pos_in_buffer = info->bitpos - start;
-      bool ret = encode_tree_to_bitpos (gimple_assign_rhs1 (info->stmt),
-					val, info->bitsize,
-					pos_in_buffer, buf_size);
-      if (dump_file && (dump_flags & TDF_DETAILS))
+      unsigned int pos_in_buffer = info->bitpos - bitregion_start;
+      tree cst = NULL_TREE;
+      if (info->ops[0].val && info->ops[0].base_addr == NULL_TREE)
+	cst = info->ops[0].val;
+      else if (info->ops[1].val && info->ops[1].base_addr == NULL_TREE)
+	cst = info->ops[1].val;
+      bool ret = true;
+      if (cst)
+	ret = encode_tree_to_bitpos (cst, val, info->bitsize,
+				     pos_in_buffer, buf_size);
+      if (cst && dump_file && (dump_flags & TDF_DETAILS))
 	{
 	  if (ret)
 	    {
 	      fprintf (dump_file, "After writing ");
-	      print_generic_expr (dump_file,
-				  gimple_assign_rhs1 (info->stmt), 0);
+	      print_generic_expr (dump_file, cst, 0);
 	      fprintf (dump_file, " of size " HOST_WIDE_INT_PRINT_DEC
 			" at position %d the merged region contains:\n",
 			info->bitsize, pos_in_buffer);
@@ -671,6 +811,13 @@ merged_store_group::apply_stores ()
         }
       if (!ret)
 	return false;
+      unsigned char *m = mask + (pos_in_buffer / BITS_PER_UNIT);
+      if (BYTES_BIG_ENDIAN)
+	clear_bit_region_be (m, (BITS_PER_UNIT - 1
+				 - (pos_in_buffer % BITS_PER_UNIT)),
+			     info->bitsize);
+      else
+	clear_bit_region (m, pos_in_buffer % BITS_PER_UNIT, info->bitsize);
     }
   return true;
 }
@@ -685,7 +832,7 @@ struct imm_store_chain_info
      See pass_store_merging::m_stores_head for more rationale.  */
   imm_store_chain_info *next, **pnxp;
   tree base_addr;
-  auto_vec<struct store_immediate_info *> m_store_info;
+  auto_vec<store_immediate_info *> m_store_info;
   auto_vec<merged_store_group *> m_merged_store_groups;
 
   imm_store_chain_info (imm_store_chain_info *&inspt, tree b_a)
@@ -733,11 +880,16 @@ public:
   {
   }
 
-  /* Pass not supported for PDP-endianness.  */
+  /* Pass not supported for PDP-endianness, nor for insane hosts
+     or target character sizes where native_{encode,interpret}_expr
+     doesn't work properly.  */
   virtual bool
   gate (function *)
   {
-    return flag_store_merging && (WORDS_BIG_ENDIAN == BYTES_BIG_ENDIAN);
+    return flag_store_merging
+	   && WORDS_BIG_ENDIAN == BYTES_BIG_ENDIAN
+	   && CHAR_BIT == 8
+	   && BITS_PER_UNIT == 8;
   }
 
   virtual unsigned int execute (function *);
@@ -756,9 +908,10 @@ private:
      decisions when going out of SSA).  */
   imm_store_chain_info *m_stores_head;
 
+  void process_store (gimple *);
   bool terminate_and_process_all_chains ();
   bool terminate_all_aliasing_chains (imm_store_chain_info **,
-				      bool, gimple *);
+				      gimple *);
   bool terminate_and_release_chain (imm_store_chain_info *);
 }; // class pass_store_merging
 
@@ -788,7 +941,6 @@ pass_store_merging::terminate_and_process_all_chains ()
 bool
 pass_store_merging::terminate_all_aliasing_chains (imm_store_chain_info
 						     **chain_info,
-						   bool var_offset_p,
 						   gimple *stmt)
 {
   bool ret = false;
@@ -802,37 +954,21 @@ pass_store_merging::terminate_all_aliasing_chains (imm_store_chain_info
      of a chain.  */
   if (chain_info)
     {
-      /* We have a chain at BASE and we're writing to [BASE + <variable>].
-	 This can interfere with any of the stores so terminate
-	 the chain.  */
-      if (var_offset_p)
-	{
-	  terminate_and_release_chain (*chain_info);
-	  ret = true;
-	}
-      /* Otherwise go through every store in the chain to see if it
-	 aliases with any of them.  */
-      else
+      store_immediate_info *info;
+      unsigned int i;
+      FOR_EACH_VEC_ELT ((*chain_info)->m_store_info, i, info)
 	{
-	  struct store_immediate_info *info;
-	  unsigned int i;
-	  FOR_EACH_VEC_ELT ((*chain_info)->m_store_info, i, info)
+	  if (ref_maybe_used_by_stmt_p (stmt, gimple_assign_lhs (info->stmt))
+	      || stmt_may_clobber_ref_p (stmt, gimple_assign_lhs (info->stmt)))
 	    {
-	      if (ref_maybe_used_by_stmt_p (stmt,
-					    gimple_assign_lhs (info->stmt))
-		  || stmt_may_clobber_ref_p (stmt,
-					     gimple_assign_lhs (info->stmt)))
+	      if (dump_file && (dump_flags & TDF_DETAILS))
 		{
-		  if (dump_file && (dump_flags & TDF_DETAILS))
-		    {
-		      fprintf (dump_file,
-			       "stmt causes chain termination:\n");
-		      print_gimple_stmt (dump_file, stmt, 0);
-		    }
-		  terminate_and_release_chain (*chain_info);
-		  ret = true;
-		  break;
+		  fprintf (dump_file, "stmt causes chain termination:\n");
+		  print_gimple_stmt (dump_file, stmt, 0);
 		}
+	      terminate_and_release_chain (*chain_info);
+	      ret = true;
+	      break;
 	    }
 	}
     }
@@ -877,6 +1013,125 @@ pass_store_merging::terminate_and_release_chain (imm_store_chain_info *chain_inf
   return ret;
 }
 
+/* Return true if stmts in between FIRST (inclusive) and LAST (exclusive)
+   may clobber REF.  FIRST and LAST must be in the same basic block and
+   have non-NULL vdef.  */
+
+bool
+stmts_may_clobber_ref_p (gimple *first, gimple *last, tree ref)
+{
+  ao_ref r;
+  ao_ref_init (&r, ref);
+  unsigned int count = 0;
+  tree vop = gimple_vdef (last);
+  gimple *stmt;
+
+  gcc_checking_assert (gimple_bb (first) == gimple_bb (last));
+  do
+    {
+      stmt = SSA_NAME_DEF_STMT (vop);
+      if (stmt_may_clobber_ref_p_1 (stmt, &r))
+	return true;
+      /* Avoid quadratic compile time by bounding the number of checks
+	 we perform.  */
+      if (++count > MAX_STORE_ALIAS_CHECKS)
+	return true;
+      vop = gimple_vuse (stmt);
+    }
+  while (stmt != first);
+  return false;
+}
+
+/* Return true if INFO->ops[IDX] is mergeable with the
+   corresponding loads already in MERGED_STORE group.
+   BASE_ADDR is the base address of the whole store group.  */
+
+bool
+compatible_load_p (merged_store_group *merged_store,
+		   store_immediate_info *info,
+		   tree base_addr, int idx)
+{
+  store_immediate_info *infof = merged_store->stores[0];
+  if (!info->ops[idx].base_addr
+      || (info->ops[idx].bitpos - infof->ops[idx].bitpos
+	  != info->bitpos - infof->bitpos)
+      || !operand_equal_p (info->ops[idx].base_addr,
+			   infof->ops[idx].base_addr, 0))
+    return false;
+
+  store_immediate_info *infol = merged_store->stores.last ();
+  tree load_vuse = gimple_vuse (info->ops[idx].stmt);
+  /* In this case all vuses should be the same, e.g.
+     _1 = s.a; _2 = s.b; _3 = _1 | 1; t.a = _3; _4 = _2 | 2; t.b = _4;
+     or
+     _1 = s.a; _2 = s.b; t.a = _1; t.b = _2;
+     and we can emit the coalesced load next to any of those loads.  */
+  if (gimple_vuse (infof->ops[idx].stmt) == load_vuse
+      && gimple_vuse (infol->ops[idx].stmt) == load_vuse)
+    return true;
+
+  /* Otherwise, at least for now require that the load has the same
+     vuse as the store.  See following examples.  */
+  if (gimple_vuse (info->stmt) != load_vuse)
+    return false;
+
+  if (gimple_vuse (infof->stmt) != gimple_vuse (infof->ops[idx].stmt)
+      || (infof != infol
+	  && gimple_vuse (infol->stmt) != gimple_vuse (infol->ops[idx].stmt)))
+    return false;
+
+  /* If the load is from the same location as the store, already
+     the construction of the immediate chain info guarantees no intervening
+     stores, so no further checks are needed.  Example:
+     _1 = s.a; _2 = _1 & -7; s.a = _2; _3 = s.b; _4 = _3 & -7; s.b = _4;  */
+  if (info->ops[idx].bitpos == info->bitpos
+      && operand_equal_p (info->ops[idx].base_addr, base_addr, 0))
+    return true;
+
+  /* Otherwise, we need to punt if any of the loads can be clobbered by any
+     of the stores in the group, or any other stores in between those.
+     Previous calls to compatible_load_p ensured that for all the
+     merged_store->stores IDX loads, no stmts starting with
+     merged_store->first_stmt and ending right before merged_store->last_stmt
+     clobbers those loads.  */
+  gimple *first = merged_store->first_stmt;
+  gimple *last = merged_store->last_stmt;
+  unsigned int i;
+  store_immediate_info *infoc;
+  /* The stores are sorted by increasing store bitpos, so if info->stmt store
+     comes before the so far first load, we'll be changing
+     merged_store->first_stmt.  In that case we need to give up if
+     any of the earlier processed loads clobber with the stmts in the new
+     range.  */
+  if (info->order < merged_store->first_order)
+    {
+      FOR_EACH_VEC_ELT (merged_store->stores, i, infoc)
+	if (stmts_may_clobber_ref_p (info->stmt, first, infoc->ops[idx].val))
+	  return false;
+      first = info->stmt;
+    }
+  /* Similarly, we could change merged_store->last_stmt, so ensure
+     in that case no stmts in the new range clobber any of the earlier
+     processed loads.  */
+  else if (info->order > merged_store->last_order)
+    {
+      FOR_EACH_VEC_ELT (merged_store->stores, i, infoc)
+	if (stmts_may_clobber_ref_p (last, info->stmt, infoc->ops[idx].val))
+	  return false;
+      last = info->stmt;
+    }
+  /* And finally, we'd be adding a new load to the set, ensure it isn't
+     clobbered in the new range.  */
+  if (stmts_may_clobber_ref_p (first, last, info->ops[idx].val))
+    return false;
+
+  /* Otherwise, we are looking for:
+     _1 = s.a; _2 = _1 ^ 15; t.a = _2; _3 = s.b; _4 = _3 ^ 15; t.b = _4;
+     or
+     _1 = s.a; t.a = _1; _2 = s.b; t.b = _2;  */
+  return true;
+}
+
 /* Go through the candidate stores recorded in m_store_info and merge them
    into merged_store_group objects recorded into m_merged_store_groups
    representing the widened stores.  Return true if coalescing was successful
@@ -924,38 +1179,63 @@ imm_store_chain_info::coalesce_immediate_stores ()
       if (IN_RANGE (start, merged_store->start,
 		    merged_store->start + merged_store->width - 1))
 	{
-	  merged_store->merge_overlapping (info);
-	  continue;
+	  /* Only allow overlapping stores of constants.  */
+	  if (info->rhs_code == INTEGER_CST
+	      && merged_store->stores[0]->rhs_code == INTEGER_CST)
+	    {
+	      merged_store->merge_overlapping (info);
+	      continue;
+	    }
 	}
-
-      /* |---store 1---| <gap> |---store 2---|.
-	 Gap between stores.  Start a new group.  */
-      if (start != merged_store->start + merged_store->width)
+      /* |---store 1---||---store 2---|
+	 This store is consecutive to the previous one.
+	 Merge it into the current store group.  There can be gaps in between
+	 the stores, but there can't be gaps in between bitregions.  */
+      else if (info->bitregion_start <= merged_store->bitregion_end
+	       && info->rhs_code == merged_store->stores[0]->rhs_code)
 	{
-	  /* Try to apply all the stores recorded for the group to determine
-	     the bitpattern they write and discard it if that fails.
-	     This will also reject single-store groups.  */
-	  if (!merged_store->apply_stores ())
-	    delete merged_store;
-	  else
-	    m_merged_store_groups.safe_push (merged_store);
+	  store_immediate_info *infof = merged_store->stores[0];
+
+	  /* All the rhs_code ops that take 2 operands are commutative,
+	     swap the operands if it could make the operands compatible.  */
+	  if (infof->ops[0].base_addr
+	      && infof->ops[1].base_addr
+	      && info->ops[0].base_addr
+	      && info->ops[1].base_addr
+	      && (info->ops[1].bitpos - infof->ops[0].bitpos
+		  == info->bitpos - infof->bitpos)
+	      && operand_equal_p (info->ops[1].base_addr,
+				  infof->ops[0].base_addr, 0))
+	    std::swap (info->ops[0], info->ops[1]);
+	  if ((!infof->ops[0].base_addr
+	       || compatible_load_p (merged_store, info, base_addr, 0))
+	      && (!infof->ops[1].base_addr
+		  || compatible_load_p (merged_store, info, base_addr, 1)))
+	    {
+	      merged_store->merge_into (info);
+	      continue;
+	    }
+	}
 
-	  merged_store = new merged_store_group (info);
+      /* |---store 1---| <gap> |---store 2---|.
+	 Gap between stores or the rhs not compatible.  Start a new group.  */
 
-	  continue;
-	}
+      /* Try to apply all the stores recorded for the group to determine
+	 the bitpattern they write and discard it if that fails.
+	 This will also reject single-store groups.  */
+      if (!merged_store->apply_stores ())
+	delete merged_store;
+      else
+	m_merged_store_groups.safe_push (merged_store);
 
-      /* |---store 1---||---store 2---|
-	 This store is consecutive to the previous one.
-	 Merge it into the current store group.  */
-       merged_store->merge_into (info);
+      merged_store = new merged_store_group (info);
     }
 
-    /* Record or discard the last store group.  */
-    if (!merged_store->apply_stores ())
-      delete merged_store;
-    else
-      m_merged_store_groups.safe_push (merged_store);
+  /* Record or discard the last store group.  */
+  if (!merged_store->apply_stores ())
+    delete merged_store;
+  else
+    m_merged_store_groups.safe_push (merged_store);
 
   gcc_assert (m_merged_store_groups.length () <= m_store_info.length ());
   bool success
@@ -964,41 +1244,63 @@ imm_store_chain_info::coalesce_immediate_stores ()
 
   if (success && dump_file)
     fprintf (dump_file, "Coalescing successful!\n"
-			 "Merged into %u stores\n",
-		m_merged_store_groups.length ());
+			"Merged into %u stores\n",
+	     m_merged_store_groups.length ());
 
   return success;
 }
 
-/* Return the type to use for the merged stores described by STMTS.
-   This is needed to get the alias sets right.  */
+/* Return the type to use for the merged stores or loads described by STMTS.
+   This is needed to get the alias sets right.  If IS_LOAD, look for rhs,
+   otherwise lhs.  Additionally set *CLIQUEP and *BASEP to MR_DEPENDENCE_*
+   of the MEM_REFs if any.  */
 
 static tree
-get_alias_type_for_stmts (auto_vec<gimple *> &stmts)
+get_alias_type_for_stmts (vec<gimple *> &stmts, bool is_load,
+			  unsigned short *cliquep, unsigned short *basep)
 {
   gimple *stmt;
   unsigned int i;
-  tree lhs = gimple_assign_lhs (stmts[0]);
-  tree type = reference_alias_ptr_type (lhs);
+  tree type = NULL_TREE;
+  tree ret = NULL_TREE;
+  *cliquep = 0;
+  *basep = 0;
 
   FOR_EACH_VEC_ELT (stmts, i, stmt)
     {
-      if (i == 0)
-	continue;
+      tree ref = is_load ? gimple_assign_rhs1 (stmt)
+			 : gimple_assign_lhs (stmt);
+      tree type1 = reference_alias_ptr_type (ref);
+      tree base = get_base_address (ref);
 
-      lhs = gimple_assign_lhs (stmt);
-      tree type1 = reference_alias_ptr_type (lhs);
+      if (i == 0)
+	{
+	  if (TREE_CODE (base) == MEM_REF)
+	    {
+	      *cliquep = MR_DEPENDENCE_CLIQUE (base);
+	      *basep = MR_DEPENDENCE_BASE (base);
+	    }
+	  ret = type = type1;
+	  continue;
+	}
       if (!alias_ptr_types_compatible_p (type, type1))
-	return ptr_type_node;
+	ret = ptr_type_node;
+      if (TREE_CODE (base) != MEM_REF
+	  || *cliquep != MR_DEPENDENCE_CLIQUE (base)
+	  || *basep != MR_DEPENDENCE_BASE (base))
+	{
+	  *cliquep = 0;
+	  *basep = 0;
+	}
     }
-  return type;
+  return ret;
 }
 
 /* Return the location_t information we can find among the statements
    in STMTS.  */
 
 static location_t
-get_location_for_stmts (auto_vec<gimple *> &stmts)
+get_location_for_stmts (vec<gimple *> &stmts)
 {
   gimple *stmt;
   unsigned int i;
@@ -1018,7 +1320,9 @@ struct split_store
   unsigned HOST_WIDE_INT bytepos;
   unsigned HOST_WIDE_INT size;
   unsigned HOST_WIDE_INT align;
-  auto_vec<gimple *> orig_stmts;
+  auto_vec<store_immediate_info *> orig_stores;
+  /* True if there is a single orig stmt covering the whole split store.  */
+  bool orig;
   split_store (unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT,
 	       unsigned HOST_WIDE_INT);
 };
@@ -1028,100 +1332,230 @@ struct split_store
 split_store::split_store (unsigned HOST_WIDE_INT bp,
 			  unsigned HOST_WIDE_INT sz,
 			  unsigned HOST_WIDE_INT al)
-			  : bytepos (bp), size (sz), align (al)
+			  : bytepos (bp), size (sz), align (al), orig (false)
 {
-  orig_stmts.create (0);
+  orig_stores.create (0);
 }
 
-/* Record all statements corresponding to stores in GROUP that write to
-   the region starting at BITPOS and is of size BITSIZE.  Record such
-   statements in STMTS.  The stores in GROUP must be sorted by
-   bitposition.  */
+/* Record all stores in GROUP that write to the region starting at BITPOS and
+   is of size BITSIZE.  Record infos for such statements in STORES if
+   non-NULL.  The stores in GROUP must be sorted by bitposition.  Return INFO
+   if there is exactly one original store in the range.  */
 
-static void
-find_constituent_stmts (struct merged_store_group *group,
-			 auto_vec<gimple *> &stmts,
+static store_immediate_info *
+find_constituent_stores (struct merged_store_group *group,
+			 vec<store_immediate_info *> *stores,
+			 unsigned int *first,
 			 unsigned HOST_WIDE_INT bitpos,
 			 unsigned HOST_WIDE_INT bitsize)
 {
-  struct store_immediate_info *info;
+  store_immediate_info *info, *ret = NULL;
   unsigned int i;
+  bool second = false;
+  bool update_first = true;
   unsigned HOST_WIDE_INT end = bitpos + bitsize;
-  FOR_EACH_VEC_ELT (group->stores, i, info)
+  for (i = *first; group->stores.iterate (i, &info); ++i)
     {
       unsigned HOST_WIDE_INT stmt_start = info->bitpos;
       unsigned HOST_WIDE_INT stmt_end = stmt_start + info->bitsize;
-      if (stmt_end < bitpos)
-	continue;
+      if (stmt_end <= bitpos)
+	{
+	  /* BITPOS passed to this function never decreases from within the
+	     same split_group call, so optimize and don't scan info records
+	     which are known to end before or at BITPOS next time.
+	     Only do it if all stores before this one also pass this.  */
+	  if (update_first)
+	    *first = i + 1;
+	  continue;
+	}
+      else
+	update_first = false;
+
       /* The stores in GROUP are ordered by bitposition so if we're past
-	  the region for this group return early.  */
-      if (stmt_start > end)
-	return;
-
-      if (IN_RANGE (stmt_start, bitpos, bitpos + bitsize)
-	  || IN_RANGE (stmt_end, bitpos, end)
-	  /* The statement writes a region that completely encloses the region
-	     that this group writes.  Unlikely to occur but let's
-	     handle it.  */
-	  || IN_RANGE (bitpos, stmt_start, stmt_end))
-	stmts.safe_push (info->stmt);
+	 the region for this group return early.  */
+      if (stmt_start >= end)
+	return ret;
+
+      if (stores)
+	{
+	  stores->safe_push (info);
+	  if (ret)
+	    {
+	      ret = NULL;
+	      second = true;
+	    }
+	}
+      else if (ret)
+	return NULL;
+      if (!second)
+	ret = info;
     }
+  return ret;
 }
 
 /* Split a merged store described by GROUP by populating the SPLIT_STORES
-   vector with split_store structs describing the byte offset (from the base),
-   the bit size and alignment of each store as well as the original statements
-   involved in each such split group.
+   vector (if non-NULL) with split_store structs describing the byte offset
+   (from the base), the bit size and alignment of each store as well as the
+   original statements involved in each such split group.
    This is to separate the splitting strategy from the statement
    building/emission/linking done in output_merged_store.
-   At the moment just start with the widest possible size and keep emitting
-   the widest we can until we have emitted all the bytes, halving the size
-   when appropriate.  */
-
-static bool
-split_group (merged_store_group *group,
-	     auto_vec<struct split_store *> &split_stores)
+   Return number of new stores.
+   If ALLOW_UNALIGNED_STORE is false, then all stores must be aligned.
+   If ALLOW_UNALIGNED_LOAD is false, then all loads must be aligned.
+   If SPLIT_STORES is NULL, it is just a dry run to count number of
+   new stores.  */
+
+static unsigned int
+split_group (merged_store_group *group, bool allow_unaligned_store,
+	     bool allow_unaligned_load,
+	     vec<struct split_store *> *split_stores)
 {
-  unsigned HOST_WIDE_INT pos = group->start;
-  unsigned HOST_WIDE_INT size = group->width;
+  unsigned HOST_WIDE_INT pos = group->bitregion_start;
+  unsigned HOST_WIDE_INT size = group->bitregion_end - pos;
   unsigned HOST_WIDE_INT bytepos = pos / BITS_PER_UNIT;
-  unsigned HOST_WIDE_INT align = group->align;
+  unsigned HOST_WIDE_INT group_align = group->align;
+  unsigned HOST_WIDE_INT align_base = group->align_base;
+  unsigned HOST_WIDE_INT group_load_align = group_align;
 
-  /* We don't handle partial bitfields for now.  We shouldn't have
-     reached this far.  */
   gcc_assert ((size % BITS_PER_UNIT == 0) && (pos % BITS_PER_UNIT == 0));
 
-  bool allow_unaligned
-    = !STRICT_ALIGNMENT && PARAM_VALUE (PARAM_STORE_MERGING_ALLOW_UNALIGNED);
-
-  unsigned int try_size = MAX_STORE_BITSIZE;
-  while (try_size > size
-	 || (!allow_unaligned
-	     && try_size > align))
-    {
-      try_size /= 2;
-      if (try_size < BITS_PER_UNIT)
-	return false;
-    }
-
+  unsigned int ret = 0, first = 0;
   unsigned HOST_WIDE_INT try_pos = bytepos;
   group->stores.qsort (sort_by_bitpos);
 
+  if (!allow_unaligned_load)
+    for (int i = 0; i < 2; ++i)
+      if (group->load_align[i])
+	group_load_align = MIN (group_load_align, group->load_align[i]);
+
   while (size > 0)
     {
-      struct split_store *store = new split_store (try_pos, try_size, align);
+      if ((allow_unaligned_store || group_align <= BITS_PER_UNIT)
+	  && group->mask[try_pos - bytepos] == (unsigned char) ~0U)
+	{
+	  /* Skip padding bytes.  */
+	  ++try_pos;
+	  size -= BITS_PER_UNIT;
+	  continue;
+	}
+
       unsigned HOST_WIDE_INT try_bitpos = try_pos * BITS_PER_UNIT;
-      find_constituent_stmts (group, store->orig_stmts, try_bitpos, try_size);
-      split_stores.safe_push (store);
+      unsigned int try_size = MAX_STORE_BITSIZE, nonmasked;
+      unsigned HOST_WIDE_INT align_bitpos
+	= (try_bitpos - align_base) & (group_align - 1);
+      unsigned HOST_WIDE_INT align = group_align;
+      if (align_bitpos)
+	align = least_bit_hwi (align_bitpos);
+      if (!allow_unaligned_store)
+	try_size = MIN (try_size, align);
+      if (!allow_unaligned_load)
+	{
+	  /* If we can't do or don't want to do unaligned stores
+	     as well as loads, we need to take the loads into account
+	     as well.  */
+	  unsigned HOST_WIDE_INT load_align = group_load_align;
+	  align_bitpos = (try_bitpos - align_base) & (load_align - 1);
+	  if (align_bitpos)
+	    load_align = least_bit_hwi (align_bitpos);
+	  for (int i = 0; i < 2; ++i)
+	    if (group->load_align[i])
+	      {
+		align_bitpos = try_bitpos - group->stores[0]->bitpos;
+		align_bitpos += group->stores[0]->ops[i].bitpos;
+		align_bitpos -= group->load_align_base[i];
+		align_bitpos &= (group_load_align - 1);
+		if (align_bitpos)
+		  {
+		    unsigned HOST_WIDE_INT a = least_bit_hwi (align_bitpos);
+		    load_align = MIN (load_align, a);
+		  }
+	      }
+	  try_size = MIN (try_size, load_align);
+	}
+      store_immediate_info *info
+	= find_constituent_stores (group, NULL, &first, try_bitpos, try_size);
+      if (info)
+	{
+	  /* If there is just one original statement for the range, see if
+	     we can just reuse the original store which could be even larger
+	     than try_size.  */
+	  unsigned HOST_WIDE_INT stmt_end
+	    = ROUND_UP (info->bitpos + info->bitsize, BITS_PER_UNIT);
+	  info = find_constituent_stores (group, NULL, &first, try_bitpos,
+					  stmt_end - try_bitpos);
+	  if (info && info->bitpos >= try_bitpos)
+	    {
+	      try_size = stmt_end - try_bitpos;
+	      goto found;
+	    }
+	}
 
-      try_pos += try_size / BITS_PER_UNIT;
+      /* Approximate store bitsize for the case when there are no padding
+	 bits.  */
+      while (try_size > size)
+	try_size /= 2;
+      /* Now look for whole padding bytes at the end of that bitsize.  */
+      for (nonmasked = try_size / BITS_PER_UNIT; nonmasked > 0; --nonmasked)
+	if (group->mask[try_pos - bytepos + nonmasked - 1]
+	    != (unsigned char) ~0U)
+	  break;
+      if (nonmasked == 0)
+	{
+	  /* If entire try_size range is padding, skip it.  */
+	  try_pos += try_size / BITS_PER_UNIT;
+	  size -= try_size;
+	  continue;
+	}
+      /* Otherwise try to decrease try_size if second half, last 3 quarters
+	 etc. are padding.  */
+      nonmasked *= BITS_PER_UNIT;
+      while (nonmasked <= try_size / 2)
+	try_size /= 2;
+      if (!allow_unaligned_store && group_align > BITS_PER_UNIT)
+	{
+	  /* Now look for whole padding bytes at the start of that bitsize.  */
+	  unsigned int try_bytesize = try_size / BITS_PER_UNIT, masked;
+	  for (masked = 0; masked < try_bytesize; ++masked)
+	    if (group->mask[try_pos - bytepos + masked] != (unsigned char) ~0U)
+	      break;
+	  masked *= BITS_PER_UNIT;
+	  gcc_assert (masked < try_size);
+	  if (masked >= try_size / 2)
+	    {
+	      while (masked >= try_size / 2)
+		{
+		  try_size /= 2;
+		  try_pos += try_size / BITS_PER_UNIT;
+		  size -= try_size;
+		  masked -= try_size;
+		}
+	      /* Need to recompute the alignment, so just retry at the new
+		 position.  */
+	      continue;
+	    }
+	}
+
+    found:
+      ++ret;
+
+      if (split_stores)
+	{
+	  struct split_store *store
+	    = new split_store (try_pos, try_size, align);
+	  info = find_constituent_stores (group, &store->orig_stores,
+					  &first, try_bitpos, try_size);
+	  if (info
+	      && info->bitpos >= try_bitpos
+	      && info->bitpos + info->bitsize <= try_bitpos + try_size)
+	    store->orig = true;
+	  split_stores->safe_push (store);
+	}
 
+      try_pos += try_size / BITS_PER_UNIT;
       size -= try_size;
-      align = try_size;
-      while (size < try_size)
-	try_size /= 2;
     }
-  return true;
+
+  return ret;
 }
 
 /* Given a merged store group GROUP output the widened version of it.
@@ -1135,81 +1569,289 @@ split_group (merged_store_group *group,
 bool
 imm_store_chain_info::output_merged_store (merged_store_group *group)
 {
-  unsigned HOST_WIDE_INT start_byte_pos = group->start / BITS_PER_UNIT;
+  unsigned HOST_WIDE_INT start_byte_pos
+    = group->bitregion_start / BITS_PER_UNIT;
 
   unsigned int orig_num_stmts = group->stores.length ();
   if (orig_num_stmts < 2)
     return false;
 
-  auto_vec<struct split_store *> split_stores;
+  auto_vec<struct split_store *, 32> split_stores;
   split_stores.create (0);
-  if (!split_group (group, split_stores))
-    return false;
+  bool allow_unaligned_store
+    = !STRICT_ALIGNMENT && PARAM_VALUE (PARAM_STORE_MERGING_ALLOW_UNALIGNED);
+  bool allow_unaligned_load = allow_unaligned_store;
+  if (allow_unaligned_store)
+    {
+      /* If unaligned stores are allowed, see how many stores we'd emit
+	 for unaligned and how many stores we'd emit for aligned stores.
+	 Only use unaligned stores if it allows fewer stores than aligned.  */
+      unsigned aligned_cnt
+	= split_group (group, false, allow_unaligned_load, NULL);
+      unsigned unaligned_cnt
+	= split_group (group, true, allow_unaligned_load, NULL);
+      if (aligned_cnt <= unaligned_cnt)
+	allow_unaligned_store = false;
+    }
+  split_group (group, allow_unaligned_store, allow_unaligned_load,
+	       &split_stores);
+
+  if (split_stores.length () >= orig_num_stmts)
+    {
+      /* We didn't manage to reduce the number of statements.  Bail out.  */
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	{
+	  fprintf (dump_file, "Exceeded original number of stmts (%u)."
+			      "  Not profitable to emit new sequence.\n",
+		   orig_num_stmts);
+	}
+      return false;
+    }
 
   gimple_stmt_iterator last_gsi = gsi_for_stmt (group->last_stmt);
   gimple_seq seq = NULL;
-  unsigned int num_stmts = 0;
   tree last_vdef, new_vuse;
   last_vdef = gimple_vdef (group->last_stmt);
   new_vuse = gimple_vuse (group->last_stmt);
 
   gimple *stmt = NULL;
-  /* The new SSA names created.  Keep track of them so that we can free them
-     if we decide to not use the new sequence.  */
-  auto_vec<tree> new_ssa_names;
   split_store *split_store;
   unsigned int i;
-  bool fail = false;
-
+  auto_vec<gimple *, 32> orig_stmts;
   tree addr = force_gimple_operand_1 (unshare_expr (base_addr), &seq,
 				      is_gimple_mem_ref_addr, NULL_TREE);
+
+  tree load_addr[2] = { NULL_TREE, NULL_TREE };
+  gimple_seq load_seq[2] = { NULL, NULL };
+  gimple_stmt_iterator load_gsi[2] = { gsi_none (), gsi_none () };
+  for (int j = 0; j < 2; ++j)
+    {
+      store_operand_info &op = group->stores[0]->ops[j];
+      if (op.base_addr == NULL_TREE)
+	continue;
+
+      store_immediate_info *infol = group->stores.last ();
+      if (gimple_vuse (op.stmt) == gimple_vuse (infol->ops[j].stmt))
+	{
+	  load_gsi[j] = gsi_for_stmt (op.stmt);
+	  load_addr[j]
+	    = force_gimple_operand_1 (unshare_expr (op.base_addr),
+				      &load_seq[j], is_gimple_mem_ref_addr,
+				      NULL_TREE);
+	}
+      else if (operand_equal_p (base_addr, op.base_addr, 0))
+	load_addr[j] = addr;
+      else
+	load_addr[j]
+	  = force_gimple_operand_1 (unshare_expr (op.base_addr),
+				    &seq, is_gimple_mem_ref_addr,
+				    NULL_TREE);
+    }
+
   FOR_EACH_VEC_ELT (split_stores, i, split_store)
     {
       unsigned HOST_WIDE_INT try_size = split_store->size;
       unsigned HOST_WIDE_INT try_pos = split_store->bytepos;
       unsigned HOST_WIDE_INT align = split_store->align;
-      tree offset_type = get_alias_type_for_stmts (split_store->orig_stmts);
-      location_t loc = get_location_for_stmts (split_store->orig_stmts);
-
-      tree int_type = build_nonstandard_integer_type (try_size, UNSIGNED);
-      int_type = build_aligned_type (int_type, align);
-      tree dest = fold_build2 (MEM_REF, int_type, addr,
-			       build_int_cst (offset_type, try_pos));
+      tree dest, src;
+      location_t loc;
+      if (split_store->orig)
+	{
+	  /* If there is just a single constituent store which covers
+	     the whole area, just reuse the lhs and rhs.  */
+	  gimple *orig_stmt = split_store->orig_stores[0]->stmt;
+	  dest = gimple_assign_lhs (orig_stmt);
+	  src = gimple_assign_rhs1 (orig_stmt);
+	  loc = gimple_location (orig_stmt);
+	}
+      else
+	{
+	  store_immediate_info *info;
+	  unsigned short clique, base;
+	  unsigned int k;
+	  FOR_EACH_VEC_ELT (split_store->orig_stores, k, info)
+	    orig_stmts.safe_push (info->stmt);
+	  tree offset_type
+	    = get_alias_type_for_stmts (orig_stmts, false, &clique, &base);
+	  loc = get_location_for_stmts (orig_stmts);
+	  orig_stmts.truncate (0);
+
+	  tree int_type = build_nonstandard_integer_type (try_size, UNSIGNED);
+	  int_type = build_aligned_type (int_type, align);
+	  dest = fold_build2 (MEM_REF, int_type, addr,
+			      build_int_cst (offset_type, try_pos));
+	  if (TREE_CODE (dest) == MEM_REF)
+	    {
+	      MR_DEPENDENCE_CLIQUE (dest) = clique;
+	      MR_DEPENDENCE_BASE (dest) = base;
+	    }
 
-      tree src = native_interpret_expr (int_type,
-					group->val + try_pos - start_byte_pos,
-					group->buf_size);
+	  tree mask
+	    = native_interpret_expr (int_type,
+				     group->mask + try_pos - start_byte_pos,
+				     group->buf_size);
 
-      stmt = gimple_build_assign (dest, src);
-      gimple_set_location (stmt, loc);
-      gimple_set_vuse (stmt, new_vuse);
-      gimple_seq_add_stmt_without_update (&seq, stmt);
+	  tree ops[2];
+	  for (int j = 0;
+	       j < 1 + (split_store->orig_stores[0]->ops[1].val != NULL_TREE);
+	       ++j)
+	    {
+	      store_operand_info &op = split_store->orig_stores[0]->ops[j];
+	      if (op.base_addr)
+		{
+		  FOR_EACH_VEC_ELT (split_store->orig_stores, k, info)
+		    orig_stmts.safe_push (info->ops[j].stmt);
+
+		  offset_type = get_alias_type_for_stmts (orig_stmts, true,
+							  &clique, &base);
+		  location_t load_loc = get_location_for_stmts (orig_stmts);
+		  orig_stmts.truncate (0);
+
+		  unsigned HOST_WIDE_INT load_align = group->load_align[j];
+		  unsigned HOST_WIDE_INT align_bitpos
+		    = (try_pos * BITS_PER_UNIT
+		       - split_store->orig_stores[0]->bitpos
+		       + op.bitpos) & (load_align - 1);
+		  if (align_bitpos)
+		    load_align = least_bit_hwi (align_bitpos);
+
+		  tree load_int_type
+		    = build_nonstandard_integer_type (try_size, UNSIGNED);
+		  load_int_type
+		    = build_aligned_type (load_int_type, load_align);
+
+		  unsigned HOST_WIDE_INT load_pos
+		    = (try_pos * BITS_PER_UNIT
+		       - split_store->orig_stores[0]->bitpos
+		       + op.bitpos) / BITS_PER_UNIT;
+		  ops[j] = fold_build2 (MEM_REF, load_int_type, load_addr[j],
+					build_int_cst (offset_type, load_pos));
+		  if (TREE_CODE (ops[j]) == MEM_REF)
+		    {
+		      MR_DEPENDENCE_CLIQUE (ops[j]) = clique;
+		      MR_DEPENDENCE_BASE (ops[j]) = base;
+		    }
+		  if (!integer_zerop (mask))
+		    /* The load might load some bits (that will be masked off
+		       later on) uninitialized, avoid -W*uninitialized
+		       warnings in that case.  */
+		    TREE_NO_WARNING (ops[j]) = 1;
+
+		  stmt = gimple_build_assign (make_ssa_name (int_type),
+					      ops[j]);
+		  gimple_set_location (stmt, load_loc);
+		  if (gsi_bb (load_gsi[j]))
+		    {
+		      gimple_set_vuse (stmt, gimple_vuse (op.stmt));
+		      gimple_seq_add_stmt_without_update (&load_seq[j], stmt);
+		    }
+		  else
+		    {
+		      gimple_set_vuse (stmt, new_vuse);
+		      gimple_seq_add_stmt_without_update (&seq, stmt);
+		    }
+		  ops[j] = gimple_assign_lhs (stmt);
+		}
+	      else
+		ops[j] = native_interpret_expr (int_type,
+						group->val + try_pos
+						- start_byte_pos,
+						group->buf_size);
+	    }
 
-      /* We didn't manage to reduce the number of statements.  Bail out.  */
-      if (++num_stmts == orig_num_stmts)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
+	  switch (split_store->orig_stores[0]->rhs_code)
 	    {
-	      fprintf (dump_file, "Exceeded original number of stmts (%u)."
-				  "  Not profitable to emit new sequence.\n",
-		       orig_num_stmts);
+	    case BIT_AND_EXPR:
+	    case BIT_IOR_EXPR:
+	    case BIT_XOR_EXPR:
+	      FOR_EACH_VEC_ELT (split_store->orig_stores, k, info)
+		{
+		  tree rhs1 = gimple_assign_rhs1 (info->stmt);
+		  orig_stmts.safe_push (SSA_NAME_DEF_STMT (rhs1));
+		}
+	      location_t bit_loc;
+	      bit_loc = get_location_for_stmts (orig_stmts);
+	      orig_stmts.truncate (0);
+
+	      stmt
+		= gimple_build_assign (make_ssa_name (int_type),
+				       split_store->orig_stores[0]->rhs_code,
+				       ops[0], ops[1]);
+	      gimple_set_location (stmt, bit_loc);
+	      /* If there is just one load and there is a separate
+		 load_seq[0], emit the bitwise op right after it.  */
+	      if (load_addr[1] == NULL_TREE && gsi_bb (load_gsi[0]))
+		gimple_seq_add_stmt_without_update (&load_seq[0], stmt);
+	      /* Otherwise, if at least one load is in seq, we need to
+		 emit the bitwise op right before the store.  If there
+		 are two loads and are emitted somewhere else, it would
+		 be better to emit the bitwise op as early as possible;
+		 we don't track where that would be possible right now
+		 though.  */
+	      else
+		gimple_seq_add_stmt_without_update (&seq, stmt);
+	      src = gimple_assign_lhs (stmt);
+	      break;
+	    default:
+	      src = ops[0];
+	      break;
 	    }
-	  unsigned int ssa_count;
-	  tree ssa_name;
-	  /* Don't forget to cleanup the temporary SSA names.  */
-	  FOR_EACH_VEC_ELT (new_ssa_names, ssa_count, ssa_name)
-	    release_ssa_name (ssa_name);
 
-	  fail = true;
-	  break;
+	  if (!integer_zerop (mask))
+	    {
+	      tree tem = make_ssa_name (int_type);
+	      tree load_src = unshare_expr (dest);
+	      /* The load might load some or all bits uninitialized,
+		 avoid -W*uninitialized warnings in that case.
+		 As optimization, it would be nice if all the bits are
+		 provably uninitialized (no stores at all yet or previous
+		 store a CLOBBER) we'd optimize away the load and replace
+		 it e.g. with 0.  */
+	      TREE_NO_WARNING (load_src) = 1;
+	      stmt = gimple_build_assign (tem, load_src);
+	      gimple_set_location (stmt, loc);
+	      gimple_set_vuse (stmt, new_vuse);
+	      gimple_seq_add_stmt_without_update (&seq, stmt);
+
+	      /* FIXME: If there is a single chunk of zero bits in mask,
+		 perhaps use BIT_INSERT_EXPR instead?  */
+	      stmt = gimple_build_assign (make_ssa_name (int_type),
+					  BIT_AND_EXPR, tem, mask);
+	      gimple_set_location (stmt, loc);
+	      gimple_seq_add_stmt_without_update (&seq, stmt);
+	      tem = gimple_assign_lhs (stmt);
+
+	      if (TREE_CODE (src) == INTEGER_CST)
+		src = wide_int_to_tree (int_type,
+					wi::bit_and_not (wi::to_wide (src),
+							 wi::to_wide (mask)));
+	      else
+		{
+		  tree nmask
+		    = wide_int_to_tree (int_type,
+					wi::bit_not (wi::to_wide (mask)));
+		  stmt = gimple_build_assign (make_ssa_name (int_type),
+					      BIT_AND_EXPR, src, nmask);
+		  gimple_set_location (stmt, loc);
+		  gimple_seq_add_stmt_without_update (&seq, stmt);
+		  src = gimple_assign_lhs (stmt);
+		}
+	      stmt = gimple_build_assign (make_ssa_name (int_type),
+					  BIT_IOR_EXPR, tem, src);
+	      gimple_set_location (stmt, loc);
+	      gimple_seq_add_stmt_without_update (&seq, stmt);
+	      src = gimple_assign_lhs (stmt);
+	    }
 	}
 
+      stmt = gimple_build_assign (dest, src);
+      gimple_set_location (stmt, loc);
+      gimple_set_vuse (stmt, new_vuse);
+      gimple_seq_add_stmt_without_update (&seq, stmt);
+
       tree new_vdef;
       if (i < split_stores.length () - 1)
-	{
-	  new_vdef = make_ssa_name (gimple_vop (cfun), stmt);
-	  new_ssa_names.safe_push (new_vdef);
-	}
+	new_vdef = make_ssa_name (gimple_vop (cfun), stmt);
       else
 	new_vdef = last_vdef;
 
@@ -1221,19 +1863,19 @@ imm_store_chain_info::output_merged_store (merged_store_group *group)
   FOR_EACH_VEC_ELT (split_stores, i, split_store)
     delete split_store;
 
-  if (fail)
-    return false;
-
   gcc_assert (seq);
   if (dump_file)
     {
       fprintf (dump_file,
 	       "New sequence of %u stmts to replace old one of %u stmts\n",
-	       num_stmts, orig_num_stmts);
+	       split_stores.length (), orig_num_stmts);
       if (dump_flags & TDF_DETAILS)
 	print_gimple_seq (dump_file, seq, 0, TDF_VOPS | TDF_MEMSYMS);
     }
   gsi_insert_seq_after (&last_gsi, seq, GSI_SAME_STMT);
+  for (int j = 0; j < 2; ++j)
+    if (load_seq[j])
+      gsi_insert_seq_after (&load_gsi[j], load_seq[j], GSI_SAME_STMT);
 
   return true;
 }
@@ -1333,10 +1975,299 @@ rhs_valid_for_store_merging_p (tree rhs)
 	  && native_encode_expr (rhs, NULL, size) != 0);
 }
 
+/* If MEM is a memory reference usable for store merging (either as
+   store destination or for loads), return the non-NULL base_addr
+   and set *PBITSIZE, *PBITPOS, *PBITREGION_START and *PBITREGION_END.
+   Otherwise return NULL, *PBITPOS should be still valid even for that
+   case.  */
+
+static tree
+mem_valid_for_store_merging (tree mem, unsigned HOST_WIDE_INT *pbitsize,
+			     unsigned HOST_WIDE_INT *pbitpos,
+			     unsigned HOST_WIDE_INT *pbitregion_start,
+			     unsigned HOST_WIDE_INT *pbitregion_end)
+{
+  poly_int64 var_bitsize, var_bitpos;
+  poly_uint64 var_bitregion_start = 0, var_bitregion_end = 0;
+  machine_mode mode;
+  int unsignedp = 0, reversep = 0, volatilep = 0;
+  tree offset;
+  tree base_addr = get_inner_reference (mem, &var_bitsize, &var_bitpos,
+					&offset, &mode, &unsignedp, &reversep,
+					&volatilep);
+  if (must_eq (var_bitsize, 0))
+    {
+      *pbitsize = 0;
+      return NULL_TREE;
+    }
+
+  *pbitsize = -1;
+  if (TREE_CODE (mem) == COMPONENT_REF
+      && DECL_BIT_FIELD_TYPE (TREE_OPERAND (mem, 1)))
+    {
+      get_bit_range (&var_bitregion_start, &var_bitregion_end, mem,
+		     &var_bitpos, &offset);
+      if (may_ne (var_bitregion_end, 0U))
+	var_bitregion_end += 1;
+    }
+
+  if (reversep)
+    return NULL_TREE;
+
+  /* We do not want to rewrite TARGET_MEM_REFs.  */
+  if (TREE_CODE (base_addr) == TARGET_MEM_REF)
+    return NULL_TREE;
+  /* In some cases get_inner_reference may return a
+     MEM_REF [ptr + byteoffset].  For the purposes of this pass
+     canonicalize the base_addr to MEM_REF [ptr] and take
+     byteoffset into account in the bitpos.  This occurs in
+     PR 23684 and this way we can catch more chains.  */
+  else if (TREE_CODE (base_addr) == MEM_REF)
+    {
+      poly_offset_int byte_off = mem_ref_offset (base_addr);
+      poly_offset_int bit_off = byte_off << LOG2_BITS_PER_UNIT;
+      bit_off += var_bitpos;
+      if (bit_off.to_shwi (&var_bitpos))
+	{
+	  if (may_ne (var_bitregion_end, 0U))
+	    {
+	      bit_off = byte_off << LOG2_BITS_PER_UNIT;
+	      bit_off += var_bitregion_start;
+	      if (bit_off.to_uhwi (&var_bitregion_start))
+		{
+		  bit_off = byte_off << LOG2_BITS_PER_UNIT;
+		  bit_off += var_bitregion_end;
+		  if (!bit_off.to_uhwi (&var_bitregion_end))
+		    var_bitregion_end = 0;
+		}
+	      else
+		var_bitregion_end = 0;
+	    }
+	}
+      else
+	return NULL_TREE;
+      base_addr = TREE_OPERAND (base_addr, 0);
+    }
+  /* get_inner_reference returns the base object, get at its
+     address now.  */
+  else
+    {
+      if (may_lt (var_bitpos, 0))
+	return NULL_TREE;
+      base_addr = build_fold_addr_expr (base_addr);
+    }
+
+  HOST_WIDE_INT bitsize, bitpos;
+  if (!var_bitsize.is_constant (&bitsize)
+      || !var_bitpos.is_constant (&bitpos))
+    return NULL_TREE;
+
+  unsigned HOST_WIDE_INT bitregion_start, bitregion_end;
+  if (!var_bitregion_start.is_constant (&bitregion_start)
+      || !var_bitregion_end.is_constant (&bitregion_end))
+    return NULL_TREE;
+
+  if (!bitregion_end)
+    {
+      bitregion_start = ROUND_DOWN (bitpos, BITS_PER_UNIT);
+      bitregion_end = ROUND_UP (bitpos + bitsize, BITS_PER_UNIT);
+    }
+
+  if (offset != NULL_TREE)
+    {
+      /* If the access is variable offset then a base decl has to be
+	 address-taken to be able to emit pointer-based stores to it.
+	 ???  We might be able to get away with re-using the original
+	 base up to the first variable part and then wrapping that inside
+	 a BIT_FIELD_REF.  */
+      tree base = get_base_address (base_addr);
+      if (! base
+	  || (DECL_P (base) && ! TREE_ADDRESSABLE (base)))
+	return NULL_TREE;
+
+      base_addr = build2 (POINTER_PLUS_EXPR, TREE_TYPE (base_addr),
+			  base_addr, offset);
+    }
+
+  *pbitsize = bitsize;
+  *pbitpos = bitpos;
+  *pbitregion_start = bitregion_start;
+  *pbitregion_end = bitregion_end;
+  return base_addr;
+}
+
+/* Return true if STMT is a load that can be used for store merging.
+   In that case fill in *OP.  BITSIZE, BITPOS, BITREGION_START and
+   BITREGION_END are properties of the corresponding store.  */
+
+static bool
+handled_load (gimple *stmt, store_operand_info *op,
+	      unsigned HOST_WIDE_INT bitsize, unsigned HOST_WIDE_INT bitpos,
+	      unsigned HOST_WIDE_INT bitregion_start,
+	      unsigned HOST_WIDE_INT bitregion_end)
+{
+  if (!is_gimple_assign (stmt) || !gimple_vuse (stmt))
+    return false;
+  if (gimple_assign_load_p (stmt)
+      && !stmt_can_throw_internal (stmt)
+      && !gimple_has_volatile_ops (stmt))
+    {
+      tree mem = gimple_assign_rhs1 (stmt);
+      op->base_addr
+	= mem_valid_for_store_merging (mem, &op->bitsize, &op->bitpos,
+				       &op->bitregion_start,
+				       &op->bitregion_end);
+      if (op->base_addr != NULL_TREE
+	  && op->bitsize == bitsize
+	  && ((op->bitpos - bitpos) % BITS_PER_UNIT) == 0
+	  && op->bitpos - op->bitregion_start >= bitpos - bitregion_start
+	  && op->bitregion_end - op->bitpos >= bitregion_end - bitpos)
+	{
+	  op->stmt = stmt;
+	  op->val = mem;
+	  return true;
+	}
+    }
+  return false;
+}
+
+/* Record the store STMT for store merging optimization if it can be
+   optimized.  */
+
+void
+pass_store_merging::process_store (gimple *stmt)
+{
+  tree lhs = gimple_assign_lhs (stmt);
+  tree rhs = gimple_assign_rhs1 (stmt);
+  unsigned HOST_WIDE_INT bitsize, bitpos;
+  unsigned HOST_WIDE_INT bitregion_start;
+  unsigned HOST_WIDE_INT bitregion_end;
+  tree base_addr
+    = mem_valid_for_store_merging (lhs, &bitsize, &bitpos,
+				   &bitregion_start, &bitregion_end);
+  if (bitsize == 0)
+    return;
+
+  bool invalid = (base_addr == NULL_TREE
+		  || ((bitsize > MAX_BITSIZE_MODE_ANY_INT)
+		       && (TREE_CODE (rhs) != INTEGER_CST)));
+  enum tree_code rhs_code = ERROR_MARK;
+  store_operand_info ops[2];
+  if (invalid)
+    ;
+  else if (rhs_valid_for_store_merging_p (rhs))
+    {
+      rhs_code = INTEGER_CST;
+      ops[0].val = rhs;
+    }
+  else if (TREE_CODE (rhs) != SSA_NAME || !has_single_use (rhs))
+    invalid = true;
+  else
+    {
+      gimple *def_stmt = SSA_NAME_DEF_STMT (rhs), *def_stmt1, *def_stmt2;
+      if (!is_gimple_assign (def_stmt))
+	invalid = true;
+      else if (handled_load (def_stmt, &ops[0], bitsize, bitpos,
+			     bitregion_start, bitregion_end))
+	rhs_code = MEM_REF;
+      else
+	switch ((rhs_code = gimple_assign_rhs_code (def_stmt)))
+	  {
+	  case BIT_AND_EXPR:
+	  case BIT_IOR_EXPR:
+	  case BIT_XOR_EXPR:
+	    tree rhs1, rhs2;
+	    rhs1 = gimple_assign_rhs1 (def_stmt);
+	    rhs2 = gimple_assign_rhs2 (def_stmt);
+	    invalid = true;
+	    if (TREE_CODE (rhs1) != SSA_NAME || !has_single_use (rhs1))
+	      break;
+	    def_stmt1 = SSA_NAME_DEF_STMT (rhs1);
+	    if (!is_gimple_assign (def_stmt1)
+		|| !handled_load (def_stmt1, &ops[0], bitsize, bitpos,
+				  bitregion_start, bitregion_end))
+	      break;
+	    if (rhs_valid_for_store_merging_p (rhs2))
+	      ops[1].val = rhs2;
+	    else if (TREE_CODE (rhs2) != SSA_NAME || !has_single_use (rhs2))
+	      break;
+	    else
+	      {
+		def_stmt2 = SSA_NAME_DEF_STMT (rhs2);
+		if (!is_gimple_assign (def_stmt2))
+		  break;
+		else if (!handled_load (def_stmt2, &ops[1], bitsize, bitpos,
+					bitregion_start, bitregion_end))
+		  break;
+	      }
+	    invalid = false;
+	    break;
+	  default:
+	    invalid = true;
+	    break;
+	  }
+    }
+
+  struct imm_store_chain_info **chain_info = NULL;
+  if (base_addr)
+    chain_info = m_stores.get (base_addr);
+
+  if (invalid)
+    {
+      terminate_all_aliasing_chains (chain_info, stmt);
+      return;
+    }
+
+  store_immediate_info *info;
+  if (chain_info)
+    {
+      unsigned int ord = (*chain_info)->m_store_info.length ();
+      info = new store_immediate_info (bitsize, bitpos, bitregion_start,
+				       bitregion_end, stmt, ord, rhs_code,
+				       ops[0], ops[1]);
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	{
+	  fprintf (dump_file, "Recording immediate store from stmt:\n");
+	  print_gimple_stmt (dump_file, stmt, 0);
+	}
+      (*chain_info)->m_store_info.safe_push (info);
+      /* If we reach the limit of stores to merge in a chain terminate and
+	 process the chain now.  */
+      if ((*chain_info)->m_store_info.length ()
+	  == (unsigned int) PARAM_VALUE (PARAM_MAX_STORES_TO_MERGE))
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file,
+		     "Reached maximum number of statements to merge:\n");
+	  terminate_and_release_chain (*chain_info);
+	}
+      return;
+    }
+
+  /* Store aliases any existing chain?  */
+  terminate_all_aliasing_chains (chain_info, stmt);
+  /* Start a new chain.  */
+  struct imm_store_chain_info *new_chain
+    = new imm_store_chain_info (m_stores_head, base_addr);
+  info = new store_immediate_info (bitsize, bitpos, bitregion_start,
+				   bitregion_end, stmt, 0, rhs_code,
+				   ops[0], ops[1]);
+  new_chain->m_store_info.safe_push (info);
+  m_stores.put (base_addr, new_chain);
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Starting new chain with statement:\n");
+      print_gimple_stmt (dump_file, stmt, 0);
+      fprintf (dump_file, "The base object is:\n");
+      print_generic_expr (dump_file, base_addr);
+      fprintf (dump_file, "\n");
+    }
+}
+
 /* Entry point for the pass.  Go over each basic block recording chains of
-  immediate stores.  Upon encountering a terminating statement (as defined
-  by stmt_terminates_chain_p) process the recorded stores and emit the widened
-  variants.  */
+   immediate stores.  Upon encountering a terminating statement (as defined
+   by stmt_terminates_chain_p) process the recorded stores and emit the widened
+   variants.  */
 
 unsigned int
 pass_store_merging::execute (function *fun)
@@ -1386,131 +2317,9 @@ pass_store_merging::execute (function *fun)
 	  if (gimple_assign_single_p (stmt) && gimple_vdef (stmt)
 	      && !stmt_can_throw_internal (stmt)
 	      && lhs_valid_for_store_merging_p (gimple_assign_lhs (stmt)))
-	    {
-	      tree lhs = gimple_assign_lhs (stmt);
-	      tree rhs = gimple_assign_rhs1 (stmt);
-
-	      poly_int64 bitsize, bitpos;
-	      machine_mode mode;
-	      int unsignedp = 0, reversep = 0, volatilep = 0;
-	      tree offset, base_addr;
-	      base_addr
-		= get_inner_reference (lhs, &bitsize, &bitpos, &offset, &mode,
-				       &unsignedp, &reversep, &volatilep);
-	      /* As a future enhancement we could handle stores with the same
-		 base and offset.  */
-	      bool invalid = reversep
-			     || !rhs_valid_for_store_merging_p (rhs);
-
-	      /* We do not want to rewrite TARGET_MEM_REFs.  */
-	      if (TREE_CODE (base_addr) == TARGET_MEM_REF)
-		invalid = true;
-	      /* In some cases get_inner_reference may return a
-		 MEM_REF [ptr + byteoffset].  For the purposes of this pass
-		 canonicalize the base_addr to MEM_REF [ptr] and take
-		 byteoffset into account in the bitpos.  This occurs in
-		 PR 23684 and this way we can catch more chains.  */
-	      else if (TREE_CODE (base_addr) == MEM_REF)
-		{
-		  poly_offset_int byte_off = mem_ref_offset (base_addr);
-		  poly_offset_int bit_off = byte_off << LOG2_BITS_PER_UNIT;
-		  bit_off += bitpos;
-		  if (!bit_off.to_shwi (&bitpos))
-		    invalid = true;
-		  base_addr = TREE_OPERAND (base_addr, 0);
-		}
-	      /* get_inner_reference returns the base object, get at its
-	         address now.  */
-	      else
-		base_addr = build_fold_addr_expr (base_addr);
-
-	      if (! invalid
-		  && offset != NULL_TREE)
-		{
-		  /* If the access is variable offset then a base
-		     decl has to be address-taken to be able to
-		     emit pointer-based stores to it.
-		     ???  We might be able to get away with
-		     re-using the original base up to the first
-		     variable part and then wrapping that inside
-		     a BIT_FIELD_REF.  */
-		  tree base = get_base_address (base_addr);
-		  if (! base
-		      || (DECL_P (base)
-			  && ! TREE_ADDRESSABLE (base)))
-		    invalid = true;
-		  else
-		    base_addr = build2 (POINTER_PLUS_EXPR,
-					TREE_TYPE (base_addr),
-					base_addr, offset);
-		}
-
-	      struct imm_store_chain_info **chain_info
-		= m_stores.get (base_addr);
-
-	      HOST_WIDE_INT const_bitsize, const_bitpos;
-	      if (!invalid
-		  && bitsize.is_constant (&const_bitsize)
-		  && bitpos.is_constant (&const_bitpos)
-		  && (const_bitsize <= MAX_BITSIZE_MODE_ANY_INT
-		      || TREE_CODE (rhs) == INTEGER_CST)
-		  && const_bitpos >= 0)
-		{
-		  store_immediate_info *info;
-		  if (chain_info)
-		    {
-		      info = new store_immediate_info (
-			const_bitsize, const_bitpos, stmt,
-			(*chain_info)->m_store_info.length ());
-		      if (dump_file && (dump_flags & TDF_DETAILS))
-			{
-			  fprintf (dump_file,
-				   "Recording immediate store from stmt:\n");
-			  print_gimple_stmt (dump_file, stmt, 0);
-			}
-		      (*chain_info)->m_store_info.safe_push (info);
-		      /* If we reach the limit of stores to merge in a chain
-			 terminate and process the chain now.  */
-		      if ((*chain_info)->m_store_info.length ()
-			   == (unsigned int)
-			      PARAM_VALUE (PARAM_MAX_STORES_TO_MERGE))
-			{
-			  if (dump_file && (dump_flags & TDF_DETAILS))
-			    fprintf (dump_file,
-				 "Reached maximum number of statements"
-				 " to merge:\n");
-			  terminate_and_release_chain (*chain_info);
-			}
-		      continue;
-		    }
-
-		  /* Store aliases any existing chain?  */
-		  terminate_all_aliasing_chains (chain_info, false, stmt);
-		  /* Start a new chain.  */
-		  struct imm_store_chain_info *new_chain
-		    = new imm_store_chain_info (m_stores_head, base_addr);
-		  info = new store_immediate_info (const_bitsize, const_bitpos,
-						   stmt, 0);
-		  new_chain->m_store_info.safe_push (info);
-		  m_stores.put (base_addr, new_chain);
-		  if (dump_file && (dump_flags & TDF_DETAILS))
-		    {
-		      fprintf (dump_file,
-			       "Starting new chain with statement:\n");
-		      print_gimple_stmt (dump_file, stmt, 0);
-		      fprintf (dump_file, "The base object is:\n");
-		      print_generic_expr (dump_file, base_addr);
-		      fprintf (dump_file, "\n");
-		    }
-		}
-	      else
-		terminate_all_aliasing_chains (chain_info,
-					       offset != NULL_TREE, stmt);
-
-	      continue;
-	    }
-
-	  terminate_all_aliasing_chains (NULL, false, stmt);
+	    process_store (stmt);
+	  else
+	    terminate_all_aliasing_chains (NULL, stmt);
 	}
       terminate_and_process_all_chains ();
     }
diff --git a/gcc/gimple-ssa-warn-alloca.c b/gcc/gimple-ssa-warn-alloca.c
index 2d255a493d0..08c2195575a 100644
--- a/gcc/gimple-ssa-warn-alloca.c
+++ b/gcc/gimple-ssa-warn-alloca.c
@@ -264,7 +264,7 @@ is_max (tree x, wide_int max)
 
 // Analyze the alloca call in STMT and return the alloca type with its
 // corresponding limit (if applicable).  IS_VLA is set if the alloca
-// call is really a BUILT_IN_ALLOCA_WITH_ALIGN, signifying a VLA.
+// call was created by the gimplifier for a VLA.
 //
 // If the alloca call may be too large because of a cast from a signed
 // type to an unsigned type, set *INVALID_CASTED_TYPE to the
@@ -278,7 +278,8 @@ alloca_call_type (gimple *stmt, bool is_vla, tree *invalid_casted_type)
   tree len = gimple_call_arg (stmt, 0);
   tree len_casted = NULL;
   wide_int min, max;
-  struct alloca_type_and_limit ret = alloca_type_and_limit (ALLOCA_UNBOUNDED);
+  edge_iterator ei;
+  edge e;
 
   gcc_assert (!is_vla || (unsigned HOST_WIDE_INT) warn_vla_limit > 0);
   gcc_assert (is_vla || (unsigned HOST_WIDE_INT) warn_alloca_limit > 0);
@@ -299,16 +300,18 @@ alloca_call_type (gimple *stmt, bool is_vla, tree *invalid_casted_type)
 				      wi::to_wide (len));
       if (integer_zerop (len))
 	return alloca_type_and_limit (ALLOCA_ARG_IS_ZERO);
-      ret = alloca_type_and_limit (ALLOCA_OK);
+
+      return alloca_type_and_limit (ALLOCA_OK);
     }
+
   // Check the range info if available.
-  else if (TREE_CODE (len) == SSA_NAME)
+  if (TREE_CODE (len) == SSA_NAME)
     {
       value_range_type range_type = get_range_info (len, &min, &max);
       if (range_type == VR_RANGE)
 	{
 	  if (wi::leu_p (max, max_size))
-	    ret = alloca_type_and_limit (ALLOCA_OK);
+	    return alloca_type_and_limit (ALLOCA_OK);
 	  else
 	    {
 	      // A cast may have created a range we don't care
@@ -391,52 +394,41 @@ alloca_call_type (gimple *stmt, bool is_vla, tree *invalid_casted_type)
   // If we couldn't find anything, try a few heuristics for things we
   // can easily determine.  Check these misc cases but only accept
   // them if all predecessors have a known bound.
-  basic_block bb = gimple_bb (stmt);
-  if (ret.type == ALLOCA_UNBOUNDED)
+  struct alloca_type_and_limit ret = alloca_type_and_limit (ALLOCA_OK);
+  FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->preds)
     {
-      ret.type = ALLOCA_OK;
-      for (unsigned ix = 0; ix < EDGE_COUNT (bb->preds); ix++)
-	{
-	  gcc_assert (!len_casted || TYPE_UNSIGNED (TREE_TYPE (len_casted)));
-	  ret = alloca_call_type_by_arg (len, len_casted,
-					 EDGE_PRED (bb, ix), max_size);
-	  if (ret.type != ALLOCA_OK)
-	    break;
-	}
+      gcc_assert (!len_casted || TYPE_UNSIGNED (TREE_TYPE (len_casted)));
+      ret = alloca_call_type_by_arg (len, len_casted, e, max_size);
+      if (ret.type != ALLOCA_OK)
+	break;
+    }
+
+  if (ret.type != ALLOCA_OK && tentative_cast_from_signed)
+    ret = alloca_type_and_limit (ALLOCA_CAST_FROM_SIGNED);
+
+  // If we have a declared maximum size, we can take it into account.
+  if (ret.type != ALLOCA_OK
+      && gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX))
+    {
+      tree arg = gimple_call_arg (stmt, 2);
+      if (compare_tree_int (arg, max_size) <= 0)
+	ret = alloca_type_and_limit (ALLOCA_OK);
+      else
+	ret = alloca_type_and_limit (ALLOCA_BOUND_MAYBE_LARGE,
+				     wi::to_wide (arg));
     }
 
-  if (tentative_cast_from_signed && ret.type != ALLOCA_OK)
-    return alloca_type_and_limit (ALLOCA_CAST_FROM_SIGNED);
   return ret;
 }
 
-// Return TRUE if the alloca call in STMT is in a loop, otherwise
-// return FALSE. As an exception, ignore alloca calls for VLAs that
-// occur in a loop since those will be cleaned up when they go out of
-// scope.
+// Return TRUE if STMT is in a loop, otherwise return FALSE.
 
 static bool
-in_loop_p (bool is_vla, gimple *stmt)
+in_loop_p (gimple *stmt)
 {
   basic_block bb = gimple_bb (stmt);
-  if (bb->loop_father
-      && bb->loop_father->header != ENTRY_BLOCK_PTR_FOR_FN (cfun))
-    {
-      // Do not warn on VLAs occurring in a loop, since VLAs are
-      // guaranteed to be cleaned up when they go out of scope.
-      // That is, there is a corresponding __builtin_stack_restore
-      // at the end of the scope in which the VLA occurs.
-      tree fndecl = gimple_call_fn (stmt);
-      while (TREE_CODE (fndecl) == ADDR_EXPR)
-	fndecl = TREE_OPERAND (fndecl, 0);
-      if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
-	  && is_vla
-	  && DECL_FUNCTION_CODE (fndecl) == BUILT_IN_ALLOCA_WITH_ALIGN)
-	return false;
-
-      return true;
-    }
-  return false;
+  return
+    bb->loop_father && bb->loop_father->header != ENTRY_BLOCK_PTR_FOR_FN (cfun);
 }
 
 unsigned int
@@ -455,8 +447,8 @@ pass_walloca::execute (function *fun)
 	    continue;
 	  gcc_assert (gimple_call_num_args (stmt) >= 1);
 
-	  bool is_vla = gimple_alloca_call_p (stmt)
-	    && gimple_call_alloca_for_var_p (as_a <gcall *> (stmt));
+	  const bool is_vla
+	    = gimple_call_alloca_for_var_p (as_a <gcall *> (stmt));
 
 	  // Strict mode whining for VLAs is handled by the front-end,
 	  // so we can safely ignore this case.  Also, ignore VLAs if
@@ -476,9 +468,10 @@ pass_walloca::execute (function *fun)
 	  struct alloca_type_and_limit t
 	    = alloca_call_type (stmt, is_vla, &invalid_casted_type);
 
-	  // Even if we think the alloca call is OK, make sure it's
-	  // not in a loop.
-	  if (t.type == ALLOCA_OK && in_loop_p (is_vla, stmt))
+	  // Even if we think the alloca call is OK, make sure it's not in a
+	  // loop, except for a VLA, since VLAs are guaranteed to be cleaned
+	  // up when they go out of scope, including in a loop.
+	  if (t.type == ALLOCA_OK && !is_vla && in_loop_p (stmt))
 	    t = alloca_type_and_limit (ALLOCA_IN_LOOP);
 
 	  enum opt_code wcode
diff --git a/gcc/gimple-streamer-in.c b/gcc/gimple-streamer-in.c
index 23cf692e321..0dabe1adcf6 100644
--- a/gcc/gimple-streamer-in.c
+++ b/gcc/gimple-streamer-in.c
@@ -266,7 +266,6 @@ input_bb (struct lto_input_block *ib, enum LTO_tags tag,
 
   bb->count = profile_count::stream_in (ib).apply_scale
 		 (count_materialization_scale, REG_BR_PROB_BASE);
-  bb->frequency = streamer_read_hwi (ib);
   bb->flags = streamer_read_hwi (ib);
 
   /* LTO_bb1 has statements.  LTO_bb0 does not.  */
diff --git a/gcc/gimple-streamer-out.c b/gcc/gimple-streamer-out.c
index cdd775388e1..c19e5f1b55f 100644
--- a/gcc/gimple-streamer-out.c
+++ b/gcc/gimple-streamer-out.c
@@ -210,7 +210,6 @@ output_bb (struct output_block *ob, basic_block bb, struct function *fn)
 
   streamer_write_uhwi (ob, bb->index);
   bb->count.stream_out (ob);
-  streamer_write_hwi (ob, bb->frequency);
   streamer_write_hwi (ob, bb->flags);
 
   if (!gsi_end_p (bsi) || phi_nodes (bb))
diff --git a/gcc/gimple.c b/gcc/gimple.c
index 26414d69ea9..af49405929a 100644
--- a/gcc/gimple.c
+++ b/gcc/gimple.c
@@ -346,7 +346,7 @@ gimple_build_call_internal_vec (enum internal_fn fn, vec<tree> args)
    this fact.  */
 
 gcall *
-gimple_build_call_from_tree (tree t)
+gimple_build_call_from_tree (tree t, tree fnptrtype)
 {
   unsigned i, nargs;
   gcall *call;
@@ -369,8 +369,7 @@ gimple_build_call_from_tree (tree t)
   gimple_call_set_return_slot_opt (call, CALL_EXPR_RETURN_SLOT_OPT (t));
   if (fndecl
       && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
-      && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_ALLOCA
-	  || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_ALLOCA_WITH_ALIGN))
+      && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (fndecl)))
     gimple_call_set_alloca_for_var (call, CALL_ALLOCA_FOR_VAR_P (t));
   else
     gimple_call_set_from_thunk (call, CALL_FROM_THUNK_P (t));
@@ -380,6 +379,23 @@ gimple_build_call_from_tree (tree t)
   gimple_set_no_warning (call, TREE_NO_WARNING (t));
   gimple_call_set_with_bounds (call, CALL_WITH_BOUNDS_P (t));
 
+  if (fnptrtype)
+    {
+      gimple_call_set_fntype (call, TREE_TYPE (fnptrtype));
+
+      /* Check if it's an indirect CALL and the type has the
+ 	 nocf_check attribute. In that case propagate the information
+	 to the gimple CALL insn.  */
+      if (!fndecl)
+	{
+	  gcc_assert (POINTER_TYPE_P (fnptrtype));
+	  tree fntype = TREE_TYPE (fnptrtype);
+
+	  if (lookup_attribute ("nocf_check", TYPE_ATTRIBUTES (fntype)))
+	    gimple_call_set_nocf_check (call, TRUE);
+	}
+    }
+
   return call;
 }
 
@@ -1824,11 +1840,35 @@ gimple_copy (gimple *stmt)
 	  gimple_omp_sections_set_clauses (copy, t);
 	  t = unshare_expr (gimple_omp_sections_control (stmt));
 	  gimple_omp_sections_set_control (copy, t);
-	  /* FALLTHRU  */
+	  goto copy_omp_body;
 
 	case GIMPLE_OMP_SINGLE:
+	  {
+	    gomp_single *omp_single_copy = as_a <gomp_single *> (copy);
+	    t = unshare_expr (gimple_omp_single_clauses (stmt));
+	    gimple_omp_single_set_clauses (omp_single_copy, t);
+	  }
+	  goto copy_omp_body;
+
 	case GIMPLE_OMP_TARGET:
+	  {
+	    gomp_target *omp_target_stmt = as_a <gomp_target *> (stmt);
+	    gomp_target *omp_target_copy = as_a <gomp_target *> (copy);
+	    t = unshare_expr (gimple_omp_target_clauses (omp_target_stmt));
+	    gimple_omp_target_set_clauses (omp_target_copy, t);
+	    t = unshare_expr (gimple_omp_target_data_arg (omp_target_stmt));
+	    gimple_omp_target_set_data_arg (omp_target_copy, t);
+	  }
+	  goto copy_omp_body;
+
 	case GIMPLE_OMP_TEAMS:
+	  {
+	    gomp_teams *omp_teams_copy = as_a <gomp_teams *> (copy);
+	    t = unshare_expr (gimple_omp_teams_clauses (stmt));
+	    gimple_omp_teams_set_clauses (omp_teams_copy, t);
+	  }
+	  /* FALLTHRU  */
+
 	case GIMPLE_OMP_SECTION:
 	case GIMPLE_OMP_MASTER:
 	case GIMPLE_OMP_TASKGROUP:
diff --git a/gcc/gimple.h b/gcc/gimple.h
index 6213c49b91f..334def89398 100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -148,6 +148,7 @@ enum gf_mask {
     GF_CALL_WITH_BOUNDS 	= 1 << 8,
     GF_CALL_MUST_TAIL_CALL	= 1 << 9,
     GF_CALL_BY_DESCRIPTOR	= 1 << 10,
+    GF_CALL_NOCF_CHECK		= 1 << 11,
     GF_OMP_PARALLEL_COMBINED	= 1 << 0,
     GF_OMP_PARALLEL_GRID_PHONY = 1 << 1,
     GF_OMP_TASK_TASKLOOP	= 1 << 0,
@@ -1425,7 +1426,7 @@ gcall *gimple_build_call (tree, unsigned, ...);
 gcall *gimple_build_call_valist (tree, unsigned, va_list);
 gcall *gimple_build_call_internal (enum internal_fn, unsigned, ...);
 gcall *gimple_build_call_internal_vec (enum internal_fn, vec<tree> );
-gcall *gimple_build_call_from_tree (tree);
+gcall *gimple_build_call_from_tree (tree, tree);
 gassign *gimple_build_assign (tree, tree CXX_MEM_STAT_INFO);
 gassign *gimple_build_assign (tree, enum tree_code,
 			      tree, tree, tree CXX_MEM_STAT_INFO);
@@ -2893,6 +2894,25 @@ gimple_call_set_with_bounds (gimple *gs, bool with_bounds)
 }
 
 
+/* Return true if call GS is marked as nocf_check.  */
+
+static inline bool
+gimple_call_nocf_check_p (const gcall *gs)
+{
+  return (gs->subcode & GF_CALL_NOCF_CHECK) != 0;
+}
+
+/* Mark statement GS as nocf_check call.  */
+
+static inline void
+gimple_call_set_nocf_check (gcall *gs, bool nocf_check)
+{
+  if (nocf_check)
+    gs->subcode |= GF_CALL_NOCF_CHECK;
+  else
+    gs->subcode &= ~GF_CALL_NOCF_CHECK;
+}
+
 /* Return the target of internal call GS.  */
 
 static inline enum internal_fn
diff --git a/gcc/gimplify.c b/gcc/gimplify.c
index 8a5c380027c..540d128a70d 100644
--- a/gcc/gimplify.c
+++ b/gcc/gimplify.c
@@ -27,6 +27,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "target.h"
 #include "rtl.h"
 #include "tree.h"
+#include "memmodel.h"
+#include "tm_p.h"
 #include "gimple.h"
 #include "gimple-predict.h"
 #include "tree-pass.h"		/* FIXME: only for PROP_gimple_any */
@@ -1576,9 +1578,8 @@ gimplify_vla_decl (tree decl, gimple_seq *seq_p)
   SET_DECL_VALUE_EXPR (decl, t);
   DECL_HAS_VALUE_EXPR_P (decl) = 1;
 
-  t = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
-  t = build_call_expr (t, 2, DECL_SIZE_UNIT (decl),
-		       size_int (DECL_ALIGN (decl)));
+  t = build_alloca_call_expr (DECL_SIZE_UNIT (decl), DECL_ALIGN (decl),
+			      max_int_size_in_bytes (TREE_TYPE (decl)));
   /* The call has been built for a variable-sized object.  */
   CALL_ALLOCA_FOR_VAR_P (t) = 1;
   t = fold_convert (ptr_type, t);
@@ -1658,6 +1659,7 @@ gimplify_decl_expr (tree *stmt_p, gimple_seq *seq_p)
 	  && TREE_ADDRESSABLE (decl)
 	  && !TREE_STATIC (decl)
 	  && !DECL_HAS_VALUE_EXPR_P (decl)
+	  && DECL_ALIGN (decl) <= MAX_SUPPORTED_STACK_ALIGNMENT
 	  && dbg_cnt (asan_use_after_scope))
 	{
 	  asan_poisoned_variables->add (decl);
@@ -3175,8 +3177,7 @@ gimplify_call_expr (tree *expr_p, gimple_seq *pre_p, bool want_value)
       && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
     switch (DECL_FUNCTION_CODE (fndecl))
       {
-      case BUILT_IN_ALLOCA:
-      case BUILT_IN_ALLOCA_WITH_ALIGN:
+      CASE_BUILT_IN_ALLOCA:
 	/* If the call has been built for a variable-sized object, then we
 	   want to restore the stack level when the enclosing BIND_EXPR is
 	   exited to reclaim the allocated space; otherwise, we precisely
@@ -3381,8 +3382,7 @@ gimplify_call_expr (tree *expr_p, gimple_seq *pre_p, bool want_value)
       /* The CALL_EXPR in *EXPR_P is already in GIMPLE form, so all we
 	 have to do is replicate it as a GIMPLE_CALL tuple.  */
       gimple_stmt_iterator gsi;
-      call = gimple_build_call_from_tree (*expr_p);
-      gimple_call_set_fntype (call, TREE_TYPE (fnptrtype));
+      call = gimple_build_call_from_tree (*expr_p, fnptrtype);
       notice_special_calls (call);
       if (EXPR_CILK_SPAWN (*expr_p))
         gimplify_cilk_detach (pre_p);
@@ -5663,8 +5663,7 @@ gimplify_modify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
 						    CALL_EXPR_ARG (*from_p, 2));
 	  else
 	    {
-	      call_stmt = gimple_build_call_from_tree (*from_p);
-	      gimple_call_set_fntype (call_stmt, TREE_TYPE (fnptrtype));
+	      call_stmt = gimple_build_call_from_tree (*from_p, fnptrtype);
 	    }
 	}
       notice_special_calls (call_stmt);
@@ -6507,7 +6506,9 @@ gimplify_target_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 	      clobber = build2 (MODIFY_EXPR, TREE_TYPE (temp), temp, clobber);
 	      gimple_push_cleanup (temp, clobber, false, pre_p, true);
 	    }
-	  if (asan_poisoned_variables && dbg_cnt (asan_use_after_scope))
+	  if (asan_poisoned_variables
+	      && DECL_ALIGN (temp) <= MAX_SUPPORTED_STACK_ALIGNMENT
+	      && dbg_cnt (asan_use_after_scope))
 	    {
 	      tree asan_cleanup = build_asan_poison_call_expr (temp);
 	      if (asan_cleanup)
@@ -7972,7 +7973,7 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
 			o1 = wi::to_poly_offset (offset);
 		      else
 			o1 = 0;
-		      if (maybe_nonzero (bitpos))
+		      if (may_ne (bitpos, 0))
 			o1 += bits_to_bytes_round_down (bitpos);
 		      sc = &OMP_CLAUSE_CHAIN (*osc);
 		      if (*sc != c
diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE
index 418e1274fdf..0fa2cccebfe 100644
--- a/gcc/go/gofrontend/MERGE
+++ b/gcc/go/gofrontend/MERGE
@@ -1,4 +1,4 @@
-44132970e4b6c1186036bf8eda8982fb6e905d6f
+64d570c590a76921cbdca4efb22e4675e19cc809
 
 The first line of this file holds the git revision number of the last
 merge done from the gofrontend repository.
diff --git a/gcc/go/gofrontend/expressions.cc b/gcc/go/gofrontend/expressions.cc
index 8337cbeb602..dad22ebd2c9 100644
--- a/gcc/go/gofrontend/expressions.cc
+++ b/gcc/go/gofrontend/expressions.cc
@@ -144,8 +144,8 @@ Expression::convert_for_assignment(Gogo*, Type* lhs_type,
       || rhs->is_error_expression())
     return Expression::make_error(location);
 
-  if (lhs_type->forwarded() != rhs_type->forwarded()
-      && lhs_type->interface_type() != NULL)
+  bool are_identical = Type::are_identical(lhs_type, rhs_type, false, NULL);
+  if (!are_identical && lhs_type->interface_type() != NULL)
     {
       if (rhs_type->interface_type() == NULL)
         return Expression::convert_type_to_interface(lhs_type, rhs, location);
@@ -153,8 +153,7 @@ Expression::convert_for_assignment(Gogo*, Type* lhs_type,
         return Expression::convert_interface_to_interface(lhs_type, rhs, false,
                                                           location);
     }
-  else if (lhs_type->forwarded() != rhs_type->forwarded()
-	   && rhs_type->interface_type() != NULL)
+  else if (!are_identical && rhs_type->interface_type() != NULL)
     return Expression::convert_interface_to_type(lhs_type, rhs, location);
   else if (lhs_type->is_slice_type() && rhs_type->is_nil_type())
     {
@@ -165,8 +164,15 @@ Expression::convert_for_assignment(Gogo*, Type* lhs_type,
     }
   else if (rhs_type->is_nil_type())
     return Expression::make_nil(location);
-  else if (Type::are_identical(lhs_type, rhs_type, false, NULL))
+  else if (are_identical)
     {
+      if (lhs_type->forwarded() != rhs_type->forwarded())
+	{
+	  // Different but identical types require an explicit
+	  // conversion.  This happens with type aliases.
+	  return Expression::make_cast(lhs_type, rhs, location);
+	}
+
       // No conversion is needed.
       return rhs;
     }
diff --git a/gcc/graphite-dependences.c b/gcc/graphite-dependences.c
index 2066b2ea59c..bd3e91ba860 100644
--- a/gcc/graphite-dependences.c
+++ b/gcc/graphite-dependences.c
@@ -67,9 +67,9 @@ add_pdr_constraints (poly_dr_p pdr, poly_bb_p pbb)
    reads are returned in READS and writes in MUST_WRITES and MAY_WRITES.  */
 
 static void
-scop_get_reads_and_writes (scop_p scop, isl_union_map *reads,
-			   isl_union_map *must_writes,
-			   isl_union_map *may_writes)
+scop_get_reads_and_writes (scop_p scop, isl_union_map *&reads,
+			   isl_union_map *&must_writes,
+			   isl_union_map *&may_writes)
 {
   int i, j;
   poly_bb_p pbb;
diff --git a/gcc/graphite-isl-ast-to-gimple.c b/gcc/graphite-isl-ast-to-gimple.c
index e7d95e22110..0858672facc 100644
--- a/gcc/graphite-isl-ast-to-gimple.c
+++ b/gcc/graphite-isl-ast-to-gimple.c
@@ -56,17 +56,9 @@ along with GCC; see the file COPYING3.  If not see
 #include "cfganal.h"
 #include "value-prof.h"
 #include "tree-ssa.h"
+#include "tree-vectorizer.h"
 #include "graphite.h"
 
-/* We always try to use signed 128 bit types, but fall back to smaller types
-   in case a platform does not provide types of these sizes. In the future we
-   should use isl to derive the optimal type for each subexpression.  */
-
-static int max_mode_int_precision =
-  GET_MODE_PRECISION (int_mode_for_size (MAX_FIXED_MODE_SIZE, 0).require ());
-static int graphite_expression_type_precision = 128 <= max_mode_int_precision ?
-						128 : max_mode_int_precision;
-
 struct ast_build_info
 {
   ast_build_info()
@@ -143,8 +135,7 @@ enum phi_node_kind
 class translate_isl_ast_to_gimple
 {
  public:
-  translate_isl_ast_to_gimple (sese_info_p r)
-    : region (r), codegen_error (false) { }
+  translate_isl_ast_to_gimple (sese_info_p r);
   edge translate_isl_ast (loop_p context_loop, __isl_keep isl_ast_node *node,
 			  edge next_e, ivs_params &ip);
   edge translate_isl_ast_node_for (loop_p context_loop,
@@ -199,14 +190,12 @@ class translate_isl_ast_to_gimple
   __isl_give isl_ast_node * scop_to_isl_ast (scop_p scop);
 
   tree get_rename_from_scev (tree old_name, gimple_seq *stmts, loop_p loop,
-			     basic_block new_bb, basic_block old_bb,
 			     vec<tree> iv_map);
-  bool graphite_copy_stmts_from_block (basic_block bb, basic_block new_bb,
+  void graphite_copy_stmts_from_block (basic_block bb, basic_block new_bb,
 				       vec<tree> iv_map);
   edge copy_bb_and_scalar_dependences (basic_block bb, edge next_e,
 				       vec<tree> iv_map);
   void set_rename (tree old_name, tree expr);
-  void set_rename_for_each_def (gimple *stmt);
   void gsi_insert_earliest (gimple_seq seq);
   bool codegen_error_p () const { return codegen_error; }
 
@@ -236,8 +225,24 @@ private:
 
   /* A vector of all the edges at if_condition merge points.  */
   auto_vec<edge, 2> merge_points;
+
+  tree graphite_expr_type;
 };
 
+translate_isl_ast_to_gimple::translate_isl_ast_to_gimple (sese_info_p r)
+  : region (r), codegen_error (false)
+{
+  /* We always try to use signed 128 bit types, but fall back to smaller types
+     in case a platform does not provide types of these sizes. In the future we
+     should use isl to derive the optimal type for each subexpression.  */
+  int max_mode_int_precision
+    = GET_MODE_PRECISION (int_mode_for_size (MAX_FIXED_MODE_SIZE, 0).require ());
+  int graphite_expr_type_precision
+    = 128 <= max_mode_int_precision ?  128 : max_mode_int_precision;
+  graphite_expr_type
+    = build_nonstandard_integer_type (graphite_expr_type_precision, 0);
+}
+
 /* Return the tree variable that corresponds to the given isl ast identifier
    expression (an isl_ast_expr of type isl_ast_expr_id).
 
@@ -260,11 +265,9 @@ gcc_expression_from_isl_ast_expr_id (tree type,
 	      "Could not map isl_id to tree expression");
   isl_ast_expr_free (expr_id);
   tree t = res->second;
-  tree *val = region->parameter_rename_map->get(t);
-
-  if (!val)
-   val = &t;
-  return fold_convert (type, *val);
+  if (useless_type_conversion_p (type, TREE_TYPE (t)))
+    return t;
+  return fold_convert (type, t);
 }
 
 /* Converts an isl_ast_expr_int expression E to a widest_int.
@@ -703,8 +706,7 @@ translate_isl_ast_node_for (loop_p context_loop, __isl_keep isl_ast_node *node,
 			    edge next_e, ivs_params &ip)
 {
   gcc_assert (isl_ast_node_get_type (node) == isl_ast_node_for);
-  tree type
-    = build_nonstandard_integer_type (graphite_expression_type_precision, 0);
+  tree type = graphite_expr_type;
 
   isl_ast_expr *for_init = isl_ast_node_for_get_init (node);
   tree lb = gcc_expression_from_isl_expression (type, for_init, ip);
@@ -743,8 +745,7 @@ build_iv_mapping (vec<tree> iv_map, gimple_poly_bb_p gbb,
   for (i = 1; i < isl_ast_expr_get_op_n_arg (user_expr); i++)
     {
       arg_expr = isl_ast_expr_get_op_arg (user_expr, i);
-      tree type =
-	build_nonstandard_integer_type (graphite_expression_type_precision, 0);
+      tree type = graphite_expr_type;
       tree t = gcc_expression_from_isl_expression (type, arg_expr, ip);
 
       /* To fail code generation, we generate wrong code until we discard it.  */
@@ -791,13 +792,12 @@ translate_isl_ast_node_user (__isl_keep isl_ast_node *node,
   isl_ast_expr_free (user_expr);
 
   basic_block old_bb = GBB_BB (gbb);
-  if (dump_file)
+  if (dump_file && (dump_flags & TDF_DETAILS))
     {
       fprintf (dump_file,
 	       "[codegen] copying from bb_%d on edge (bb_%d, bb_%d)\n",
 	       old_bb->index, next_e->src->index, next_e->dest->index);
       print_loops_bb (dump_file, GBB_BB (gbb), 0, 3);
-
     }
 
   next_e = copy_bb_and_scalar_dependences (old_bb, next_e, iv_map);
@@ -807,7 +807,7 @@ translate_isl_ast_node_user (__isl_keep isl_ast_node *node,
   if (codegen_error_p ())
     return NULL;
 
-  if (dump_file)
+  if (dump_file && (dump_flags & TDF_DETAILS))
     {
       fprintf (dump_file, "[codegen] (after copy) new basic block\n");
       print_loops_bb (dump_file, next_e->src, 0, 3);
@@ -842,8 +842,7 @@ edge translate_isl_ast_to_gimple::
 graphite_create_new_guard (edge entry_edge, __isl_take isl_ast_expr *if_cond,
 			   ivs_params &ip)
 {
-  tree type =
-    build_nonstandard_integer_type (graphite_expression_type_precision, 0);
+  tree type = graphite_expr_type;
   tree cond_expr = gcc_expression_from_isl_expression (type, if_cond, ip);
 
   /* To fail code generation, we generate wrong code until we discard it.  */
@@ -933,32 +932,12 @@ set_rename (tree old_name, tree expr)
     {
       fprintf (dump_file, "[codegen] setting rename: old_name = ");
       print_generic_expr (dump_file, old_name);
-      fprintf (dump_file, ", new_name = ");
+      fprintf (dump_file, ", new decl = ");
       print_generic_expr (dump_file, expr);
       fprintf (dump_file, "\n");
     }
-
-  if (old_name == expr)
-    return;
-
-  vec <tree> *renames = region->rename_map->get (old_name);
-
-  if (renames)
-    renames->safe_push (expr);
-  else
-    {
-      vec<tree> r;
-      r.create (2);
-      r.safe_push (expr);
-      region->rename_map->put (old_name, r);
-    }
-
-  tree t;
-  int i;
-  /* For a parameter of a scop we don't want to rename it.  */
-  FOR_EACH_VEC_ELT (region->params, i, t)
-    if (old_name == t)
-      region->parameter_rename_map->put(old_name, expr);
+  bool res = region->rename_map->put (old_name, expr);
+  gcc_assert (! res);
 }
 
 /* Return an iterator to the instructions comes last in the execution order.
@@ -1070,9 +1049,9 @@ gsi_insert_earliest (gimple_seq seq)
 
       if (dump_file)
 	{
-	  fprintf (dump_file, "[codegen] inserting statement: ");
+	  fprintf (dump_file, "[codegen] inserting statement in BB %d: ",
+		   gimple_bb (use_stmt)->index);
 	  print_gimple_stmt (dump_file, use_stmt, 0, TDF_VOPS | TDF_MEMSYMS);
-	  print_loops_bb (dump_file, gimple_bb (use_stmt), 0, 3);
 	}
     }
 }
@@ -1082,7 +1061,6 @@ gsi_insert_earliest (gimple_seq seq)
 
 tree translate_isl_ast_to_gimple::
 get_rename_from_scev (tree old_name, gimple_seq *stmts, loop_p loop,
-		      basic_block new_bb, basic_block,
 		      vec<tree> iv_map)
 {
   tree scev = scalar_evolution_in_region (region->region, loop, old_name);
@@ -1111,16 +1089,6 @@ get_rename_from_scev (tree old_name, gimple_seq *stmts, loop_p loop,
       return build_zero_cst (TREE_TYPE (old_name));
     }
 
-  if (TREE_CODE (new_expr) == SSA_NAME)
-    {
-      basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (new_expr));
-      if (bb && !dominated_by_p (CDI_DOMINATORS, new_bb, bb))
-	{
-	  set_codegen_error ();
-	  return build_zero_cst (TREE_TYPE (old_name));
-	}
-    }
-
   /* Replace the old_name with the new_expr.  */
   return force_gimple_operand (unshare_expr (new_expr), stmts,
 			       true, NULL_TREE);
@@ -1148,36 +1116,13 @@ should_copy_to_new_region (gimple *stmt, sese_info_p region)
       && scev_analyzable_p (lhs, region->region))
     return false;
 
-  /* Do not copy parameters that have been generated in the header of the
-     scop.  */
-  if (is_gimple_assign (stmt)
-      && (lhs = gimple_assign_lhs (stmt))
-      && TREE_CODE (lhs) == SSA_NAME
-      && region->parameter_rename_map->get(lhs))
-    return false;
-
   return true;
 }
 
-/* Create new names for all the definitions created by COPY and add replacement
-   mappings for each new name.  */
-
-void translate_isl_ast_to_gimple::
-set_rename_for_each_def (gimple *stmt)
-{
-  def_operand_p def_p;
-  ssa_op_iter op_iter;
-  FOR_EACH_SSA_DEF_OPERAND (def_p, stmt, op_iter, SSA_OP_ALL_DEFS)
-    {
-      tree old_name = DEF_FROM_PTR (def_p);
-      create_new_def_for (old_name, stmt, def_p);
-    }
-}
-
 /* Duplicates the statements of basic block BB into basic block NEW_BB
    and compute the new induction variables according to the IV_MAP.  */
 
-bool translate_isl_ast_to_gimple::
+void translate_isl_ast_to_gimple::
 graphite_copy_stmts_from_block (basic_block bb, basic_block new_bb,
 				vec<tree> iv_map)
 {
@@ -1194,7 +1139,6 @@ graphite_copy_stmts_from_block (basic_block bb, basic_block new_bb,
       /* Create a new copy of STMT and duplicate STMT's virtual
 	 operands.  */
       gimple *copy = gimple_copy (stmt);
-      gsi_insert_after (&gsi_tgt, copy, GSI_NEW_STMT);
 
       /* Rather than not copying debug stmts we reset them.
          ???  Where we can rewrite uses without inserting new
@@ -1209,57 +1153,54 @@ graphite_copy_stmts_from_block (basic_block bb, basic_block new_bb,
 	    gcc_unreachable ();
 	}
 
-      if (dump_file)
-	{
-	  fprintf (dump_file, "[codegen] inserting statement: ");
-	  print_gimple_stmt (dump_file, copy, 0);
-	}
-
       maybe_duplicate_eh_stmt (copy, stmt);
       gimple_duplicate_stmt_histograms (cfun, copy, cfun, stmt);
 
       /* Crete new names for each def in the copied stmt.  */
-      set_rename_for_each_def (copy);
+      def_operand_p def_p;
+      ssa_op_iter op_iter;
+      FOR_EACH_SSA_DEF_OPERAND (def_p, copy, op_iter, SSA_OP_ALL_DEFS)
+	{
+	  tree old_name = DEF_FROM_PTR (def_p);
+	  create_new_def_for (old_name, copy, def_p);
+	}
 
-      if (codegen_error_p ())
-	return false;
+      gsi_insert_after (&gsi_tgt, copy, GSI_NEW_STMT);
+      if (dump_file)
+	{
+	  fprintf (dump_file, "[codegen] inserting statement: ");
+	  print_gimple_stmt (dump_file, copy, 0);
+	}
 
-      /* For each SSA_NAME in the parameter_rename_map rename their usage.  */
+      /* For each SCEV analyzable SSA_NAME, rename their usage.  */
       ssa_op_iter iter;
       use_operand_p use_p;
       if (!is_gimple_debug (copy))
-	FOR_EACH_SSA_USE_OPERAND (use_p, copy, iter, SSA_OP_USE)
-	  {
-	    tree old_name = USE_FROM_PTR (use_p);
-
-	    if (TREE_CODE (old_name) != SSA_NAME
-		|| SSA_NAME_IS_DEFAULT_DEF (old_name))
-	      continue;
-
-	    tree *new_expr = region->parameter_rename_map->get (old_name);
-	    tree new_name;
-	    if (!new_expr
-		&& scev_analyzable_p (old_name, region->region))
-	      {
-		gimple_seq stmts = NULL;
-		new_name = get_rename_from_scev (old_name, &stmts,
-						 bb->loop_father,
-						 new_bb, bb, iv_map);
-		if (! codegen_error_p ())
-		  gsi_insert_earliest (stmts);
-		new_expr = &new_name;
-	      }
-
-	    if (!new_expr)
-	      continue;
-
-	    replace_exp (use_p, *new_expr);
-	  }
+	{
+	  bool changed = false;
+	  FOR_EACH_SSA_USE_OPERAND (use_p, copy, iter, SSA_OP_USE)
+	    {
+	      tree old_name = USE_FROM_PTR (use_p);
+
+	      if (TREE_CODE (old_name) != SSA_NAME
+		  || SSA_NAME_IS_DEFAULT_DEF (old_name)
+		  || ! scev_analyzable_p (old_name, region->region))
+		continue;
+
+	      gimple_seq stmts = NULL;
+	      tree new_name = get_rename_from_scev (old_name, &stmts,
+						    bb->loop_father, iv_map);
+	      if (! codegen_error_p ())
+		gsi_insert_earliest (stmts);
+	      replace_exp (use_p, new_name);
+	      changed = true;
+	    }
+	  if (changed)
+	    fold_stmt_inplace (&gsi_tgt);
+	}
 
       update_stmt (copy);
     }
-
-  return true;
 }
 
 
@@ -1282,39 +1223,21 @@ copy_bb_and_scalar_dependences (basic_block bb, edge next_e, vec<tree> iv_map)
 	continue;
 
       tree new_phi_def;
-      vec <tree> *renames = region->rename_map->get (res);
-      if (! renames || renames->is_empty ())
+      tree *rename = region->rename_map->get (res);
+      if (! rename)
 	{
 	  new_phi_def = create_tmp_reg (TREE_TYPE (res));
 	  set_rename (res, new_phi_def);
 	}
       else
-	{
-	  gcc_assert (renames->length () == 1);
-	  new_phi_def = (*renames)[0];
-	}
+	new_phi_def = *rename;
 
       gassign *ass = gimple_build_assign (NULL_TREE, new_phi_def);
       create_new_def_for (res, ass, NULL);
       gsi_insert_after (&gsi_tgt, ass, GSI_NEW_STMT);
     }
 
-  vec <basic_block> *copied_bbs = region->copied_bb_map->get (bb);
-  if (copied_bbs)
-    copied_bbs->safe_push (new_bb);
-  else
-    {
-      vec<basic_block> bbs;
-      bbs.create (2);
-      bbs.safe_push (new_bb);
-      region->copied_bb_map->put (bb, bbs);
-    }
-
-  if (!graphite_copy_stmts_from_block (bb, new_bb, iv_map))
-    {
-      set_codegen_error ();
-      return NULL;
-    }
+  graphite_copy_stmts_from_block (bb, new_bb, iv_map);
 
   /* Insert out-of SSA copies on the original BB outgoing edges.  */
   gsi_tgt = gsi_last_bb (new_bb);
@@ -1340,17 +1263,14 @@ copy_bb_and_scalar_dependences (basic_block bb, edge next_e, vec<tree> iv_map)
 		continue;
 
 	      tree new_phi_def;
-	      vec <tree> *renames = region->rename_map->get (res);
-	      if (! renames || renames->is_empty ())
+	      tree *rename = region->rename_map->get (res);
+	      if (! rename)
 		{
 		  new_phi_def = create_tmp_reg (TREE_TYPE (res));
 		  set_rename (res, new_phi_def);
 		}
 	      else
-		{
-		  gcc_assert (renames->length () == 1);
-		  new_phi_def = (*renames)[0];
-		}
+		new_phi_def = *rename;
 
 	      tree arg = PHI_ARG_DEF_FROM_EDGE (phi, e);
 	      if (TREE_CODE (arg) == SSA_NAME
@@ -1359,7 +1279,7 @@ copy_bb_and_scalar_dependences (basic_block bb, edge next_e, vec<tree> iv_map)
 		  gimple_seq stmts = NULL;
 		  tree new_name = get_rename_from_scev (arg, &stmts,
 							bb->loop_father,
-							new_bb, bb, iv_map);
+							iv_map);
 		  if (! codegen_error_p ())
 		    gsi_insert_earliest (stmts);
 		  arg = new_name;
@@ -1385,13 +1305,14 @@ add_parameters_to_ivs_params (scop_p scop, ivs_params &ip)
 {
   sese_info_p region = scop->scop_info;
   unsigned nb_parameters = isl_set_dim (scop->param_context, isl_dim_param);
-  gcc_assert (nb_parameters == region->params.length ());
+  gcc_assert (nb_parameters == sese_nb_params (region));
   unsigned i;
-  for (i = 0; i < nb_parameters; i++)
+  tree param;
+  FOR_EACH_VEC_ELT (region->params, i, param)
     {
       isl_id *tmp_id = isl_set_get_dim_id (scop->param_context,
 					   isl_dim_param, i);
-      ip[tmp_id] = region->params[i];
+      ip[tmp_id] = param;
     }
 }
 
@@ -1427,6 +1348,13 @@ ast_build_before_for (__isl_keep isl_ast_build *build, void *user)
 __isl_give isl_ast_node *translate_isl_ast_to_gimple::
 scop_to_isl_ast (scop_p scop)
 {
+  int old_err = isl_options_get_on_error (scop->isl_context);
+  int old_max_operations = isl_ctx_get_max_operations (scop->isl_context);
+  int max_operations = PARAM_VALUE (PARAM_MAX_ISL_OPERATIONS);
+  if (max_operations)
+    isl_ctx_set_max_operations (scop->isl_context, max_operations);
+  isl_options_set_on_error (scop->isl_context, ISL_ON_ERROR_CONTINUE);
+
   gcc_assert (scop->transformed_schedule);
 
   /* Set the separate option to reduce control flow overhead.  */
@@ -1445,70 +1373,56 @@ scop_to_isl_ast (scop_p scop)
   isl_ast_node *ast_isl = isl_ast_build_node_from_schedule
     (context_isl, schedule);
   isl_ast_build_free (context_isl);
-  return ast_isl;
-}
-
-/* Copy def from sese REGION to the newly created TO_REGION. TR is defined by
-   DEF_STMT. GSI points to entry basic block of the TO_REGION.  */
-
-static void
-copy_def (tree tr, gimple *def_stmt, sese_info_p region, sese_info_p to_region,
-	  gimple_stmt_iterator *gsi)
-{
-  if (!defined_in_sese_p (tr, region->region))
-    return;
 
-  ssa_op_iter iter;
-  use_operand_p use_p;
-  FOR_EACH_SSA_USE_OPERAND (use_p, def_stmt, iter, SSA_OP_USE)
+  isl_options_set_on_error (scop->isl_context, old_err);
+  isl_ctx_reset_operations (scop->isl_context);
+  isl_ctx_set_max_operations (scop->isl_context, old_max_operations);
+  if (isl_ctx_last_error (scop->isl_context) != isl_error_none)
     {
-      tree use_tr = USE_FROM_PTR (use_p);
-
-      /* Do not copy parameters that have been generated in the header of the
-	 scop.  */
-      if (region->parameter_rename_map->get(use_tr))
-	continue;
-
-      gimple *def_of_use = SSA_NAME_DEF_STMT (use_tr);
-      if (!def_of_use)
-	continue;
-
-      copy_def (use_tr, def_of_use, region, to_region, gsi);
-    }
-
-  gimple *copy = gimple_copy (def_stmt);
-  gsi_insert_after (gsi, copy, GSI_NEW_STMT);
-
-  /* Create new names for all the definitions created by COPY and
-     add replacement mappings for each new name.  */
-  def_operand_p def_p;
-  ssa_op_iter op_iter;
-  FOR_EACH_SSA_DEF_OPERAND (def_p, copy, op_iter, SSA_OP_ALL_DEFS)
-    {
-      tree old_name = DEF_FROM_PTR (def_p);
-      tree new_name = create_new_def_for (old_name, copy, def_p);
-      region->parameter_rename_map->put(old_name, new_name);
+      location_t loc = find_loop_location
+	(scop->scop_info->region.entry->dest->loop_father);
+      if (isl_ctx_last_error (scop->isl_context) == isl_error_quota)
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, loc,
+			 "loop nest not optimized, AST generation timed out "
+			 "after %d operations [--param max-isl-operations]\n",
+			 max_operations);
+      else
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, loc,
+			 "loop nest not optimized, ISL AST generation "
+			 "signalled an error\n");
+      isl_ast_node_free (ast_isl);
+      return NULL;
     }
 
-  update_stmt (copy);
+  return ast_isl;
 }
 
+/* Generate out-of-SSA copies for the entry edge FALSE_ENTRY/TRUE_ENTRY
+   in REGION.  */
+
 static void
-copy_internal_parameters (sese_info_p region, sese_info_p to_region)
+generate_entry_out_of_ssa_copies (edge false_entry,
+				  edge true_entry,
+				  sese_info_p region)
 {
-  /* For all the parameters which definitino is in the if_region->false_region,
-     insert code on true_region (if_region->true_region->entry). */
-
-  int i;
-  tree tr;
-  gimple_stmt_iterator gsi = gsi_start_bb(to_region->region.entry->dest);
-
-  FOR_EACH_VEC_ELT (region->params, i, tr)
+  gimple_stmt_iterator gsi_tgt = gsi_start_bb (true_entry->dest);
+  for (gphi_iterator psi = gsi_start_phis (false_entry->dest);
+       !gsi_end_p (psi); gsi_next (&psi))
     {
-      // If def is not in region.
-      gimple *def_stmt = SSA_NAME_DEF_STMT (tr);
-      if (def_stmt)
-	copy_def (tr, def_stmt, region, to_region, &gsi);
+      gphi *phi = psi.phi ();
+      tree res = gimple_phi_result (phi);
+      if (virtual_operand_p (res))
+	continue;
+      /* When there's no out-of-SSA var registered do not bother
+         to create one.  */
+      tree *rename = region->rename_map->get (res);
+      if (! rename)
+	continue;
+      tree new_phi_def = *rename;
+      gassign *ass = gimple_build_assign (new_phi_def,
+					  PHI_ARG_DEF_FROM_EDGE (phi,
+								 false_entry));
+      gsi_insert_after (&gsi_tgt, ass, GSI_NEW_STMT);
     }
 }
 
@@ -1528,6 +1442,12 @@ graphite_regenerate_ast_isl (scop_p scop)
   timevar_push (TV_GRAPHITE_CODE_GEN);
   t.add_parameters_to_ivs_params (scop, ip);
   root_node = t.scop_to_isl_ast (scop);
+  if (! root_node)
+    {
+      ivs_params_clear (ip);
+      timevar_pop (TV_GRAPHITE_CODE_GEN);
+      return false;
+    }
 
   if (dump_file && (dump_flags & TDF_DETAILS))
     {
@@ -1546,10 +1466,6 @@ graphite_regenerate_ast_isl (scop_p scop)
   region->if_region = if_region;
 
   loop_p context_loop = region->region.entry->src->loop_father;
-
-  /* Copy all the parameters which are defined in the region.  */
-  copy_internal_parameters(if_region->false_region, if_region->true_region);
-
   edge e = single_succ_edge (if_region->true_region->region.entry->dest);
   basic_block bb = split_edge (e);
 
@@ -1559,35 +1475,24 @@ graphite_regenerate_ast_isl (scop_p scop)
   t.translate_isl_ast (context_loop, root_node, e, ip);
   if (! t.codegen_error_p ())
     {
+      generate_entry_out_of_ssa_copies (if_region->false_region->region.entry,
+					if_region->true_region->region.entry,
+					region);
       sese_insert_phis_for_liveouts (region,
 				     if_region->region->region.exit->src,
 				     if_region->false_region->region.exit,
 				     if_region->true_region->region.exit);
       if (dump_file)
 	fprintf (dump_file, "[codegen] isl AST to Gimple succeeded.\n");
-
-      mark_virtual_operands_for_renaming (cfun);
-      update_ssa (TODO_update_ssa);
-      checking_verify_ssa (true, true);
-      rewrite_into_loop_closed_ssa (NULL, 0);
-      /* We analyzed evolutions of all SCOPs during SCOP detection
-         which cached evolutions.  Now we've introduced PHIs for
-	 liveouts which causes those cached solutions to be invalid
-	 for code-generation purposes given we'd insert references
-	 to SSA names not dominating their new use.  */
-      scev_reset ();
     }
 
   if (t.codegen_error_p ())
     {
-      if (dump_file)
-	fprintf (dump_file, "codegen error: "
-		 "reverting back to the original code.\n");
-      set_ifsese_condition (if_region, integer_zero_node);
+      location_t loc = find_loop_location
+	(scop->scop_info->region.entry->dest->loop_father);
+      dump_printf_loc (MSG_MISSED_OPTIMIZATION, loc,
+		       "loop nest not optimized, code generation error\n");
 
-      /* We registered new names, scrap that.  */
-      if (need_ssa_update_p (cfun))
-	delete_update_ssa ();
       /* Remove the unreachable region.  */
       remove_edge_and_dominated_blocks (if_region->true_region->region.entry);
       basic_block ifb = if_region->false_region->region.entry->src;
@@ -1603,9 +1508,11 @@ graphite_regenerate_ast_isl (scop_p scop)
 	  delete_loop (loop);
     }
 
-  /* Verifies properties that GRAPHITE should maintain during translation.  */
-  checking_verify_loop_structure ();
-  checking_verify_loop_closed_ssa (true);
+  /* We are delaying SSA update to after code-generating all SCOPs.
+     This is because we analyzed DRs and parameters on the unmodified
+     IL and thus rely on SSA update to pick up new dominating definitions
+     from for example SESE liveout PHIs.  This is also for efficiency
+     as SSA update does work depending on the size of the function.  */
 
   free (if_region->true_region);
   free (if_region->region);
diff --git a/gcc/graphite-scop-detection.c b/gcc/graphite-scop-detection.c
index f9d69247b0c..1bef380b32a 100644
--- a/gcc/graphite-scop-detection.c
+++ b/gcc/graphite-scop-detection.c
@@ -1005,15 +1005,10 @@ scop_detection::graphite_can_represent_expr (sese_l scop, loop_p loop,
 bool
 scop_detection::stmt_has_simple_data_refs_p (sese_l scop, gimple *stmt)
 {
-  edge nest;
+  edge nest = scop.entry;;
   loop_p loop = loop_containing_stmt (stmt);
   if (!loop_in_sese_p (loop, scop))
-    {
-      nest = scop.entry;
-      loop = NULL;
-    }
-  else
-    nest = loop_preheader_edge (outermost_loop_in_sese (scop, gimple_bb (stmt)));
+    loop = NULL;
 
   auto_vec<data_reference_p> drs;
   if (! graphite_find_data_references_in_stmt (nest, loop, stmt, &drs))
@@ -1108,7 +1103,7 @@ scop_detection::stmt_simple_for_scop_p (sese_l scop, gimple *stmt,
 	    tree op = gimple_op (stmt, i);
 	    if (!graphite_can_represent_expr (scop, loop, op)
 		/* We can only constrain on integer type.  */
-		|| (TREE_CODE (TREE_TYPE (op)) != INTEGER_TYPE))
+		|| ! INTEGRAL_TYPE_P (TREE_TYPE (op)))
 	      {
 		DEBUG_PRINT (dp << "[scop-detection-fail] "
 				<< "Graphite cannot represent stmt:\n";
@@ -1151,49 +1146,23 @@ scop_detection::nb_pbbs_in_loops (scop_p scop)
   return res;
 }
 
-/* When parameter NAME is in REGION, returns its index in SESE_PARAMS.
-   Otherwise returns -1.  */
+/* Assigns the parameter NAME an index in REGION.  */
 
-static inline int
-parameter_index_in_region_1 (tree name, sese_info_p region)
+static void
+assign_parameter_index_in_region (tree name, sese_info_p region)
 {
+  gcc_assert (TREE_CODE (name) == SSA_NAME
+	      && INTEGRAL_TYPE_P (TREE_TYPE (name))
+	      && ! defined_in_sese_p (name, region->region));
+
   int i;
   tree p;
-
-  gcc_assert (TREE_CODE (name) == SSA_NAME);
-
   FOR_EACH_VEC_ELT (region->params, i, p)
     if (p == name)
-      return i;
-
-  return -1;
-}
-
-/* When the parameter NAME is in REGION, returns its index in
-   SESE_PARAMS.  Otherwise this function inserts NAME in SESE_PARAMS
-   and returns the index of NAME.  */
-
-static int
-parameter_index_in_region (tree name, sese_info_p region)
-{
-  int i;
-
-  gcc_assert (TREE_CODE (name) == SSA_NAME);
-
-  /* Cannot constrain on anything else than INTEGER_TYPE parameters.  */
-  if (TREE_CODE (TREE_TYPE (name)) != INTEGER_TYPE)
-    return -1;
-
-  if (!invariant_in_sese_p_rec (name, region->region, NULL))
-    return -1;
-
-  i = parameter_index_in_region_1 (name, region);
-  if (i != -1)
-    return i;
+      return;
 
   i = region->params.length ();
   region->params.safe_push (name);
-  return i;
 }
 
 /* In the context of sese S, scan the expression E and translate it to
@@ -1235,7 +1204,7 @@ scan_tree_for_params (sese_info_p s, tree e)
       break;
 
     case SSA_NAME:
-      parameter_index_in_region (e, s);
+      assign_parameter_index_in_region (e, s);
       break;
 
     case INTEGER_CST:
@@ -1383,15 +1352,10 @@ try_generate_gimple_bb (scop_p scop, basic_block bb)
   vec<scalar_use> reads = vNULL;
 
   sese_l region = scop->scop_info->region;
-  edge nest;
+  edge nest = region.entry;
   loop_p loop = bb->loop_father;
   if (!loop_in_sese_p (loop, region))
-    {
-      nest = region.entry;
-      loop = NULL;
-    }
-  else
-    nest = loop_preheader_edge (outermost_loop_in_sese (region, bb));
+    loop = NULL;
 
   for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
        gsi_next (&gsi))
@@ -1710,10 +1674,6 @@ build_scops (vec<scop_p> *scops)
   sese_l *s;
   FOR_EACH_VEC_ELT (scops_l, i, s)
     {
-      /* For our out-of-SSA we need a block on s->entry, similar to how
-         we include the LCSSA block in the region.  */
-      s->entry = single_pred_edge (split_edge (s->entry));
-
       scop_p scop = new_scop (s->entry, s->exit);
 
       /* Record all basic blocks and their conditions in REGION.  */
diff --git a/gcc/graphite-sese-to-poly.c b/gcc/graphite-sese-to-poly.c
index ed6cbeccca1..248c34a41c9 100644
--- a/gcc/graphite-sese-to-poly.c
+++ b/gcc/graphite-sese-to-poly.c
@@ -142,11 +142,8 @@ isl_id_for_dr (scop_p s)
 /* Extract an affine expression from the ssa_name E.  */
 
 static isl_pw_aff *
-extract_affine_name (scop_p s, tree e, __isl_take isl_space *space)
+extract_affine_name (int dimension, __isl_take isl_space *space)
 {
-  isl_id *id = isl_id_for_ssa_name (s, e);
-  int dimension = isl_space_find_dim_by_id (space, isl_dim_param, id);
-  isl_id_free (id);
   isl_set *dom = isl_set_universe (isl_space_copy (space));
   isl_aff *aff = isl_aff_zero_on_domain (isl_local_space_from_space (space));
   aff = isl_aff_add_coefficient_si (aff, isl_dim_param, dimension, 1);
@@ -211,17 +208,13 @@ wrap (isl_pw_aff *pwaff, unsigned width)
    Otherwise returns -1.  */
 
 static inline int
-parameter_index_in_region_1 (tree name, sese_info_p region)
+parameter_index_in_region (tree name, sese_info_p region)
 {
   int i;
   tree p;
-
-  gcc_assert (TREE_CODE (name) == SSA_NAME);
-
   FOR_EACH_VEC_ELT (region->params, i, p)
     if (p == name)
       return i;
-
   return -1;
 }
 
@@ -288,10 +281,13 @@ extract_affine (scop_p s, tree e, __isl_take isl_space *space)
       break;
 
     case SSA_NAME:
-      gcc_assert (-1 != parameter_index_in_region_1 (e, s->scop_info)
-		  || defined_in_sese_p (e, s->scop_info->region));
-      res = extract_affine_name (s, e, space);
-      break;
+      {
+	gcc_assert (! defined_in_sese_p (e, s->scop_info->region));
+	int dim = parameter_index_in_region (e, s->scop_info);
+	gcc_assert (dim != -1);
+	res = extract_affine_name (dim, space);
+	break;
+      }
 
     case INTEGER_CST:
       res = extract_affine_int (e, space);
@@ -431,54 +427,40 @@ add_conditions_to_domain (poly_bb_p pbb)
    of P.  */
 
 static void
-add_param_constraints (scop_p scop, graphite_dim_t p)
+add_param_constraints (scop_p scop, graphite_dim_t p, tree parameter)
 {
-  tree parameter = scop->scop_info->params[p];
   tree type = TREE_TYPE (parameter);
-  tree lb = NULL_TREE;
-  tree ub = NULL_TREE;
+  wide_int min, max;
 
-  if (POINTER_TYPE_P (type) || !TYPE_MIN_VALUE (type))
-    lb = lower_bound_in_type (type, type);
-  else
-    lb = TYPE_MIN_VALUE (type);
+  gcc_assert (INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type));
 
-  if (POINTER_TYPE_P (type) || !TYPE_MAX_VALUE (type))
-    ub = upper_bound_in_type (type, type);
+  if (INTEGRAL_TYPE_P (type)
+      && get_range_info (parameter, &min, &max) == VR_RANGE)
+    ;
   else
-    ub = TYPE_MAX_VALUE (type);
-
-  if (lb)
     {
-      isl_space *space = isl_set_get_space (scop->param_context);
-      isl_constraint *c;
-      isl_val *v;
-
-      c = isl_inequality_alloc (isl_local_space_from_space (space));
-      v = isl_val_int_from_wi (scop->isl_context, wi::to_widest (lb));
-      v = isl_val_neg (v);
-      c = isl_constraint_set_constant_val (c, v);
-      c = isl_constraint_set_coefficient_si (c, isl_dim_param, p, 1);
-
-      scop->param_context = isl_set_coalesce
-	(isl_set_add_constraint (scop->param_context, c));
+      min = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
+      max = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
     }
 
-  if (ub)
-    {
-      isl_space *space = isl_set_get_space (scop->param_context);
-      isl_constraint *c;
-      isl_val *v;
-
-      c = isl_inequality_alloc (isl_local_space_from_space (space));
-
-      v = isl_val_int_from_wi (scop->isl_context, wi::to_widest (ub));
-      c = isl_constraint_set_constant_val (c, v);
-      c = isl_constraint_set_coefficient_si (c, isl_dim_param, p, -1);
-
-      scop->param_context = isl_set_coalesce
-	(isl_set_add_constraint (scop->param_context, c));
-    }
+  isl_space *space = isl_set_get_space (scop->param_context);
+  isl_constraint *c = isl_inequality_alloc (isl_local_space_from_space (space));
+  isl_val *v = isl_val_int_from_wi (scop->isl_context,
+				    widest_int::from (min, TYPE_SIGN (type)));
+  v = isl_val_neg (v);
+  c = isl_constraint_set_constant_val (c, v);
+  c = isl_constraint_set_coefficient_si (c, isl_dim_param, p, 1);
+  scop->param_context = isl_set_coalesce
+      (isl_set_add_constraint (scop->param_context, c));
+
+  space = isl_set_get_space (scop->param_context);
+  c = isl_inequality_alloc (isl_local_space_from_space (space));
+  v = isl_val_int_from_wi (scop->isl_context,
+			   widest_int::from (max, TYPE_SIGN (type)));
+  c = isl_constraint_set_constant_val (c, v);
+  c = isl_constraint_set_coefficient_si (c, isl_dim_param, p, -1);
+  scop->param_context = isl_set_coalesce
+      (isl_set_add_constraint (scop->param_context, c));
 }
 
 /* Add a constrain to the ACCESSES polyhedron for the alias set of
@@ -930,9 +912,8 @@ build_scop_context (scop_p scop)
 
   scop->param_context = isl_set_universe (space);
 
-  graphite_dim_t p;
-  for (p = 0; p < nbp; p++)
-    add_param_constraints (scop, p);
+  FOR_EACH_VEC_ELT (region->params, i, e)
+    add_param_constraints (scop, i, e);
 }
 
 /* Return true when loop A is nested in loop B.  */
@@ -1194,7 +1175,7 @@ build_schedule_loop_nest (scop_p scop, int *index, loop_p context_loop)
 
 /* Build the schedule of the SCOP.  */
 
-static bool
+static void
 build_original_schedule (scop_p scop)
 {
   int i = 0;
@@ -1216,9 +1197,6 @@ build_original_schedule (scop_p scop)
       fprintf (dump_file, "[sese-to-poly] original schedule:\n");
       print_isl_schedule (dump_file, scop->original_schedule);
     }
-  if (!scop->original_schedule)
-    return false;
-  return true;
 }
 
 /* Builds the polyhedral representation for a SESE region.  */
diff --git a/gcc/graphite.c b/gcc/graphite.c
index 0bdcc28cba8..22d83307bd2 100644
--- a/gcc/graphite.c
+++ b/gcc/graphite.c
@@ -55,6 +55,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-cfgcleanup.h"
 #include "tree-vectorizer.h"
 #include "tree-ssa-loop-manip.h"
+#include "tree-ssa.h"
+#include "tree-into-ssa.h"
 #include "graphite.h"
 
 /* Print global statistics to FILE.  */
@@ -109,7 +111,7 @@ print_global_statistics (FILE* file)
   fprintf (file, "LOOPS:%ld, ", n_loops);
   fprintf (file, "CONDITIONS:%ld, ", n_conditions);
   fprintf (file, "STMTS:%ld)\n", n_stmts);
-  fprintf (file, "\nGlobal profiling statistics (");
+  fprintf (file, "Global profiling statistics (");
   fprintf (file, "BBS:");
   n_p_bbs.dump (file);
   fprintf (file, ", LOOPS:");
@@ -118,7 +120,7 @@ print_global_statistics (FILE* file)
   n_p_conditions.dump (file);
   fprintf (file, ", STMTS:");
   n_p_stmts.dump (file);
-  fprintf (file, ")\n");
+  fprintf (file, ")\n\n");
 }
 
 /* Print statistics for SCOP to FILE.  */
@@ -183,7 +185,7 @@ print_graphite_scop_statistics (FILE* file, scop_p scop)
   fprintf (file, "LOOPS:%ld, ", n_loops);
   fprintf (file, "CONDITIONS:%ld, ", n_conditions);
   fprintf (file, "STMTS:%ld)\n", n_stmts);
-  fprintf (file, "\nSCoP profiling statistics (");
+  fprintf (file, "SCoP profiling statistics (");
   fprintf (file, "BBS:");
   n_p_bbs.dump (file);
   fprintf (file, ", LOOPS:");
@@ -192,7 +194,7 @@ print_graphite_scop_statistics (FILE* file, scop_p scop)
   n_p_conditions.dump (file);
   fprintf (file, ", STMTS:");
   n_p_stmts.dump (file);
-  fprintf (file, ")\n");
+  fprintf (file, ")\n\n");
 }
 
 /* Print statistics for SCOPS to FILE.  */
@@ -201,73 +203,10 @@ static void
 print_graphite_statistics (FILE* file, vec<scop_p> scops)
 {
   int i;
-
   scop_p scop;
 
   FOR_EACH_VEC_ELT (scops, i, scop)
     print_graphite_scop_statistics (file, scop);
-
-  /* Print the loop structure.  */
-  print_loops (file, 2);
-  print_loops (file, 3);
-}
-
-/* Initialize graphite: when there are no loops returns false.  */
-
-static bool
-graphite_initialize (void)
-{
-  int min_loops = PARAM_VALUE (PARAM_GRAPHITE_MIN_LOOPS_PER_FUNCTION);
-  int nloops = number_of_loops (cfun);
-
-  if (nloops <= min_loops)
-    {
-      if (dump_file && (dump_flags & TDF_DETAILS))
-	{
-	  if (nloops <= min_loops)
-	    fprintf (dump_file, "\nFunction does not have enough loops: "
-		     "PARAM_GRAPHITE_MIN_LOOPS_PER_FUNCTION = %d.\n",
-		     min_loops);
-
-	  fprintf (dump_file, "\nnumber of SCoPs: 0\n");
-	  print_global_statistics (dump_file);
-	}
-
-      return false;
-    }
-
-  calculate_dominance_info (CDI_DOMINATORS);
-  initialize_original_copy_tables ();
-
-  if (dump_file && dump_flags)
-    {
-      dump_function_to_file (current_function_decl, dump_file, dump_flags);
-      print_loops (dump_file, 3);
-    }
-
-  return true;
-}
-
-/* Finalize graphite: perform CFG cleanup when NEED_CFG_CLEANUP_P is
-   true.  */
-
-static void
-graphite_finalize (bool need_cfg_cleanup_p)
-{
-  if (need_cfg_cleanup_p)
-    {
-      free_dominance_info (CDI_DOMINATORS);
-      scev_reset ();
-      cleanup_tree_cfg ();
-      profile_status_for_fn (cfun) = PROFILE_ABSENT;
-      release_recorded_exits (cfun);
-      tree_estimate_probability (false);
-    }
-
-  free_original_copy_tables ();
-
-  if (dump_file && dump_flags)
-    print_loops (dump_file, 3);
 }
 
 /* Deletes all scops in SCOPS.  */
@@ -396,7 +335,7 @@ graphite_transform_loops (void)
 {
   int i;
   scop_p scop;
-  bool need_cfg_cleanup_p = false;
+  bool changed = false;
   vec<scop_p> scops = vNULL;
   isl_ctx *ctx;
 
@@ -405,8 +344,7 @@ graphite_transform_loops (void)
   if (parallelized_function_p (cfun->decl))
     return;
 
-  if (!graphite_initialize ())
-    return;
+  calculate_dominance_info (CDI_DOMINATORS);
 
   ctx = isl_ctx_alloc ();
   isl_options_set_on_error (ctx, ISL_ON_ERROR_ABORT);
@@ -415,6 +353,13 @@ graphite_transform_loops (void)
   sort_sibling_loops (cfun);
   canonicalize_loop_closed_ssa_form ();
 
+  /* Print the loop structure.  */
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      print_loops (dump_file, 2);
+      print_loops (dump_file, 3);
+    }
+
   calculate_dominance_info (CDI_POST_DOMINATORS);
   build_scops (&scops);
   free_dominance_info (CDI_POST_DOMINATORS);
@@ -435,18 +380,26 @@ graphite_transform_loops (void)
 	if (!apply_poly_transforms (scop))
 	  continue;
 
-	location_t loc = find_loop_location
-	  (scops[i]->scop_info->region.entry->dest->loop_father);
-
-	need_cfg_cleanup_p = true;
-	if (!graphite_regenerate_ast_isl (scop))
-	  dump_printf_loc (MSG_MISSED_OPTIMIZATION, loc,
-			   "loop nest not optimized, code generation error\n");
-	else
-	  dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
-			   "loop nest optimized\n");
+	changed = true;
+	if (graphite_regenerate_ast_isl (scop))
+	  {
+	    location_t loc = find_loop_location
+	      (scops[i]->scop_info->region.entry->dest->loop_father);
+	    dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
+			     "loop nest optimized\n");
+	  }
       }
 
+  if (changed)
+    {
+      mark_virtual_operands_for_renaming (cfun);
+      update_ssa (TODO_update_ssa);
+      checking_verify_ssa (true, true);
+      rewrite_into_loop_closed_ssa (NULL, 0);
+      scev_reset ();
+      checking_verify_loop_structure ();
+    }
+
   if (dump_file && (dump_flags & TDF_DETAILS))
     {
       loop_p loop;
@@ -461,9 +414,17 @@ graphite_transform_loops (void)
     }
 
   free_scops (scops);
-  graphite_finalize (need_cfg_cleanup_p);
   the_isl_ctx = NULL;
   isl_ctx_free (ctx);
+
+  if (changed)
+    {
+      cleanup_tree_cfg ();
+      profile_status_for_fn (cfun) = PROFILE_ABSENT;
+      release_recorded_exits (cfun);
+      tree_estimate_probability (false);
+    }
+
 }
 
 #else /* If isl is not available: #ifndef HAVE_isl.  */
diff --git a/gcc/haifa-sched.c b/gcc/haifa-sched.c
index d6dab57101b..f5c06a95bb6 100644
--- a/gcc/haifa-sched.c
+++ b/gcc/haifa-sched.c
@@ -3084,8 +3084,7 @@ ready_sort_real (struct ready_list *ready)
   if (n_ready_real == 2)
     swap_sort (first, n_ready_real);
   else if (n_ready_real > 2)
-    /* HACK: Disable qsort checking for now (PR82396).  */
-    (qsort) (first, n_ready_real, sizeof (rtx), rank_for_schedule);
+    qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
 
   if (sched_verbose >= 4)
     {
@@ -3918,8 +3917,8 @@ sched_pressure_start_bb (basic_block bb)
       - call_saved_regs_num[cl]).  */
   {
     int i;
-    int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
-    int bb_freq = bb->frequency;
+    int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
+    int bb_freq = bb->count.to_frequency (cfun);
 
     if (bb_freq == 0)
       {
@@ -5569,9 +5568,7 @@ autopref_multipass_init (const rtx_insn *insn, int write)
 
   gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
   data->base = NULL_RTX;
-  data->min_offset = 0;
-  data->max_offset = 0;
-  data->multi_mem_insn_p = false;
+  data->offset = 0;
   /* Set insn entry initialized, but not relevant for auto-prefetcher.  */
   data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
 
@@ -5586,10 +5583,9 @@ autopref_multipass_init (const rtx_insn *insn, int write)
     {
       int n_elems = XVECLEN (pat, 0);
 
-      int i = 0;
-      rtx prev_base = NULL_RTX;
-      int min_offset = 0;
-      int max_offset = 0;
+      int i, offset;
+      rtx base, prev_base = NULL_RTX;
+      int min_offset = INT_MAX;
 
       for (i = 0; i < n_elems; i++)
 	{
@@ -5597,38 +5593,23 @@ autopref_multipass_init (const rtx_insn *insn, int write)
 	  if (GET_CODE (set) != SET)
 	    return;
 
-	  rtx base = NULL_RTX;
-	  int offset = 0;
 	  if (!analyze_set_insn_for_autopref (set, write, &base, &offset))
 	    return;
 
-	  if (i == 0)
-	    {
-	      prev_base = base;
-	      min_offset = offset;
-	      max_offset = offset;
-	    }
 	  /* Ensure that all memory operations in the PARALLEL use the same
 	     base register.  */
-	  else if (REGNO (base) != REGNO (prev_base))
+	  if (i > 0 && REGNO (base) != REGNO (prev_base))
 	    return;
-	  else
-	    {
-	      min_offset = MIN (min_offset, offset);
-	      max_offset = MAX (max_offset, offset);
-	    }
+	  prev_base = base;
+	  min_offset = MIN (min_offset, offset);
 	}
 
-      /* If we reached here then we have a valid PARALLEL of multiple memory
-	 ops with prev_base as the base and min_offset and max_offset
-	 containing the offsets range.  */
+      /* If we reached here then we have a valid PARALLEL of multiple memory ops
+	 with prev_base as the base and min_offset containing the offset.  */
       gcc_assert (prev_base);
       data->base = prev_base;
-      data->min_offset = min_offset;
-      data->max_offset = max_offset;
-      data->multi_mem_insn_p = true;
+      data->offset = min_offset;
       data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
-
       return;
     }
 
@@ -5638,7 +5619,7 @@ autopref_multipass_init (const rtx_insn *insn, int write)
     return;
 
   if (!analyze_set_insn_for_autopref (set, write, &data->base,
-				       &data->min_offset))
+				       &data->offset))
     return;
 
   /* This insn is relevant for the auto-prefetcher.
@@ -5647,63 +5628,6 @@ autopref_multipass_init (const rtx_insn *insn, int write)
   data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
 }
 
-
-/* Helper for autopref_rank_for_schedule.  Given the data of two
-   insns relevant to the auto-prefetcher modelling code DATA1 and DATA2
-   return their comparison result.  Return 0 if there is no sensible
-   ranking order for the two insns.  */
-
-static int
-autopref_rank_data (autopref_multipass_data_t data1,
-		     autopref_multipass_data_t data2)
-{
-  /* Simple case when both insns are simple single memory ops.  */
-  if (!data1->multi_mem_insn_p && !data2->multi_mem_insn_p)
-    return data1->min_offset - data2->min_offset;
-
-  /* Two load/store multiple insns.  Return 0 if the offset ranges
-     overlap and the difference between the minimum offsets otherwise.  */
-  else if (data1->multi_mem_insn_p && data2->multi_mem_insn_p)
-    {
-      int min1 = data1->min_offset;
-      int max1 = data1->max_offset;
-      int min2 = data2->min_offset;
-      int max2 = data2->max_offset;
-
-      if (max1 < min2 || min1 > max2)
-	return min1 - min2;
-      else
-	return 0;
-    }
-
-  /* The other two cases is a pair of a load/store multiple and
-     a simple memory op.  Return 0 if the single op's offset is within the
-     range of the multi-op insn and the difference between the single offset
-     and the minimum offset of the multi-set insn otherwise.  */
-  else if (data1->multi_mem_insn_p && !data2->multi_mem_insn_p)
-    {
-      int max1 = data1->max_offset;
-      int min1 = data1->min_offset;
-
-      if (data2->min_offset >= min1
-	  && data2->min_offset <= max1)
-	return 0;
-      else
-	return min1 - data2->min_offset;
-    }
-  else
-    {
-      int max2 = data2->max_offset;
-      int min2 = data2->min_offset;
-
-      if (data1->min_offset >= min2
-	  && data1->min_offset <= max2)
-	return 0;
-      else
-	return data1->min_offset - min2;
-    }
-}
-
 /* Helper function for rank_for_schedule sorting.  */
 static int
 autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
@@ -5726,7 +5650,7 @@ autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
       int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
 
       if (!irrel1 && !irrel2)
-	r = autopref_rank_data (data1, data2);
+	r = data1->offset - data2->offset;
       else
 	r = irrel2 - irrel1;
     }
@@ -5754,7 +5678,7 @@ autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
     return 0;
 
   if (rtx_equal_p (data1->base, data2->base)
-      && autopref_rank_data (data1, data2) > 0)
+      && data1->offset > data2->offset)
     {
       if (sched_verbose >= 2)
 	{
@@ -8217,8 +8141,6 @@ init_before_recovery (basic_block *before_recovery_ptr)
 
       single->count = last->count;
       empty->count = last->count;
-      single->frequency = last->frequency;
-      empty->frequency = last->frequency;
       BB_COPY_PARTITION (single, last);
       BB_COPY_PARTITION (empty, last);
 
@@ -8311,11 +8233,8 @@ sched_create_recovery_edges (basic_block first_bb, basic_block rec,
      'todo_spec' variable in create_check_block_twin and
      in sel-sched.c `check_ds' in create_speculation_check.  */
   e->probability = profile_probability::very_unlikely ();
-  e->count = first_bb->count.apply_probability (e->probability);
-  rec->count = e->count;
-  rec->frequency = EDGE_FREQUENCY (e);
+  rec->count = e->count ();
   e2->probability = e->probability.invert ();
-  e2->count = first_bb->count - e2->count;
 
   rtx_code_label *label = block_label (second_bb);
   rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
diff --git a/gcc/hsa-gen.c b/gcc/hsa-gen.c
index 593b92f11c1..a462ae5aa11 100644
--- a/gcc/hsa-gen.c
+++ b/gcc/hsa-gen.c
@@ -4231,12 +4231,11 @@ gen_hsa_alloca (gcall *call, hsa_bb *hbb)
 
   built_in_function fn = DECL_FUNCTION_CODE (gimple_call_fndecl (call));
 
-  gcc_checking_assert (fn == BUILT_IN_ALLOCA
-		       || fn == BUILT_IN_ALLOCA_WITH_ALIGN);
+  gcc_checking_assert (ALLOCA_FUNCTION_CODE_P (fn));
 
   unsigned bit_alignment = 0;
 
-  if (fn == BUILT_IN_ALLOCA_WITH_ALIGN)
+  if (fn != BUILT_IN_ALLOCA)
     {
       tree alignment_tree = gimple_call_arg (call, 1);
       if (TREE_CODE (alignment_tree) != INTEGER_CST)
@@ -5716,8 +5715,7 @@ gen_hsa_insns_for_call (gimple *stmt, hsa_bb *hbb)
 
 	break;
       }
-    case BUILT_IN_ALLOCA:
-    case BUILT_IN_ALLOCA_WITH_ALIGN:
+    CASE_BUILT_IN_ALLOCA:
       {
 	gen_hsa_alloca (call, hbb);
 	break;
@@ -6331,7 +6329,7 @@ convert_switch_statements (void)
 	    tree label = gimple_switch_label (s, i);
 	    basic_block label_bb = label_to_block_fn (func, CASE_LABEL (label));
 	    edge e = find_edge (bb, label_bb);
-	    edge_counts.safe_push (e->count);
+	    edge_counts.safe_push (e->count ());
 	    edge_probabilities.safe_push (e->probability);
 	    gphi_iterator phi_gsi;
 
@@ -6421,7 +6419,6 @@ convert_switch_statements (void)
 	    if (prob_sum.initialized_p ())
 	      new_edge->probability = edge_probabilities[i] / prob_sum;
 
-	    new_edge->count = edge_counts[i];
 	    new_edges.safe_push (new_edge);
 
 	    if (i < labels - 1)
@@ -6437,10 +6434,7 @@ convert_switch_statements (void)
 
 		edge next_edge = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
 		next_edge->probability = new_edge->probability.invert ();
-		next_edge->count = edge_counts[0]
-		  + sum_slice <profile_count> (edge_counts, i, labels,
-					       profile_count::zero ());
-		next_bb->frequency = EDGE_FREQUENCY (next_edge);
+		next_bb->count = next_edge->count ();
 		cur_bb = next_bb;
 	      }
 	    else /* Link last IF statement and default label
@@ -6448,7 +6442,6 @@ convert_switch_statements (void)
 	      {
 		edge e = make_edge (cur_bb, default_label_bb, EDGE_FALSE_VALUE);
 		e->probability = new_edge->probability.invert ();
-		e->count = edge_counts[0];
 		new_edges.safe_insert (0, e);
 	      }
 	  }
diff --git a/gcc/ifcvt.c b/gcc/ifcvt.c
index 47169270d6b..4041a5ba9ba 100644
--- a/gcc/ifcvt.c
+++ b/gcc/ifcvt.c
@@ -5283,8 +5283,6 @@ dead_or_predicable (basic_block test_bb, basic_block merge_bb,
       redirect_edge_succ (BRANCH_EDGE (test_bb), new_dest);
       if (reversep)
 	{
-	  std::swap (BRANCH_EDGE (test_bb)->count,
-		     FALLTHRU_EDGE (test_bb)->count);
 	  std::swap (BRANCH_EDGE (test_bb)->probability,
 		     FALLTHRU_EDGE (test_bb)->probability);
 	  update_br_prob_note (test_bb);
diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index 4264bb81fe1..c4dcb7fb13e 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -1791,8 +1791,8 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
 		}
 
 	      /* At this point hipart{0,1} are both in [-1, 0].  If they are
-		 the same, overflow happened if res is negative, if they are
-		 different, overflow happened if res is positive.  */
+		 the same, overflow happened if res is non-positive, if they
+		 are different, overflow happened if res is positive.  */
 	      if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
 		emit_jump (hipart_different);
 	      else if (op0_sign == 1 || op1_sign == 1)
@@ -1800,7 +1800,7 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
 					 NULL_RTX, NULL, hipart_different,
 					 profile_probability::even ());
 
-	      do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode,
+	      do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
 				       NULL_RTX, NULL, do_error,
 				       profile_probability::very_unlikely ());
 	      emit_jump (done_label);
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index a98d35c5b74..43885e7988d 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -97,6 +97,11 @@ along with GCC; see the file COPYING3.  If not see
   DEF_INTERNAL_OPTAB_FN (NAME, FLAGS, OPTAB, TYPE)
 #endif
 
+#ifndef DEF_INTERNAL_FLT_FLOATN_FN
+#define DEF_INTERNAL_FLT_FLOATN_FN(NAME, FLAGS, OPTAB, TYPE) \
+  DEF_INTERNAL_FLT_FN (NAME, FLAGS, OPTAB, TYPE)
+#endif
+
 #ifndef DEF_INTERNAL_INT_FN
 #define DEF_INTERNAL_INT_FN(NAME, FLAGS, OPTAB, TYPE) \
   DEF_INTERNAL_OPTAB_FN (NAME, FLAGS, OPTAB, TYPE)
@@ -207,7 +212,7 @@ DEF_INTERNAL_FLT_FN (LOG2, ECF_CONST, log2, unary)
 DEF_INTERNAL_FLT_FN (LOGB, ECF_CONST, logb, unary)
 DEF_INTERNAL_FLT_FN (SIGNIFICAND, ECF_CONST, significand, unary)
 DEF_INTERNAL_FLT_FN (SIN, ECF_CONST, sin, unary)
-DEF_INTERNAL_FLT_FN (SQRT, ECF_CONST, sqrt, unary)
+DEF_INTERNAL_FLT_FLOATN_FN (SQRT, ECF_CONST, sqrt, unary)
 DEF_INTERNAL_FLT_FN (TAN, ECF_CONST, tan, unary)
 
 /* FP rounding.  */
@@ -220,13 +225,13 @@ DEF_INTERNAL_FLT_FN (TRUNC, ECF_CONST, btrunc, unary)
 
 /* Binary math functions.  */
 DEF_INTERNAL_FLT_FN (ATAN2, ECF_CONST, atan2, binary)
-DEF_INTERNAL_FLT_FN (COPYSIGN, ECF_CONST, copysign, binary)
+DEF_INTERNAL_FLT_FLOATN_FN (COPYSIGN, ECF_CONST, copysign, binary)
 DEF_INTERNAL_FLT_FN (FMOD, ECF_CONST, fmod, binary)
 DEF_INTERNAL_FLT_FN (POW, ECF_CONST, pow, binary)
 DEF_INTERNAL_FLT_FN (REMAINDER, ECF_CONST, remainder, binary)
 DEF_INTERNAL_FLT_FN (SCALB, ECF_CONST, scalb, binary)
-DEF_INTERNAL_FLT_FN (FMIN, ECF_CONST, fmin, binary)
-DEF_INTERNAL_FLT_FN (FMAX, ECF_CONST, fmax, binary)
+DEF_INTERNAL_FLT_FLOATN_FN (FMIN, ECF_CONST, fmin, binary)
+DEF_INTERNAL_FLT_FLOATN_FN (FMAX, ECF_CONST, fmax, binary)
 DEF_INTERNAL_OPTAB_FN (XORSIGN, ECF_CONST, xorsign, binary)
 
 /* FP scales.  */
@@ -331,6 +336,7 @@ DEF_INTERNAL_FN (DIVMOD, ECF_CONST | ECF_LEAF, NULL)
 
 #undef DEF_INTERNAL_INT_FN
 #undef DEF_INTERNAL_FLT_FN
+#undef DEF_INTERNAL_FLT_FLOATN_FN
 #undef DEF_INTERNAL_COND_OPTAB_FN
 #undef DEF_INTERNAL_OPTAB_FN
 #undef DEF_INTERNAL_FN
diff --git a/gcc/ipa-cp.c b/gcc/ipa-cp.c
index d23c1d8ba3e..24d2be79103 100644
--- a/gcc/ipa-cp.c
+++ b/gcc/ipa-cp.c
@@ -3257,6 +3257,8 @@ ipcp_propagate_stage (struct ipa_topo_info *topo)
   if (dump_file)
     fprintf (dump_file, "\n Propagating constants:\n\n");
 
+  max_count = profile_count::uninitialized ();
+
   FOR_EACH_DEFINED_FUNCTION (node)
   {
     struct ipa_node_params *info = IPA_NODE_REF (node);
@@ -3270,8 +3272,7 @@ ipcp_propagate_stage (struct ipa_topo_info *topo)
       }
     if (node->definition && !node->alias)
       overall_size += ipa_fn_summaries->get (node)->self_size;
-    if (node->count > max_count)
-      max_count = node->count;
+    max_count = max_count.max (node->count);
   }
 
   max_new_size = overall_size;
@@ -5125,7 +5126,7 @@ make_pass_ipa_cp (gcc::context *ctxt)
 void
 ipa_cp_c_finalize (void)
 {
-  max_count = profile_count::zero ();
+  max_count = profile_count::uninitialized ();
   overall_size = 0;
   max_new_size = 0;
 }
diff --git a/gcc/ipa-fnsummary.c b/gcc/ipa-fnsummary.c
index 076ccd40bd7..f6841104a32 100644
--- a/gcc/ipa-fnsummary.c
+++ b/gcc/ipa-fnsummary.c
@@ -542,6 +542,7 @@ void
 ipa_call_summary::reset ()
 {
   call_stmt_size = call_stmt_time = 0;
+  is_return_callee_uncaptured = false;
   if (predicate)
     edge_predicate_pool.remove (predicate);
   predicate = NULL;
@@ -1607,7 +1608,7 @@ static basic_block
 get_minimal_bb (basic_block init_bb, basic_block use_bb)
 {
   struct loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
-  if (l && l->header->frequency < init_bb->frequency)
+  if (l && l->header->count < init_bb->count)
     return l->header;
   return init_bb;
 }
@@ -1663,20 +1664,21 @@ param_change_prob (gimple *stmt, int i)
     {
       int init_freq;
 
-      if (!bb->frequency)
+      if (!bb->count.to_frequency (cfun))
 	return REG_BR_PROB_BASE;
 
       if (SSA_NAME_IS_DEFAULT_DEF (base))
-	init_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
+	init_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
       else
 	init_freq = get_minimal_bb
 		      (gimple_bb (SSA_NAME_DEF_STMT (base)),
-		       gimple_bb (stmt))->frequency;
+		       gimple_bb (stmt))->count.to_frequency (cfun);
 
       if (!init_freq)
 	init_freq = 1;
-      if (init_freq < bb->frequency)
-	return MAX (GCOV_COMPUTE_SCALE (init_freq, bb->frequency), 1);
+      if (init_freq < bb->count.to_frequency (cfun))
+	return MAX (GCOV_COMPUTE_SCALE (init_freq,
+					bb->count.to_frequency (cfun)), 1);
       else
 	return REG_BR_PROB_BASE;
     }
@@ -1691,7 +1693,7 @@ param_change_prob (gimple *stmt, int i)
 
       if (init != error_mark_node)
 	return 0;
-      if (!bb->frequency)
+      if (!bb->count.to_frequency (cfun))
 	return REG_BR_PROB_BASE;
       ao_ref_init (&refd, op);
       info.stmt = stmt;
@@ -1707,17 +1709,17 @@ param_change_prob (gimple *stmt, int i)
       /* Assume that every memory is initialized at entry.
          TODO: Can we easilly determine if value is always defined
          and thus we may skip entry block?  */
-      if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency)
-	max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
+      if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun))
+	max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
       else
 	max = 1;
 
       EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
-	max = MIN (max, BASIC_BLOCK_FOR_FN (cfun, index)->frequency);
+	max = MIN (max, BASIC_BLOCK_FOR_FN (cfun, index)->count.to_frequency (cfun));
 
       BITMAP_FREE (info.bb_set);
-      if (max < bb->frequency)
-	return MAX (GCOV_COMPUTE_SCALE (max, bb->frequency), 1);
+      if (max < bb->count.to_frequency (cfun))
+	return MAX (GCOV_COMPUTE_SCALE (max, bb->count.to_frequency (cfun)), 1);
       else
 	return REG_BR_PROB_BASE;
     }
@@ -3204,6 +3206,10 @@ read_ipa_call_summary (struct lto_input_block *ib, struct cgraph_edge *e)
   es->call_stmt_size = streamer_read_uhwi (ib);
   es->call_stmt_time = streamer_read_uhwi (ib);
   es->loop_depth = streamer_read_uhwi (ib);
+
+  bitpack_d bp = streamer_read_bitpack (ib);
+  es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
+
   p.stream_in (ib);
   edge_set_predicate (e, &p);
   length = streamer_read_uhwi (ib);
@@ -3360,6 +3366,11 @@ write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
   streamer_write_uhwi (ob, es->call_stmt_size);
   streamer_write_uhwi (ob, es->call_stmt_time);
   streamer_write_uhwi (ob, es->loop_depth);
+
+  bitpack_d bp = bitpack_create (ob->main_stream);
+  bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
+  streamer_write_bitpack (&bp);
+
   if (es->predicate)
     es->predicate->stream_out (ob);
   else
diff --git a/gcc/ipa-fnsummary.h b/gcc/ipa-fnsummary.h
index f50d6806e61..a794bd09318 100644
--- a/gcc/ipa-fnsummary.h
+++ b/gcc/ipa-fnsummary.h
@@ -197,7 +197,9 @@ struct ipa_call_summary
   int call_stmt_time;
   /* Depth of loop nest, 0 means no nesting.  */
   unsigned int loop_depth;
-  
+  /* Indicates whether the caller returns the value of it's callee.  */
+  bool is_return_callee_uncaptured;
+
   /* Keep all field empty so summary dumping works during its computation.
      This is useful for debugging.  */
   ipa_call_summary ()
diff --git a/gcc/ipa-icf.c b/gcc/ipa-icf.c
index 7b4cd9d49e8..cb66aa5f7a0 100644
--- a/gcc/ipa-icf.c
+++ b/gcc/ipa-icf.c
@@ -1422,6 +1422,7 @@ sem_function::init (void)
 	      }
 	  }
 
+	hstate.commit_flag ();
 	gcode_hash = hstate.end ();
 	bb_sizes.safe_push (nondbg_stmt_count);
 
@@ -1646,6 +1647,11 @@ sem_function::hash_stmt (gimple *stmt, inchash::hash &hstate)
 	  if (gimple_op (stmt, i))
 	    add_type (TREE_TYPE (gimple_op (stmt, i)), hstate);
 	}
+      /* Consider nocf_check attribute in hash as it affects code
+ 	 generation.  */
+      if (code == GIMPLE_CALL
+	  && flag_cf_protection & CF_BRANCH)
+	hstate.add_flag (gimple_call_nocf_check_p (as_a <gcall *> (stmt)));
     default:
       break;
     }
diff --git a/gcc/ipa-inline-transform.c b/gcc/ipa-inline-transform.c
index dc224f7a394..886e8edd473 100644
--- a/gcc/ipa-inline-transform.c
+++ b/gcc/ipa-inline-transform.c
@@ -676,9 +676,9 @@ inline_transform (struct cgraph_node *node)
     {
       profile_count num = node->count;
       profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
-      bool scale = num.initialized_p ()
-		   && (den > 0 || num == profile_count::zero ())
-		   && !(num == den);
+      bool scale = num.initialized_p () && den.ipa_p ()
+		   && (den.nonzero_p () || num == profile_count::zero ())
+		   && !(num == den.ipa ());
       if (scale)
 	{
 	  if (dump_file)
@@ -692,14 +692,7 @@ inline_transform (struct cgraph_node *node)
 
 	  basic_block bb;
 	  FOR_ALL_BB_FN (bb, cfun)
-	    {
-	      bb->count = bb->count.apply_scale (num, den);
-	
-	      edge e;
-	      edge_iterator ei;
-	      FOR_EACH_EDGE (e, ei, bb->succs)
-		e->count = e->count.apply_scale (num, den);
-	    }
+	    bb->count = bb->count.apply_scale (num, den);
 	  ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
 	}
       todo = optimize_inline_calls (current_function_decl);
diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c
index dd46cb61362..687996876ce 100644
--- a/gcc/ipa-inline.c
+++ b/gcc/ipa-inline.c
@@ -640,8 +640,8 @@ compute_uninlined_call_time (struct cgraph_edge *edge,
 			 ? edge->caller->global.inlined_to
 			 : edge->caller);
 
-  if (edge->count > profile_count::zero ()
-      && caller->count > profile_count::zero ())
+  if (edge->count.nonzero_p ()
+      && caller->count.nonzero_p ())
     uninlined_call_time *= (sreal)edge->count.to_gcov_type ()
 			   / caller->count.to_gcov_type ();
   if (edge->frequency)
@@ -665,8 +665,8 @@ compute_inlined_call_time (struct cgraph_edge *edge,
 			 : edge->caller);
   sreal caller_time = ipa_fn_summaries->get (caller)->time;
 
-  if (edge->count > profile_count::zero ()
-      && caller->count > profile_count::zero ())
+  if (edge->count.nonzero_p ()
+      && caller->count.nonzero_p ())
     time *= (sreal)edge->count.to_gcov_type () / caller->count.to_gcov_type ();
   if (edge->frequency)
     time *= cgraph_freq_base_rec * edge->frequency;
@@ -733,7 +733,7 @@ want_inline_small_function_p (struct cgraph_edge *e, bool report)
       want_inline = false;
     }
   else if ((DECL_DECLARED_INLINE_P (callee->decl)
-	    || e->count > profile_count::zero ())
+	    || e->count.nonzero_p ())
 	   && ipa_fn_summaries->get (callee)->min_size
 		- ipa_call_summaries->get (e)->call_stmt_size
 	      > 16 * MAX_INLINE_INSNS_SINGLE)
@@ -843,7 +843,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
       reason = "recursive call is cold";
       want_inline = false;
     }
-  else if (outer_node->count == profile_count::zero ())
+  else if (!outer_node->count.nonzero_p ())
     {
       reason = "not executed in profile";
       want_inline = false;
@@ -881,7 +881,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
       int i;
       for (i = 1; i < depth; i++)
 	max_prob = max_prob * max_prob / CGRAPH_FREQ_BASE;
-      if (max_count > profile_count::zero () && edge->count > profile_count::zero ()
+      if (max_count.nonzero_p () && edge->count.nonzero_p () 
 	  && (edge->count.to_gcov_type () * CGRAPH_FREQ_BASE
 	      / outer_node->count.to_gcov_type ()
 	      >= max_prob))
@@ -889,7 +889,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
 	  reason = "profile of recursive call is too large";
 	  want_inline = false;
 	}
-      if (max_count == profile_count::zero ()
+      if (!max_count.nonzero_p ()
 	  && (edge->frequency * CGRAPH_FREQ_BASE / caller_freq
 	      >= max_prob))
 	{
@@ -915,7 +915,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
      methods.  */
   else
     {
-      if (max_count > profile_count::zero () && edge->count.initialized_p ()
+      if (max_count.nonzero_p () && edge->count.initialized_p ()
 	  && (edge->count.to_gcov_type () * 100
 	      / outer_node->count.to_gcov_type ()
 	      <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY)))
@@ -923,7 +923,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
 	  reason = "profile of recursive call is too small";
 	  want_inline = false;
 	}
-      else if ((max_count == profile_count::zero ()
+      else if ((!max_count.nonzero_p ()
 	        || !edge->count.initialized_p ())
 	       && (edge->frequency * 100 / caller_freq
 	           <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY)))
@@ -1070,7 +1070,7 @@ edge_badness (struct cgraph_edge *edge, bool dump)
      then calls without.
   */
   else if (opt_for_fn (caller->decl, flag_guess_branch_prob)
-	   || caller->count > profile_count::zero ())
+	   || caller->count.nonzero_p ())
     {
       sreal numerator, denominator;
       int overall_growth;
@@ -1080,7 +1080,7 @@ edge_badness (struct cgraph_edge *edge, bool dump)
 		   - inlined_time);
       if (numerator == 0)
 	numerator = ((sreal) 1 >> 8);
-      if (caller->count > profile_count::zero ())
+      if (caller->count.nonzero_p ())
 	numerator *= caller->count.to_gcov_type ();
       else if (caller->count.initialized_p ())
 	numerator = numerator >> 11;
@@ -1521,7 +1521,7 @@ recursive_inlining (struct cgraph_edge *edge,
 	{
 	  fprintf (dump_file,
 		   "   Inlining call of depth %i", depth);
-	  if (node->count > profile_count::zero ())
+	  if (node->count.nonzero_p ())
 	    {
 	      fprintf (dump_file, " called approx. %.2f times per call",
 		       (double)curr->count.to_gcov_type ()
@@ -1684,7 +1684,8 @@ resolve_noninline_speculation (edge_heap_t *edge_heap, struct cgraph_edge *edge)
 				  ? node->global.inlined_to : node;
       auto_bitmap updated_nodes;
 
-      spec_rem += edge->count;
+      if (edge->count.initialized_p ())
+        spec_rem += edge->count;
       edge->resolve_speculation ();
       reset_edge_caches (where);
       ipa_update_overall_fn_summary (where);
@@ -1789,8 +1790,7 @@ inline_small_functions (void)
 	  }
 
 	for (edge = node->callers; edge; edge = edge->next_caller)
-	  if (!(max_count >= edge->count))
-	    max_count = edge->count;
+	  max_count = max_count.max (edge->count);
       }
   ipa_free_postorder_info ();
   initialize_growth_caches ();
@@ -2049,7 +2049,7 @@ inline_small_functions (void)
       update_caller_keys (&edge_heap, where, updated_nodes, NULL);
       /* Offline copy count has possibly changed, recompute if profile is
 	 available.  */
-      if (max_count > profile_count::zero ())
+      if (max_count.nonzero_p ())
         {
 	  struct cgraph_node *n = cgraph_node::get (edge->callee->decl);
 	  if (n != edge->callee && n->analyzed)
@@ -2392,6 +2392,7 @@ ipa_inline (void)
     ipa_dump_fn_summaries (dump_file);
 
   nnodes = ipa_reverse_postorder (order);
+  spec_rem = profile_count::zero ();
 
   FOR_EACH_FUNCTION (node)
     {
@@ -2487,8 +2488,9 @@ ipa_inline (void)
 	      next = edge->next_callee;
 	      if (edge->speculative && !speculation_useful_p (edge, false))
 		{
+		  if (edge->count.initialized_p ())
+		    spec_rem += edge->count;
 		  edge->resolve_speculation ();
-		  spec_rem += edge->count;
 		  update = true;
 		  remove_functions = true;
 		}
@@ -2526,9 +2528,6 @@ ipa_inline (void)
 
   if (dump_file)
     ipa_dump_fn_summaries (dump_file);
-  /* In WPA we use inline summaries for partitioning process.  */
-  if (!flag_wpa)
-    ipa_free_fn_summary ();
   return remove_functions ? TODO_remove_functions : 0;
 }
 
diff --git a/gcc/ipa-profile.c b/gcc/ipa-profile.c
index f149d0196fa..8eb03dd7c24 100644
--- a/gcc/ipa-profile.c
+++ b/gcc/ipa-profile.c
@@ -179,53 +179,54 @@ ipa_profile_generate_summary (void)
   hash_table<histogram_hash> hashtable (10);
   
   FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
-    FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
-      {
-	int time = 0;
-	int size = 0;
-        for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
-	  {
-	    gimple *stmt = gsi_stmt (gsi);
-	    if (gimple_code (stmt) == GIMPLE_CALL
-		&& !gimple_call_fndecl (stmt))
-	      {
-		histogram_value h;
-		h = gimple_histogram_value_of_type
-		      (DECL_STRUCT_FUNCTION (node->decl),
-		       stmt, HIST_TYPE_INDIR_CALL);
-		/* No need to do sanity check: gimple_ic_transform already
-		   takes away bad histograms.  */
-		if (h)
-		  {
-		    /* counter 0 is target, counter 1 is number of execution we called target,
-		       counter 2 is total number of executions.  */
-		    if (h->hvalue.counters[2])
-		      {
-			struct cgraph_edge * e = node->get_edge (stmt);
-			if (e && !e->indirect_unknown_callee)
-			  continue;
-			e->indirect_info->common_target_id
-			  = h->hvalue.counters [0];
-			e->indirect_info->common_target_probability
-			  = GCOV_COMPUTE_SCALE (h->hvalue.counters [1], h->hvalue.counters [2]);
-			if (e->indirect_info->common_target_probability > REG_BR_PROB_BASE)
-			  {
-			    if (dump_file)
-			      fprintf (dump_file, "Probability capped to 1\n");
-			    e->indirect_info->common_target_probability = REG_BR_PROB_BASE;
-			  }
-		      }
-		    gimple_remove_histogram_value (DECL_STRUCT_FUNCTION (node->decl),
-						    stmt, h);
-		  }
-	      }
-	    time += estimate_num_insns (stmt, &eni_time_weights);
-	    size += estimate_num_insns (stmt, &eni_size_weights);
-	  }
-	if (bb->count.initialized_p ())
-	  account_time_size (&hashtable, histogram, bb->count.to_gcov_type (),
-			     time, size);
-      }
+    if (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (node->decl))->count.ipa_p ())
+      FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
+	{
+	  int time = 0;
+	  int size = 0;
+	  for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+	    {
+	      gimple *stmt = gsi_stmt (gsi);
+	      if (gimple_code (stmt) == GIMPLE_CALL
+		  && !gimple_call_fndecl (stmt))
+		{
+		  histogram_value h;
+		  h = gimple_histogram_value_of_type
+			(DECL_STRUCT_FUNCTION (node->decl),
+			 stmt, HIST_TYPE_INDIR_CALL);
+		  /* No need to do sanity check: gimple_ic_transform already
+		     takes away bad histograms.  */
+		  if (h)
+		    {
+		      /* counter 0 is target, counter 1 is number of execution we called target,
+			 counter 2 is total number of executions.  */
+		      if (h->hvalue.counters[2])
+			{
+			  struct cgraph_edge * e = node->get_edge (stmt);
+			  if (e && !e->indirect_unknown_callee)
+			    continue;
+			  e->indirect_info->common_target_id
+			    = h->hvalue.counters [0];
+			  e->indirect_info->common_target_probability
+			    = GCOV_COMPUTE_SCALE (h->hvalue.counters [1], h->hvalue.counters [2]);
+			  if (e->indirect_info->common_target_probability > REG_BR_PROB_BASE)
+			    {
+			      if (dump_file)
+				fprintf (dump_file, "Probability capped to 1\n");
+			      e->indirect_info->common_target_probability = REG_BR_PROB_BASE;
+			    }
+			}
+		      gimple_remove_histogram_value (DECL_STRUCT_FUNCTION (node->decl),
+						      stmt, h);
+		    }
+		}
+	      time += estimate_num_insns (stmt, &eni_time_weights);
+	      size += estimate_num_insns (stmt, &eni_size_weights);
+	    }
+	  if (bb->count.ipa_p () && bb->count.initialized_p ())
+	    account_time_size (&hashtable, histogram, bb->count.ipa ().to_gcov_type (),
+			       time, size);
+	}
   histogram.qsort (cmp_counts);
 }
 
diff --git a/gcc/ipa-pure-const.c b/gcc/ipa-pure-const.c
index 915423559cb..bdc752207b1 100644
--- a/gcc/ipa-pure-const.c
+++ b/gcc/ipa-pure-const.c
@@ -56,6 +56,11 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-scalar-evolution.h"
 #include "intl.h"
 #include "opts.h"
+#include "ssa.h"
+#include "alloc-pool.h"
+#include "symbol-summary.h"
+#include "ipa-prop.h"
+#include "ipa-fnsummary.h"
 
 /* Lattice values for const and pure functions.  Everything starts out
    being const, then may drop to pure and then neither depending on
@@ -67,7 +72,16 @@ enum pure_const_state_e
   IPA_NEITHER
 };
 
-const char *pure_const_names[3] = {"const", "pure", "neither"};
+static const char *pure_const_names[3] = {"const", "pure", "neither"};
+
+enum malloc_state_e
+{
+  STATE_MALLOC_TOP,
+  STATE_MALLOC,
+  STATE_MALLOC_BOTTOM
+};
+
+static const char *malloc_state_names[] = {"malloc_top", "malloc", "malloc_bottom"};
 
 /* Holder for the const_state.  There is one of these per function
    decl.  */
@@ -92,11 +106,13 @@ struct funct_state_d
   /* If function can call free, munmap or otherwise make previously
      non-trapping memory accesses trapping.  */
   bool can_free;
+
+  enum malloc_state_e malloc_state;
 };
 
 /* State used when we know nothing about function.  */
 static struct funct_state_d varying_state
-   = { IPA_NEITHER, IPA_NEITHER, true, true, true, true };
+   = { IPA_NEITHER, IPA_NEITHER, true, true, true, true, STATE_MALLOC_BOTTOM };
 
 
 typedef struct funct_state_d * funct_state;
@@ -216,6 +232,19 @@ warn_function_const (tree decl, bool known_finite)
 			 known_finite, warned_about, "const");
 }
 
+/* Emit suggestion about __attribute__((malloc)) for DECL.  */
+
+static void
+warn_function_malloc (tree decl)
+{
+  static hash_set<tree> *warned_about;
+  warned_about
+    = suggest_attribute (OPT_Wsuggest_attribute_malloc, decl,
+			 false, warned_about, "malloc");
+}
+
+/* Emit suggestion about __attribute__((noreturn)) for DECL.  */
+
 static void
 warn_function_noreturn (tree decl)
 {
@@ -518,8 +547,7 @@ special_builtin_state (enum pure_const_state_e *state, bool *looping,
       {
 	case BUILT_IN_RETURN:
 	case BUILT_IN_UNREACHABLE:
-	case BUILT_IN_ALLOCA:
-	case BUILT_IN_ALLOCA_WITH_ALIGN:
+	CASE_BUILT_IN_ALLOCA:
 	case BUILT_IN_STACK_SAVE:
 	case BUILT_IN_STACK_RESTORE:
 	case BUILT_IN_EH_POINTER:
@@ -828,6 +856,149 @@ check_stmt (gimple_stmt_iterator *gsip, funct_state local, bool ipa)
     }
 }
 
+/* Check that RETVAL is used only in STMT and in comparisons against 0.
+   RETVAL is return value of the function and STMT is return stmt.  */
+
+static bool
+check_retval_uses (tree retval, gimple *stmt)
+{
+  imm_use_iterator use_iter;
+  gimple *use_stmt;
+
+  FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, retval)
+    if (gcond *cond = dyn_cast<gcond *> (use_stmt))
+      {
+	tree op2 = gimple_cond_rhs (cond);
+	if (!integer_zerop (op2))
+	  RETURN_FROM_IMM_USE_STMT (use_iter, false);
+      }
+    else if (gassign *ga = dyn_cast<gassign *> (use_stmt))
+      {
+	enum tree_code code = gimple_assign_rhs_code (ga);
+	if (TREE_CODE_CLASS (code) != tcc_comparison)
+	  RETURN_FROM_IMM_USE_STMT (use_iter, false);
+	if (!integer_zerop (gimple_assign_rhs2 (ga)))
+	  RETURN_FROM_IMM_USE_STMT (use_iter, false);
+      }
+    else if (is_gimple_debug (use_stmt))
+      ;
+    else if (use_stmt != stmt)
+      RETURN_FROM_IMM_USE_STMT (use_iter, false);
+
+  return true;
+}
+
+/* malloc_candidate_p() checks if FUN can possibly be annotated with malloc
+   attribute. Currently this function does a very conservative analysis.
+   FUN is considered to be a candidate if
+   1) It returns a value of pointer type.
+   2) SSA_NAME_DEF_STMT (return_value) is either a function call or
+      a phi, and element of phi is either NULL or
+      SSA_NAME_DEF_STMT(element) is function call.
+   3) The return-value has immediate uses only within comparisons (gcond or gassign)
+      and return_stmt (and likewise a phi arg has immediate use only within comparison
+      or the phi stmt).  */
+
+static bool
+malloc_candidate_p (function *fun, bool ipa)
+{
+  basic_block exit_block = EXIT_BLOCK_PTR_FOR_FN (fun);
+  edge e;
+  edge_iterator ei;
+  cgraph_node *node = cgraph_node::get_create (fun->decl);
+
+#define DUMP_AND_RETURN(reason)  \
+{  \
+  if (dump_file && (dump_flags & TDF_DETAILS))  \
+    fprintf (dump_file, "%s", (reason));  \
+  return false;  \
+}
+
+  if (EDGE_COUNT (exit_block->preds) == 0)
+    return false;
+
+  FOR_EACH_EDGE (e, ei, exit_block->preds)
+    {
+      gimple_stmt_iterator gsi = gsi_last_bb (e->src);
+      greturn *ret_stmt = dyn_cast<greturn *> (gsi_stmt (gsi));
+
+      if (!ret_stmt)
+	return false;
+
+      tree retval = gimple_return_retval (ret_stmt);
+      if (!retval)
+	DUMP_AND_RETURN("No return value.")
+
+      if (TREE_CODE (retval) != SSA_NAME
+	  || TREE_CODE (TREE_TYPE (retval)) != POINTER_TYPE)
+	DUMP_AND_RETURN("Return value is not SSA_NAME or not a pointer type.")
+
+      if (!check_retval_uses (retval, ret_stmt))
+	DUMP_AND_RETURN("Return value has uses outside return stmt"
+			" and comparisons against 0.")
+
+      gimple *def = SSA_NAME_DEF_STMT (retval);
+      if (gcall *call_stmt = dyn_cast<gcall *> (def))
+	{
+	  tree callee_decl = gimple_call_fndecl (call_stmt);
+	  if (!callee_decl)
+	    return false;
+
+	  if (!ipa && !DECL_IS_MALLOC (callee_decl))
+	    DUMP_AND_RETURN("callee_decl does not have malloc attribute for"
+			    " non-ipa mode.")
+
+	  cgraph_edge *cs = node->get_edge (call_stmt);
+	  if (cs)
+	    {
+	      ipa_call_summary *es = ipa_call_summaries->get (cs);
+	      gcc_assert (es);
+	      es->is_return_callee_uncaptured = true;
+	    }
+	}
+
+      else if (gphi *phi = dyn_cast<gphi *> (def))
+	for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
+	  {
+	    tree arg = gimple_phi_arg_def (phi, i);
+	    if (TREE_CODE (arg) != SSA_NAME)
+	      DUMP_AND_RETURN("phi arg is not SSA_NAME.")
+	    if (!(arg == null_pointer_node || check_retval_uses (arg, phi)))
+	      DUMP_AND_RETURN("phi arg has uses outside phi"
+			      " and comparisons against 0.")
+
+	    gimple *arg_def = SSA_NAME_DEF_STMT (arg);
+	    gcall *call_stmt = dyn_cast<gcall *> (arg_def);
+	    if (!call_stmt)
+	      return false;
+	    tree callee_decl = gimple_call_fndecl (call_stmt);
+	    if (!callee_decl)
+	      return false;
+	    if (!ipa && !DECL_IS_MALLOC (callee_decl))
+	      DUMP_AND_RETURN("callee_decl does not have malloc attribute for"
+			      " non-ipa mode.")
+
+	    cgraph_edge *cs = node->get_edge (call_stmt);
+	    if (cs)
+	      {
+		ipa_call_summary *es = ipa_call_summaries->get (cs);
+		gcc_assert (es);
+		es->is_return_callee_uncaptured = true;
+	      }
+	  }
+
+      else
+	DUMP_AND_RETURN("def_stmt of return value is not a call or phi-stmt.")
+    }
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    fprintf (dump_file, "\nFound %s to be candidate for malloc attribute\n",
+	     IDENTIFIER_POINTER (DECL_NAME (fun->decl)));
+  return true;
+
+#undef DUMP_AND_RETURN
+}
+
 
 /* This is the main routine for finding the reference patterns for
    global variables within a function FN.  */
@@ -937,6 +1108,14 @@ end:
   if (TREE_NOTHROW (decl))
     l->can_throw = false;
 
+  l->malloc_state = STATE_MALLOC_BOTTOM;
+  if (DECL_IS_MALLOC (decl))
+    l->malloc_state = STATE_MALLOC;
+  else if (ipa && malloc_candidate_p (DECL_STRUCT_FUNCTION (decl), true))
+    l->malloc_state = STATE_MALLOC_TOP;
+  else if (malloc_candidate_p (DECL_STRUCT_FUNCTION (decl), false))
+    l->malloc_state = STATE_MALLOC;
+
   pop_cfun ();
   if (dump_file)
     {
@@ -950,6 +1129,8 @@ end:
         fprintf (dump_file, "Function is locally pure.\n");
       if (l->can_free)
 	fprintf (dump_file, "Function can locally free.\n");
+      if (l->malloc_state == STATE_MALLOC)
+	fprintf (dump_file, "Function is locally malloc.\n");
     }
   return l;
 }
@@ -1083,6 +1264,7 @@ pure_const_write_summary (void)
 	  bp_pack_value (&bp, fs->looping, 1);
 	  bp_pack_value (&bp, fs->can_throw, 1);
 	  bp_pack_value (&bp, fs->can_free, 1);
+	  bp_pack_value (&bp, fs->malloc_state, 2);
 	  streamer_write_bitpack (&bp);
 	}
     }
@@ -1143,6 +1325,9 @@ pure_const_read_summary (void)
 	      fs->looping = bp_unpack_value (&bp, 1);
 	      fs->can_throw = bp_unpack_value (&bp, 1);
 	      fs->can_free = bp_unpack_value (&bp, 1);
+	      fs->malloc_state
+			= (enum malloc_state_e) bp_unpack_value (&bp, 2);
+
 	      if (dump_file)
 		{
 		  int flags = flags_from_decl_or_type (node->decl);
@@ -1165,6 +1350,8 @@ pure_const_read_summary (void)
 		    fprintf (dump_file,"  function is locally throwing\n");
 		  if (fs->can_free)
 		    fprintf (dump_file,"  function can locally free\n");
+		  fprintf (dump_file, "\n malloc state: %s\n",
+			   malloc_state_names[fs->malloc_state]);
 		}
 	    }
 
@@ -1675,6 +1862,131 @@ propagate_nothrow (void)
   free (order);
 }
 
+/* Debugging function to dump state of malloc lattice.  */
+
+DEBUG_FUNCTION
+static void
+dump_malloc_lattice (FILE *dump_file, const char *s)
+{
+  if (!dump_file)
+    return;
+
+  fprintf (dump_file, "\n\nMALLOC LATTICE %s:\n", s);
+  cgraph_node *node;
+  FOR_EACH_FUNCTION (node)
+    {
+      funct_state fs = get_function_state (node);
+      malloc_state_e state = fs->malloc_state;
+      fprintf (dump_file, "%s: %s\n", node->name (), malloc_state_names[state]);
+    }
+}
+
+/* Propagate malloc attribute across the callgraph.  */
+
+static void
+propagate_malloc (void)
+{
+  cgraph_node *node;
+  FOR_EACH_FUNCTION (node)
+    {
+      if (DECL_IS_MALLOC (node->decl))
+	if (!has_function_state (node))
+	  {
+	    funct_state l = XCNEW (struct funct_state_d);
+	    *l = varying_state;
+	    l->malloc_state = STATE_MALLOC;
+	    set_function_state (node, l);
+	  }
+    }
+
+  dump_malloc_lattice (dump_file, "Initial");
+  struct cgraph_node **order
+    = XNEWVEC (struct cgraph_node *, symtab->cgraph_count);
+  int order_pos = ipa_reverse_postorder (order);
+  bool changed = true;
+
+  while (changed)
+    {
+      changed = false;
+      /* Walk in postorder.  */
+      for (int i = order_pos - 1; i >= 0; --i)
+	{
+	  cgraph_node *node = order[i];
+	  if (node->alias
+	      || !node->definition
+	      || !has_function_state (node))
+	    continue;
+
+	  funct_state l = get_function_state (node);
+
+	  /* FIXME: add support for indirect-calls.  */
+	  if (node->indirect_calls)
+	    {
+	      l->malloc_state = STATE_MALLOC_BOTTOM;
+	      continue;
+	    }
+
+	  if (node->get_availability () <= AVAIL_INTERPOSABLE)
+	    {
+	      l->malloc_state = STATE_MALLOC_BOTTOM;
+	      continue;
+	    }
+
+	  if (l->malloc_state == STATE_MALLOC_BOTTOM)
+	    continue;
+
+	  vec<cgraph_node *> callees = vNULL;
+	  for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
+	    {
+	      ipa_call_summary *es = ipa_call_summaries->get (cs);
+	      if (es && es->is_return_callee_uncaptured)
+		callees.safe_push (cs->callee);
+	    }
+
+	  malloc_state_e new_state = l->malloc_state;
+	  for (unsigned j = 0; j < callees.length (); j++)
+	    {
+	      cgraph_node *callee = callees[j];
+	      if (!has_function_state (callee))
+		{
+		  new_state = STATE_MALLOC_BOTTOM;
+		  break;
+		}
+	      malloc_state_e callee_state = get_function_state (callee)->malloc_state;
+	      if (new_state < callee_state)
+		new_state = callee_state;
+	    }
+	  if (new_state != l->malloc_state)
+	    {
+	      changed = true;
+	      l->malloc_state = new_state;
+	    }
+	}
+    }
+
+  FOR_EACH_DEFINED_FUNCTION (node)
+    if (has_function_state (node))
+      {
+	funct_state l = get_function_state (node);
+	if (!node->alias
+	    && l->malloc_state == STATE_MALLOC
+	    && !node->global.inlined_to)
+	  {
+	    if (dump_file && (dump_flags & TDF_DETAILS))
+	      fprintf (dump_file, "Function %s found to be malloc\n",
+		       node->name ());
+
+	    bool malloc_decl_p = DECL_IS_MALLOC (node->decl);
+	    node->set_malloc_flag (true);
+	    if (!malloc_decl_p && warn_suggest_attribute_malloc)
+		warn_function_malloc (node->decl);
+	  }
+      }
+
+  dump_malloc_lattice (dump_file, "after propagation");
+  ipa_free_postorder_info ();
+  free (order);
+}
 
 /* Produce the global information by preforming a transitive closure
    on the local information that was produced by generate_summary.  */
@@ -1693,6 +2005,7 @@ execute (function *)
   /* Nothrow makes more function to not lead to return and improve
      later analysis.  */
   propagate_nothrow ();
+  propagate_malloc ();
   remove_p = propagate_pure_const ();
 
   /* Cleanup. */
@@ -1700,6 +2013,10 @@ execute (function *)
     if (has_function_state (node))
       free (get_function_state (node));
   funct_state_vec.release ();
+
+  /* In WPA we use inline summaries for partitioning process.  */
+  if (!flag_wpa)
+    ipa_free_fn_summary ();
   return remove_p ? TODO_remove_functions : 0;
 }
 
@@ -1894,6 +2211,19 @@ pass_local_pure_const::execute (function *fun)
 	fprintf (dump_file, "Function found to be nothrow: %s\n",
 		 current_function_name ());
     }
+
+  if (l->malloc_state == STATE_MALLOC
+      && !DECL_IS_MALLOC (current_function_decl))
+    {
+      node->set_malloc_flag (true);
+      if (warn_suggest_attribute_malloc)
+	warn_function_malloc (node->decl);
+      changed = true;
+      if (dump_file)
+	fprintf (dump_file, "Function found to be malloc: %s\n",
+		 node->name ());
+    }
+
   free (l);
   if (changed)
     return execute_fixup_cfg ();
diff --git a/gcc/ipa-split.c b/gcc/ipa-split.c
index e3759d6c50e..252ea053e2a 100644
--- a/gcc/ipa-split.c
+++ b/gcc/ipa-split.c
@@ -444,7 +444,7 @@ consider_split (struct split_point *current, bitmap non_ssa_vars,
 
   /* Do not split when we would end up calling function anyway.  */
   if (incoming_freq
-      >= (ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency
+      >= (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun)
 	  * PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100))
     {
       /* When profile is guessed, we can not expect it to give us
@@ -454,13 +454,14 @@ consider_split (struct split_point *current, bitmap non_ssa_vars,
 	 is likely noticeable win.  */
       if (back_edge
 	  && profile_status_for_fn (cfun) != PROFILE_READ
-	  && incoming_freq < ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency)
+	  && incoming_freq
+		 < ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun))
 	{
 	  if (dump_file && (dump_flags & TDF_DETAILS))
 	    fprintf (dump_file,
 		     "  Split before loop, accepting despite low frequencies %i %i.\n",
 		     incoming_freq,
-		     ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency);
+		     ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun));
 	}
       else
 	{
@@ -714,8 +715,10 @@ consider_split (struct split_point *current, bitmap non_ssa_vars,
      out smallest size of header.
      In future we might re-consider this heuristics.  */
   if (!best_split_point.split_bbs
-      || best_split_point.entry_bb->frequency > current->entry_bb->frequency
-      || (best_split_point.entry_bb->frequency == current->entry_bb->frequency
+      || best_split_point.entry_bb->count.to_frequency (cfun)
+	 > current->entry_bb->count.to_frequency (cfun)
+      || (best_split_point.entry_bb->count.to_frequency (cfun)
+	  == current->entry_bb->count.to_frequency (cfun)
 	  && best_split_point.split_size < current->split_size))
 	
     {
@@ -1285,8 +1288,7 @@ split_function (basic_block return_bb, struct split_point *split_point,
 	  FOR_EACH_EDGE (e, ei, return_bb->preds)
 	    if (bitmap_bit_p (split_point->split_bbs, e->src->index))
 	      {
-		new_return_bb->count += e->count;
-		new_return_bb->frequency += EDGE_FREQUENCY (e);
+		new_return_bb->count += e->count ();
 		redirect_edge_and_branch (e, new_return_bb);
 		redirected = true;
 		break;
diff --git a/gcc/ipa-utils.c b/gcc/ipa-utils.c
index 708710d6135..e9ab78cdabb 100644
--- a/gcc/ipa-utils.c
+++ b/gcc/ipa-utils.c
@@ -524,20 +524,36 @@ ipa_merge_profiles (struct cgraph_node *dst,
 	  unsigned int i;
 
 	  dstbb = BASIC_BLOCK_FOR_FN (dstcfun, srcbb->index);
-	  if (dstbb->count.initialized_p ())
-	    dstbb->count += srcbb->count;
-	  else
-	    dstbb->count = srcbb->count;
-	  for (i = 0; i < EDGE_COUNT (srcbb->succs); i++)
+
+	  /* Either sum the profiles if both are IPA and not global0, or
+	     pick more informative one (that is nonzero IPA if other is
+	     uninitialized, guessed or global0).   */
+	  if (!dstbb->count.ipa ().initialized_p ()
+	      || (dstbb->count.ipa () == profile_count::zero ()
+		  && (srcbb->count.ipa ().initialized_p ()
+		      && !(srcbb->count.ipa () == profile_count::zero ()))))
 	    {
-	      edge srce = EDGE_SUCC (srcbb, i);
-	      edge dste = EDGE_SUCC (dstbb, i);
-	      if (dstbb->count.initialized_p ())
-	        dste->count += srce->count;
-	      else
-		dste->count = srce->count;
-	      if (dstbb->count > 0 && dste->count.initialized_p ())
-		dste->probability = dste->count.probability_in (dstbb->count);
+	      dstbb->count = srcbb->count;
+	      for (i = 0; i < EDGE_COUNT (srcbb->succs); i++)
+		{
+		  edge srce = EDGE_SUCC (srcbb, i);
+		  edge dste = EDGE_SUCC (dstbb, i);
+		  if (srce->probability.initialized_p ())
+		    dste->probability = srce->probability;
+		}
+	    }	
+	  else if (srcbb->count.ipa ().initialized_p ()
+		   && !(srcbb->count.ipa () == profile_count::zero ()))
+	    {
+	      for (i = 0; i < EDGE_COUNT (srcbb->succs); i++)
+		{
+		  edge srce = EDGE_SUCC (srcbb, i);
+		  edge dste = EDGE_SUCC (dstbb, i);
+		  dste->probability = 
+		    dste->probability * dstbb->count.probability_in (dstbb->count + srcbb->count)
+		    + srce->probability * srcbb->count.probability_in (dstbb->count + srcbb->count);
+		}
+	      dstbb->count += srcbb->count;
 	    }
 	}
       push_cfun (dstcfun);
@@ -548,7 +564,7 @@ ipa_merge_profiles (struct cgraph_node *dst,
 	{
 	  if (e->speculative)
 	    continue;
-	  e->count = gimple_bb (e->call_stmt)->count;
+	  e->count = gimple_bb (e->call_stmt)->count.ipa ();
 	  e->frequency = compute_call_stmt_bb_frequency
 			     (dst->decl,
 			      gimple_bb (e->call_stmt));
@@ -626,7 +642,7 @@ ipa_merge_profiles (struct cgraph_node *dst,
 	      ipa_ref *ref;
 
 	      e2->speculative_call_info (direct, indirect, ref);
-	      e->count = count;
+	      e->count = count.ipa ();
 	      e->frequency = freq;
 	      int prob = direct->count.probability_in (e->count)
 			 .to_reg_br_prob_base ();
@@ -635,7 +651,7 @@ ipa_merge_profiles (struct cgraph_node *dst,
 	    }
 	  else
 	    {
-	      e->count = count;
+	      e->count = count.ipa ();
 	      e->frequency = freq;
 	    }
 	}
diff --git a/gcc/ira-build.c b/gcc/ira-build.c
index 366b83e6df1..67c0305a168 100644
--- a/gcc/ira-build.c
+++ b/gcc/ira-build.c
@@ -2202,7 +2202,8 @@ loop_compare_func (const void *v1p, const void *v2p)
     return -1;
   if (! l1->to_remove_p && l2->to_remove_p)
     return 1;
-  if ((diff = l1->loop->header->frequency - l2->loop->header->frequency) != 0)
+  if ((diff = l1->loop->header->count.to_frequency (cfun)
+	      - l2->loop->header->count.to_frequency (cfun)) != 0)
     return diff;
   if ((diff = (int) loop_depth (l1->loop) - (int) loop_depth (l2->loop)) != 0)
     return diff;
@@ -2260,7 +2261,7 @@ mark_loops_for_removal (void)
 	  (ira_dump_file,
 	   "  Mark loop %d (header %d, freq %d, depth %d) for removal (%s)\n",
 	   sorted_loops[i]->loop_num, sorted_loops[i]->loop->header->index,
-	   sorted_loops[i]->loop->header->frequency,
+	   sorted_loops[i]->loop->header->count.to_frequency (cfun),
 	   loop_depth (sorted_loops[i]->loop),
 	   low_pressure_loop_node_p (sorted_loops[i]->parent)
 	   && low_pressure_loop_node_p (sorted_loops[i])
@@ -2293,7 +2294,7 @@ mark_all_loops_for_removal (void)
 	     "  Mark loop %d (header %d, freq %d, depth %d) for removal\n",
 	     ira_loop_nodes[i].loop_num,
 	     ira_loop_nodes[i].loop->header->index,
-	     ira_loop_nodes[i].loop->header->frequency,
+	     ira_loop_nodes[i].loop->header->count.to_frequency (cfun),
 	     loop_depth (ira_loop_nodes[i].loop));
       }
 }
diff --git a/gcc/ira-color.c b/gcc/ira-color.c
index 8be7f31c5e9..72f7dd9ba21 100644
--- a/gcc/ira-color.c
+++ b/gcc/ira-color.c
@@ -3004,14 +3004,13 @@ allocno_priority_compare_func (const void *v1p, const void *v2p)
 {
   ira_allocno_t a1 = *(const ira_allocno_t *) v1p;
   ira_allocno_t a2 = *(const ira_allocno_t *) v2p;
-  int pri1, pri2;
+  int pri1, pri2, diff;
 
   /* Assign hard reg to static chain pointer pseudo first when
      non-local goto is used.  */
-  if (non_spilled_static_chain_regno_p (ALLOCNO_REGNO (a1)))
-    return 1;
-  else if (non_spilled_static_chain_regno_p (ALLOCNO_REGNO (a2)))
-    return -1;
+  if ((diff = (non_spilled_static_chain_regno_p (ALLOCNO_REGNO (a2))
+	       - non_spilled_static_chain_regno_p (ALLOCNO_REGNO (a1)))) != 0)
+    return diff;
   pri1 = allocno_priorities[ALLOCNO_NUM (a1)];
   pri2 = allocno_priorities[ALLOCNO_NUM (a2)];
   if (pri2 != pri1)
diff --git a/gcc/ira.c b/gcc/ira.c
index 4345f7595d0..93d02093757 100644
--- a/gcc/ira.c
+++ b/gcc/ira.c
@@ -4419,6 +4419,12 @@ rtx_moveable_p (rtx *loc, enum op_type type)
 	 for a reason.  */
       return false;
 
+    case ASM_OPERANDS:
+      /* The same is true for volatile asm: it has unknown side effects, it
+         cannot be moved at will.  */
+      if (MEM_VOLATILE_P (x))
+	return false;
+
     default:
       break;
     }
diff --git a/gcc/jit/ChangeLog b/gcc/jit/ChangeLog
index 24df99057a0..87b5473922a 100644
--- a/gcc/jit/ChangeLog
+++ b/gcc/jit/ChangeLog
@@ -1,3 +1,10 @@
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* docs/internals/index.rst (Running the test suite): Document
+	PRESERVE_EXECUTABLES.
+	(Running under valgrind): Add markup to RUN_UNDER_VALGRIND.
+	* docs/_build/texinfo/libgccjit.texi: Regenerate.
+
 2017-10-04  David Malcolm  <dmalcolm@redhat.com>
 
 	* docs/cp/topics/expressions.rst (Vector expressions): New
diff --git a/gcc/jit/docs/_build/texinfo/libgccjit.texi b/gcc/jit/docs/_build/texinfo/libgccjit.texi
index 344c93e4ccf..a3b206f2dd4 100644
--- a/gcc/jit/docs/_build/texinfo/libgccjit.texi
+++ b/gcc/jit/docs/_build/texinfo/libgccjit.texi
@@ -19,7 +19,7 @@
 
 @copying
 @quotation
-libgccjit 8.0.0 (experimental 20171004), October 04, 2017
+libgccjit 8.0.0 (experimental 20171031), October 31, 2017
 
 David Malcolm
 
@@ -15016,7 +15016,12 @@ jit/build/gcc/testsuite/jit/jit.log
 
 @noindent
 
-The test executables can be seen as:
+The test executables are normally deleted after each test is run.  For
+debugging, they can be preserved by setting 
+@geindex PRESERVE_EXECUTABLES
+@geindex environment variable; PRESERVE_EXECUTABLES
+@code{PRESERVE_EXECUTABLES}
+in the environment.  If so, they can then be seen as:
 
 @example
 jit/build/gcc/testsuite/jit/*.exe
@@ -15029,7 +15034,9 @@ which can be run independently.
 You can compile and run individual tests by passing "jit.exp=TESTNAME" to RUNTESTFLAGS e.g.:
 
 @example
-[gcc] $ make check-jit RUNTESTFLAGS="-v -v -v jit.exp=test-factorial.c"
+[gcc] $ PRESERVE_EXECUTABLES= \
+          make check-jit \
+            RUNTESTFLAGS="-v -v -v jit.exp=test-factorial.c"
 @end example
 
 @noindent
@@ -15056,7 +15063,10 @@ and once a test has been compiled, you can debug it directly:
 @subsection Running under valgrind
 
 
-The jit testsuite detects if RUN_UNDER_VALGRIND is present in the
+The jit testsuite detects if 
+@geindex RUN_UNDER_VALGRIND
+@geindex environment variable; RUN_UNDER_VALGRIND
+@code{RUN_UNDER_VALGRIND} is present in the
 environment (with any value).  If it is present, it runs the test client
 code under valgrind@footnote{http://valgrind.org},
 specifcally, the default
diff --git a/gcc/jit/docs/internals/index.rst b/gcc/jit/docs/internals/index.rst
index cadf36283ef..4ad7f61f774 100644
--- a/gcc/jit/docs/internals/index.rst
+++ b/gcc/jit/docs/internals/index.rst
@@ -103,7 +103,9 @@ and detailed logs in:
 
   jit/build/gcc/testsuite/jit/jit.log
 
-The test executables can be seen as:
+The test executables are normally deleted after each test is run.  For
+debugging, they can be preserved by setting :envvar:`PRESERVE_EXECUTABLES`
+in the environment.  If so, they can then be seen as:
 
 .. code-block:: console
 
@@ -115,7 +117,9 @@ You can compile and run individual tests by passing "jit.exp=TESTNAME" to RUNTES
 
 .. code-block:: console
 
-   [gcc] $ make check-jit RUNTESTFLAGS="-v -v -v jit.exp=test-factorial.c"
+   [gcc] $ PRESERVE_EXECUTABLES= \
+             make check-jit \
+               RUNTESTFLAGS="-v -v -v jit.exp=test-factorial.c"
 
 and once a test has been compiled, you can debug it directly:
 
@@ -130,7 +134,7 @@ and once a test has been compiled, you can debug it directly:
 Running under valgrind
 **********************
 
-The jit testsuite detects if RUN_UNDER_VALGRIND is present in the
+The jit testsuite detects if :envvar:`RUN_UNDER_VALGRIND` is present in the
 environment (with any value).  If it is present, it runs the test client
 code under `valgrind <http://valgrind.org>`_,
 specifcally, the default
diff --git a/gcc/langhooks.c b/gcc/langhooks.c
index c54b790f0cc..9b3212b90cf 100644
--- a/gcc/langhooks.c
+++ b/gcc/langhooks.c
@@ -266,8 +266,8 @@ lhd_gimplify_expr (tree *expr_p ATTRIBUTE_UNUSED,
 }
 
 /* lang_hooks.tree_size: Determine the size of a tree with code C,
-   which is a language-specific tree code in category tcc_constant or
-   tcc_exceptional.  The default expects never to be called.  */
+   which is a language-specific tree code in category tcc_constant,
+   tcc_exceptional or tcc_type.  The default expects never to be called.  */
 size_t
 lhd_tree_size (enum tree_code c ATTRIBUTE_UNUSED)
 {
diff --git a/gcc/langhooks.h b/gcc/langhooks.h
index b0c9829a6cd..d1288f1965d 100644
--- a/gcc/langhooks.h
+++ b/gcc/langhooks.h
@@ -307,10 +307,10 @@ struct lang_hooks
   /* Remove any parts of the tree that are used only by the FE. */
   void (*free_lang_data) (tree);
 
-  /* Determines the size of any language-specific tcc_constant or
-     tcc_exceptional nodes.  Since it is called from make_node, the
-     only information available is the tree code.  Expected to die
-     on unrecognized codes.  */
+  /* Determines the size of any language-specific tcc_constant,
+     tcc_exceptional or tcc_type nodes.  Since it is called from
+     make_node, the only information available is the tree code.
+     Expected to die on unrecognized codes.  */
   size_t (*tree_size) (enum tree_code);
 
   /* Return the language mask used for converting argv into a sequence
diff --git a/gcc/loop-doloop.c b/gcc/loop-doloop.c
index 5769d9deccb..3a1d838affd 100644
--- a/gcc/loop-doloop.c
+++ b/gcc/loop-doloop.c
@@ -393,9 +393,7 @@ add_test (rtx cond, edge *e, basic_block dest)
 
   edge e2 = make_edge (bb, dest, (*e)->flags & ~EDGE_FALLTHRU);
   e2->probability = prob;
-  e2->count = e2->src->count.apply_probability (prob);
   (*e)->probability = prob.invert ();
-  (*e)->count = (*e)->count.apply_probability (prob);
   update_br_prob_note (e2->src);
   return true;
 }
@@ -508,7 +506,6 @@ doloop_modify (struct loop *loop, struct niter_desc *desc,
       set_immediate_dominator (CDI_DOMINATORS, new_preheader, preheader);
 
       set_zero->count = profile_count::uninitialized ();
-      set_zero->frequency = 0;
 
       te = single_succ_edge (preheader);
       for (; ass; ass = XEXP (ass, 1))
@@ -524,7 +521,6 @@ doloop_modify (struct loop *loop, struct niter_desc *desc,
 	     also be very hard to show that it is impossible, so we must
 	     handle this case.  */
 	  set_zero->count = preheader->count;
-	  set_zero->frequency = preheader->frequency;
 	}
 
       if (EDGE_COUNT (set_zero->preds) == 0)
diff --git a/gcc/loop-iv.c b/gcc/loop-iv.c
index 45e822980ff..1d0c66f2b2f 100644
--- a/gcc/loop-iv.c
+++ b/gcc/loop-iv.c
@@ -353,7 +353,7 @@ iv_get_reaching_def (rtx_insn *insn, rtx reg, df_ref *def)
   adef = DF_REF_CHAIN (use)->ref;
 
   /* We do not handle setting only part of the register.  */
-  if (DF_REF_FLAGS (adef) & (DF_REF_READ_WRITE | DF_REF_SUBREG))
+  if (DF_REF_FLAGS (adef) & DF_REF_READ_WRITE)
     return GRD_INVALID;
 
   def_insn = DF_REF_INSN (adef);
diff --git a/gcc/loop-unroll.c b/gcc/loop-unroll.c
index 322f151ac5d..91bf5dddeed 100644
--- a/gcc/loop-unroll.c
+++ b/gcc/loop-unroll.c
@@ -863,7 +863,7 @@ unroll_loop_runtime_iterations (struct loop *loop)
   unsigned i, j;
   profile_probability p;
   basic_block preheader, *body, swtch, ezc_swtch = NULL;
-  int may_exit_copy, iter_freq, new_freq;
+  int may_exit_copy;
   profile_count iter_count, new_count;
   unsigned n_peel;
   edge e;
@@ -970,14 +970,11 @@ unroll_loop_runtime_iterations (struct loop *loop)
   /* Record the place where switch will be built for preconditioning.  */
   swtch = split_edge (loop_preheader_edge (loop));
 
-  /* Compute frequency/count increments for each switch block and initialize
+  /* Compute count increments for each switch block and initialize
      innermost switch block.  Switch blocks and peeled loop copies are built
      from innermost outward.  */
-  iter_freq = new_freq = swtch->frequency / (max_unroll + 1);
   iter_count = new_count = swtch->count.apply_scale (1, max_unroll + 1);
-  swtch->frequency = new_freq;
   swtch->count = new_count;
-  single_succ_edge (swtch)->count = new_count;
 
   for (i = 0; i < n_peel; i++)
     {
@@ -996,10 +993,8 @@ unroll_loop_runtime_iterations (struct loop *loop)
       p = profile_probability::always ().apply_scale (1, i + 2);
 
       preheader = split_edge (loop_preheader_edge (loop));
-      /* Add in frequency/count of edge from switch block.  */
-      preheader->frequency += iter_freq;
+      /* Add in count of edge from switch block.  */
       preheader->count += iter_count;
-      single_succ_edge (preheader)->count = preheader->count;
       branch_code = compare_and_jump_seq (copy_rtx (niter), GEN_INT (j), EQ,
 					  block_label (preheader), p,
 					  NULL);
@@ -1011,14 +1006,10 @@ unroll_loop_runtime_iterations (struct loop *loop)
       swtch = split_edge_and_insert (single_pred_edge (swtch), branch_code);
       set_immediate_dominator (CDI_DOMINATORS, preheader, swtch);
       single_succ_edge (swtch)->probability = p.invert ();
-      single_succ_edge (swtch)->count = new_count;
-      new_freq += iter_freq;
       new_count += iter_count;
-      swtch->frequency = new_freq;
       swtch->count = new_count;
       e = make_edge (swtch, preheader,
 		     single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP);
-      e->count = iter_count;
       e->probability = p;
     }
 
@@ -1028,14 +1019,11 @@ unroll_loop_runtime_iterations (struct loop *loop)
       p = profile_probability::always ().apply_scale (1, max_unroll + 1);
       swtch = ezc_swtch;
       preheader = split_edge (loop_preheader_edge (loop));
-      /* Recompute frequency/count adjustments since initial peel copy may
+      /* Recompute count adjustments since initial peel copy may
 	 have exited and reduced those values that were computed above.  */
-      iter_freq = swtch->frequency / (max_unroll + 1);
       iter_count = swtch->count.apply_scale (1, max_unroll + 1);
-      /* Add in frequency/count of edge from switch block.  */
-      preheader->frequency += iter_freq;
+      /* Add in count of edge from switch block.  */
       preheader->count += iter_count;
-      single_succ_edge (preheader)->count = preheader->count;
       branch_code = compare_and_jump_seq (copy_rtx (niter), const0_rtx, EQ,
 					  block_label (preheader), p,
 					  NULL);
@@ -1044,10 +1032,8 @@ unroll_loop_runtime_iterations (struct loop *loop)
       swtch = split_edge_and_insert (single_succ_edge (swtch), branch_code);
       set_immediate_dominator (CDI_DOMINATORS, preheader, swtch);
       single_succ_edge (swtch)->probability = p.invert ();
-      single_succ_edge (swtch)->count -= iter_count;
       e = make_edge (swtch, preheader,
 		     single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP);
-      e->count = iter_count;
       e->probability = p;
     }
 
diff --git a/gcc/lower-subreg.c b/gcc/lower-subreg.c
index dd853d799bc..0e76a718a1e 100644
--- a/gcc/lower-subreg.c
+++ b/gcc/lower-subreg.c
@@ -670,7 +670,7 @@ simplify_gen_subreg_concatn (machine_mode outermode, rtx op,
 
       if (must_eq (GET_MODE_SIZE (GET_MODE (op)),
 		   GET_MODE_SIZE (GET_MODE (SUBREG_REG (op))))
-	  && known_zero (SUBREG_BYTE (op)))
+	  && must_eq (SUBREG_BYTE (op), 0))
 	return simplify_gen_subreg_concatn (outermode, SUBREG_REG (op),
 					    GET_MODE (SUBREG_REG (op)), byte);
 
@@ -869,7 +869,7 @@ resolve_simple_move (rtx set, rtx_insn *insn)
 
   if (GET_CODE (src) == SUBREG
       && resolve_reg_p (SUBREG_REG (src))
-      && (maybe_nonzero (SUBREG_BYTE (src))
+      && (may_ne (SUBREG_BYTE (src), 0)
 	  || may_ne (orig_size, GET_MODE_SIZE (GET_MODE (SUBREG_REG (src))))))
     {
       real_dest = dest;
@@ -883,7 +883,7 @@ resolve_simple_move (rtx set, rtx_insn *insn)
 
   if (GET_CODE (dest) == SUBREG
       && resolve_reg_p (SUBREG_REG (dest))
-      && (maybe_nonzero (SUBREG_BYTE (dest))
+      && (may_ne (SUBREG_BYTE (dest), 0)
 	  || may_ne (orig_size, GET_MODE_SIZE (GET_MODE (SUBREG_REG (dest))))))
     {
       rtx reg, smove;
diff --git a/gcc/lra-constraints.c b/gcc/lra-constraints.c
index ff192733955..c2902fd682f 100644
--- a/gcc/lra-constraints.c
+++ b/gcc/lra-constraints.c
@@ -3170,7 +3170,7 @@ equiv_address_substitution (struct address_info *ad)
 	  change_p = true;
 	}
     }
-  if (maybe_nonzero (disp))
+  if (may_ne (disp, 0))
     {
       if (ad->disp != NULL)
 	*ad->disp = plus_constant (GET_MODE (*ad->inner), *ad->disp, disp);
@@ -4013,7 +4013,7 @@ curr_insn_transform (bool check_only_p)
       if (INSN_CODE (curr_insn) >= 0
           && (p = get_insn_name (INSN_CODE (curr_insn))) != NULL)
         fprintf (lra_dump_file, " {%s}", p);
-      if (maybe_nonzero (curr_id->sp_offset))
+      if (may_ne (curr_id->sp_offset, 0))
 	{
 	  fprintf (lra_dump_file, " (sp_off=");
 	  print_dec (curr_id->sp_offset, lra_dump_file);
@@ -4224,8 +4224,9 @@ curr_insn_transform (bool check_only_p)
 	      reg = SUBREG_REG (*loc);
 	      poly_int64 byte = SUBREG_BYTE (*loc);
 	      if (REG_P (reg)
-		  /* Strict_low_part requires reload the register not
-		     the sub-register.	*/
+		  /* Strict_low_part requires reloading the register and not
+		     just the subreg.  Likewise for a strict subreg no wider
+		     than a word for WORD_REGISTER_OPERATIONS targets.  */
 		  && (curr_static_id->operand[i].strict_low
 		      || (!paradoxical_subreg_p (mode, GET_MODE (reg))
 			  && (hard_regno
@@ -4236,7 +4237,11 @@ curr_insn_transform (bool check_only_p)
 			  && (goal_alt[i] == NO_REGS
 			      || (simplify_subreg_regno
 				  (ira_class_hard_regs[goal_alt[i]][0],
-				   GET_MODE (reg), byte, mode) >= 0)))))
+				   GET_MODE (reg), byte, mode) >= 0)))
+		      || (partial_subreg_p (mode, GET_MODE (reg))
+			  && must_le (GET_MODE_SIZE (GET_MODE (reg)),
+				      UNITS_PER_WORD)
+			  && WORD_REGISTER_OPERATIONS)))
 		{
 		  /* An OP_INOUT is required when reloading a subreg of a
 		     mode wider than a word to ensure that data beyond the
@@ -4283,7 +4288,13 @@ curr_insn_transform (bool check_only_p)
 	}
       else if (curr_static_id->operand[i].type == OP_IN
 	       && (curr_static_id->operand[goal_alt_matched[i][0]].type
-		   == OP_OUT))
+		   == OP_OUT
+		   || (curr_static_id->operand[goal_alt_matched[i][0]].type
+		       == OP_INOUT
+		       && (operands_match_p
+			   (*curr_id->operand_loc[i],
+			    *curr_id->operand_loc[goal_alt_matched[i][0]],
+			    -1)))))
 	{
 	  /* generate reloads for input and matched outputs.  */
 	  match_inputs[0] = i;
@@ -4294,9 +4305,14 @@ curr_insn_transform (bool check_only_p)
 			[goal_alt_number * n_operands + goal_alt_matched[i][0]]
 			.earlyclobber);
 	}
-      else if (curr_static_id->operand[i].type == OP_OUT
+      else if ((curr_static_id->operand[i].type == OP_OUT
+		|| (curr_static_id->operand[i].type == OP_INOUT
+		    && (operands_match_p
+			(*curr_id->operand_loc[i],
+			 *curr_id->operand_loc[goal_alt_matched[i][0]],
+			 -1))))
 	       && (curr_static_id->operand[goal_alt_matched[i][0]].type
-		   == OP_IN))
+		    == OP_IN))
 	/* Generate reloads for output and matched inputs.  */
 	match_reload (i, goal_alt_matched[i], outputs, goal_alt[i], &before,
 		      &after, curr_static_id->operand_alternative
diff --git a/gcc/lra-eliminations.c b/gcc/lra-eliminations.c
index b958190c2c7..bea8b023b7c 100644
--- a/gcc/lra-eliminations.c
+++ b/gcc/lra-eliminations.c
@@ -264,7 +264,7 @@ get_elimination (rtx reg)
   if ((ep = elimination_map[hard_regno]) != NULL)
     return ep->from_rtx != reg ? NULL : ep;
   poly_int64 offset = self_elim_offsets[hard_regno];
-  if (known_zero (offset))
+  if (must_eq (offset, 0))
     return NULL;
   /* This is an iteration to restore offsets just after HARD_REGNO
      stopped to be eliminable.	*/
@@ -340,7 +340,7 @@ lra_eliminate_regs_1 (rtx_insn *insn, rtx x, machine_mode mem_mode,
   int copied = 0;
 
   lra_assert (!update_p || !full_p);
-  lra_assert (known_zero (update_sp_offset)
+  lra_assert (must_eq (update_sp_offset, 0)
 	      || (!subst_p && update_p && !full_p));
   if (! current_function_decl)
     return x;
@@ -366,7 +366,7 @@ lra_eliminate_regs_1 (rtx_insn *insn, rtx x, machine_mode mem_mode,
 	{
 	  rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
 
-	  if (maybe_nonzero (update_sp_offset))
+	  if (may_ne (update_sp_offset, 0))
 	    {
 	      if (ep->to_rtx == stack_pointer_rtx)
 		return plus_constant (Pmode, to, update_sp_offset);
@@ -399,7 +399,7 @@ lra_eliminate_regs_1 (rtx_insn *insn, rtx x, machine_mode mem_mode,
 	      if (! update_p && ! full_p)
 		return gen_rtx_PLUS (Pmode, to, XEXP (x, 1));
 	      
-	      if (maybe_nonzero (update_sp_offset))
+	      if (may_ne (update_sp_offset, 0))
 		offset = ep->to_rtx == stack_pointer_rtx ? update_sp_offset : 0;
 	      else
 		offset = (update_p
@@ -456,7 +456,7 @@ lra_eliminate_regs_1 (rtx_insn *insn, rtx x, machine_mode mem_mode,
 	{
 	  rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
 
-	  if (maybe_nonzero (update_sp_offset))
+	  if (may_ne (update_sp_offset, 0))
 	    {
 	      if (ep->to_rtx == stack_pointer_rtx)
 		return plus_constant (Pmode,
@@ -952,7 +952,7 @@ eliminate_regs_in_insn (rtx_insn *insn, bool replace_p, bool first_p,
 
 		/* We should never process such insn with non-zero
 		   UPDATE_SP_OFFSET.  */
-		lra_assert (known_zero (update_sp_offset));
+		lra_assert (must_eq (update_sp_offset, 0));
 		
 		if (remove_reg_equal_offset_note (insn, ep->to_rtx, &offset)
 		    || strip_offset (src, &offset) == ep->to_rtx)
@@ -1032,7 +1032,7 @@ eliminate_regs_in_insn (rtx_insn *insn, bool replace_p, bool first_p,
 
 	  if (! replace_p)
 	    {
-	      if (known_zero (update_sp_offset))
+	      if (must_eq (update_sp_offset, 0))
 		offset += (ep->offset - ep->previous_offset);
 	      if (ep->to_rtx == stack_pointer_rtx)
 		{
@@ -1051,7 +1051,7 @@ eliminate_regs_in_insn (rtx_insn *insn, bool replace_p, bool first_p,
 	     the cost of the insn by replacing a simple REG with (plus
 	     (reg sp) CST).  So try only when we already had a PLUS
 	     before.  */
-	  if (known_zero (offset) || plus_src)
+	  if (must_eq (offset, 0) || plus_src)
 	    {
 	      rtx new_src = plus_constant (GET_MODE (to_rtx), to_rtx, offset);
 
@@ -1239,7 +1239,7 @@ update_reg_eliminate (bitmap insns_with_changed_offsets)
 	      if (lra_dump_file != NULL)
 		fprintf (lra_dump_file, "    Using elimination %d to %d now\n",
 			 ep1->from, ep1->to);
-	      lra_assert (known_zero (ep1->previous_offset));
+	      lra_assert (must_eq (ep1->previous_offset, 0));
 	      ep1->previous_offset = ep->offset;
 	    }
 	  else
@@ -1251,7 +1251,7 @@ update_reg_eliminate (bitmap insns_with_changed_offsets)
 		fprintf (lra_dump_file, "    %d is not eliminable at all\n",
 			 ep->from);
 	      self_elim_offsets[ep->from] = -ep->offset;
-	      if (maybe_nonzero (ep->offset))
+	      if (may_ne (ep->offset, 0))
 		bitmap_ior_into (insns_with_changed_offsets,
 				 &lra_reg_info[ep->from].insn_bitmap);
 	    }
@@ -1357,13 +1357,13 @@ init_elimination (void)
 	    if (NONDEBUG_INSN_P (insn))
 	      {
 		mark_not_eliminable (PATTERN (insn), VOIDmode);
-		if (maybe_nonzero (curr_sp_change)
+		if (may_ne (curr_sp_change, 0)
 		    && find_reg_note (insn, REG_LABEL_OPERAND, NULL_RTX))
 		  stop_to_sp_elimination_p = true;
 	      }
 	  }
       if (! frame_pointer_needed
-	  && (maybe_nonzero (curr_sp_change) || stop_to_sp_elimination_p)
+	  && (may_ne (curr_sp_change, 0) || stop_to_sp_elimination_p)
 	  && bb->succs && bb->succs->length () != 0)
 	for (ep = reg_eliminate; ep < &reg_eliminate[NUM_ELIMINABLE_REGS]; ep++)
 	  if (ep->to == STACK_POINTER_REGNUM)
diff --git a/gcc/lra-lives.c b/gcc/lra-lives.c
index 4648eca5ace..df7e2537dd0 100644
--- a/gcc/lra-lives.c
+++ b/gcc/lra-lives.c
@@ -220,6 +220,9 @@ lra_intersected_live_ranges_p (lra_live_range_t r1, lra_live_range_t r2)
   return false;
 }
 
+/* The corresponding bitmaps of BB currently being processed.  */
+static bitmap bb_killed_pseudos, bb_gen_pseudos;
+
 /* The function processing birth of hard register REGNO.  It updates
    living hard regs, START_LIVING, and conflict hard regs for living
    pseudos.  Conflict hard regs for the pic pseudo is not updated if
@@ -243,6 +246,8 @@ make_hard_regno_born (int regno, bool check_pic_pseudo_p ATTRIBUTE_UNUSED)
 	|| i != REGNO (pic_offset_table_rtx))
 #endif
       SET_HARD_REG_BIT (lra_reg_info[i].conflict_hard_regs, regno);
+  if (fixed_regs[regno])
+    bitmap_set_bit (bb_gen_pseudos, regno);
 }
 
 /* Process the death of hard register REGNO.  This updates
@@ -255,6 +260,11 @@ make_hard_regno_dead (int regno)
     return;
   sparseset_set_bit (start_dying, regno);
   CLEAR_HARD_REG_BIT (hard_regs_live, regno);
+  if (fixed_regs[regno])
+    {
+      bitmap_clear_bit (bb_gen_pseudos, regno);
+      bitmap_set_bit (bb_killed_pseudos, regno);
+    }
 }
 
 /* Mark pseudo REGNO as living at program point POINT, update conflicting
@@ -299,9 +309,6 @@ mark_pseudo_dead (int regno, int point)
     }
 }
 
-/* The corresponding bitmaps of BB currently being processed.  */
-static bitmap bb_killed_pseudos, bb_gen_pseudos;
-
 /* Mark register REGNO (pseudo or hard register) in MODE as live at
    program point POINT.  Update BB_GEN_PSEUDOS.
    Return TRUE if the liveness tracking sets were modified, or FALSE
diff --git a/gcc/lra-remat.c b/gcc/lra-remat.c
index 2828d42cd13..d549271128d 100644
--- a/gcc/lra-remat.c
+++ b/gcc/lra-remat.c
@@ -1242,7 +1242,7 @@ do_remat (void)
 	  if (remat_insn != NULL)
 	    {
 	      poly_int64 sp_offset_change = cand_sp_offset - id->sp_offset;
-	      if (maybe_nonzero (sp_offset_change))
+	      if (may_ne (sp_offset_change, 0))
 		change_sp_offset (remat_insn, sp_offset_change);
 	      update_scratch_ops (remat_insn);
 	      lra_process_new_insns (insn, remat_insn, NULL,
diff --git a/gcc/lra.c b/gcc/lra.c
index 64c5cfbea1c..8d44c75b0b4 100644
--- a/gcc/lra.c
+++ b/gcc/lra.c
@@ -820,7 +820,8 @@ collect_non_operand_hard_regs (rtx *x, lra_insn_recog_data_t data,
   const char *fmt = GET_RTX_FORMAT (code);
 
   for (i = 0; i < data->insn_static_data->n_operands; i++)
-    if (x == data->operand_loc[i])
+    if (! data->insn_static_data->operand[i].is_operator
+	&& x == data->operand_loc[i])
       /* It is an operand loc. Stop here.  */
       return list;
   for (i = 0; i < data->insn_static_data->n_dups; i++)
@@ -2371,7 +2372,7 @@ lra (FILE *f)
   bitmap_initialize (&lra_optional_reload_pseudos, &reg_obstack);
   bitmap_initialize (&lra_subreg_reload_pseudos, &reg_obstack);
   live_p = false;
-  if (maybe_nonzero (get_frame_size ()) && crtl->stack_alignment_needed)
+  if (may_ne (get_frame_size (), 0) && crtl->stack_alignment_needed)
     /* If we have a stack frame, we must align it now.  The stack size
        may be a part of the offset computation for register
        elimination.  */
diff --git a/gcc/lto-streamer-in.c b/gcc/lto-streamer-in.c
index babdcfb7cf6..4682be089c4 100644
--- a/gcc/lto-streamer-in.c
+++ b/gcc/lto-streamer-in.c
@@ -715,8 +715,7 @@ make_new_block (struct function *fn, unsigned int index)
 
 static void
 input_cfg (struct lto_input_block *ib, struct data_in *data_in,
-	   struct function *fn,
-	   int count_materialization_scale)
+	   struct function *fn)
 {
   unsigned int bb_count;
   basic_block p_bb;
@@ -756,13 +755,10 @@ input_cfg (struct lto_input_block *ib, struct data_in *data_in,
 	  unsigned int edge_flags;
 	  basic_block dest;
 	  profile_probability probability;
-	  profile_count count;
 	  edge e;
 
 	  dest_index = streamer_read_uhwi (ib);
 	  probability = profile_probability::stream_in (ib);
-	  count = profile_count::stream_in (ib).apply_scale
-			 (count_materialization_scale, REG_BR_PROB_BASE);
 	  edge_flags = streamer_read_uhwi (ib);
 
 	  dest = BASIC_BLOCK_FOR_FN (fn, dest_index);
@@ -772,7 +768,6 @@ input_cfg (struct lto_input_block *ib, struct data_in *data_in,
 
 	  e = make_edge (bb, dest, edge_flags);
 	  e->probability = probability;
-	  e->count = count;
 	}
 
       index = streamer_read_hwi (ib);
@@ -1070,7 +1065,7 @@ input_function (tree fn_decl, struct data_in *data_in,
   if (!node)
     node = cgraph_node::create (fn_decl);
   input_struct_function_base (fn, data_in, ib);
-  input_cfg (ib_cfg, data_in, fn, node->count_materialization_scale);
+  input_cfg (ib_cfg, data_in, fn);
 
   /* Read all the SSA names.  */
   input_ssa_names (ib, data_in, fn);
@@ -1197,6 +1192,7 @@ input_function (tree fn_decl, struct data_in *data_in,
     gimple_set_body (fn_decl, bb_seq (ei_edge (ei)->dest));
   }
 
+  counts_to_freqs ();
   fixup_call_stmt_edges (node, stmts);
   execute_all_ipa_stmt_fixups (node, stmts);
 
diff --git a/gcc/lto-streamer-out.c b/gcc/lto-streamer-out.c
index cf98eed52a3..12a3249debb 100644
--- a/gcc/lto-streamer-out.c
+++ b/gcc/lto-streamer-out.c
@@ -1885,7 +1885,6 @@ output_cfg (struct output_block *ob, struct function *fn)
 	{
 	  streamer_write_uhwi (ob, e->dest->index);
 	  e->probability.stream_out (ob);
-	  e->count.stream_out (ob);
 	  streamer_write_uhwi (ob, e->flags);
 	}
     }
diff --git a/gcc/lto/ChangeLog b/gcc/lto/ChangeLog
index 3e6b00bc487..173cde67369 100644
--- a/gcc/lto/ChangeLog
+++ b/gcc/lto/ChangeLog
@@ -1,3 +1,7 @@
+2017-10-13  Jan Hubicka  <hubicka@ucw.cz>
+
+	* lto-lang.c (lto_post_options): Clean shlib flag when not doing PIC.
+
 2017-10-11  Nathan Sidwell  <nathan@acm.org>
 
 	* lto.c (mentions_vars_p_decl_with_vis): Use
diff --git a/gcc/lto/lto-lang.c b/gcc/lto/lto-lang.c
index 6062deea5f3..a2f2c931338 100644
--- a/gcc/lto/lto-lang.c
+++ b/gcc/lto/lto-lang.c
@@ -854,11 +854,13 @@ lto_post_options (const char **pfilename ATTRIBUTE_UNUSED)
          flag_pie is 2.  */
       flag_pie = MAX (flag_pie, flag_pic);
       flag_pic = flag_pie;
+      flag_shlib = 0;
       break;
 
     case LTO_LINKER_OUTPUT_EXEC: /* Normal executable */
       flag_pic = 0;
       flag_pie = 0;
+      flag_shlib = 0;
       break;
 
     case LTO_LINKER_OUTPUT_UNKNOWN:
diff --git a/gcc/machmode.def b/gcc/machmode.def
index afe685195ef..dcf10565958 100644
--- a/gcc/machmode.def
+++ b/gcc/machmode.def
@@ -142,6 +142,12 @@ along with GCC; see the file COPYING3.  If not see
 	than two bytes (if CLASS is FLOAT).  CLASS must be INT or
 	FLOAT.  The names follow the same rule as VECTOR_MODE uses.
 
+     VECTOR_BOOL_MODE (COUNT, BYTESIZE)
+	Create a vector of booleans with COUNT elements and BYTESIZE bytes.
+	Each boolean occupies (COUNT * BITS_PER_UNIT) / BYTESIZE bits,
+	with the element at index 0 occupying the lsb of the first byte
+	in memory.  Only the lowest bit of each element is significant.
+
      COMPLEX_MODES (CLASS);
         For all modes presently declared in class CLASS, construct
 	corresponding complex modes.  Modes smaller than one byte
@@ -163,6 +169,12 @@ along with GCC; see the file COPYING3.  If not see
 	Unlike a FORMAT argument, if you are adjusting a float format
 	you must put an & in front of the name of each format structure.
 
+     ADJUST_NUNITS (MODE, EXPR);
+	Like the above, but set the number of nunits of MODE to EXPR.
+	This changes the size and precision of the mode in proportion
+	to the change in the number of units; for example, doubling
+	the number of units doubles the size and precision as well.
+
    Note: If a mode is ever made which is more than 255 bytes wide,
    machmode.h and genmodes.c will have to be changed to allocate
    more space for the mode_size and mode_alignment arrays.  */
diff --git a/gcc/match.pd b/gcc/match.pd
index 11c04dba77d..63566df3205 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -1514,12 +1514,31 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
        t1 = TYPE_OVERFLOW_WRAPS (type) ? type : TREE_TYPE (@1);
     }
     (convert (plus (convert:t1 @0) (convert:t1 @1))))))
- /* -(-A) -> A */
+ /* -(T)(-A) -> (T)A
+    Sign-extension is ok except for INT_MIN, which thankfully cannot
+    happen without overflow.  */
  (simplify
-  (negate (convert? (negate @1)))
-  (if (tree_nop_conversion_p (type, TREE_TYPE (@1))
-       && !TYPE_OVERFLOW_SANITIZED (type))
+  (negate (convert (negate @1)))
+  (if (INTEGRAL_TYPE_P (type)
+       && (TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@1))
+	   || (!TYPE_UNSIGNED (TREE_TYPE (@1))
+	       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@1))))
+       && !TYPE_OVERFLOW_SANITIZED (type)
+       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@1)))
    (convert @1)))
+ (simplify
+  (negate (convert negate_expr_p@1))
+  (if (SCALAR_FLOAT_TYPE_P (type)
+       && ((DECIMAL_FLOAT_TYPE_P (type)
+	    == DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@1))
+	    && TYPE_PRECISION (type) >= TYPE_PRECISION (TREE_TYPE (@1)))
+	   || !HONOR_SIGN_DEPENDENT_ROUNDING (type)))
+   (convert (negate @1))))
+ (simplify
+  (negate (nop_convert (negate @1)))
+  (if (!TYPE_OVERFLOW_SANITIZED (type)
+       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@1)))
+   (view_convert @1)))
 
  /* We can't reassociate floating-point unless -fassociative-math
     or fixed-point plus or minus because of saturation to +-Inf.  */
diff --git a/gcc/modulo-sched.c b/gcc/modulo-sched.c
index f85011e90c2..71b2a616096 100644
--- a/gcc/modulo-sched.c
+++ b/gcc/modulo-sched.c
@@ -1422,15 +1422,15 @@ sms_schedule (void)
       get_ebb_head_tail (bb, bb, &head, &tail);
       latch_edge = loop_latch_edge (loop);
       gcc_assert (single_exit (loop));
-      if (single_exit (loop)->count > profile_count::zero ())
-	trip_count = latch_edge->count.to_gcov_type ()
-		     / single_exit (loop)->count.to_gcov_type ();
+      if (single_exit (loop)->count () > profile_count::zero ())
+	trip_count = latch_edge->count ().to_gcov_type ()
+		     / single_exit (loop)->count ().to_gcov_type ();
 
       /* Perform SMS only on loops that their average count is above threshold.  */
 
-      if ( latch_edge->count > profile_count::zero ()
-          && (latch_edge->count
-	      < single_exit (loop)->count.apply_scale
+      if ( latch_edge->count () > profile_count::zero ()
+          && (latch_edge->count()
+	      < single_exit (loop)->count ().apply_scale
 				 (SMS_LOOP_AVERAGE_COUNT_THRESHOLD, 1)))
 	{
 	  if (dump_file)
@@ -1552,9 +1552,9 @@ sms_schedule (void)
 
       latch_edge = loop_latch_edge (loop);
       gcc_assert (single_exit (loop));
-      if (single_exit (loop)->count > profile_count::zero ())
-	trip_count = latch_edge->count.to_gcov_type ()
-		     / single_exit (loop)->count.to_gcov_type ();
+      if (single_exit (loop)->count ()> profile_count::zero ())
+	trip_count = latch_edge->count ().to_gcov_type ()
+		     / single_exit (loop)->count ().to_gcov_type ();
 
       if (dump_file)
 	{
diff --git a/gcc/objc/ChangeLog b/gcc/objc/ChangeLog
index 20b0fe44b29..7d865928999 100644
--- a/gcc/objc/ChangeLog
+++ b/gcc/objc/ChangeLog
@@ -1,3 +1,12 @@
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* objc-gnu-runtime-abi-01.c (objc_gnu_runtime_abi_01_init): Use
+	UNKNOWN_LOCATION rather than 0.
+
+2017-10-17  Nathan Sidwell  <nathan@acm.org>
+
+	* objc-act.c (objc_common_tree_size): Return size of TYPE nodes.
+
 2017-10-10  Richard Sandiford  <richard.sandiford@linaro.org>
 
 	* objc-act.c (objc_decl_method_attributes): Use wi::to_wide when
diff --git a/gcc/objc/objc-act.c b/gcc/objc/objc-act.c
index ce2adcc0ded..765192c82aa 100644
--- a/gcc/objc/objc-act.c
+++ b/gcc/objc/objc-act.c
@@ -10118,11 +10118,14 @@ objc_common_tree_size (enum tree_code code)
     case CLASS_METHOD_DECL:
     case INSTANCE_METHOD_DECL:
     case KEYWORD_DECL:
-    case PROPERTY_DECL:
-      return sizeof (struct tree_decl_non_common);
+    case PROPERTY_DECL:			return sizeof (tree_decl_non_common);
+    case CLASS_INTERFACE_TYPE:
+    case CLASS_IMPLEMENTATION_TYPE:
+    case CATEGORY_INTERFACE_TYPE:
+    case CATEGORY_IMPLEMENTATION_TYPE:
+    case PROTOCOL_INTERFACE_TYPE:	return sizeof (tree_type_non_common);
     default:
       gcc_unreachable ();
-  
     }
 }
 
diff --git a/gcc/objc/objc-gnu-runtime-abi-01.c b/gcc/objc/objc-gnu-runtime-abi-01.c
index b53d1820db3..4321b365358 100644
--- a/gcc/objc/objc-gnu-runtime-abi-01.c
+++ b/gcc/objc/objc-gnu-runtime-abi-01.c
@@ -130,7 +130,8 @@ objc_gnu_runtime_abi_01_init (objc_runtime_hooks *rthooks)
   /* GNU runtime does not need the compiler to change code in order to do GC. */
   if (flag_objc_gc)
     {
-      warning_at (0, 0, "%<-fobjc-gc%> is ignored for %<-fgnu-runtime%>");
+      warning_at (UNKNOWN_LOCATION, 0,
+		  "%<-fobjc-gc%> is ignored for %<-fgnu-runtime%>");
       flag_objc_gc = 0;
     }
 
diff --git a/gcc/omp-expand.c b/gcc/omp-expand.c
index 130814e68f2..34a95aa15b6 100644
--- a/gcc/omp-expand.c
+++ b/gcc/omp-expand.c
@@ -1399,6 +1399,7 @@ expand_omp_taskreg (struct omp_region *region)
 
       if (optimize)
 	optimize_omp_library_calls (entry_stmt);
+      counts_to_freqs ();
       cgraph_edge::rebuild_edges ();
 
       /* Some EH regions might become dead, see PR34608.  If
diff --git a/gcc/omp-grid.c b/gcc/omp-grid.c
index a7b6f60aeaf..121c96ebe39 100644
--- a/gcc/omp-grid.c
+++ b/gcc/omp-grid.c
@@ -1315,6 +1315,7 @@ grid_attempt_target_gridification (gomp_target *target,
       n1 = fold_convert (itype, n1);
       n2 = fold_convert (itype, n2);
 
+      tree cond = fold_build2 (cond_code, boolean_type_node, n1, n2);
       tree step
 	= omp_get_for_step_from_incr (loc, gimple_omp_for_incr (inner_loop, i));
 
@@ -1328,6 +1329,7 @@ grid_attempt_target_gridification (gomp_target *target,
 			 fold_build1 (NEGATE_EXPR, itype, step));
       else
 	t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
+      t = fold_build3 (COND_EXPR, itype, cond, t, build_zero_cst (itype));
       if (grid.tiling)
 	{
 	  if (cond_code == GT_EXPR)
diff --git a/gcc/omp-low.c b/gcc/omp-low.c
index ec838c5a175..c17226c8ffc 100644
--- a/gcc/omp-low.c
+++ b/gcc/omp-low.c
@@ -3466,7 +3466,7 @@ omp_clause_aligned_alignment (tree clause)
 	machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
 	if (GET_MODE_CLASS (vmode) != classes[i + 1])
 	  continue;
-	while (maybe_nonzero (vs)
+	while (may_ne (vs, 0U)
 	       && must_lt (GET_MODE_SIZE (vmode), vs)
 	       && GET_MODE_2XWIDER_MODE (vmode).exists ())
 	  vmode = GET_MODE_2XWIDER_MODE (vmode).require ();
@@ -3506,7 +3506,7 @@ static bool
 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
 			      omplow_simd_context *sctx, tree &ivar, tree &lvar)
 {
-  if (known_zero (sctx->max_vf))
+  if (must_eq (sctx->max_vf, 0U))
     {
       sctx->max_vf = sctx->is_simt ? omp_max_simt_vf () : omp_max_vf ();
       if (may_gt (sctx->max_vf, 1U))
@@ -4670,7 +4670,7 @@ lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
 
   /* If max_vf is non-zero, then we can use only a vectorization factor
      up to the max_vf we chose.  So stick it into the safelen clause.  */
-  if (maybe_nonzero (sctx.max_vf))
+  if (may_ne (sctx.max_vf, 0U))
     {
       tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
 				OMP_CLAUSE_SAFELEN);
diff --git a/gcc/omp-simd-clone.c b/gcc/omp-simd-clone.c
index 30d60e636d2..9cd66e26c27 100644
--- a/gcc/omp-simd-clone.c
+++ b/gcc/omp-simd-clone.c
@@ -1141,6 +1141,7 @@ simd_clone_adjust (struct cgraph_node *node)
     {
       basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
       incr_bb = create_empty_bb (orig_exit);
+      incr_bb->count = profile_count::zero ();
       add_bb_to_loop (incr_bb, body_bb->loop_father);
       /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
 	 flag.  Set it now to be a FALLTHRU_EDGE.  */
@@ -1151,11 +1152,13 @@ simd_clone_adjust (struct cgraph_node *node)
 	{
 	  edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
 	  redirect_edge_succ (e, incr_bb);
+	  incr_bb->count += e->count ();
 	}
     }
   else if (node->simdclone->inbranch)
     {
       incr_bb = create_empty_bb (entry_bb);
+      incr_bb->count = profile_count::zero ();
       add_bb_to_loop (incr_bb, body_bb->loop_father);
     }
 
@@ -1252,6 +1255,7 @@ simd_clone_adjust (struct cgraph_node *node)
       gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
       edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
       e->probability = profile_probability::unlikely ().guessed ();
+      incr_bb->count += e->count ();
       edge fallthru = FALLTHRU_EDGE (loop->header);
       fallthru->flags = EDGE_FALSE_VALUE;
       fallthru->probability = profile_probability::likely ().guessed ();
diff --git a/gcc/optabs.c b/gcc/optabs.c
index a6635e15116..7b8c0f60c99 100644
--- a/gcc/optabs.c
+++ b/gcc/optabs.c
@@ -6329,10 +6329,10 @@ expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
   return true;
 }
 
-/* Generate asm volatile("" : : : "memory") as the memory barrier.  */
+/* Generate asm volatile("" : : : "memory") as the memory blockage.  */
 
 static void
-expand_asm_memory_barrier (void)
+expand_asm_memory_blockage (void)
 {
   rtx asm_op, clob;
 
@@ -6348,6 +6348,17 @@ expand_asm_memory_barrier (void)
   emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
 }
 
+/* Do not propagate memory accesses across this point.  */
+
+static void
+expand_memory_blockage (void)
+{
+  if (targetm.have_memory_blockage ())
+    emit_insn (targetm.gen_memory_blockage ());
+  else
+    expand_asm_memory_blockage ();
+}
+
 /* This routine will either emit the mem_thread_fence pattern or issue a 
    sync_synchronize to generate a fence for memory model MEMMODEL.  */
 
@@ -6359,14 +6370,14 @@ expand_mem_thread_fence (enum memmodel model)
   if (targetm.have_mem_thread_fence ())
     {
       emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
-      expand_asm_memory_barrier ();
+      expand_memory_blockage ();
     }
   else if (targetm.have_memory_barrier ())
     emit_insn (targetm.gen_memory_barrier ());
   else if (synchronize_libfunc != NULL_RTX)
     emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
   else
-    expand_asm_memory_barrier ();
+    expand_memory_blockage ();
 }
 
 /* Emit a signal fence with given memory model.  */
@@ -6377,7 +6388,7 @@ expand_mem_signal_fence (enum memmodel model)
   /* No machine barrier is required to implement a signal fence, but
      a compiler memory barrier must be issued, except for relaxed MM.  */
   if (!is_mm_relaxed (model))
-    expand_asm_memory_barrier ();
+    expand_memory_blockage ();
 }
 
 /* This function expands the atomic load operation:
@@ -6399,7 +6410,7 @@ expand_atomic_load (rtx target, rtx mem, enum memmodel model)
       struct expand_operand ops[3];
       rtx_insn *last = get_last_insn ();
       if (is_mm_seq_cst (model))
-	expand_asm_memory_barrier ();
+	expand_memory_blockage ();
 
       create_output_operand (&ops[0], target, mode);
       create_fixed_operand (&ops[1], mem);
@@ -6407,7 +6418,7 @@ expand_atomic_load (rtx target, rtx mem, enum memmodel model)
       if (maybe_expand_insn (icode, 3, ops))
 	{
 	  if (!is_mm_relaxed (model))
-	    expand_asm_memory_barrier ();
+	    expand_memory_blockage ();
 	  return ops[0].value;
 	}
       delete_insns_since (last);
@@ -6457,14 +6468,14 @@ expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
     {
       rtx_insn *last = get_last_insn ();
       if (!is_mm_relaxed (model))
-	expand_asm_memory_barrier ();
+	expand_memory_blockage ();
       create_fixed_operand (&ops[0], mem);
       create_input_operand (&ops[1], val, mode);
       create_integer_operand (&ops[2], model);
       if (maybe_expand_insn (icode, 3, ops))
 	{
 	  if (is_mm_seq_cst (model))
-	    expand_asm_memory_barrier ();
+	    expand_memory_blockage ();
 	  return const0_rtx;
 	}
       delete_insns_since (last);
@@ -7095,7 +7106,10 @@ maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
       if (mode != VOIDmode
 	  && must_eq (trunc_int_for_mode (op->int_value, mode),
 		      op->int_value))
-	goto input;
+	{
+	  op->value = gen_int_mode (op->int_value, mode);
+	  goto input;
+	}
       break;
     }
   return insn_operand_matches (icode, opno, op->value);
diff --git a/gcc/opts.c b/gcc/opts.c
index adf3d89851d..ac383d48ec1 100644
--- a/gcc/opts.c
+++ b/gcc/opts.c
@@ -37,7 +37,7 @@ static void set_Wstrict_aliasing (struct gcc_options *opts, int onoff);
 /* Indexed by enum debug_info_type.  */
 const char *const debug_type_names[] =
 {
-  "none", "stabs", "coff", "dwarf-2", "xcoff", "vms"
+  "none", "stabs", "dwarf-2", "xcoff", "vms"
 };
 
 /* Parse the -femit-struct-debug-detailed option value
@@ -1521,6 +1521,7 @@ const struct sanitizer_opts_s sanitizer_opts[] =
   SANITIZER_OPT (object-size, SANITIZE_OBJECT_SIZE, true),
   SANITIZER_OPT (vptr, SANITIZE_VPTR, true),
   SANITIZER_OPT (pointer-overflow, SANITIZE_POINTER_OVERFLOW, true),
+  SANITIZER_OPT (builtin, SANITIZE_BUILTIN, true),
   SANITIZER_OPT (all, ~0U, true),
 #undef SANITIZER_OPT
   { NULL, 0U, 0UL, false }
@@ -2350,10 +2351,6 @@ common_handle_option (struct gcc_options *opts,
                        loc);
       break;
 
-    case OPT_gcoff:
-      set_debug_level (SDB_DEBUG, false, arg, opts, opts_set, loc);
-      break;
-
     case OPT_gdwarf:
       if (arg && strlen (arg) != 0)
         {
diff --git a/gcc/output.h b/gcc/output.h
index e98a911c647..ede44476a76 100644
--- a/gcc/output.h
+++ b/gcc/output.h
@@ -308,11 +308,6 @@ extern void output_quoted_string (FILE *, const char *);
    This variable is defined  in final.c.  */
 extern rtx_sequence *final_sequence;
 
-/* The line number of the beginning of the current function.  Various
-   md code needs this so that it can output relative linenumbers.  */
-
-extern int sdb_begin_function_line;
-
 /* File in which assembler code is being written.  */
 
 #ifdef BUFSIZ
diff --git a/gcc/params.def b/gcc/params.def
index bc4e9e40fd6..4a96268c102 100644
--- a/gcc/params.def
+++ b/gcc/params.def
@@ -882,13 +882,6 @@ DEFPARAM (PARAM_GRAPHITE_MAX_ARRAYS_PER_SCOP,
 	  "maximum number of arrays per scop.",
 	  100, 0, 0)
 
-/* Maximal number of basic blocks in the functions analyzed by Graphite.  */
-
-DEFPARAM (PARAM_GRAPHITE_MIN_LOOPS_PER_FUNCTION,
-	  "graphite-min-loops-per-function",
-	  "minimal number of loops per function to be analyzed by Graphite.",
-	  2, 0, 0)
-
 DEFPARAM (PARAM_MAX_ISL_OPERATIONS,
 	  "max-isl-operations",
 	  "maximum number of isl operations, 0 means unlimited",
diff --git a/gcc/poly-int.h b/gcc/poly-int.h
index 61cf3213db8..73c0efd47e4 100644
--- a/gcc/poly-int.h
+++ b/gcc/poly-int.h
@@ -152,20 +152,24 @@ struct if_lossless<T1, T2, T3, true>
 /* poly_int_traits<T> describes an integer type T that might be polynomial
    or non-polynomial:
 
-   - poly_int_coeffs<T>::is_poly is true if T is a poly_int-based type
+   - poly_int_traits<T>::is_poly is true if T is a poly_int-based type
      and false otherwise.
 
-   - poly_int_coeffs<T>::num_coeffs gives the number of coefficients in T
+   - poly_int_traits<T>::num_coeffs gives the number of coefficients in T
      if T is a poly_int and 1 otherwise.
 
-   - poly_int_coeffs<T>::coeff_type gives the coefficent type of T if T
-     is a poly_int and T itself otherwise.  */
+   - poly_int_traits<T>::coeff_type gives the coefficent type of T if T
+     is a poly_int and T itself otherwise
+
+   - poly_int_traits<T>::int_type is a shorthand for
+     typename poly_coeff_traits<coeff_type>::int_type.  */
 template<typename T>
 struct poly_int_traits
 {
   static const bool is_poly = false;
   static const unsigned int num_coeffs = 1;
   typedef T coeff_type;
+  typedef typename poly_coeff_traits<T>::int_type int_type;
 };
 template<unsigned int N, typename C>
 struct poly_int_traits<poly_int_pod<N, C> >
@@ -173,6 +177,7 @@ struct poly_int_traits<poly_int_pod<N, C> >
   static const bool is_poly = true;
   static const unsigned int num_coeffs = N;
   typedef C coeff_type;
+  typedef typename poly_coeff_traits<C>::int_type int_type;
 };
 template<unsigned int N, typename C>
 struct poly_int_traits<poly_int<N, C> > : poly_int_traits<poly_int_pod<N, C> >
@@ -304,8 +309,7 @@ struct poly_result<T1, T2, 2>
 
 /* The type to which an integer constant should be cast before
    comparing it with T.  */
-#define POLY_INT_TYPE(T) \
-  typename poly_coeff_traits<typename poly_int_traits<T>::coeff_type>::int_type
+#define POLY_INT_TYPE(T) typename poly_int_traits<T>::int_type
 
 /* RES is a poly_int result that has coefficients of type C and that
    is being built up a coefficient at a time.  Set coefficient number I
@@ -1332,52 +1336,6 @@ may_ne (const Ca &a, const Cb &b)
 /* Return true if A must be unequal to B.  */
 #define must_ne(A, B) (!may_eq (A, B))
 
-/* Return true if A is known to be zero.  */
-
-template<typename T>
-inline bool
-known_zero (const T &a)
-{
-  typedef POLY_INT_TYPE (T) int_type;
-  return must_eq (a, int_type (0));
-}
-
-/* Return true if A is known to be nonzero.  */
-
-template<typename T>
-inline bool
-known_nonzero (const T &a)
-{
-  typedef POLY_INT_TYPE (T) int_type;
-  return must_ne (a, int_type (0));
-}
-
-/* Return true if A might be equal to zero.  */
-#define maybe_zero(A) (!known_nonzero (A))
-
-/* Return true if A might not be equal to zero.  */
-#define maybe_nonzero(A) (!known_zero (A))
-
-/* Return true if A is known to be equal to 1.  */
-
-template<typename T>
-inline bool
-known_one (const T &a)
-{
-  typedef POLY_INT_TYPE (T) int_type;
-  return must_eq (a, int_type (1));
-}
-
-/* Return true if A is known to be all ones.  */
-
-template<typename T>
-inline bool
-known_all_ones (const T &a)
-{
-  typedef POLY_INT_TYPE (T) int_type;
-  return must_eq (a, int_type (-1));
-}
-
 /* Return true if A might be less than or equal to B for some
    indeterminate values.  */
 
@@ -1596,8 +1554,7 @@ template<unsigned int N, typename Ca>
 inline Ca
 constant_lower_bound (const poly_int_pod<N, Ca> &a)
 {
-  typedef POLY_INT_TYPE (Ca) ICa;
-  gcc_checking_assert (must_ge (a, ICa (0)));
+  gcc_checking_assert (must_ge (a, POLY_INT_TYPE (Ca) (0)));
   return a.coeffs[0];
 }
 
@@ -2509,7 +2466,12 @@ struct poly_span_traits<T1, T2, T3, HOST_WIDE_INT, unsigned HOST_WIDE_INT>
 /* Return true if SIZE represents a known size, assuming that all-ones
    indicates an unknown size.  */
 
-#define known_size_p(X) (!known_all_ones (X))
+template<typename T>
+inline bool
+known_size_p (const T &a)
+{
+  return may_ne (a, POLY_INT_TYPE (T) (-1));
+}
 
 /* Return true if range [POS, POS + SIZE) might include VAL.
    SIZE can be the special value -1, in which case the range is
@@ -2557,9 +2519,9 @@ ranges_may_overlap_p (const T1 &pos1, const T2 &size1,
 		      const T3 &pos2, const T4 &size2)
 {
   if (maybe_in_range_p (pos2, pos1, size1))
-    return maybe_nonzero (size2);
+    return may_ne (size2, POLY_INT_TYPE (T4) (0));
   if (maybe_in_range_p (pos1, pos2, size2))
-    return maybe_nonzero (size1);
+    return may_ne (size1, POLY_INT_TYPE (T2) (0));
   return false;
 }
 
@@ -2602,10 +2564,9 @@ known_subrange_p (const T1 &pos1, const T2 &size1,
 		  const T3 &pos2, const T4 &size2)
 {
   typedef typename poly_int_traits<T2>::coeff_type C2;
-  typedef POLY_INT_TYPE (C2) IC2;
   typedef POLY_BINARY_COEFF (T2, T4) size_diff_type;
   typedef poly_span_traits<T1, T3, size_diff_type> span;
-  return (must_gt (size1, IC2 (0))
+  return (must_gt (size1, POLY_INT_TYPE (T2) (0))
 	  && (poly_coeff_traits<C2>::signedness > 0
 	      || known_size_p (size1))
 	  && known_size_p (size2)
diff --git a/gcc/postreload-gcse.c b/gcc/postreload-gcse.c
index a1dcac2600c..15fdb7e0cfe 100644
--- a/gcc/postreload-gcse.c
+++ b/gcc/postreload-gcse.c
@@ -1108,14 +1108,14 @@ eliminate_partially_redundant_load (basic_block bb, rtx_insn *insn,
 	    avail_insn = NULL;
 	}
 
-      if (EDGE_CRITICAL_P (pred) && pred->count.initialized_p ())
-	critical_count += pred->count;
+      if (EDGE_CRITICAL_P (pred) && pred->count ().initialized_p ())
+	critical_count += pred->count ();
 
       if (avail_insn != NULL_RTX)
 	{
 	  npred_ok++;
-	  if (pred->count.initialized_p ())
-	    ok_count = ok_count + pred->count;
+	  if (pred->count ().initialized_p ())
+	    ok_count = ok_count + pred->count ();
 	  if (! set_noop_p (PATTERN (gen_move_insn (copy_rtx (dest),
 						    copy_rtx (avail_reg)))))
 	    {
@@ -1139,8 +1139,8 @@ eliminate_partially_redundant_load (basic_block bb, rtx_insn *insn,
 	  /* Adding a load on a critical edge will cause a split.  */
 	  if (EDGE_CRITICAL_P (pred))
 	    critical_edge_split = true;
-	  if (pred->count.initialized_p ())
-	    not_ok_count = not_ok_count + pred->count;
+	  if (pred->count ().initialized_p ())
+	    not_ok_count = not_ok_count + pred->count ();
 	  unoccr = (struct unoccr *) obstack_alloc (&unoccr_obstack,
 						    sizeof (struct unoccr));
 	  unoccr->insn = NULL;
diff --git a/gcc/postreload.c b/gcc/postreload.c
index f5a26c55c48..a70d11a6c87 100644
--- a/gcc/postreload.c
+++ b/gcc/postreload.c
@@ -1706,7 +1706,7 @@ move2add_valid_value_p (int regno, scalar_int_mode mode)
 	 regno of the lowpart might be different.  */
       poly_int64 s_off = subreg_lowpart_offset (mode, old_mode);
       s_off = subreg_regno_offset (regno, old_mode, s_off, mode);
-      if (maybe_nonzero (s_off))
+      if (may_ne (s_off, 0))
 	/* We could in principle adjust regno, check reg_mode[regno] to be
 	   BLKmode, and return s_off to the caller (vs. -1 for failure),
 	   but we currently have no callers that could make use of this
diff --git a/gcc/predict.c b/gcc/predict.c
index e534502aaf5..cf42ccbd903 100644
--- a/gcc/predict.c
+++ b/gcc/predict.c
@@ -137,12 +137,12 @@ maybe_hot_frequency_p (struct function *fun, int freq)
   if (profile_status_for_fn (fun) == PROFILE_ABSENT)
     return true;
   if (node->frequency == NODE_FREQUENCY_EXECUTED_ONCE
-      && freq < (ENTRY_BLOCK_PTR_FOR_FN (fun)->frequency * 2 / 3))
+      && freq < (ENTRY_BLOCK_PTR_FOR_FN (fun)->count.to_frequency (cfun) * 2 / 3))
     return false;
   if (PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION) == 0)
     return false;
   if (freq * PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION)
-      < ENTRY_BLOCK_PTR_FOR_FN (fun)->frequency)
+      < ENTRY_BLOCK_PTR_FOR_FN (fun)->count.to_frequency (cfun))
     return false;
   return true;
 }
@@ -175,10 +175,14 @@ set_hot_bb_threshold (gcov_type min)
 /* Return TRUE if frequency FREQ is considered to be hot.  */
 
 bool
-maybe_hot_count_p (struct function *, profile_count count)
+maybe_hot_count_p (struct function *fun, profile_count count)
 {
   if (!count.initialized_p ())
     return true;
+  if (!count.ipa_p ())
+    return maybe_hot_frequency_p (fun, count.to_frequency (fun));
+  if (count.ipa () == profile_count::zero ())
+    return false;
   /* Code executed at most once is not hot.  */
   if (count <= MAX (profile_info ? profile_info->runs : 1, 1))
     return false;
@@ -192,9 +196,7 @@ bool
 maybe_hot_bb_p (struct function *fun, const_basic_block bb)
 {
   gcc_checking_assert (fun);
-  if (!maybe_hot_count_p (fun, bb->count))
-    return false;
-  return maybe_hot_frequency_p (fun, bb->frequency);
+  return maybe_hot_count_p (fun, bb->count);
 }
 
 /* Return true in case BB can be CPU intensive and should be optimized
@@ -203,9 +205,7 @@ maybe_hot_bb_p (struct function *fun, const_basic_block bb)
 bool
 maybe_hot_edge_p (edge e)
 {
-  if (!maybe_hot_count_p (cfun, e->count))
-    return false;
-  return maybe_hot_frequency_p (cfun, EDGE_FREQUENCY (e));
+  return maybe_hot_count_p (cfun, e->count ());
 }
 
 /* Return true if profile COUNT and FREQUENCY, or function FUN static
@@ -213,7 +213,7 @@ maybe_hot_edge_p (edge e)
    
 static bool
 probably_never_executed (struct function *fun,
-                         profile_count count, int)
+                         profile_count count)
 {
   gcc_checking_assert (fun);
   if (count == profile_count::zero ())
@@ -238,7 +238,7 @@ probably_never_executed (struct function *fun,
 bool
 probably_never_executed_bb_p (struct function *fun, const_basic_block bb)
 {
-  return probably_never_executed (fun, bb->count, bb->frequency);
+  return probably_never_executed (fun, bb->count);
 }
 
 
@@ -247,7 +247,7 @@ probably_never_executed_bb_p (struct function *fun, const_basic_block bb)
 static bool
 unlikely_executed_edge_p (edge e)
 {
-  return (e->count == profile_count::zero ()
+  return (e->count () == profile_count::zero ()
 	  || e->probability == profile_probability::never ())
 	 || (e->flags & (EDGE_EH | EDGE_FAKE));
 }
@@ -259,7 +259,7 @@ probably_never_executed_edge_p (struct function *fun, edge e)
 {
   if (unlikely_executed_edge_p (e))
     return true;
-  return probably_never_executed (fun, e->count, EDGE_FREQUENCY (e));
+  return probably_never_executed (fun, e->count ());
 }
 
 /* Return true when current function should always be optimized for size.  */
@@ -746,8 +746,8 @@ dump_prediction (FILE *file, enum br_predictor predictor, int probability,
       if (e)
 	{
 	  fprintf (file, " hit ");
-	  e->count.dump (file);
-	  fprintf (file, " (%.1f%%)", e->count.to_gcov_type() * 100.0
+	  e->count ().dump (file);
+	  fprintf (file, " (%.1f%%)", e->count ().to_gcov_type() * 100.0
 		   / bb->count.to_gcov_type ());
 	}
     }
@@ -1289,7 +1289,8 @@ combine_predictions_for_bb (basic_block bb, bool dry_run)
     }
   clear_bb_predictions (bb);
 
-  if (!bb->count.initialized_p () && !dry_run)
+  if ((!bb->count.nonzero_p () || !first->probability.initialized_p ())
+      && !dry_run)
     {
       first->probability
 	 = profile_probability::from_reg_br_prob_base (combined_probability);
@@ -3014,10 +3015,7 @@ propagate_freq (basic_block head, bitmap tovisit)
       BLOCK_INFO (bb)->npredecessors = count;
       /* When function never returns, we will never process exit block.  */
       if (!count && bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
-	{
-	  bb->count = profile_count::zero ();
-	  bb->frequency = 0;
-	}
+	bb->count = profile_count::zero ();
     }
 
   BLOCK_INFO (head)->frequency = 1;
@@ -3050,7 +3048,10 @@ propagate_freq (basic_block head, bitmap tovisit)
 				  * BLOCK_INFO (e->src)->frequency /
 				  REG_BR_PROB_BASE);  */
 
-		sreal tmp = e->probability.to_reg_br_prob_base ();
+		/* FIXME: Graphite is producing edges with no profile. Once
+		   this is fixed, drop this.  */
+		sreal tmp = e->probability.initialized_p () ?
+			    e->probability.to_reg_br_prob_base () : 0;
 		tmp *= BLOCK_INFO (e->src)->frequency;
 		tmp *= real_inv_br_prob_base;
 		frequency += tmp;
@@ -3082,7 +3083,10 @@ propagate_freq (basic_block head, bitmap tovisit)
 	     = ((e->probability * BLOCK_INFO (bb)->frequency)
 	     / REG_BR_PROB_BASE); */
 
-	  sreal tmp = e->probability.to_reg_br_prob_base ();
+	  /* FIXME: Graphite is producing edges with no profile. Once
+	     this is fixed, drop this.  */
+	  sreal tmp = e->probability.initialized_p () ?
+		      e->probability.to_reg_br_prob_base () : 0;
 	  tmp *= BLOCK_INFO (bb)->frequency;
 	  EDGE_INFO (e)->back_edge_prob = tmp * real_inv_br_prob_base;
 	}
@@ -3196,24 +3200,33 @@ drop_profile (struct cgraph_node *node, profile_count call_count)
     }
 
   basic_block bb;
-  FOR_ALL_BB_FN (bb, fn)
+  push_cfun (DECL_STRUCT_FUNCTION (node->decl));
+  if (flag_guess_branch_prob)
     {
-      bb->count = profile_count::uninitialized ();
-
-      edge_iterator ei;
-      edge e;
-      FOR_EACH_EDGE (e, ei, bb->preds)
-	e->count = profile_count::uninitialized ();
+      bool clear_zeros
+	 = ENTRY_BLOCK_PTR_FOR_FN
+		 (DECL_STRUCT_FUNCTION (node->decl))->count.nonzero_p ();
+      FOR_ALL_BB_FN (bb, fn)
+	if (clear_zeros || !(bb->count == profile_count::zero ()))
+	  bb->count = bb->count.guessed_local ();
+      DECL_STRUCT_FUNCTION (node->decl)->cfg->count_max =
+        DECL_STRUCT_FUNCTION (node->decl)->cfg->count_max.guessed_local ();
     }
+  else
+    {
+      FOR_ALL_BB_FN (bb, fn)
+	bb->count = profile_count::uninitialized ();
+      DECL_STRUCT_FUNCTION (node->decl)->cfg->count_max
+	 = profile_count::uninitialized ();
+    }
+  pop_cfun ();
 
   struct cgraph_edge *e;
   for (e = node->callees; e; e = e->next_caller)
     {
-      e->count = profile_count::uninitialized ();
       e->frequency = compute_call_stmt_bb_frequency (e->caller->decl,
 						     gimple_bb (e->call_stmt));
     }
-  node->count = profile_count::uninitialized ();
   
   profile_status_for_fn (fn)
       = (flag_guess_branch_prob ? PROFILE_GUESSED : PROFILE_ABSENT);
@@ -3307,33 +3320,16 @@ handle_missing_profiles (void)
 bool
 counts_to_freqs (void)
 {
-  gcov_type count_max;
-  profile_count true_count_max = profile_count::zero ();
+  profile_count true_count_max = profile_count::uninitialized ();
   basic_block bb;
 
-  /* Don't overwrite the estimated frequencies when the profile for
-     the function is missing.  We may drop this function PROFILE_GUESSED
-     later in drop_profile ().  */
-  if (!ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ()
-      || ENTRY_BLOCK_PTR_FOR_FN (cfun)->count == profile_count::zero ())
-    return false;
-
   FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, next_bb)
-    if (bb->count > true_count_max)
-      true_count_max = bb->count;
+    if (!(bb->count < true_count_max))
+      true_count_max = true_count_max.max (bb->count);
 
-  /* If we have no counts to base frequencies on, keep those that are
-     already there.  */
-  if (!(true_count_max > 0))
-    return false;
-
-  count_max = true_count_max.to_gcov_type ();
-
-  FOR_ALL_BB_FN (bb, cfun)
-    if (bb->count.initialized_p ())
-      bb->frequency = RDIV (bb->count.to_gcov_type () * BB_FREQ_MAX, count_max);
+  cfun->cfg->count_max = true_count_max;
 
-  return true;
+  return true_count_max.nonzero_p ();
 }
 
 /* Return true if function is likely to be expensive, so there is no point to
@@ -3355,11 +3351,11 @@ expensive_function_p (int threshold)
   /* Frequencies are out of range.  This either means that function contains
      internal loop executing more than BB_FREQ_MAX times or profile feedback
      is available and function has not been executed at all.  */
-  if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency == 0)
+  if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun) == 0)
     return true;
 
   /* Maximally BB_FREQ_MAX^2 so overflow won't happen.  */
-  limit = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency * threshold;
+  limit = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun) * threshold;
   FOR_EACH_BB_FN (bb, cfun)
     {
       rtx_insn *insn;
@@ -3367,7 +3363,7 @@ expensive_function_p (int threshold)
       FOR_BB_INSNS (bb, insn)
 	if (active_insn_p (insn))
 	  {
-	    sum += bb->frequency;
+	    sum += bb->count.to_frequency (cfun);
 	    if (sum > limit)
 	      return true;
 	}
@@ -3396,7 +3392,7 @@ propagate_unlikely_bbs_forward (void)
 	{
 	  bb = worklist.pop ();
 	  FOR_EACH_EDGE (e, ei, bb->succs)
-	    if (!(e->count == profile_count::zero ())
+	    if (!(e->count () == profile_count::zero ())
 		&& !(e->dest->count == profile_count::zero ())
 		&& !e->dest->aux)
 	      {
@@ -3416,9 +3412,6 @@ propagate_unlikely_bbs_forward (void)
 		     "Basic block %i is marked unlikely by forward prop\n",
 		     bb->index);
 	  bb->count = profile_count::zero ();
-	  bb->frequency = 0;
-          FOR_EACH_EDGE (e, ei, bb->succs)
-	    e->count = profile_count::zero ();
 	}
       else
         bb->aux = NULL;
@@ -3449,21 +3442,14 @@ determine_unlikely_bbs ()
 	  bb->count = profile_count::zero ();
 	}
 
-      if (bb->count == profile_count::zero ())
-	{
-	  bb->frequency = 0;
-          FOR_EACH_EDGE (e, ei, bb->preds)
-	    e->count = profile_count::zero ();
-	}
-
       FOR_EACH_EDGE (e, ei, bb->succs)
-	if (!(e->count == profile_count::zero ())
+	if (!(e->probability == profile_probability::never ())
 	    && unlikely_executed_edge_p (e))
 	  {
             if (dump_file && (dump_flags & TDF_DETAILS))
 	      fprintf (dump_file, "Edge %i->%i is locally unlikely\n",
 		       bb->index, e->dest->index);
-	    e->count = profile_count::zero ();
+	    e->probability = profile_probability::never ();
 	  }
 
       gcc_checking_assert (!bb->aux);
@@ -3477,7 +3463,8 @@ determine_unlikely_bbs ()
       {
 	nsuccs[bb->index] = 0;
         FOR_EACH_EDGE (e, ei, bb->succs)
-	  if (!(e->count == profile_count::zero ()))
+	  if (!(e->probability == profile_probability::never ())
+	      && !(e->dest->count == profile_count::zero ()))
 	    nsuccs[bb->index]++;
 	if (!nsuccs[bb->index])
 	  worklist.safe_push (bb);
@@ -3509,11 +3496,10 @@ determine_unlikely_bbs ()
 		 "Basic block %i is marked unlikely by backward prop\n",
 		 bb->index);
       bb->count = profile_count::zero ();
-      bb->frequency = 0;
       FOR_EACH_EDGE (e, ei, bb->preds)
-	if (!(e->count == profile_count::zero ()))
+	if (!(e->probability == profile_probability::never ()))
 	  {
-	    e->count = profile_count::zero ();
+	    e->probability = profile_probability::never ();
 	    if (!(e->src->count == profile_count::zero ()))
 	      {
 	        nsuccs[e->src->index]--;
@@ -3566,8 +3552,13 @@ estimate_bb_frequencies (bool force)
 
 	  FOR_EACH_EDGE (e, ei, bb->succs)
 	    {
-	      EDGE_INFO (e)->back_edge_prob
-		 = e->probability.to_reg_br_prob_base ();
+	      /* FIXME: Graphite is producing edges with no profile. Once
+		 this is fixed, drop this.  */
+	      if (e->probability.initialized_p ())
+	        EDGE_INFO (e)->back_edge_prob
+		   = e->probability.to_reg_br_prob_base ();
+	      else
+		EDGE_INFO (e)->back_edge_prob = REG_BR_PROB_BASE / 2;
 	      EDGE_INFO (e)->back_edge_prob *= real_inv_br_prob_base;
 	    }
 	}
@@ -3576,16 +3567,28 @@ estimate_bb_frequencies (bool force)
          to outermost to examine frequencies for back edges.  */
       estimate_loops ();
 
+      bool global0 = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.initialized_p ()
+		     && ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa_p ();
+
       freq_max = 0;
       FOR_EACH_BB_FN (bb, cfun)
 	if (freq_max < BLOCK_INFO (bb)->frequency)
 	  freq_max = BLOCK_INFO (bb)->frequency;
 
       freq_max = real_bb_freq_max / freq_max;
+      cfun->cfg->count_max = profile_count::uninitialized ();
       FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, next_bb)
 	{
 	  sreal tmp = BLOCK_INFO (bb)->frequency * freq_max + real_one_half;
-	  bb->frequency = tmp.to_int ();
+	  profile_count count = profile_count::from_gcov_type (tmp.to_int ());	
+
+	  /* If we have profile feedback in which this function was never
+	     executed, then preserve this info.  */
+	  if (global0)
+	    bb->count = count.global0 ();
+	  else if (!(bb->count == profile_count::zero ()))
+	    bb->count = count.guessed_local ();
+          cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count);
 	}
 
       free_aux_for_blocks ();
@@ -3610,7 +3613,8 @@ compute_function_frequency (void)
   if (profile_status_for_fn (cfun) != PROFILE_READ)
     {
       int flags = flags_from_decl_or_type (current_function_decl);
-      if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count == profile_count::zero ()
+      if ((ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa_p ()
+	   && ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa() == profile_count::zero ())
 	  || lookup_attribute ("cold", DECL_ATTRIBUTES (current_function_decl))
 	     != NULL)
 	{
@@ -3729,7 +3733,7 @@ pass_profile::execute (function *fun)
    {
      struct loop *loop;
      FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)
-       if (loop->header->frequency)
+       if (loop->header->count.initialized_p ())
          fprintf (dump_file, "Loop got predicted %d to iterate %i times.\n",
        	   loop->num,
        	   (int)expected_loop_iterations_unbounded (loop));
@@ -3855,15 +3859,12 @@ rebuild_frequencies (void)
      which may also lead to frequencies incorrectly reduced to 0. There
      is less precision in the probabilities, so we only do this for small
      max counts.  */
-  profile_count count_max = profile_count::zero ();
+  cfun->cfg->count_max = profile_count::uninitialized ();
   basic_block bb;
   FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, next_bb)
-    if (bb->count > count_max)
-      count_max = bb->count;
+    cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count);
 
-  if (profile_status_for_fn (cfun) == PROFILE_GUESSED
-      || (!flag_auto_profile && profile_status_for_fn (cfun) == PROFILE_READ
-	  && count_max < REG_BR_PROB_BASE / 10))
+  if (profile_status_for_fn (cfun) == PROFILE_GUESSED)
     {
       loop_optimizer_init (0);
       add_noreturn_fake_exit_edges ();
@@ -3928,8 +3929,6 @@ force_edge_cold (edge e, bool impossible)
   profile_probability prob_sum = profile_probability::never ();
   edge_iterator ei;
   edge e2;
-  profile_count old_count = e->count;
-  profile_probability old_probability = e->probability;
   bool uninitialized_exit = false;
 
   profile_probability goal = (impossible ? profile_probability::never ()
@@ -3937,13 +3936,13 @@ force_edge_cold (edge e, bool impossible)
 
   /* If edge is already improbably or cold, just return.  */
   if (e->probability <= goal
-      && (!impossible || e->count == profile_count::zero ()))
+      && (!impossible || e->count () == profile_count::zero ()))
     return;
   FOR_EACH_EDGE (e2, ei, e->src->succs)
     if (e2 != e)
       {
-	if (e2->count.initialized_p ())
-	  count_sum += e2->count;
+	if (e2->count ().initialized_p ())
+	  count_sum += e2->count ();
 	else
 	  uninitialized_exit = true;
 	if (e2->probability.initialized_p ())
@@ -3956,13 +3955,6 @@ force_edge_cold (edge e, bool impossible)
     {
       if (!(e->probability < goal))
 	e->probability = goal;
-      if (impossible)
-	e->count = profile_count::zero ();
-      else if (old_probability > profile_probability::never ())
-	e->count = e->count.apply_probability (e->probability
-					       / old_probability);
-      else
-        e->count = e->count.apply_scale (1, REG_BR_PROB_BASE);
 
       profile_probability prob_comp = prob_sum / e->probability.invert ();
 
@@ -3971,12 +3963,9 @@ force_edge_cold (edge e, bool impossible)
 		 "probability to other edges.\n",
 		 e->src->index, e->dest->index,
 		 impossible ? "impossible" : "cold");
-      profile_count count_sum2 = count_sum + old_count - e->count;
       FOR_EACH_EDGE (e2, ei, e->src->succs)
 	if (e2 != e)
 	  {
-	    if (count_sum > 0)
-	      e2->count.apply_scale (count_sum2, count_sum);
 	    e2->probability /= prob_comp;
 	  }
       if (current_ir_type () != IR_GIMPLE
@@ -4027,7 +4016,6 @@ force_edge_cold (edge e, bool impossible)
 		fprintf (dump_file,
 			 "Making bb %i impossible and dropping count to 0.\n",
 			 e->src->index);
-	      e->count = profile_count::zero ();
 	      e->src->count = profile_count::zero ();
 	      FOR_EACH_EDGE (e2, ei, e->src->preds)
 		force_edge_cold (e2, impossible);
@@ -4042,18 +4030,20 @@ force_edge_cold (edge e, bool impossible)
 	 after loop transforms.  */
       if (!(prob_sum > profile_probability::never ())
 	  && count_sum == profile_count::zero ()
-	  && single_pred_p (e->src) && e->src->frequency > (impossible ? 0 : 1))
+	  && single_pred_p (e->src) && e->src->count.to_frequency (cfun)
+	     > (impossible ? 0 : 1))
 	{
-	  int old_frequency = e->src->frequency;
+	  int old_frequency = e->src->count.to_frequency (cfun);
 	  if (dump_file && (dump_flags & TDF_DETAILS))
 	    fprintf (dump_file, "Making bb %i %s.\n", e->src->index,
 		     impossible ? "impossible" : "cold");
-	  e->src->frequency = MIN (e->src->frequency, impossible ? 0 : 1);
+	  int new_frequency = MIN (e->src->count.to_frequency (cfun),
+				   impossible ? 0 : 1);
 	  if (impossible)
-	    e->src->count = e->count = profile_count::zero ();
+	    e->src->count = profile_count::zero ();
 	  else
-	    e->src->count = e->count = e->count.apply_scale (e->src->frequency,
-							     old_frequency);
+	    e->src->count = e->count ().apply_scale (new_frequency,
+						     old_frequency);
 	  force_edge_cold (single_pred_edge (e->src), impossible);
 	}
       else if (dump_file && (dump_flags & TDF_DETAILS)
diff --git a/gcc/print-rtl.c b/gcc/print-rtl.c
index a72f9cda188..2ecdbb4299e 100644
--- a/gcc/print-rtl.c
+++ b/gcc/print-rtl.c
@@ -516,7 +516,7 @@ rtx_writer::print_rtx_operand_code_r (const_rtx in_rtx)
       if (REG_EXPR (in_rtx))
 	print_mem_expr (m_outfile, REG_EXPR (in_rtx));
 
-      if (maybe_nonzero (REG_OFFSET (in_rtx)))
+      if (may_ne (REG_OFFSET (in_rtx), 0))
 	{
 	  fprintf (m_outfile, "+");
 	  print_poly_int (m_outfile, REG_OFFSET (in_rtx));
diff --git a/gcc/profile-count.c b/gcc/profile-count.c
index 44ceaed2d66..d7031404645 100644
--- a/gcc/profile-count.c
+++ b/gcc/profile-count.c
@@ -42,7 +42,11 @@ profile_count::dump (FILE *f) const
   else
     {
       fprintf (f, "%" PRId64, m_val);
-      if (m_quality == profile_adjusted)
+      if (m_quality == profile_guessed_local)
+	fprintf (f, " (estimated locally)");
+      else if (m_quality == profile_guessed_global0)
+	fprintf (f, " (estimated locally, globally 0)");
+      else if (m_quality == profile_adjusted)
 	fprintf (f, " (adjusted)");
       else if (m_quality == profile_afdo)
 	fprintf (f, " (auto FDO)");
@@ -65,6 +69,7 @@ profile_count::debug () const
 bool
 profile_count::differs_from_p (profile_count other) const
 {
+  gcc_checking_assert (compatible_p (other));
   if (!initialized_p () || !other.initialized_p ())
     return false;
   if ((uint64_t)m_val - (uint64_t)other.m_val < 100
@@ -213,3 +218,40 @@ slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
   *res = (uint64_t) -1;
   return false;
 }
+
+/* Return count as frequency within FUN scaled in range 0 to REG_FREQ_MAX
+   Used for legacy code and should not be used anymore.  */
+
+int
+profile_count::to_frequency (struct function *fun) const
+{
+  if (!initialized_p ())
+    return BB_FREQ_MAX;
+  if (*this == profile_count::zero ())
+    return 0;
+  gcc_assert (REG_BR_PROB_BASE == BB_FREQ_MAX
+	      && fun->cfg->count_max.initialized_p ());
+  profile_probability prob = probability_in (fun->cfg->count_max);
+  if (!prob.initialized_p ())
+    return REG_BR_PROB_BASE;
+  return prob.to_reg_br_prob_base ();
+}
+
+/* Return count as frequency within FUN scaled in range 0 to CGRAPH_FREQ_MAX
+   where CGRAPH_FREQ_BASE means that count equals to entry block count.
+   Used for legacy code and should not be used anymore.  */
+
+int
+profile_count::to_cgraph_frequency (profile_count entry_bb_count) const
+{
+  if (!initialized_p ())
+    return CGRAPH_FREQ_BASE;
+  if (*this == profile_count::zero ())
+    return 0;
+  gcc_checking_assert (entry_bb_count.initialized_p ());
+  uint64_t scale;
+  if (!safe_scale_64bit (!entry_bb_count.m_val ? m_val + 1 : m_val,
+			 CGRAPH_FREQ_BASE, MAX (1, entry_bb_count.m_val), &scale))
+    return CGRAPH_FREQ_MAX;
+  return MIN (scale, CGRAPH_FREQ_MAX);
+}
diff --git a/gcc/profile-count.h b/gcc/profile-count.h
index 4546e199f24..d793d11c830 100644
--- a/gcc/profile-count.h
+++ b/gcc/profile-count.h
@@ -21,21 +21,37 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_PROFILE_COUNT_H
 #define GCC_PROFILE_COUNT_H
 
+struct function;
+
 /* Quality of the profile count.  Because gengtype does not support enums
    inside of classes, this is in global namespace.  */
 enum profile_quality {
+  /* Profile is based on static branch prediction heuristics and may
+     or may not match reality.  It is local to function and can not be compared
+     inter-procedurally.  Never used by probabilities (they are always local).
+   */
+  profile_guessed_local = 0,
+  /* Profile was read by feedback and was 0, we used local heuristics to guess
+     better.  This is the case of functions not run in profile fedback.
+     Never used by probabilities.  */
+  profile_guessed_global0 = 1,
+
+
   /* Profile is based on static branch prediction heuristics.  It may or may
-     not reflect the reality.  */
-  profile_guessed = 0,
+     not reflect the reality but it can be compared interprocedurally
+     (for example, we inlined function w/o profile feedback into function
+      with feedback and propagated from that).
+     Never used by probablities.  */
+  profile_guessed = 2,
   /* Profile was determined by autofdo.  */
-  profile_afdo = 1,
+  profile_afdo = 3,
   /* Profile was originally based on feedback but it was adjusted
      by code duplicating optimization.  It may not precisely reflect the
      particular code path.  */
-  profile_adjusted = 2,
+  profile_adjusted = 4,
   /* Profile was read from profile feedback or determined by accurate static
      method.  */
-  profile_precise = 3
+  profile_precise = 5
 };
 
 /* The base value for branch probability notes and edge probabilities.  */
@@ -114,15 +130,15 @@ safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
 
 class GTY((user)) profile_probability
 {
-  static const int n_bits = 30;
+  static const int n_bits = 29;
   /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
      will lead to harder multiplication sequences.  */
   static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
   static const uint32_t uninitialized_probability
 		 = ((uint32_t) 1 << (n_bits - 1)) - 1;
 
-  uint32_t m_val : 30;
-  enum profile_quality m_quality : 2;
+  uint32_t m_val : 29;
+  enum profile_quality m_quality : 3;
 
   friend class profile_count;
 public:
@@ -226,14 +242,14 @@ public:
   static profile_probability from_reg_br_prob_note (int v)
     {
       profile_probability ret;
-      ret.m_val = ((unsigned int)v) / 4;
-      ret.m_quality = (enum profile_quality)(v & 3);
+      ret.m_val = ((unsigned int)v) / 8;
+      ret.m_quality = (enum profile_quality)(v & 7);
       return ret;
     }
   int to_reg_br_prob_note () const
     {
       gcc_checking_assert (initialized_p ());
-      int ret = m_val * 4 + m_quality;
+      int ret = m_val * 8 + m_quality;
       gcc_checking_assert (profile_probability::from_reg_br_prob_note (ret)
 			   == *this);
       return ret;
@@ -489,8 +505,9 @@ public:
     {
       if (m_val == uninitialized_probability)
 	return m_quality == profile_guessed;
-      else
-	return m_val <= max_probability;
+      else if (m_quality < profile_guessed)
+	return false;
+      return m_val <= max_probability;
     }
 
   /* Comparsions are three-state and conservative.  False is returned if
@@ -530,9 +547,32 @@ public:
   void stream_out (struct lto_output_stream *);
 };
 
-/* Main data type to hold profile counters in GCC.  In most cases profile
-   counts originate from profile feedback. They are 64bit integers
-   representing number of executions during the train run.
+/* Main data type to hold profile counters in GCC. Profile counts originate
+   either from profile feedback, static profile estimation or both.  We do not
+   perform whole program profile propagation and thus profile estimation
+   counters are often local to function, while counters from profile feedback
+   (or special cases of profile estimation) can be used inter-procedurally.
+
+   There are 3 basic types
+     1) local counters which are result of intra-procedural static profile
+        estimation.
+     2) ipa counters which are result of profile feedback or special case
+        of static profile estimation (such as in function main).
+     3) counters which counts as 0 inter-procedurally (beause given function
+        was never run in train feedback) but they hold local static profile
+        estimate.
+
+   Counters of type 1 and 3 can not be mixed with counters of different type
+   within operation (because whole function should use one type of counter)
+   with exception that global zero mix in most operations where outcome is
+   well defined.
+
+   To take local counter and use it inter-procedurally use ipa member function
+   which strips information irelevant at the inter-procedural level.
+
+   Counters are 61bit integers representing number of executions during the
+   train run or normalized frequency within the function.
+
    As the profile is maintained during the compilation, many adjustments are
    made.  Not all transformations can be made precisely, most importantly
    when code is being duplicated.  It also may happen that part of CFG has
@@ -567,12 +607,25 @@ class GTY(()) profile_count
      64bit.  Although a counter cannot be negative, we use a signed
      type to hold various extra stages.  */
 
-  static const int n_bits = 62;
+  static const int n_bits = 61;
   static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
   static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
 
   uint64_t m_val : n_bits;
-  enum profile_quality m_quality : 2;
+  enum profile_quality m_quality : 3;
+
+  /* Return true if both values can meaningfully appear in single function
+     body.  We have either all counters in function local or global, otherwise
+     operations between them are not really defined well.  */
+  bool compatible_p (const profile_count other) const
+    {
+      if (!initialized_p () || !other.initialized_p ())
+	return true;
+      if (*this == profile_count::zero ()
+	  || other == profile_count::zero ())
+	return true;
+      return ipa_p () == other.ipa_p ();
+    }
 public:
 
   /* Used for counters which are expected to be never executed.  */
@@ -597,7 +650,7 @@ public:
     {
       profile_count c;
       c.m_val = uninitialized_count;
-      c.m_quality = profile_guessed;
+      c.m_quality = profile_guessed_local;
       return c;
     }
 
@@ -630,6 +683,11 @@ public:
     {
       return m_quality >= profile_adjusted;
     }
+  /* Return true if vlaue can be operated inter-procedurally.  */
+  bool ipa_p () const
+    {
+      return !initialized_p () || m_quality >= profile_guessed_global0;
+    }
 
   /* When merging basic blocks, the two different profile counts are unified.
      Return true if this can be done without losing info about profile.
@@ -671,6 +729,7 @@ public:
 	return profile_count::uninitialized ();
 
       profile_count ret;
+      gcc_checking_assert (compatible_p (other));
       ret.m_val = m_val + other.m_val;
       ret.m_quality = MIN (m_quality, other.m_quality);
       return ret;
@@ -688,6 +747,7 @@ public:
 	return *this = profile_count::uninitialized ();
       else
 	{
+          gcc_checking_assert (compatible_p (other));
 	  m_val += other.m_val;
 	  m_quality = MIN (m_quality, other.m_quality);
 	}
@@ -699,6 +759,7 @@ public:
 	return *this;
       if (!initialized_p () || !other.initialized_p ())
 	return profile_count::uninitialized ();
+      gcc_checking_assert (compatible_p (other));
       profile_count ret;
       ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
       ret.m_quality = MIN (m_quality, other.m_quality);
@@ -712,6 +773,7 @@ public:
 	return *this = profile_count::uninitialized ();
       else
 	{
+          gcc_checking_assert (compatible_p (other));
 	  m_val = m_val >= other.m_val ? m_val - other.m_val: 0;
 	  m_quality = MIN (m_quality, other.m_quality);
 	}
@@ -721,48 +783,115 @@ public:
   /* Return false if profile_count is bogus.  */
   bool verify () const
     {
-      return m_val != uninitialized_count || m_quality == profile_guessed;
+      return m_val != uninitialized_count || m_quality == profile_guessed_local;
     }
 
   /* Comparsions are three-state and conservative.  False is returned if
      the inequality can not be decided.  */
   bool operator< (const profile_count &other) const
     {
-      return initialized_p () && other.initialized_p () && m_val < other.m_val;
+      if (!initialized_p () || !other.initialized_p ())
+	return false;
+      if (*this == profile_count::zero ())
+	return !(other == profile_count::zero ());
+      if (other == profile_count::zero ())
+	return false;
+      gcc_checking_assert (compatible_p (other));
+      return m_val < other.m_val;
     }
   bool operator> (const profile_count &other) const
     {
+      if (!initialized_p () || !other.initialized_p ())
+	return false;
+      if (*this  == profile_count::zero ())
+	return false;
+      if (other == profile_count::zero ())
+	return !(*this == profile_count::zero ());
+      gcc_checking_assert (compatible_p (other));
       return initialized_p () && other.initialized_p () && m_val > other.m_val;
     }
   bool operator< (const gcov_type other) const
     {
+      gcc_checking_assert (ipa_p ());
       gcc_checking_assert (other >= 0);
       return initialized_p () && m_val < (uint64_t) other;
     }
   bool operator> (const gcov_type other) const
     {
+      gcc_checking_assert (ipa_p ());
       gcc_checking_assert (other >= 0);
       return initialized_p () && m_val > (uint64_t) other;
     }
 
   bool operator<= (const profile_count &other) const
     {
-      return initialized_p () && other.initialized_p () && m_val <= other.m_val;
+      if (!initialized_p () || !other.initialized_p ())
+	return false;
+      if (*this == profile_count::zero ())
+	return true;
+      if (other == profile_count::zero ())
+	return (*this == profile_count::zero ());
+      gcc_checking_assert (compatible_p (other));
+      return m_val <= other.m_val;
     }
   bool operator>= (const profile_count &other) const
     {
-      return initialized_p () && other.initialized_p () && m_val >= other.m_val;
+      if (!initialized_p () || !other.initialized_p ())
+	return false;
+      if (other == profile_count::zero ())
+	return true;
+      if (*this == profile_count::zero ())
+	return !(other == profile_count::zero ());
+      gcc_checking_assert (compatible_p (other));
+      return m_val >= other.m_val;
     }
   bool operator<= (const gcov_type other) const
     {
+      gcc_checking_assert (ipa_p ());
       gcc_checking_assert (other >= 0);
       return initialized_p () && m_val <= (uint64_t) other;
     }
   bool operator>= (const gcov_type other) const
     {
+      gcc_checking_assert (ipa_p ());
       gcc_checking_assert (other >= 0);
       return initialized_p () && m_val >= (uint64_t) other;
     }
+  /* Return true when value is not zero and can be used for scaling. 
+     This is different from *this > 0 because that requires counter to
+     be IPA.  */
+  bool nonzero_p () const
+    {
+      return initialized_p () && m_val != 0;
+    }
+
+  /* Make counter forcingly nonzero.  */
+  profile_count force_nonzero () const
+    {
+      if (!initialized_p ())
+	return *this;
+      profile_count ret = *this;
+      if (ret.m_val == 0)
+	ret.m_val = 1;
+      return ret;
+    }
+
+  profile_count max (profile_count other) const
+    {
+      if (!initialized_p ())
+	return other;
+      if (!other.initialized_p ())
+	return *this;
+      if (*this == profile_count::zero ())
+	return other;
+      if (other == profile_count::zero ())
+	return *this;
+      gcc_checking_assert (compatible_p (other));
+      if (m_val < other.m_val || (m_val == other.m_val
+				  && m_quality < other.m_quality))
+	return other;
+      return *this;
+    }
 
   /* PROB is a probability in scale 0...REG_BR_PROB_BASE.  Scale counter
      accordingly.  */
@@ -814,13 +943,13 @@ public:
     }
   profile_count apply_scale (profile_count num, profile_count den) const
     {
-      if (m_val == 0)
+      if (*this == profile_count::zero ())
 	return *this;
-      if (num.m_val == 0)
+      if (num == profile_count::zero ())
 	return num;
       if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
 	return profile_count::uninitialized ();
-      gcc_checking_assert (den > 0);
+      gcc_checking_assert (den.m_val);
       if (num == den)
 	return *this;
 
@@ -828,7 +957,30 @@ public:
       uint64_t val;
       safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
       ret.m_val = MIN (val, max_count);
-      ret.m_quality = MIN (m_quality, profile_adjusted);
+      ret.m_quality = MIN (MIN (MIN (m_quality, profile_adjusted),
+			        num.m_quality), den.m_quality);
+      if (num.ipa_p () && !ret.ipa_p ())
+	ret.m_quality = MIN (num.m_quality, profile_guessed);
+      return ret;
+    }
+
+  /* Return THIS with quality dropped to GUESSED_LOCAL.  */
+  profile_count guessed_local () const
+    {
+      profile_count ret = *this;
+      if (!initialized_p ())
+	return *this;
+      ret.m_quality = profile_guessed_local;
+      return ret;
+    }
+
+  /* We know that profile is globally0 but keep local profile if present.  */
+  profile_count global0 () const
+    {
+      profile_count ret = *this;
+      if (!initialized_p ())
+	return *this;
+      ret.m_quality = profile_guessed_global0;
       return ret;
     }
 
@@ -836,10 +988,21 @@ public:
   profile_count guessed () const
     {
       profile_count ret = *this;
-      ret.m_quality = profile_guessed;
+      ret.m_quality = MIN (ret.m_quality, profile_guessed);
       return ret;
     }
 
+  /* Return variant of profile counte which is always safe to compare
+     acorss functions.  */
+  profile_count ipa () const
+    {
+      if (m_quality > profile_guessed_global0)
+	return *this;
+      if (m_quality == profile_guessed_global0)
+	return profile_count::zero ();
+      return profile_count::uninitialized ();
+    }
+
   /* Return THIS with quality dropped to AFDO.  */
   profile_count afdo () const
     {
@@ -852,21 +1015,26 @@ public:
      OVERALL.  */
   profile_probability probability_in (const profile_count overall) const
     {
-      if (!m_val)
+      if (*this == profile_count::zero ())
 	return profile_probability::never ();
       if (!initialized_p () || !overall.initialized_p ()
 	  || !overall.m_val)
 	return profile_probability::uninitialized ();
       profile_probability ret;
-      if (overall < m_val)
+      gcc_checking_assert (compatible_p (overall));
+
+      if (overall.m_val < m_val)
 	ret.m_val = profile_probability::max_probability;
       else
 	ret.m_val = RDIV (m_val * profile_probability::max_probability,
 			  overall.m_val);
-      ret.m_quality = MIN (m_quality, overall.m_quality);
+      ret.m_quality = MAX (MIN (m_quality, overall.m_quality), profile_guessed);
       return ret;
     }
 
+  int to_frequency (struct function *fun) const;
+  int to_cgraph_frequency (profile_count entry_bb_count) const;
+
   /* Output THIS to F.  */
   void dump (FILE *f) const;
 
diff --git a/gcc/profile.c b/gcc/profile.c
index 6d40241a37b..2b30a9e6754 100644
--- a/gcc/profile.c
+++ b/gcc/profile.c
@@ -476,38 +476,6 @@ read_profile_edge_counts (gcov_type *exec_counts)
     return num_edges;
 }
 
-#define OVERLAP_BASE 10000
-
-/* Compare the static estimated profile to the actual profile, and
-   return the "degree of overlap" measure between them.
-
-   Degree of overlap is a number between 0 and OVERLAP_BASE. It is
-   the sum of each basic block's minimum relative weights between
-   two profiles. And overlap of OVERLAP_BASE means two profiles are
-   identical.  */
-
-static int
-compute_frequency_overlap (void)
-{
-  gcov_type count_total = 0, freq_total = 0;
-  int overlap = 0;
-  basic_block bb;
-
-  FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, next_bb)
-    {
-      count_total += bb_gcov_count (bb);
-      freq_total += bb->frequency;
-    }
-
-  if (count_total == 0 || freq_total == 0)
-    return 0;
-
-  FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, next_bb)
-    overlap += MIN (bb_gcov_count (bb) * OVERLAP_BASE / count_total,
-		    bb->frequency * OVERLAP_BASE / freq_total);
-
-  return overlap;
-}
 
 /* Compute the branch probabilities for the various branches.
    Annotate them accordingly.  
@@ -676,14 +644,6 @@ compute_branch_probabilities (unsigned cfg_checksum, unsigned lineno_checksum)
 	    }
 	}
     }
-  if (dump_file)
-    {
-      int overlap = compute_frequency_overlap ();
-      gimple_dump_cfg (dump_file, dump_flags);
-      fprintf (dump_file, "Static profile overlap: %d.%d%%\n",
-	       overlap / (OVERLAP_BASE / 100),
-	       overlap % (OVERLAP_BASE / 100));
-    }
 
   total_num_passes += passes;
   if (dump_file)
@@ -829,15 +789,18 @@ compute_branch_probabilities (unsigned cfg_checksum, unsigned lineno_checksum)
 	}
     }
 
-  FOR_ALL_BB_FN (bb, cfun)
-    {
-      edge e;
-      edge_iterator ei;
-
+  /* If we have real data, use them!  */
+  if (bb_gcov_count (ENTRY_BLOCK_PTR_FOR_FN (cfun))
+      || !flag_guess_branch_prob)
+    FOR_ALL_BB_FN (bb, cfun)
       bb->count = profile_count::from_gcov_type (bb_gcov_count (bb));
-      FOR_EACH_EDGE (e, ei, bb->succs)
-        e->count = profile_count::from_gcov_type (edge_gcov_count (e));
-    }
+  /* If function was not trained, preserve local estimates including statically
+     determined zero counts.  */
+  else
+    FOR_ALL_BB_FN (bb, cfun)
+      if (!(bb->count == profile_count::zero ()))
+        bb->count = bb->count.global0 ();
+
   bb_gcov_counts.release ();
   delete edge_gcov_counts;
   edge_gcov_counts = NULL;
diff --git a/gcc/recog.c b/gcc/recog.c
index 0ac16b9b87f..05e69134236 100644
--- a/gcc/recog.c
+++ b/gcc/recog.c
@@ -1007,7 +1007,7 @@ general_operand (rtx op, machine_mode mode)
 
 	 ??? This is a kludge.  */
       if (!reload_completed
-	  && maybe_nonzero (SUBREG_BYTE (op))
+	  && may_ne (SUBREG_BYTE (op), 0)
 	  && MEM_P (sub))
 	return 0;
 
@@ -1380,7 +1380,7 @@ indirect_operand (rtx op, machine_mode mode)
 	 operand.  */
       poly_int64 offset;
       rtx addr = strip_offset (XEXP (SUBREG_REG (op), 0), &offset);
-      return (known_zero (offset + SUBREG_BYTE (op))
+      return (must_eq (offset + SUBREG_BYTE (op), 0)
 	      && general_operand (addr, Pmode));
     }
 
@@ -1967,7 +1967,7 @@ offsettable_address_addr_space_p (int strictp, machine_mode mode, rtx y,
      Clearly that depends on the situation in which it's being used.
      However, the current situation in which we test 0xffffffff is
      less than ideal.  Caveat user.  */
-  if (known_zero (mode_sz))
+  if (must_eq (mode_sz, 0))
     mode_sz = BIGGEST_ALIGNMENT / BITS_PER_UNIT;
 
   /* If the expression contains a constant term,
@@ -3379,6 +3379,7 @@ peep2_attempt (basic_block bb, rtx_insn *insn, int match_len, rtx_insn *attempt)
 	  case REG_NORETURN:
 	  case REG_SETJMP:
 	  case REG_TM:
+	  case REG_CALL_NOCF_CHECK:
 	    add_reg_note (new_insn, REG_NOTE_KIND (note),
 			  XEXP (note, 0));
 	    break;
@@ -3860,7 +3861,7 @@ const pass_data pass_data_split_all_insns =
   OPTGROUP_NONE, /* optinfo_flags */
   TV_NONE, /* tv_id */
   0, /* properties_required */
-  0, /* properties_provided */
+  PROP_rtl_split_insns, /* properties_provided */
   0, /* properties_destroyed */
   0, /* todo_flags_start */
   0, /* todo_flags_finish */
diff --git a/gcc/reg-notes.def b/gcc/reg-notes.def
index a542990cde2..d83fc45ef72 100644
--- a/gcc/reg-notes.def
+++ b/gcc/reg-notes.def
@@ -232,3 +232,10 @@ REG_NOTE (STACK_CHECK)
    The decl might not be available in the call due to splitting of the call
    insn.  This note is a SYMBOL_REF.  */
 REG_NOTE (CALL_DECL)
+
+/* Indicate that a call should not be verified for control-flow consistency.
+   The target address of the call is assumed as a valid address and no check
+   to validate a branch to the target address is needed.  The call is marked
+   when a called function has a 'notrack' attribute.  This note is used by the
+   compiler when the option -fcf-protection=branch is specified.  */
+REG_NOTE (CALL_NOCF_CHECK)
diff --git a/gcc/reg-stack.c b/gcc/reg-stack.c
index f2381067f5e..83fc4762671 100644
--- a/gcc/reg-stack.c
+++ b/gcc/reg-stack.c
@@ -262,7 +262,7 @@ static bool move_for_stack_reg (rtx_insn *, stack_ptr, rtx);
 static bool move_nan_for_stack_reg (rtx_insn *, stack_ptr, rtx);
 static int swap_rtx_condition_1 (rtx);
 static int swap_rtx_condition (rtx_insn *);
-static void compare_for_stack_reg (rtx_insn *, stack_ptr, rtx);
+static void compare_for_stack_reg (rtx_insn *, stack_ptr, rtx, bool);
 static bool subst_stack_regs_pat (rtx_insn *, stack_ptr, rtx);
 static void subst_asm_stack_regs (rtx_insn *, stack_ptr);
 static bool subst_stack_regs (rtx_insn *, stack_ptr);
@@ -1325,7 +1325,8 @@ swap_rtx_condition (rtx_insn *insn)
    set up.  */
 
 static void
-compare_for_stack_reg (rtx_insn *insn, stack_ptr regstack, rtx pat_src)
+compare_for_stack_reg (rtx_insn *insn, stack_ptr regstack,
+		       rtx pat_src, bool can_pop_second_op)
 {
   rtx *src1, *src2;
   rtx src1_note, src2_note;
@@ -1366,8 +1367,18 @@ compare_for_stack_reg (rtx_insn *insn, stack_ptr regstack, rtx pat_src)
 
   if (src1_note)
     {
-      pop_stack (regstack, REGNO (XEXP (src1_note, 0)));
-      replace_reg (&XEXP (src1_note, 0), FIRST_STACK_REG);
+      if (*src2 == CONST0_RTX (GET_MODE (*src2)))
+	{
+	  /* This is `ftst' insn that can't pop register.  */
+	  remove_regno_note (insn, REG_DEAD, REGNO (XEXP (src1_note, 0)));
+	  emit_pop_insn (insn, regstack, XEXP (src1_note, 0),
+			 EMIT_AFTER);
+	}
+      else
+	{
+	  pop_stack (regstack, REGNO (XEXP (src1_note, 0)));
+	  replace_reg (&XEXP (src1_note, 0), FIRST_STACK_REG);
+	}
     }
 
   /* If the second operand dies, handle that.  But if the operands are
@@ -1384,7 +1395,7 @@ compare_for_stack_reg (rtx_insn *insn, stack_ptr regstack, rtx pat_src)
 	 at top (FIRST_STACK_REG) now.  */
 
       if (get_hard_regnum (regstack, XEXP (src2_note, 0)) == FIRST_STACK_REG
-	  && src1_note)
+	  && src1_note && can_pop_second_op)
 	{
 	  pop_stack (regstack, REGNO (XEXP (src2_note, 0)));
 	  replace_reg (&XEXP (src2_note, 0), FIRST_STACK_REG + 1);
@@ -1549,10 +1560,6 @@ subst_stack_regs_pat (rtx_insn *insn, stack_ptr regstack, rtx pat)
 
 	switch (GET_CODE (pat_src))
 	  {
-	  case COMPARE:
-	    compare_for_stack_reg (insn, regstack, pat_src);
-	    break;
-
 	  case CALL:
 	    {
 	      int count;
@@ -1953,31 +1960,35 @@ subst_stack_regs_pat (rtx_insn *insn, stack_ptr regstack, rtx pat)
 		replace_reg (src2, FIRST_STACK_REG + 1);
 		break;
 
-	      case UNSPEC_SAHF:
-		/* (unspec [(unspec [(compare)] UNSPEC_FNSTSW)] UNSPEC_SAHF)
-		   The combination matches the PPRO fcomi instruction.  */
-
-		pat_src = XVECEXP (pat_src, 0, 0);
-		gcc_assert (GET_CODE (pat_src) == UNSPEC);
-		gcc_assert (XINT (pat_src, 1) == UNSPEC_FNSTSW);
-		/* Fall through.  */
-
 	      case UNSPEC_FNSTSW:
 		/* Combined fcomp+fnstsw generated for doing well with
 		   CSE.  When optimizing this would have been broken
 		   up before now.  */
 
 		pat_src = XVECEXP (pat_src, 0, 0);
-		gcc_assert (GET_CODE (pat_src) == COMPARE);
+		if (GET_CODE (pat_src) == COMPARE)
+		  goto do_compare;
 
-		compare_for_stack_reg (insn, regstack, pat_src);
-		break;
+		/* Fall through.  */
+
+	      case UNSPEC_NOTRAP:
+
+		pat_src = XVECEXP (pat_src, 0, 0);
+		gcc_assert (GET_CODE (pat_src) == COMPARE);
+		goto do_compare;
 
 	      default:
 		gcc_unreachable ();
 	      }
 	    break;
 
+	  case COMPARE:
+	  do_compare:
+	    /* `fcomi' insn can't pop two regs.  */
+	    compare_for_stack_reg (insn, regstack, pat_src,
+				   REGNO (*dest) != FLAGS_REG);
+	    break;
+
 	  case IF_THEN_ELSE:
 	    /* This insn requires the top of stack to be the destination.  */
 
@@ -2948,9 +2959,9 @@ better_edge (edge e1, edge e2)
   if (EDGE_FREQUENCY (e1) < EDGE_FREQUENCY (e2))
     return e2;
 
-  if (e1->count > e2->count)
+  if (e1->count () > e2->count ())
     return e1;
-  if (e1->count < e2->count)
+  if (e1->count () < e2->count ())
     return e2;
 
   /* Prefer critical edges to minimize inserting compensation code on
diff --git a/gcc/regcprop.c b/gcc/regcprop.c
index 55f4ea36a7d..4ca10f58a58 100644
--- a/gcc/regcprop.c
+++ b/gcc/regcprop.c
@@ -345,8 +345,8 @@ copy_value (rtx dest, rtx src, struct value_data *vd)
      We can't properly represent the latter case in our tables, so don't
      record anything then.  */
   else if (sn < hard_regno_nregs (sr, vd->e[sr].mode)
-	   && maybe_nonzero (subreg_lowpart_offset (GET_MODE (dest),
-						    vd->e[sr].mode)))
+	   && may_ne (subreg_lowpart_offset (GET_MODE (dest),
+					     vd->e[sr].mode), 0U))
     return;
 
   /* If SRC had been assigned a mode narrower than the copy, we can't
@@ -870,8 +870,8 @@ copyprop_hardreg_forward_1 (basic_block bb, struct value_data *vd)
 	      /* And likewise, if we are narrowing on big endian the transformation
 		 is also invalid.  */
 	      if (REG_NREGS (src) < hard_regno_nregs (regno, vd->e[regno].mode)
-		  && maybe_nonzero (subreg_lowpart_offset (mode,
-							   vd->e[regno].mode)))
+		  && may_ne (subreg_lowpart_offset (mode,
+						    vd->e[regno].mode), 0U))
 		goto no_move_special_case;
 	    }
 
diff --git a/gcc/regs.h b/gcc/regs.h
index 8225355e3f3..2dc94a929d3 100644
--- a/gcc/regs.h
+++ b/gcc/regs.h
@@ -130,8 +130,10 @@ extern size_t reg_info_p_size;
    frequency.  */
 #define REG_FREQ_FROM_BB(bb) (optimize_function_for_size_p (cfun)	      \
 			      ? REG_FREQ_MAX				      \
-			      : ((bb)->frequency * REG_FREQ_MAX / BB_FREQ_MAX)\
-			      ? ((bb)->frequency * REG_FREQ_MAX / BB_FREQ_MAX)\
+			      : ((bb)->count.to_frequency (cfun)	      \
+				* REG_FREQ_MAX / BB_FREQ_MAX)		      \
+			      ? ((bb)->count.to_frequency (cfun)	      \
+				 * REG_FREQ_MAX / BB_FREQ_MAX)		      \
 			      : 1)
 
 /* Indexed by N, gives number of insns in which register N dies.
diff --git a/gcc/reload.c b/gcc/reload.c
index c09a9c6a3f8..744cd51a2b0 100644
--- a/gcc/reload.c
+++ b/gcc/reload.c
@@ -1785,7 +1785,7 @@ combine_reloads (void)
 	&& (ira_reg_class_max_nregs [(int)rld[i].rclass][(int) rld[i].inmode]
 	    == ira_reg_class_max_nregs [(int) rld[output_reload].rclass]
 				       [(int) rld[output_reload].outmode])
-	&& known_zero (rld[i].inc)
+	&& must_eq (rld[i].inc, 0)
 	&& rld[i].reg_rtx == 0
 	/* Don't combine two reloads with different secondary
 	   memory locations.  */
@@ -6170,7 +6170,7 @@ find_reloads_subreg_address (rtx x, int opnum, enum reload_type type,
 				   XEXP (tem, 0), &XEXP (tem, 0),
 				   opnum, type, ind_levels, insn);
   /* ??? Do we need to handle nonzero offsets somehow?  */
-  if (known_zero (offset) && !rtx_equal_p (tem, orig))
+  if (must_eq (offset, 0) && !rtx_equal_p (tem, orig))
     push_reg_equiv_alt_mem (regno, tem);
 
   /* For some processors an address may be valid in the original mode but
@@ -7120,7 +7120,7 @@ find_inc_amount (rtx x, rtx inced)
       if (fmt[i] == 'e')
 	{
 	  poly_int64 tem = find_inc_amount (XEXP (x, i), inced);
-	  if (maybe_nonzero (tem))
+	  if (may_ne (tem, 0))
 	    return tem;
 	}
       if (fmt[i] == 'E')
@@ -7129,7 +7129,7 @@ find_inc_amount (rtx x, rtx inced)
 	  for (j = XVECLEN (x, i) - 1; j >= 0; j--)
 	    {
 	      poly_int64 tem = find_inc_amount (XVECEXP (x, i, j), inced);
-	      if (maybe_nonzero (tem))
+	      if (may_ne (tem, 0))
 		return tem;
 	    }
 	}
@@ -7290,7 +7290,7 @@ debug_reload_to_stream (FILE *f)
       if (rld[r].nongroup)
 	fprintf (f, ", nongroup");
 
-      if (maybe_nonzero (rld[r].inc))
+      if (may_ne (rld[r].inc, 0))
 	{
 	  fprintf (f, ", inc by ");
 	  print_dec (rld[r].inc, f, SIGNED);
diff --git a/gcc/reload1.c b/gcc/reload1.c
index 2ec09c4a7cc..902d940245d 100644
--- a/gcc/reload1.c
+++ b/gcc/reload1.c
@@ -955,7 +955,7 @@ reload (rtx_insn *first, int global)
       if (caller_save_needed)
 	setup_save_areas ();
 
-      if (maybe_nonzero (starting_frame_size) && crtl->stack_alignment_needed)
+      if (may_ne (starting_frame_size, 0) && crtl->stack_alignment_needed)
 	{
 	  /* If we have a stack frame, we must align it now.  The
 	     stack size may be a part of the offset computation for
@@ -2196,7 +2196,7 @@ alter_reg (int i, int from_reg, bool dont_share_p)
 	  if (BYTES_BIG_ENDIAN)
 	    {
 	      adjust = inherent_size - total_size;
-	      if (maybe_nonzero (adjust))
+	      if (may_ne (adjust, 0))
 		{
 		  poly_uint64 total_bits = total_size * BITS_PER_UNIT;
 		  machine_mode mem_mode
@@ -2254,7 +2254,7 @@ alter_reg (int i, int from_reg, bool dont_share_p)
 	  if (BYTES_BIG_ENDIAN)
 	    {
 	      adjust = GET_MODE_SIZE (mode) - total_size;
-	      if (maybe_nonzero (adjust))
+	      if (may_ne (adjust, 0))
 		{
 		  poly_uint64 total_bits = total_size * BITS_PER_UNIT;
 		  machine_mode mem_mode
@@ -3383,7 +3383,7 @@ eliminate_regs_in_insn (rtx_insn *insn, int replace)
 	       increase the cost of the insn by replacing a simple REG
 	       with (plus (reg sp) CST).  So try only when we already
 	       had a PLUS before.  */
-	    if (known_zero (offset) || plus_src)
+	    if (must_eq (offset, 0) || plus_src)
 	      {
 		rtx new_src = plus_constant (GET_MODE (to_rtx),
 					     to_rtx, offset);
diff --git a/gcc/rtlanal.c b/gcc/rtlanal.c
index 3c297eb501f..79a5ae197c1 100644
--- a/gcc/rtlanal.c
+++ b/gcc/rtlanal.c
@@ -494,7 +494,7 @@ rtx_addr_can_trap_p_1 (const_rtx x, poly_int64 offset, poly_int64 size,
 
 	  if (may_lt (offset, 0))
 	    return 1;
-	  if (known_zero (offset))
+	  if (must_eq (offset, 0))
 	    return 0;
 	  if (!known_size_p (size))
 	    return 1;
@@ -649,7 +649,7 @@ rtx_addr_can_trap_p_1 (const_rtx x, poly_int64 offset, poly_int64 size,
       if (XEXP (x, 0) == pic_offset_table_rtx
 	  && GET_CODE (XEXP (x, 1)) == CONST
 	  && GET_CODE (XEXP (XEXP (x, 1), 0)) == UNSPEC
-	  && known_zero (offset))
+	  && must_eq (offset, 0))
 	return 0;
 
       /* - or it is an address that can't trap plus a constant integer.  */
@@ -3641,7 +3641,7 @@ subreg_size_offset_from_lsb (poly_uint64 outer_bytes, poly_uint64 inner_bytes,
   gcc_checking_assert (ordered_p (outer_bytes, inner_bytes));
   if (may_gt (outer_bytes, inner_bytes))
     {
-      gcc_checking_assert (known_zero (lsb_shift));
+      gcc_checking_assert (must_eq (lsb_shift, 0U));
       return 0;
     }
 
@@ -3745,7 +3745,7 @@ subreg_get_info (unsigned int xregno, machine_mode xmode,
   gcc_checking_assert (ordered_p (xsize, ysize));
 
   /* Paradoxical subregs are otherwise valid.  */
-  if (!rknown && known_zero (offset) && may_gt (ysize, xsize))
+  if (!rknown && must_eq (offset, 0U) && may_gt (ysize, xsize))
     {
       info->representable_p = true;
       /* If this is a big endian paradoxical subreg, which uses more
@@ -3822,7 +3822,7 @@ subreg_get_info (unsigned int xregno, machine_mode xmode,
       info->representable_p = true;
       rknown = true;
 
-      if (known_zero (offset) || nregs_xmode == nregs_ymode)
+      if (must_eq (offset, 0U) || nregs_xmode == nregs_ymode)
 	{
 	  info->offset = 0;
 	  info->nregs = nregs_ymode;
diff --git a/gcc/sanitizer.def b/gcc/sanitizer.def
index 9d963f05c21..00e7ae031e6 100644
--- a/gcc/sanitizer.def
+++ b/gcc/sanitizer.def
@@ -424,8 +424,8 @@ DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_VLA_BOUND_NOT_POSITIVE,
 		      "__ubsan_handle_vla_bound_not_positive",
 		      BT_FN_VOID_PTR_PTR,
 		      ATTR_COLD_NOTHROW_LEAF_LIST)
-DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH,
-		      "__ubsan_handle_type_mismatch",
+DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1,
+		      "__ubsan_handle_type_mismatch_v1",
 		      BT_FN_VOID_PTR_PTR,
 		      ATTR_COLD_NOTHROW_LEAF_LIST)
 DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_ADD_OVERFLOW,
@@ -464,8 +464,8 @@ DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_VLA_BOUND_NOT_POSITIVE_ABORT,
 		      "__ubsan_handle_vla_bound_not_positive_abort",
 		      BT_FN_VOID_PTR_PTR,
 		      ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST)
-DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_ABORT,
-		      "__ubsan_handle_type_mismatch_abort",
+DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1_ABORT,
+		      "__ubsan_handle_type_mismatch_v1_abort",
 		      BT_FN_VOID_PTR_PTR,
 		      ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST)
 DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_ADD_OVERFLOW_ABORT,
@@ -516,12 +516,20 @@ DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_NONNULL_ARG_ABORT,
 		      "__ubsan_handle_nonnull_arg_abort",
 		      BT_FN_VOID_PTR,
 		      ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST)
-DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN,
-		      "__ubsan_handle_nonnull_return",
+DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_V1,
+		      "__ubsan_handle_nonnull_return_v1",
+		      BT_FN_VOID_PTR_PTR,
+		      ATTR_COLD_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_V1_ABORT,
+		      "__ubsan_handle_nonnull_return_v1_abort",
+		      BT_FN_VOID_PTR_PTR,
+		      ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_INVALID_BUILTIN,
+		      "__ubsan_handle_invalid_builtin",
 		      BT_FN_VOID_PTR,
 		      ATTR_COLD_NOTHROW_LEAF_LIST)
-DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_ABORT,
-		      "__ubsan_handle_nonnull_return_abort",
+DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_INVALID_BUILTIN_ABORT,
+		      "__ubsan_handle_invalid_builtin_abort",
 		      BT_FN_VOID_PTR,
 		      ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST)
 DEF_SANITIZER_BUILTIN(BUILT_IN_UBSAN_HANDLE_DYNAMIC_TYPE_CACHE_MISS,
diff --git a/gcc/sbitmap.c b/gcc/sbitmap.c
index baef4d05f0d..df933f6516c 100644
--- a/gcc/sbitmap.c
+++ b/gcc/sbitmap.c
@@ -180,6 +180,8 @@ sbitmap_vector_alloc (unsigned int n_vecs, unsigned int n_elms)
 void
 bitmap_copy (sbitmap dst, const_sbitmap src)
 {
+  gcc_checking_assert (src->size <= dst->size);
+
   memcpy (dst->elms, src->elms, sizeof (SBITMAP_ELT_TYPE) * dst->size);
 }
 
@@ -187,6 +189,8 @@ bitmap_copy (sbitmap dst, const_sbitmap src)
 int
 bitmap_equal_p (const_sbitmap a, const_sbitmap b)
 {
+  bitmap_check_sizes (a, b);
+
   return !memcmp (a->elms, b->elms, sizeof (SBITMAP_ELT_TYPE) * a->size);
 }
 
@@ -211,6 +215,8 @@ bitmap_clear_range (sbitmap bmap, unsigned int start, unsigned int count)
   if (count == 0)
     return;
 
+  bitmap_check_index (bmap, start + count - 1);
+
   unsigned int start_word = start / SBITMAP_ELT_BITS;
   unsigned int start_bitno = start % SBITMAP_ELT_BITS;
 
@@ -267,6 +273,8 @@ bitmap_set_range (sbitmap bmap, unsigned int start, unsigned int count)
   if (count == 0)
     return;
 
+  bitmap_check_index (bmap, start + count - 1);
+
   unsigned int start_word = start / SBITMAP_ELT_BITS;
   unsigned int start_bitno = start % SBITMAP_ELT_BITS;
 
@@ -324,6 +332,8 @@ bool
 bitmap_bit_in_range_p (const_sbitmap bmap, unsigned int start, unsigned int end)
 {
   gcc_checking_assert (start <= end);
+  bitmap_check_index (bmap, end);
+
   unsigned int start_word = start / SBITMAP_ELT_BITS;
   unsigned int start_bitno = start % SBITMAP_ELT_BITS;
 
@@ -467,6 +477,9 @@ bitmap_vector_ones (sbitmap *bmap, unsigned int n_vecs)
 bool
 bitmap_ior_and_compl (sbitmap dst, const_sbitmap a, const_sbitmap b, const_sbitmap c)
 {
+  bitmap_check_sizes (a, b);
+  bitmap_check_sizes (b, c);
+
   unsigned int i, n = dst->size;
   sbitmap_ptr dstp = dst->elms;
   const_sbitmap_ptr ap = a->elms;
@@ -489,6 +502,8 @@ bitmap_ior_and_compl (sbitmap dst, const_sbitmap a, const_sbitmap b, const_sbitm
 void
 bitmap_not (sbitmap dst, const_sbitmap src)
 {
+  bitmap_check_sizes (src, dst);
+
   unsigned int i, n = dst->size;
   sbitmap_ptr dstp = dst->elms;
   const_sbitmap_ptr srcp = src->elms;
@@ -510,6 +525,9 @@ bitmap_not (sbitmap dst, const_sbitmap src)
 void
 bitmap_and_compl (sbitmap dst, const_sbitmap a, const_sbitmap b)
 {
+  bitmap_check_sizes (a, b);
+  bitmap_check_sizes (b, dst);
+
   unsigned int i, dst_size = dst->size;
   unsigned int min_size = dst->size;
   sbitmap_ptr dstp = dst->elms;
@@ -537,6 +555,8 @@ bitmap_and_compl (sbitmap dst, const_sbitmap a, const_sbitmap b)
 bool
 bitmap_intersect_p (const_sbitmap a, const_sbitmap b)
 {
+  bitmap_check_sizes (a, b);
+
   const_sbitmap_ptr ap = a->elms;
   const_sbitmap_ptr bp = b->elms;
   unsigned int i, n;
@@ -555,6 +575,9 @@ bitmap_intersect_p (const_sbitmap a, const_sbitmap b)
 bool
 bitmap_and (sbitmap dst, const_sbitmap a, const_sbitmap b)
 {
+  bitmap_check_sizes (a, b);
+  bitmap_check_sizes (b, dst);
+
   unsigned int i, n = dst->size;
   sbitmap_ptr dstp = dst->elms;
   const_sbitmap_ptr ap = a->elms;
@@ -577,6 +600,9 @@ bitmap_and (sbitmap dst, const_sbitmap a, const_sbitmap b)
 bool
 bitmap_xor (sbitmap dst, const_sbitmap a, const_sbitmap b)
 {
+  bitmap_check_sizes (a, b);
+  bitmap_check_sizes (b, dst);
+
   unsigned int i, n = dst->size;
   sbitmap_ptr dstp = dst->elms;
   const_sbitmap_ptr ap = a->elms;
@@ -599,6 +625,9 @@ bitmap_xor (sbitmap dst, const_sbitmap a, const_sbitmap b)
 bool
 bitmap_ior (sbitmap dst, const_sbitmap a, const_sbitmap b)
 {
+  bitmap_check_sizes (a, b);
+  bitmap_check_sizes (b, dst);
+
   unsigned int i, n = dst->size;
   sbitmap_ptr dstp = dst->elms;
   const_sbitmap_ptr ap = a->elms;
@@ -620,6 +649,8 @@ bitmap_ior (sbitmap dst, const_sbitmap a, const_sbitmap b)
 bool
 bitmap_subset_p (const_sbitmap a, const_sbitmap b)
 {
+  bitmap_check_sizes (a, b);
+
   unsigned int i, n = a->size;
   const_sbitmap_ptr ap, bp;
 
@@ -636,6 +667,10 @@ bitmap_subset_p (const_sbitmap a, const_sbitmap b)
 bool
 bitmap_or_and (sbitmap dst, const_sbitmap a, const_sbitmap b, const_sbitmap c)
 {
+  bitmap_check_sizes (a, b);
+  bitmap_check_sizes (b, c);
+  bitmap_check_sizes (c, dst);
+
   unsigned int i, n = dst->size;
   sbitmap_ptr dstp = dst->elms;
   const_sbitmap_ptr ap = a->elms;
@@ -659,6 +694,10 @@ bitmap_or_and (sbitmap dst, const_sbitmap a, const_sbitmap b, const_sbitmap c)
 bool
 bitmap_and_or (sbitmap dst, const_sbitmap a, const_sbitmap b, const_sbitmap c)
 {
+  bitmap_check_sizes (a, b);
+  bitmap_check_sizes (b, c);
+  bitmap_check_sizes (c, dst);
+
   unsigned int i, n = dst->size;
   sbitmap_ptr dstp = dst->elms;
   const_sbitmap_ptr ap = a->elms;
@@ -823,11 +862,64 @@ namespace selftest {
 
 /* Selftests for sbitmaps.  */
 
+/* Checking function that uses both bitmap_bit_in_range_p and
+   loop of bitmap_bit_p and verifies consistent results.  */
+
+static bool
+bitmap_bit_in_range_p_checking (sbitmap s, unsigned int start,
+				unsigned end)
+{
+  bool r1 = bitmap_bit_in_range_p (s, start, end);
+  bool r2 = false;
+
+  for (unsigned int i = start; i <= end; i++)
+    if (bitmap_bit_p (s, i))
+      {
+	r2 = true;
+	break;
+      }
+
+  ASSERT_EQ (r1, r2);
+  return r1;
+}
+
+/* Verify bitmap_set_range functions for sbitmap.  */
+
+static void
+test_set_range ()
+{
+  sbitmap s = sbitmap_alloc (16);
+  bitmap_clear (s);
+
+  bitmap_set_range (s, 0, 1);
+  ASSERT_TRUE (bitmap_bit_in_range_p_checking (s, 0, 0));
+  ASSERT_FALSE (bitmap_bit_in_range_p_checking (s, 1, 15));
+  bitmap_set_range (s, 15, 1);
+  ASSERT_FALSE (bitmap_bit_in_range_p_checking (s, 1, 14));
+  ASSERT_TRUE (bitmap_bit_in_range_p_checking (s, 15, 15));
+
+  s = sbitmap_alloc (1024);
+  bitmap_clear (s);
+  bitmap_set_range (s, 512, 1);
+  ASSERT_FALSE (bitmap_bit_in_range_p_checking (s, 0, 511));
+  ASSERT_FALSE (bitmap_bit_in_range_p_checking (s, 513, 1023));
+  ASSERT_TRUE (bitmap_bit_in_range_p_checking (s, 512, 512));
+  ASSERT_TRUE (bitmap_bit_in_range_p_checking (s, 508, 512));
+  ASSERT_TRUE (bitmap_bit_in_range_p_checking (s, 508, 513));
+  ASSERT_FALSE (bitmap_bit_in_range_p_checking (s, 508, 511));
+
+  bitmap_clear (s);
+  bitmap_set_range (s, 512, 64);
+  ASSERT_FALSE (bitmap_bit_in_range_p_checking (s, 0, 511));
+  ASSERT_FALSE (bitmap_bit_in_range_p_checking (s, 512 + 64, 1023));
+  ASSERT_TRUE (bitmap_bit_in_range_p_checking (s, 512, 512));
+  ASSERT_TRUE (bitmap_bit_in_range_p_checking (s, 512 + 63, 512 + 63));
+}
 
-/* Verify range functions for sbitmap.  */
+/* Verify bitmap_bit_in_range_p functions for sbitmap.  */
 
 static void
-test_range_functions ()
+test_bit_in_range ()
 {
   sbitmap s = sbitmap_alloc (1024);
   bitmap_clear (s);
@@ -900,7 +992,8 @@ test_range_functions ()
 void
 sbitmap_c_tests ()
 {
-  test_range_functions ();
+  test_set_range ();
+  test_bit_in_range ();
 }
 
 } // namespace selftest
diff --git a/gcc/sbitmap.h b/gcc/sbitmap.h
index ff52e939bf3..a5ff0685e43 100644
--- a/gcc/sbitmap.h
+++ b/gcc/sbitmap.h
@@ -96,10 +96,29 @@ struct simple_bitmap_def
 /* Return the number of bits in BITMAP.  */
 #define SBITMAP_SIZE(BITMAP) ((BITMAP)->n_bits)
 
+/* Verify that access at INDEX in bitmap MAP is valid.  */ 
+
+static inline void
+bitmap_check_index (const_sbitmap map, int index)
+{
+  gcc_checking_assert (index >= 0);
+  gcc_checking_assert ((unsigned int)index < map->n_bits);
+}
+
+/* Verify that bitmaps A and B have same size.  */ 
+
+static inline void
+bitmap_check_sizes (const_sbitmap a, const_sbitmap b)
+{
+  gcc_checking_assert (a->n_bits == b->n_bits);
+}
+
 /* Test if bit number bitno in the bitmap is set.  */
 static inline SBITMAP_ELT_TYPE
 bitmap_bit_p (const_sbitmap map, int bitno)
 {
+  bitmap_check_index (map, bitno);
+
   size_t i = bitno / SBITMAP_ELT_BITS;
   unsigned int s = bitno % SBITMAP_ELT_BITS;
   return (map->elms[i] >> s) & (SBITMAP_ELT_TYPE) 1;
@@ -110,6 +129,8 @@ bitmap_bit_p (const_sbitmap map, int bitno)
 static inline void
 bitmap_set_bit (sbitmap map, int bitno)
 {
+  bitmap_check_index (map, bitno);
+
   map->elms[bitno / SBITMAP_ELT_BITS]
     |= (SBITMAP_ELT_TYPE) 1 << (bitno) % SBITMAP_ELT_BITS;
 }
@@ -119,6 +140,8 @@ bitmap_set_bit (sbitmap map, int bitno)
 static inline void
 bitmap_clear_bit (sbitmap map, int bitno)
 {
+  bitmap_check_index (map, bitno);
+
   map->elms[bitno / SBITMAP_ELT_BITS]
     &= ~((SBITMAP_ELT_TYPE) 1 << (bitno) % SBITMAP_ELT_BITS);
 }
@@ -148,6 +171,8 @@ static inline void
 bmp_iter_set_init (sbitmap_iterator *i, const_sbitmap bmp,
 		   unsigned int min, unsigned *bit_no ATTRIBUTE_UNUSED)
 {
+  bitmap_check_index (bmp, min);
+
   i->word_num = min / (unsigned int) SBITMAP_ELT_BITS;
   i->bit_num = min;
   i->size = bmp->size;
diff --git a/gcc/sched-ebb.c b/gcc/sched-ebb.c
index a0422f4b1ba..e51749c60cc 100644
--- a/gcc/sched-ebb.c
+++ b/gcc/sched-ebb.c
@@ -231,11 +231,9 @@ rank (rtx_insn *insn1, rtx_insn *insn2)
   basic_block bb1 = BLOCK_FOR_INSN (insn1);
   basic_block bb2 = BLOCK_FOR_INSN (insn2);
 
-  if (bb1->count > bb2->count
-      || bb1->frequency > bb2->frequency)
+  if (bb1->count > bb2->count)
     return -1;
-  if (bb1->count < bb2->count
-      || bb1->frequency < bb2->frequency)
+  if (bb1->count < bb2->count)
     return 1;
   return 0;
 }
diff --git a/gcc/sched-int.h b/gcc/sched-int.h
index 2af8f9fc32c..6832589e3d0 100644
--- a/gcc/sched-int.h
+++ b/gcc/sched-int.h
@@ -819,15 +819,8 @@ struct autopref_multipass_data_
   /* Base part of memory address.  */
   rtx base;
 
-  /* Memory offsets from the base.  For single simple sets
-     only min_offset is valid.  For multi-set insns min_offset
-     and max_offset record the minimum and maximum offsets from the same
-     base among the sets inside the PARALLEL.  */
-  int min_offset;
-  int max_offset;
-
-  /* True if this is a load/store-multiple instruction.  */
-  bool multi_mem_insn_p;
+  /* Memory offsets from the base.  */
+  int offset;
 
   /* Entry status.  */
   enum autopref_multipass_data_status status;
diff --git a/gcc/sdbout.c b/gcc/sdbout.c
deleted file mode 100644
index acd25a3c765..00000000000
--- a/gcc/sdbout.c
+++ /dev/null
@@ -1,1661 +0,0 @@
-/* Output sdb-format symbol table information from GNU compiler.
-   Copyright (C) 1988-2017 Free Software Foundation, Inc.
-
-This file is part of GCC.
-
-GCC is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 3, or (at your option) any later
-version.
-
-GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with GCC; see the file COPYING3.  If not see
-<http://www.gnu.org/licenses/>.  */
-
-/*  mike@tredysvr.Tredydev.Unisys.COM says:
-I modified the struct.c example and have a nm of a .o resulting from the
-AT&T C compiler.  From the example below I would conclude the following:
-
-1. All .defs from structures are emitted as scanned.  The example below
-   clearly shows the symbol table entries for BoxRec2 are after the first
-   function.
-
-2. All functions and their locals (including statics) are emitted as scanned.
-
-3. All nested unnamed union and structure .defs must be emitted before
-   the structure in which they are nested.  The AT&T assembler is a
-   one pass beast as far as symbolics are concerned.
-
-4. All structure .defs are emitted before the typedefs that refer to them.
-
-5. All top level static and external variable definitions are moved to the
-   end of file with all top level statics occurring first before externs.
-
-6. All undefined references are at the end of the file.
-*/
-
-#include "config.h"
-#include "system.h"
-#include "coretypes.h"
-#include "gsyms.h"
-#include "tm.h"
-#include "debug.h"
-#include "tree.h"
-#include "varasm.h"
-#include "stor-layout.h"
-
-static GTY(()) tree anonymous_types;
-
-/* Counter to generate unique "names" for nameless struct members.  */
-
-static GTY(()) int unnamed_struct_number;
-
-/* Declarations whose debug info was deferred till end of compilation.  */
-
-static GTY(()) vec<tree, va_gc> *deferred_global_decls;
-
-/* The C front end may call sdbout_symbol before sdbout_init runs.
-   We save all such decls in this list and output them when we get
-   to sdbout_init.  */
-
-static GTY(()) tree preinit_symbols;
-static GTY(()) bool sdbout_initialized;
-
-#include "rtl.h"
-#include "regs.h"
-#include "function.h"
-#include "memmodel.h"
-#include "emit-rtl.h"
-#include "flags.h"
-#include "insn-config.h"
-#include "reload.h"
-#include "output.h"
-#include "diagnostic-core.h"
-#include "tm_p.h"
-#include "langhooks.h"
-#include "target.h"
-
-/* 1 if PARM is passed to this function in memory.  */
-
-#define PARM_PASSED_IN_MEMORY(PARM) \
- (MEM_P (DECL_INCOMING_RTL (PARM)))
-
-/* A C expression for the integer offset value of an automatic variable
-   (C_AUTO) having address X (an RTX).  */
-#ifndef DEBUGGER_AUTO_OFFSET
-#define DEBUGGER_AUTO_OFFSET(X) \
-  (GET_CODE (X) == PLUS ? INTVAL (XEXP (X, 1)) : 0)
-#endif
-
-/* A C expression for the integer offset value of an argument (C_ARG)
-   having address X (an RTX).  The nominal offset is OFFSET.  */
-#ifndef DEBUGGER_ARG_OFFSET
-#define DEBUGGER_ARG_OFFSET(OFFSET, X) (OFFSET)
-#endif
-
-/* Line number of beginning of current function, minus one.
-   Negative means not in a function or not using sdb.  */
-
-int sdb_begin_function_line = -1;
-
-
-extern FILE *asm_out_file;
-
-extern tree current_function_decl;
-
-#include "sdbout.h"
-
-static void sdbout_init			(const char *);
-static void sdbout_finish		(const char *);
-static void sdbout_start_source_file	(unsigned int, const char *);
-static void sdbout_end_source_file	(unsigned int);
-static void sdbout_begin_block		(unsigned int, unsigned int);
-static void sdbout_end_block		(unsigned int, unsigned int);
-static void sdbout_source_line		(unsigned int, unsigned int,
-					 const char *, int, bool);
-static void sdbout_end_epilogue		(unsigned int, const char *);
-static void sdbout_early_global_decl	(tree);
-static void sdbout_late_global_decl	(tree);
-static void sdbout_begin_prologue	(unsigned int, unsigned int,
-					 const char *);
-static void sdbout_end_prologue		(unsigned int, const char *);
-static void sdbout_begin_function	(tree);
-static void sdbout_end_function		(unsigned int);
-static void sdbout_toplevel_data	(tree);
-static void sdbout_label		(rtx_code_label *);
-static char *gen_fake_label		(void);
-static int plain_type			(tree);
-static int template_name_p		(tree);
-static void sdbout_record_type_name	(tree);
-static int plain_type_1			(tree, int);
-static void sdbout_block		(tree);
-static void sdbout_syms			(tree);
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-static void sdbout_queue_anonymous_type	(tree);
-static void sdbout_dequeue_anonymous_types (void);
-#endif
-static void sdbout_type			(tree);
-static void sdbout_field_types		(tree);
-static void sdbout_one_type		(tree);
-static void sdbout_parms		(tree);
-static void sdbout_reg_parms		(tree);
-
-/* Random macros describing parts of SDB data.  */
-
-/* Default value of delimiter is ";".  */
-#ifndef SDB_DELIM
-#define SDB_DELIM	";"
-#endif
-
-/* Maximum number of dimensions the assembler will allow.  */
-#ifndef SDB_MAX_DIM
-#define SDB_MAX_DIM 4
-#endif
-
-#ifndef PUT_SDB_SCL
-#define PUT_SDB_SCL(a) fprintf (asm_out_file, "\t.scl\t%d%s", (a), SDB_DELIM)
-#endif
-
-#ifndef PUT_SDB_INT_VAL
-#define PUT_SDB_INT_VAL(a) \
- do {									\
-   fprintf (asm_out_file, "\t.val\t" HOST_WIDE_INT_PRINT_DEC "%s",	\
-	    (HOST_WIDE_INT) (a), SDB_DELIM);				\
- } while (0)
-
-#endif
-
-#ifndef PUT_SDB_VAL
-#define PUT_SDB_VAL(a)				\
-( fputs ("\t.val\t", asm_out_file),		\
-  output_addr_const (asm_out_file, (a)),	\
-  fprintf (asm_out_file, SDB_DELIM))
-#endif
-
-#ifndef PUT_SDB_DEF
-#define PUT_SDB_DEF(a)				\
-do { fprintf (asm_out_file, "\t.def\t");	\
-     assemble_name (asm_out_file, a);	\
-     fprintf (asm_out_file, SDB_DELIM); } while (0)
-#endif
-
-#ifndef PUT_SDB_PLAIN_DEF
-#define PUT_SDB_PLAIN_DEF(a) \
-  fprintf (asm_out_file, "\t.def\t.%s%s", a, SDB_DELIM)
-#endif
-
-#ifndef PUT_SDB_ENDEF
-#define PUT_SDB_ENDEF fputs ("\t.endef\n", asm_out_file)
-#endif
-
-#ifndef PUT_SDB_TYPE
-#define PUT_SDB_TYPE(a) fprintf (asm_out_file, "\t.type\t0%o%s", a, SDB_DELIM)
-#endif
-
-#ifndef PUT_SDB_SIZE
-#define PUT_SDB_SIZE(a) \
- do {									\
-   fprintf (asm_out_file, "\t.size\t" HOST_WIDE_INT_PRINT_DEC "%s",	\
-	    (HOST_WIDE_INT) (a), SDB_DELIM);				\
- } while (0)
-#endif
-
-#ifndef PUT_SDB_START_DIM
-#define PUT_SDB_START_DIM fprintf (asm_out_file, "\t.dim\t")
-#endif
-
-#ifndef PUT_SDB_NEXT_DIM
-#define PUT_SDB_NEXT_DIM(a) fprintf (asm_out_file, "%d,", a)
-#endif
-
-#ifndef PUT_SDB_LAST_DIM
-#define PUT_SDB_LAST_DIM(a) fprintf (asm_out_file, "%d%s", a, SDB_DELIM)
-#endif
-
-#ifndef PUT_SDB_TAG
-#define PUT_SDB_TAG(a)				\
-do { fprintf (asm_out_file, "\t.tag\t");	\
-     assemble_name (asm_out_file, a);	\
-     fprintf (asm_out_file, SDB_DELIM); } while (0)
-#endif
-
-#ifndef PUT_SDB_BLOCK_START
-#define PUT_SDB_BLOCK_START(LINE)		\
-  fprintf (asm_out_file,			\
-	   "\t.def\t.bb%s\t.val\t.%s\t.scl\t100%s\t.line\t%d%s\t.endef\n", \
-	   SDB_DELIM, SDB_DELIM, SDB_DELIM, (LINE), SDB_DELIM)
-#endif
-
-#ifndef PUT_SDB_BLOCK_END
-#define PUT_SDB_BLOCK_END(LINE)			\
-  fprintf (asm_out_file,			\
-	   "\t.def\t.eb%s\t.val\t.%s\t.scl\t100%s\t.line\t%d%s\t.endef\n",  \
-	   SDB_DELIM, SDB_DELIM, SDB_DELIM, (LINE), SDB_DELIM)
-#endif
-
-#ifndef PUT_SDB_FUNCTION_START
-#define PUT_SDB_FUNCTION_START(LINE)		\
-  fprintf (asm_out_file,			\
-	   "\t.def\t.bf%s\t.val\t.%s\t.scl\t101%s\t.line\t%d%s\t.endef\n", \
-	   SDB_DELIM, SDB_DELIM, SDB_DELIM, (LINE), SDB_DELIM)
-#endif
-
-#ifndef PUT_SDB_FUNCTION_END
-#define PUT_SDB_FUNCTION_END(LINE)		\
-  fprintf (asm_out_file,			\
-	   "\t.def\t.ef%s\t.val\t.%s\t.scl\t101%s\t.line\t%d%s\t.endef\n", \
-	   SDB_DELIM, SDB_DELIM, SDB_DELIM, (LINE), SDB_DELIM)
-#endif
-
-/* Return the sdb tag identifier string for TYPE
-   if TYPE has already been defined; otherwise return a null pointer.  */
-
-#define KNOWN_TYPE_TAG(type)  TYPE_SYMTAB_POINTER (type)
-
-/* Set the sdb tag identifier string for TYPE to NAME.  */
-
-#define SET_KNOWN_TYPE_TAG(TYPE, NAME) \
-  TYPE_SYMTAB_POINTER (TYPE) = (const char *)(NAME)
-
-/* Return the name (a string) of the struct, union or enum tag
-   described by the TREE_LIST node LINK.  This is 0 for an anonymous one.  */
-
-#define TAG_NAME(link) \
-  (((link) && TREE_PURPOSE ((link)) \
-    && IDENTIFIER_POINTER (TREE_PURPOSE ((link)))) \
-   ? IDENTIFIER_POINTER (TREE_PURPOSE ((link))) : (char *) 0)
-
-/* Ensure we don't output a negative line number.  */
-#define MAKE_LINE_SAFE(line)  \
-  if ((int) line <= sdb_begin_function_line) \
-    line = sdb_begin_function_line + 1
-
-/* The debug hooks structure.  */
-const struct gcc_debug_hooks sdb_debug_hooks =
-{
-  sdbout_init,			         /* init */
-  sdbout_finish,		         /* finish */
-  debug_nothing_charstar,		 /* early_finish */
-  debug_nothing_void,			 /* assembly_start */
-  debug_nothing_int_charstar,	         /* define */
-  debug_nothing_int_charstar,	         /* undef */
-  sdbout_start_source_file,	         /* start_source_file */
-  sdbout_end_source_file,	         /* end_source_file */
-  sdbout_begin_block,		         /* begin_block */
-  sdbout_end_block,		         /* end_block */
-  debug_true_const_tree,	         /* ignore_block */
-  sdbout_source_line,		         /* source_line */
-  sdbout_begin_prologue,	         /* begin_prologue */
-  debug_nothing_int_charstar,	         /* end_prologue */
-  debug_nothing_int_charstar,	         /* begin_epilogue */
-  sdbout_end_epilogue,		         /* end_epilogue */
-  sdbout_begin_function,	         /* begin_function */
-  sdbout_end_function,		         /* end_function */
-  debug_nothing_tree,		         /* register_main_translation_unit */
-  debug_nothing_tree,		         /* function_decl */
-  sdbout_early_global_decl,		 /* early_global_decl */
-  sdbout_late_global_decl,		 /* late_global_decl */
-  sdbout_symbol,			 /* type_decl */
-  debug_nothing_tree_tree_tree_bool_bool,/* imported_module_or_decl */
-  debug_false_tree_charstarstar_uhwistar,/* die_ref_for_decl */
-  debug_nothing_tree_charstar_uhwi,      /* register_external_die */
-  debug_nothing_tree,		         /* deferred_inline_function */
-  debug_nothing_tree,		         /* outlining_inline_function */
-  sdbout_label,			         /* label */
-  debug_nothing_int,		         /* handle_pch */
-  debug_nothing_rtx_insn,	         /* var_location */
-  debug_nothing_tree,			 /* size_function */
-  debug_nothing_void,                    /* switch_text_section */
-  debug_nothing_tree_tree,		 /* set_name */
-  0,                                     /* start_end_main_source_file */
-  TYPE_SYMTAB_IS_POINTER                 /* tree_type_symtab_field */
-};
-
-/* Return a unique string to name an anonymous type.  */
-
-static char *
-gen_fake_label (void)
-{
-  char label[10];
-  char *labelstr;
-  sprintf (label, ".%dfake", unnamed_struct_number);
-  unnamed_struct_number++;
-  labelstr = xstrdup (label);
-  return labelstr;
-}
-
-/* Return the number which describes TYPE for SDB.
-   For pointers, etc., this function is recursive.
-   Each record, union or enumeral type must already have had a
-   tag number output.  */
-
-/* The number is given by d6d5d4d3d2d1bbbb
-   where bbbb is 4 bit basic type, and di indicate  one of notype,ptr,fn,array.
-   Thus, char *foo () has bbbb=T_CHAR
-			  d1=D_FCN
-			  d2=D_PTR
- N_BTMASK=     017       1111     basic type field.
- N_TSHIFT=       2                derived type shift
- N_BTSHFT=       4                Basic type shift */
-
-/* Produce the number that describes a pointer, function or array type.
-   PREV is the number describing the target, value or element type.
-   DT_type describes how to transform that type.  */
-#define PUSH_DERIVED_LEVEL(DT_type,PREV)		\
-  ((((PREV) & ~(int) N_BTMASK) << (int) N_TSHIFT)		\
-   | ((int) DT_type << (int) N_BTSHFT)			\
-   | ((PREV) & (int) N_BTMASK))
-
-/* Number of elements used in sdb_dims.  */
-static int sdb_n_dims = 0;
-
-/* Table of array dimensions of current type.  */
-static int sdb_dims[SDB_MAX_DIM];
-
-/* Size of outermost array currently being processed.  */
-static int sdb_type_size = -1;
-
-static int
-plain_type (tree type)
-{
-  int val = plain_type_1 (type, 0);
-
-  /* If we have already saved up some array dimensions, print them now.  */
-  if (sdb_n_dims > 0)
-    {
-      int i;
-      PUT_SDB_START_DIM;
-      for (i = sdb_n_dims - 1; i > 0; i--)
-	PUT_SDB_NEXT_DIM (sdb_dims[i]);
-      PUT_SDB_LAST_DIM (sdb_dims[0]);
-      sdb_n_dims = 0;
-
-      sdb_type_size = int_size_in_bytes (type);
-      /* Don't kill sdb if type is not laid out or has variable size.  */
-      if (sdb_type_size < 0)
-	sdb_type_size = 0;
-    }
-  /* If we have computed the size of an array containing this type,
-     print it now.  */
-  if (sdb_type_size >= 0)
-    {
-      PUT_SDB_SIZE (sdb_type_size);
-      sdb_type_size = -1;
-    }
-  return val;
-}
-
-static int
-template_name_p (tree name)
-{
-  const char *ptr = IDENTIFIER_POINTER (name);
-  while (*ptr && *ptr != '<')
-    ptr++;
-
-  return *ptr != '\0';
-}
-
-static void
-sdbout_record_type_name (tree type)
-{
-  const char *name = 0;
-  int no_name;
-
-  if (KNOWN_TYPE_TAG (type))
-    return;
-
-  if (TYPE_NAME (type) != 0)
-    {
-      tree t = 0;
-
-      /* Find the IDENTIFIER_NODE for the type name.  */
-      if (TREE_CODE (TYPE_NAME (type)) == IDENTIFIER_NODE)
-	t = TYPE_NAME (type);
-      else if (TREE_CODE (TYPE_NAME (type)) == TYPE_DECL)
-	{
-	  t = DECL_NAME (TYPE_NAME (type));
-	  /* The DECL_NAME for templates includes "<>", which breaks
-	     most assemblers.  Use its assembler name instead, which
-	     has been mangled into being safe.  */
-	  if (t && template_name_p (t))
-	    t = DECL_ASSEMBLER_NAME (TYPE_NAME (type));
-	}
-
-      /* Now get the name as a string, or invent one.  */
-      if (t != NULL_TREE)
-	name = IDENTIFIER_POINTER (t);
-    }
-
-  no_name = (name == 0 || *name == 0);
-  if (no_name)
-    name = gen_fake_label ();
-
-  SET_KNOWN_TYPE_TAG (type, name);
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-  if (no_name)
-    sdbout_queue_anonymous_type (type);
-#endif
-}
-
-/* Return the .type value for type TYPE.
-
-   LEVEL indicates how many levels deep we have recursed into the type.
-   The SDB debug format can only represent 6 derived levels of types.
-   After that, we must output inaccurate debug info.  We deliberately
-   stop before the 7th level, so that ADA recursive types will not give an
-   infinite loop.  */
-
-static int
-plain_type_1 (tree type, int level)
-{
-  if (type == 0)
-    type = void_type_node;
-  else if (type == error_mark_node)
-    type = integer_type_node;
-  else
-    type = TYPE_MAIN_VARIANT (type);
-
-  switch (TREE_CODE (type))
-    {
-    case VOID_TYPE:
-    case NULLPTR_TYPE:
-      return T_VOID;
-    case BOOLEAN_TYPE:
-    case INTEGER_TYPE:
-      {
-	int size = int_size_in_bytes (type) * BITS_PER_UNIT;
-
-	/* Carefully distinguish all the standard types of C,
-	   without messing up if the language is not C.
-	   Note that we check only for the names that contain spaces;
-	   other names might occur by coincidence in other languages.  */
-	if (TYPE_NAME (type) != 0
-	    && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
-	    && DECL_NAME (TYPE_NAME (type)) != 0
-	    && TREE_CODE (DECL_NAME (TYPE_NAME (type))) == IDENTIFIER_NODE)
-	  {
-	    const char *const name
-	      = IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type)));
-
-	    if (!strcmp (name, "char"))
-	      return T_CHAR;
-	    if (!strcmp (name, "unsigned char"))
-	      return T_UCHAR;
-	    if (!strcmp (name, "signed char"))
-	      return T_CHAR;
-	    if (!strcmp (name, "int"))
-	      return T_INT;
-	    if (!strcmp (name, "unsigned int"))
-	      return T_UINT;
-	    if (!strcmp (name, "short int"))
-	      return T_SHORT;
-	    if (!strcmp (name, "short unsigned int"))
-	      return T_USHORT;
-	    if (!strcmp (name, "long int"))
-	      return T_LONG;
-	    if (!strcmp (name, "long unsigned int"))
-	      return T_ULONG;
-	  }
-
-	if (size == INT_TYPE_SIZE)
-	  return (TYPE_UNSIGNED (type) ? T_UINT : T_INT);
-	if (size == CHAR_TYPE_SIZE)
-	  return (TYPE_UNSIGNED (type) ? T_UCHAR : T_CHAR);
-	if (size == SHORT_TYPE_SIZE)
-	  return (TYPE_UNSIGNED (type) ? T_USHORT : T_SHORT);
-	if (size == LONG_TYPE_SIZE)
-	  return (TYPE_UNSIGNED (type) ? T_ULONG : T_LONG);
-	if (size == LONG_LONG_TYPE_SIZE)	/* better than nothing */
-	  return (TYPE_UNSIGNED (type) ? T_ULONG : T_LONG);
-	return 0;
-      }
-
-    case REAL_TYPE:
-      {
-	int precision = TYPE_PRECISION (type);
-	if (precision == FLOAT_TYPE_SIZE)
-	  return T_FLOAT;
-	if (precision == DOUBLE_TYPE_SIZE)
-	  return T_DOUBLE;
-	if (precision == LONG_DOUBLE_TYPE_SIZE)
-	  return T_DOUBLE;	/* better than nothing */
-
-	return 0;
-      }
-
-    case ARRAY_TYPE:
-      {
-	int m;
-	if (level >= 6)
-	  return T_VOID;
-	else
-	  m = plain_type_1 (TREE_TYPE (type), level+1);
-	if (sdb_n_dims < SDB_MAX_DIM)
-	  sdb_dims[sdb_n_dims++]
-	    = (TYPE_DOMAIN (type)
-	       && TYPE_MIN_VALUE (TYPE_DOMAIN (type)) != 0
-	       && TYPE_MAX_VALUE (TYPE_DOMAIN (type)) != 0
-	       && tree_fits_shwi_p (TYPE_MAX_VALUE (TYPE_DOMAIN (type)))
-	       && tree_fits_shwi_p (TYPE_MIN_VALUE (TYPE_DOMAIN (type)))
-	       ? (tree_to_shwi (TYPE_MAX_VALUE (TYPE_DOMAIN (type)))
-		  - tree_to_shwi (TYPE_MIN_VALUE (TYPE_DOMAIN (type))) + 1)
-	       : 0);
-
-	return PUSH_DERIVED_LEVEL (DT_ARY, m);
-      }
-
-    case RECORD_TYPE:
-    case UNION_TYPE:
-    case QUAL_UNION_TYPE:
-    case ENUMERAL_TYPE:
-      {
-	const char *tag;
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-	sdbout_record_type_name (type);
-#endif
-#ifndef SDB_ALLOW_UNKNOWN_REFERENCES
-	if ((TREE_ASM_WRITTEN (type) && KNOWN_TYPE_TAG (type) != 0)
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-	    || TYPE_MODE (type) != VOIDmode
-#endif
-	    )
-#endif
-	  {
-	    /* Output the referenced structure tag name
-	       only if the .def has already been finished.
-	       At least on 386, the Unix assembler
-	       cannot handle forward references to tags.  */
-	    /* But the 88100, it requires them, sigh...  */
-	    /* And the MIPS requires unknown refs as well...  */
-	    tag = KNOWN_TYPE_TAG (type);
-	    PUT_SDB_TAG (tag);
-	    /* These 3 lines used to follow the close brace.
-	       However, a size of 0 without a tag implies a tag of 0,
-	       so if we don't know a tag, we can't mention the size.  */
-	    sdb_type_size = int_size_in_bytes (type);
-	    if (sdb_type_size < 0)
-	      sdb_type_size = 0;
-	  }
-	return ((TREE_CODE (type) == RECORD_TYPE) ? T_STRUCT
-		: (TREE_CODE (type) == UNION_TYPE) ? T_UNION
-		: (TREE_CODE (type) == QUAL_UNION_TYPE) ? T_UNION
-		: T_ENUM);
-      }
-    case POINTER_TYPE:
-    case REFERENCE_TYPE:
-      {
-	int m;
-	if (level >= 6)
-	  return T_VOID;
-	else
-	  m = plain_type_1 (TREE_TYPE (type), level+1);
-	return PUSH_DERIVED_LEVEL (DT_PTR, m);
-      }
-    case FUNCTION_TYPE:
-    case METHOD_TYPE:
-      {
-	int m;
-	if (level >= 6)
-	  return T_VOID;
-	else
-	  m = plain_type_1 (TREE_TYPE (type), level+1);
-	return PUSH_DERIVED_LEVEL (DT_FCN, m);
-      }
-    default:
-      return 0;
-    }
-}
-
-/* Output the symbols defined in block number DO_BLOCK.
-
-   This function works by walking the tree structure of blocks,
-   counting blocks until it finds the desired block.  */
-
-static int do_block = 0;
-
-static void
-sdbout_block (tree block)
-{
-  while (block)
-    {
-      /* Ignore blocks never expanded or otherwise marked as real.  */
-      if (TREE_USED (block))
-	{
-	  /* When we reach the specified block, output its symbols.  */
-	  if (BLOCK_NUMBER (block) == do_block)
-	    sdbout_syms (BLOCK_VARS (block));
-
-	  /* If we are past the specified block, stop the scan.  */
-	  if (BLOCK_NUMBER (block) > do_block)
-	    return;
-
-	  /* Scan the blocks within this block.  */
-	  sdbout_block (BLOCK_SUBBLOCKS (block));
-	}
-
-      block = BLOCK_CHAIN (block);
-    }
-}
-
-/* Call sdbout_symbol on each decl in the chain SYMS.  */
-
-static void
-sdbout_syms (tree syms)
-{
-  while (syms)
-    {
-      if (TREE_CODE (syms) != LABEL_DECL)
-	sdbout_symbol (syms, 1);
-      syms = TREE_CHAIN (syms);
-    }
-}
-
-/* Output SDB information for a symbol described by DECL.
-   LOCAL is nonzero if the symbol is not file-scope.  */
-
-void
-sdbout_symbol (tree decl, int local)
-{
-  tree type = TREE_TYPE (decl);
-  tree context = NULL_TREE;
-  rtx value;
-  int regno = -1;
-  const char *name;
-
-  /* If we are called before sdbout_init is run, just save the symbol
-     for later.  */
-  if (!sdbout_initialized)
-    {
-      preinit_symbols = tree_cons (0, decl, preinit_symbols);
-      return;
-    }
-
-  sdbout_one_type (type);
-
-  switch (TREE_CODE (decl))
-    {
-    case CONST_DECL:
-      /* Enum values are defined by defining the enum type.  */
-      return;
-
-    case FUNCTION_DECL:
-      /* Don't mention a nested function under its parent.  */
-      context = decl_function_context (decl);
-      if (context == current_function_decl)
-	return;
-      /* Check DECL_INITIAL to distinguish declarations from definitions.
-	 Don't output debug info here for declarations; they will have
-	 a DECL_INITIAL value of 0.  */
-      if (! DECL_INITIAL (decl))
-	return;
-      if (!MEM_P (DECL_RTL (decl))
-	  || GET_CODE (XEXP (DECL_RTL (decl), 0)) != SYMBOL_REF)
-	return;
-      PUT_SDB_DEF (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
-      PUT_SDB_VAL (XEXP (DECL_RTL (decl), 0));
-      PUT_SDB_SCL (TREE_PUBLIC (decl) ? C_EXT : C_STAT);
-      break;
-
-    case TYPE_DECL:
-      /* Done with tagged types.  */
-      if (DECL_NAME (decl) == 0)
-	return;
-      if (DECL_IGNORED_P (decl))
-	return;
-      /* Don't output intrinsic types.  GAS chokes on SDB .def
-	 statements that contain identifiers with embedded spaces
-	 (eg "unsigned long").  */
-      if (DECL_IS_BUILTIN (decl))
-	return;
-
-      /* Output typedef name.  */
-      if (template_name_p (DECL_NAME (decl)))
-	PUT_SDB_DEF (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
-      else
-	PUT_SDB_DEF (IDENTIFIER_POINTER (DECL_NAME (decl)));
-      PUT_SDB_SCL (C_TPDEF);
-      break;
-
-    case PARM_DECL:
-      /* Parm decls go in their own separate chains
-	 and are output by sdbout_reg_parms and sdbout_parms.  */
-      gcc_unreachable ();
-
-    case VAR_DECL:
-      /* Don't mention a variable that is external.
-	 Let the file that defines it describe it.  */
-      if (DECL_EXTERNAL (decl))
-	return;
-
-      /* Ignore __FUNCTION__, etc.  */
-      if (DECL_IGNORED_P (decl))
-	return;
-
-      /* If there was an error in the declaration, don't dump core
-	 if there is no RTL associated with the variable doesn't
-	 exist.  */
-      if (!DECL_RTL_SET_P (decl))
-	return;
-
-      value = DECL_RTL (decl);
-
-      if (!is_global_var (decl))
-	value = eliminate_regs (value, VOIDmode, NULL_RTX);
-
-      SET_DECL_RTL (decl, value);
-#ifdef LEAF_REG_REMAP
-      if (crtl->uses_only_leaf_regs)
-	leaf_renumber_regs_insn (value);
-#endif
-
-      /* Don't mention a variable at all
-	 if it was completely optimized into nothingness.
-
-	 If DECL was from an inline function, then its rtl
-	 is not identically the rtl that was used in this
-	 particular compilation.  */
-      if (REG_P (value))
-	{
-	  regno = REGNO (value);
-	  if (regno >= FIRST_PSEUDO_REGISTER)
-	    return;
-	}
-      else if (GET_CODE (value) == SUBREG)
-	{
-	  while (GET_CODE (value) == SUBREG)
-	    value = SUBREG_REG (value);
-	  if (REG_P (value))
-	    {
-	      if (REGNO (value) >= FIRST_PSEUDO_REGISTER)
-		return;
-	    }
-	  regno = REGNO (alter_subreg (&value, true));
-	  SET_DECL_RTL (decl, value);
-	}
-      /* Don't output anything if an auto variable
-	 gets RTL that is static.
-	 GAS version 2.2 can't handle such output.  */
-      else if (MEM_P (value) && CONSTANT_P (XEXP (value, 0))
-	       && ! TREE_STATIC (decl))
-	return;
-
-      /* Emit any structure, union, or enum type that has not been output.
-	 This occurs for tag-less structs (et al) used to declare variables
-	 within functions.  */
-      if (TREE_CODE (type) == ENUMERAL_TYPE
-	  || TREE_CODE (type) == RECORD_TYPE
-	  || TREE_CODE (type) == UNION_TYPE
-	  || TREE_CODE (type) == QUAL_UNION_TYPE)
-	{
-	  if (COMPLETE_TYPE_P (type)		/* not a forward reference */
-	      && KNOWN_TYPE_TAG (type) == 0)	/* not yet declared */
-	    sdbout_one_type (type);
-	}
-
-      /* Defer SDB information for top-level initialized variables! */
-      if (! local
-	  && MEM_P (value)
-	  && DECL_INITIAL (decl))
-	return;
-
-      /* C++ in 2.3 makes nameless symbols.  That will be fixed later.
-	 For now, avoid crashing.  */
-      if (DECL_NAME (decl) == NULL_TREE)
-	return;
-
-      /* Record the name for, starting a symtab entry.  */
-      if (local)
-	name = IDENTIFIER_POINTER (DECL_NAME (decl));
-      else
-	name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
-
-      if (MEM_P (value)
-	  && GET_CODE (XEXP (value, 0)) == SYMBOL_REF)
-	{
-	  PUT_SDB_DEF (name);
-	  if (TREE_PUBLIC (decl))
-	    {
-	      PUT_SDB_VAL (XEXP (value, 0));
-	      PUT_SDB_SCL (C_EXT);
-	    }
-	  else
-	    {
-	      PUT_SDB_VAL (XEXP (value, 0));
-	      PUT_SDB_SCL (C_STAT);
-	    }
-	}
-      else if (regno >= 0)
-	{
-	  PUT_SDB_DEF (name);
-	  PUT_SDB_INT_VAL (DBX_REGISTER_NUMBER (regno));
-	  PUT_SDB_SCL (C_REG);
-	}
-      else if (MEM_P (value)
-	       && (MEM_P (XEXP (value, 0))
-		   || (REG_P (XEXP (value, 0))
-		       && REGNO (XEXP (value, 0)) != HARD_FRAME_POINTER_REGNUM
-		       && REGNO (XEXP (value, 0)) != STACK_POINTER_REGNUM)))
-	/* If the value is indirect by memory or by a register
-	   that isn't the frame pointer
-	   then it means the object is variable-sized and address through
-	   that register or stack slot.  COFF has no way to represent this
-	   so all we can do is output the variable as a pointer.  */
-	{
-	  PUT_SDB_DEF (name);
-	  if (REG_P (XEXP (value, 0)))
-	    {
-	      PUT_SDB_INT_VAL (DBX_REGISTER_NUMBER (REGNO (XEXP (value, 0))));
-	      PUT_SDB_SCL (C_REG);
-	    }
-	  else
-	    {
-	      /* DECL_RTL looks like (MEM (MEM (PLUS (REG...)
-		 (CONST_INT...)))).
-		 We want the value of that CONST_INT.  */
-	      /* Encore compiler hates a newline in a macro arg, it seems.  */
-	      PUT_SDB_INT_VAL (DEBUGGER_AUTO_OFFSET
-			       (XEXP (XEXP (value, 0), 0)));
-	      PUT_SDB_SCL (C_AUTO);
-	    }
-
-	  /* Effectively do build_pointer_type, but don't cache this type,
-	     since it might be temporary whereas the type it points to
-	     might have been saved for inlining.  */
-	  /* Don't use REFERENCE_TYPE because dbx can't handle that.  */
-	  type = make_node (POINTER_TYPE);
-	  TREE_TYPE (type) = TREE_TYPE (decl);
-	}
-      else if (MEM_P (value)
-	       && ((GET_CODE (XEXP (value, 0)) == PLUS
-		    && REG_P (XEXP (XEXP (value, 0), 0))
-		    && CONST_INT_P (XEXP (XEXP (value, 0), 1)))
-		   /* This is for variables which are at offset zero from
-		      the frame pointer.  This happens on the Alpha.
-		      Non-frame pointer registers are excluded above.  */
-		   || (REG_P (XEXP (value, 0)))))
-	{
-	  /* DECL_RTL looks like (MEM (PLUS (REG...) (CONST_INT...)))
-	     or (MEM (REG...)).  We want the value of that CONST_INT
-	     or zero.  */
-	  PUT_SDB_DEF (name);
-	  PUT_SDB_INT_VAL (DEBUGGER_AUTO_OFFSET (XEXP (value, 0)));
-	  PUT_SDB_SCL (C_AUTO);
-	}
-      else
-	{
-	  /* It is something we don't know how to represent for SDB.  */
-	  return;
-	}
-      break;
-
-    default:
-      break;
-    }
-  PUT_SDB_TYPE (plain_type (type));
-  PUT_SDB_ENDEF;
-}
-
-/* Output SDB information for a top-level initialized variable
-   that has been delayed.  */
-
-static void
-sdbout_toplevel_data (tree decl)
-{
-  tree type = TREE_TYPE (decl);
-
-  if (DECL_IGNORED_P (decl))
-    return;
-
-  gcc_assert (VAR_P (decl));
-  gcc_assert (MEM_P (DECL_RTL (decl)));
-  gcc_assert (DECL_INITIAL (decl));
-
-  PUT_SDB_DEF (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
-  PUT_SDB_VAL (XEXP (DECL_RTL (decl), 0));
-  if (TREE_PUBLIC (decl))
-    {
-      PUT_SDB_SCL (C_EXT);
-    }
-  else
-    {
-      PUT_SDB_SCL (C_STAT);
-    }
-  PUT_SDB_TYPE (plain_type (type));
-  PUT_SDB_ENDEF;
-}
-
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-
-/* Machinery to record and output anonymous types.  */
-
-static void
-sdbout_queue_anonymous_type (tree type)
-{
-  anonymous_types = tree_cons (NULL_TREE, type, anonymous_types);
-}
-
-static void
-sdbout_dequeue_anonymous_types (void)
-{
-  tree types, link;
-
-  while (anonymous_types)
-    {
-      types = nreverse (anonymous_types);
-      anonymous_types = NULL_TREE;
-
-      for (link = types; link; link = TREE_CHAIN (link))
-	{
-	  tree type = TREE_VALUE (link);
-
-	  if (type && ! TREE_ASM_WRITTEN (type))
-	    sdbout_one_type (type);
-	}
-    }
-}
-
-#endif
-
-/* Given a chain of ..._TYPE nodes, all of which have names,
-   output definitions of those names, as typedefs.  */
-
-void
-sdbout_types (tree types)
-{
-  tree link;
-
-  for (link = types; link; link = TREE_CHAIN (link))
-    sdbout_one_type (link);
-
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-  sdbout_dequeue_anonymous_types ();
-#endif
-}
-
-static void
-sdbout_type (tree type)
-{
-  if (type == error_mark_node)
-    type = integer_type_node;
-  PUT_SDB_TYPE (plain_type (type));
-}
-
-/* Output types of the fields of type TYPE, if they are structs.
-
-   Formerly did not chase through pointer types, since that could be circular.
-   They must come before TYPE, since forward refs are not allowed.
-   Now james@bigtex.cactus.org says to try them.  */
-
-static void
-sdbout_field_types (tree type)
-{
-  tree tail;
-
-  for (tail = TYPE_FIELDS (type); tail; tail = TREE_CHAIN (tail))
-    /* This condition should match the one for emitting the actual
-       members below.  */
-    if (TREE_CODE (tail) == FIELD_DECL
-	&& DECL_NAME (tail)
-	&& DECL_SIZE (tail)
-	&& tree_fits_uhwi_p (DECL_SIZE (tail))
-	&& tree_fits_shwi_p (bit_position (tail)))
-      {
-	if (POINTER_TYPE_P (TREE_TYPE (tail)))
-	  sdbout_one_type (TREE_TYPE (TREE_TYPE (tail)));
-	else
-	  sdbout_one_type (TREE_TYPE (tail));
-      }
-}
-
-/* Use this to put out the top level defined record and union types
-   for later reference.  If this is a struct with a name, then put that
-   name out.  Other unnamed structs will have .xxfake labels generated so
-   that they may be referred to later.
-   The label will be stored in the KNOWN_TYPE_TAG slot of a type.
-   It may NOT be called recursively.  */
-
-static void
-sdbout_one_type (tree type)
-{
-  if (current_function_decl != NULL_TREE
-      && DECL_SECTION_NAME (current_function_decl) != NULL)
-    ; /* Don't change section amid function.  */
-  else
-    switch_to_section (current_function_section ());
-
-  switch (TREE_CODE (type))
-    {
-    case RECORD_TYPE:
-    case UNION_TYPE:
-    case QUAL_UNION_TYPE:
-    case ENUMERAL_TYPE:
-      type = TYPE_MAIN_VARIANT (type);
-      /* Don't output a type twice.  */
-      if (TREE_ASM_WRITTEN (type))
-	/* James said test TREE_ASM_BEING_WRITTEN here.  */
-	return;
-
-      /* Output nothing if type is not yet defined.  */
-      if (!COMPLETE_TYPE_P (type))
-	return;
-
-      TREE_ASM_WRITTEN (type) = 1;
-
-      /* This is reputed to cause trouble with the following case,
-	 but perhaps checking TYPE_SIZE above will fix it.  */
-
-      /* Here is a testcase:
-
-	struct foo {
-	  struct badstr *bbb;
-	} forwardref;
-
-	typedef struct intermediate {
-	  int aaaa;
-	} intermediate_ref;
-
-	typedef struct badstr {
-	  int ccccc;
-	} badtype;   */
-
-      /* This change, which ought to make better output,
-	 used to make the COFF assembler unhappy.
-	 Changes involving KNOWN_TYPE_TAG may fix the problem.  */
-      /* Before really doing anything, output types we want to refer to.  */
-      /* Note that in version 1 the following two lines
-	 are not used if forward references are in use.  */
-      if (TREE_CODE (type) != ENUMERAL_TYPE)
-	sdbout_field_types (type);
-
-      /* Output a structure type.  */
-      {
-	int size = int_size_in_bytes (type);
-	int member_scl = 0;
-	tree tem;
-
-	/* Record the type tag, but not in its permanent place just yet.  */
-	sdbout_record_type_name (type);
-
-	PUT_SDB_DEF (KNOWN_TYPE_TAG (type));
-
-	switch (TREE_CODE (type))
-	  {
-	  case UNION_TYPE:
-	  case QUAL_UNION_TYPE:
-	    PUT_SDB_SCL (C_UNTAG);
-	    PUT_SDB_TYPE (T_UNION);
-	    member_scl = C_MOU;
-	    break;
-
-	  case RECORD_TYPE:
-	    PUT_SDB_SCL (C_STRTAG);
-	    PUT_SDB_TYPE (T_STRUCT);
-	    member_scl = C_MOS;
-	    break;
-
-	  case ENUMERAL_TYPE:
-	    PUT_SDB_SCL (C_ENTAG);
-	    PUT_SDB_TYPE (T_ENUM);
-	    member_scl = C_MOE;
-	    break;
-
-	  default:
-	    break;
-	  }
-
-	PUT_SDB_SIZE (size);
-	PUT_SDB_ENDEF;
-
-	/* Print out the base class information with fields
-	   named after the types they hold.  */
-	/* This is only relevant to aggregate types.  TYPE_BINFO is used
-	   for other purposes in an ENUMERAL_TYPE, so we must exclude that
-	   case.  */
-	if (TREE_CODE (type) != ENUMERAL_TYPE && TYPE_BINFO (type))
-	  {
-	    int i;
-	    tree binfo, child;
-
-	    for (binfo = TYPE_BINFO (type), i = 0;
-		 BINFO_BASE_ITERATE (binfo, i, child); i++)
-	      {
-		tree child_type = BINFO_TYPE (child);
-		tree child_type_name;
-
-		if (TYPE_NAME (child_type) == 0)
-		  continue;
-		if (TREE_CODE (TYPE_NAME (child_type)) == IDENTIFIER_NODE)
-		  child_type_name = TYPE_NAME (child_type);
-		else if (TREE_CODE (TYPE_NAME (child_type)) == TYPE_DECL)
-		  {
-		    child_type_name = DECL_NAME (TYPE_NAME (child_type));
-		    if (child_type_name && template_name_p (child_type_name))
-		      child_type_name
-			= DECL_ASSEMBLER_NAME (TYPE_NAME (child_type));
-		  }
-		else
-		  continue;
-
-		PUT_SDB_DEF (IDENTIFIER_POINTER (child_type_name));
-		PUT_SDB_INT_VAL (tree_to_shwi (BINFO_OFFSET (child)));
-		PUT_SDB_SCL (member_scl);
-		sdbout_type (BINFO_TYPE (child));
-		PUT_SDB_ENDEF;
-	      }
-	  }
-
-	/* Output the individual fields.  */
-
-	if (TREE_CODE (type) == ENUMERAL_TYPE)
-	  {
-	    for (tem = TYPE_VALUES (type); tem; tem = TREE_CHAIN (tem))
-	      {
-	        tree value = TREE_VALUE (tem);
-
-	        if (TREE_CODE (value) == CONST_DECL)
-	          value = DECL_INITIAL (value);
-
-	        if (tree_fits_shwi_p (value))
-		  {
-		    PUT_SDB_DEF (IDENTIFIER_POINTER (TREE_PURPOSE (tem)));
-		    PUT_SDB_INT_VAL (tree_to_shwi (value));
-		    PUT_SDB_SCL (C_MOE);
-		    PUT_SDB_TYPE (T_MOE);
-		    PUT_SDB_ENDEF;
-		  }
-	      }
-	  }
-	else			/* record or union type */
-	  for (tem = TYPE_FIELDS (type); tem; tem = TREE_CHAIN (tem))
-	    /* Output the name, type, position (in bits), size (in bits)
-	       of each field.  */
-
-	    /* Omit here the nameless fields that are used to skip bits.
-	       Also omit fields with variable size or position.
-	       Also omit non FIELD_DECL nodes that GNU C++ may put here.  */
-	    if (TREE_CODE (tem) == FIELD_DECL
-		&& DECL_NAME (tem)
-		&& DECL_SIZE (tem)
-		&& tree_fits_uhwi_p (DECL_SIZE (tem))
-		&& tree_fits_shwi_p (bit_position (tem)))
-	      {
-		const char *name;
-
-		name = IDENTIFIER_POINTER (DECL_NAME (tem));
-		PUT_SDB_DEF (name);
-		if (DECL_BIT_FIELD_TYPE (tem))
-		  {
-		    PUT_SDB_INT_VAL (int_bit_position (tem));
-		    PUT_SDB_SCL (C_FIELD);
-		    sdbout_type (DECL_BIT_FIELD_TYPE (tem));
-		    PUT_SDB_SIZE (tree_to_uhwi (DECL_SIZE (tem)));
-		  }
-		else
-		  {
-		    PUT_SDB_INT_VAL (int_bit_position (tem) / BITS_PER_UNIT);
-		    PUT_SDB_SCL (member_scl);
-		    sdbout_type (TREE_TYPE (tem));
-		  }
-		PUT_SDB_ENDEF;
-	      }
-	/* Output end of a structure,union, or enumeral definition.  */
-
-	PUT_SDB_PLAIN_DEF ("eos");
-	PUT_SDB_INT_VAL (size);
-	PUT_SDB_SCL (C_EOS);
-	PUT_SDB_TAG (KNOWN_TYPE_TAG (type));
-	PUT_SDB_SIZE (size);
-	PUT_SDB_ENDEF;
-	break;
-      }
-
-    default:
-      break;
-    }
-}
-
-/* The following two functions output definitions of function parameters.
-   Each parameter gets a definition locating it in the parameter list.
-   Each parameter that is a register variable gets a second definition
-   locating it in the register.
-
-   Printing or argument lists in gdb uses the definitions that
-   locate in the parameter list.  But reference to the variable in
-   expressions uses preferentially the definition as a register.  */
-
-/* Output definitions, referring to storage in the parmlist,
-   of all the parms in PARMS, which is a chain of PARM_DECL nodes.  */
-
-static void
-sdbout_parms (tree parms)
-{
-  for (; parms; parms = TREE_CHAIN (parms))
-    if (DECL_NAME (parms)
-	&& TREE_TYPE (parms) != error_mark_node
-	&& DECL_RTL_SET_P (parms)
-	&& DECL_INCOMING_RTL (parms))
-      {
-	int current_sym_value = 0;
-	const char *name = IDENTIFIER_POINTER (DECL_NAME (parms));
-
-	if (name == 0 || *name == 0)
-	  name = gen_fake_label ();
-
-	/* Perform any necessary register eliminations on the parameter's rtl,
-	   so that the debugging output will be accurate.  */
-	DECL_INCOMING_RTL (parms)
-	  = eliminate_regs (DECL_INCOMING_RTL (parms), VOIDmode, NULL_RTX);
-	SET_DECL_RTL (parms,
-		      eliminate_regs (DECL_RTL (parms), VOIDmode, NULL_RTX));
-
-	if (PARM_PASSED_IN_MEMORY (parms))
-	  {
-	    rtx addr = XEXP (DECL_INCOMING_RTL (parms), 0);
-	    tree type;
-
-	    /* ??? Here we assume that the parm address is indexed
-	       off the frame pointer or arg pointer.
-	       If that is not true, we produce meaningless results,
-	       but do not crash.  */
-	    if (GET_CODE (addr) == PLUS
-		&& CONST_INT_P (XEXP (addr, 1)))
-	      current_sym_value = INTVAL (XEXP (addr, 1));
-	    else
-	      current_sym_value = 0;
-
-	    if (REG_P (DECL_RTL (parms))
-		&& REGNO (DECL_RTL (parms)) < FIRST_PSEUDO_REGISTER)
-	      type = DECL_ARG_TYPE (parms);
-	    else
-	      {
-		int original_sym_value = current_sym_value;
-
-		/* This is the case where the parm is passed as an int or
-		   double and it is converted to a char, short or float
-		   and stored back in the parmlist.  In this case, describe
-		   the parm with the variable's declared type, and adjust
-		   the address if the least significant bytes (which we are
-		   using) are not the first ones.  */
-		scalar_mode from_mode, to_mode;
-		if (BYTES_BIG_ENDIAN
-		    && TREE_TYPE (parms) != DECL_ARG_TYPE (parms)
-		    && is_a <scalar_mode> (TYPE_MODE (DECL_ARG_TYPE (parms)),
-					   &from_mode)
-		    && is_a <scalar_mode> (GET_MODE (DECL_RTL (parms)),
-					   &to_mode))
-		  current_sym_value += (GET_MODE_SIZE (from_mode)
-					- GET_MODE_SIZE (to_mode));
-
-		if (MEM_P (DECL_RTL (parms))
-		    && GET_CODE (XEXP (DECL_RTL (parms), 0)) == PLUS
-		    && (GET_CODE (XEXP (XEXP (DECL_RTL (parms), 0), 1))
-			== CONST_INT)
-		    && (INTVAL (XEXP (XEXP (DECL_RTL (parms), 0), 1))
-			== current_sym_value))
-		  type = TREE_TYPE (parms);
-		else
-		  {
-		    current_sym_value = original_sym_value;
-		    type = DECL_ARG_TYPE (parms);
-		  }
-	      }
-
-	    PUT_SDB_DEF (name);
-	    PUT_SDB_INT_VAL (DEBUGGER_ARG_OFFSET (current_sym_value, addr));
-	    PUT_SDB_SCL (C_ARG);
-	    PUT_SDB_TYPE (plain_type (type));
-	    PUT_SDB_ENDEF;
-	  }
-	else if (REG_P (DECL_RTL (parms)))
-	  {
-	    rtx best_rtl;
-	    /* Parm passed in registers and lives in registers or nowhere.  */
-
-	    /* If parm lives in a register, use that register;
-	       pretend the parm was passed there.  It would be more consistent
-	       to describe the register where the parm was passed,
-	       but in practice that register usually holds something else.  */
-	    if (REGNO (DECL_RTL (parms)) < FIRST_PSEUDO_REGISTER)
-	      best_rtl = DECL_RTL (parms);
-	    /* If the parm lives nowhere,
-	       use the register where it was passed.  */
-	    else
-	      best_rtl = DECL_INCOMING_RTL (parms);
-
-	    PUT_SDB_DEF (name);
-	    PUT_SDB_INT_VAL (DBX_REGISTER_NUMBER (REGNO (best_rtl)));
-	    PUT_SDB_SCL (C_REGPARM);
-	    PUT_SDB_TYPE (plain_type (TREE_TYPE (parms)));
-	    PUT_SDB_ENDEF;
-	  }
-	else if (MEM_P (DECL_RTL (parms))
-		 && XEXP (DECL_RTL (parms), 0) != const0_rtx)
-	  {
-	    /* Parm was passed in registers but lives on the stack.  */
-
-	    /* DECL_RTL looks like (MEM (PLUS (REG...) (CONST_INT...))),
-	       in which case we want the value of that CONST_INT,
-	       or (MEM (REG ...)) or (MEM (MEM ...)),
-	       in which case we use a value of zero.  */
-	    if (REG_P (XEXP (DECL_RTL (parms), 0))
-		|| MEM_P (XEXP (DECL_RTL (parms), 0)))
-	      current_sym_value = 0;
-	    else
-	      current_sym_value = INTVAL (XEXP (XEXP (DECL_RTL (parms), 0), 1));
-
-	    /* Again, this assumes the offset is based on the arg pointer.  */
-	    PUT_SDB_DEF (name);
-	    PUT_SDB_INT_VAL (DEBUGGER_ARG_OFFSET (current_sym_value,
-						  XEXP (DECL_RTL (parms), 0)));
-	    PUT_SDB_SCL (C_ARG);
-	    PUT_SDB_TYPE (plain_type (TREE_TYPE (parms)));
-	    PUT_SDB_ENDEF;
-	  }
-      }
-}
-
-/* Output definitions for the places where parms live during the function,
-   when different from where they were passed, when the parms were passed
-   in memory.
-
-   It is not useful to do this for parms passed in registers
-   that live during the function in different registers, because it is
-   impossible to look in the passed register for the passed value,
-   so we use the within-the-function register to begin with.
-
-   PARMS is a chain of PARM_DECL nodes.  */
-
-static void
-sdbout_reg_parms (tree parms)
-{
-  for (; parms; parms = TREE_CHAIN (parms))
-    if (DECL_NAME (parms)
-        && TREE_TYPE (parms) != error_mark_node
-        && DECL_RTL_SET_P (parms)
-        && DECL_INCOMING_RTL (parms))
-      {
-	const char *name = IDENTIFIER_POINTER (DECL_NAME (parms));
-
-	/* Report parms that live in registers during the function
-	   but were passed in memory.  */
-	if (REG_P (DECL_RTL (parms))
-	    && REGNO (DECL_RTL (parms)) < FIRST_PSEUDO_REGISTER
-	    && PARM_PASSED_IN_MEMORY (parms))
-	  {
-	    if (name == 0 || *name == 0)
-	      name = gen_fake_label ();
-	    PUT_SDB_DEF (name);
-	    PUT_SDB_INT_VAL (DBX_REGISTER_NUMBER (REGNO (DECL_RTL (parms))));
-	    PUT_SDB_SCL (C_REG);
-	    PUT_SDB_TYPE (plain_type (TREE_TYPE (parms)));
-	    PUT_SDB_ENDEF;
-	  }
-	/* Report parms that live in memory but not where they were passed.  */
-	else if (MEM_P (DECL_RTL (parms))
-		 && GET_CODE (XEXP (DECL_RTL (parms), 0)) == PLUS
-		 && CONST_INT_P (XEXP (XEXP (DECL_RTL (parms), 0), 1))
-		 && PARM_PASSED_IN_MEMORY (parms)
-		 && ! rtx_equal_p (DECL_RTL (parms), DECL_INCOMING_RTL (parms)))
-	  {
-#if 0 /* ??? It is not clear yet what should replace this.  */
-	    int offset = DECL_OFFSET (parms) / BITS_PER_UNIT;
-	    /* A parm declared char is really passed as an int,
-	       so it occupies the least significant bytes.
-	       On a big-endian machine those are not the low-numbered ones.  */
-	    if (BYTES_BIG_ENDIAN
-		&& offset != -1
-		&& TREE_TYPE (parms) != DECL_ARG_TYPE (parms))
-	      offset += (GET_MODE_SIZE (TYPE_MODE (DECL_ARG_TYPE (parms)))
-			 - GET_MODE_SIZE (GET_MODE (DECL_RTL (parms))));
-	    if (INTVAL (XEXP (XEXP (DECL_RTL (parms), 0), 1)) != offset) {...}
-#endif
-	      {
-		if (name == 0 || *name == 0)
-		  name = gen_fake_label ();
-		PUT_SDB_DEF (name);
-		PUT_SDB_INT_VAL (DEBUGGER_AUTO_OFFSET
-				 (XEXP (DECL_RTL (parms), 0)));
-		PUT_SDB_SCL (C_AUTO);
-		PUT_SDB_TYPE (plain_type (TREE_TYPE (parms)));
-		PUT_SDB_ENDEF;
-	      }
-	  }
-      }
-}
-
-/* Output early debug information for a global DECL.  Called from
-   rest_of_decl_compilation during parsing.  */
-
-static void
-sdbout_early_global_decl (tree decl ATTRIBUTE_UNUSED)
-{
-  /* NYI for non-dwarf.  */
-}
-
-/* Output late debug information for a global DECL after location
-   information is available.  */
-
-static void
-sdbout_late_global_decl (tree decl)
-{
-  if (VAR_P (decl) && !DECL_EXTERNAL (decl) && DECL_RTL_SET_P (decl))
-    {
-      /* The COFF linker can move initialized global vars to the end.
-	 And that can screw up the symbol ordering.  Defer those for
-	 sdbout_finish ().  */
-      if (!DECL_INITIAL (decl) || !TREE_PUBLIC (decl))
-	sdbout_symbol (decl, 0);
-      else
-	vec_safe_push (deferred_global_decls, decl);
-
-      /* Output COFF information for non-global file-scope initialized
-	 variables.  */
-      if (DECL_INITIAL (decl) && MEM_P (DECL_RTL (decl)))
-	sdbout_toplevel_data (decl);
-    }
-}
-
-/* Output initialized global vars at the end, in the order of
-   definition.  See comment in sdbout_global_decl.  */
-
-static void
-sdbout_finish (const char *main_filename ATTRIBUTE_UNUSED)
-{
-  size_t i;
-  tree decl;
-
-  FOR_EACH_VEC_SAFE_ELT (deferred_global_decls, i, decl)
-    sdbout_symbol (decl, 0);
-}
-
-/* Describe the beginning of an internal block within a function.
-   Also output descriptions of variables defined in this block.
-
-   N is the number of the block, by order of beginning, counting from 1,
-   and not counting the outermost (function top-level) block.
-   The blocks match the BLOCKs in DECL_INITIAL (current_function_decl),
-   if the count starts at 0 for the outermost one.  */
-
-static void
-sdbout_begin_block (unsigned int line, unsigned int n)
-{
-  tree decl = current_function_decl;
-  MAKE_LINE_SAFE (line);
-
-  /* The SCO compiler does not emit a separate block for the function level
-     scope, so we avoid it here also.  */
-  PUT_SDB_BLOCK_START (line - sdb_begin_function_line);
-
-  if (n == 1)
-    {
-      /* Include the outermost BLOCK's variables in block 1.  */
-      do_block = BLOCK_NUMBER (DECL_INITIAL (decl));
-      sdbout_block (DECL_INITIAL (decl));
-    }
-  /* If -g1, suppress all the internal symbols of functions
-     except for arguments.  */
-  if (debug_info_level != DINFO_LEVEL_TERSE)
-    {
-      do_block = n;
-      sdbout_block (DECL_INITIAL (decl));
-    }
-
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-  sdbout_dequeue_anonymous_types ();
-#endif
-}
-
-/* Describe the end line-number of an internal block within a function.  */
-
-static void
-sdbout_end_block (unsigned int line, unsigned int n ATTRIBUTE_UNUSED)
-{
-  MAKE_LINE_SAFE (line);
-
-  /* The SCO compiler does not emit a separate block for the function level
-     scope, so we avoid it here also.  */
-  if (n != 1)
-    PUT_SDB_BLOCK_END (line - sdb_begin_function_line);
-}
-
-/* Output a line number symbol entry for source file FILENAME and line
-   number LINE.  */
-
-static void
-sdbout_source_line (unsigned int line, unsigned int column ATTRIBUTE_UNUSED,
-		    const char *filename ATTRIBUTE_UNUSED,
-                    int discriminator ATTRIBUTE_UNUSED,
-                    bool is_stmt ATTRIBUTE_UNUSED)
-{
-  /* COFF relative line numbers must be positive.  */
-  if ((int) line > sdb_begin_function_line)
-    {
-#ifdef SDB_OUTPUT_SOURCE_LINE
-      SDB_OUTPUT_SOURCE_LINE (asm_out_file, line);
-#else
-      fprintf (asm_out_file, "\t.ln\t%d\n",
-	       ((sdb_begin_function_line > -1)
-		? line - sdb_begin_function_line : 1));
-#endif
-    }
-}
-
-/* Output sdb info for the current function name.
-   Called from assemble_start_function.  */
-
-static void
-sdbout_begin_function (tree decl ATTRIBUTE_UNUSED)
-{
-  sdbout_symbol (current_function_decl, 0);
-}
-
-/* Called at beginning of function body after prologue.  Record the
-   function's starting line number, so we can output relative line numbers
-   for the other lines.  Describe beginning of outermost block.  Also
-   describe the parameter list.  */
-
-static void
-sdbout_begin_prologue (unsigned int line, unsigned int column ATTRIBUTE_UNUSED,
-		       const char *file ATTRIBUTE_UNUSED)
-{
-  sdbout_end_prologue (line, file);
-}
-
-static void
-sdbout_end_prologue (unsigned int line, const char *file ATTRIBUTE_UNUSED)
-{
-  sdb_begin_function_line = line - 1;
-  PUT_SDB_FUNCTION_START (line);
-  sdbout_parms (DECL_ARGUMENTS (current_function_decl));
-  sdbout_reg_parms (DECL_ARGUMENTS (current_function_decl));
-}
-
-/* Called at end of function (before epilogue).
-   Describe end of outermost block.  */
-
-static void
-sdbout_end_function (unsigned int line)
-{
-#ifdef SDB_ALLOW_FORWARD_REFERENCES
-  sdbout_dequeue_anonymous_types ();
-#endif
-
-  MAKE_LINE_SAFE (line);
-  PUT_SDB_FUNCTION_END (line - sdb_begin_function_line);
-
-  /* Indicate we are between functions, for line-number output.  */
-  sdb_begin_function_line = -1;
-}
-
-/* Output sdb info for the absolute end of a function.
-   Called after the epilogue is output.  */
-
-static void
-sdbout_end_epilogue (unsigned int line ATTRIBUTE_UNUSED,
-		     const char *file ATTRIBUTE_UNUSED)
-{
-  const char *const name ATTRIBUTE_UNUSED
-    = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (current_function_decl));
-
-#ifdef PUT_SDB_EPILOGUE_END
-  PUT_SDB_EPILOGUE_END (name);
-#else
-  fprintf (asm_out_file, "\t.def\t");
-  assemble_name (asm_out_file, name);
-  fprintf (asm_out_file, "%s\t.val\t.%s\t.scl\t-1%s\t.endef\n",
-	   SDB_DELIM, SDB_DELIM, SDB_DELIM);
-#endif
-}
-
-/* Output sdb info for the given label.  Called only if LABEL_NAME (insn)
-   is present.  */
-
-static void
-sdbout_label (rtx_code_label *insn)
-{
-  PUT_SDB_DEF (LABEL_NAME (insn));
-  PUT_SDB_VAL (insn);
-  PUT_SDB_SCL (C_LABEL);
-  PUT_SDB_TYPE (T_NULL);
-  PUT_SDB_ENDEF;
-}
-
-/* Change to reading from a new source file.  */
-
-static void
-sdbout_start_source_file (unsigned int line ATTRIBUTE_UNUSED,
-			  const char *filename ATTRIBUTE_UNUSED)
-{
-}
-
-/* Revert to reading a previous source file.  */
-
-static void
-sdbout_end_source_file (unsigned int line ATTRIBUTE_UNUSED)
-{
-}
-
-/* Set up for SDB output at the start of compilation.  */
-
-static void
-sdbout_init (const char *input_file_name ATTRIBUTE_UNUSED)
-{
-  tree t;
-
-  vec_alloc (deferred_global_decls, 12);
-
-  /* Emit debug information which was queued by sdbout_symbol before
-     we got here.  */
-  sdbout_initialized = true;
-
-  for (t = nreverse (preinit_symbols); t; t = TREE_CHAIN (t))
-    sdbout_symbol (TREE_VALUE (t), 0);
-  preinit_symbols = 0;
-}
-
-#include "gt-sdbout.h"
diff --git a/gcc/sdbout.h b/gcc/sdbout.h
deleted file mode 100644
index 204b68790ce..00000000000
--- a/gcc/sdbout.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/* sdbout.h - Various declarations for functions found in sdbout.c
-   Copyright (C) 1998-2017 Free Software Foundation, Inc.
-
-This file is part of GCC.
-
-GCC is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 3, or (at your option) any later
-version.
-
-GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with GCC; see the file COPYING3.  If not see
-<http://www.gnu.org/licenses/>.  */
-
-#ifndef GCC_SDBOUT_H
-#define GCC_SDBOUT_H
-
-extern void sdbout_symbol (tree, int);
-extern void sdbout_types (tree);
-
-#endif /* GCC_SDBOUT_H */
diff --git a/gcc/selftest-diagnostic.c b/gcc/selftest-diagnostic.c
new file mode 100644
index 00000000000..201806288e4
--- /dev/null
+++ b/gcc/selftest-diagnostic.c
@@ -0,0 +1,62 @@
+/* Selftest support for diagnostics.
+   Copyright (C) 2016-2017 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "diagnostic.h"
+#include "selftest.h"
+#include "selftest-diagnostic.h"
+
+/* The selftest code should entirely disappear in a production
+   configuration, hence we guard all of it with #if CHECKING_P.  */
+
+#if CHECKING_P
+
+namespace selftest {
+
+/* Implementation of class selftest::test_diagnostic_context.  */
+
+test_diagnostic_context::test_diagnostic_context ()
+{
+  diagnostic_initialize (this, 0);
+  show_caret = true;
+  show_column = true;
+  start_span = start_span_cb;
+}
+
+test_diagnostic_context::~test_diagnostic_context ()
+{
+  diagnostic_finish (this);
+}
+
+/* Implementation of diagnostic_start_span_fn, hiding the
+   real filename (to avoid printing the names of tempfiles).  */
+
+void
+test_diagnostic_context::start_span_cb (diagnostic_context *context,
+					expanded_location exploc)
+{
+  exploc.file = "FILENAME";
+  default_diagnostic_start_span_fn (context, exploc);
+}
+
+} // namespace selftest
+
+#endif /* #if CHECKING_P */
diff --git a/gcc/selftest-diagnostic.h b/gcc/selftest-diagnostic.h
new file mode 100644
index 00000000000..61525dcfd43
--- /dev/null
+++ b/gcc/selftest-diagnostic.h
@@ -0,0 +1,49 @@
+/* Selftest support for diagnostics.
+   Copyright (C) 2016-2017 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_SELFTEST_DIAGNOSTIC_H
+#define GCC_SELFTEST_DIAGNOSTIC_H
+
+/* The selftest code should entirely disappear in a production
+   configuration, hence we guard all of it with #if CHECKING_P.  */
+
+#if CHECKING_P
+
+namespace selftest {
+
+/* Convenience subclass of diagnostic_context for testing
+   the diagnostic subsystem.  */
+
+class test_diagnostic_context : public diagnostic_context
+{
+ public:
+  test_diagnostic_context ();
+  ~test_diagnostic_context ();
+
+  /* Implementation of diagnostic_start_span_fn, hiding the
+     real filename (to avoid printing the names of tempfiles).  */
+  static void
+  start_span_cb (diagnostic_context *context, expanded_location exploc);
+};
+
+} // namespace selftest
+
+#endif /* #if CHECKING_P */
+
+#endif /* GCC_SELFTEST_DIAGNOSTIC_H */
diff --git a/gcc/selftest-run-tests.c b/gcc/selftest-run-tests.c
index 80ae8f9799b..6030d3b22e7 100644
--- a/gcc/selftest-run-tests.c
+++ b/gcc/selftest-run-tests.c
@@ -67,6 +67,7 @@ selftest::run_tests ()
   sreal_c_tests ();
   fibonacci_heap_c_tests ();
   typed_splay_tree_c_tests ();
+  unique_ptr_tests_cc_tests ();
 
   /* Mid-level data structures.  */
   input_c_tests ();
diff --git a/gcc/selftest.h b/gcc/selftest.h
index c5135b0cb60..253cfc2d732 100644
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -195,6 +195,7 @@ extern void store_merging_c_tests ();
 extern void typed_splay_tree_c_tests ();
 extern void tree_c_tests ();
 extern void tree_cfg_c_tests ();
+extern void unique_ptr_tests_cc_tests ();
 extern void vec_c_tests ();
 extern void wide_int_cc_tests ();
 extern void predict_c_tests ();
diff --git a/gcc/sese.c b/gcc/sese.c
index 8aa8015290d..89cddf0ec97 100644
--- a/gcc/sese.c
+++ b/gcc/sese.c
@@ -156,12 +156,8 @@ new_sese_info (edge entry, edge exit)
   region->liveout = NULL;
   region->debug_liveout = NULL;
   region->params.create (3);
-  region->rename_map = new rename_map_t;
-  region->parameter_rename_map = new parameter_rename_map_t;
-  region->copied_bb_map = new bb_map_t;
+  region->rename_map = new hash_map <tree, tree>;
   region->bbs.create (3);
-  region->incomplete_phis.create (3);
-
 
   return region;
 }
@@ -175,24 +171,9 @@ free_sese_info (sese_info_p region)
   BITMAP_FREE (region->liveout);
   BITMAP_FREE (region->debug_liveout);
 
-  for (rename_map_t::iterator it = region->rename_map->begin ();
-       it != region->rename_map->end (); ++it)
-    (*it).second.release ();
-
-  for (bb_map_t::iterator it = region->copied_bb_map->begin ();
-       it != region->copied_bb_map->end (); ++it)
-    (*it).second.release ();
-
   delete region->rename_map;
-  delete region->parameter_rename_map;
-  delete region->copied_bb_map;
-
   region->rename_map = NULL;
-  region->parameter_rename_map = NULL;
-  region->copied_bb_map = NULL;
-
   region->bbs.release ();
-  region->incomplete_phis.release ();
 
   XDELETE (region);
 }
@@ -459,41 +440,16 @@ scev_analyzable_p (tree def, sese_l &region)
 tree
 scalar_evolution_in_region (const sese_l &region, loop_p loop, tree t)
 {
-  gimple *def;
-  struct loop *def_loop;
-
   /* SCOP parameters.  */
   if (TREE_CODE (t) == SSA_NAME
       && !defined_in_sese_p (t, region))
     return t;
 
-  if (TREE_CODE (t) != SSA_NAME
-      || loop_in_sese_p (loop, region))
-    /* FIXME: we would need instantiate SCEV to work on a region, and be more
-       flexible wrt. memory loads that may be invariant in the region.  */
-    return instantiate_scev (region.entry, loop,
-			     analyze_scalar_evolution (loop, t));
-
-  def = SSA_NAME_DEF_STMT (t);
-  def_loop = loop_containing_stmt (def);
-
-  if (loop_in_sese_p (def_loop, region))
-    {
-      t = analyze_scalar_evolution (def_loop, t);
-      def_loop = superloop_at_depth (def_loop, loop_depth (loop) + 1);
-      t = compute_overall_effect_of_inner_loop (def_loop, t);
-      return t;
-    }
-
-  bool has_vdefs = false;
-  if (invariant_in_sese_p_rec (t, region, &has_vdefs))
-    return t;
-
-  /* T variates in REGION.  */
-  if (has_vdefs)
-    return chrec_dont_know;
+  if (!loop_in_sese_p (loop, region))
+    loop = NULL;
 
-  return instantiate_scev (region.entry, loop, t);
+  return instantiate_scev (region.entry, loop,
+			   analyze_scalar_evolution (loop, t));
 }
 
 /* Return true if BB is empty, contains only DEBUG_INSNs.  */
diff --git a/gcc/sese.h b/gcc/sese.h
index faefd806d9d..cbc20ab1064 100644
--- a/gcc/sese.h
+++ b/gcc/sese.h
@@ -22,14 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_SESE_H
 #define GCC_SESE_H
 
-typedef hash_map<tree, tree> parameter_rename_map_t;
-typedef hash_map<basic_block, vec<basic_block> > bb_map_t;
-typedef hash_map<tree, vec<tree> > rename_map_t;
 typedef struct ifsese_s *ifsese;
-/* First phi is the new codegenerated phi second one is original phi.  */
-typedef std::pair <gphi *, gphi *> phi_rename;
-/* First edge is the init edge and second is the back edge w.r.t. a loop.  */
-typedef std::pair<edge, edge> init_back_edge_pair_t;
 
 /* A Single Entry, Single Exit region is a part of the CFG delimited
    by two edges.  */
@@ -92,24 +85,12 @@ typedef struct sese_info_t
   /* Parameters used within the SCOP.  */
   vec<tree> params;
 
-  /* Maps an old name to one or more new names.  When there are several new
-     names, one has to select the definition corresponding to the immediate
-     dominator.  */
-  rename_map_t *rename_map;
-
-  /* Parameters to be renamed.  */
-  parameter_rename_map_t *parameter_rename_map;
+  /* Maps an old name to a new decl.  */
+  hash_map<tree, tree> *rename_map;
 
   /* Basic blocks contained in this SESE.  */
   vec<basic_block> bbs;
 
-  /* Copied basic blocks indexed by the original bb.  */
-  bb_map_t *copied_bb_map;
-
-  /* A vector of phi nodes to be updated when all arguments are available.  The
-     pair contains first the old_phi and second the new_phi.  */
-  vec<phi_rename> incomplete_phis;
-
   /* The condition region generated for this sese.  */
   ifsese if_region;
 
diff --git a/gcc/shrink-wrap.c b/gcc/shrink-wrap.c
index 3cad7760f9c..0e4ff6cd46a 100644
--- a/gcc/shrink-wrap.c
+++ b/gcc/shrink-wrap.c
@@ -561,8 +561,7 @@ handle_simple_exit (edge e)
       BB_END (old_bb) = end;
 
       redirect_edge_succ (e, new_bb);
-      new_bb->count = e->count;
-      new_bb->frequency = EDGE_FREQUENCY (e);
+      new_bb->count = e->count ();
       e->flags |= EDGE_FALLTHRU;
 
       e = make_single_succ_edge (new_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
@@ -888,7 +887,7 @@ try_shrink_wrapping (edge *entry_edge, rtx_insn *prologue_seq)
     if (!dominated_by_p (CDI_DOMINATORS, e->src, pro))
       {
 	num += EDGE_FREQUENCY (e);
-	den += e->src->frequency;
+	den += e->src->count.to_frequency (cfun);
       }
 
   if (den == 0)
@@ -921,8 +920,6 @@ try_shrink_wrapping (edge *entry_edge, rtx_insn *prologue_seq)
 	if (dump_file)
 	  fprintf (dump_file, "Duplicated %d to %d\n", bb->index, dup->index);
 
-	bb->frequency = RDIV (num * bb->frequency, den);
-	dup->frequency -= bb->frequency;
 	bb->count = bb->count.apply_scale (num, den);
 	dup->count -= bb->count;
       }
@@ -996,8 +993,7 @@ try_shrink_wrapping (edge *entry_edge, rtx_insn *prologue_seq)
 	  continue;
 	}
 
-      new_bb->count += e->src->count.apply_probability (e->probability);
-      new_bb->frequency += EDGE_FREQUENCY (e);
+      new_bb->count += e->count ();
 
       redirect_edge_and_branch_force (e, new_bb);
       if (dump_file)
@@ -1182,7 +1178,7 @@ place_prologue_for_one_component (unsigned int which, basic_block head)
 	     work: this does not always add up to the block frequency at
 	     all, and even if it does, rounding error makes for bad
 	     decisions.  */
-	  SW (bb)->own_cost = bb->frequency;
+	  SW (bb)->own_cost = bb->count.to_frequency (cfun);
 
 	  edge e;
 	  edge_iterator ei;
diff --git a/gcc/simplify-rtx.c b/gcc/simplify-rtx.c
index b1b4767d8c4..212b5068cd0 100644
--- a/gcc/simplify-rtx.c
+++ b/gcc/simplify-rtx.c
@@ -357,7 +357,7 @@ delegitimize_mem_from_attrs (rtx x)
 		x = adjust_address_nv (newx, mode, offset);
 	    }
 	  else if (GET_MODE (x) == GET_MODE (newx)
-		   && known_zero (offset))
+		   && must_eq (offset, 0))
 	    x = newx;
 	}
     }
@@ -6210,7 +6210,7 @@ simplify_subreg (machine_mode outermode, rtx op,
   if (may_ge (byte, innersize))
     return NULL_RTX;
 
-  if (outermode == innermode && known_zero (byte))
+  if (outermode == innermode && must_eq (byte, 0U))
     return op;
 
   if (multiple_p (byte, GET_MODE_UNIT_SIZE (innermode)))
@@ -6256,8 +6256,8 @@ simplify_subreg (machine_mode outermode, rtx op,
       rtx newx;
 
       if (outermode == innermostmode
-	  && known_zero (byte)
-	  && known_zero (SUBREG_BYTE (op)))
+	  && must_eq (byte, 0U)
+	  && must_eq (SUBREG_BYTE (op), 0))
 	return SUBREG_REG (op);
 
       /* Work out the memory offset of the final OUTERMODE value relative
@@ -6639,7 +6639,8 @@ test_vector_ops_duplicate (machine_mode mode, rtx scalar_reg)
 				      mode, offset));
 
   machine_mode narrower_mode;
-  if (may_gt (nunits, 2U)
+  if (may_ne (nunits, 2U)
+      && multiple_p (nunits, 2)
       && mode_for_vector (inner_mode, 2).exists (&narrower_mode)
       && VECTOR_MODE_P (narrower_mode))
     {
diff --git a/gcc/ssa-iterators.h b/gcc/ssa-iterators.h
index c8aa77bd4f3..740cbf13cb2 100644
--- a/gcc/ssa-iterators.h
+++ b/gcc/ssa-iterators.h
@@ -93,6 +93,12 @@ struct imm_use_iterator
      break;							\
    }
 
+/* Similarly for return.  */
+#define RETURN_FROM_IMM_USE_STMT(ITER, VAL)			\
+  {								\
+    end_imm_use_stmt_traverse (&(ITER));			\
+    return (VAL);						\
+  }
 
 /* Use this iterator in combination with FOR_EACH_IMM_USE_STMT to
    get access to each occurrence of ssavar on the stmt returned by
diff --git a/gcc/stor-layout.c b/gcc/stor-layout.c
index 95f0afa994d..bd7d44471d8 100644
--- a/gcc/stor-layout.c
+++ b/gcc/stor-layout.c
@@ -2761,7 +2761,7 @@ bit_field_mode_iterator
   m_bitregion_end (bitregion_end), m_align (align),
   m_volatilep (volatilep), m_count (0)
 {
-  if (known_zero (m_bitregion_end))
+  if (must_eq (m_bitregion_end, 0))
     {
       /* We can assume that any aligned chunk of ALIGN bits that overlaps
 	 the bitfield is mapped and won't trap, provided that ALIGN isn't
@@ -2809,7 +2809,7 @@ bit_field_mode_iterator::next_mode (scalar_int_mode *out_mode)
 
       /* Stop if the mode goes outside the bitregion.  */
       HOST_WIDE_INT start = m_bitpos - substart;
-      if (maybe_nonzero (m_bitregion_start)
+      if (may_ne (m_bitregion_start, 0)
 	  && may_lt (start, m_bitregion_start))
 	break;
       HOST_WIDE_INT end = start + unit;
diff --git a/gcc/substring-locations.c b/gcc/substring-locations.c
index 433023d9845..7de435b969f 100644
--- a/gcc/substring-locations.c
+++ b/gcc/substring-locations.c
@@ -63,7 +63,7 @@ along with GCC; see the file COPYING3.  If not see
      printf(fmt, msg);
             ^~~
 
-   For each of cases 1-3, if param_range is non-NULL, then it is used
+   For each of cases 1-3, if param_loc is not UNKNOWN_LOCATION, then it is used
    as a secondary range within the warning.  For example, here it
    is used with case 1:
 
@@ -100,7 +100,7 @@ along with GCC; see the file COPYING3.  If not see
 ATTRIBUTE_GCC_DIAG (5,0)
 bool
 format_warning_va (const substring_loc &fmt_loc,
-		   const source_range *param_range,
+		   location_t param_loc,
 		   const char *corrected_substring,
 		   int opt, const char *gmsgid, va_list *ap)
 {
@@ -136,13 +136,8 @@ format_warning_va (const substring_loc &fmt_loc,
 
   rich_location richloc (line_table, primary_loc);
 
-  if (param_range)
-    {
-      location_t param_loc = make_location (param_range->m_start,
-					    param_range->m_start,
-					    param_range->m_finish);
-      richloc.add_range (param_loc, false);
-    }
+  if (param_loc != UNKNOWN_LOCATION)
+    richloc.add_range (param_loc, false);
 
   if (!err && corrected_substring && substring_within_range)
     richloc.add_fixit_replace (fmt_substring_range, corrected_substring);
@@ -160,8 +155,8 @@ format_warning_va (const substring_loc &fmt_loc,
 	if (corrected_substring)
 	  substring_richloc.add_fixit_replace (fmt_substring_range,
 					       corrected_substring);
-	inform_at_rich_loc (&substring_richloc,
-			    "format string is defined here");
+	inform (&substring_richloc,
+		"format string is defined here");
       }
 
   return warned;
@@ -171,13 +166,13 @@ format_warning_va (const substring_loc &fmt_loc,
 
 bool
 format_warning_at_substring (const substring_loc &fmt_loc,
-			     const source_range *param_range,
+			     location_t param_loc,
 			     const char *corrected_substring,
 			     int opt, const char *gmsgid, ...)
 {
   va_list ap;
   va_start (ap, gmsgid);
-  bool warned = format_warning_va (fmt_loc, param_range, corrected_substring,
+  bool warned = format_warning_va (fmt_loc, param_loc, corrected_substring,
 				   opt, gmsgid, &ap);
   va_end (ap);
 
diff --git a/gcc/substring-locations.h b/gcc/substring-locations.h
index a91cc6c8b4a..3d7796db3e6 100644
--- a/gcc/substring-locations.h
+++ b/gcc/substring-locations.h
@@ -77,13 +77,13 @@ class substring_loc
 /* Functions for emitting a warning about a format string.  */
 
 extern bool format_warning_va (const substring_loc &fmt_loc,
-			       const source_range *param_range,
+			       location_t param_loc,
 			       const char *corrected_substring,
 			       int opt, const char *gmsgid, va_list *ap)
   ATTRIBUTE_GCC_DIAG (5,0);
 
 extern bool format_warning_at_substring (const substring_loc &fmt_loc,
-					 const source_range *param_range,
+					 location_t param_loc,
 					 const char *corrected_substring,
 					 int opt, const char *gmsgid, ...)
   ATTRIBUTE_GCC_DIAG (5,0);
diff --git a/gcc/system.h b/gcc/system.h
index 01bc134d1cc..187193ff485 100644
--- a/gcc/system.h
+++ b/gcc/system.h
@@ -720,6 +720,16 @@ extern int vsnprintf (char *, size_t, const char *, va_list);
 #define __builtin_expect(a, b) (a)
 #endif
 
+/* Some of the headers included by <memory> can use "abort" within a
+   namespace, e.g. "_VSTD::abort();", which fails after we use the
+   preprocessor to redefine "abort" as "fancy_abort" below.
+   Given that unique-ptr.h can use "free", we need to do this after "free"
+   is declared but before "abort" is overridden.  */
+
+#ifdef INCLUDE_UNIQUE_PTR
+# include "unique-ptr.h"
+#endif
+
 /* Redefine abort to report an internal error w/o coredump, and
    reporting the location of the error in the source file.  */
 extern void fancy_abort (const char *, int, const char *)
@@ -1008,7 +1018,8 @@ extern void fancy_abort (const char *, int, const char *)
 	ROUND_TOWARDS_ZERO SF_SIZE DF_SIZE XF_SIZE TF_SIZE LIBGCC2_TF_CEXT \
 	LIBGCC2_LONG_DOUBLE_TYPE_SIZE STRUCT_VALUE			   \
 	EH_FRAME_IN_DATA_SECTION TARGET_FLT_EVAL_METHOD_NON_DEFAULT	   \
-	JCR_SECTION_NAME TARGET_USE_JCR_SECTION
+	JCR_SECTION_NAME TARGET_USE_JCR_SECTION SDB_DEBUGGING_INFO	   \
+	SDB_DEBUG
 
 /* Hooks that are no longer used.  */
  #pragma GCC poison LANG_HOOKS_FUNCTION_MARK LANG_HOOKS_FUNCTION_FREE	\
diff --git a/gcc/target-insns.def b/gcc/target-insns.def
index 4669439c7e1..75976b2f8d9 100644
--- a/gcc/target-insns.def
+++ b/gcc/target-insns.def
@@ -60,6 +60,7 @@ DEF_TARGET_INSN (jump, (rtx x0))
 DEF_TARGET_INSN (load_multiple, (rtx x0, rtx x1, rtx x2))
 DEF_TARGET_INSN (mem_thread_fence, (rtx x0))
 DEF_TARGET_INSN (memory_barrier, (void))
+DEF_TARGET_INSN (memory_blockage, (void))
 DEF_TARGET_INSN (movstr, (rtx x0, rtx x1, rtx x2))
 DEF_TARGET_INSN (nonlocal_goto, (rtx x0, rtx x1, rtx x2, rtx x3))
 DEF_TARGET_INSN (nonlocal_goto_receiver, (void))
diff --git a/gcc/target.def b/gcc/target.def
index 3129c3f3210..6593df14ee0 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -876,9 +876,8 @@ to generate it on the spot.",
 
 DEFHOOK
 (output_source_filename,
- "Output COFF information or DWARF debugging information which indicates\
- that filename @var{name} is the current source file to the stdio\
- stream @var{file}.\n\
+ "Output DWARF debugging information which indicates that filename\
+ @var{name} is the current source file to the stdio stream @var{file}.\n\
  \n\
  This target hook need not be defined if the standard form of output\
  for the file format in use is appropriate.",
@@ -2837,7 +2836,7 @@ DEFHOOK
  "This hook should return true if @var{x} should not be emitted into\n\
 debug sections.",
  bool, (rtx x),
- hook_bool_rtx_false)
+ default_const_not_ok_for_debug_p)
 
 /* Given an address RTX, say whether it is valid.  */
 DEFHOOK
@@ -3489,6 +3488,19 @@ if @var{extended} is false, 16 or greater than 128 and a multiple of 32.",
  opt_scalar_float_mode, (int n, bool extended),
  default_floatn_mode)
 
+DEFHOOK
+(floatn_builtin_p,
+  "Define this to return true if the @code{_Float@var{n}} and\n\
+@code{_Float@var{n}x} built-in functions should implicitly enable the\n\
+built-in function without the @code{__builtin_} prefix in addition to the\n\
+normal built-in function with the @code{__builtin_} prefix.  The default is\n\
+to only enable built-in functions without the @code{__builtin_} prefix for\n\
+the GNU C langauge.  In strict ANSI/ISO mode, the built-in function without\n\
+the @code{__builtin_} prefix is not enabled.  The argument @code{FUNC} is the\n\
+@code{enum built_in_function} id of the function to be enabled.",
+ bool, (int func),
+ default_floatn_builtin_p)
+
 /* Compute cost of moving data from a register of class FROM to one of
    TO, using MODE.  */
 DEFHOOK
diff --git a/gcc/target.h b/gcc/target.h
index 1b8decd1b49..e7bdad33d34 100644
--- a/gcc/target.h
+++ b/gcc/target.h
@@ -171,9 +171,11 @@ enum vect_cost_for_stmt
   scalar_store,
   vector_stmt,
   vector_load,
+  vector_gather_load,
   unaligned_load,
   unaligned_store,
   vector_store,
+  vector_scatter_store,
   vec_to_scalar,
   scalar_to_vec,
   cond_branch_not_taken,
diff --git a/gcc/targhooks.c b/gcc/targhooks.c
index 1251e452ff5..5d8ecd31b8c 100644
--- a/gcc/targhooks.c
+++ b/gcc/targhooks.c
@@ -81,7 +81,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "predict.h"
 #include "params.h"
 #include "real.h"
-
+#include "langhooks.h"
 
 bool
 default_legitimate_address_p (machine_mode mode ATTRIBUTE_UNUSED,
@@ -176,6 +176,14 @@ default_legitimize_address_displacement (rtx *, rtx *, poly_int64,
   return false;
 }
 
+bool
+default_const_not_ok_for_debug_p (rtx x)
+{
+  if (GET_CODE (x) == UNSPEC)
+    return true;
+  return false;
+}
+
 rtx
 default_expand_builtin_saveregs (void)
 {
@@ -554,6 +562,28 @@ default_floatn_mode (int n, bool extended)
   return opt_scalar_float_mode ();
 }
 
+/* Define this to return true if the _Floatn and _Floatnx built-in functions
+   should implicitly enable the built-in function without the __builtin_ prefix
+   in addition to the normal built-in function with the __builtin_ prefix.  The
+   default is to only enable built-in functions without the __builtin_ prefix
+   for the GNU C langauge.  The argument FUNC is the enum builtin_in_function
+   id of the function to be enabled.  */
+
+bool
+default_floatn_builtin_p (int func ATTRIBUTE_UNUSED)
+{
+  static bool first_time_p = true;
+  static bool c_or_objective_c;
+
+  if (first_time_p)
+    {
+      first_time_p = false;
+      c_or_objective_c = lang_GNU_C () || lang_GNU_OBJC ();
+    }
+
+  return c_or_objective_c;
+}
+
 /* Make some target macros useable by target-independent code.  */
 bool
 targhook_words_big_endian (void)
diff --git a/gcc/targhooks.h b/gcc/targhooks.h
index ac315a476e4..917431f17ee 100644
--- a/gcc/targhooks.h
+++ b/gcc/targhooks.h
@@ -26,6 +26,7 @@ extern void default_external_libcall (rtx);
 extern rtx default_legitimize_address (rtx, rtx, machine_mode);
 extern bool default_legitimize_address_displacement (rtx *, rtx *,
 						     poly_int64, machine_mode);
+extern bool default_const_not_ok_for_debug_p (rtx);
 
 extern int default_unspec_may_trap_p (const_rtx, unsigned);
 extern machine_mode default_promote_function_mode (const_tree, machine_mode,
@@ -74,6 +75,7 @@ extern tree default_mangle_assembler_name (const char *);
 extern bool default_scalar_mode_supported_p (scalar_mode);
 extern bool default_libgcc_floating_mode_supported_p (scalar_float_mode);
 extern opt_scalar_float_mode default_floatn_mode (int, bool);
+extern bool default_floatn_builtin_p (int);
 extern bool targhook_words_big_endian (void);
 extern bool targhook_float_words_big_endian (void);
 extern bool default_float_exceptions_rounding_supported_p (void);
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 566864c2183..10331b39929 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,1301 @@
+2017-11-04  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/81735
+	* gfortran.dg/pr81735.f90: New test.
+
+2017-11-03  Steven G. Kargl  <kargl@gcc.gnu.org>
+
+	PR fortran/82796
+	* gfortran.dg/equiv_pure.f90: New test.
+
+2017-11-03  Jeff Law  <law@redhat.com>
+
+	PR target/82823
+	* g++.dg/torture/pr82823.C: New test.
+
+	* gcc.target/i386/stack-check-12.c: New test.
+
+2017-11-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/78821
+	* gcc.dg/store_merging_13.c: New test.
+	* gcc.dg/store_merging_14.c: New test.
+
+2017-11-03  Steven G. Kargl  <kargl@gcc.gnu.org>
+
+	* gfortran.dg/large_real_kind_2.F90: Test passes on FreeBSD.  Remove
+	dg-xfail-if directive.
+
+2017-11-03  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gcc.target/mips/msa.c: Add -fcommon to dg-options.
+
+2017-11-03  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR testsuite/82828
+	PR rtl-optimization/70263
+	* gcc.target/i386/pr70263-2.c: Fix invalid testcase.
+
+2017-11-03  Marc Glisse  <marc.glisse@inria.fr>
+
+	* gcc.dg/tree-ssa/negneg-1.c: New file.
+	* gcc.dg/tree-ssa/negneg-2.c: Likewise.
+	* gcc.dg/tree-ssa/negneg-3.c: Likewise.
+	* gcc.dg/tree-ssa/negneg-4.c: Likewise.
+
+2017-11-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/no-strict-overflow-3.c (foo): Update magic
+	value to not clash with frequency.
+	* gcc.dg/strict-overflow-3.c (foo): Likewise.
+	* gcc.dg/tree-ssa/builtin-sprintf-2.c: Update template.
+	* gcc.dg/tree-ssa/dump-2.c: Update template.
+	* gcc.dg/tree-ssa/ifc-10.c: Update template.
+	* gcc.dg/tree-ssa/ifc-11.c: Update template.
+	* gcc.dg/tree-ssa/ifc-12.c: Update template.
+	* gcc.dg/tree-ssa/ifc-20040816-1.c: Update template.
+	* gcc.dg/tree-ssa/ifc-20040816-2.c: Update template.
+	* gcc.dg/tree-ssa/ifc-5.c: Update template.
+	* gcc.dg/tree-ssa/ifc-8.c: Update template.
+	* gcc.dg/tree-ssa/ifc-9.c: Update template.
+	* gcc.dg/tree-ssa/ifc-cd.c: Update template.
+	* gcc.dg/tree-ssa/ifc-pr56541.c: Update template.
+	* gcc.dg/tree-ssa/ifc-pr68583.c: Update template.
+	* gcc.dg/tree-ssa/ifc-pr69489-1.c: Update template.
+	* gcc.dg/tree-ssa/ifc-pr69489-2.c: Update template.
+	* gcc.target/i386/pr61403.c: Update template.
+
+2017-11-03  Nathan Sidwell  <nathan@acm.org>
+
+	* lib/scanlang.exp: Fix error message to refer to scan-lang-dump.
+
+	PR c++/82710
+	* g++.dg/warn/pr82710.C: More cases.
+
+2017-11-03  Richard Sandiford  <richard.sandiford@linaro.org>
+
+	* gcc.dg/pr82809.c: New test.
+
+2017-11-02  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/81957
+	* g++.dg/cpp0x/variadic-crash5.C: New.
+
+2017-11-02  Steve Ellcey  <sellcey@cavium.com>
+
+	PR target/79868
+	* gcc.target/aarch64/spellcheck_1.c: Update dg-error string to match
+	new format.
+	* gcc.target/aarch64/spellcheck_2.c: Ditto.
+	* gcc.target/aarch64/spellcheck_3.c: Ditto.
+	* gcc.target/aarch64/target_attr_11.c: Ditto.
+	* gcc.target/aarch64/target_attr_12.c: Ditto.
+	* gcc.target/aarch64/target_attr_17.c: Ditto.
+
+2017-11-02  Nathan Sidwell  <nathan@acm.org>
+
+	PR c++/82710
+	* g++.dg/warn/pr82710.C: New.
+
+	* g++.dg/lang-dump.C: New.
+
+2017-11-02  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82795
+	* gcc.target/i386/pr82795.c: New testcase.
+
+2017-11-02  Claudiu Zissulescu <claziss@synopsys.com>
+
+	* gcc.target/arc/loop-1.c: Add test.
+
+2017-11-02  Tom de Vries  <tom@codesourcery.com>
+
+	PR testsuite/82415
+	* gcc.target/i386/naked-1.c: Make scan patterns more precise.
+	* gcc.target/i386/naked-2.c: Same.
+
+2017-11-02  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/82765
+	* gcc.dg/pr82765.c: New testcase.
+
+2017-11-02  Tom de Vries  <tom@codesourcery.com>
+
+	* gfortran.dg/implied_do_io_1.f90: Fix scan-tree-dump-times pattern.
+
+2017-11-01  Jakub Jelinek  <jakub@redhat.com>
+
+	PR rtl-optimization/82778
+	* g++.dg/opt/pr82778.C: New test.
+
+2017-11-01  Michael Collison  <michael.collison@arm.com>
+
+	PR rtl-optimization/82597
+	* gcc.dg/pr82597.c: New test.
+
+2017-11-01  Uros Bizjak  <ubizjak@gmail.com>
+
+	* gcc.target/alpha/sqrt.c: New test.
+
+2017-10-31  Daniel Santos  <daniel.santos@pobox.com>
+
+	* gcc.target/i386/pr82002-1.c: New test.
+	* gcc.target/i386/pr82002-2a.c: New xfail test.
+	* gcc.target/i386/pr82002-2b.c: New xfail test.
+
+2017-10-31  Martin Jambor  <mjambor@suse.cz>
+
+	PR c++/81702
+	* g++.dg/tree-ssa/pr81702.C: New test.
+
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* jit.dg/jit.exp (jit-dg-test): If PRESERVE_EXECUTABLES is set in
+	the environment, don't delete the generated executable.
+
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/cpp0x/auto21.C: Update dg-error to reflect addition of quotes.
+	* g++.dg/cpp0x/missing-initializer_list-include.C: Likewise.
+
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_show_trees.c (show_tree): Update
+	for renaming of error_at_rich_loc and inform_at_rich_loc.
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.c
+	(test_show_locus): Likewise for renaming of warning_at_rich_loc.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* g++.dg/gcov/loop.C: New test.
+	* lib/gcov.exp: Support human readable format for counts.
+
+2017-10-31  Martin Liska  <mliska@suse.cz>
+
+	* g++.dg/gcov/ternary.C: New test.
+	* g++.dg/gcov/gcov-threads-1.C (main): Update expected line count.
+	* lib/gcov.exp: Support new format for intermediate file format.
+
+2017-11-01  Julia Koval  <julia.koval@intel.com>
+
+	* gcc.target/i386/avx-1.c: Handle new intrinsics.
+	* gcc.target/i386/avx512-check.h: Check GFNI bit.
+	* gcc.target/i386/avx512f-gf2p8affineinvqb-2.c: Runtime test.
+	* gcc.target/i386/avx512vl-gf2p8affineinvqb-2.c: Runtime test.
+	* gcc.target/i386/gfni-1.c: New.
+	* gcc.target/i386/gfni-2.c: New.
+	* gcc.target/i386/gfni-3.c: New.
+	* gcc.target/i386/gfni-4.c: New.
+	* gcc.target/i386/i386.exp: (check_effective_target_gfni): New.
+	* gcc.target/i386/sse-12.c: Handle new intrinsics.
+	* gcc.target/i386/sse-13.c: Ditto.
+	* gcc.target/i386/sse-14.c: Ditto.
+	* gcc.target/i386/sse-22.c: Ditto.
+	* gcc.target/i386/sse-23.c: Ditto.
+	* g++.dg/other/i386-2.C: Ditto.
+	* g++.dg/other/i386-3.C: Ditto.
+
+2017-11-01  Michael Collison  <michael.collison@arm.com>
+
+	PR rtl-optimization/82597
+	* gcc.dg/pr82597.c: New test.
+
+2017-10-30  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/67595
+	* g++.dg/concepts/pr67595.C: New.
+
+2017-10-30  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/80850
+	* gfortran.dg/class_64_f90 : New test.
+
+2017-10-30  Uros Bizjak  <ubizjak@gmail.com>
+
+	* g++.dg/pr82725.C: Move to ...
+	* g++.dg/cpp0x/pr82725.C: ... here.  Add c++11 target directive.
+
+2017-10-30  Steven G. Kargl   <kargl@gcc.gnu.org>
+
+	* gfortran.dg/dtio_13.f90: Remove TODO comment and dg-error test.
+
+2017-10-30  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82085
+	* g++.dg/cpp1y/var-templ56.C: New.
+
+2017-10-30  Nathan Sidwell  <nathan@acm.org>
+
+	* g++.dg/other/operator2.C: Adjust diagnostic.
+	* g++.old-deja/g++.jason/operator.C: Likewise.
+
+2017-10-30  Steven Munroe  <munroesj@gcc.gnu.org>
+
+	* sse2-check.h: New file.
+	* sse2-addpd-1.c: New file.
+	* sse2-addsd-1.c: New file.
+	* sse2-andnpd-1.c: New file.
+	* sse2-andpd-1.c: New file.
+	* sse2-cmppd-1.c: New file.
+	* sse2-cmpsd-1.c: New file.
+	* sse2-comisd-1.c: New file.
+	* sse2-comisd-2.c: New file.
+	* sse2-comisd-3.c: New file.
+	* sse2-comisd-4.c: New file.
+	* sse2-comisd-5.c: New file.
+	* sse2-comisd-6.c: New file.
+	* sse2-cvtdq2pd-1.c: New file.
+	* sse2-cvtdq2ps-1.c: New file.
+	* sse2-cvtpd2dq-1.c: New file.
+	* sse2-cvtpd2ps-1.c: New file.
+	* sse2-cvtps2dq-1.c: New file.
+	* sse2-cvtps2pd-1.c: New file.
+	* sse2-cvtsd2si-1.c: New file.
+	* sse2-cvtsd2si-2.c: New file.
+	* sse2-cvtsd2ss-1.c: New file.
+	* sse2-cvtsi2sd-1.c: New file.
+	* sse2-cvtsi2sd-2.c: New file.
+	* sse2-cvtss2sd-1.c: New file.
+	* sse2-cvttpd2dq-1.c: New file.
+	* sse2-cvttps2dq-1.c: New file.
+	* sse2-cvttsd2si-1.c: New file.
+	* sse2-cvttsd2si-2.c: New file.
+	* sse2-divpd-1.c: New file.
+	* sse2-divsd-1.c: New file.
+	* sse2-maxpd-1.c: New file.
+	* sse2-maxsd-1.c: New file.
+	* sse2-minpd-1.c: New file.
+	* sse2-minsd-1.c: New file.
+	* sse2-mmx.c: New file.
+	* sse2-movhpd-1.c: New file.
+	* sse2-movhpd-2.c: New file.
+	* sse2-movlpd-1.c: New file.
+	* sse2-movlpd-2.c: New file.
+	* sse2-movmskpd-1.c: New file.
+	* sse2-movq-1.c: New file.
+	* sse2-movq-2.c: New file.
+	* sse2-movq-3.c: New file.
+	* sse2-movsd-1.c: New file.
+	* sse2-movsd-2.c: New file.
+	* sse2-movsd-3.c: New file.
+	* sse2-mulpd-1.c: New file.
+	* sse2-mulsd-1.c: New file.
+	* sse2-orpd-1.c: New file.
+	* sse2-packssdw-1.c: New file.
+	* sse2-packsswb-1.c: New file.
+	* sse2-packuswb-1.c: New file.
+	* sse2-paddb-1.c: New file.
+	* sse2-paddd-1.c: New file.
+	* sse2-paddq-1.c: New file.
+	* sse2-paddsb-1.c: New file.
+	* sse2-paddsw-1.c: New file.
+	* sse2-paddusb-1.c: New file.
+	* sse2-paddusw-1.c: New file.
+	* sse2-paddw-1.c: New file.
+	* sse2-pavgb-1.c: New file.
+	* sse2-pavgw-1.c: New file.
+	* sse2-pcmpeqb-1.c: New file.
+	* sse2-pcmpeqd-1.c: New file.
+	* sse2-pcmpeqw-1.c: New file.
+	* sse2-pcmpgtb-1.c: New file.
+	* sse2-pcmpgtd-1.c: New file.
+	* sse2-pcmpgtw-1.c: New file.
+	* sse2-pextrw.c: New file.
+	* sse2-pinsrw.c: New file.
+	* sse2-pmaddwd-1.c: New file.
+	* sse2-pmaxsw-1.c: New file.
+	* sse2-pmaxub-1.c: New file.
+	* sse2-pminsw-1.c: New file.
+	* sse2-pminub-1.c: New file.
+	* sse2-pmovmskb-1.c: New file.
+	* sse2-pmulhuw-1.c: New file.
+	* sse2-pmulhw-1.c: New file.
+	* sse2-pmullw-1.c: New file.
+	* sse2-pmuludq-1.c: New file.
+	* sse2-psadbw-1.c: New file.
+	* sse2-pshufd-1.c: New file.
+	* sse2-pshufhw-1.c: New file.
+	* sse2-pshuflw-1.c: New file.
+	* sse2-pslld-1.c: New file.
+	* sse2-pslld-2.c: New file.
+	* sse2-pslldq-1.c: New file.
+	* sse2-psllq-1.c: New file.
+	* sse2-psllq-2.c: New file.
+	* sse2-psllw-1.c: New file.
+	* sse2-psllw-2.c: New file.
+	* sse2-psrad-1.c: New file.
+	* sse2-psrad-2.c: New file.
+	* sse2-psraw-1.c: New file.
+	* sse2-psraw-2.c: New file.
+	* sse2-psrld-1.c: New file.
+	* sse2-psrld-2.c: New file.
+	* sse2-psrldq-1.c: New file.
+	* sse2-psrlq-1.c: New file.
+	* sse2-psrlq-2.c: New file.
+	* sse2-psrlw-1.c: New file.
+	* sse2-psrlw-2.c: New file.
+	* sse2-psubb-1.c: New file.
+	* sse2-psubd-1.c: New file.
+
+2017-10-30  Will Schmidt  <will_schmidt@vnet.ibm.com>
+
+	* gcc.target/powerpc/fold-vec-perm-longlong.c: Update to use long long
+	types for testcase arguments.
+
+2017-10-30  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82762
+	* gcc.dg/torture/pr82762.c: New testcase.
+
+2017-10-30  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/gimplefe-27.c: New testcase.
+
+2017-10-30  Joseph Myers  <joseph@codesourcery.com>
+
+	* gcc.dg/c17-version-1.c, gcc.dg/c17-version-2.c: New tests.
+
+2017-10-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/22141
+	* gcc.dg/store_merging_10.c: New test.
+	* gcc.dg/store_merging_11.c: New test.
+	* gcc.dg/store_merging_12.c: New test.
+	* g++.dg/pr71694.C: Add -fno-store-merging to dg-options.
+
+2017-10-30  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82725
+	* g++.dg/pr82725.C: New test.
+
+2017-10-29  Jim Wilson  <wilson@tuliptree.org>
+
+	* lib/gcc-dg.exp (gcc-dg-debug-runtest): Delete -gcoff.
+	* lib/gfortran-dg.exp (gfortran-dg-debug-runtest): Delete -gcoff.
+
+2017-10-28  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/70971
+	* g++.dg/torture/pr70971.C: New.
+
+2017-10-28  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/81758
+	* gfortran.dg/class_63.f90: New test.
+
+2017-10-27  Steven G. Kargl  <kargl@gcc.gnu.org>
+
+	PR fortran/82620
+	* gfortran.dg/allocate_error_7.f90: new test.
+
+2017-10-27  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82218
+	* g++.dg/cpp1y/constexpr-82218.C: New.
+
+2017-10-27  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gnat.dg/opt68.ad[sb]: New test.
+
+2017-10-27  Daniel Santos  <daniel.santos@pobox.com>
+
+	* gcc.target/i386/pr82196-1.c (dg-options): Add -mno-avx.
+
+2017-10-27  Michael Meissner  <meissner@linux.vnet.ibm.com>
+
+	* gcc.target/powerpc/float128-hw.c: Add support for all 4 FMA
+	variants.  Check various conversions to/from float128.  Check
+	negation.  Use {\m...\M} in the tests.
+	* gcc.target/powerpc/float128-hw2.c: New test for implicit
+	_Float128 math functions.
+	* gcc.target/powerpc/float128-hw3.c: New test for strict ansi mode
+	not implicitly adding the _Float128 math functions.
+	* gcc.target/powerpc/float128-fma2.c: Delete, test is no longer
+	valid.
+	* gcc.target/powerpc/float128-sqrt2.c: Likewise.
+
+2017-10-27  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82692
+	* gcc.dg/torture/pr82692.c: New test.
+
+2017-10-27  Will Schmidt  <will_schmidt@vnet.ibm.com>
+
+	* gcc.target/powerpc/fold-vec-neg-char.c: New.
+	* gcc.target/powerpc/fold-vec-neg-floatdouble.c: New.
+	* gcc.target/powerpc/fold-vec-neg-int.c: New.
+	* gcc.target/powerpc/fold-vec-neg-longlong.c: New.
+	* gcc.target/powerpc/fold-vec-neg-short.c: New.
+
+2017-10-27  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/56342
+	* gfortran.dg/matmul_const.f90: New test.
+
+2017-10-25  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.target/i386/pr70021.c: Add -mtune=skylake.
+
+2017-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82703
+	* gcc.dg/pr82703.c: New test.
+
+2017-10-27  Prathamesh Kulkarni  <prathamesh.kulkarni@linaro.org>
+
+	* gcc.dg/ipa/propmalloc-1.c: New test-case.
+	* gcc.dg/ipa/propmalloc-2.c: Likewise.
+	* gcc.dg/ipa/propmalloc-3.c: Likewise.
+
+2017-10-27  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/71385
+	* g++.dg/concepts/pr71385.C: New.
+
+2017-10-27  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/80739
+	* g++.dg/cpp1y/constexpr-80739.C: New.
+
+2017-10-27  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/81659
+	* g++.dg/torture/pr81659.C: New testcase.
+
+2017-10-26  Michael Collison  <michael.collison@arm.com>
+
+	* gcc.target/aarch64/fix_trunc1.c: New testcase.
+	* gcc.target/aarch64/vect-vcvt.c: Fix scan-assembler
+	directives to allow float or integer destination registers for
+	fcvtz[su].
+
+2017-10-26  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gcc.target/nios2/gpopt-r0rel-sec.c: New.
+
+2017-10-26  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gcc.target/nios2/gpopt-gprel-sec.c: New.
+
+2017-10-26  Olga Makhotina  <olga.makhotina@intel.com>
+
+	* gcc.target/i386/avx512f-vcmpps-1.c (_mm512_cmpeq_ps_mask,
+	_mm512_cmple_ps_mask, _mm512_cmplt_ps_mask,
+	_mm512_cmpneq_ps_mask, _mm512_cmpnle_ps_mask,
+	_mm512_cmpnlt_ps_mask, _mm512_cmpord_ps_mask,
+	_mm512_cmpunord_ps_mask, _mm512_mask_cmpeq_ps_mask,
+	_mm512_mask_cmple_ps_mask, _mm512_mask_cmplt_ps_mask,
+	_mm512_mask_cmpneq_ps_mask, _mm512_mask_cmpnle_ps_mask,
+	_mm512_mask_cmpnlt_ps_mask, _mm512_mask_cmpord_ps_mask,
+	_mm512_mask_cmpunord_ps_mask): Test new intrinsics.
+	* gcc.target/i386/avx512f-vcmpps-2.c (_mm512_cmpeq_ps_mask,
+	_mm512_cmple_ps_mask, _mm512_cmplt_ps_mask,
+	_mm512_cmpneq_ps_mask, _mm512_cmpnle_ps_mask,
+	_mm512_cmpnlt_ps_mask, _mm512_cmpord_ps_mask,
+	_mm512_cmpunord_ps_mask, _mm512_mask_cmpeq_ps_mask,
+	_mm512_mask_cmple_ps_mask, _mm512_mask_cmplt_ps_mask,
+	_mm512_mask_cmpneq_ps_mask, _mm512_mask_cmpnle_ps_mask,
+	_mm512_mask_cmpnlt_ps_mask, _mm512_mask_cmpord_ps_mask,
+	_mm512_mask_cmpunord_ps_mask): Test new intrinsics.
+	* gcc.target/i386/avx512f-vcmppd-1.c (_mm512_cmpeq_pd_mask,
+	_mm512_cmple_pd_mask, _mm512_cmplt_pd_mask,
+	_mm512_cmpneq_pd_mask, _mm512_cmpnle_pd_mask,
+	_mm512_cmpnlt_pd_mask, _mm512_cmpord_pd_mask,
+	_mm512_cmpunord_pd_mask, _mm512_mask_cmpeq_pd_mask,
+	_mm512_mask_cmple_pd_mask, _mm512_mask_cmplt_pd_mask,
+	_mm512_mask_cmpneq_pd_mask, _mm512_mask_cmpnle_pd_mask,
+	_mm512_mask_cmpnlt_pd_mask, _mm512_mask_cmpord_pd_mask,
+	_mm512_mask_cmpunord_pd_mask): Test new intrinsics.
+	* gcc.target/i386/avx512f-vcmppd-2.c (_mm512_cmpeq_pd_mask,
+	_mm512_cmple_pd_mask, _mm512_cmplt_pd_mask,
+	_mm512_cmpneq_pd_mask, _mm512_cmpnle_pd_mask,
+	_mm512_cmpnlt_pd_mask, _mm512_cmpord_pd_mask,
+	_mm512_cmpunord_pd_mask, _mm512_mask_cmpeq_pd_mask,
+	_mm512_mask_cmple_pd_mask, _mm512_mask_cmplt_pd_mask,
+	_mm512_mask_cmpneq_pd_mask, _mm512_mask_cmpnle_pd_mask,
+	_mm512_mask_cmpnlt_pd_mask, _mm512_mask_cmpord_pd_mask,
+	_mm512_mask_cmpunord_pd_mask): Test new intrinsics.
+
+2017-10-26  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* gcc.target/aarch64/ldp_stp_unaligned_2.c: New file.
+
+2017-10-26  James Greenhalgh  <james.greenhalgh@arm.com>
+
+	* gcc.target/arm/require-pic-register-loc.c: Use wider regex for
+	column information.
+
+2017-10-26  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.dg/vect/vect-reduc-dot-s8a.c
+	(dg-additional-options, dg-require-effective-target): Add +dotprod.
+	* gcc.dg/vect/vect-reduc-dot-u8a.c
+	(dg-additional-options, dg-require-effective-target): Add +dotprod.
+
+2017-10-26  Tamar Christina  <tamar.christina@arm.com>
+
+	* lib/target-supports.exp
+	(check_effective_target_arm_v8_2a_dotprod_neon_ok_nocache): New.
+	(check_effective_target_arm_v8_2a_dotprod_neon_ok): New.
+	(add_options_for_arm_v8_2a_dotprod_neon): New.
+	(check_effective_target_arm_v8_2a_dotprod_neon_hw): New.
+	(check_effective_target_vect_sdot_qi): Add ARM && AArch64.
+	(check_effective_target_vect_udot_qi): Likewise.
+	* gcc.target/arm/simd/vdot-exec.c: New.
+	* gcc.target/aarch64/advsimd-intrinsics/vdot-exec.c: New.
+	* gcc/doc/sourcebuild.texi: Document arm_v8_2a_dotprod_neon.
+
+2017-10-26  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.dg/vect/vect-multitypes-1.c: Correct target selector.
+
+2017-10-26  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.target/aarch64/inline-lrint_2.c (dg-options): Add -fno-trapping-math.
+
+2017-10-26  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.target/aarch64/advsimd-intrinsics/vect-dot-qi.h: New.
+	* gcc.target/aarch64/advsimd-intrinsics/vdot-compile.c: New.
+	* gcc.target/aarch64/advsimd-intrinsics/vect-dot-s8.c: New.
+	* gcc.target/aarch64/advsimd-intrinsics/vect-dot-u8.c: New.
+
+2017-10-25  David Malcolm  <dmalcolm@redhat.com>
+
+	PR c/7356
+	PR c/44515
+	* c-c++-common/pr44515.c: New test case.
+	* gcc.dg/pr7356-2.c: New test case.
+	* gcc.dg/pr7356.c: New test case.
+	* gcc.dg/spellcheck-typenames.c: Update the "singed" char "TODO"
+	case to reflect changes to output.
+	* gcc.dg/noncompile/920923-1.c: Add dg-warning to reflect changes
+	to output.
+
+2017-10-25  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc.dg/fold-cond_expr-1.c: Rename to...
+	* gcc.dg/fold-cond-2.c: ...this.
+	* gcc.dg/fold-cond-3.c: New test.
+
+2017-10-25  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82436
+	* gcc.dg/torture/pr82436-2.c: New testcase.
+
+2017-10-25  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/71820
+	* g++.dg/ext/typeof12.C: New.
+
+2017-10-25  Tom de Vries  <tom@codesourcery.com>
+
+	* gcc.dg/tree-ssa/loop-1.c: Add xfail for nvptx in scan-assembler-times
+	line, and add nvptx-specific version.
+
+2017-10-25  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.target/i386/cet-sjlj-5.c: Allow for emtpy user label prefix
+	in setjmp call.
+
+2017-10-25  Jakub Jelinek  <jakub@redhat.com>
+
+	PR libstdc++/81706
+	* gcc.target/i386/pr81706.c: New test.
+	* g++.dg/ext/pr81706.C: New test.
+
+2017-10-24  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82460
+	* gcc.target/i386/pr82460-1.c: New test.
+	* gcc.target/i386/pr82460-2.c: New test.
+	* gcc.target/i386/avx512f-vpermt2pd-1.c: Adjust scan-assembler*
+	regexps to allow vpermt2* to vpermi2* replacement or vice versa
+	where possible.
+	* gcc.target/i386/avx512vl-vpermt2pd-1.c: Likewise.
+	* gcc.target/i386/avx512f-vpermt2d-1.c: Likewise.
+	* gcc.target/i386/vect-pack-trunc-2.c: Likewise.
+	* gcc.target/i386/avx512vl-vpermt2ps-1.c: Likewise.
+	* gcc.target/i386/avx512vl-vpermt2q-1.c: Likewise.
+	* gcc.target/i386/avx512f-vpermt2ps-1.c: Likewise.
+	* gcc.target/i386/avx512vl-vpermt2d-1.c: Likewise.
+	* gcc.target/i386/avx512bw-vpermt2w-1.c: Likewise.
+	* gcc.target/i386/avx512vbmi-vpermt2b-1.c: Likewise.
+	* gcc.target/i386/avx512f-vpermt2q-1.c: Likewise.
+
+	PR target/82370
+	* gcc.target/i386/pr82370.c: New test.
+
+2017-10-24  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82466
+	* c-c++-common/Wbuiltin-declaration-mismatch-1.c: New.
+	* c-c++-common/Wno-builtin-declaration-mismatch-1.c: Likewise.
+	* g++.dg/warn/Wbuiltin_declaration_mismatch-1.C: Likewise.
+	* g++.dg/parse/builtin2.C: Adjust.
+	* g++.old-deja/g++.mike/p811.C: Likewise.
+
+2017-10-24  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/80991
+	* g++.dg/ext/is_trivially_constructible5.C: New.
+
+2017-10-24  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.target/i386/387-ficom-1.c: Allow for ficomp without s
+	suffix.
+	* gcc.target/i386/387-ficom-2.c: Likewise.
+
+2017-10-24  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.target/i386/cet-sjlj-3.c: Allow for emtpy user label prefix
+	in setjmp call.
+
+2017-10-24  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82697
+	* gcc.dg/torture/pr82697.c: New testcase.
+
+2017-10-24  Mukesh Kapoor  <mukesh.kapoor@oracle.com>
+	    Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82307
+	* g++.dg/cpp0x/enum35.C: New.
+	* g++.dg/cpp0x/enum36.C: Likewise.
+
+2017-10-24  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/82659
+	* gcc.target/i386/cet-label-2.c: New test.
+	* gcc.target/i386/cet-sjlj-4.c: Likewise.
+	* gcc.target/i386/cet-sjlj-5.c: Likewise.
+	* gcc.target/i386/cet-switch-3.c: Likewise.
+	* gcc.target/i386/pr82659-1.c: Likewise.
+	* gcc.target/i386/pr82659-2.c: Likewise.
+	* gcc.target/i386/pr82659-3.c: Likewise.
+	* gcc.target/i386/pr82659-4.c: Likewise.
+	* gcc.target/i386/pr82659-5.c: Likewise.
+	* gcc.target/i386/pr82659-6.c: Likewise.
+
+2017-10-23  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gcc.target/nios2/cdx-branch.c:  Fix broken test.
+	* gcc.target/nios2/lo-addr-bypass.c: New.
+	* gcc.target/nios2/lo-addr-char.c: New.
+	* gcc.target/nios2/lo-addr-int.c: New.
+	* gcc.target/nios2/lo-addr-pic.c: New.
+	* gcc.target/nios2/lo-addr-short.c: New.
+	* gcc.target/nios2/lo-addr-tls.c: New.
+	* gcc.target/nios2/lo-addr-uchar.c: New.
+	* gcc.target/nios2/lo-addr-ushort.c: New.
+	* gcc.target/nios2/lo-addr-volatile.c: New.
+
+2017-10-23  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/80449
+	* g++.dg/cpp1z/class-deduction46.C: New.
+
+2017-10-23  Jakub Jelinek  <jakub@redhat.com>
+
+	PR debug/82630
+	* g++.dg/guality/pr82630.C: New test.
+
+2017-10-23  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/82662
+	* gcc.target/i386/pr82662.c: New test.
+
+2017-10-23  Marek Polacek  <polacek@redhat.com>
+
+	PR c/82681
+	* gcc.dg/c90-const-expr-11.c: Fix typos in dg-warning.
+	* gcc.dg/overflow-warn-5.c: Likewise.
+	* gcc.dg/overflow-warn-8.c: Likewise.
+
+2017-10-23  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/82673
+	* gcc.target/i386/pr82673.c: New test.
+
+2017-10-23  Jakub Jelinek  <jakub@redhat.com>
+
+	* lib/scanasm.exp (dg-function-on-line): Accept optional column info.
+	* gcc.dg/debug/dwarf2/pr53948.c: Likewise.
+	* g++.dg/debug/dwarf2/pr77363.C: Likewise.
+	* gcc.dg/debug/dwarf2/asm-line1.c: Add -gno-column-info to dg-options.
+	* gcc.dg/debug/dwarf2/discriminator.c: Likewise.
+	* g++.dg/debug/dwarf2/typedef6.C: Likewise.
+
+2017-10-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82672
+	* gfortran.dg/graphite/pr82672.f90: New testcase.
+
+2017-10-23  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/77555
+	* g++.dg/torture/pr77555.C: New.
+
+2017-10-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82129
+	* gcc.dg/torture/pr82129.c: New testcase.
+
+2017-10-22  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/52451
+	* gcc.dg/torture/pr52451.c: New test.
+
+2017-10-22  Uros Bizjak  <ubizjak@gmail.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82628
+	* gcc.dg/torture/pr82628.c: New test.
+
+2017-10-22  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+	* c-c++-common/attr-nocf-check-1a.c: Remove test.
+	* c-c++-common/attr-nocf-check-3a.c: Likewise.
+	* gcc.target/i386/attr-nocf-check-1a.c: Add test.
+	* gcc.target/i386/attr-nocf-check-3a.c: Likewise.
+
+2017-10-21  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+	* c-c++-common/attr-nocf-check-1.c: Shorten a cheking message.
+	* c-c++-common/attr-nocf-check-3.c: Likewise.
+	* c-c++-common/fcf-protection-1.c: Add x86 specific message.
+	* c-c++-common/fcf-protection-2.c: Likewise.
+	* c-c++-common/fcf-protection-3.c: Likewise.
+	* c-c++-common/fcf-protection-5.c: Likewise.
+	* c-c++-common/attr-nocf-check-1a.c: New test.
+	* c-c++-common/attr-nocf-check-3a.c: Likewise.
+	* g++.dg/cet-notrack-1.C: Likewise.
+	* gcc.target/i386/cet-intrin-1.c: Likewise.
+	* gcc.target/i386/cet-intrin-10.c: Likewise.
+	* gcc.target/i386/cet-intrin-2.c: Likewise.
+	* gcc.target/i386/cet-intrin-3.c: Likewise.
+	* gcc.target/i386/cet-intrin-4.c: Likewise.
+	* gcc.target/i386/cet-intrin-5.c: Likewise.
+	* gcc.target/i386/cet-intrin-6.c: Likewise.
+	* gcc.target/i386/cet-intrin-7.c: Likewise.
+	* gcc.target/i386/cet-intrin-8.c: Likewise.
+	* gcc.target/i386/cet-intrin-9.c: Likewise.
+	* gcc.target/i386/cet-label.c: Likewise.
+	* gcc.target/i386/cet-notrack-1a.c: Likewise.
+	* gcc.target/i386/cet-notrack-1b.c: Likewise.
+	* gcc.target/i386/cet-notrack-2a.c: Likewise.
+	* gcc.target/i386/cet-notrack-2b.c: Likewise.
+	* gcc.target/i386/cet-notrack-3.c: Likewise.
+	* gcc.target/i386/cet-notrack-4a.c: Likewise.
+	* gcc.target/i386/cet-notrack-4b.c: Likewise.
+	* gcc.target/i386/cet-notrack-5a.c: Likewise.
+	* gcc.target/i386/cet-notrack-5b.c: Likewise.
+	* gcc.target/i386/cet-notrack-6a.c: Likewise.
+	* gcc.target/i386/cet-notrack-6b.c: Likewise.
+	* gcc.target/i386/cet-notrack-7.c: Likewise.
+	* gcc.target/i386/cet-property-1.c: Likewise.
+	* gcc.target/i386/cet-property-2.c: Likewise.
+	* gcc.target/i386/cet-rdssp-1.c: Likewise.
+	* gcc.target/i386/cet-sjlj-1.c: Likewise.
+	* gcc.target/i386/cet-sjlj-2.c: Likewise.
+	* gcc.target/i386/cet-sjlj-3.c: Likewise.
+	* gcc.target/i386/cet-switch-1.c: Likewise.
+	* gcc.target/i386/cet-switch-2.c: Likewise.
+	* lib/target-supports.exp (check_effective_target_cet): New proc.
+
+2017-10-20  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.target/i386/pr79683.c: Disable costmodel.
+
+2017-10-21  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gnat.dg/specs/discr_private.ads: Rename into ...
+	* gnat.dg/specs/discr2.ads: ...this.
+	* gnat.dg/specs/discr_record_constant.ads: Rename into...
+	* gnat.dg/specs/discr3.ads: ...this.
+	* gnat.dg/specs/discr4.ads: New test.
+	* gnat.dg/specs/discr4_pkg.ads: New helper.
+
+2017-10-21  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/82586
+	* gfortran.dg/pdt_16.f03 : New test.
+	* gfortran.dg/pdt_4.f03 : Catch the changed messages.
+	* gfortran.dg/pdt_8.f03 : Ditto.
+
+	PR fortran/82587
+	* gfortran.dg/pdt_17.f03 : New test.
+
+	PR fortran/82589
+	* gfortran.dg/pdt_18.f03 : New test.
+
+2017-10-20  Igor Tsimbalist  <igor.v.tsimbalist@intel.com>
+
+	* c-c++-common/fcf-protection-1.c: New test.
+	* c-c++-common/fcf-protection-2.c: Likewise.
+	* c-c++-common/fcf-protection-3.c: Likewise.
+	* c-c++-common/fcf-protection-4.c: Likewise.
+	* c-c++-common/fcf-protection-5.c: Likewise.
+	* c-c++-common/attr-nocf-check-1.c: Likewise.
+	* c-c++-common/attr-nocf-check-2.c: Likewise.
+	* c-c++-common/attr-nocf-check-3.c: Likewise.
+
+2017-10-20  Ed Schonberg  <schonberg@adacore.com>
+
+	* gnat.dg/sync_iface_call.adb, gnat.dg/sync_iface_call_pkg.ads,
+	gnat.dg/sync_iface_call_pkg2.adb, gnat.dg/sync_iface_call_pkg2.ads:
+	New testcase.
+
+2017-10-20  Justin Squirek  <squirek@adacore.com>
+
+	* gnat.dg/default_pkg_actual.adb, gnat.dg/default_pkg_actual2.adb: New
+	testcases.
+
+2017-10-20  Ed Schonberg  <schonberg@adacore.com>
+
+	* gnat.dg/dimensions.adb, gnat.dg/dimensions.ads: New testcase.
+
+2017-10-20  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82473
+	* gcc.dg/torture/pr82473.c: New testcase.
+
+2017-10-20  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82603
+	* gcc.dg/torture/pr82603.c: New testcase.
+
+2017-10-20  Tom de Vries  <tom@codesourcery.com>
+
+	* gcc.dg/tree-ssa/ldist-27.c: Remove dg-require-stack-size.
+	(main): Move s ...
+	(s): ... here.
+
+2017-10-20  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82158
+	* gcc.dg/tree-ssa/noreturn-1.c: New test.
+
+	PR target/82370
+	* gcc.target/i386/avx-pr82370.c: New test.
+	* gcc.target/i386/avx2-pr82370.c: New test.
+	* gcc.target/i386/avx512f-pr82370.c: New test.
+	* gcc.target/i386/avx512bw-pr82370.c: New test.
+	* gcc.target/i386/avx512vl-pr82370.c: New test.
+	* gcc.target/i386/avx512vlbw-pr82370.c: New test.
+
+2017-10-20  Orlando Arias  <oarias@knights.ucf.edu>
+
+	* lib/target-supports.exp (check_effective_target_keeps_null_pointer_checks):
+	Add msp430 to the list.
+
+2017-10-19  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82308
+	* g++.dg/cpp1z/class-deduction45.C: New.
+
+2017-10-19  Uros Bizjak  <ubizjak@gmail.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82618
+	* gcc.target/i386/pr82618.c: New test.
+
+2017-10-19  Martin Sebor  <msebor@redhat.com>
+
+	PR tree-optimization/82596
+	* gcc/testsuite/gcc.dg/pr82596.c: New test.
+
+2017-10-19  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc.dg/Walloca-15.c: New test.
+	* gnat.dg/stack_usage4.adb: Likewise.
+	* gnat.dg/stack_usage4_pkg.ads: New helper.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/82600
+	* g++.dg/warn/Wreturn-local-addr-4.C: New test.
+
+2017-10-19  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc.dg/debug/dwarf2/sso.c: Rename into...
+	* gcc.dg/debug/dwarf2/sso-1.c: ...this.
+	* gcc.dg/debug/dwarf2/sso-2.c: New test.
+	* gcc.dg/debug/dwarf2/sso-3.c: Likewise.
+
+2017-10-19  Richard Earnshaw  <rearnsha@arm.com>
+
+	PR target/82445
+	* gcc.target/arm/peep-ldrd-1.c: Tighten test scan pattern.
+	* gcc.target/arm/peep-strd-1.c: Likewise.
+	* gcc.target/arm/peep-ldrd-2.c: New test.
+	* gcc.target/arm/peep-strd-2.c: New test.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-c++-common/ubsan/builtin-1.c: New test.
+
+	* c-c++-common/ubsan/float-cast-overflow-1.c: Drop value keyword
+	from expected output regexps.
+	* c-c++-common/ubsan/float-cast-overflow-2.c: Likewise.
+	* c-c++-common/ubsan/float-cast-overflow-3.c: Likewise.
+	* c-c++-common/ubsan/float-cast-overflow-4.c: Likewise.
+	* c-c++-common/ubsan/float-cast-overflow-5.c: Likewise.
+	* c-c++-common/ubsan/float-cast-overflow-6.c: Likewise.
+	* c-c++-common/ubsan/float-cast-overflow-8.c: Likewise.
+	* c-c++-common/ubsan/float-cast-overflow-9.c: Likewise.
+	* c-c++-common/ubsan/float-cast-overflow-10.c: Likewise.
+	* g++.dg/ubsan/float-cast-overflow-bf.C: Likewise.
+	* gcc.dg/ubsan/float-cast-overflow-bf.c: Likewise.
+	* g++.dg/asan/default-options-1.C (__asan_default_options): Add
+	used attribute.
+	* g++.dg/asan/asan_test.C: Run with ASAN_OPTIONS=handle_segv=2
+	in the environment.
+
+	PR target/82580
+	* gcc.target/i386/pr82580.c: Use {\msbb} instead of "sbb" in
+	scan-assembler-times.  Check that there are no movzb* instructions
+	if lp64.
+
+2017-10-19  Tom de Vries  <tom@codesourcery.com>
+
+	* gcc.dg/tree-ssa/ldist-27.c: Use dg-require-stack-size.
+
+2017-10-19  Tom de Vries  <tom@codesourcery.com>
+
+	* lib/target-supports-dg.exp (dg-require-stack-size): New proc.
+	* gcc.c-torture/execute/20030209-1.c: Use dg-require-stack-size.
+	* gcc.c-torture/execute/20040805-1.c: Same.
+	* gcc.c-torture/execute/920410-1.c: Same.
+	* gcc.c-torture/execute/921113-1.c: Same.
+	* gcc.c-torture/execute/921208-2.c: Same.
+	* gcc.c-torture/execute/comp-goto-1.c: Same.
+	* gcc.c-torture/execute/pr20621-1.c: Same.
+	* gcc.c-torture/execute/pr28982b.c: Same.
+	* gcc.dg/tree-prof/comp-goto-1.c: Same.
+
+2017-10-19  Martin Liska  <mliska@suse.cz>
+
+	PR sanitizer/82517
+	* gcc.dg/asan/pr82517.c: New test.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	PR fortran/82568
+	* gfortran.dg/gomp/pr82568.f90: New test.
+
+2017-10-19  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* gfortran.dg/spellcheck-operator.f90: New testcase.
+	* gfortran.dg/spellcheck-procedure_1.f90: New testcase.
+	* gfortran.dg/spellcheck-procedure_2.f90: New testcase.
+	* gfortran.dg/spellcheck-structure.f90: New testcase.
+	* gfortran.dg/spellcheck-parameter.f90: New testcase.
+
+2017-10-18  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/82567
+	* gfortran.dg/array_constructor_51.f90: New test.
+
+2017-10-18  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/79795
+	* gfortran.dg/assumed_size_2.f90: New test.
+
+2017-10-18  Uros Bizjak  <ubizjak@gmail.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82580
+	* gcc.target/i386/pr82580.c: New test.
+
+2017-10-18  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR libfortran/82233
+	* gfortran.dg/execute_command_line_3.f90:  Remove unneeded output.
+	Move test with wait=.false. before the last test.
+
+2017-10-18  Vladimir Makarov  <vmakarov@redhat.com>
+
+	PR middle-end/82556
+	* gcc.target/i386/pr82556.c: New.
+
+2017-10-18  Bin Cheng  <bin.cheng@arm.com>
+
+	* gcc.dg/tree-ssa/ldist-17.c: Adjust test string.
+	* gcc.dg/tree-ssa/ldist-32.c: New test.
+	* gcc.dg/tree-ssa/ldist-35.c: New test.
+	* gcc.dg/tree-ssa/ldist-36.c: New test.
+
+2017-10-18  Bin Cheng  <bin.cheng@arm.com>
+
+	PR tree-optimization/82574
+	* gcc.dg/tree-ssa/pr82574.c: New test.
+
+2017-10-18  Martin Liska  <mliska@suse.cz>
+
+	* gcc.dg/tree-prof/switch-case-2.c: Scan IPA profile dump
+	file instead of expand. Reason is that switch statement is
+	not yet expanded as decision tree, which also contains a BB
+	with count == 2000.
+
+017-10-18  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/82550
+	* gfortran.dg/submodule_30.f08 : New test.
+
+2017-10-18  Andreas Krebbel  <krebbel@linux.vnet.ibm.com>
+
+	* gcc.target/s390/zvector/vec-cmp-2.c
+	(all_eq_double, all_ne_double, all_gt_double)
+	(all_lt_double, all_ge_double, all_le_double)
+	(any_eq_double, any_ne_double, any_gt_double)
+	(any_lt_double, any_ge_double, any_le_double)
+	(all_eq_int, all_ne_int, all_gt_int)
+	(all_lt_int, all_ge_int, all_le_int)
+	(any_eq_int, any_ne_int, any_gt_int)
+	(any_lt_int, any_ge_int, any_le_int): Set global variable instead
+	of calling foo().  Fix return type.
+
+2017-10-18  Martin Liska  <mliska@suse.cz>
+
+	PR sanitizer/82545
+	* gcc.dg/asan/pr82545.c: New test.
+
+2017-10-18  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/69057
+	* g++.dg/cpp1y/auto-fn45.C: New.
+
+2017-10-18  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/68884
+	* g++.dg/cpp0x/variadic-crash4.C: New.
+
+2017-10-18  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/79474
+	* g++.dg/cpp1y/auto-fn44.C: New.
+
+2017-10-17  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc.dg/attr-alloc_size-11.c: UnXFAIL for visium-*-*.
+
+2017-10-17  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/71821
+	* g++.dg/cpp0x/alignas12.C: New.
+
+2017-10-17  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/71368
+	* g++.dg/concepts/pr71368.C: New.
+
+2017-10-17  Nathan Sidwell  <nathan@acm.org>
+
+	PR c++/82560
+	* g++.dg/cpp0x/pr82560.C: New.
+
+	PR middle-end/82577
+	* g++.dg/opt/pr82577.C: New.
+
+2017-10-17  Qing Zhao <qing.zhao@oracle.com>
+	    Wilco Dijkstra <wilco.dijkstra@arm.com>
+
+	PR middle-end/80295
+	* gcc.target/aarch64/pr80295.c: New test.
+
+2017-10-17  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/82563
+	* gcc.dg/graphite/pr82563.c: New testcase.
+
+2017-10-17  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/67831
+	* g++.dg/cpp0x/constexpr-ice18.C: New.
+
+2017-10-17  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/82570
+	* g++.dg/cpp1z/constexpr-lambda18.C: New.
+
+2017-10-17  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/82549
+	* gcc.c-torture/compile/pr82549.c: New test.
+
+2017-10-17  Martin Liska  <mliska@suse.cz>
+
+	* lib/scanasm.exp: Print how many times a regex pattern is
+	found.
+	* lib/scandump.exp: Likewise.
+
+2017-10-17  Olga Makhotina  <olga.makhotina@intel.com>
+
+	* gcc.target/i386/avx512dq-vreducesd-1.c (_mm_mask_reduce_sd,
+	_mm_maskz_reduce_sd): Test new intrinsics.
+	* gcc.target/i386/avx512dq-vreducesd-2.c: New.
+	* gcc.target/i386/avx512dq-vreducess-1.c (_mm_mask_reduce_ss,
+	_mm_maskz_reduce_ss): Test new intrinsics.
+	* gcc.target/i386/avx512dq-vreducess-2.c: New.
+	* gcc.target/i386/avx-1.c (__builtin_ia32_reducesd,
+	__builtin_ia32_reducess): Remove builtin.
+	(__builtin_ia32_reducesd_mask,
+	__builtin_ia32_reducess_mask): Test new builtin.
+	* gcc.target/i386/sse-13.c: Ditto.
+	* gcc.target/i386/sse-23.c: Ditto.
+
+2017-10-16  Martin Liska  <mliska@suse.cz>
+
+	* c-c++-common/ubsan/attrib-5.c (float_cast2): Fix warning scan
+	so that it will work for both C and C++ FEs.
+
+2017-10-16  Fritz Reese <fritzoreese@gmail.com>
+
+	PR fortran/82511
+	* gfortran.dg/dec_structure_22.f90: New testcase.
+
+2017-10-16  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/64931
+	* g++.dg/cpp1y/auto-fn43.C: New.
+
+2017-10-16  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	PR target/82442
+	* gcc.dg/vect/pr31699.c: Fix testcase.
+
+2017-10-16  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.target/aarch64/advsimd-intrinsics/vect-dot-qi.h: New.
+	* gcc.target/aarch64/advsimd-intrinsics/vdot-compile.c: New.
+	* gcc.target/aarch64/advsimd-intrinsics/vect-dot-s8.c: New.
+	* gcc.target/aarch64/advsimd-intrinsics/vect-dot-u8.c: New.
+
+2017-10-16  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/53574
+	* g++.dg/other/pr53574.C: New test.
+
+2017-10-16  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/61323
+	* g++.dg/cpp0x/constexpr-61323.C: New.
+
+2017-10-15  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/54090
+	* g++.dg/template/crash128.C: New.
+
+2017-10-15  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	PR fortran/82372
+	* gfortran.dg/illegal_char.f90: New test.
+
+2017-10-14  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>
+	    Michael Collison <michael.collison@arm.com>
+
+	* gcc.target/aarch64/cmpelim_mult_uses_1.c: New test.
+
+2017-10-14  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/80908
+	* g++.dg/cpp1z/noexcept-type18.C: New.
+
+2017-10-14  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/81016
+	* g++.dg/cpp1z/pr81016.C: New.
+
+2017-10-14  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/62263
+	PR middle-end/82498
+	* c-c++-common/rotate-8.c: Expect no PHIs in optimized dump.
+
+	PR middle-end/62263
+	PR middle-end/82498
+	* c-c++-common/rotate-5.c (f2): New function.  Move old
+	function to ...
+	(f4): ... this.  Use 127 instead of 128.
+	(f3, f5, f6): New functions.
+	(main): Test all f[1-6] functions, with both 0 and 1 as
+	second arguments.
+	* c-c++-common/rotate-6.c: New test.
+	* c-c++-common/rotate-6a.c: New test.
+	* c-c++-common/rotate-7.c: New test.
+	* c-c++-common/rotate-7a.c: New test.
+	* c-c++-common/rotate-8.c: New test.
+
+2017-10-14  Hristian Kirtchev  <kirtchev@adacore.com>
+
+	* gnat.dg/remote_call_iface.ads, gnat.dg/remote_call_iface.adb: New
+	testcase.
+
+2017-10-14  Jakub Jelinek  <jakub@redhat.com>
+
+	PR rtl-optimization/81423
+	* gcc.c-torture/execute/pr81423.c (foo): Add missing cast.  Change L
+	suffixes to LL.
+	(main): Punt if either long long isn't 64-bit or int isn't 32-bit.
+
+2017-10-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR sanitizer/82353
+	* g++.dg/ubsan/pr82353-2.C: New test.
+	* g++.dg/ubsan/pr82353-2-aux.cc: New file.
+	* g++.dg/ubsan/pr82353-2.h: New file.
+
+2017-10-13  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/81048
+	* gfortran.dg/derived_init_4.f90 : New test.
+
+2017-10-13  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/69078
+	* g++.dg/cpp1y/lambda-generic-69078-1.C: New.
+	* g++.dg/cpp1y/lambda-generic-69078-2.C: Likewise.
+
+2017-10-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82274
+	* gcc.dg/pr82274-1.c: New test.
+	* gcc.dg/pr82274-2.c: New test.
+
+2017-10-13  Paolo Carlini  <paolo.carlini@oracle.com>
+
+	PR c++/80873
+	* g++.dg/cpp1y/auto-fn41.C: New.
+	* g++.dg/cpp1y/auto-fn42.C: Likewise.
+
+2017-10-13  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/cpp0x/udlit-extern-c.C: New test case.
+	* g++.dg/diagnostic/unclosed-extern-c.C: Add example of a template
+	erroneously covered by an unclosed extern "C".
+	* g++.dg/template/extern-c.C: New test case.
+
+2017-10-13  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/graphite/pr35356-3.c: XFAIL again.
+	* gcc.dg/graphite/pr81373-2.c: Copy from gcc.dg/graphite/pr81373.c
+	with alternate flags.
+
+2017-10-13  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/graphite/scop-10.c: Enlarge array to avoid undefined
+	behavior.
+	* gcc.dg/graphite/scop-7.c: Likewise.
+	* gcc.dg/graphite/scop-8.c: Likewise.
+
+2017-10-13  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/82499
+	* gcc.target/i386/pr82499-1.c: New file.
+	* gcc.target/i386/pr82499-2.c: Likewise.
+	* gcc.target/i386/pr82499-3.c: Likewise.
+
 2017-10-13  Jakub Jelinek  <jakub@redhat.com>
 
 	PR target/82524
@@ -11631,6 +12929,11 @@
 	PR lto/79587
 	* gcc.dg/tree-prof/pr79587.c: New test.
 
+2017-02-22  Bill Schmidt  <wschmidt@linux.vnet.ibm.com>
+
+	PR tree-optimization/68644
+	* gcc.dg/tree-ssa/ivopts-lt-2.c: Skip for powerpc*-*-*.
+
 2017-02-21  Marek Polacek  <polacek@redhat.com>
 
 	PR c++/79535
diff --git a/gcc/testsuite/c-c++-common/Wbuiltin-declaration-mismatch-1.c b/gcc/testsuite/c-c++-common/Wbuiltin-declaration-mismatch-1.c
new file mode 100644
index 00000000000..63343b8bfee
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wbuiltin-declaration-mismatch-1.c
@@ -0,0 +1,4 @@
+/* PR c++/82466 */
+/* { dg-options "-Wbuiltin-declaration-mismatch" } */
+
+int printf;  /* { dg-warning "declared as non-function" } */
diff --git a/gcc/testsuite/c-c++-common/Wno-builtin-declaration-mismatch-1.c b/gcc/testsuite/c-c++-common/Wno-builtin-declaration-mismatch-1.c
new file mode 100644
index 00000000000..6409412ac6a
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wno-builtin-declaration-mismatch-1.c
@@ -0,0 +1,4 @@
+/* PR c++/82466 */
+/* { dg-options "-Wno-builtin-declaration-mismatch" } */
+
+int printf;
diff --git a/gcc/testsuite/c-c++-common/attr-nocf-check-1.c b/gcc/testsuite/c-c++-common/attr-nocf-check-1.c
new file mode 100644
index 00000000000..15f69731b91
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/attr-nocf-check-1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+
+int func (int) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored" } */
+int (*fptr) (int) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored" } */
+typedef void (*nocf_check_t) (void) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored" } */
+
+int
+foo1 (int arg)
+{
+  return func (arg) + fptr (arg);
+}
+
+void
+foo2 (void (*foo) (void))
+{
+  void (*func) (void) __attribute__((nocf_check)) = foo; /* { dg-warning "'nocf_check' attribute ignored" } */
+  func ();
+}
+
+void
+foo3 (nocf_check_t foo)
+{
+  foo ();
+}
+
+void
+foo4 (void (*foo) (void) __attribute__((nocf_check))) /* { dg-warning "'nocf_check' attribute ignored" } */
+{
+  foo ();
+}
diff --git a/gcc/testsuite/c-c++-common/attr-nocf-check-2.c b/gcc/testsuite/c-c++-common/attr-nocf-check-2.c
new file mode 100644
index 00000000000..9ab01804782
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/attr-nocf-check-2.c
@@ -0,0 +1,5 @@
+/* { dg-do compile } */
+
+int var1 __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
+int *var2 __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
+void (**var3) (void) __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
diff --git a/gcc/testsuite/c-c++-common/attr-nocf-check-3.c b/gcc/testsuite/c-c++-common/attr-nocf-check-3.c
new file mode 100644
index 00000000000..ad1ca7eec9b
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/attr-nocf-check-3.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+
+int  foo (void) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored" } */
+void (*foo1) (void) __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored" } */
+void (*foo2) (void);
+
+int
+foo (void) /* The function's address is not tracked.  */
+{
+  /* This call site is not tracked for
+     control-flow instrumentation.  */
+  (*foo1)();
+
+  foo1 = foo2;
+  /* This call site is still not tracked for
+     control-flow instrumentation.  */
+  (*foo1)();
+
+  /* This call site is tracked for
+     control-flow instrumentation.  */
+  (*foo2)();
+
+  foo2 = foo1;
+  /* This call site is still tracked for
+     control-flow instrumentation.  */
+  (*foo2)();
+
+  return 0;
+}
diff --git a/gcc/testsuite/c-c++-common/fcf-protection-1.c b/gcc/testsuite/c-c++-common/fcf-protection-1.c
new file mode 100644
index 00000000000..2e9337c3051
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/fcf-protection-1.c
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection=full" } */
+/* { dg-error "'-fcf-protection=full' requires CET support on this target" "" { target { "i?86-*-* x86_64-*-*" } } 0 } */
+/* { dg-error "'-fcf-protection=full' is not supported for this target" "" { target { ! "i?86-*-* x86_64-*-*" } } 0 } */
diff --git a/gcc/testsuite/c-c++-common/fcf-protection-2.c b/gcc/testsuite/c-c++-common/fcf-protection-2.c
new file mode 100644
index 00000000000..aa0d2a04645
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/fcf-protection-2.c
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection=branch" } */
+/* { dg-error "'-fcf-protection=branch' requires CET support on this target" "" { target { "i?86-*-* x86_64-*-*" } } 0 } */
+/* { dg-error "'-fcf-protection=branch' is not supported for this target" "" { target { ! "i?86-*-* x86_64-*-*" } } 0 } */
diff --git a/gcc/testsuite/c-c++-common/fcf-protection-3.c b/gcc/testsuite/c-c++-common/fcf-protection-3.c
new file mode 100644
index 00000000000..028775adc35
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/fcf-protection-3.c
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection=return" } */
+/* { dg-error "'-fcf-protection=return' requires CET support on this target" "" { target { "i?86-*-* x86_64-*-*" } } 0 } */
+/* { dg-error "'-fcf-protection=return' is not supported for this target" "" { target { ! "i?86-*-* x86_64-*-*" } } 0 } */
diff --git a/gcc/testsuite/c-c++-common/fcf-protection-4.c b/gcc/testsuite/c-c++-common/fcf-protection-4.c
new file mode 100644
index 00000000000..af4fc0b2812
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/fcf-protection-4.c
@@ -0,0 +1,2 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection=none" } */
diff --git a/gcc/testsuite/c-c++-common/fcf-protection-5.c b/gcc/testsuite/c-c++-common/fcf-protection-5.c
new file mode 100644
index 00000000000..a5f8e116992
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/fcf-protection-5.c
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection" } */
+/* { dg-error "'-fcf-protection=full' requires CET support on this target" "" { target { "i?86-*-* x86_64-*-*" } } 0 } */
+/* { dg-error "'-fcf-protection=full' is not supported for this target" "" { target { ! "i?86-*-* x86_64-*-*" } } 0 } */
diff --git a/gcc/testsuite/c-c++-common/pr44515.c b/gcc/testsuite/c-c++-common/pr44515.c
new file mode 100644
index 00000000000..dbb7750907c
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/pr44515.c
@@ -0,0 +1,14 @@
+/* { dg-options "-fdiagnostics-show-caret" } */
+
+void bar(void);
+void foo(void)
+{
+  bar() /* { dg-error "expected ';' before '.' token" } */
+}
+/* { dg-begin-multiline-output "" }
+   bar()
+        ^
+        ;
+ }
+ ~       
+   { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/c-c++-common/rotate-5.c b/gcc/testsuite/c-c++-common/rotate-5.c
index 35b14b86c3a..629ab2f7274 100644
--- a/gcc/testsuite/c-c++-common/rotate-5.c
+++ b/gcc/testsuite/c-c++-common/rotate-5.c
@@ -15,12 +15,40 @@ f1 (unsigned long long x, unsigned int y)
   return (x << y) | (x >> ((-y) & 63));
 }
 
+__attribute__((noinline, noclone))
+unsigned long long
+f2 (unsigned long long x, unsigned int y)
+{
+  return (x << y) + (x >> ((-y) & 63));
+}
+
+__attribute__((noinline, noclone))
+unsigned long long
+f3 (unsigned long long x, unsigned int y)
+{
+  return (x << y) ^ (x >> ((-y) & 63));
+}
+
 #if __CHAR_BIT__ * __SIZEOF_INT128__ == 128
 __attribute__((noinline, noclone))
 unsigned __int128
-f2 (unsigned __int128 x, unsigned int y)
+f4 (unsigned __int128 x, unsigned int y)
+{
+  return (x << y) | (x >> ((-y) & 127));
+}
+
+__attribute__((noinline, noclone))
+unsigned __int128
+f5 (unsigned __int128 x, unsigned int y)
 {
-  return (x << y) | (x >> ((-y) & 128));
+  return (x << y) + (x >> ((-y) & 127));
+}
+
+__attribute__((noinline, noclone))
+unsigned __int128
+f6 (unsigned __int128 x, unsigned int y)
+{
+  return (x << y) ^ (x >> ((-y) & 127));
 }
 #endif
 #endif
@@ -31,12 +59,45 @@ main ()
 #if __CHAR_BIT__ * __SIZEOF_LONG_LONG__ == 64
   if (f1 (0x123456789abcdef0ULL, 0) != 0x123456789abcdef0ULL)
     abort ();
+  if (f2 (0x123456789abcdef0ULL, 0) != 0x2468acf13579bde0ULL)
+    abort ();
+  if (f3 (0x123456789abcdef0ULL, 0) != 0)
+    abort ();
+  if (f1 (0x123456789abcdef0ULL, 1) != 0x2468acf13579bde0ULL)
+    abort ();
+  if (f2 (0x123456789abcdef0ULL, 1) != 0x2468acf13579bde0ULL)
+    abort ();
+  if (f3 (0x123456789abcdef0ULL, 1) != 0x2468acf13579bde0ULL)
+    abort ();
 #if __CHAR_BIT__ * __SIZEOF_INT128__ == 128
-  if (f2 ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
+  if (f4 ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
 	  | 0x0fedcba987654321ULL, 0)
       != ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
           | 0x0fedcba987654321ULL))
     abort ();
+  if (f5 ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
+	  | 0x0fedcba987654321ULL, 0)
+      != ((((unsigned __int128) 0x2468acf13579bde0ULL) << 64)
+          | 0x1fdb97530eca8642ULL))
+    abort ();
+  if (f6 ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
+	  | 0x0fedcba987654321ULL, 0) != 0)
+    abort ();
+  if (f4 ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
+	  | 0x0fedcba987654321ULL, 1)
+      != ((((unsigned __int128) 0x2468acf13579bde0ULL) << 64)
+          | 0x1fdb97530eca8642ULL))
+    abort ();
+  if (f5 ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
+	  | 0x0fedcba987654321ULL, 1)
+      != ((((unsigned __int128) 0x2468acf13579bde0ULL) << 64)
+          | 0x1fdb97530eca8642ULL))
+    abort ();
+  if (f6 ((((unsigned __int128) 0x123456789abcdef0ULL) << 64)
+	  | 0x0fedcba987654321ULL, 1)
+      != ((((unsigned __int128) 0x2468acf13579bde0ULL) << 64)
+          | 0x1fdb97530eca8642ULL))
+    abort ();
 #endif
 #endif
   return 0;
diff --git a/gcc/testsuite/c-c++-common/rotate-6.c b/gcc/testsuite/c-c++-common/rotate-6.c
new file mode 100644
index 00000000000..715f8a48c93
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/rotate-6.c
@@ -0,0 +1,582 @@
+/* Check rotate pattern detection.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-ipa-icf -fdump-tree-optimized" } */
+/* Rotates should be recognized only in functions with | instead of + or ^,
+   or in functions that have constant shift counts (unused attribute on y).  */
+/* { dg-final { scan-tree-dump-times "r\[<>]\[<>]" 48 "optimized" } } */
+
+unsigned int
+f1 (unsigned int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f2 (unsigned int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f3 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) | (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f4 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> 1);
+}
+
+unsigned short int
+f5 (unsigned short int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f6 (unsigned short int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f7 (unsigned char x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f8 (unsigned char x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f9 (unsigned int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f10 (unsigned int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f11 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) | (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f12 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x >> 1);
+}
+
+unsigned short int
+f13 (unsigned short int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f14 (unsigned short int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f15 (unsigned char x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f16 (unsigned char x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f17 (unsigned int x, unsigned int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f18 (unsigned int x, unsigned long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f19 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x << 1);
+}
+
+unsigned int
+f20 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) ^ (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned short int
+f21 (unsigned short int x, unsigned int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f22 (unsigned short int x, unsigned long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f23 (unsigned char x, unsigned int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ - 1))) ^ (x << (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f24 (unsigned char x, unsigned long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ - 1))) ^ (x << (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f25 (unsigned int x, unsigned int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f26 (unsigned int x, unsigned long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f27 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x << 1);
+}
+
+unsigned int
+f28 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) ^ (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned short int
+f29 (unsigned short int x, unsigned int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f30 (unsigned short int x, unsigned long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f31 (unsigned char x, unsigned int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f32 (unsigned char x, unsigned long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f33 (unsigned int x, unsigned int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f34 (unsigned int x, unsigned long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f35 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) | (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f36 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x << 1);
+}
+
+unsigned short int
+f37 (unsigned short int x, unsigned int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f38 (unsigned short int x, unsigned long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f39 (unsigned char x, unsigned int y)
+{
+  return (x >> (y & (__CHAR_BIT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f40 (unsigned char x, unsigned long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f41 (unsigned int x, unsigned int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f42 (unsigned int x, unsigned long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f43 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) | (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f44 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x << 1);
+}
+
+unsigned short int
+f45 (unsigned short int x, unsigned int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f46 (unsigned short int x, unsigned long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f47 (unsigned char x, unsigned int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f48 (unsigned char x, unsigned long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f49 (unsigned int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f50 (unsigned int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f51 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x >> 1);
+}
+
+unsigned int
+f52 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) ^ (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned short int
+f53 (unsigned short int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f54 (unsigned short int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f55 (unsigned char x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f56 (unsigned char x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f57 (unsigned int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f58 (unsigned int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f59 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x >> 1);
+}
+
+unsigned int
+f60 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) ^ (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned short int
+f61 (unsigned short int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f62 (unsigned short int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f63 (unsigned char x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f64 (unsigned char x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f65 (unsigned int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f66 (unsigned int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f67 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f68 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> 1);
+}
+
+unsigned short int
+f69 (unsigned short int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f70 (unsigned short int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f71 (unsigned char x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f72 (unsigned char x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f73 (unsigned int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f74 (unsigned int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f75 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f76 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> 1);
+}
+
+unsigned short int
+f77 (unsigned short int x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f78 (unsigned short int x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f79 (unsigned char x, unsigned int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f80 (unsigned char x, unsigned long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f81 (unsigned int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f82 (unsigned int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f83 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> 1);
+}
+
+unsigned int
+f84 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned short int
+f85 (unsigned short int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f86 (unsigned short int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f87 (unsigned char x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) + (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f88 (unsigned char x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) + (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f89 (unsigned int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f90 (unsigned int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f91 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> 1);
+}
+
+unsigned int
+f92 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned short int
+f93 (unsigned short int x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f94 (unsigned short int x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f95 (unsigned char x, unsigned int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f96 (unsigned char x, unsigned long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
diff --git a/gcc/testsuite/c-c++-common/rotate-6a.c b/gcc/testsuite/c-c++-common/rotate-6a.c
new file mode 100644
index 00000000000..06ba56a5dde
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/rotate-6a.c
@@ -0,0 +1,6 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -Wno-overflow" } */
+
+#define ROTATE_N "rotate-6.c"
+
+#include "rotate-1a.c"
diff --git a/gcc/testsuite/c-c++-common/rotate-7.c b/gcc/testsuite/c-c++-common/rotate-7.c
new file mode 100644
index 00000000000..390cef680d9
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/rotate-7.c
@@ -0,0 +1,582 @@
+/* Check rotate pattern detection.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-ipa-icf -fdump-tree-optimized" } */
+/* Rotates should be recognized only in functions with | instead of + or ^,
+   or in functions that have constant shift counts (unused attribute on y).  */
+/* { dg-final { scan-tree-dump-times "r\[<>]\[<>]" 48 "optimized" } } */
+
+unsigned int
+f1 (unsigned int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f2 (unsigned int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f3 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) | (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f4 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> 1);
+}
+
+unsigned short int
+f5 (unsigned short int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f6 (unsigned short int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f7 (unsigned char x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f8 (unsigned char x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f9 (unsigned int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f10 (unsigned int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f11 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) | (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f12 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x >> 1);
+}
+
+unsigned short int
+f13 (unsigned short int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f14 (unsigned short int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f15 (unsigned char x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f16 (unsigned char x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f17 (unsigned int x, int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f18 (unsigned int x, long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f19 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x << 1);
+}
+
+unsigned int
+f20 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) ^ (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned short int
+f21 (unsigned short int x, int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f22 (unsigned short int x, long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f23 (unsigned char x, int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ - 1))) ^ (x << (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f24 (unsigned char x, long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ - 1))) ^ (x << (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f25 (unsigned int x, int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f26 (unsigned int x, long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f27 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x << 1);
+}
+
+unsigned int
+f28 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) ^ (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned short int
+f29 (unsigned short int x, int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f30 (unsigned short int x, long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f31 (unsigned char x, int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f32 (unsigned char x, long int y)
+{
+  return (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f33 (unsigned int x, int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f34 (unsigned int x, long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f35 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) | (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f36 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x << 1);
+}
+
+unsigned short int
+f37 (unsigned short int x, int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f38 (unsigned short int x, long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f39 (unsigned char x, int y)
+{
+  return (x >> (y & (__CHAR_BIT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f40 (unsigned char x, long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ - 1))) | (x << ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f41 (unsigned int x, int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f42 (unsigned int x, long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f43 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> 1) | (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f44 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) | (x << 1);
+}
+
+unsigned short int
+f45 (unsigned short int x, int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f46 (unsigned short int x, long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f47 (unsigned char x, int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f48 (unsigned char x, long int y)
+{
+  return (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) | (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f49 (unsigned int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f50 (unsigned int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f51 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) ^ (x >> 1);
+}
+
+unsigned int
+f52 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) ^ (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned short int
+f53 (unsigned short int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f54 (unsigned short int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f55 (unsigned char x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f56 (unsigned char x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) ^ (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f57 (unsigned int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f58 (unsigned int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f59 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) ^ (x >> 1);
+}
+
+unsigned int
+f60 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) ^ (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned short int
+f61 (unsigned short int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f62 (unsigned short int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f63 (unsigned char x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f64 (unsigned char x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) ^ (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f65 (unsigned int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f66 (unsigned int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f67 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f68 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> 1);
+}
+
+unsigned short int
+f69 (unsigned short int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f70 (unsigned short int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f71 (unsigned char x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f72 (unsigned char x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ - 1))) + (x >> ((-y) & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f73 (unsigned int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f74 (unsigned int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f75 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f76 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> 1);
+}
+
+unsigned short int
+f77 (unsigned short int x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f78 (unsigned short int x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f79 (unsigned char x, int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f80 (unsigned char x, long int y)
+{
+  return (x << (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned int
+f81 (unsigned int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f82 (unsigned int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f83 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) + (x >> 1);
+}
+
+unsigned int
+f84 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned short int
+f85 (unsigned short int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned short int
+f86 (unsigned short int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1))) + (x >> (y & (__CHAR_BIT__ * __SIZEOF_SHORT__ - 1)));
+}
+
+unsigned char
+f87 (unsigned char x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) + (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned char
+f88 (unsigned char x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ - 1))) + (x >> (y & (__CHAR_BIT__ - 1)));
+}
+
+unsigned int
+f89 (unsigned int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f90 (unsigned int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned int
+f91 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1))) + (x >> 1);
+}
+
+unsigned int
+f92 (unsigned int x, int y __attribute__((unused)))
+{
+  return (x << 1) + (x >> ((-1) & (__CHAR_BIT__ * sizeof (unsigned int) - 1)));
+}
+
+unsigned short int
+f93 (unsigned short int x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned short int
+f94 (unsigned short int x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned short) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned short) - 1)));
+}
+
+unsigned char
+f95 (unsigned char x, int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
+
+unsigned char
+f96 (unsigned char x, long int y)
+{
+  return (x << ((-y) & (__CHAR_BIT__ * sizeof (unsigned char) - 1))) + (x >> (y & (__CHAR_BIT__ * sizeof (unsigned char) - 1)));
+}
diff --git a/gcc/testsuite/c-c++-common/rotate-7a.c b/gcc/testsuite/c-c++-common/rotate-7a.c
new file mode 100644
index 00000000000..4fb08465403
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/rotate-7a.c
@@ -0,0 +1,6 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -Wno-overflow" } */
+
+#define ROTATE_N "rotate-7.c"
+
+#include "rotate-1a.c"
diff --git a/gcc/testsuite/c-c++-common/rotate-8.c b/gcc/testsuite/c-c++-common/rotate-8.c
new file mode 100644
index 00000000000..9ba3e940930
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/rotate-8.c
@@ -0,0 +1,171 @@
+/* PR middle-end/62263 */
+/* PR middle-end/82498 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-ipa-icf -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times "r\[<>]\[<>]" 23 "optimized" } } */
+/* { dg-final { scan-tree-dump-not "PHI <" "optimized" } } */
+
+unsigned int
+f1 (unsigned int x, unsigned char y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned int
+f2 (unsigned int x, signed char y)
+{
+  y &= __CHAR_BIT__ * __SIZEOF_INT__ - 1;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned int
+f3 (unsigned int x, unsigned char y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))));
+}
+
+unsigned int
+f4 (unsigned int x, unsigned char y)
+{
+  y = y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1);
+  return y ? (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y)) : x;
+}
+
+unsigned int
+f5 (unsigned int x, unsigned char y)
+{
+  y = y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1);
+  return (x << y) | (x >> ((__CHAR_BIT__ * __SIZEOF_INT__ - y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f6 (unsigned int x, unsigned char y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> ((__CHAR_BIT__ * __SIZEOF_INT__ - (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f7 (unsigned int x, unsigned char y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> ((__CHAR_BIT__ * __SIZEOF_INT__ - y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f8 (unsigned int x, unsigned char y)
+{
+  return (x << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (x >> ((-y) & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f9 (unsigned int x, int y)
+{
+  return (0x12345678U << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (0x12345678U >> (-y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f10 (unsigned int x, int y)
+{
+  return (0x12345678U >> (-y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (0x12345678U << (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f11 (unsigned int x, int y)
+{
+  return (0x12345678U >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (0x12345678U << (-y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned int
+f12 (unsigned int x, int y)
+{
+  return (0x12345678U << (-y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1))) | (0x12345678U >> (y & (__CHAR_BIT__ * __SIZEOF_INT__ - 1)));
+}
+
+unsigned
+f13 (unsigned x, unsigned char y)
+{
+  if (y == 0)
+    return x;
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned
+f14 (unsigned x, unsigned y)
+{
+  if (y == 0)
+    return x;
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned
+f15 (unsigned x, unsigned short y)
+{
+  if (y == 0)
+    return x;
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned
+f16 (unsigned x, unsigned char y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  if (y == 0)
+    return x;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned
+f17 (unsigned x, unsigned y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  if (y == 0)
+    return x;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned
+f18 (unsigned x, unsigned short y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  if (y == 0)
+    return x;
+  return (x << y) | (x >> (__CHAR_BIT__ * __SIZEOF_INT__ - y));
+}
+
+unsigned
+f19 (unsigned x, unsigned char y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (((unsigned char) -y) % (__CHAR_BIT__ * __SIZEOF_INT__)));
+}
+
+unsigned
+f20 (unsigned x, unsigned int y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (-y % (__CHAR_BIT__ * __SIZEOF_INT__)));
+}
+
+unsigned
+f21 (unsigned x, unsigned short y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (((unsigned short) -y) % (__CHAR_BIT__ * __SIZEOF_INT__)));
+}
+
+unsigned
+f22 (unsigned x, unsigned char y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (-y & ((__CHAR_BIT__ * __SIZEOF_INT__) - 1)));
+}
+
+unsigned
+f23 (unsigned x, unsigned short y)
+{
+  y %= __CHAR_BIT__ * __SIZEOF_INT__;
+  return (x << y) | (x >> (-y & ((__CHAR_BIT__ * __SIZEOF_INT__) - 1)));
+}
diff --git a/gcc/testsuite/c-c++-common/ubsan/attrib-5.c b/gcc/testsuite/c-c++-common/ubsan/attrib-5.c
index fee1df1c433..209b5dd7d2b 100644
--- a/gcc/testsuite/c-c++-common/ubsan/attrib-5.c
+++ b/gcc/testsuite/c-c++-common/ubsan/attrib-5.c
@@ -3,8 +3,7 @@
 
 __attribute__((no_sanitize("foobar")))
 static void
-float_cast2 (void)
-{ /* { dg-warning "attribute directive ignored" } */
+float_cast2 (void) { /* { dg-warning "attribute directive ignored" } */
   volatile double d = 300;
   volatile signed char c;
   c = d;
diff --git a/gcc/testsuite/c-c++-common/ubsan/builtin-1.c b/gcc/testsuite/c-c++-common/ubsan/builtin-1.c
new file mode 100644
index 00000000000..2f340e3e70f
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/ubsan/builtin-1.c
@@ -0,0 +1,36 @@
+/* { dg-do run } */
+/* { dg-options "-fsanitize=undefined" } */
+
+#include <stdio.h>
+
+__attribute__((noinline, noclone)) unsigned long long
+foo (unsigned int x, unsigned long int y, unsigned long long int z, __UINTMAX_TYPE__ w)
+{
+  unsigned long long ret = 0;
+  fprintf (stderr, "FOO MARKER1\n");
+  ret += __builtin_ctz (x);
+  ret += __builtin_ctzl (y);
+  ret += __builtin_ctzll (z);
+  ret += __builtin_ctzimax (w);
+  fprintf (stderr, "FOO MARKER2\n");
+  ret += __builtin_clz (x);
+  ret += __builtin_clzl (y);
+  ret += __builtin_clzll (z);
+  ret += __builtin_clzimax (w);
+  fprintf (stderr, "FOO MARKER3\n");
+  return ret;
+}
+
+int
+main ()
+{
+  volatile __UINTMAX_TYPE__ t = 0;
+  t = foo (t, t, t, t);
+  return 0;
+}
+
+/* { dg-output "FOO MARKER1(\n|\r\n|\r)" } */
+/* { dg-output "(\[^\n\r]*runtime error: passing zero to ctz\\\(\\\), which is not a valid argument\[^\n\r]*(\n|\r\n|\r)){4}" } */
+/* { dg-output "FOO MARKER2(\n|\r\n|\r)" } */
+/* { dg-output "(\[^\n\r]*runtime error: passing zero to clz\\\(\\\), which is not a valid argument\[^\n\r]*(\n|\r\n|\r)){4}" } */
+/* { dg-output "FOO MARKER3" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-1.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-1.c
index aae88aa3180..8139cc1723f 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-1.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-1.c
@@ -91,115 +91,115 @@ main (void)
   return 0;
 }
 
-/* { dg-output "value -133 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -129.5 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 128 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 128.5 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 132 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 256 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 256.5 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 260 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -32773 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -32769.5 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 32768 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 32768.5 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 32772 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 65536 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 65536.5 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 65540 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type 'long long unsigned int'" } */
+/* { dg-output " -133 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -129.5 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 128 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 128.5 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 132 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 256 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 256.5 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 260 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -32773 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -32769.5 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 32768 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 32768.5 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 32772 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 65536 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 65536.5 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 65540 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 9.22337e\\\+18 is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.84467e\\\+19 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type 'long long unsigned int'" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-10.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-10.c
index a54a838870b..a4e8ec457b5 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-10.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-10.c
@@ -9,38 +9,38 @@
 #include "float-cast-overflow-8.c"
 
 /* _Decimal32 */
-/* { dg-output "value <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output " <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
 /* _Decimal64 */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
 /* _Decimal128 */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-2.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-2.c
index b25e312b61b..426c625fc6b 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-2.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-2.c
@@ -30,44 +30,44 @@ main (void)
   return 0;
 }
 
-/* { dg-output "runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value nan is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value -?nan is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value inf is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value -inf is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value -5 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value -1.5 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value nan is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value -?nan is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value inf is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*runtime error: value -inf is outside the range of representable values of type '__int128 unsigned'" } */
+/* { dg-output "runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 1.70141e\\\+38 is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: nan is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: -?nan is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: inf is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: -inf is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: 3.40282e\\\+38 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: -5 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: -1.5 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: nan is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: -?nan is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: inf is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]*runtime error: -inf is outside the range of representable values of type '__int128 unsigned'" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-3.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-3.c
index ba82111a4df..6567ca9a444 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-3.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-3.c
@@ -26,15 +26,15 @@ main (void)
   return 0;
 }
 
-/* { dg-output "value -133* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -129.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -129 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 128 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 128.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 132 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 256 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 256.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 260 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type" } */
+/* { dg-output " -133* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -129.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -129 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 128 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 128.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 132 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 256 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 256.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 260 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-4.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-4.c
index af76e4a3343..48ad257c641 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-4.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-4.c
@@ -30,23 +30,23 @@ main (void)
   return 0;
 }
 
-/* { dg-output "value -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -?nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value inf is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -inf is outside the range of representable values of type" } */
+/* { dg-output " -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -?nan is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* inf is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -inf is outside the range of representable values of type" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-5.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-5.c
index 4c2fbb4d9ea..25a94950970 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-5.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-5.c
@@ -26,15 +26,15 @@ main (void)
   return 0;
 }
 
-/* { dg-output "value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[^\n\r]* is outside the range of representable values of type" } */
+/* { dg-output " \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[^\n\r]* is outside the range of representable values of type" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-6.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-6.c
index a2b5f9a28ce..90ec26838f8 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-6.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-6.c
@@ -26,15 +26,15 @@ main (void)
   return 0;
 }
 
-/* { dg-output "value -133 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -129.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -129 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 128 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 128.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 132 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 256 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 256.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 260 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type" } */
+/* { dg-output " -133 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -129.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -129 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 128 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 128.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 132 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 256 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 256.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 260 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type" } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-8.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-8.c
index 4adb22ae3b4..4e7beeb08db 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-8.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-8.c
@@ -99,45 +99,45 @@ main ()
 }
 
 /* float */
-/* { dg-output "value -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
+/* { dg-output " -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
 /* No error for float and __int128 unsigned max value, as ui128_MAX is +Inf in float.  */
 /* double */
-/* { dg-output "\[^\n\r]*value -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
+/* { dg-output "\[^\n\r]* -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
 /* long double */
-/* { dg-output "\[^\n\r]*value -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
+/* { dg-output "\[^\n\r]* -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" { target { ilp32 || lp64 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target { int128 } } } */
diff --git a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-9.c b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-9.c
index f2d71f6a533..ca9b425d23e 100644
--- a/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-9.c
+++ b/gcc/testsuite/c-c++-common/ubsan/float-cast-overflow-9.c
@@ -6,30 +6,30 @@
 #include "float-cast-overflow-8.c"
 
 /* __float80 */
-/* { dg-output "value -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
+/* { dg-output " -129 is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* (-129|-1) is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -32769 is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* \[0-9.e+-]* is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
 /* __float128 */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
-/* { dg-output "\[^\n\r]*value <unknown> is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'signed char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned char'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'short unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type 'long long unsigned int'\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type '__int128'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
+/* { dg-output "\[^\n\r]* <unknown> is outside the range of representable values of type '__int128 unsigned'\[^\n\r]*(\n|\r\n|\r)" { target int128 } } */
diff --git a/gcc/testsuite/g++.dg/asan/asan_test.C b/gcc/testsuite/g++.dg/asan/asan_test.C
index 410e4ce72d4..f3f7626ef3b 100644
--- a/gcc/testsuite/g++.dg/asan/asan_test.C
+++ b/gcc/testsuite/g++.dg/asan/asan_test.C
@@ -8,6 +8,7 @@
 // { dg-additional-options "-DASAN_AVOID_EXPENSIVE_TESTS=1" { target { ! run_expensive_tests } } }
 // { dg-additional-options "-msse2" { target { i?86-*-linux* x86_64-*-linux* } } }
 // { dg-additional-options "-D__NO_INLINE__" { target { *-*-linux-gnu } } }
+// { dg-set-target-env-var ASAN_OPTIONS "handle_segv=2" }
 // { dg-final { asan-gtest } }
 
 #include "asan_test.cc"
diff --git a/gcc/testsuite/g++.dg/asan/default-options-1.C b/gcc/testsuite/g++.dg/asan/default-options-1.C
index dc818917ddc..98abdfbd3ff 100644
--- a/gcc/testsuite/g++.dg/asan/default-options-1.C
+++ b/gcc/testsuite/g++.dg/asan/default-options-1.C
@@ -3,7 +3,7 @@
 const char *kAsanDefaultOptions="verbosity=1 foo=bar";
 
 extern "C"
-__attribute__((no_sanitize_address))
+__attribute__((no_sanitize_address, used))
 const char *__asan_default_options() {
   return kAsanDefaultOptions;
 }
diff --git a/gcc/testsuite/g++.dg/cet-notrack-1.C b/gcc/testsuite/g++.dg/cet-notrack-1.C
new file mode 100644
index 00000000000..43dbbd6a7f3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cet-notrack-1.C
@@ -0,0 +1,25 @@
+/* { dg-do compile { target i?86-*-* x86_64-*-* } } */
+/* { dg-options "-fcf-protection -mcet" } */
+/* { dg-final { scan-assembler "endbr32|endbr64" } } */
+/* { dg-final { scan-assembler-times "\tcall\[ \t]+puts" 2 } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+" 1 } } */
+#include <stdio.h>
+
+struct A {
+virtual int foo() __attribute__((nocf_check)) { return 42; }
+};
+
+struct B : A {
+int foo() __attribute__((nocf_check)) { return 73; }
+};
+
+int main() {
+B b;
+A& a = b;
+int (A::*amem) () __attribute__((nocf_check)) = &A::foo; // take address
+if ((a.*amem)() == 73) // use the address
+  printf("pass\n");
+else
+  printf("fail\n");
+return 0;
+}
diff --git a/gcc/testsuite/g++.dg/concepts/pr67595.C b/gcc/testsuite/g++.dg/concepts/pr67595.C
new file mode 100644
index 00000000000..63162fb4c72
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/pr67595.C
@@ -0,0 +1,13 @@
+// { dg-options "-std=c++17 -fconcepts" }
+
+template <class X> concept bool allocatable = requires{{new X}->X * };
+template <class X> concept bool semiregular = allocatable<X>;
+template <class X> concept bool readable = requires{requires semiregular<X>};
+template <class> int weak_input_iterator = requires{{0}->readable};
+template <class X> bool input_iterator{weak_input_iterator<X>};
+template <class X> bool forward_iterator{input_iterator<X>};
+template <class X> bool bidirectional_iterator{forward_iterator<X>};
+template <class X>
+concept bool random_access_iterator{bidirectional_iterator<X>};
+void fn1(random_access_iterator);
+int main() { fn1(0); }  // { dg-error "" }
diff --git a/gcc/testsuite/g++.dg/concepts/pr71368.C b/gcc/testsuite/g++.dg/concepts/pr71368.C
new file mode 100644
index 00000000000..f0e0a956366
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/pr71368.C
@@ -0,0 +1,25 @@
+// { dg-options "-std=c++17 -fconcepts" }
+
+struct inner;
+
+template<typename X> concept bool CompoundReq = requires {
+    // fine with concrete type in trailing type, i.e. inner& instead of X&
+    { X::inner_member() } -> X&;
+};
+
+template<typename X> concept bool Concept = requires {
+    { X::outer_member() } -> CompoundReq;
+};
+
+struct inner { static inner& inner_member(); };
+struct outer { static inner outer_member(); };
+
+int main()
+{
+    // fine
+    static_assert( CompoundReq<inner> );
+    static_assert( CompoundReq<decltype( outer::outer_member() )> );
+
+    // ICE
+    static_assert( Concept<outer> );
+}
diff --git a/gcc/testsuite/g++.dg/concepts/pr71385.C b/gcc/testsuite/g++.dg/concepts/pr71385.C
new file mode 100644
index 00000000000..bd5d08cb6f0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/pr71385.C
@@ -0,0 +1,12 @@
+// { dg-options "-std=c++17 -fconcepts" }
+
+template<class T>
+concept bool Addable(){
+ return requires(T x){
+  {x + x} -> T;
+ };
+}
+
+int main(){
+ Addable t = 0;
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/alignas12.C b/gcc/testsuite/g++.dg/cpp0x/alignas12.C
new file mode 100644
index 00000000000..bc163441529
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/alignas12.C
@@ -0,0 +1,6 @@
+// PR c++/71821
+// { dg-do compile { target c++11 } }
+
+template < typename > constexpr int f () {  return 4; }
+
+alignas (f < int >) char c;  // { dg-error "non-integral type" }
diff --git a/gcc/testsuite/g++.dg/cpp0x/auto21.C b/gcc/testsuite/g++.dg/cpp0x/auto21.C
index a827b3df853..346e98c254e 100644
--- a/gcc/testsuite/g++.dg/cpp0x/auto21.C
+++ b/gcc/testsuite/g++.dg/cpp0x/auto21.C
@@ -1,5 +1,5 @@
 // Origin PR c++/47208
 // { dg-do compile { target c++11 } }
 
-constexpr auto list = { }; // { dg-error "deducing from brace-enclosed initializer list requires #include <initializer_list>" }
+constexpr auto list = { }; // { dg-error "deducing from brace-enclosed initializer list requires '#include <initializer_list>'" }
 static const int l = list.size();
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-61323.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-61323.C
new file mode 100644
index 00000000000..f194bb8be82
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-61323.C
@@ -0,0 +1,26 @@
+// PR c++/61323
+// { dg-do compile { target c++11 } }
+
+char* table1[10];
+template<unsigned size, char*(&table)[size]> void test1() { }
+void tester1() { test1<10,table1>(); }
+
+static char* table2[10];
+template<unsigned size, char*(&table)[size]> void test2() { }
+void tester2() { test2<10,table2>(); }
+
+const char* table3[10];
+template<unsigned size, const char*(&table)[size]> void test3() { }
+void tester3() { test3<10,table3>(); }
+
+const char* const table4[10] = {};
+template<unsigned size, const char*const (&table)[size]> void test4() { }
+void tester4() { test4<10,table4>(); }
+
+const char* volatile table5[10] = {};
+template<unsigned size, const char* volatile (&table)[size]> void test5() { }
+void tester5() { test5<10,table5>(); }
+
+const char* const table6[10] = {};
+template<unsigned size, const char*const (&table)[size]> void test6() { }
+void tester6() { test6<10,table6>(); }
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-ice18.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-ice18.C
new file mode 100644
index 00000000000..0b5ff701306
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-ice18.C
@@ -0,0 +1,11 @@
+// PR c++/67831
+// { dg-do compile { target c++11 } }
+
+struct Task {
+  struct TaskStaticData {
+    constexpr TaskStaticData() {}
+  } const &tsd;
+  constexpr Task() : tsd(TaskStaticData()) {}
+};
+
+Task tasks{Task()};
diff --git a/gcc/testsuite/g++.dg/cpp0x/enum35.C b/gcc/testsuite/g++.dg/cpp0x/enum35.C
new file mode 100644
index 00000000000..bcc1b26b390
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/enum35.C
@@ -0,0 +1,14 @@
+// PR c++/82307
+// { dg-do run { target c++11 } }
+
+#include <cassert>
+
+enum : unsigned long long { VAL };
+
+bool foo (unsigned long long) { return true; }
+bool foo (int) { return false; }
+
+int main()
+{
+  assert (foo(VAL));
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/enum36.C b/gcc/testsuite/g++.dg/cpp0x/enum36.C
new file mode 100644
index 00000000000..4859670309f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/enum36.C
@@ -0,0 +1,14 @@
+// PR c++/82307
+// { dg-do run { target c++11 } }
+
+#include <cassert>
+
+enum : short { VAL };
+
+bool foo (int) { return true; }
+bool foo (unsigned long long) { return false; }
+
+int main()
+{
+  assert (foo (VAL));
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/missing-initializer_list-include.C b/gcc/testsuite/g++.dg/cpp0x/missing-initializer_list-include.C
index 8e803c82f24..7d72ec45de4 100644
--- a/gcc/testsuite/g++.dg/cpp0x/missing-initializer_list-include.C
+++ b/gcc/testsuite/g++.dg/cpp0x/missing-initializer_list-include.C
@@ -7,7 +7,7 @@
 
 void test (int i)
 {
-  auto a = { &i }; // { dg-error "deducing from brace-enclosed initializer list requires #include <initializer_list>" }
+  auto a = { &i }; // { dg-error "deducing from brace-enclosed initializer list requires '#include <initializer_list>'" }
 }
 
 /* Verify the output from -fdiagnostics-generate-patch.
diff --git a/gcc/testsuite/g++.dg/cpp0x/noexcept31.C b/gcc/testsuite/g++.dg/cpp0x/noexcept31.C
new file mode 100644
index 00000000000..c4c0e7dd466
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/noexcept31.C
@@ -0,0 +1,12 @@
+// PR c++/77369
+// { dg-do compile { target c++11 } }
+
+template<typename F> int caller(F f) noexcept(noexcept(f())) { f(); return 0; }
+
+void func1() noexcept { }
+
+void func2() { throw 1; }
+
+int instantiate_caller_with_func1 = caller(func1);
+
+static_assert( !noexcept(caller(func2)), "" );
diff --git a/gcc/testsuite/g++.dg/cpp0x/pr82560.C b/gcc/testsuite/g++.dg/cpp0x/pr82560.C
new file mode 100644
index 00000000000..3408bae518e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/pr82560.C
@@ -0,0 +1,28 @@
+// { dg-do run { target c++11 } }
+// PR82560, failed to destruct default arg inside new
+
+static int liveness = 0;
+
+struct Foo {
+
+  Foo (int) {
+    liveness++;
+  }
+
+  ~Foo() {
+    liveness--;
+  }
+
+};
+
+struct Bar {
+  Bar (Foo = 0) { }
+  ~Bar() { }
+};
+
+int main()
+{
+  delete new Bar();
+
+  return liveness != 0;;
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/pr82725.C b/gcc/testsuite/g++.dg/cpp0x/pr82725.C
new file mode 100644
index 00000000000..14cb6d897c9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/pr82725.C
@@ -0,0 +1,16 @@
+// { dg-do compile { target { { i?86-*-* x86_64-*-* } && c++11 } } }
+// { dg-require-effective-target pie }
+// { dg-options "-O2 -fpie -mtls-direct-seg-refs" }
+
+struct string
+{
+  __SIZE_TYPE__ length;
+  const char *ptr;
+};
+
+string
+tempDir ()
+{
+  thread_local string cache;
+  return cache;
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/udlit-extern-c.C b/gcc/testsuite/g++.dg/cpp0x/udlit-extern-c.C
new file mode 100644
index 00000000000..d47a49c3fa8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/udlit-extern-c.C
@@ -0,0 +1,7 @@
+// { dg-do compile { target c++11 } }
+
+extern "C" { // { dg-message "1: 'extern .C.' linkage started here" }
+
+constexpr double operator"" _deg ( double degrees ); // { dg-error "literal operator with C linkage" }
+
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/variadic-crash4.C b/gcc/testsuite/g++.dg/cpp0x/variadic-crash4.C
new file mode 100644
index 00000000000..2974fe933e1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/variadic-crash4.C
@@ -0,0 +1,14 @@
+// PR c++/68884
+// { dg-do compile { target c++11 } }
+
+namespace std {
+  template <typename _Tp, _Tp __v> struct A { static constexpr _Tp value = __v; };
+typedef A<bool, true> true_type;
+}
+template <int> struct VsA;
+template <class ValueType> struct ValueTemplate {
+  template <template <ValueType> class, class> struct IsInstanceOf;
+  template <template <ValueType> class TemplateA, ValueType... TypesA>
+  struct IsInstanceOf<TemplateA, TemplateA<TypesA...>> : std::true_type {};
+};
+bool foo = ValueTemplate<int>::IsInstanceOf<VsA, VsA<0>>::value;
diff --git a/gcc/testsuite/g++.dg/cpp0x/variadic-crash5.C b/gcc/testsuite/g++.dg/cpp0x/variadic-crash5.C
new file mode 100644
index 00000000000..6866f39975a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/variadic-crash5.C
@@ -0,0 +1,28 @@
+// PR c++/81957
+// { dg-do compile { target c++11 } }
+
+template <class T, T v>
+struct integral_constant { };
+
+struct f {
+  template<bool b, typename Int>
+  void operator()(integral_constant<bool,b>, Int i) {
+  }
+};
+
+template<bool...Bs, typename F, typename ...T>
+auto dispatch(F f, T...t) -> decltype(f(integral_constant<bool,Bs>()..., t...)) {
+  return f(integral_constant<bool,Bs>()..., t...);
+}
+
+template<bool...Bs, typename F, typename ...T>
+auto dispatch(F f, bool b, T...t) -> decltype(dispatch<Bs..., true>(f, t...)) {
+  if (b)
+    return dispatch<Bs..., true>(f, t...);
+  else
+    return dispatch<Bs..., false>(f, t...);
+}
+
+int main() {
+  dispatch(f(), true, 5);
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/auto-fn41.C b/gcc/testsuite/g++.dg/cpp1y/auto-fn41.C
new file mode 100644
index 00000000000..25a879da118
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/auto-fn41.C
@@ -0,0 +1,23 @@
+// PR c++/80873
+// { dg-do compile { target c++14 } }
+
+struct S {};
+
+auto overloaded(S &);
+
+template <typename T>
+int overloaded(T &) {
+    return 0;
+}
+
+template <typename T>
+auto returns_lambda(T &param) {
+	return [&] {
+		overloaded(param);  // { dg-error "before deduction" }
+	};
+}
+
+int main() {
+	S s;
+	returns_lambda(s);
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/auto-fn42.C b/gcc/testsuite/g++.dg/cpp1y/auto-fn42.C
new file mode 100644
index 00000000000..0f2b68efa42
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/auto-fn42.C
@@ -0,0 +1,21 @@
+// PR c++/80873
+// { dg-do compile { target c++14 } }
+
+struct Buffer {};
+
+auto parse(Buffer b);
+template <typename T> void parse(T target);
+
+template <typename T>
+auto field(T target) {
+	return [&] {
+		parse(target);
+	};
+}
+
+template <typename T>
+void parse(T target) {}
+
+auto parse(Buffer b) {
+	field(0);
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/auto-fn43.C b/gcc/testsuite/g++.dg/cpp1y/auto-fn43.C
new file mode 100644
index 00000000000..7256ecb0d01
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/auto-fn43.C
@@ -0,0 +1,13 @@
+// PR c++/64931
+// { dg-do compile { target c++14 } }
+
+template<typename T>
+struct S {
+  T data[32];
+};
+
+auto
+foo (S<int> & x)
+{
+  return x;
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/auto-fn44.C b/gcc/testsuite/g++.dg/cpp1y/auto-fn44.C
new file mode 100644
index 00000000000..e35215d64c7
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/auto-fn44.C
@@ -0,0 +1,12 @@
+// PR c++/79474
+// { dg-do compile { target c++14 } }
+
+struct Funject
+{  
+  operator auto() { return +[](bool b) {return b;}; }
+  operator auto() { return +[](bool b, bool, bool) {return b;}; }  // { dg-error "cannot be overloaded" }
+};
+
+Funject fun;
+auto bbb = fun(true);
+auto bbbb = fun(true, false, true);  // { dg-error "no match" }
diff --git a/gcc/testsuite/g++.dg/cpp1y/auto-fn45.C b/gcc/testsuite/g++.dg/cpp1y/auto-fn45.C
new file mode 100644
index 00000000000..a9c163dd736
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/auto-fn45.C
@@ -0,0 +1,27 @@
+// PR c++/69057
+// { dg-do compile { target c++14 } }
+
+#include <cassert>
+
+using GLenum = unsigned int;
+
+template <typename T>
+inline constexpr auto from_enum(const T& x) noexcept
+{
+    // Comment this line to prevent segmentation fault:
+    assert(true);
+    // ------------------------------------------------
+
+    return (GLenum)x;
+}
+
+enum class buffer_target : GLenum
+{
+    array
+};
+
+struct vbo
+{
+    static constexpr GLenum target_value{from_enum(buffer_target::array)};
+    GLenum x{target_value};
+};
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-80739.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-80739.C
new file mode 100644
index 00000000000..5bfa082866a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-80739.C
@@ -0,0 +1,20 @@
+// PR c++/80739
+// { dg-do compile { target c++14 } }
+
+using size_t = decltype(sizeof(0));
+template <class T> struct element {
+    constexpr element() noexcept: x0(0), x1(0), x2(0), x3(0) {}
+    T x0; int x1, x2, x3;
+};
+template <class T> struct container {
+    constexpr container() noexcept: data() {data = element<T>();}
+    element<T> data;
+};
+template <class T> constexpr bool test() {
+    return (container<T>(), true);
+}
+int main() {
+    constexpr bool tmp0 = test<int>();
+    constexpr bool tmp1 = test<size_t>();
+    return tmp0 && tmp1;
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-82218.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-82218.C
new file mode 100644
index 00000000000..06507a9f437
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-82218.C
@@ -0,0 +1,128 @@
+// PR c++/82218
+// { dg-do compile { target c++14 } }
+
+template<typename _Tp>
+struct identity
+{
+  typedef _Tp type;
+};
+
+template<typename _Tp>
+inline _Tp&&
+forward(typename identity<_Tp>::type&& __t)
+{ return __t; }
+
+template < typename T >
+class delegate;
+
+template < typename R, typename... Params >
+class delegate< R(Params...) > final
+{
+private:
+  using CallbackType = R (*)(void*, Params...);
+
+  using FunctionPtr = R (*)(Params...);
+
+  template < typename Object >
+  using MethodPtr = R (Object::*)(Params...);
+
+  template < typename Object >
+  using ConstMethodPtr = R (Object::*)(Params...) const;
+
+  void* obj_;
+  CallbackType cb_;
+
+  template < typename Object, MethodPtr< Object > Mptr >
+  constexpr static R invoke_method(void* obj, Params... params) noexcept(
+      noexcept((static_cast< Object* >(obj)->*Mptr)(params...)))
+  {
+    return (static_cast< Object* >(obj)->*Mptr)(params...);
+  }
+
+  template < typename Object, ConstMethodPtr< Object > Mptr >
+  constexpr static R invoke_method(void* obj, Params... params) noexcept(
+      noexcept((static_cast< Object* >(obj)->*Mptr)(params...)))
+  {
+    return (static_cast< Object* >(obj)->*Mptr)(params...);
+  }
+
+  template < FunctionPtr Fptr >
+  constexpr static R invoke_function(void*, Params... params) noexcept(
+      noexcept((*Fptr)(params...)))
+  {
+    return (*Fptr)(params...);
+  }
+
+  constexpr delegate(void* obj, CallbackType callback) noexcept : obj_(obj),
+                                                                  cb_(callback)
+  {
+  }
+
+  constexpr static R error_function(Params...)
+  {
+    while(1);
+  }
+
+public:
+  using base_type = delegate< R(Params...) >;
+
+  delegate()
+  {
+    *this = from< error_function >();
+  }
+
+  delegate(const base_type&) = default;
+  delegate(base_type&&)      = default;
+
+  base_type& operator=(const base_type&)  = default;
+  base_type& operator=(base_type&&)       = default;
+
+  template < typename Object, MethodPtr< Object > Mptr >
+  constexpr static auto from(Object& obj) noexcept
+  {
+    return delegate(&obj, &invoke_method< Object, Mptr >);
+  }
+
+  template < typename Object, ConstMethodPtr< Object > Mptr >
+  constexpr static auto from(Object& obj) noexcept
+  {
+    return delegate(&obj, &invoke_method< Object, Mptr >);
+  }
+
+  template < FunctionPtr Fptr >
+  constexpr static auto from() noexcept
+  {
+    static_assert(Fptr != nullptr, "Function pointer must not be null");
+
+    return delegate(nullptr, &invoke_function< Fptr >);
+  }
+
+  template < typename... Args >
+  constexpr auto operator()(Args&&... params) const
+      noexcept(noexcept((*cb_)(obj_, forward< Args >(params)...)))
+  {
+    return (*cb_)(obj_, forward< Args >(params)...);
+  }
+
+  constexpr bool valid() const noexcept
+  {
+    return (cb_ != &invoke_function< error_function >);
+  }
+
+  constexpr bool operator==(const delegate& other) const noexcept
+  {
+    return (obj_ == other.obj_) && (cb_ == other.cb_);
+  }
+
+  constexpr bool operator!=(const delegate& other) const noexcept
+  {
+    return (obj_ != other.obj_) || (cb_ != other.cb_);
+  }
+};
+
+delegate< void(void) > a;
+
+void test()
+{
+  a();
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-69078-1.C b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-69078-1.C
new file mode 100644
index 00000000000..3f10f82672d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-69078-1.C
@@ -0,0 +1,26 @@
+// PR c++/69078
+// { dg-do run { target c++14 } }
+// { dg-options "-Wall" }
+
+struct Class {
+    Class(void (*_param)()) : data(_param) {}
+    void (*data)();
+};
+
+void funUser(void (*test)(int)) {
+    test(60);
+}
+
+void user(Class& c, int i) {
+    (void)i;
+    if (!c.data) __builtin_abort();
+}
+
+void probe() {}
+
+int main() {
+    static Class instance = { probe };
+    funUser([](auto... p) {
+        user(instance, p...);
+    });
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-69078-2.C b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-69078-2.C
new file mode 100644
index 00000000000..318e0967250
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-69078-2.C
@@ -0,0 +1,21 @@
+// PR c++/69078
+// { dg-do run { target c++14 } }
+
+#include <cassert>
+
+template<typename F>
+void run( F &&f ) {
+  f(nullptr);
+}
+
+struct V {
+  int i;
+};
+
+int main() {
+  static V const s={2};
+  assert (s.i == 2);
+  run([](auto){
+      assert (s.i == 2);
+    });
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/var-templ56.C b/gcc/testsuite/g++.dg/cpp1y/var-templ56.C
new file mode 100644
index 00000000000..d0f762b8e11
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/var-templ56.C
@@ -0,0 +1,11 @@
+// PR c++/82085
+// { dg-do compile { target c++14 } }
+
+template <const char& V>
+using char_sequence_t = int;
+
+template <typename T>
+constexpr char name_of_v = 'x';
+
+template <typename T>
+using type = char_sequence_t<name_of_v<T>>;
diff --git a/gcc/testsuite/g++.dg/cpp1z/class-deduction45.C b/gcc/testsuite/g++.dg/cpp1z/class-deduction45.C
new file mode 100644
index 00000000000..3fe8dd33b79
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/class-deduction45.C
@@ -0,0 +1,24 @@
+// PR c++/82308
+// { dg-options -std=c++17 }
+
+template<typename, unsigned>
+struct array {};
+
+template <unsigned R>
+class X {
+public:
+  using T = array<int, R>;
+
+  enum class C : char { A, B };
+  X(T bounds, C c = C::B) : t(bounds) {}
+
+private:
+  T t;
+};
+
+int main()
+{
+  array<int, 2> a;
+  X    d{a};
+  X<2> e{a};
+}
diff --git a/gcc/testsuite/g++.dg/cpp1z/class-deduction46.C b/gcc/testsuite/g++.dg/cpp1z/class-deduction46.C
new file mode 100644
index 00000000000..cf38ed65fa8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/class-deduction46.C
@@ -0,0 +1,6 @@
+// PR c++/80449
+// { dg-options -std=c++17 }
+
+template<class S> struct C;
+template<> struct C<int> { C(int, int) {} };
+auto k = C{0, 0};  // { dg-error "cannot deduce" }
diff --git a/gcc/testsuite/g++.dg/cpp1z/constexpr-lambda18.C b/gcc/testsuite/g++.dg/cpp1z/constexpr-lambda18.C
new file mode 100644
index 00000000000..639018ba945
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/constexpr-lambda18.C
@@ -0,0 +1,30 @@
+// PR c++/82570
+// { dg-options "-std=c++17" }
+
+template< typename Body >
+inline void iterate(Body body)
+{
+	body(10);
+}
+
+template< typename Pred >
+inline void foo(Pred pred)
+{
+	iterate([&](int param)
+	{
+		if (pred(param))
+		{
+			unsigned char buf[4];
+			buf[0] = 0;
+			buf[1] = 1;
+			buf[2] = 2;
+			buf[3] = 3;
+		}
+	});
+}
+
+int main()
+{
+	foo([](int x) { return x > 0; });
+	return 0;
+}
diff --git a/gcc/testsuite/g++.dg/cpp1z/noexcept-type13.C b/gcc/testsuite/g++.dg/cpp1z/noexcept-type13.C
index 8eb3be0bd61..b51d7af2b11 100644
--- a/gcc/testsuite/g++.dg/cpp1z/noexcept-type13.C
+++ b/gcc/testsuite/g++.dg/cpp1z/noexcept-type13.C
@@ -5,7 +5,7 @@
 void foo () throw () {}		// { dg-bogus "mangled name" }
 
 template <class T>
-T bar (T x) { return x; }	// { dg-warning "mangled name" "" { target c++14_down } }
+T bar (T x) { return x; }
 
 void baz () {			// { dg-bogus "mangled name" }
   return (bar (foo)) ();
diff --git a/gcc/testsuite/g++.dg/cpp1z/noexcept-type18.C b/gcc/testsuite/g++.dg/cpp1z/noexcept-type18.C
new file mode 100644
index 00000000000..e01fd0a2030
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/noexcept-type18.C
@@ -0,0 +1,15 @@
+// { dg-options "-std=c++17" }
+
+template<typename T>
+struct S;
+
+template<bool IsNoexcept>
+struct S<void(*)() noexcept(IsNoexcept)> {
+	S() {}
+};
+
+void f() {}
+
+int main() {
+	S<decltype(&f)> {};
+}
diff --git a/gcc/testsuite/g++.dg/cpp1z/pr81016.C b/gcc/testsuite/g++.dg/cpp1z/pr81016.C
new file mode 100644
index 00000000000..4826fbfb775
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/pr81016.C
@@ -0,0 +1,4 @@
+// { dg-options "-std=c++17" }
+
+template <typename a, a> struct b;
+template <typename c> struct b<bool, c::d>; // { dg-error "template parameter" }
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/pr77363.C b/gcc/testsuite/g++.dg/debug/dwarf2/pr77363.C
index 47b71433815..cd06c360a98 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/pr77363.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/pr77363.C
@@ -1,9 +1,9 @@
 // PR debug/77363
 // { dg-options "-gdwarf-2 -dA -fno-merge-debug-strings" }
-// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type2\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_type" } }
-// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type3\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_type" } }
-// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type4\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_type" } }
-// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type5\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_type" } }
+// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type2\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*(\[^\n\r\]* DW_AT_decl_column\[^\n\r\]*\[\n\r]*)?\[^\n\r\]* DW_AT_type" } }
+// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type3\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*(\[^\n\r\]* DW_AT_decl_column\[^\n\r\]*\[\n\r]*)?\[^\n\r\]* DW_AT_type" } }
+// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type4\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*(\[^\n\r\]* DW_AT_decl_column\[^\n\r\]*\[\n\r]*)?\[^\n\r\]* DW_AT_type" } }
+// { dg-final { scan-assembler "DIE \\(\[^\n\r\]*\\) DW_TAG_typedef\[^\n\r\]*\[\n\r]*\[^\n\r\]*type5\[^\n\r\]* DW_AT_name\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_file\[^\n\r\]*\[\n\r]*\[^\n\r\]* DW_AT_decl_line\[^\n\r\]*\[\n\r]*(\[^\n\r\]* DW_AT_decl_column\[^\n\r\]*\[\n\r]*)?\[^\n\r\]* DW_AT_type" } }
 
 typedef unsigned short type1;
 typedef unsigned char type2;
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/typedef6.C b/gcc/testsuite/g++.dg/debug/dwarf2/typedef6.C
index 7945deadaa2..654eba023da 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/typedef6.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/typedef6.C
@@ -1,5 +1,5 @@
 // Origin PR debug/
-// { dg-options "-gdwarf-2 -dA" }
+// { dg-options "-gdwarf-2 -dA -gno-column-info" }
 
 class C {
 public:
diff --git a/gcc/testsuite/g++.dg/diagnostic/unclosed-extern-c.C b/gcc/testsuite/g++.dg/diagnostic/unclosed-extern-c.C
index fda3532266d..44f538e33ec 100644
--- a/gcc/testsuite/g++.dg/diagnostic/unclosed-extern-c.C
+++ b/gcc/testsuite/g++.dg/diagnostic/unclosed-extern-c.C
@@ -1,3 +1,12 @@
-extern "C" { /* { dg-message "12: to match this '.'" } */
+extern "C" { // { dg-line open_extern_c }
+
+  int foo (void);
+
+/* Missing close-brace for the extern "C" here.  */
+
+template <typename T> // { dg-error "template with C linkage" }
+void bar (void);
+// { dg-message "1: 'extern .C.' linkage started here" "" { target *-*-* } open_extern_c }
 
 void test (void); /* { dg-error "17: expected '.' at end of input" } */
+// { message "12: to match this '.'" "" { target *-*-* } open_extern_c }
diff --git a/gcc/testsuite/g++.dg/ext/is_trivially_constructible5.C b/gcc/testsuite/g++.dg/ext/is_trivially_constructible5.C
new file mode 100644
index 00000000000..15ea33675ed
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_trivially_constructible5.C
@@ -0,0 +1,12 @@
+// PR c++/80991
+// { dg-do compile { target c++11 } }
+
+template<bool> void foo()
+{
+  static_assert(__is_trivially_constructible(int, int), "");
+}
+
+void bar()
+{
+  foo<true>();
+}
diff --git a/gcc/testsuite/g++.dg/ext/pr81706.C b/gcc/testsuite/g++.dg/ext/pr81706.C
new file mode 100644
index 00000000000..f0ed8ab6d71
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/pr81706.C
@@ -0,0 +1,32 @@
+// PR libstdc++/81706
+// { dg-do compile { target i?86-*-* x86_64-*-* } }
+// { dg-options "-O3 -mavx2 -mno-avx512f" }
+// { dg-final { scan-assembler "call\[^\n\r]_ZGVdN4v_cos" } }
+// { dg-final { scan-assembler "call\[^\n\r]_ZGVdN4v_sin" } }
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern double cos (double) __attribute__ ((nothrow, leaf, simd ("notinbranch")));
+extern double sin (double) __attribute__ ((nothrow, leaf, simd ("notinbranch")));
+#ifdef __cplusplus
+}
+#endif
+double p[1024] = { 1.0 };
+double q[1024] = { 1.0 };
+
+void
+foo (void)
+{
+  int i;
+  for (i = 0; i < 1024; i++)
+    p[i] = cos (q[i]);
+}
+
+void
+bar (void)
+{
+  int i;
+  for (i = 0; i < 1024; i++)
+    p[i] = __builtin_sin (q[i]);
+}
diff --git a/gcc/testsuite/g++.dg/ext/typeof12.C b/gcc/testsuite/g++.dg/ext/typeof12.C
new file mode 100644
index 00000000000..4ba75732db1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/typeof12.C
@@ -0,0 +1,11 @@
+// PR c++/71820
+
+void f (void (*) (int, int)) {}
+
+template < typename T > void g (T x, __typeof__ x) {}  // { dg-message "sorry, unimplemented: mangling" }
+
+int main ()
+{
+  f (g < int >); 
+  return 0; 
+}
diff --git a/gcc/testsuite/g++.dg/gcov/gcov-threads-1.C b/gcc/testsuite/g++.dg/gcov/gcov-threads-1.C
index cc9266ab8ea..cc912f9ddf4 100644
--- a/gcc/testsuite/g++.dg/gcov/gcov-threads-1.C
+++ b/gcc/testsuite/g++.dg/gcov/gcov-threads-1.C
@@ -31,14 +31,14 @@ int main(int argc, char **argv) {
   {
     ids[i] = i;
     int r = pthread_create (&t[i], NULL, ContentionNoDeadlock_thread, &ids[i]);
-    assert (r == 0);				/* count(5) */
+    assert (r == 0);				/* count(5*) */
   }
 
   int ret;
   for (int i = 0; i < NR; i++)
     {
       int r = pthread_join (t[i], (void**)&ret);
-      assert (r == 0);				/* count(5) */
+      assert (r == 0);				/* count(5*) */
     }
 
   return 0;					/* count(1) */
diff --git a/gcc/testsuite/g++.dg/gcov/loop.C b/gcc/testsuite/g++.dg/gcov/loop.C
new file mode 100644
index 00000000000..7f3be5587af
--- /dev/null
+++ b/gcc/testsuite/g++.dg/gcov/loop.C
@@ -0,0 +1,27 @@
+/* { dg-options "-fprofile-arcs -ftest-coverage" } */
+/* { dg-do run { target native } } */
+
+unsigned
+loop (unsigned n, int value)		  /* count(14k) */
+{
+  for (unsigned i = 0; i < n - 1; i++)
+  {
+    value += i;				  /* count(21M) */
+  }
+
+  return value;
+}
+
+int main(int argc, char **argv)
+{
+  unsigned sum = 0;
+  for (unsigned i = 0; i < 7 * 1000; i++)
+  {
+    sum += loop (1000, sum);
+    sum += loop (2000, sum);		  /* count(7k) */
+  }
+
+  return 0;				  /* count(1) */
+}
+
+/* { dg-final { run-gcov branches { -abj loop.C } } } */
diff --git a/gcc/testsuite/g++.dg/gcov/ternary.C b/gcc/testsuite/g++.dg/gcov/ternary.C
new file mode 100644
index 00000000000..d055928c295
--- /dev/null
+++ b/gcc/testsuite/g++.dg/gcov/ternary.C
@@ -0,0 +1,12 @@
+// { dg-options "-fprofile-arcs -ftest-coverage" }
+// { dg-do run { target native } }
+
+int b, c, d, e;
+
+int main()
+{
+  int a = b < 1 ? (c < 3 ? d : c) : e;	/* count(1*) */
+  return a;
+}
+
+// { dg-final { run-gcov remove-gcda ternary.C } }
diff --git a/gcc/testsuite/g++.dg/guality/pr82630.C b/gcc/testsuite/g++.dg/guality/pr82630.C
new file mode 100644
index 00000000000..71d11acf5e2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/guality/pr82630.C
@@ -0,0 +1,58 @@
+// PR debug/82630
+// { dg-do run }
+// { dg-additional-options "-fPIC" { target fpic } }
+
+struct C
+{
+  int &c;
+  long d;
+  __attribute__((always_inline)) C (int &x) : c(x), d() {}
+};
+int v;
+
+__attribute__((noipa)) void
+fn1 (const void *x)
+{
+  asm volatile ("" : : "g" (x) : "memory");
+}
+
+__attribute__((noipa)) void
+fn2 (C x)
+{
+  int a = x.c + x.d;
+  asm volatile ("" : : "g" (a) : "memory");
+}
+
+__attribute__((noipa)) void
+fn3 (void)
+{
+  asm volatile ("" : : : "memory");
+}
+
+__attribute__((noipa))
+#ifdef __i386__
+__attribute__((regparm (2)))
+#endif
+static void
+fn4 (int *x, const char *y, C z)
+{
+  fn2 (C (*x));
+  fn1 ("baz");
+  fn2 (z);	// { dg-final { gdb-test 41 "y\[0\]" "'f'" } }
+  fn1 ("baz");	// { dg-final { gdb-test 41 "y\[1\]" "'o'" } }
+}
+
+__attribute__((noipa)) void
+fn5 (int *x)
+{
+  fn4 (x, "foo", C (*x));
+  fn3 ();
+}
+
+int
+main ()
+{
+  int a = 10;
+  fn5 (&a);
+  return 0;
+}
diff --git a/gcc/testsuite/g++.dg/lang-dump.C b/gcc/testsuite/g++.dg/lang-dump.C
new file mode 100644
index 00000000000..b2eddafa79e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/lang-dump.C
@@ -0,0 +1,21 @@
+// { dg-additional-options "-fdump-lang-all" }
+// Just check we don't explode when asking for language dumps.  Does
+// not necessarily mean any particular language dump is useful.
+
+struct X 
+{
+  int m;
+  virtual ~X ();
+};
+
+X::~X () {}
+
+struct Y : X
+{
+};
+
+int frob (int a)
+{
+  return 2 * a;
+}
+
diff --git a/gcc/testsuite/g++.dg/opt/pr82577.C b/gcc/testsuite/g++.dg/opt/pr82577.C
new file mode 100644
index 00000000000..1a06897a403
--- /dev/null
+++ b/gcc/testsuite/g++.dg/opt/pr82577.C
@@ -0,0 +1,22 @@
+// { dg-additional-options "-O2" }
+// PR c++/82577 ICE when optimizing
+
+#if __cplusplus > 201500L
+// register is no longer a keyword in C++17.
+#define register
+#endif
+
+class a {
+public:
+  int *b();
+};
+struct c {
+  int d;
+  a e;
+} f;
+void fn1(register c *g) {
+  register int *h;
+  do
+    (h) = g->e.b() + (g)->d;
+  while (&f);
+}
diff --git a/gcc/testsuite/g++.dg/opt/pr82778.C b/gcc/testsuite/g++.dg/opt/pr82778.C
new file mode 100644
index 00000000000..eeac0c5f38b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/opt/pr82778.C
@@ -0,0 +1,37 @@
+// PR rtl-optimization/82778
+// { dg-do compile }
+// { dg-options "-O2" }
+
+template <typename a, int b> struct c {
+  typedef a d[b];
+  static a e(d f, int g) { return f[g]; }
+};
+template <typename a, int b> struct B {
+  typedef c<a, b> h;
+  typename h::d i;
+  long j;
+  a at() { return h::e(i, j); }
+};
+int k, m, r, s, t;
+char l, n, q;
+short o, p, w;
+struct C {
+  int u;
+};
+B<C, 4> v;
+void x() {
+  if (((p > (q ? v.at().u : k)) >> l - 226) + !(n ^ r * m))
+    s = ((-(((p > (q ? v.at().u : k)) >> l - 226) + !(n ^ r * m)) < 0) /
+             (-(((p > (q ? v.at().u : k)) >> l - 226) + !(n ^ r * m)) ^
+              -25 & o) &&
+         p) >>
+        (0 <= 0
+             ? 0 ||
+                   (-(((p > (q ? v.at().u : k)) >> l - 226) + !(n ^ r * m)) <
+                    0) /
+                       (-(((p > (q ? v.at().u : k)) >> l - 226) +
+                          !(n ^ r * m)) ^ -25 & o)
+             : 0);
+  w = (p > (q ? v.at().u : k)) >> l - 226;
+  t = !(n ^ r * m);
+}
diff --git a/gcc/testsuite/g++.dg/other/i386-2.C b/gcc/testsuite/g++.dg/other/i386-2.C
index 63c5f738baa..7e35e686cff 100644
--- a/gcc/testsuite/g++.dg/other/i386-2.C
+++ b/gcc/testsuite/g++.dg/other/i386-2.C
@@ -1,12 +1,12 @@
 /* { dg-do compile { target i?86-*-* x86_64-*-* } } */
-/* { dg-options "-O -pedantic-errors -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt  -mavx512dq -mavx512bw -mavx512vl -mavx512ifma -mavx512vbmi -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid" } */
+/* { dg-options "-O -pedantic-errors -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt  -mavx512dq -mavx512bw -mavx512vl -mavx512ifma -mavx512vbmi -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid -mgfni" } */
 
 
 /* Test that {,x,e,p,t,s,w,a,b,i}mmintrin.h, mm3dnow.h, fma4intrin.h,
    xopintrin.h, abmintrin.h, bmiintrin.h, tbmintrin.h, lwpintrin.h,
    popcntintrin.h, fmaintrin.h, pkuintrin.h, avx5124fmapsintrin.h,
-   avx5124vnniwintrin.h, avx512vpopcntdqintrin.h and mm_malloc.h.h are usable
-   with -O -pedantic-errors.  */
+   avx5124vnniwintrin.h, avx512vpopcntdqintrin.h gfniintrin.h
+   and mm_malloc.h.h are usable with -O -pedantic-errors.  */
 
 #include <x86intrin.h>
 
diff --git a/gcc/testsuite/g++.dg/other/i386-3.C b/gcc/testsuite/g++.dg/other/i386-3.C
index 16a96efe2a5..7e44d47a93c 100644
--- a/gcc/testsuite/g++.dg/other/i386-3.C
+++ b/gcc/testsuite/g++.dg/other/i386-3.C
@@ -1,10 +1,10 @@
 /* { dg-do compile { target i?86-*-* x86_64-*-* } } */
-/* { dg-options "-O -fkeep-inline-functions -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512dq -mavx512bw -mavx512vl -mavx512ifma -mavx512vbmi -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid" } */
+/* { dg-options "-O -fkeep-inline-functions -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512dq -mavx512bw -mavx512vl -mavx512ifma -mavx512vbmi -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid -mgfni" } */
 
 /* Test that {,x,e,p,t,s,w,a,b,i}mmintrin.h, mm3dnow.h, fma4intrin.h,
    xopintrin.h, abmintrin.h, bmiintrin.h, tbmintrin.h, lwpintrin.h,
    popcntintrin.h, fmaintrin.h, pkuintrin.h, avx5124fmapsintrin.h,
-   avx5124vnniwintrin.h, avx512vpopcntdqintrin.h and mm_malloc.h are
-   usable with -O -fkeep-inline-functions.  */
+   avx5124vnniwintrin.h, avx512vpopcntdqintrin.h gfniintrin.h and
+   mm_malloc.h are usable with -O -fkeep-inline-functions.  */
 
 #include <x86intrin.h>
diff --git a/gcc/testsuite/g++.dg/other/operator2.C b/gcc/testsuite/g++.dg/other/operator2.C
index 4b952bf11eb..cc68d53354e 100644
--- a/gcc/testsuite/g++.dg/other/operator2.C
+++ b/gcc/testsuite/g++.dg/other/operator2.C
@@ -3,7 +3,7 @@
 
 struct A
 {
-  operator int&(int);  // { dg-error "void" }
+  operator int&(int);  // { dg-error "no arguments" }
 };
 
 A a;
diff --git a/gcc/testsuite/g++.dg/other/pr53574.C b/gcc/testsuite/g++.dg/other/pr53574.C
new file mode 100644
index 00000000000..cc899a552c8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/other/pr53574.C
@@ -0,0 +1,48 @@
+// PR c++/53574
+// { dg-do compile { target c++11 } }
+// { dg-options "-fstack-usage" }
+
+template <typename> struct A { typedef int type; };
+struct B {
+  typedef __SIZE_TYPE__ H;
+};
+template <typename> class allocator : B {};
+template <typename _Alloc> struct C {
+  template <typename T>
+  static typename T::H foo(T *);
+  typedef decltype(foo((_Alloc *)0)) H;
+  template <typename U>
+  static typename A<H>::type bar(U) { return typename A<H>::type (); }
+  static int baz(_Alloc p1) { bar(p1); return 0; }
+};
+template <typename _Alloc> struct I : C<_Alloc> {};
+template <typename, typename> struct J {
+  typedef I<allocator<int>> K;
+  K k;
+};
+struct D : J<int, allocator<int>> {
+  void fn(int, int) {
+    K m;
+    I<K>::baz(m);
+  }
+};
+template <class Ch, class = int, class = int> struct F {
+  F();
+  F(const Ch *);
+  F test();
+  D d;
+};
+int l;
+struct G {
+  G(F<char>);
+};
+char n;
+template <class Ch, class Tr, class Alloc> F<Ch, Tr, Alloc>::F(const Ch *) {
+  test();
+}
+template <class Ch, class Tr, class Alloc>
+F<Ch, Tr, Alloc> F<Ch, Tr, Alloc>::test() {
+  d.fn(l, 0);
+  return F<Ch, Tr, Alloc> ();
+}
+G fn1() { return G(&n); }
diff --git a/gcc/testsuite/g++.dg/parse/builtin2.C b/gcc/testsuite/g++.dg/parse/builtin2.C
index c524ea68416..daa80bb11b0 100644
--- a/gcc/testsuite/g++.dg/parse/builtin2.C
+++ b/gcc/testsuite/g++.dg/parse/builtin2.C
@@ -1,5 +1,5 @@
 // PR c++/14432
-// { dg-options "" }
+// { dg-options "-Wno-builtin-declaration-mismatch" }
 
 struct Y {}; 
 Y y1; 
diff --git a/gcc/testsuite/g++.dg/pr71694.C b/gcc/testsuite/g++.dg/pr71694.C
index e79f62aeb13..0a8baf230bf 100644
--- a/gcc/testsuite/g++.dg/pr71694.C
+++ b/gcc/testsuite/g++.dg/pr71694.C
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -fno-store-merging" } */
 
 struct B {
     B() {}
diff --git a/gcc/testsuite/g++.dg/template/bitfield4.C b/gcc/testsuite/g++.dg/template/bitfield4.C
new file mode 100644
index 00000000000..4927b7ab144
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/bitfield4.C
@@ -0,0 +1,6 @@
+// PR c++/82357
+
+template <typename> struct A {
+  A() { x |= 0; }
+  int x : 8;
+};
diff --git a/gcc/testsuite/g++.dg/template/cast4.C b/gcc/testsuite/g++.dg/template/cast4.C
new file mode 100644
index 00000000000..2f46c7189eb
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/cast4.C
@@ -0,0 +1,4 @@
+template <class T> void f()
+{
+  static_cast<int&>(42);	// { dg-error "static_cast" }
+}
diff --git a/gcc/testsuite/g++.dg/template/crash128.C b/gcc/testsuite/g++.dg/template/crash128.C
new file mode 100644
index 00000000000..2682e3dc3ce
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/crash128.C
@@ -0,0 +1,19 @@
+// PR c++/54090
+
+template <int n>
+struct X {
+
+  template <int N, bool = (n >= N), typename T = void> struct Y;
+
+  template <int N, typename T>
+  struct Y<N, true, T> {};
+
+  static const int M = n / 2;
+
+  template <typename T>
+  struct Y<X::M, true, T> {};
+};
+
+void foo() {
+  X<10>::Y<10/2> y;
+}
diff --git a/gcc/testsuite/g++.dg/template/extern-c.C b/gcc/testsuite/g++.dg/template/extern-c.C
new file mode 100644
index 00000000000..c0dd7cb66d5
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/extern-c.C
@@ -0,0 +1,66 @@
+template <typename T> void specializable (T);
+
+/* Invalid template: within "extern C".  */
+
+extern "C" { // { dg-message "1: 'extern .C.' linkage started here" }
+
+template <typename T> // { dg-error "template with C linkage" }
+void within_extern_c_braces (void);
+
+}
+
+/* Valid template: not within "extern C".  */
+
+template <typename T>
+void not_within_extern_c (void);
+
+
+/* Invalid specialization: within "extern C".  */
+
+extern "C" { // { dg-message "1: 'extern .C.' linkage started here" }
+
+template <>  // { dg-error "template specialization with C linkage" }
+void specializable (int);
+
+}
+
+
+/* Valid specialization: not within "extern C".  */
+template <>
+void specializable (char);
+
+
+/* Example of extern C without braces.  */
+
+extern "C" template <typename T> // { dg-line open_extern_c_no_braces }
+void within_extern_c_no_braces (void);
+// { dg-error "12: template with C linkage" "" { target *-*-* } open_extern_c_no_braces }
+// { dg-message "1: 'extern .C.' linkage started here" "" { target *-*-* } open_extern_c_no_braces }
+
+
+/* Nested extern "C" specifications.
+   We should report within the innermost extern "C" that's still open.  */
+
+extern "C" {
+  extern "C" { // { dg-line middle_open_extern_c }
+    extern "C" {
+    }
+
+    template <typename T>  // { dg-error "template with C linkage" }
+    void within_nested_extern_c (void);
+    // { dg-message "3: 'extern .C.' linkage started here" "" { target *-*-* } middle_open_extern_c }
+
+    extern "C++" {
+      /* Valid template: within extern "C++".  */
+      template <typename T>
+      void within_nested_extern_cpp (void);
+
+      extern "C" {  // { dg-line last_open_extern_c }
+	/* Invalid template: within "extern C".  */
+	template <typename T> // { dg-error "template with C linkage" }
+	void within_extern_c_within_extern_cpp (void);
+	// { dg-message "7: 'extern .C.' linkage started here" "" { target *-*-* } last_open_extern_c }	
+      }
+    }
+  }
+}
diff --git a/gcc/testsuite/g++.dg/torture/pr70971.C b/gcc/testsuite/g++.dg/torture/pr70971.C
new file mode 100644
index 00000000000..23f33aafaba
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr70971.C
@@ -0,0 +1,48 @@
+// { dg-additional-options "-std=c++14" }
+
+template<typename Signature>
+class function;
+
+template<typename R, typename... Args>
+class invoker_base
+{
+ public:
+  virtual ~invoker_base() { }
+};
+
+template<typename F, typename R, typename... Args>
+class functor_invoker : public invoker_base<R, Args...>
+{
+ public:
+  explicit functor_invoker(const F& f) : f(f) { }
+ private:
+  F f;
+};
+
+template<typename R, typename... Args>
+class function<R (Args...)> {
+ public:
+  template<typename F>
+  function(const F& f) : invoker(0) {
+    invoker = new functor_invoker<F, R, Args...>(f); 
+  }
+  ~function() {
+    if (invoker)
+      delete invoker;
+  }
+ private:
+  invoker_base<R, Args...>* invoker;
+};
+
+template<typename>
+struct unique_ptr { };
+
+struct A {};
+template <class...> struct typelist {};
+template <class... Cs> unique_ptr<A> chooseB(typelist<Cs...>);
+template <class... Cs, class Idx, class... Rest>
+unique_ptr<A> chooseB(typelist<Cs...> choices, Idx, Rest... rest) {
+  auto f = [=](auto) { return [=] { return chooseB(choices, rest...); }; };
+  function<unique_ptr<A>()> fs[]{f(Cs{})...};
+}
+main() { chooseB(typelist<double, char>{}, 0, 1, 2); }
diff --git a/gcc/testsuite/g++.dg/torture/pr77555.C b/gcc/testsuite/g++.dg/torture/pr77555.C
new file mode 100644
index 00000000000..540d1a09a5f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr77555.C
@@ -0,0 +1,20 @@
+// { dg-do link }
+// { dg-options "-std=c++11" }
+
+extern "C" int printf(const char*, ...);
+struct A {
+  A(int, char *p2) { printf(p2); }
+};
+template <int, typename> struct B { static A static_var; };
+template <int LINE, typename GETTER>
+A B<LINE, GETTER>::static_var{0, GETTER::get()};
+struct C {
+  void unused() {
+    static char function_static;
+    struct D {
+      static char *get() { return &function_static; }
+    };
+    auto addr = B<0, D>::static_var;
+  }
+};
+int main() {}
diff --git a/gcc/testsuite/g++.dg/torture/pr81659.C b/gcc/testsuite/g++.dg/torture/pr81659.C
new file mode 100644
index 00000000000..3696957532e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr81659.C
@@ -0,0 +1,19 @@
+// { dg-do compile }
+
+void
+a (int b)
+{
+  if (b)
+    throw;
+  try
+    {
+      a (3);
+    }
+  catch (int)
+    {
+    }
+  catch (int)
+    {
+    }
+}
+
diff --git a/gcc/testsuite/g++.dg/torture/pr82823.C b/gcc/testsuite/g++.dg/torture/pr82823.C
new file mode 100644
index 00000000000..dab369e7ad3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr82823.C
@@ -0,0 +1,26 @@
+// { dg-do compile }
+// { dg-additional-options "-fstack-clash-protection" }
+// { dg-require-effective-target supports_stack_clash_protection }
+
+
+class a
+{
+public:
+  ~a ();
+  int b;
+};
+class c
+{
+public:
+  a m_fn1 ();
+};
+class d
+{
+  int e ();
+  c f;
+};
+int
+d::e ()
+{
+  return f.m_fn1 ().b;
+}
diff --git a/gcc/testsuite/g++.dg/tree-ssa/pr81702.C b/gcc/testsuite/g++.dg/tree-ssa/pr81702.C
new file mode 100644
index 00000000000..85acd857e67
--- /dev/null
+++ b/gcc/testsuite/g++.dg/tree-ssa/pr81702.C
@@ -0,0 +1,110 @@
+// { dg-do compile }
+// { dg-options "-O2" }
+
+namespace std {
+  struct type_info
+  {
+    virtual bool __do_catch(const type_info *__thr_type, void **__thr_obj,
+       unsigned __outer) const;
+  };
+}
+
+template< typename VALUE_T, typename TYPE >
+struct List_policy
+{
+  typedef VALUE_T *Value_type;
+  typedef TYPE **Type;
+  typedef TYPE *Head_type;
+  typedef TYPE Item_type;
+};
+
+template< typename POLICY >
+class List
+{
+public:
+  typedef typename POLICY::Value_type Value_type;
+  class Iterator
+  {
+    typedef typename POLICY::Type Internal_type;
+  public:
+    typedef typename POLICY::Value_type value_type;
+    typedef typename POLICY::Value_type Value_type;
+    Value_type operator -> () const { return static_cast<Value_type>(*_c); }
+    Internal_type _c;
+  };
+  Iterator begin() { return Iterator(); }
+  Iterator end() { return Iterator(); }
+  typename POLICY::Head_type _f;
+};
+
+template<typename ELEM_TYPE> class H_list_item_t { };
+
+template< typename T, typename POLICY >
+class H_list : public List<POLICY>
+{
+public:
+  typedef typename POLICY::Item_type Item;
+  typedef List<POLICY> Base;
+  typedef typename Base::Iterator Iterator;
+  Iterator insert(T *e, Iterator const &pred)
+  {
+    Item **x = &this->_f;
+    *x = static_cast<Item*>(e);
+    return Iterator();
+  }
+};
+
+template< typename T >
+struct H_list_t : H_list<T, List_policy< T, H_list_item_t<T> > >
+{
+  H_list_t(bool b) : H_list<T, List_policy< T, H_list_item_t<T> > >(b) {}
+};
+
+template< typename BASE, typename MATCH_RESULT >
+struct Type_matcher : H_list_item_t<BASE>
+{
+  explicit Type_matcher(std::type_info const *type);
+  typedef MATCH_RESULT Match_result;
+
+private:
+  std::type_info *_type;
+  typedef H_list_t<BASE> List;
+  typedef typename List::Iterator Iterator;
+  static List _for_type;
+};
+
+template< typename BASE, typename MR >
+Type_matcher<BASE, MR>::Type_matcher(std::type_info const *t)
+{
+  Iterator c = _for_type.begin();
+  t->__do_catch(c->_type, 0, 0);
+  _for_type.insert(static_cast<BASE*>(this), _for_type.begin());
+}
+
+template< typename VI, typename HW >
+class Fa : public Type_matcher<Fa<VI, HW>, VI*>
+{
+public:
+  typedef Fa<VI, HW> Self;
+  virtual VI *do_match(HW *f) = 0;
+  explicit Fa(std::type_info const *type) : Type_matcher<Self, VI*>(type) {}
+};
+
+class Res {};
+typedef Fa<Res, Res> R_fac;
+
+template< typename VI, typename HW_BASE, typename HW, typename BASE >
+class Fa_t : public BASE
+{
+public:
+  Fa_t() : BASE(&typeid(HW)) {}
+  VI *do_match(HW_BASE *) { return 0; }
+};
+
+template< typename VI, typename HW >
+class Resource_factory_t : public Fa_t<VI, Res, HW, R_fac > {};
+
+class Foo {};
+class Foo2;
+class Foo3 : public Res {};
+Resource_factory_t<Foo3, Foo> _x;
diff --git a/gcc/testsuite/g++.dg/ubsan/float-cast-overflow-bf.C b/gcc/testsuite/g++.dg/ubsan/float-cast-overflow-bf.C
index f01c576c3db..385a109c359 100644
--- a/gcc/testsuite/g++.dg/ubsan/float-cast-overflow-bf.C
+++ b/gcc/testsuite/g++.dg/ubsan/float-cast-overflow-bf.C
@@ -52,11 +52,11 @@ main (void)
   return 0;
 }
 
-/* { dg-output "value -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type" } */
+/* { dg-output " -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.14748e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 4.29497e\\\+09 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type" } */
diff --git a/gcc/testsuite/g++.dg/ubsan/pr82353-2-aux.cc b/gcc/testsuite/g++.dg/ubsan/pr82353-2-aux.cc
new file mode 100644
index 00000000000..75d466b39bb
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ubsan/pr82353-2-aux.cc
@@ -0,0 +1,32 @@
+// PR sanitizer/82353
+
+#include "pr82353-2.h"
+
+B a;
+E b;
+B C::c0;
+unsigned D::d0;
+
+void
+foo ()
+{
+  a.b1 = p.f2.e2.b1 = 5;
+}
+
+void
+bar ()
+{
+  int c = p.f2.e4.d1.a0 - -~p.f4 * 89;
+  q.c0.b0 = i > g * a.b0 * h - k % a.b1;
+  if ((~(m * j) && -~p.f4 * 90284000534361) % ~m * j)
+    b.e2.b0 << l << f;
+  o = -~p.f4 * 89;
+  int d = p.f4;
+  if (b.e2.b0)
+    b.e2.b1 = c;
+  bool e = ~-~p.f4;
+  a.b1 % e;
+  if (k / p.f2.e2.b1)
+    b.e4.d0 = g * a.b0 * h;
+  n = j;
+}
diff --git a/gcc/testsuite/g++.dg/ubsan/pr82353-2.C b/gcc/testsuite/g++.dg/ubsan/pr82353-2.C
new file mode 100644
index 00000000000..31a35ac3a02
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ubsan/pr82353-2.C
@@ -0,0 +1,20 @@
+// PR sanitizer/82353
+// { dg-do run }
+// { dg-options "-fsanitize=undefined -fno-sanitize-recover=undefined -std=c++11 -O2 -w" }
+// { dg-additional-sources "pr82353-2-aux.cc" }
+
+#include "pr82353-2.h"
+
+unsigned long f, g;
+bool h, k, j, i;
+unsigned char l, m;
+short n;
+unsigned o;
+F p;
+
+int
+main ()
+{
+  foo ();
+  bar ();
+}
diff --git a/gcc/testsuite/g++.dg/ubsan/pr82353-2.h b/gcc/testsuite/g++.dg/ubsan/pr82353-2.h
new file mode 100644
index 00000000000..4693d2299f2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ubsan/pr82353-2.h
@@ -0,0 +1,31 @@
+extern unsigned long f, g;
+extern bool h, i, j, k;
+extern unsigned char l, m;
+extern short n;
+extern unsigned o;
+struct B {
+  short b0 : 27;
+  long b1 : 10;
+};
+struct A {
+  int a0 : 5;
+};
+struct C {
+  static B c0;
+};
+struct D {
+  static unsigned d0;
+  A d1;
+};
+struct E {
+  B e2;
+  D e4;
+};
+struct F {
+  E f2;
+  short f4;
+};
+extern F p;
+extern C q;
+void foo ();
+void bar ();
diff --git a/gcc/testsuite/g++.dg/vect/slp-pr56812.cc b/gcc/testsuite/g++.dg/vect/slp-pr56812.cc
index 53032330142..8b24b337efa 100644
--- a/gcc/testsuite/g++.dg/vect/slp-pr56812.cc
+++ b/gcc/testsuite/g++.dg/vect/slp-pr56812.cc
@@ -17,7 +17,6 @@ void mydata::Set (float x)
     data[i] = x;

 }

 

-/* 256-bit vectors will be handled by loop vectorisation instead, since there

-   is no prologue or epilogue that would raise the cost.  SLP isn't yet

-   possible with variable-length vectors.  */

-/* { dg-final { scan-tree-dump-times "basic block vectorized" 1 "slp1" { xfail { { vect256 || vect_variable_length } || aarch64*-*-* } } } } */

+/* For targets without vector loop peeling the loop becomes cheap

+   enough to be vectorized.  */

+/* { dg-final { scan-tree-dump-times "basic block vectorized" 1 "slp1" { xfail { ! vect_peeling_profitable } } } } */

diff --git a/gcc/testsuite/g++.dg/warn/Wbuiltin_declaration_mismatch-1.C b/gcc/testsuite/g++.dg/warn/Wbuiltin_declaration_mismatch-1.C
new file mode 100644
index 00000000000..713073cb421
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wbuiltin_declaration_mismatch-1.C
@@ -0,0 +1,7 @@
+// PR c++/82466
+// { dg-options "-Wbuiltin-declaration-mismatch" }
+
+namespace N
+{
+  int printf;
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wreturn-local-addr-4.C b/gcc/testsuite/g++.dg/warn/Wreturn-local-addr-4.C
new file mode 100644
index 00000000000..492dcb9e76f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wreturn-local-addr-4.C
@@ -0,0 +1,18 @@
+// PR c++/82600
+// { dg-do compile }
+
+void *b[10];
+
+template <int N>
+void **
+foo (int x)
+{
+  void **a = b;		// { dg-bogus "address of local variable 'a' returned" }
+  return &a[x];
+}
+
+void **
+bar (int x)
+{
+  return foo <0> (x);
+}
diff --git a/gcc/testsuite/g++.dg/warn/pr82710.C b/gcc/testsuite/g++.dg/warn/pr82710.C
new file mode 100644
index 00000000000..93585eacf99
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr82710.C
@@ -0,0 +1,48 @@
+// { dg-do compile { target c++11 } }
+// { dg-additional-options "-Wparentheses -Wno-non-template-friend" }
+
+// the MVP warning triggered on a friend decl.  */
+class X;
+enum class Q {}; // C++ 11ness
+enum R {};
+
+namespace here 
+{
+  // these friends
+  X friendFunc1();
+  X *friendFunc2 ();
+  int friendFunc3 ();
+  int bob ();
+  Q bill ();
+  R ben ();
+}
+
+namespace nm
+{
+  namespace here 
+  {
+    // Not these friends
+    void friendFunc1 ();
+    void friendFunc2 ();
+    void friendFunc3 ();
+    int bob ();
+    Q bill ();
+    R ben ();
+  }
+
+  class TestClass
+  {
+    friend X (::here::friendFunc1 ()); // parens are needed
+    friend X *(::here::friendFunc2 ()); // { dg-warning "" }
+    friend X *::here::friendFunc2 ();
+    friend int (::here::friendFunc3 ()); // { dg-warning "" }
+  };
+
+  template <typename T> class X
+  {
+    friend typename T::frob (::here::bob ());
+    friend Q (::here::bill ());
+    friend R (::here::ben ());
+  };
+}
+
diff --git a/gcc/testsuite/g++.old-deja/g++.jason/operator.C b/gcc/testsuite/g++.old-deja/g++.jason/operator.C
index 339e6a447b4..bdcd5493a97 100644
--- a/gcc/testsuite/g++.old-deja/g++.jason/operator.C
+++ b/gcc/testsuite/g++.old-deja/g++.jason/operator.C
@@ -9,7 +9,7 @@ struct A {
   static int operator()(int a);	   // { dg-error "must be a nonstatic member" }
   static int operator+(A,A);	   // { dg-error "either a non-static member" } 
   int operator+(int a, int b = 1); // { dg-error "either zero or one" }
-  int operator++(char);		   // { dg-error "must take 'int'" } 
+  int operator++(char);		   // { dg-error "must have 'int'" }
   void operator delete (void *);   
   void operator delete (void *, unsigned long);	
 };
diff --git a/gcc/testsuite/g++.old-deja/g++.mike/p811.C b/gcc/testsuite/g++.old-deja/g++.mike/p811.C
index 5c8260aa1f8..2ca04abdcba 100644
--- a/gcc/testsuite/g++.old-deja/g++.mike/p811.C
+++ b/gcc/testsuite/g++.old-deja/g++.mike/p811.C
@@ -1,5 +1,5 @@
 // { dg-do assemble  }
-// { dg-options "" }
+// { dg-options "-Wno-builtin-declaration-mismatch" }
 // This test case caused the compiler to abort at one point in time.
 // prms-id: 811
 
diff --git a/gcc/testsuite/g++.target/aarch64/aarch64.exp b/gcc/testsuite/g++.target/aarch64/aarch64.exp
new file mode 100644
index 00000000000..5eaa8725c9d
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/aarch64.exp
@@ -0,0 +1,38 @@
+#  Specific regression driver for AArch64.
+#  Copyright (C) 2009-2017 Free Software Foundation, Inc.
+#  Contributed by ARM Ltd.
+#
+#  This file is part of GCC.
+#
+#  GCC is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 3, or (at your option)
+#  any later version.
+#
+#  GCC is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with GCC; see the file COPYING3.  If not see
+#  <http://www.gnu.org/licenses/>.  */
+
+# GCC testsuite that uses the `dg.exp' driver.
+
+# Exit immediately if this isn't an AArch64 target.
+if {![istarget aarch64*-*-*] } then {
+  return
+}
+
+# Load support procs.
+load_lib g++-dg.exp
+
+# Initialize `dg'.
+dg-init
+
+# Main loop.
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/*.C]] "" ""
+
+# All done.
+dg-finish
diff --git a/gcc/testsuite/g++.target/aarch64/sve_catch_1.C b/gcc/testsuite/g++.target/aarch64/sve_catch_1.C
new file mode 100644
index 00000000000..48b007fc1be
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve_catch_1.C
@@ -0,0 +1,70 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fopenmp-simd -fno-omit-frame-pointer" } */
+/* { dg-options "-O3 -fopenmp-simd -fno-omit-frame-pointer -march=armv8-a+sve" { target aarch64_sve_hw } } */
+
+/* Invoke X (P##n) for n in [0, 7].  */
+#define REPEAT8(X, P) \
+  X (P##0) X (P##1) X (P##2) X (P##3) X (P##4) X (P##5) X (P##6) X (P##7)
+
+/* Invoke X (n) for all octal n in [0, 39].  */
+#define REPEAT40(X) \
+  REPEAT8 (X, 0) REPEAT8 (X, 1)  REPEAT8 (X, 2) REPEAT8 (X, 3) REPEAT8 (X, 4)
+
+volatile int testi;
+
+/* Throw to f3.  */
+void __attribute__ ((weak))
+f1 (int x[40][100], int *y)
+{
+  /* A wild write to x and y.  */
+  asm volatile ("" ::: "memory");
+  if (y[testi] == x[testi][testi])
+    throw 100;
+}
+
+/* Expect vector work to be done, with spilling of vector registers.  */
+void __attribute__ ((weak))
+f2 (int x[40][100], int *y)
+{
+  /* Try to force some spilling.  */
+#define DECLARE(N) int y##N = y[N];
+  REPEAT40 (DECLARE);
+  for (int j = 0; j < 20; ++j)
+    {
+      f1 (x, y);
+#pragma omp simd
+      for (int i = 0; i < 100; ++i)
+	{
+#define INC(N) x[N][i] += y##N;
+	  REPEAT40 (INC);
+	}
+    }
+}
+
+/* Catch an exception thrown from f1, via f2.  */
+void __attribute__ ((weak))
+f3 (int x[40][100], int *y, int *z)
+{
+  volatile int extra = 111;
+  try
+    {
+      f2 (x, y);
+    }
+  catch (int val)
+    {
+      *z = val + extra;
+    }
+}
+
+static int x[40][100];
+static int y[40];
+static int z;
+
+int
+main (void)
+{
+  f3 (x, y, &z);
+  if (z != 211)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/g++.target/aarch64/sve_catch_2.C b/gcc/testsuite/g++.target/aarch64/sve_catch_2.C
new file mode 100644
index 00000000000..4acdefd235a
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve_catch_2.C
@@ -0,0 +1,5 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fopenmp-simd -fomit-frame-pointer" } */
+/* { dg-options "-O3 -fopenmp-simd -fomit-frame-pointer -march=armv8-a+sve" { target aarch64_sve_hw } } */
+
+#include "sve_catch_1.C"
diff --git a/gcc/testsuite/g++.target/aarch64/sve_catch_3.C b/gcc/testsuite/g++.target/aarch64/sve_catch_3.C
new file mode 100644
index 00000000000..b7e701668e5
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve_catch_3.C
@@ -0,0 +1,79 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fopenmp-simd -fno-omit-frame-pointer" } */
+/* { dg-options "-O3 -fopenmp-simd -fno-omit-frame-pointer -march=armv8-a+sve" { target aarch64_sve_hw } } */
+
+/* Invoke X (P##n) for n in [0, 7].  */
+#define REPEAT8(X, P) \
+  X (P##0) X (P##1) X (P##2) X (P##3) X (P##4) X (P##5) X (P##6) X (P##7)
+
+/* Invoke X (n) for all octal n in [0, 39].  */
+#define REPEAT40(X) \
+  REPEAT8 (X, 0) REPEAT8 (X, 1)  REPEAT8 (X, 2) REPEAT8 (X, 3) REPEAT8 (X, 4)
+
+volatile int testi, sink;
+
+/* Take 2 stack arguments and throw to f3.  */
+void __attribute__ ((weak))
+f1 (int x[40][100], int *y, int z1, int z2, int z3, int z4,
+    int z5, int z6, int z7, int z8)
+{
+  /* A wild write to x and y.  */
+  sink = z1;
+  sink = z2;
+  sink = z3;
+  sink = z4;
+  sink = z5;
+  sink = z6;
+  sink = z7;
+  sink = z8;
+  asm volatile ("" ::: "memory");
+  if (y[testi] == x[testi][testi])
+    throw 100;
+}
+
+/* Expect vector work to be done, with spilling of vector registers.  */
+void __attribute__ ((weak))
+f2 (int x[40][100], int *y)
+{
+  /* Try to force some spilling.  */
+#define DECLARE(N) int y##N = y[N];
+  REPEAT40 (DECLARE);
+  for (int j = 0; j < 20; ++j)
+    {
+      f1 (x, y, 1, 2, 3, 4, 5, 6, 7, 8);
+#pragma omp simd
+      for (int i = 0; i < 100; ++i)
+	{
+#define INC(N) x[N][i] += y##N;
+	  REPEAT40 (INC);
+	}
+    }
+}
+
+/* Catch an exception thrown from f1, via f2.  */
+void __attribute__ ((weak))
+f3 (int x[40][100], int *y, int *z)
+{
+  volatile int extra = 111;
+  try
+    {
+      f2 (x, y);
+    }
+  catch (int val)
+    {
+      *z = val + extra;
+    }
+}
+
+static int x[40][100];
+static int y[40];
+static int z;
+
+int
+main (void)
+{
+  f3 (x, y, &z);
+  if (z != 211)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/g++.target/aarch64/sve_catch_4.C b/gcc/testsuite/g++.target/aarch64/sve_catch_4.C
new file mode 100644
index 00000000000..cb75672e6b6
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve_catch_4.C
@@ -0,0 +1,5 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fopenmp-simd -fomit-frame-pointer" } */
+/* { dg-options "-O3 -fopenmp-simd -fomit-frame-pointer -march=armv8-a+sve" { target aarch64_sve_hw } } */
+
+#include "sve_catch_3.C"
diff --git a/gcc/testsuite/g++.target/aarch64/sve_catch_5.C b/gcc/testsuite/g++.target/aarch64/sve_catch_5.C
new file mode 100644
index 00000000000..7d0d430fd91
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve_catch_5.C
@@ -0,0 +1,82 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fopenmp-simd -fno-omit-frame-pointer" } */
+/* { dg-options "-O3 -fopenmp-simd -fno-omit-frame-pointer -march=armv8-a+sve" { target aarch64_sve_hw } } */
+
+/* Invoke X (P##n) for n in [0, 7].  */
+#define REPEAT8(X, P) \
+  X (P##0) X (P##1) X (P##2) X (P##3) X (P##4) X (P##5) X (P##6) X (P##7)
+
+/* Invoke X (n) for all octal n in [0, 39].  */
+#define REPEAT40(X) \
+  REPEAT8 (X, 0) REPEAT8 (X, 1)  REPEAT8 (X, 2) REPEAT8 (X, 3) REPEAT8 (X, 4)
+
+volatile int testi, sink;
+volatile void *ptr;
+
+/* Take 2 stack arguments and throw to f3.  */
+void __attribute__ ((weak))
+f1 (int x[40][100], int *y, int z1, int z2, int z3, int z4,
+    int z5, int z6, int z7, int z8)
+{
+  /* A wild write to x and y.  */
+  sink = z1;
+  sink = z2;
+  sink = z3;
+  sink = z4;
+  sink = z5;
+  sink = z6;
+  sink = z7;
+  sink = z8;
+  asm volatile ("" ::: "memory");
+  if (y[testi] == x[testi][testi])
+    throw 100;
+}
+
+/* Expect vector work to be done, with spilling of vector registers.  */
+void __attribute__ ((weak))
+f2 (int x[40][100], int *y)
+{
+  /* Create a true variable-sized frame.  */
+  ptr = __builtin_alloca (testi + 40);
+  /* Try to force some spilling.  */
+#define DECLARE(N) int y##N = y[N];
+  REPEAT40 (DECLARE);
+  for (int j = 0; j < 20; ++j)
+    {
+      f1 (x, y, 1, 2, 3, 4, 5, 6, 7, 8);
+#pragma omp simd
+      for (int i = 0; i < 100; ++i)
+	{
+#define INC(N) x[N][i] += y##N;
+	  REPEAT40 (INC);
+	}
+    }
+}
+
+/* Catch an exception thrown from f1, via f2.  */
+void __attribute__ ((weak))
+f3 (int x[40][100], int *y, int *z)
+{
+  volatile int extra = 111;
+  try
+    {
+      f2 (x, y);
+    }
+  catch (int val)
+    {
+      *z = val + extra;
+    }
+}
+
+static int x[40][100];
+static int y[40];
+static int z;
+
+int
+main (void)
+{
+  f3 (x, y, &z);
+  if (z != 211)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/g++.target/aarch64/sve_catch_6.C b/gcc/testsuite/g++.target/aarch64/sve_catch_6.C
new file mode 100644
index 00000000000..184d7ee111e
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve_catch_6.C
@@ -0,0 +1,5 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fopenmp-simd -fomit-frame-pointer" } */
+/* { dg-options "-O3 -fopenmp-simd -fomit-frame-pointer -march=armv8-a+sve" { target aarch64_sve_hw } } */
+
+#include "sve_catch_5.C"
diff --git a/gcc/testsuite/gcc.c-torture/compile/pr82549.c b/gcc/testsuite/gcc.c-torture/compile/pr82549.c
new file mode 100644
index 00000000000..11525cde032
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/compile/pr82549.c
@@ -0,0 +1,9 @@
+/* PR tree-optimization/82549 */
+
+int a, b[1];
+
+int
+main ()
+{
+  return !a || b[-2] || b[-2];
+}
diff --git a/gcc/testsuite/gcc.c-torture/compile/pr82816.c b/gcc/testsuite/gcc.c-torture/compile/pr82816.c
new file mode 100644
index 00000000000..8e9bd001bac
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/compile/pr82816.c
@@ -0,0 +1,12 @@
+struct A
+{
+  int b:3;
+} d, e;
+
+int c;
+
+void f ()
+{
+  char g = d.b * e.b;
+  c = g;
+}
diff --git a/gcc/testsuite/gcc.c-torture/execute/20030209-1.c b/gcc/testsuite/gcc.c-torture/execute/20030209-1.c
index 8f076ecb0c7..52f71ec3543 100644
--- a/gcc/testsuite/gcc.c-torture/execute/20030209-1.c
+++ b/gcc/testsuite/gcc.c-torture/execute/20030209-1.c
@@ -1,12 +1,5 @@
-/* { dg-add-options stack_size } */
+/* { dg-require-stack-size "8*100*100" } */
 
-#ifdef STACK_SIZE
-#if STACK_SIZE < 8*100*100
-#define SKIP
-#endif
-#endif
-
-#ifndef SKIP
 double x[100][100];
 int main ()
 {
@@ -18,10 +11,3 @@ int main ()
     abort ();
   exit (0);
 }
-#else
-int
-main ()
-{
-  exit (0);
-}
-#endif
diff --git a/gcc/testsuite/gcc.c-torture/execute/20040805-1.c b/gcc/testsuite/gcc.c-torture/execute/20040805-1.c
index d3208d69f9d..f31109266b1 100644
--- a/gcc/testsuite/gcc.c-torture/execute/20040805-1.c
+++ b/gcc/testsuite/gcc.c-torture/execute/20040805-1.c
@@ -1,6 +1,6 @@
-/* { dg-add-options stack_size } */
+/* { dg-require-stack-size "0x12000" } */
 
-#if __INT_MAX__ < 32768 || (defined(STACK_SIZE) && STACK_SIZE < 0x12000)
+#if __INT_MAX__ < 32768
 int main () { exit (0); }
 #else
 int a[2] = { 2, 3 };
diff --git a/gcc/testsuite/gcc.c-torture/execute/920410-1.c b/gcc/testsuite/gcc.c-torture/execute/920410-1.c
index 44a72bd7bb5..daeff5e3990 100644
--- a/gcc/testsuite/gcc.c-torture/execute/920410-1.c
+++ b/gcc/testsuite/gcc.c-torture/execute/920410-1.c
@@ -1,8 +1,4 @@
-/* { dg-add-options stack_size } */
+/* { dg-require-stack-size "40000 * 4 + 256" } */
 
-#define STACK_REQUIREMENT (40000 * 4 + 256)
-#if defined (STACK_SIZE) && STACK_SIZE < STACK_REQUIREMENT
-main () { exit (0); }
-#else
 main(){int d[40000];d[0]=0;exit(0);}
-#endif
+
diff --git a/gcc/testsuite/gcc.c-torture/execute/921113-1.c b/gcc/testsuite/gcc.c-torture/execute/921113-1.c
index d3e44e358d2..824e69f04c4 100644
--- a/gcc/testsuite/gcc.c-torture/execute/921113-1.c
+++ b/gcc/testsuite/gcc.c-torture/execute/921113-1.c
@@ -1,9 +1,4 @@
-/* { dg-add-options stack_size } */
-
-#define STACK_REQUIREMENT (128 * 128 * 4 + 1024)
-#if defined (STACK_SIZE) && STACK_SIZE < STACK_REQUIREMENT
-main () { exit (0); }
-#else
+/* { dg-require-stack-size "128 * 128 * 4 + 1024" } */
 
 typedef struct {
   float wsx;
@@ -62,4 +57,3 @@ main()
   exit(0);
 }
 
-#endif
diff --git a/gcc/testsuite/gcc.c-torture/execute/921208-2.c b/gcc/testsuite/gcc.c-torture/execute/921208-2.c
index da9ee524924..01e14f8cffe 100644
--- a/gcc/testsuite/gcc.c-torture/execute/921208-2.c
+++ b/gcc/testsuite/gcc.c-torture/execute/921208-2.c
@@ -1,10 +1,5 @@
 /* { dg-require-effective-target untyped_assembly } */
-/* { dg-add-options stack_size } */
-
-#define STACK_REQUIREMENT (100000 * 4 + 1024)
-#if defined (STACK_SIZE) && STACK_SIZE < STACK_REQUIREMENT
-main () { exit (0); }
-#else
+/* { dg-require-stack-size "100000 * 4 + 1024" } */
 
 g(){}
 
@@ -25,5 +20,3 @@ main ()
   f();
   exit(0);
 }
-
-#endif
diff --git a/gcc/testsuite/gcc.c-torture/execute/comp-goto-1.c b/gcc/testsuite/gcc.c-torture/execute/comp-goto-1.c
index 2a840521487..4379fe70e9c 100644
--- a/gcc/testsuite/gcc.c-torture/execute/comp-goto-1.c
+++ b/gcc/testsuite/gcc.c-torture/execute/comp-goto-1.c
@@ -1,9 +1,9 @@
 /* { dg-require-effective-target label_values } */
-/* { dg-add-options stack_size } */
+/* { dg-require-stack-size "4000" } */
 
 #include <stdlib.h>
 
-#if (!defined(STACK_SIZE) || STACK_SIZE >= 4000) && __INT_MAX__ >= 2147483647
+#if __INT_MAX__ >= 2147483647
 typedef unsigned int uint32;
 typedef signed int sint32;
 
diff --git a/gcc/testsuite/gcc.c-torture/execute/pr20621-1.c b/gcc/testsuite/gcc.c-torture/execute/pr20621-1.c
index 9d0119b9689..b2a9785cd6f 100644
--- a/gcc/testsuite/gcc.c-torture/execute/pr20621-1.c
+++ b/gcc/testsuite/gcc.c-torture/execute/pr20621-1.c
@@ -1,12 +1,9 @@
-/* { dg-add-options stack_size } */
+/* { dg-require-stack-size "0x10000" } */
 
 /* When generating o32 MIPS PIC, main's $gp save slot was out of range
    of a single load instruction.  */
 struct big { int i[sizeof (int) >= 4 && sizeof (void *) >= 4 ? 0x4000 : 4]; };
 struct big gb;
 int foo (struct big b, int x) { return b.i[x]; }
-#if defined(STACK_SIZE) && STACK_SIZE <= 0x10000
-int main (void) { return 0; }
-#else
 int main (void) { return foo (gb, 0) + foo (gb, 1); }
-#endif
+
diff --git a/gcc/testsuite/gcc.c-torture/execute/pr28982b.c b/gcc/testsuite/gcc.c-torture/execute/pr28982b.c
index f28425e8fd7..b68fa9a7051 100644
--- a/gcc/testsuite/gcc.c-torture/execute/pr28982b.c
+++ b/gcc/testsuite/gcc.c-torture/execute/pr28982b.c
@@ -1,11 +1,8 @@
-/* { dg-add-options stack_size } */
+/* { dg-require-stack-size "0x80100" } */
 
 /* Like pr28982a.c, but with the spill slots outside the range of
    a single sp-based load on ARM.  This test tests for cases where
    the addresses in the base and index reloads require further reloads.  */
-#if defined(STACK_SIZE) && STACK_SIZE <= 0x80100
-int main (void) { return 0; }
-#else
 #define NITER 4
 #define NVARS 20
 #define MULTI(X) \
@@ -57,4 +54,3 @@ main (void)
       return 1;
   return 0;
 }
-#endif
diff --git a/gcc/testsuite/gcc.c-torture/execute/pr81423.c b/gcc/testsuite/gcc.c-torture/execute/pr81423.c
index 731aa8f1c65..be7413be334 100644
--- a/gcc/testsuite/gcc.c-torture/execute/pr81423.c
+++ b/gcc/testsuite/gcc.c-torture/execute/pr81423.c
@@ -1,3 +1,5 @@
+/* PR rtl-optimization/81423 */
+
 extern void abort (void);
 
 unsigned long long int ll = 0;
@@ -10,11 +12,11 @@ foo (void)
 {
   ll = -5597998501375493990LL;
 
-  ll = (5677365550390624949L - ll) - (ull1 > 0);
+  ll = (unsigned int) (5677365550390624949LL - ll) - (ull1 > 0);
   unsigned long long int ull3;
   ull3 = (unsigned int)
-    (2067854353L <<
-     (((ll + -2129105131L) ^ 10280750144413668236ULL) -
+    (2067854353LL <<
+     (((ll + -2129105131LL) ^ 10280750144413668236ULL) -
       10280750143997242009ULL)) >> ((2873442921854271231ULL | ull2)
 				    - 12098357307243495419ULL);
 
@@ -24,9 +26,10 @@ foo (void)
 int
 main (void)
 {
-  /* We need a long long of exactly 64 bits for this test.  */
-  ll--;
-  if (ll != 0xffffffffffffffffULL)
+  /* We need a long long of exactly 64 bits and int of exactly 32 bits
+     for this test.  */
+  if (__SIZEOF_LONG_LONG__ * __CHAR_BIT__ != 64
+      || __SIZEOF_INT__ * __CHAR_BIT__ != 32)
     return 0;
 
   ull3 = foo ();
diff --git a/gcc/testsuite/gcc.dg/Walloca-15.c b/gcc/testsuite/gcc.dg/Walloca-15.c
new file mode 100644
index 00000000000..f34ffd98b61
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Walloca-15.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target alloca } */
+/* { dg-options "-Walloca-larger-than=128 -O2" } */
+
+typedef __SIZE_TYPE__ size_t;
+
+void bar (void*);
+
+void foo1 (size_t len)
+{
+  bar (__builtin_alloca_with_align_and_max (len, 8, 128));
+}
+
+void foo2 (size_t len)
+{
+  bar (__builtin_alloca_with_align_and_max (len, 8, 256)); /* { dg-warning "may be too large" } */
+}
diff --git a/gcc/testsuite/gcc.dg/asan/pr82517.c b/gcc/testsuite/gcc.dg/asan/pr82517.c
new file mode 100644
index 00000000000..c7743ecb8b1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asan/pr82517.c
@@ -0,0 +1,43 @@
+/* PR sanitizer/82517.  */
+
+static int *pp;
+
+void
+baz ()
+{
+  return;
+}
+
+void
+bar (int *p)
+{
+  *p = 1;
+}
+
+void
+foo (int a)
+{
+  if (a == 2)
+    {
+    lab:
+      baz ();
+      return;
+    }
+  if (a > 1)
+    {
+      int x __attribute__ ((aligned (256)));
+      pp = &x;
+      bar (&x);
+      if (!x)
+	goto lab;
+    }
+}
+
+int
+main (int argc, char **argv)
+{
+  foo (4);
+  foo (3);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/asan/pr82545.c b/gcc/testsuite/gcc.dg/asan/pr82545.c
new file mode 100644
index 00000000000..8870db3653f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asan/pr82545.c
@@ -0,0 +1,17 @@
+/* PR sanitizer/82545.  */
+/* { dg-do compile } */
+
+extern void c(int);
+extern void d(void);
+
+void *buf[5];
+
+void a(void) {
+  {
+    int b;
+    &b;
+    __builtin_setjmp(buf);
+    c(b);
+  }
+  d();
+}
diff --git a/gcc/testsuite/gcc.dg/attr-alloc_size-11.c b/gcc/testsuite/gcc.dg/attr-alloc_size-11.c
index fe6154a0d77..6e109955183 100644
--- a/gcc/testsuite/gcc.dg/attr-alloc_size-11.c
+++ b/gcc/testsuite/gcc.dg/attr-alloc_size-11.c
@@ -47,8 +47,8 @@ typedef __SIZE_TYPE__    size_t;
 
 /* The following tests fail because of missing range information.  The xfail
    exclusions are PR79356.  */
-TEST (signed char, SCHAR_MIN + 2, ALLOC_MAX);   /* { dg-warning "argument 1 range \\\[13, \[0-9\]+\\\] exceeds maximum object size 12" "missing range info for signed char" { xfail { ! { aarch64*-*-* arm*-*-* alpha*-*-* ia64-*-* mips*-*-* powerpc*-*-* sparc*-*-* s390*-*-* } } } } */
-TEST (short, SHRT_MIN + 2, ALLOC_MAX); /* { dg-warning "argument 1 range \\\[13, \[0-9\]+\\\] exceeds maximum object size 12" "missing range info for short" { xfail { ! { aarch64*-*-* arm*-*-* alpha*-*-* ia64-*-* mips*-*-* powerpc*-*-* sparc*-*-* s390x-*-* } } } } */
+TEST (signed char, SCHAR_MIN + 2, ALLOC_MAX);   /* { dg-warning "argument 1 range \\\[13, \[0-9\]+\\\] exceeds maximum object size 12" "missing range info for signed char" { xfail { ! { aarch64*-*-* arm*-*-* alpha*-*-* ia64-*-* mips*-*-* powerpc*-*-* sparc*-*-* s390*-*-* visium-*-* } } } } */
+TEST (short, SHRT_MIN + 2, ALLOC_MAX); /* { dg-warning "argument 1 range \\\[13, \[0-9\]+\\\] exceeds maximum object size 12" "missing range info for short" { xfail { ! { aarch64*-*-* arm*-*-* alpha*-*-* ia64-*-* mips*-*-* powerpc*-*-* sparc*-*-* s390x-*-* visium-*-* } } } } */
 TEST (int, INT_MIN + 2, ALLOC_MAX);    /* { dg-warning "argument 1 range \\\[13, \[0-9\]+\\\] exceeds maximum object size 12" } */
 TEST (int, -3, ALLOC_MAX);             /* { dg-warning "argument 1 range \\\[13, \[0-9\]+\\\] exceeds maximum object size 12" } */
 TEST (int, -2, ALLOC_MAX);             /* { dg-warning "argument 1 range \\\[13, \[0-9\]+\\\] exceeds maximum object size 12" } */
diff --git a/gcc/testsuite/gcc.dg/c17-version-1.c b/gcc/testsuite/gcc.dg/c17-version-1.c
new file mode 100644
index 00000000000..4e69a6eec11
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/c17-version-1.c
@@ -0,0 +1,9 @@
+/* Test __STDC_VERSION__ for C17.  Test -std=c17.  */
+/* { dg-do compile } */
+/* { dg-options "-std=c17 -pedantic-errors" } */
+
+#if __STDC_VERSION__ == 201710L
+int i;
+#else
+#error "Bad __STDC_VERSION__."
+#endif
diff --git a/gcc/testsuite/gcc.dg/c17-version-2.c b/gcc/testsuite/gcc.dg/c17-version-2.c
new file mode 100644
index 00000000000..3f367204094
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/c17-version-2.c
@@ -0,0 +1,9 @@
+/* Test __STDC_VERSION__ for C17.  Test -std=iso9899:2017.  */
+/* { dg-do compile } */
+/* { dg-options "-std=iso9899:2017 -pedantic-errors" } */
+
+#if __STDC_VERSION__ == 201710L
+int i;
+#else
+#error "Bad __STDC_VERSION__."
+#endif
diff --git a/gcc/testsuite/gcc.dg/c90-const-expr-11.c b/gcc/testsuite/gcc.dg/c90-const-expr-11.c
index e4f2aff7874..a2720c47bf4 100644
--- a/gcc/testsuite/gcc.dg/c90-const-expr-11.c
+++ b/gcc/testsuite/gcc.dg/c90-const-expr-11.c
@@ -20,7 +20,7 @@ f (void)
   /* Overflow.  */
   struct t b = { INT_MAX + 1 }; /* { dg-warning "integer overflow in expression" } */
   /* { dg-error "overflow in constant expression" "constant" { target *-*-* } .-1 } */
-  struct t c = { DBL_MAX }; /* { dg-warning "overflow in conversion from .double. to .int. chages value " } */
+  struct t c = { DBL_MAX }; /* { dg-warning "overflow in conversion from .double. to .int. changes value " } */
   /* { dg-error "overflow in constant expression" "constant" { target *-*-* } .-1 } */
   /* Bad operator outside sizeof.  */
   struct s d = { 1 ? 1.0 : atan (a.d) }; /* { dg-error "is not a constant expression|near initialization" } */
diff --git a/gcc/testsuite/gcc.dg/debug/dwarf2/asm-line1.c b/gcc/testsuite/gcc.dg/debug/dwarf2/asm-line1.c
index 3773e1c83c3..aebfcad6008 100644
--- a/gcc/testsuite/gcc.dg/debug/dwarf2/asm-line1.c
+++ b/gcc/testsuite/gcc.dg/debug/dwarf2/asm-line1.c
@@ -1,6 +1,6 @@
 /* PR debug/50983 */
 /* { dg-do compile { target *-*-gnu* } } */
-/* { dg-options "-O0 -gdwarf" } */
+/* { dg-options "-O0 -gdwarf -gno-column-info" } */
 /* { dg-final { scan-assembler "is_stmt 1" } } */
 
 int i;
diff --git a/gcc/testsuite/gcc.dg/debug/dwarf2/discriminator.c b/gcc/testsuite/gcc.dg/debug/dwarf2/discriminator.c
index b77f7b1bfff..fa24de8d7d4 100644
--- a/gcc/testsuite/gcc.dg/debug/dwarf2/discriminator.c
+++ b/gcc/testsuite/gcc.dg/debug/dwarf2/discriminator.c
@@ -1,7 +1,7 @@
 /* HAVE_AS_DWARF2_DEBUG_LINE macro needs to be defined to pass the unittest.
    However, dg cannot access it, so we restrict to GNU targets.  */
 /* { dg-do compile { target *-*-gnu* } } */
-/* { dg-options "-O0 -gdwarf" } */
+/* { dg-options "-O0 -gdwarf -gno-column-info" } */
 /* { dg-final { scan-assembler "loc \[0-9] 11 \[0-9]( is_stmt \[0-9])?\n" } } */
 /* { dg-final { scan-assembler "loc \[0-9] 11 \[0-9]( is_stmt \[0-9])? discriminator 2\n" } } */
 /* { dg-final { scan-assembler "loc \[0-9] 11 \[0-9]( is_stmt \[0-9])? discriminator 1\n" } } */
diff --git a/gcc/testsuite/gcc.dg/debug/dwarf2/pr53948.c b/gcc/testsuite/gcc.dg/debug/dwarf2/pr53948.c
index 0ec3e84d704..4485e19c1cd 100644
--- a/gcc/testsuite/gcc.dg/debug/dwarf2/pr53948.c
+++ b/gcc/testsuite/gcc.dg/debug/dwarf2/pr53948.c
@@ -1,7 +1,7 @@
 /* Test that we have line information for the line
    with local variable initializations.  */
 /* { dg-options "-O0 -gdwarf -dA" } */
-/* { dg-final { scan-assembler ".loc 1 8 0|\[#/!\]\[ \t\]+line 8" } } */
+/* { dg-final { scan-assembler ".loc 1 8 \[0-9\]|\[#/!\]\[ \t\]+line 8" } } */
 
 
 int f (register int a, register int b) {
diff --git a/gcc/testsuite/gcc.dg/debug/dwarf2/sso.c b/gcc/testsuite/gcc.dg/debug/dwarf2/sso-1.c
index 698c636a130..698c636a130 100644
--- a/gcc/testsuite/gcc.dg/debug/dwarf2/sso.c
+++ b/gcc/testsuite/gcc.dg/debug/dwarf2/sso-1.c
diff --git a/gcc/testsuite/gcc.dg/debug/dwarf2/sso-2.c b/gcc/testsuite/gcc.dg/debug/dwarf2/sso-2.c
new file mode 100644
index 00000000000..0965084d260
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/debug/dwarf2/sso-2.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-gdwarf-3 -dA" } */
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define REVERSE_SSO __attribute__((scalar_storage_order("big-endian")));
+#else
+#define REVERSE_SSO __attribute__((scalar_storage_order("little-endian")));
+#endif
+
+struct reverse
+{
+  int i;
+  short a[4];
+} REVERSE_SSO;
+
+struct native
+{
+  int i;
+  short a[4];
+};
+
+struct reverse R;
+struct native  N;
+
+/* Verify that we have endianity on the common base type of 'i' and the
+ *  element of 'a' in the first 2 structures.  */
+/* { dg-final { scan-assembler-times " DW_AT_endianity" 2 } } */
+/* { dg-final { scan-assembler-times "DIE \\(\[0-9a-z\]*\\) DW_TAG_base_type" 5 } } */
diff --git a/gcc/testsuite/gcc.dg/debug/dwarf2/sso-3.c b/gcc/testsuite/gcc.dg/debug/dwarf2/sso-3.c
new file mode 100644
index 00000000000..004327c78ad
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/debug/dwarf2/sso-3.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-gdwarf-3 -dA" } */
+
+typedef int   int_t;
+typedef short short_t;
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define REVERSE_SSO __attribute__((scalar_storage_order("big-endian")));
+#else
+#define REVERSE_SSO __attribute__((scalar_storage_order("little-endian")));
+#endif
+
+struct reverse
+{
+  int_t i;
+  short_t a[4];
+} REVERSE_SSO;
+
+struct native
+{
+  int_t i;
+  short_t a[4];
+};
+
+struct reverse R;
+struct native  N;
+
+/* Verify that we have endianity on the common base type of 'i' and the
+ *  element of 'a' in the first 2 structures.  */
+/* { dg-final { scan-assembler-times " DW_AT_endianity" 2 } } */
+/* { dg-final { scan-assembler-times "DIE \\(\[0-9a-z\]*\\) DW_TAG_base_type" 5 } } */
diff --git a/gcc/testsuite/gcc.dg/fold-cond_expr-1.c b/gcc/testsuite/gcc.dg/fold-cond-2.c
index 68ec75480ad..68ec75480ad 100644
--- a/gcc/testsuite/gcc.dg/fold-cond_expr-1.c
+++ b/gcc/testsuite/gcc.dg/fold-cond-2.c
diff --git a/gcc/testsuite/gcc.dg/fold-cond-3.c b/gcc/testsuite/gcc.dg/fold-cond-3.c
new file mode 100644
index 00000000000..fe0ba65ebac
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/fold-cond-3.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-fdump-tree-original" } */
+
+unsigned long f1 (int x)
+{
+  return x > 0 ? (unsigned long) x : 0;
+}
+
+unsigned long f2 (int x, int y)
+{
+  return x > y ? (unsigned long) x : (unsigned long) y;
+}
+
+unsigned long f3 (int x)
+{
+  return x < 0 ? (unsigned long) x : 0;
+}
+
+unsigned long f4 (int x, int y)
+{
+  return x < y ? (unsigned long) x : (unsigned long) y;
+}
+
+unsigned long f5 (unsigned int x, unsigned int y)
+{
+  return x > y ? (unsigned long) x : (unsigned long) y;
+}
+
+unsigned long f6 (unsigned int x, unsigned int y)
+{
+  return x < y ? (unsigned long) x : (unsigned long) y;
+}
+
+/* { dg-final { scan-tree-dump-times "MAX_EXPR" 3 "original"} } */
+/* { dg-final { scan-tree-dump-times "MIN_EXPR" 3 "original"} } */
diff --git a/gcc/testsuite/gcc.dg/gimplefe-27.c b/gcc/testsuite/gcc.dg/gimplefe-27.c
new file mode 100644
index 00000000000..604a2cc2fcc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/gimplefe-27.c
@@ -0,0 +1,9 @@
+/* { dg-options "-O -fgimple" } */
+
+int __GIMPLE ()
+p (int n)
+{
+  int _2;
+  _2 = n_1(D) != 0 ? 2 : 0;
+  return _2;
+}
diff --git a/gcc/testsuite/gcc.dg/graphite/interchange-3.c b/gcc/testsuite/gcc.dg/graphite/interchange-3.c
index 4aec824183a..cb93f5d0920 100644
--- a/gcc/testsuite/gcc.dg/graphite/interchange-3.c
+++ b/gcc/testsuite/gcc.dg/graphite/interchange-3.c
@@ -47,4 +47,4 @@ main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump "tiled" "graphite" } } */
+/* { dg-final { scan-tree-dump "tiled" "graphite" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/graphite/interchange-7.c b/gcc/testsuite/gcc.dg/graphite/interchange-7.c
index 81a6d832327..81a0a4daf55 100644
--- a/gcc/testsuite/gcc.dg/graphite/interchange-7.c
+++ b/gcc/testsuite/gcc.dg/graphite/interchange-7.c
@@ -46,4 +46,4 @@ main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump "tiled" "graphite" } } */
+/* { dg-final { scan-tree-dump "tiled" "graphite" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/graphite/interchange-9.c b/gcc/testsuite/gcc.dg/graphite/interchange-9.c
index 88a357893e9..75d269e4527 100644
--- a/gcc/testsuite/gcc.dg/graphite/interchange-9.c
+++ b/gcc/testsuite/gcc.dg/graphite/interchange-9.c
@@ -44,4 +44,4 @@ main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump "tiled" "graphite" } } */
+/* { dg-final { scan-tree-dump "tiled" "graphite" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/graphite/pr35356-3.c b/gcc/testsuite/gcc.dg/graphite/pr35356-3.c
index f2827a2bb6d..8db042ffc6f 100644
--- a/gcc/testsuite/gcc.dg/graphite/pr35356-3.c
+++ b/gcc/testsuite/gcc.dg/graphite/pr35356-3.c
@@ -36,4 +36,5 @@ match (void)
    "Y[winner].y > 0".  This could be fixed when we will use predicates
    for such cases.  */
 
-/* { dg-final { scan-tree-dump-times "loop_1" 0 "graphite" } } */
+/* { dg-final { scan-tree-dump-times "loop_1" 0 "graphite" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "number of SCoPs: 0" "graphite" } } */
diff --git a/gcc/testsuite/gcc.dg/graphite/pr81373-2.c b/gcc/testsuite/gcc.dg/graphite/pr81373-2.c
new file mode 100644
index 00000000000..6a654bec977
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/graphite/pr81373-2.c
@@ -0,0 +1,40 @@
+/* { dg-options "-fno-tree-scev-cprop -floop-nest-optimize -fgraphite-identity -O -fdump-tree-graphite-all" } */
+
+void bar (void);
+
+int toto()
+{
+  int i, j, k;
+  int a[101][100];
+  int b[100];
+
+  for (i = 1; i < 100; i++)
+    {
+      for (j = 1; j < 100; j++)
+	for (k = 1; k < 100; k++)
+	  a[j][k] = a[j+1][i-1] + 2;
+
+      b[i] = b[i-1] + 2;
+
+      bar ();
+
+      for (j = 1; j < 100; j++)
+	a[j][i] = a[j+1][i-1] + 2;
+
+      b[i] = b[i-1] + 2;
+
+      bar ();
+
+      for (j = 1; j < 100; j++)
+	a[j][i] = a[j+1][i-1] + 2;
+
+      b[i] = a[i-1][i] + 2;
+
+      for (j = 1; j < 100; j++)
+	a[j][i] = a[j+1][i-1] + 2;
+    }
+
+  return a[3][5] + b[1];
+}
+
+/* { dg-final { scan-tree-dump-times "number of SCoPs: 2" 1 "graphite"} } */
diff --git a/gcc/testsuite/gcc.dg/graphite/pr82563.c b/gcc/testsuite/gcc.dg/graphite/pr82563.c
new file mode 100644
index 00000000000..cd492fa79c8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/graphite/pr82563.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -floop-nest-optimize" } */
+
+int tj, cw, xf;
+
+void
+zp (int *ei)
+{
+  for (;;)
+    {
+      int hd = 0;
+
+      if (cw != 0 && xf != 0)
+	{
+	  for (hd = 0; hd < 3; ++hd)
+	    cw = (tj != 0) ? 0 : *ei;
+	  for (;;)
+	    ;
+	}
+
+      while (tj != 0)
+	tj = (__UINTPTR_TYPE__)&hd;
+    }
+}
diff --git a/gcc/testsuite/gcc.dg/graphite/scop-10.c b/gcc/testsuite/gcc.dg/graphite/scop-10.c
index 39ed5d7ea7b..20d53510b4e 100644
--- a/gcc/testsuite/gcc.dg/graphite/scop-10.c
+++ b/gcc/testsuite/gcc.dg/graphite/scop-10.c
@@ -4,7 +4,7 @@ int toto()
 {
   int i, j, k;
   int a[100][100];
-  int b[100];
+  int b[200];
 
   for (i = 1; i < 100; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/graphite/scop-7.c b/gcc/testsuite/gcc.dg/graphite/scop-7.c
index 3e337d0c603..2f0a50470e9 100644
--- a/gcc/testsuite/gcc.dg/graphite/scop-7.c
+++ b/gcc/testsuite/gcc.dg/graphite/scop-7.c
@@ -4,7 +4,7 @@ int toto()
 {
   int i, j, k;
   int a[100][100];
-  int b[100];
+  int b[200];
 
   for (i = 1; i < 100; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/graphite/scop-8.c b/gcc/testsuite/gcc.dg/graphite/scop-8.c
index 71d5c531fb8..3ceb5d874d6 100644
--- a/gcc/testsuite/gcc.dg/graphite/scop-8.c
+++ b/gcc/testsuite/gcc.dg/graphite/scop-8.c
@@ -4,7 +4,7 @@ int toto()
 {
   int i, j, k;
   int a[100][100];
-  int b[100];
+  int b[200];
 
   for (i = 1; i < 100; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/graphite/uns-interchange-9.c b/gcc/testsuite/gcc.dg/graphite/uns-interchange-9.c
index cc108c2bbc3..fb36afe003e 100644
--- a/gcc/testsuite/gcc.dg/graphite/uns-interchange-9.c
+++ b/gcc/testsuite/gcc.dg/graphite/uns-interchange-9.c
@@ -45,4 +45,4 @@ main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump "tiled" "graphite" } } */
+/* { dg-final { scan-tree-dump "tiled" "graphite" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/ipa/propmalloc-1.c b/gcc/testsuite/gcc.dg/ipa/propmalloc-1.c
new file mode 100644
index 00000000000..9a95f817079
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ipa/propmalloc-1.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-ipa-pure-const-details" } */
+
+__attribute__((noinline, no_icf, used))
+static void *f(__SIZE_TYPE__ n)
+{
+  void *p = __builtin_malloc (n);
+  if (p == 0)
+    __builtin_abort ();
+  return p;
+}
+
+__attribute__((noinline, no_icf, used))
+static void *bar(__SIZE_TYPE__ n)
+{
+  void *p = f (n);
+  return p;
+}
+
+/* { dg-final { scan-ipa-dump "Function f found to be malloc" "pure-const" } } */
+/* { dg-final { scan-ipa-dump "Function bar found to be malloc" "pure-const" } } */
diff --git a/gcc/testsuite/gcc.dg/ipa/propmalloc-2.c b/gcc/testsuite/gcc.dg/ipa/propmalloc-2.c
new file mode 100644
index 00000000000..95b2fd74a7a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ipa/propmalloc-2.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-ipa-pure-const-details" } */
+
+__attribute__((noinline, used, no_icf))
+static void *foo (__SIZE_TYPE__ n)
+{
+  return __builtin_malloc (n * 10);
+}
+
+__attribute__((noinline, used, no_icf))
+static void *bar(__SIZE_TYPE__ n, int cond)
+{
+  void *p;
+  if (cond)
+    p = foo (n);
+  else
+    p = __builtin_malloc (n);
+
+  return p;
+}
+
+/* { dg-final { scan-ipa-dump "Function foo found to be malloc" "pure-const" } } */
+/* { dg-final { scan-ipa-dump "Function bar found to be malloc" "pure-const" } } */
diff --git a/gcc/testsuite/gcc.dg/ipa/propmalloc-3.c b/gcc/testsuite/gcc.dg/ipa/propmalloc-3.c
new file mode 100644
index 00000000000..13558ddd07d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ipa/propmalloc-3.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-ipa-pure-const-details" } */
+
+static void *foo(__SIZE_TYPE__, int) __attribute__((noinline, no_icf, used));
+
+__attribute__((noinline, used, no_icf))
+static void *bar(__SIZE_TYPE__ n, int m)
+{
+  return foo (n, m);
+}
+
+static void *foo(__SIZE_TYPE__ n, int m)
+{
+  void *p;
+  if (m > 0)
+    p = bar (n, --m);
+  else
+    p = __builtin_malloc (n);
+
+  return p;
+}
+
+/* { dg-final { scan-ipa-dump "Function foo found to be malloc" "pure-const" } } */
+/* { dg-final { scan-ipa-dump "Function bar found to be malloc" "pure-const" } } */
diff --git a/gcc/testsuite/gcc.dg/no-strict-overflow-3.c b/gcc/testsuite/gcc.dg/no-strict-overflow-3.c
index fd4defbd447..d68008a3dde 100644
--- a/gcc/testsuite/gcc.dg/no-strict-overflow-3.c
+++ b/gcc/testsuite/gcc.dg/no-strict-overflow-3.c
@@ -9,7 +9,7 @@
 int
 foo (int i, int j)
 {
-  return i + 100 < j + 1000;
+  return i + 100 < j + 1234;
 }
 
-/* { dg-final { scan-tree-dump "1000" "optimized" } } */
+/* { dg-final { scan-tree-dump "1234" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/noncompile/920923-1.c b/gcc/testsuite/gcc.dg/noncompile/920923-1.c
index 1cb140ebabc..006a07131f9 100644
--- a/gcc/testsuite/gcc.dg/noncompile/920923-1.c
+++ b/gcc/testsuite/gcc.dg/noncompile/920923-1.c
@@ -1,5 +1,6 @@
 /* { dg-message "undeclared identifier is reported only once" "reminder for mmu_base" { target *-*-* } 0 } */
 typedef BYTE unsigned char;	/* { dg-error "expected" } */
+/* { dg-warning "useless type name in empty declaration" ""  { target *-*-* } .-1 } */
 typedef int item_n;
 typedef int perm_set;
 struct PENT { caddr_t v_addr; };/* { dg-error "unknown type name" } */
diff --git a/gcc/testsuite/gcc.dg/overflow-warn-5.c b/gcc/testsuite/gcc.dg/overflow-warn-5.c
index b2c8dc31d95..1a5aa0c6059 100644
--- a/gcc/testsuite/gcc.dg/overflow-warn-5.c
+++ b/gcc/testsuite/gcc.dg/overflow-warn-5.c
@@ -3,5 +3,5 @@
 /* { dg-options "-Woverflow" } */
 
 unsigned char rx_async(unsigned char p) {
-    return p & 512; /* { dg-warning "overflow in conversion from .int. to .unsigned char. chages value" } */
+    return p & 512; /* { dg-warning "overflow in conversion from .int. to .unsigned char. changes value" } */
 }
diff --git a/gcc/testsuite/gcc.dg/overflow-warn-8.c b/gcc/testsuite/gcc.dg/overflow-warn-8.c
index ace605517dc..e76bcac5e07 100644
--- a/gcc/testsuite/gcc.dg/overflow-warn-8.c
+++ b/gcc/testsuite/gcc.dg/overflow-warn-8.c
@@ -7,7 +7,7 @@ void foo (int j)
   int i3 = 1 + INT_MAX; /* { dg-warning "integer overflow" } */
   int i4 = +1 + INT_MAX; /* { dg-warning "integer overflow" } */
   int i5 = (int)((double)1.0 + INT_MAX);
-  int i6 = (double)1.0 + INT_MAX; /* { dg-warning "overflow in conversion from .double. to .int. chages value" } */
+  int i6 = (double)1.0 + INT_MAX; /* { dg-warning "overflow in conversion from .double. to .int. changes value" } */
   int i7 = 0 ? (int)(double)1.0 + INT_MAX : 1;
   int i8 = 1 ? 1 : (int)(double)1.0 + INT_MAX;
   int i9 = j ? (int)(double)1.0 + INT_MAX : 1; /* { dg-warning "integer overflow" } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.c b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.c
index f025f963e69..0bdd877dbd5 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.c
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.c
@@ -45,14 +45,14 @@ show_tree (tree node)
 
   if (richloc.get_num_locations () < 2)
     {
-      error_at_rich_loc (&richloc, "range not found");
+      error_at (&richloc, "range not found");
       return;
     }
 
   enum tree_code code = TREE_CODE (node);
 
   location_range *range = richloc.get_range (1);
-  inform_at_rich_loc (&richloc, "%s", get_tree_code_name (code));
+  inform (&richloc, "%s", get_tree_code_name (code));
 
   /* Recurse.  */
   int min_idx = 0;
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.c b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.c
index 0a8eeba1846..9751e1cd25e 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.c
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.c
@@ -176,7 +176,7 @@ test_show_locus (function *fun)
       rich_location richloc (line_table, get_loc (line, 15));
       add_range (&richloc, get_loc (line, 10), get_loc (line, 14), false);
       add_range (&richloc, get_loc (line, 16), get_loc (line, 16), false);
-      warning_at_rich_loc (&richloc, 0, "test");
+      warning_at (&richloc, 0, "test");
     }
 
   if (0 == strcmp (fnname, "test_simple_2"))
@@ -185,7 +185,7 @@ test_show_locus (function *fun)
       rich_location richloc (line_table, get_loc (line, 24));
       add_range (&richloc, get_loc (line, 6), get_loc (line, 22), false);
       add_range (&richloc, get_loc (line, 26), get_loc (line, 43), false);
-      warning_at_rich_loc (&richloc, 0, "test");
+      warning_at (&richloc, 0, "test");
     }
 
   if (0 == strcmp (fnname, "test_multiline"))
@@ -195,7 +195,7 @@ test_show_locus (function *fun)
       add_range (&richloc, get_loc (line, 7), get_loc (line, 23), false);
       add_range (&richloc, get_loc (line + 1, 9), get_loc (line + 1, 26),
 		 false);
-      warning_at_rich_loc (&richloc, 0, "test");
+      warning_at (&richloc, 0, "test");
     }
 
   if (0 == strcmp (fnname, "test_many_lines"))
@@ -205,7 +205,7 @@ test_show_locus (function *fun)
       add_range (&richloc, get_loc (line, 7), get_loc (line + 4, 65), false);
       add_range (&richloc, get_loc (line + 5, 9), get_loc (line + 10, 61),
 		 false);
-      warning_at_rich_loc (&richloc, 0, "test");
+      warning_at (&richloc, 0, "test");
     }
 
   /* Example of a rich_location where the range is larger than
@@ -216,7 +216,7 @@ test_show_locus (function *fun)
       location_t start = get_loc (line, 12);
       location_t finish = get_loc (line, 16);
       rich_location richloc (line_table, make_location (start, start, finish));
-      warning_at_rich_loc (&richloc, 0, "test");
+      warning_at (&richloc, 0, "test");
     }
 
   /* Example of a single-range location where the range starts
@@ -251,7 +251,7 @@ test_show_locus (function *fun)
       add_range (&richloc, caret_b, caret_b, true);
       global_dc->caret_chars[0] = 'A';
       global_dc->caret_chars[1] = 'B';
-      warning_at_rich_loc (&richloc, 0, "test");
+      warning_at (&richloc, 0, "test");
       global_dc->caret_chars[0] = '^';
       global_dc->caret_chars[1] = '^';
     }
@@ -265,7 +265,7 @@ test_show_locus (function *fun)
       rich_location richloc (line_table, make_location (start, start, finish));
       richloc.add_fixit_insert_before ("{");
       richloc.add_fixit_insert_after ("}");
-      warning_at_rich_loc (&richloc, 0, "example of insertion hints");
+      warning_at (&richloc, 0, "example of insertion hints");
     }
 
   if (0 == strcmp (fnname, "test_fixit_insert_newline"))
@@ -277,7 +277,7 @@ test_show_locus (function *fun)
       location_t case_loc = make_location (case_start, case_start, case_finish);
       rich_location richloc (line_table, case_loc);
       richloc.add_fixit_insert_before (line_start, "      break;\n");
-      warning_at_rich_loc (&richloc, 0, "example of newline insertion hint");
+      warning_at (&richloc, 0, "example of newline insertion hint");
     }
 
   if (0 == strcmp (fnname, "test_fixit_remove"))
@@ -290,7 +290,7 @@ test_show_locus (function *fun)
       src_range.m_start = start;
       src_range.m_finish = finish;
       richloc.add_fixit_remove (src_range);
-      warning_at_rich_loc (&richloc, 0, "example of a removal hint");
+      warning_at (&richloc, 0, "example of a removal hint");
     }
 
   if (0 == strcmp (fnname, "test_fixit_replace"))
@@ -303,7 +303,7 @@ test_show_locus (function *fun)
       src_range.m_start = start;
       src_range.m_finish = finish;
       richloc.add_fixit_replace (src_range, "gtk_widget_show_all");
-      warning_at_rich_loc (&richloc, 0, "example of a replacement hint");
+      warning_at (&richloc, 0, "example of a replacement hint");
     }
 
   if (0 == strcmp (fnname, "test_mutually_exclusive_suggestions"))
@@ -319,14 +319,14 @@ test_show_locus (function *fun)
 	rich_location richloc (line_table, make_location (start, start, finish));
 	richloc.add_fixit_replace (src_range, "replacement_1");
 	richloc.fixits_cannot_be_auto_applied ();
-	warning_at_rich_loc (&richloc, 0, "warning 1");
+	warning_at (&richloc, 0, "warning 1");
       }
 
       {
 	rich_location richloc (line_table, make_location (start, start, finish));
 	richloc.add_fixit_replace (src_range, "replacement_2");
 	richloc.fixits_cannot_be_auto_applied ();
-	warning_at_rich_loc (&richloc, 0, "warning 2");
+	warning_at (&richloc, 0, "warning 2");
       }
     }  
 
@@ -346,7 +346,7 @@ test_show_locus (function *fun)
       richloc.add_range (caret_b, true);
       global_dc->caret_chars[0] = '1';
       global_dc->caret_chars[1] = '2';
-      warning_at_rich_loc (&richloc, 0, "test");
+      warning_at (&richloc, 0, "test");
       global_dc->caret_chars[0] = '^';
       global_dc->caret_chars[1] = '^';
     }
@@ -411,8 +411,8 @@ test_show_locus (function *fun)
 	 statically-allocated buffer in class rich_location,
 	 and then trigger a reallocation of the dynamic buffer.  */
       gcc_assert (richloc.get_num_locations () > 3 + (2 * 16));
-      warning_at_rich_loc (&richloc, 0, "test of %i locations",
-			   richloc.get_num_locations ());
+      warning_at (&richloc, 0, "test of %i locations",
+		  richloc.get_num_locations ());
     }
 }
 
diff --git a/gcc/testsuite/gcc.dg/plugin/poly-int-tests.h b/gcc/testsuite/gcc.dg/plugin/poly-int-tests.h
index 9409ec7bc0f..b7a93856003 100644
--- a/gcc/testsuite/gcc.dg/plugin/poly-int-tests.h
+++ b/gcc/testsuite/gcc.dg/plugin/poly-int-tests.h
@@ -437,64 +437,6 @@ test_must_eq ()
 			 ph::make (0, 3, 5)));
 }
 
-/* Test known_zero.  */
-
-template<unsigned int N, typename C, typename T>
-static void
-test_known_zero ()
-{
-  typedef poly_helper<T> ph;
-
-  ASSERT_EQ (known_zero (ph::make (0, 0, 1)), N <= 2);
-  ASSERT_EQ (known_zero (ph::make (0, 1, 0)), N == 1);
-  ASSERT_TRUE (known_zero (ph::make (0, 0, 0)));
-  ASSERT_FALSE (known_zero (ph::make (1, 0, 0)));
-}
-
-/* Test maybe_nonzero.  */
-
-template<unsigned int N, typename C, typename T>
-static void
-test_maybe_nonzero ()
-{
-  typedef poly_helper<T> ph;
-
-  ASSERT_EQ (maybe_nonzero (ph::make (0, 0, 1)), N == 3);
-  ASSERT_EQ (maybe_nonzero (ph::make (0, 1, 0)), N >= 2);
-  ASSERT_FALSE (maybe_nonzero (ph::make (0, 0, 0)));
-  ASSERT_TRUE (maybe_nonzero (ph::make (1, 0, 0)));
-}
-
-/* Test known_one.  */
-
-template<unsigned int N, typename C, typename T>
-static void
-test_known_one ()
-{
-  typedef poly_helper<T> ph;
-
-  ASSERT_EQ (known_one (ph::make (1, 0, 1)), N <= 2);
-  ASSERT_EQ (known_one (ph::make (1, 1, 0)), N == 1);
-  ASSERT_TRUE (known_one (ph::make (1, 0, 0)));
-  ASSERT_FALSE (known_one (ph::make (0, 0, 0)));
-}
-
-/* Test known_all_ones.  */
-
-template<unsigned int N, typename C, typename T>
-static void
-test_known_all_ones ()
-{
-  typedef poly_helper<T> ph;
-
-  ASSERT_EQ (known_all_ones (ph::make (-1, 0, -1)), N <= 2);
-  ASSERT_EQ (known_all_ones (ph::make (-1, -1, 0)), N == 1);
-  ASSERT_EQ (known_all_ones (ph::make (-1, -1, -1)), N == 1);
-  ASSERT_TRUE (known_all_ones (ph::make (-1, 0, 0)));
-  ASSERT_FALSE (known_all_ones (ph::make (0, 0, 0)));
-  ASSERT_FALSE (known_all_ones (ph::make (1, 0, 0)));
-}
-
 /* Test can_align_p.  */
 
 template<unsigned int N, typename C, typename T>
@@ -903,44 +845,6 @@ test_must_ne_2 ()
   ASSERT_TRUE (must_ne (T (11, 0), T (4, 2)));
 }
 
-/* Test maybe_zero for poly_int<2, C>.  */
-
-template<typename C>
-static void
-test_maybe_zero_2 ()
-{
-  typedef poly_int<2, C> T;
-
-  ASSERT_TRUE (maybe_zero (T (0, 0)));
-  ASSERT_TRUE (maybe_zero (T (0, 1)));
-  ASSERT_TRUE (maybe_zero (T (0, -1)));
-  ASSERT_FALSE (maybe_zero (T (1, 0)));
-  ASSERT_FALSE (maybe_zero (T (1, 2)));
-  ASSERT_FALSE (maybe_zero (T (1, -2)));
-  ASSERT_FALSE (maybe_zero (T (-1, 0)));
-  ASSERT_FALSE (maybe_zero (T (-1, 2)));
-  ASSERT_FALSE (maybe_zero (T (-1, -2)));
-}
-
-/* Test known_nonzero for poly_int<2, C>.  */
-
-template<typename C>
-static void
-test_known_nonzero_2 ()
-{
-  typedef poly_int<2, C> T;
-
-  ASSERT_FALSE (known_nonzero (T (0, 0)));
-  ASSERT_FALSE (known_nonzero (T (0, 1)));
-  ASSERT_FALSE (known_nonzero (T (0, -1)));
-  ASSERT_TRUE (known_nonzero (T (1, 0)));
-  ASSERT_TRUE (known_nonzero (T (1, 2)));
-  ASSERT_TRUE (known_nonzero (T (1, -2)));
-  ASSERT_TRUE (known_nonzero (T (-1, 0)));
-  ASSERT_TRUE (known_nonzero (T (-1, 2)));
-  ASSERT_TRUE (known_nonzero (T (-1, -2)));
-}
-
 /* Test may_le for both signed and unsigned C.  */
 
 template<unsigned int N, typename C, typename T>
@@ -2235,6 +2139,22 @@ test_can_div_away_from_zero_p ()
   ASSERT_EQ (const_quot, C (0));
 }
 
+/* Test known_size_p.  */
+
+template<unsigned int N, typename C, typename T>
+static void
+test_known_size_p ()
+{
+  typedef poly_helper<T> ph;
+
+  ASSERT_EQ (known_size_p (ph::make (-1, 0, -1)), N == 3);
+  ASSERT_EQ (known_size_p (ph::make (-1, -1, 0)), N >= 2);
+  ASSERT_EQ (known_size_p (ph::make (-1, -1, -1)), N >= 2);
+  ASSERT_FALSE (known_size_p (ph::make (-1, 0, 0)));
+  ASSERT_TRUE (known_size_p (ph::make (0, 0, 0)));
+  ASSERT_TRUE (known_size_p (ph::make (1, 0, 0)));
+}
+
 /* Test maybe_in_range_p for both signed and unsigned C.  */
 
 template<unsigned int N, typename C, typename T>
@@ -2633,44 +2553,6 @@ test_signed_must_ne_2 ()
   ASSERT_TRUE (must_ne (T (-3, 4), T (6, -1)));
 }
 
-/* Test maybe_zero for poly_int<2, C>, given that C is signed.  */
-
-template<typename C>
-static void
-test_signed_maybe_zero_2 ()
-{
-  typedef poly_int<2, C> T;
-
-  ASSERT_TRUE (maybe_zero (T (3, -3)));
-  ASSERT_TRUE (maybe_zero (T (16, -4)));
-  ASSERT_TRUE (maybe_zero (T (-15, 5)));
-  ASSERT_FALSE (maybe_zero (T (3, -4)));
-  ASSERT_FALSE (maybe_zero (T (3, -6)));
-  ASSERT_FALSE (maybe_zero (T (15, -4)));
-  ASSERT_FALSE (maybe_zero (T (17, -4)));
-  ASSERT_FALSE (maybe_zero (T (-14, 5)));
-  ASSERT_FALSE (maybe_zero (T (-16, 5)));
-}
-
-/* Test known_nonzero for poly_int<2, C>, given that C is signed.  */
-
-template<typename C>
-static void
-test_signed_known_nonzero_2 ()
-{
-  typedef poly_int<2, C> T;
-
-  ASSERT_FALSE (known_nonzero (T (3, -3)));
-  ASSERT_FALSE (known_nonzero (T (16, -4)));
-  ASSERT_FALSE (known_nonzero (T (-15, 5)));
-  ASSERT_TRUE (known_nonzero (T (3, -4)));
-  ASSERT_TRUE (known_nonzero (T (3, -6)));
-  ASSERT_TRUE (known_nonzero (T (15, -4)));
-  ASSERT_TRUE (known_nonzero (T (17, -4)));
-  ASSERT_TRUE (known_nonzero (T (-14, 5)));
-  ASSERT_TRUE (known_nonzero (T (-16, 5)));
-}
-
 /* Test negation for signed C, both via operators and wi::.  */
 
 template<unsigned int N, typename C, typename RC, typename T>
@@ -4623,76 +4505,16 @@ test_uhwi ()
 				  wi::uhwi (210, 16)));
 }
 
-/* Test known_zero for non-polynomial T.  */
-
-template<typename T>
-static void
-test_nonpoly_known_zero ()
-{
-  ASSERT_TRUE (known_zero (T (0)));
-  ASSERT_FALSE (known_zero (T (1)));
-  ASSERT_FALSE (known_zero (T (2)));
-  ASSERT_FALSE (known_zero (T (-1)));
-}
-
-/* Test maybe_zero for non-polynomial T.  */
-
-template<typename T>
-static void
-test_nonpoly_maybe_zero ()
-{
-  ASSERT_TRUE (maybe_zero (T (0)));
-  ASSERT_FALSE (maybe_zero (T (1)));
-  ASSERT_FALSE (maybe_zero (T (2)));
-  ASSERT_FALSE (maybe_zero (T (-1)));
-}
-
-/* Test known_nonzero for non-polynomial T.  */
-
-template<typename T>
-static void
-test_nonpoly_known_nonzero ()
-{
-  ASSERT_FALSE (known_nonzero (T (0)));
-  ASSERT_TRUE (known_nonzero (T (1)));
-  ASSERT_TRUE (known_nonzero (T (2)));
-  ASSERT_TRUE (known_nonzero (T (-1)));
-}
-
-/* Test maybe_nonzero for non-polynomial T.  */
-
-template<typename T>
-static void
-test_nonpoly_maybe_nonzero ()
-{
-  ASSERT_FALSE (maybe_nonzero (T (0)));
-  ASSERT_TRUE (maybe_nonzero (T (1)));
-  ASSERT_TRUE (maybe_nonzero (T (2)));
-  ASSERT_TRUE (maybe_nonzero (T (-1)));
-}
-
-/* Test known_one for non-polynomial T.  */
-
-template<typename T>
-static void
-test_nonpoly_known_one ()
-{
-  ASSERT_FALSE (known_one (T (0)));
-  ASSERT_TRUE (known_one (T (1)));
-  ASSERT_FALSE (known_one (T (2)));
-  ASSERT_FALSE (known_one (T (-1)));
-}
-
-/* Test known_all_ones for non-polynomial T.  */
+/* Test known_size_p for non-polynomial T.  */
 
 template<typename T>
 static void
-test_nonpoly_known_all_ones ()
+test_nonpoly_known_size_p ()
 {
-  ASSERT_FALSE (known_all_ones (T (0)));
-  ASSERT_FALSE (known_all_ones (T (1)));
-  ASSERT_FALSE (known_all_ones (T (2)));
-  ASSERT_TRUE (known_all_ones (T (-1)));
+  ASSERT_TRUE (known_size_p (T (0)));
+  ASSERT_TRUE (known_size_p (T (1)));
+  ASSERT_TRUE (known_size_p (T (2)));
+  ASSERT_FALSE (known_size_p (T (-1)));
 }
 
 /* Test poly-int.h operations on non-polynomial type T.  */
@@ -4701,12 +4523,7 @@ template<typename T>
 static void
 test_nonpoly_type ()
 {
-  test_nonpoly_known_zero<T> ();
-  test_nonpoly_maybe_zero<T> ();
-  test_nonpoly_known_nonzero<T> ();
-  test_nonpoly_maybe_nonzero<T> ();
-  test_nonpoly_known_one<T> ();
-  test_nonpoly_known_all_ones<T> ();
+  test_nonpoly_known_size_p<T> ();
 }
 
 /* Test poly-int.h operations on non-polynomial values.  */
@@ -4747,10 +4564,6 @@ test_general ()
   test_shift_left<N, C, T> ();
   test_may_ne<N, C, T> ();
   test_must_eq<N, C, T> ();
-  test_known_zero<N, C, T> ();
-  test_maybe_nonzero<N, C, T> ();
-  test_known_one<N, C, T> ();
-  test_known_all_ones<N, C, T> ();
   test_can_align_p<N, C, T> ();
   test_can_align_up<N, C, T> ();
   test_can_align_down<N, C, T> ();
@@ -4764,6 +4577,7 @@ test_general ()
   test_force_get_misalignment<N, C, T> ();
   test_known_alignment<N, C, T> ();
   test_can_ior_p<N, C, T> ();
+  test_known_size_p<N, C, T> ();
 }
 
 /* Test things that work for poly_int<2, C>, given that C is signed.  */
@@ -4774,8 +4588,6 @@ test_ordered_2 ()
 {
   test_may_eq_2<C> ();
   test_must_ne_2<C> ();
-  test_maybe_zero_2<C> ();
-  test_known_nonzero_2<C> ();
 }
 
 /* Test things that work for poly_int-based types T, given that the
@@ -4829,8 +4641,6 @@ test_signed_2 ()
   test_ordered_2<C> ();
   test_signed_may_eq_2<C> ();
   test_signed_must_ne_2<C> ();
-  test_signed_maybe_zero_2<C> ();
-  test_signed_known_nonzero_2<C> ();
 }
 
 /* Test things that work for poly_int-based types T, given that the
diff --git a/gcc/testsuite/gcc.dg/pr7356-2.c b/gcc/testsuite/gcc.dg/pr7356-2.c
new file mode 100644
index 00000000000..ad679756978
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr7356-2.c
@@ -0,0 +1,33 @@
+/* { dg-options "-fdiagnostics-show-caret" } */
+
+int i /* { dg-error "6: expected ';' before 'int'" } */
+int j;
+/* { dg-begin-multiline-output "" }
+ int i 
+      ^
+      ;
+ int j;
+ ~~~   
+   { dg-end-multiline-output "" } */
+
+
+void test (void)
+{
+  int i /* { dg-error "8: expected ';' before 'int'" } */
+  int j;
+
+  /* { dg-begin-multiline-output "" }
+   int i 
+        ^
+        ;
+   int j;
+   ~~~   
+     { dg-end-multiline-output "" } */
+}
+
+int old_style_params (first, second)
+     int first;
+     int second;
+{
+  return first + second;
+}
diff --git a/gcc/testsuite/gcc.dg/pr7356.c b/gcc/testsuite/gcc.dg/pr7356.c
new file mode 100644
index 00000000000..84baf078b96
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr7356.c
@@ -0,0 +1,17 @@
+/* { dg-options "-fdiagnostics-show-caret" } */
+
+a /* { dg-line stray_token } */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+int main(int argc, char** argv)
+{
+  return 0;
+}
+
+/* { dg-error "expected ';' before '.*'" "" { target *-*-* } stray_token } */
+/* { dg-begin-multiline-output "" }
+ a
+  ^
+  ;
+   { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/gcc.dg/pr82274-1.c b/gcc/testsuite/gcc.dg/pr82274-1.c
new file mode 100644
index 00000000000..f96b7338fc4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr82274-1.c
@@ -0,0 +1,16 @@
+/* PR target/82274 */
+/* { dg-do run } */
+/* { dg-shouldfail "trapv" } */
+/* { dg-options "-ftrapv" } */
+
+int
+main ()
+{
+#ifdef __SIZEOF_INT128__
+  volatile __int128 m = -(((__int128) 1) << (__CHAR_BIT__ * __SIZEOF_INT128__ / 2));
+#else
+  volatile long long m = -(1LL << (__CHAR_BIT__ * __SIZEOF_LONG_LONG__ / 2));
+#endif
+  m = m * m;
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr82274-2.c b/gcc/testsuite/gcc.dg/pr82274-2.c
new file mode 100644
index 00000000000..a9643b5a923
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr82274-2.c
@@ -0,0 +1,26 @@
+/* PR target/82274 */
+/* { dg-do run } */
+/* { dg-options "-O2" } */
+
+int
+main ()
+{
+#ifdef __SIZEOF_INT128__
+  __int128 m = -(((__int128) 1) << (__CHAR_BIT__ * __SIZEOF_INT128__ / 2));
+  volatile __int128 mv = m;
+  __int128 r;
+#else
+  long long m = -(1LL << (__CHAR_BIT__ * __SIZEOF_LONG_LONG__ / 2));
+  volatile long long mv = m;
+  long long r;
+#endif
+  if (!__builtin_mul_overflow (mv, mv, &r))
+    __builtin_abort ();
+  if (!__builtin_mul_overflow_p (mv, mv, r))
+    __builtin_abort ();
+  if (!__builtin_mul_overflow (m, m, &r))
+    __builtin_abort ();
+  if (!__builtin_mul_overflow_p (m, m, r))
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr82596.c b/gcc/testsuite/gcc.dg/pr82596.c
new file mode 100644
index 00000000000..5dc67c28e8c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr82596.c
@@ -0,0 +1,27 @@
+/* PR tree-optimization/82596 - missing -Warray-bounds on an out-of-bounds
+   index into string literal
+   { dg-do compile }
+   { dg-options "-O2 -Warray-bounds" } */
+
+#define SIZE_MAX  __SIZE_MAX__
+#define SSIZE_MAX __PTRDIFF_MAX__
+#define SSIZE_MIN (-SSIZE_MAX - 1)
+
+void sink (int, ...);
+
+#define T(arg) sink (arg)
+
+void test_cststring (int i)
+{
+  T (""[SSIZE_MIN]);                      /* { dg-warning "below array bounds" "string" { xfail lp64 } } */
+  T (""[SSIZE_MIN + 1]);                  /* { dg-warning "below array bounds" "string" } */
+  T (""[-1]);                             /* { dg-warning "below array bounds" "string" } */
+  T (""[0]);
+  T (""[1]);                              /* { dg-warning "above array bounds" "string" } */
+  T ("0"[2]);                             /* { dg-warning "above array bounds" "string" } */
+  T ("012"[2]);
+  T ("012"[3]);
+  T ("012"[4]);                           /* { dg-warning "above array bounds" "string" } */
+  T ("0123"[SSIZE_MAX]);                  /* { dg-warning "above array bounds" "string" } */
+  T ("0123"[SIZE_MAX]);                   /* { dg-warning "above array bounds" "string" } */
+}
diff --git a/gcc/testsuite/gcc.dg/pr82597.c b/gcc/testsuite/gcc.dg/pr82597.c
new file mode 100644
index 00000000000..98ae264d1c9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr82597.c
@@ -0,0 +1,40 @@
+/* PR rtl-optimization/82597 */
+/* { dg-do compile  }*/
+/* { dg-options "-O2 -funroll-loops" } */
+
+int pb;
+
+void
+ch (unsigned char np, char fc)
+{
+  unsigned char *y6 = &np;
+
+  if (fc != 0)
+    {
+      unsigned char *z1 = &np;
+
+      for (;;)
+        if (*y6 != 0)
+          for (fc = 0; fc < 12; ++fc)
+            {
+              int hh;
+              int tp;
+
+              if (fc != 0)
+                hh = (*z1 != 0) ? fc : 0;
+              else
+                hh = pb;
+
+              tp = fc > 0;
+              if (hh == tp)
+                *y6 = 1;
+            }
+    }
+
+  if (np != 0)
+    y6 = (unsigned char *)&fc;
+  if (pb != 0 && *y6 != 0)
+    for (;;)
+      {
+      }
+}
diff --git a/gcc/testsuite/gcc.dg/pr82703.c b/gcc/testsuite/gcc.dg/pr82703.c
new file mode 100644
index 00000000000..0bd2f91eea4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr82703.c
@@ -0,0 +1,28 @@
+/* PR target/82703 */
+/* { dg-do run } */
+/* { dg-options "-O2 -fno-tree-sra -ftree-vectorize" } */
+
+__attribute__((noinline, noclone)) void
+compare (const double *p, const double *q)
+{
+  for (int i = 0; i < 3; ++i)
+    if (p[i] != q[i])
+      __builtin_abort ();
+}
+
+double vr[3] = { 4, 4, 4 };
+
+int
+main ()
+{
+  double v1[3] = { 1, 2, 3 };
+  double v2[3] = { 3, 2, 1 };
+  double v3[3];
+  __builtin_memcpy (v3, v1, sizeof (v1));
+  for (int i = 0; i < 3; ++i)
+    v3[i] += v2[i];
+  for (int i = 0; i < 3; ++i)
+    v1[i] += v2[i];
+  compare (v3, vr);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr82765.c b/gcc/testsuite/gcc.dg/pr82765.c
new file mode 100644
index 00000000000..dde0aeba7ef
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr82765.c
@@ -0,0 +1,5 @@
+/* { dg-do compile } */
+/* { dg-options "-Os -w" } */
+
+int a[1][1];
+int main() { int *b[] = {a, a[1820408606019012862278468], a, a, a}; }
diff --git a/gcc/testsuite/gcc.dg/pr82809.c b/gcc/testsuite/gcc.dg/pr82809.c
new file mode 100644
index 00000000000..9f74ee86534
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr82809.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -fno-tree-dominator-opts" } */
+
+struct locale_time_t
+{
+  const char *abday[7];
+  const unsigned int *wabday[7];
+};
+
+static const unsigned int empty_wstr[1] = { 0 };
+
+void
+time_read (struct locale_time_t *time)
+{
+  int cnt;
+
+  for (cnt=0; cnt < 7; cnt++)
+    {
+      time->abday[cnt] = "";
+      time->wabday[cnt] = empty_wstr;
+    }
+}
diff --git a/gcc/testsuite/gcc.dg/spellcheck-typenames.c b/gcc/testsuite/gcc.dg/spellcheck-typenames.c
index f3b8102d5a4..3717ad89f1b 100644
--- a/gcc/testsuite/gcc.dg/spellcheck-typenames.c
+++ b/gcc/testsuite/gcc.dg/spellcheck-typenames.c
@@ -100,8 +100,9 @@ baz value; /* { dg-error "1: unknown type name .baz.; use .enum. keyword to refe
    { dg-end-multiline-output "" } */
 
 /* TODO: it would be better to detect the "singed" vs "signed" typo here.  */
-singed char ch; /* { dg-error "8: before .char." } */
+singed char ch; /* { dg-error "7: before .char." } */
 /* { dg-begin-multiline-output "" }
  singed char ch;
-        ^~~~
+       ^~~~~
+       ;
    { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/gcc.dg/store_merging_10.c b/gcc/testsuite/gcc.dg/store_merging_10.c
new file mode 100644
index 00000000000..440f6e1f6c3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/store_merging_10.c
@@ -0,0 +1,56 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target store_merge } */
+/* { dg-options "-O2 -fdump-tree-store-merging" } */
+
+struct S {
+  unsigned int b1:1;
+  unsigned int b2:1;
+  unsigned int b3:1;
+  unsigned int b4:1;
+  unsigned int b5:1;
+  unsigned int b6:27;
+};
+
+struct T {
+  unsigned int b1:1;
+  unsigned int b2:16;
+  unsigned int b3:14;
+  unsigned int b4:1;
+};
+
+__attribute__((noipa)) void
+foo (struct S *x)
+{
+  x->b1 = 1;
+  x->b2 = 0;
+  x->b3 = 1;
+  x->b4 = 1;
+  x->b5 = 0;
+}
+
+__attribute__((noipa)) void
+bar (struct T *x)
+{
+  x->b1 = 1;
+  x->b2 = 0;
+  x->b4 = 0;
+}
+
+struct S s = { 0, 1, 0, 0, 1, 0x3a5f05a };
+struct T t = { 0, 0xf5af, 0x3a5a, 1 };
+
+int
+main ()
+{
+  asm volatile ("" : : : "memory");
+  foo (&s);
+  bar (&t);
+  asm volatile ("" : : : "memory");
+  if (s.b1 != 1 || s.b2 != 0 || s.b3 != 1 || s.b4 != 1 || s.b5 != 0 || s.b6 != 0x3a5f05a)
+    __builtin_abort ();
+  if (t.b1 != 1 || t.b2 != 0 || t.b3 != 0x3a5a || t.b4 != 0)
+    __builtin_abort ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "Merging successful" 2 "store-merging" } } */
diff --git a/gcc/testsuite/gcc.dg/store_merging_11.c b/gcc/testsuite/gcc.dg/store_merging_11.c
new file mode 100644
index 00000000000..399538e522e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/store_merging_11.c
@@ -0,0 +1,47 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target store_merge } */
+/* { dg-options "-O2 -fdump-tree-store-merging" } */
+
+struct S { unsigned char b[2]; unsigned short c; unsigned char d[4]; unsigned long e; };
+
+__attribute__((noipa)) void
+foo (struct S *p)
+{
+  p->b[1] = 1;
+  p->c = 23;
+  p->d[0] = 4;
+  p->d[1] = 5;
+  p->d[2] = 6;
+  p->d[3] = 7;
+  p->e = 8;
+}
+
+__attribute__((noipa)) void
+bar (struct S *p)
+{
+  p->b[1] = 9;
+  p->c = 112;
+  p->d[0] = 10;
+  p->d[1] = 11;
+}
+
+struct S s = { { 30, 31 }, 32, { 33, 34, 35, 36 }, 37 };
+
+int
+main ()
+{
+  asm volatile ("" : : : "memory");
+  foo (&s);
+  asm volatile ("" : : : "memory");
+  if (s.b[0] != 30 || s.b[1] != 1 || s.c != 23 || s.d[0] != 4 || s.d[1] != 5
+      || s.d[2] != 6 || s.d[3] != 7 || s.e != 8)
+    __builtin_abort ();
+  bar (&s);
+  asm volatile ("" : : : "memory");
+  if (s.b[0] != 30 || s.b[1] != 9 || s.c != 112 || s.d[0] != 10 || s.d[1] != 11
+      || s.d[2] != 6 || s.d[3] != 7 || s.e != 8)
+    __builtin_abort ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "Merging successful" 2 "store-merging" } } */
diff --git a/gcc/testsuite/gcc.dg/store_merging_12.c b/gcc/testsuite/gcc.dg/store_merging_12.c
new file mode 100644
index 00000000000..67f23449e93
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/store_merging_12.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wall" } */
+
+struct S { unsigned int b1:1, b2:1, b3:1, b4:1, b5:1, b6:27; };
+void bar (struct S *);
+void foo (int x)
+{
+  struct S s;
+  s.b2 = 1; s.b3 = 0; s.b4 = 1; s.b5 = 0; s.b1 = x; s.b6 = x;	/* { dg-bogus "is used uninitialized in this function" } */
+  bar (&s);
+}
diff --git a/gcc/testsuite/gcc.dg/store_merging_13.c b/gcc/testsuite/gcc.dg/store_merging_13.c
new file mode 100644
index 00000000000..d4e9ad2d260
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/store_merging_13.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target store_merge } */
+/* { dg-options "-O2 -fdump-tree-store-merging" } */
+
+struct S { unsigned char a, b; unsigned short c; unsigned char d, e, f, g; unsigned long long h; };
+
+__attribute__((noipa)) void
+f1 (struct S *p)
+{
+  p->a = 1;
+  p->b = 2;
+  p->c = 3;
+  p->d = 4;
+  p->e = 5;
+  p->f = 6;
+  p->g = 7;
+}
+
+__attribute__((noipa)) void
+f2 (struct S *__restrict p, struct S *__restrict q)
+{
+  p->a = q->a;
+  p->b = q->b;
+  p->c = q->c;
+  p->d = q->d;
+  p->e = q->e;
+  p->f = q->f;
+  p->g = q->g;
+}
+
+__attribute__((noipa)) void
+f3 (struct S *p, struct S *q)
+{
+  unsigned char pa = q->a;
+  unsigned char pb = q->b;
+  unsigned short pc = q->c;
+  unsigned char pd = q->d;
+  unsigned char pe = q->e;
+  unsigned char pf = q->f;
+  unsigned char pg = q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+__attribute__((noipa)) void
+f4 (struct S *p, struct S *q)
+{
+  unsigned char pa = p->a | q->a;
+  unsigned char pb = p->b | q->b;
+  unsigned short pc = p->c | q->c;
+  unsigned char pd = p->d | q->d;
+  unsigned char pe = p->e | q->e;
+  unsigned char pf = p->f | q->f;
+  unsigned char pg = p->g | q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+__attribute__((noipa)) void
+f5 (struct S *p, struct S *q)
+{
+  unsigned char pa = p->a & q->a;
+  unsigned char pb = p->b & q->b;
+  unsigned short pc = p->c & q->c;
+  unsigned char pd = p->d & q->d;
+  unsigned char pe = p->e & q->e;
+  unsigned char pf = p->f & q->f;
+  unsigned char pg = p->g & q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+__attribute__((noipa)) void
+f6 (struct S *p, struct S *q)
+{
+  unsigned char pa = p->a ^ q->a;
+  unsigned char pb = p->b ^ q->b;
+  unsigned short pc = p->c ^ q->c;
+  unsigned char pd = p->d ^ q->d;
+  unsigned char pe = p->e ^ q->e;
+  unsigned char pf = p->f ^ q->f;
+  unsigned char pg = p->g ^ q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+struct S s = { 20, 21, 22, 23, 24, 25, 26, 27 };
+struct S t = { 0x71, 0x72, 0x7f04, 0x78, 0x31, 0x32, 0x34, 0xf1f2f3f4f5f6f7f8ULL };
+struct S u = { 28, 29, 30, 31, 32, 33, 34, 35 };
+struct S v = { 36, 37, 38, 39, 40, 41, 42, 43 };
+
+int
+main ()
+{
+  asm volatile ("" : : : "memory");
+  f1 (&s);
+  asm volatile ("" : : : "memory");
+  if (s.a != 1 || s.b != 2 || s.c != 3 || s.d != 4
+      || s.e != 5 || s.f != 6 || s.g != 7 || s.h != 27)
+    __builtin_abort ();
+  f2 (&s, &u);
+  asm volatile ("" : : : "memory");
+  if (s.a != 28 || s.b != 29 || s.c != 30 || s.d != 31
+      || s.e != 32 || s.f != 33 || s.g != 34 || s.h != 27)
+    __builtin_abort ();
+  f3 (&s, &v);
+  asm volatile ("" : : : "memory");
+  if (s.a != 36 || s.b != 37 || s.c != 38 || s.d != 39
+      || s.e != 40 || s.f != 41 || s.g != 42 || s.h != 27)
+    __builtin_abort ();
+  f4 (&s, &t);
+  asm volatile ("" : : : "memory");
+  if (s.a != (36 | 0x71) || s.b != (37 | 0x72)
+      || s.c != (38 | 0x7f04) || s.d != (39 | 0x78)
+      || s.e != (40 | 0x31) || s.f != (41 | 0x32)
+      || s.g != (42 | 0x34) || s.h != 27)
+    __builtin_abort ();
+  f3 (&s, &u);
+  f5 (&s, &t);
+  asm volatile ("" : : : "memory");
+  if (s.a != (28 & 0x71) || s.b != (29 & 0x72)
+      || s.c != (30 & 0x7f04) || s.d != (31 & 0x78)
+      || s.e != (32 & 0x31) || s.f != (33 & 0x32)
+      || s.g != (34 & 0x34) || s.h != 27)
+    __builtin_abort ();
+  f2 (&s, &v);
+  f6 (&s, &t);
+  asm volatile ("" : : : "memory");
+  if (s.a != (36 ^ 0x71) || s.b != (37 ^ 0x72)
+      || s.c != (38 ^ 0x7f04) || s.d != (39 ^ 0x78)
+      || s.e != (40 ^ 0x31) || s.f != (41 ^ 0x32)
+      || s.g != (42 ^ 0x34) || s.h != 27)
+    __builtin_abort ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "Merging successful" 6 "store-merging" } } */
diff --git a/gcc/testsuite/gcc.dg/store_merging_14.c b/gcc/testsuite/gcc.dg/store_merging_14.c
new file mode 100644
index 00000000000..49af24951cb
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/store_merging_14.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target store_merge } */
+/* { dg-options "-O2 -fdump-tree-store-merging" } */
+
+struct S { unsigned int i : 8, a : 7, b : 7, j : 10, c : 15, d : 7, e : 10, f : 7, g : 9, k : 16; unsigned long long h; };
+
+__attribute__((noipa)) void
+f1 (struct S *p)
+{
+  p->a = 1;
+  p->b = 2;
+  p->c = 3;
+  p->d = 4;
+  p->e = 5;
+  p->f = 6;
+  p->g = 7;
+}
+
+__attribute__((noipa)) void
+f2 (struct S *__restrict p, struct S *__restrict q)
+{
+  p->a = q->a;
+  p->b = q->b;
+  p->c = q->c;
+  p->d = q->d;
+  p->e = q->e;
+  p->f = q->f;
+  p->g = q->g;
+}
+
+__attribute__((noipa)) void
+f3 (struct S *p, struct S *q)
+{
+  unsigned char pa = q->a;
+  unsigned char pb = q->b;
+  unsigned short pc = q->c;
+  unsigned char pd = q->d;
+  unsigned short pe = q->e;
+  unsigned char pf = q->f;
+  unsigned short pg = q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+__attribute__((noipa)) void
+f4 (struct S *p, struct S *q)
+{
+  unsigned char pa = p->a | q->a;
+  unsigned char pb = p->b | q->b;
+  unsigned short pc = p->c | q->c;
+  unsigned char pd = p->d | q->d;
+  unsigned short pe = p->e | q->e;
+  unsigned char pf = p->f | q->f;
+  unsigned short pg = p->g | q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+__attribute__((noipa)) void
+f5 (struct S *p, struct S *q)
+{
+  unsigned char pa = p->a & q->a;
+  unsigned char pb = p->b & q->b;
+  unsigned short pc = p->c & q->c;
+  unsigned char pd = p->d & q->d;
+  unsigned short pe = p->e & q->e;
+  unsigned char pf = p->f & q->f;
+  unsigned short pg = p->g & q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+__attribute__((noipa)) void
+f6 (struct S *p, struct S *q)
+{
+  unsigned char pa = p->a ^ q->a;
+  unsigned char pb = p->b ^ q->b;
+  unsigned short pc = p->c ^ q->c;
+  unsigned char pd = p->d ^ q->d;
+  unsigned short pe = p->e ^ q->e;
+  unsigned char pf = p->f ^ q->f;
+  unsigned short pg = p->g ^ q->g;
+  p->a = pa;
+  p->b = pb;
+  p->c = pc;
+  p->d = pd;
+  p->e = pe;
+  p->f = pf;
+  p->g = pg;
+}
+
+struct S s = { 72, 20, 21, 73, 22, 23, 24, 25, 26, 74, 27 };
+struct S t = { 75, 0x71, 0x72, 76, 0x7f04, 0x78, 0x31, 0x32, 0x34, 77, 0xf1f2f3f4f5f6f7f8ULL };
+struct S u = { 78, 28, 29, 79, 30, 31, 32, 33, 34, 80, 35 };
+struct S v = { 81, 36, 37, 82, 38, 39, 40, 41, 42, 83, 43 };
+
+int
+main ()
+{
+  asm volatile ("" : : : "memory");
+  f1 (&s);
+  asm volatile ("" : : : "memory");
+  if (s.i != 72 || s.a != 1 || s.b != 2 || s.j != 73 || s.c != 3 || s.d != 4
+      || s.e != 5 || s.f != 6 || s.g != 7 || s.k != 74 || s.h != 27)
+    __builtin_abort ();
+  f2 (&s, &u);
+  asm volatile ("" : : : "memory");
+  if (s.i != 72 || s.a != 28 || s.b != 29 || s.j != 73 || s.c != 30 || s.d != 31
+      || s.e != 32 || s.f != 33 || s.g != 34 || s.k != 74 || s.h != 27)
+    __builtin_abort ();
+  f3 (&s, &v);
+  asm volatile ("" : : : "memory");
+  if (s.i != 72 || s.a != 36 || s.b != 37 || s.j != 73 || s.c != 38 || s.d != 39
+      || s.e != 40 || s.f != 41 || s.g != 42 || s.k != 74 || s.h != 27)
+    __builtin_abort ();
+  f4 (&s, &t);
+  asm volatile ("" : : : "memory");
+  if (s.i != 72 || s.a != (36 | 0x71) || s.b != (37 | 0x72) || s.j != 73
+      || s.c != (38 | 0x7f04) || s.d != (39 | 0x78)
+      || s.e != (40 | 0x31) || s.f != (41 | 0x32)
+      || s.g != (42 | 0x34) || s.k != 74 || s.h != 27)
+    __builtin_abort ();
+  f3 (&s, &u);
+  f5 (&s, &t);
+  asm volatile ("" : : : "memory");
+  if (s.i != 72 || s.a != (28 & 0x71) || s.b != (29 & 0x72) || s.j != 73
+      || s.c != (30 & 0x7f04) || s.d != (31 & 0x78)
+      || s.e != (32 & 0x31) || s.f != (33 & 0x32)
+      || s.g != (34 & 0x34) || s.k != 74 || s.h != 27)
+    __builtin_abort ();
+  f2 (&s, &v);
+  f6 (&s, &t);
+  asm volatile ("" : : : "memory");
+  if (s.i != 72 || s.a != (36 ^ 0x71) || s.b != (37 ^ 0x72) || s.j != 73
+      || s.c != (38 ^ 0x7f04) || s.d != (39 ^ 0x78)
+      || s.e != (40 ^ 0x31) || s.f != (41 ^ 0x32)
+      || s.g != (42 ^ 0x34) || s.k != 74 || s.h != 27)
+    __builtin_abort ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "Merging successful" 6 "store-merging" } } */
diff --git a/gcc/testsuite/gcc.dg/strict-overflow-3.c b/gcc/testsuite/gcc.dg/strict-overflow-3.c
index 6215a501a72..8ef91476200 100644
--- a/gcc/testsuite/gcc.dg/strict-overflow-3.c
+++ b/gcc/testsuite/gcc.dg/strict-overflow-3.c
@@ -9,7 +9,7 @@
 int
 foo (int i, int j)
 {
-  return i + 100 < j + 1000;
+  return i + 100 < j + 1234;
 }
 
-/* { dg-final { scan-tree-dump-not "1000" "optimized" } } */
+/* { dg-final { scan-tree-dump-not "1234" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/torture/pr52451.c b/gcc/testsuite/gcc.dg/torture/pr52451.c
new file mode 100644
index 00000000000..81a3d4d158d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr52451.c
@@ -0,0 +1,55 @@
+/* { dg-do run } */
+/* { dg-add-options ieee } */
+/* { dg-require-effective-target fenv_exceptions } */
+
+#include <fenv.h>
+
+#define TEST_C_NOEX(CMP, S)			\
+  r = nan##S CMP arg##S;			\
+  if (fetestexcept (FE_INVALID))		\
+    __builtin_abort ()
+
+#define TEST_B_NOEX(FN, S)			\
+  r = __builtin_##FN (nan##S, arg##S);		\
+  if (fetestexcept (FE_INVALID))		\
+    __builtin_abort ()
+
+#define TEST_C_EX(CMP, S)			\
+  r = nan##S CMP arg##S;			\
+  if (!fetestexcept (FE_INVALID))		\
+    __builtin_abort ();				\
+  feclearexcept (FE_INVALID)
+
+#define TEST(TYPE, S)				\
+  volatile TYPE nan##S = __builtin_nan##S ("");	\
+  volatile TYPE arg##S = 1.0##S;		\
+						\
+  TEST_C_NOEX (==, S);				\
+  TEST_C_NOEX (!=, S);				\
+						\
+  TEST_B_NOEX (isgreater, S);			\
+  TEST_B_NOEX (isless, S);			\
+  TEST_B_NOEX (isgreaterequal, S);		\
+  TEST_B_NOEX (islessequal, S);			\
+						\
+  TEST_B_NOEX (islessgreater, S);		\
+  TEST_B_NOEX (isunordered, S);			\
+						\
+  TEST_C_EX (>, S);				\
+  TEST_C_EX (<, S);				\
+  TEST_C_EX (>=, S);				\
+  TEST_C_EX (<=, S)
+
+int
+main (void)
+{
+  volatile int r;
+
+  feclearexcept (FE_INVALID);
+
+  TEST (float, f);
+  TEST (double, );
+  TEST (long double, l);
+  
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr82129.c b/gcc/testsuite/gcc.dg/torture/pr82129.c
new file mode 100644
index 00000000000..b1161491fe6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr82129.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-ftree-pre" } */
+
+int pj;
+
+void
+g4 (unsigned long int *bc, unsigned long int *h5)
+{
+  if (pj != 0)
+    {
+      int ib = 0;
+
+      while (bc != 0)
+	{
+m6:
+	  for (pj = 0; pj < 2; ++pj)
+	    pj = 0;
+
+	  while (pj != 0)
+	    {
+	      for (;;)
+		{
+		}
+
+	      while (ib != 0)
+		{
+		  unsigned long int tv = *bc;
+		  unsigned long int n7;
+
+		  *bc = 1;
+		  while (*bc != 0)
+		    {
+		    }
+
+ut:
+		  if (pj == 0)
+		    n7 = *h5 > 0;
+		  else
+		    {
+		      *h5 = tv;
+		      n7 = *h5;
+		    }
+		  ib += n7;
+		}
+	    }
+	}
+
+      goto ut;
+    }
+
+  goto m6;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr82436-2.c b/gcc/testsuite/gcc.dg/torture/pr82436-2.c
new file mode 100644
index 00000000000..32eda186ff0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr82436-2.c
@@ -0,0 +1,45 @@
+/* { dg-do compile } */
+
+enum
+{
+  a, b, c, d,  e,  f,  g,  h,  j,  k
+};
+
+int l;
+void m (short *s)
+{
+  short n, o, p;
+  float(*q)[k];
+  int r, i;
+  while (l > 0)
+    r = l;
+  for (;;)
+    {
+      i = 0;
+      for (; i < r; i++)
+	{
+	    {
+	      float ab = q[i][a];
+	      int i = ab;
+	      p = i;
+	    }
+	  ((short *) s)[0] = p;
+	    {
+	      float ab = q[i][b];
+	      int i = ab;
+	      o = i;
+	    }
+	  ((short *) s)[1] = o;
+	    {
+	      float ab = q[i][f];
+	      int i = ab;
+	      n = i;
+	    }
+	  ((short *) s)[2] = n;
+	  float ab = q[i][g];
+	  int i = ab;
+	  ((short *) s)[3] = i;
+	  s = (short *) s + 4;
+	}
+    }
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr82473.c b/gcc/testsuite/gcc.dg/torture/pr82473.c
new file mode 100644
index 00000000000..b12de21d7db
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr82473.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-ftree-vectorize" } */
+
+void
+zz (int x9, short int gt)
+{
+  if (0)
+    {
+      while (gt < 1)
+	{
+	  int pz;
+
+k6:
+	  for (pz = 0; pz < 3; ++pz)
+	    x9 += gt;
+	  ++gt;
+	}
+    }
+
+  if (x9 != 0)
+    goto k6;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr82603.c b/gcc/testsuite/gcc.dg/torture/pr82603.c
new file mode 100644
index 00000000000..960a48bbd3a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr82603.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-ftree-loop-vectorize" } */
+
+int
+mr (unsigned int lf, int ms)
+{
+  unsigned int sw = 0;
+  char *cu = (char *)&ms;
+
+  while (ms < 1)
+    {
+      if (lf == 0)
+	ms = 0;
+      else
+	ms = 0;
+      ms += ((lf > 0) && ((lf > sw) ? 1 : ++*cu));
+    }
+
+  if (lf != 0)
+    cu = (char *)&sw;
+  *cu = lf;
+
+  return ms;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr82692.c b/gcc/testsuite/gcc.dg/torture/pr82692.c
new file mode 100644
index 00000000000..254ace15ada
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr82692.c
@@ -0,0 +1,25 @@
+/* { dg-do run } */
+/* { dg-add-options ieee } */
+/* { dg-require-effective-target fenv_exceptions } */
+
+#include <fenv.h>
+
+extern void abort (void);
+extern void exit (int);
+
+double __attribute__ ((noinline, noclone))
+foo (double x)
+{
+  if (__builtin_islessequal (x, 0.0) || __builtin_isgreater (x, 1.0))
+    return x + x;
+  return x * x;
+}
+
+int
+main (void)
+{
+  volatile double x = foo (__builtin_nan (""));
+  if (fetestexcept (FE_INVALID))
+    abort ();
+  exit (0);
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr82697.c b/gcc/testsuite/gcc.dg/torture/pr82697.c
new file mode 100644
index 00000000000..57da8a264b9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr82697.c
@@ -0,0 +1,23 @@
+/* { dg-do run } */
+
+__attribute__((noinline,noclone))
+void test(int *pi, long *pl, int f)
+{
+  *pl = 0;
+
+  *pi = 1;
+
+  if (f)
+    *pl = 2;
+}
+
+int main()
+{
+  void *p = __builtin_malloc(sizeof (long));
+
+  test(p, p, 0);
+
+  if (*(int *)p != 1)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr82762.c b/gcc/testsuite/gcc.dg/torture/pr82762.c
new file mode 100644
index 00000000000..d4f57bc55f7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr82762.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+
+int printf (const char *, ...);
+
+int b, c, d, e, f, g, j, k;
+char h, i;
+volatile int l;
+
+int m (int n, int o)
+{ 
+  return o < 0 || o > 1 ? n : o;
+}
+
+int p (int n, unsigned o)
+{ 
+  return n - o;
+}
+
+int q ()
+{ 
+  char r;
+  int a, s, t, u, v, w;
+L:
+  if (t)
+    printf ("%d", d);
+  u = v;
+  while (j)
+    { 
+      while (e)
+	for (w = 0; w != 54; w += 6)
+	  { 
+	    l;
+	    s = p (u < 1, i || c);
+	    r = s < 0 || s > 1 ? 0 : 1 >> s;
+	    v = r;
+	    g = h;
+	  }
+      if (h)
+	return f;
+      if (u)
+	for (a = 0; a != 54; a += 6)
+	  f = m (2, -(k || b));
+    }
+  d = t;
+  goto L;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-prof/comp-goto-1.c b/gcc/testsuite/gcc.dg/tree-prof/comp-goto-1.c
index fe768f9a98d..baed1e3fa78 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/comp-goto-1.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/comp-goto-1.c
@@ -1,11 +1,11 @@
 /* { dg-require-effective-target freorder } */
 /* { dg-require-effective-target label_values } */
 /* { dg-options "-O2 -freorder-blocks-and-partition" } */
-/* { dg-add-options stack_size } */
+/* { dg-require-stack-size "4000" } */
 
 #include <stdlib.h>
 
-#if (!defined(STACK_SIZE) || STACK_SIZE >= 4000) && __INT_MAX__ >= 2147483647
+#if __INT_MAX__ >= 2147483647
 typedef unsigned int uint32;
 typedef signed int sint32;
 
diff --git a/gcc/testsuite/gcc.dg/tree-prof/switch-case-2.c b/gcc/testsuite/gcc.dg/tree-prof/switch-case-2.c
index dcd50241eb9..9b0dfc2dbb5 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/switch-case-2.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/switch-case-2.c
@@ -1,4 +1,4 @@
-/* { dg-options "-O2 -fdump-rtl-expand-all" } */
+/* { dg-options "-O2 -fdump-ipa-profile-all" } */
 int g;
 
 __attribute__((noinline)) void foo (int  n)
@@ -36,5 +36,5 @@ int main ()
  return 0;
 }
 /* autofdo cannot do that precise execution numbers: */
-/* { dg-final-use-not-autofdo { scan-rtl-dump-times ";; basic block\[^\\n\]*count 4000" 2 "expand"} } */
-/* { dg-final-use-not-autofdo { scan-rtl-dump-times ";; basic block\[^\\n\]*count 2000" 1 "expand" { xfail *-*-* } } } */
+/* { dg-final-use-not-autofdo { scan-ipa-dump-times ";;   basic block\[^\\n\]*count 4000" 2 "profile"} } */
+/* { dg-final-use-not-autofdo { scan-ipa-dump-times ";;   basic block\[^\\n\]*count 2000" 1 "profile"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-2.c b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-2.c
index 2323b7fa3e9..75d3db37ade 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-2.c
@@ -290,7 +290,7 @@ RNG (0,  6,   8, "%s%ls", "1", L"2");
 
 /*  Only conditional calls to must_not_eliminate must be made (with
     any probability):
-    { dg-final { scan-tree-dump-times "> \\\[\[0-9.\]+%\\\] \\\[count: \[0-9INV\]*\\\]:\n *must_not_eliminate" 127 "optimized" { target { ilp32 || lp64 } } } }
-    { dg-final { scan-tree-dump-times "> \\\[\[0-9.\]+%\\\] \\\[count: \[0-9INV\]*\\\]:\n *must_not_eliminate" 96 "optimized" { target { { ! ilp32 } && { ! lp64 } } } } }
+    { dg-final { scan-tree-dump-times "> \\\[local count: \[0-9INV\]*\\\]:\n *must_not_eliminate" 127 "optimized" { target { ilp32 || lp64 } } } }
+    { dg-final { scan-tree-dump-times "> \\\[local count: \[0-9INV\]*\\\]:\n *must_not_eliminate" 96 "optimized" { target { { ! ilp32 } && { ! lp64 } } } } }
     No unconditional calls to abort should be made:
     { dg-final { scan-tree-dump-not ";\n *must_not_eliminate" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/dump-2.c b/gcc/testsuite/gcc.dg/tree-ssa/dump-2.c
index 6ae2ef5bf39..20f99c2df12 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/dump-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/dump-2.c
@@ -6,4 +6,4 @@ int f(void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump "<bb \[0-9\]> \\\[100\\\.00%\\\] \\\[count: INV\\\]:" "optimized" } } */
+/* { dg-final { scan-tree-dump "<bb \[0-9\]> \\\[local count: 10000\\\]:" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-10.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-10.c
index 4097145eba6..75a8ab9b1d5 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-10.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-10.c
@@ -26,5 +26,5 @@ int foo (int x, int n)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 1 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-11.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-11.c
index a0333fbb28c..10f3d534adc 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-11.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-11.c
@@ -24,5 +24,4 @@ int foo (float *x)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* Sum is wrong here, but not enough for error to be reported.  */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 0 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-12.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-12.c
index 535c1f0eb6c..9468c070489 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-12.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-12.c
@@ -29,6 +29,5 @@ int foo (int x)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* Sum is wrong here, but not enough for error to be reported.  */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 0 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-1.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-1.c
index 8badc762267..b55a533e374 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-1.c
@@ -39,4 +39,4 @@ int main1 ()
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 1 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-2.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-2.c
index a517f6552e6..9249f3020b7 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-20040816-2.c
@@ -43,5 +43,4 @@ void foo(const int * __restrict__ zr_in,
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* Sum is wrong here, but not enough for error to be reported.  */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 0 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-5.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-5.c
index 58260dd878b..35595aa98e3 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-5.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-5.c
@@ -27,4 +27,4 @@ dct_unquantize_h263_inter_c (short *block, int n, int qscale, int nCoeffs)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 1 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-8.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-8.c
index 6c26c209212..c2007486500 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-8.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-8.c
@@ -22,5 +22,4 @@ void test ()
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* Sum is wrong here, but not enough for error to be reported.  */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 0 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-9.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-9.c
index 789cb6ae23a..fce181e2fee 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-9.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-9.c
@@ -26,4 +26,4 @@ int foo (int x, int n)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 1 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-cd.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-cd.c
index 11e142af321..4932cd75a13 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-cd.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-cd.c
@@ -32,5 +32,4 @@ void foo (int *x1, int *x2, int *x3, int *x4, int *y)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* Sum is wrong here, but not enough for error to be reported.  */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 0 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr56541.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr56541.c
index 9682fbc15df..71d6398897a 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr56541.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr56541.c
@@ -29,5 +29,4 @@ void foo()
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* Sum is wrong here, but not enough for error to be reported.  */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 0 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr68583.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr68583.c
index b128deb4a21..6739fad9f6c 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr68583.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr68583.c
@@ -26,5 +26,5 @@ void foo (long *a)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 1 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-1.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-1.c
index 3ba7de5e6a5..a9f4ff669f8 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-1.c
@@ -20,5 +20,4 @@ void foo (int a[], int b[])
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* Sum is wrong here, but not enough for error to be reported.  */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 0 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-2.c b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-2.c
index 07589fd7928..c9e7c1b96ea 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ifc-pr69489-2.c
@@ -21,4 +21,4 @@ foo (const char *u, const char *v, long n)
    which is folded by vectorizer.  Both outgoing edges must have probability
    100% so the resulting profile match after folding.  */
 /* { dg-final { scan-tree-dump-times "Invalid sum of outgoing probabilities 200.0" 1 "ifcvt" } } */
-/* { dg-final { scan-tree-dump-times "Invalid sum of incoming frequencies" 1 "ifcvt" } } */
+/* { dg-final { scan-tree-dump-times "Invalid sum of incoming counts" 1 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-17.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-17.c
index 4efc0a4a696..b3617f685a1 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ldist-17.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-17.c
@@ -45,5 +45,5 @@ mad_synth_mute (struct mad_synth *synth)
   return;
 }
 
-/* { dg-final { scan-tree-dump "distributed: split to 0 loops and 4 library calls" "ldist" } } */
-/* { dg-final { scan-tree-dump-times "generated memset zero" 4 "ldist" } } */
+/* { dg-final { scan-tree-dump "Loop nest . distributed: split to 0 loops and 1 library calls" "ldist" } } */
+/* { dg-final { scan-tree-dump-times "generated memset zero" 1 "ldist" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-27.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-27.c
index 3580c65f09b..b1fd024a942 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ldist-27.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-27.c
@@ -11,7 +11,8 @@ struct st
   double c[M][N];
 };
 
-int __attribute__ ((noinline)) foo (struct st *s)
+int __attribute__ ((noinline))
+foo (struct st *s)
 {
   int i, j;
   for (i = 0; i != M;)
@@ -29,9 +30,11 @@ L2:
   return 0;
 }
 
-int main (void)
+struct st s;
+
+int
+main (void)
 {
-  struct st s;
   return foo (&s);
 }
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-32.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-32.c
new file mode 100644
index 00000000000..477d222fb3b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-32.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-loop-distribution -ftree-loop-distribute-patterns -fdump-tree-ldist-details" } */
+
+#define M (256)
+#define N (512)
+
+struct st
+{
+  int a[M][N];
+  int c[M];
+  int b[M][N];
+};
+
+void
+foo (struct st *p)
+{
+  for (unsigned i = 0; i < M; ++i)
+    {
+      p->c[i] = 0;
+      for (unsigned j = N; j > 0; --j)
+	{
+	  p->a[i][j - 1] = 0;
+	  p->b[i][j - 1] = 0;
+	}
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "Loop nest . distributed: split to 0 loops and 1 library" 1 "ldist" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_memset \\(.*, 0, 1049600\\);" 1 "ldist" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-35.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-35.c
new file mode 100644
index 00000000000..445d23d114b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-35.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-loop-distribution -ftree-loop-distribute-patterns -fdump-tree-ldist-details" } */
+
+#define M (256)
+#define N (512)
+
+struct st
+{
+  int a[M][N];
+  int c[M];
+  int b[M][N];
+};
+
+void
+foo (struct st * restrict p, struct st * restrict q)
+{
+  for (unsigned i = 0; i < M; ++i)
+    {
+      p->c[i] = 0;
+      for (unsigned j = N; j > 0; --j)
+	{
+	  p->a[i][j - 1] = q->a[i][j - 1];
+	  p->b[i][j - 1] = 0;
+	}
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "Loop nest . distributed: split to 0 loops and 1 library" 1 "ldist" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-36.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-36.c
new file mode 100644
index 00000000000..0e843f4dd55
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-36.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-loop-distribution -ftree-loop-distribute-patterns -fdump-tree-ldist-details" } */
+
+#define M (256)
+#define N (512)
+
+struct st
+{
+  int a[M][N];
+  int c[M];
+  int b[M][N];
+};
+
+void
+foo (struct st * restrict p)
+{
+  for (unsigned i = 0; i < M; ++i)
+    {
+      p->c[i] = 0;
+      for (unsigned j = N; j > 0; --j)
+	{
+	  p->b[i][j - 1] = p->a[i][j - 1];
+	  p->a[i][j - 1] = 0;
+	}
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "Loop nest . distributed: split to 0 loops and 3 library" 1 "ldist" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/loop-1.c b/gcc/testsuite/gcc.dg/tree-ssa/loop-1.c
index 0193c6e52fc..01c37a56671 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/loop-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/loop-1.c
@@ -46,7 +46,7 @@ int xxx(void)
 /* CRIS keeps the address in a register.  */
 /* m68k sometimes puts the address in a register, depending on CPU and PIC.  */
 
-/* { dg-final { scan-assembler-times "foo" 5 { xfail hppa*-*-* ia64*-*-* sh*-*-* cris-*-* crisv32-*-* fido-*-* m68k-*-* i?86-*-mingw* i?86-*-cygwin* x86_64-*-mingw* visium-*-* } } } */
+/* { dg-final { scan-assembler-times "foo" 5 { xfail hppa*-*-* ia64*-*-* sh*-*-* cris-*-* crisv32-*-* fido-*-* m68k-*-* i?86-*-mingw* i?86-*-cygwin* x86_64-*-mingw* visium-*-* nvptx*-*-* } } } */
 /* { dg-final { scan-assembler-times "foo,%r" 5 { target hppa*-*-* } } } */
 /* { dg-final { scan-assembler-times "= foo"  5 { target ia64*-*-* } } } */
 /* { dg-final { scan-assembler-times "call\[ \t\]*_foo" 5 { target i?86-*-mingw* i?86-*-cygwin* } } } */
@@ -55,3 +55,4 @@ int xxx(void)
 /* { dg-final { scan-assembler-times "Jsr \\\$r" 5 { target cris-*-* } } } */
 /* { dg-final { scan-assembler-times "\[jb\]sr" 5 { target fido-*-* m68k-*-* } } } */
 /* { dg-final { scan-assembler-times "bra *tr,r\[1-9\]*,r21" 5 { target visium-*-* } } } */
+/* { dg-final { scan-assembler-times "(?n)\[ \t\]call\[ \t\].*\[ \t\]foo," 5 { target nvptx*-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/negneg-1.c b/gcc/testsuite/gcc.dg/tree-ssa/negneg-1.c
new file mode 100644
index 00000000000..9c6c36998e5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/negneg-1.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O -frounding-math -fdump-tree-optimized-raw -Wno-psabi" } */
+
+#define DEF(num, T1, T2) T2 f##num(T1 x) { \
+    T1 y = -x; \
+    T2 z = (T2)y; \
+    return -z; \
+}
+DEF(0, int, long long)
+DEF(1, int, unsigned long long)
+DEF(2, long long, int)
+DEF(3, unsigned long long, int)
+DEF(4, long long, unsigned)
+DEF(5, unsigned long long, unsigned)
+DEF(6, float, double)
+
+typedef int vec __attribute__((vector_size(4*sizeof(int))));
+typedef unsigned uvec __attribute__((vector_size(4*sizeof(int))));
+void h(vec*p,uvec*q){
+    vec a = -*p;
+    *q = -(uvec)a;
+}
+
+/* { dg-final { scan-tree-dump-not "negate_expr" "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/negneg-2.c b/gcc/testsuite/gcc.dg/tree-ssa/negneg-2.c
new file mode 100644
index 00000000000..bd6198e633b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/negneg-2.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fno-rounding-math -fdump-tree-optimized-raw" } */
+
+#define DEF(num, T1, T2) T2 f##num(T1 x) { \
+    T1 y = -x; \
+    T2 z = (T2)y; \
+    return -z; \
+}
+DEF(0, double, float)
+
+/* { dg-final { scan-tree-dump-not "negate_expr" "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/negneg-3.c b/gcc/testsuite/gcc.dg/tree-ssa/negneg-3.c
new file mode 100644
index 00000000000..9deb9f6f320
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/negneg-3.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O -frounding-math -fdump-tree-optimized-raw" } */
+
+// This assumes that long long is strictly larger than int
+
+#define DEF(num, T1, T2) T2 f##num(T1 x) { \
+    T1 y = -x; \
+    T2 z = (T2)y; \
+    return -z; \
+}
+DEF(0, unsigned, long long)
+DEF(1, unsigned, unsigned long long)
+DEF(2, double, float)
+
+/* { dg-final { scan-tree-dump-times "negate_expr" 6 "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/negneg-4.c b/gcc/testsuite/gcc.dg/tree-ssa/negneg-4.c
new file mode 100644
index 00000000000..e1131d06f64
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/negneg-4.c
@@ -0,0 +1,18 @@
+/* { dg-do run } */
+/* { dg-options "-O -fwrapv" } */
+
+#define DEF(num, T1, T2) T2 f##num(T1 x) { \
+    T1 y = -x; \
+    T2 z = (T2)y; \
+    return -z; \
+}
+DEF(0, int, long long)
+
+int main(){
+    volatile int a = -1 - __INT_MAX__;
+    volatile long long b = f0 (a);
+    volatile long long c = a;
+    volatile long long d = -c;
+    if (b != d)
+      __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/noreturn-1.c b/gcc/testsuite/gcc.dg/tree-ssa/noreturn-1.c
new file mode 100644
index 00000000000..ae7ee42fabc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/noreturn-1.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } *
+/* { dg-options "-O2 -fdump-tree-ssa -std=gnu11" } */
+/* { dg-final { scan-tree-dump-times "__builtin_unreachable" 4 "ssa" } } */
+
+void bar1 (void);
+void bar2 (void);
+void bar3 (void);
+void bar4 (void);
+
+_Noreturn void
+foo1 (int *p, int y)
+{
+  bar1 ();
+  *p = y;
+  return;	/* { dg-warning "function declared 'noreturn' has a 'return' statement" } */
+}		/* { dg-warning "'noreturn' function does return" "" { target *-*-* } .-1 } */
+
+_Noreturn void
+foo2 (int *p, int y)
+{
+  bar2 ();
+  *p = y;
+}		/* { dg-warning "'noreturn' function does return" } */
+
+_Noreturn void
+foo3 (int *p, int y)
+{
+  if (y > 10)
+    return;	/* { dg-warning "function declared 'noreturn' has a 'return' statement" } */
+  bar3 ();
+  *p = y;
+  return;	/* { dg-warning "function declared 'noreturn' has a 'return' statement" } */
+}		/* { dg-warning "'noreturn' function does return" } */
+
+_Noreturn void
+foo4 (int *p, int y)
+{
+  if (y > 10)
+    return;	/* { dg-warning "function declared 'noreturn' has a 'return' statement" } */
+  bar4 ();
+  *p = y;
+}		/* { dg-warning "'noreturn' function does return" } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr82574.c b/gcc/testsuite/gcc.dg/tree-ssa/pr82574.c
new file mode 100644
index 00000000000..8fc459631ef
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr82574.c
@@ -0,0 +1,19 @@
+/* { dg-do run } */
+/* { dg-options "-O3" } */
+
+unsigned char a, b, c, d[200][200];
+
+void abort (void);
+
+int main ()
+{
+  for (; a < 200; a++)
+    for (b = 0; b < 200; b++)
+      if (c)
+	d[a][b] = 1;
+
+  if ((c && d[0][0] != 1) || (!c && d[0][0] != 0))
+    abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp101.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp101.c
index c9feb256857..aad41f91f47 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp101.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp101.c
@@ -10,4 +10,4 @@ int main ()
   return 0;
 }
 
-/* { dg-final { scan-tree-dump "<bb 2> \\\[\[0-9.\]+%\\\] \\\[count: \[0-9INV\]*\\\]:\[\n\r \]*return 0;" "optimized" } } */
+/* { dg-final { scan-tree-dump "<bb 2> \\\[\[0-9.\]+%\\\] \\\[count: \[0-9INV\]*\\\]:\[\n\r \]*return 0;" "optimized" { xfail aarch64*-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/float-cast-overflow-bf.c b/gcc/testsuite/gcc.dg/ubsan/float-cast-overflow-bf.c
index 16268603375..538d900b0ab 100644
--- a/gcc/testsuite/gcc.dg/ubsan/float-cast-overflow-bf.c
+++ b/gcc/testsuite/gcc.dg/ubsan/float-cast-overflow-bf.c
@@ -48,25 +48,25 @@ main (void)
   return 0;
 }
 
-/* { dg-output "value -2.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*value 2.5 is outside the range of representable values of type" } */
+/* { dg-output " -2.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1.5 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* -1 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2 is outside the range of representable values of type\[^\n\r]*(\n|\r\n|\r)" } */
+/* { dg-output "\[^\n\r]* 2.5 is outside the range of representable values of type" } */
diff --git a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-101.c b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-101.c
index cc04959d187..91eb28218bd 100644
--- a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-101.c
+++ b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-101.c
@@ -45,6 +45,6 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 0 "vect" } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 1 "vect" { target vect_1_size } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 2 "vect" { target vect_2_sizes } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 3 "vect" { target vect_3_sizes } } } */
+/* { dg-final { scan-tree-dump-times "can't determine dependence" 1 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "can't determine dependence" "vect" { target vect_multiple_sizes } } } */
+
diff --git a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102.c b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102.c
index f32561dc4fd..51f62788dbf 100644
--- a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102.c
+++ b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102.c
@@ -50,6 +50,6 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 0 "vect" } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 1 "vect" { target vect_1_size } } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 2 "vect" { target vect_2_sizes } } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 3 "vect" { target vect_3_sizes } } } */
+/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 1 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "possible dependence between data-refs" "vect" { target vect_multiple_sizes } } } */
+
diff --git a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102a.c b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102a.c
index 79a6ee45f7a..581438823fd 100644
--- a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102a.c
+++ b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-102a.c
@@ -50,6 +50,6 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 0 "vect" } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 1 "vect" { target vect_1_size } } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 2 "vect" { target vect_2_sizes } } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 3 "vect" { target vect_3_sizes } } } */
+/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 1 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "possible dependence between data-refs" "vect" { target vect_multiple_sizes } } } */
+
diff --git a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-37.c b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-37.c
index d0673e93fb2..6f4c84b4cd2 100644
--- a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-37.c
+++ b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-37.c
@@ -58,6 +58,5 @@ int main (void)
    If/when the aliasing problems are resolved, unalignment may
    prevent vectorization on some targets.  */
 /* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 2 "vect" { target vect_1_size } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 4 "vect" { target vect_2_sizes } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 6 "vect" { target vect_3_sizes } } } */
+/* { dg-final { scan-tree-dump-times "can't determine dependence" 2 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "can't determine dependence" "vect" { target vect_multiple_sizes } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-79.c b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-79.c
index 232d8e526ed..6e9ddcfa5ce 100644
--- a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-79.c
+++ b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-79.c
@@ -46,6 +46,5 @@ int main (void)
   If/when the aliasing problems are resolved, unalignment may
   prevent vectorization on some targets.  */
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect"  { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 1 "vect" { target vect_1_size } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 2 "vect" { target vect_2_sizes } } } */
-/* { dg-final { scan-tree-dump-times "can't determine dependence" 3 "vect" { target vect_3_sizes } } } */
+/* { dg-final { scan-tree-dump-times "can't determine dependence" 1 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "can't determine dependence" "vect" { target vect_multiple_sizes } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr25413a.c b/gcc/testsuite/gcc.dg/vect/pr25413a.c
index 36e786f4f7d..a80ca868112 100644
--- a/gcc/testsuite/gcc.dg/vect/pr25413a.c
+++ b/gcc/testsuite/gcc.dg/vect/pr25413a.c
@@ -124,5 +124,5 @@ int main (void)
 } 
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "vector alignment may not be reachable" 1 "vect" { target { ! vector_alignment_reachable } xfail { vect_element_align_preferred } } } }  */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 1 "vect" { target { ! vector_alignment_reachable } xfail { vect_element_align_preferred } } } } */
+/* { dg-final { scan-tree-dump-times "vector alignment may not be reachable" 1 "vect" { target { ! vector_alignment_reachable  } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 1 "vect" { target { ! vector_alignment_reachable } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr31699.c b/gcc/testsuite/gcc.dg/vect/pr31699.c
index c73d11a224e..b0b9971fcfc 100644
--- a/gcc/testsuite/gcc.dg/vect/pr31699.c
+++ b/gcc/testsuite/gcc.dg/vect/pr31699.c
@@ -7,9 +7,9 @@
 float x[256];
 
 __attribute__ ((noinline))
-double *foo(void)
+float *foo(void)
 {
- double *z = malloc (sizeof(double) * 256);
+ float *z = malloc (sizeof(float) * 256);
 
  int i;
  for (i=0; i<256; ++i)
@@ -34,5 +34,5 @@ int main()
 }
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_intfloat_cvt } } } */
-/* { dg-final { scan-tree-dump-times "vector alignment may not be reachable" 1 "vect" { target { ! vector_alignment_reachable } xfail { vect_element_align_preferred } } } } */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 1 "vect" { target { ! vector_alignment_reachable } xfail { vect_element_align_preferred } } } } */
+/* { dg-final { scan-tree-dump-times "vector alignment may not be reachable" 1 "vect" { target { ! vector_alignment_reachable } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 1 "vect" { target { ! vector_alignment_reachable } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr45752.c b/gcc/testsuite/gcc.dg/vect/pr45752.c
index 22a398f0acc..755205b275a 100644
--- a/gcc/testsuite/gcc.dg/vect/pr45752.c
+++ b/gcc/testsuite/gcc.dg/vect/pr45752.c
@@ -103,8 +103,8 @@ int main (int argc, const char* argv[])
 	26776, 9542, 363804, 169059, 25853, 36596, 12962, 503404, 224634,
 	35463 };
 #else
-  unsigned int check_results[N];
-  unsigned int check_results2[N];
+  volatile unsigned int check_results[N];
+  volatile unsigned int check_results2[N];
 
   for (i = 0; i < N / 5; i++)
     {
@@ -140,7 +140,7 @@ int main (int argc, const char* argv[])
       check_results2[i * 5 + 4] = (M40 * a + M41 * b + M42 * c
 				   + M43 * d + M44 * e);
 
-      asm volatile ("");
+      asm volatile ("" ::: "memory");
     }
 #endif
 
diff --git a/gcc/testsuite/gcc.dg/vect/pr65947-5.c b/gcc/testsuite/gcc.dg/vect/pr65947-5.c
index 3e34b7a2644..709f17f80a4 100644
--- a/gcc/testsuite/gcc.dg/vect/pr65947-5.c
+++ b/gcc/testsuite/gcc.dg/vect/pr65947-5.c
@@ -37,7 +37,7 @@ main (void)
   for (int i = 32; i < N; ++i)
     {
       a[i] = 70 + (i & 3);
-      asm volatile ("");
+      asm volatile ("" ::: "memory");
     }
 
   check_vect ();
diff --git a/gcc/testsuite/gcc.dg/vect/section-anchors-vect-69.c b/gcc/testsuite/gcc.dg/vect/section-anchors-vect-69.c
index b5aaa924bb2..e3466d0da1d 100644
--- a/gcc/testsuite/gcc.dg/vect/section-anchors-vect-69.c
+++ b/gcc/testsuite/gcc.dg/vect/section-anchors-vect-69.c
@@ -120,4 +120,4 @@ int main (void)
 /* { dg-final { scan-tree-dump-times "vectorized 4 loops" 1 "vect" { target vect_int } } } */
 /* Alignment forced using versioning until the pass that increases alignment
   is extended to handle structs.  */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 4 "vect" { target { vect_int && { ! vector_alignment_reachable } } xfail { vect_element_align_preferred } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 4 "vect" { target {vect_int && {! vector_alignment_reachable} } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/slp-19c.c b/gcc/testsuite/gcc.dg/vect/slp-19c.c
index de47f7760c4..cda6a096332 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-19c.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-19c.c
@@ -21,7 +21,7 @@ main1 ()
   for (unsigned int i = 0; i < N * 8; ++i)
     {
       in[i] = i & 63;
-      asm volatile ("");
+      asm volatile ("" ::: "memory");
     }
 #endif
   unsigned int ia[N*2], a0, a1, a2, a3;
diff --git a/gcc/testsuite/gcc.dg/vect/slp-23.c b/gcc/testsuite/gcc.dg/vect/slp-23.c
index 8dd95528cae..88708e645d6 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-23.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-23.c
@@ -107,6 +107,8 @@ int main (void)
 
 /* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target { vect_strided8 && { ! { vect_no_align} } } } } } */
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { ! { vect_strided8 || vect_no_align } } } } } */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" { target { ! vect_any_perm } } } } */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" { target vect_any_perm xfail vect_variable_length } } } */
+/* We fail to vectorize the second loop with variable-length SVE but
+   fall back to 128-bit vectors, which does use SLP.  */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" { target { ! vect_perm } xfail aarch64_sve } } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" { target vect_perm } } } */
   
diff --git a/gcc/testsuite/gcc.dg/vect/slp-28.c b/gcc/testsuite/gcc.dg/vect/slp-28.c
index d9f50d1c097..4211b94ad7f 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-28.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-28.c
@@ -57,7 +57,8 @@ main1 ()
         abort ();
     }
   
-  /* Vectorizable with a fully-masked loop or if VF==8.  */
+  /* Not vectorizable because of data dependencies: distance 3 is greater than 
+     the actual VF with SLP (2), but the analysis fail to detect that for now.  */
   for (i = 3; i < N/4; i++)
     {
       in3[i*4] = in3[(i-3)*4] + 5;
diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-1.c b/gcc/testsuite/gcc.dg/vect/slp-perm-1.c
index 9a0835575d3..6bd16ef43b0 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-perm-1.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-perm-1.c
@@ -51,7 +51,7 @@ int main (int argc, const char* argv[])
 #if N == 16
   unsigned int check_results[N] = {1470, 395, 28271, 5958, 1655, 111653, 10446, 2915, 195035, 14934, 4175, 278417, 19422, 5435, 361799, 0};
 #else
-  unsigned int check_results[N] = {};
+  volatile unsigned int check_results[N] = {};
 
   for (unsigned int i = 0; i < N / 3; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-4.c b/gcc/testsuite/gcc.dg/vect/slp-perm-4.c
index a706f1792f5..3a4420c53e4 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-perm-4.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-perm-4.c
@@ -70,15 +70,15 @@ int main (int argc, const char* argv[])
     {
       input[i] = i%256;
       output[i] = 0;
-      __asm__ volatile ("");
+      asm volatile ("" ::: "memory");
     }
 
 #if N == 20
   unsigned int check_results[N]
-    = {3208, 1334, 28764, 35679, 2789, 13028, 4754, 168364, 91254, 12399, 
+    = {3208, 1334, 28764, 35679, 2789, 13028, 4754, 168364, 91254, 12399,
     22848, 8174, 307964, 146829, 22009, 32668, 11594, 447564, 202404, 31619};
 #else
-  unsigned int check_results[N];
+  volatile unsigned int check_results[N];
 
   for (i = 0; i < N / 5; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-5.c b/gcc/testsuite/gcc.dg/vect/slp-perm-5.c
index edd36ab5255..52939133ca8 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-perm-5.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-perm-5.c
@@ -72,8 +72,8 @@ int main (int argc, const char* argv[])
   int check_results2[N] = { 4322, 135, 13776, 629, 23230, 1123, 32684, 1617,
 			    42138, 2111, 0, 0, 0, 0, 0, 0 };
 #else
-  int check_results[N] = {};
-  int check_results2[N] = {};
+  volatile int check_results[N] = {};
+  volatile int check_results2[N] = {};
 
   for (int i = 0; i < N / 3; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-6.c b/gcc/testsuite/gcc.dg/vect/slp-perm-6.c
index e44ab555d98..4eb648ac71b 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-perm-6.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-perm-6.c
@@ -71,8 +71,8 @@ int main (int argc, const char* argv[])
   int check_results2[N] = { 0, 112, 810, 336, 1620, 560, 2430, 784, 3240, 1008,
 			    0, 0, 0, 0, 0, 0 };
 #else
-  int check_results[N] = {};
-  int check_results2[N] = {};
+  volatile int check_results[N] = {};
+  volatile int check_results2[N] = {};
 
   for (int i = 0; i < N / 3; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-7.c b/gcc/testsuite/gcc.dg/vect/slp-perm-7.c
index bc06a78d2e5..baf7f7888a3 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-perm-7.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-perm-7.c
@@ -66,8 +66,8 @@ int main (int argc, const char* argv[])
   int check_results[N] = {1470, 395, 28271, 5958, 1655, 111653, 10446, 2915, 195035, 14934, 4175, 278417, 19422, 5435, 361799, 0};
   int check_results2[N] = {0, 405, 810, 1215, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 #else
-  int check_results[N] = {};
-  int check_results2[N] = {};
+  volatile int check_results[N] = {};
+  volatile int check_results2[N] = {};
 
   for (int i = 0; i < N / 3; i++)
     {
diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-8.c b/gcc/testsuite/gcc.dg/vect/slp-perm-8.c
index c94125f5fb4..94d4455dfd9 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-perm-8.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-perm-8.c
@@ -31,11 +31,7 @@ int main (int argc, const char* argv[])
 {
   unsigned char input[N], output[N];
   unsigned char check_results[N];
-#if N < 256
-  unsigned char i;
-#else
   unsigned int i;
-#endif
 
   check_vect ();
 
diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-9.c b/gcc/testsuite/gcc.dg/vect/slp-perm-9.c
index ad0832348dc..b01d493b6e7 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-perm-9.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-perm-9.c
@@ -37,7 +37,7 @@ int main (int argc, const char* argv[])
     {
       input[i] = i;
       output[i] = 0;
-      asm volatile ("");
+      asm volatile ("" ::: "memory");
     }
 
   for (i = 0; i < N / 3; i++)
diff --git a/gcc/testsuite/gcc.dg/vect/trapv-vect-reduc-4.c b/gcc/testsuite/gcc.dg/vect/trapv-vect-reduc-4.c
index d68687930de..5121414260b 100644
--- a/gcc/testsuite/gcc.dg/vect/trapv-vect-reduc-4.c
+++ b/gcc/testsuite/gcc.dg/vect/trapv-vect-reduc-4.c
@@ -47,10 +47,8 @@ int main (void)
 }
 
 /* 2 for the first loop.  */
-/* { dg-final { scan-tree-dump-times "Detected reduction\\." 3 "vect" { target { vect_1_size } } } } */
-/* { dg-final { scan-tree-dump-times "Detected reduction\\." 4 "vect" { target { vect_2_sizes } } } } */
-/* { dg-final { scan-tree-dump-times "Detected reduction\\." 5 "vect" { target { vect_3_sizes } } } } */
-/* { dg-final { scan-tree-dump-times "not vectorized" 1 "vect" { target vect_1_size } } } */
-/* { dg-final { scan-tree-dump-times "not vectorized" 2 "vect" { target vect_2_sizes } } } */
-/* { dg-final { scan-tree-dump-times "not vectorized" 3 "vect" { target vect_3_sizes } } } */
+/* { dg-final { scan-tree-dump-times "Detected reduction\\." 3 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "Detected reduction\\." "vect" { target vect_multiple_sizes } } } */
+/* { dg-final { scan-tree-dump-times "not vectorized" 1 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "not vectorized" "vect" { target vect_multiple_sizes } } } */
 /* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target { ! vect_no_int_min_max } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-104.c b/gcc/testsuite/gcc.dg/vect/vect-104.c
index f86043e6231..a77c98735eb 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-104.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-104.c
@@ -62,6 +62,6 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 0 "vect" } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 1 "vect" { target vect_1_size } } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 2 "vect" { target vect_2_sizes } } } */
-/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 3 "vect" { target vect_3_sizes } } } */
+/* { dg-final { scan-tree-dump-times "possible dependence between data-refs" 1 "vect" { target { ! vect_multiple_sizes } } } } */
+/* { dg-final { scan-tree-dump "possible dependence between data-refs" "vect" { target vect_multiple_sizes } } } */
+
diff --git a/gcc/testsuite/gcc.dg/vect/vect-109.c b/gcc/testsuite/gcc.dg/vect/vect-109.c
index 566cc8f7ab3..9a507105899 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-109.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-109.c
@@ -76,5 +76,5 @@ int main (void)
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target vect_element_align } } } */
 /* { dg-final { scan-tree-dump-times "not vectorized: unsupported unaligned store" 2 "vect" { xfail vect_element_align } } } */
-/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 3 "vect" { target { vect_element_align } xfail { ! vect_unaligned_possible } } } } */
+/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 3 "vect" { target vect_element_align xfail { ! vect_unaligned_possible } } } } */
 
diff --git a/gcc/testsuite/gcc.dg/vect/vect-33.c b/gcc/testsuite/gcc.dg/vect/vect-33.c
index af5aefa2455..e215052ff77 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-33.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-33.c
@@ -37,6 +37,6 @@ int main (void)
 } 
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect"  } } */
-/* { dg-final { scan-tree-dump "Vectorizing an unaligned access" "vect" { target { { { ! powerpc*-*-* } && vect_hw_misalign } && { ! { vect64 && vect_1_size } } } xfail { ! vect_unaligned_possible } } } } */
-/* { dg-final { scan-tree-dump "Alignment of access forced using peeling" "vect" { target { vector_alignment_reachable && { vect64 && vect_1_size } } } } } */
+/* { dg-final { scan-tree-dump "Vectorizing an unaligned access" "vect" { target { { { ! powerpc*-*-* } && vect_hw_misalign } && { { ! vect64 } || vect_multiple_sizes } } xfail { ! vect_unaligned_possible } } } }  */
+/* { dg-final { scan-tree-dump "Alignment of access forced using peeling" "vect" { target { vector_alignment_reachable && { vect64 && {! vect_multiple_sizes} } } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 1 "vect" { target { { {! vector_alignment_reachable} || {! vect64} } && {! vect_hw_misalign} } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-42.c b/gcc/testsuite/gcc.dg/vect/vect-42.c
index 55adfc93df3..a65b4a62276 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-42.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-42.c
@@ -67,5 +67,5 @@ int main (void)
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 3 "vect" { target { vect_no_align && { ! vect_hw_misalign } } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 1 "vect" { target { { ! vector_alignment_reachable } && { ! vect_element_align } } } } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 4 "vect" { xfail { vect_no_align || { { !  vector_alignment_reachable } || vect_element_align  } } } } }  */
-/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 3 "vect" { target { vect_element_align } xfail { ! { vect_unaligned_possible && vect_align_stack_vars } } } } } */
+/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 3 "vect" { target vect_element_align xfail { ! { vect_unaligned_possible && vect_align_stack_vars } } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 1 "vect" { xfail { vect_no_align || { { ! vector_alignment_reachable } || vect_element_align } } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-44.c b/gcc/testsuite/gcc.dg/vect/vect-44.c
index 96dab0ac03e..03ef2c0f671 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-44.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-44.c
@@ -66,6 +66,6 @@ int main (void)
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 3 "vect" { xfail { ! vect_unaligned_possible } } } } */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { xfail { { vect_no_align && { ! vect_hw_misalign } } || { { ! vector_alignment_reachable } && { ! vect_element_align_preferred } } } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { xfail { { vect_no_align && { ! vect_hw_misalign } } || {! vector_alignment_reachable} } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning." 3 "vect" { target { vect_no_align && { ! vect_hw_misalign } } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning." 1 "vect" { target { {! vector_alignment_reachable} && {{! vect_no_align} && {! vect_hw_misalign} } } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-50.c b/gcc/testsuite/gcc.dg/vect/vect-50.c
index e11036bf130..c9500ca91e5 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-50.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-50.c
@@ -62,6 +62,6 @@ int main (void)
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 3 "vect" { xfail { ! vect_unaligned_possible } } } } */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { xfail { { vect_no_align && { ! vect_hw_misalign } } || { { ! vector_alignment_reachable } && { ! vect_element_align_preferred } } } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { xfail { { vect_no_align && { ! vect_hw_misalign } } || {! vector_alignment_reachable} } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning." 3 "vect" { target { vect_no_align && { ! vect_hw_misalign } } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning." 1 "vect" { target { {! vector_alignment_reachable} && { {! vect_no_align } && {! vect_hw_misalign } } } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-56.c b/gcc/testsuite/gcc.dg/vect/vect-56.c
index 673ab23a2b4..8060b05e781 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-56.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-56.c
@@ -70,5 +70,5 @@ int main (void)
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_align && { ! vect_hw_misalign } } } } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 2 "vect" { target { ! vect_element_align } xfail { ! vect_unaligned_possible } } } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { target { vect_element_align } xfail { ! vect_unaligned_possible } } } } */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { target { ! vect_element_align } xfail { vect_element_align_preferred } } } } */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 1 "vect" { target { vect_element_align } xfail { vect_element_align_preferred } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { target { { ! vect_element_align } || vect_element_align_preferred} } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 1 "vect" { target { vect_element_align && { ! vect_element_align_preferred } } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-60.c b/gcc/testsuite/gcc.dg/vect/vect-60.c
index 9dcfd85ce9c..3b7477c96ab 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-60.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-60.c
@@ -71,5 +71,5 @@ int main (void)
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_align && { ! vect_hw_misalign } } } } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 2 "vect" { target { ! vect_element_align } xfail { ! vect_unaligned_possible } } } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { target { vect_element_align } xfail { ! vect_unaligned_possible } } } } */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { target { ! vect_element_align } xfail { vect_element_align_preferred } } } } */
-/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 1 "vect" { target { vect_element_align } xfail { vect_element_align_preferred } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" { target { { ! vect_element_align } || vect_element_align_preferred } } } } */
+/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 1 "vect" { target { vect_element_align && { ! vect_element_align_preferred } } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-70.c b/gcc/testsuite/gcc.dg/vect/vect-70.c
index 8f212571693..793dbfb7481 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-70.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-70.c
@@ -12,6 +12,7 @@
 
 #define N (NINTS * 6)
 
+/* Keep execution time down.  */
 #if N <= 24
 #define OUTERN N
 #else
@@ -30,6 +31,7 @@ struct test1{
   struct s e[N]; /* array e.n is aligned */
 };
 
+/* Avoid big local temporaries.  */
 #if NINTS > 8
 struct test1 tmp1;
 #endif
diff --git a/gcc/testsuite/gcc.dg/vect/vect-91.c b/gcc/testsuite/gcc.dg/vect/vect-91.c
index ffa95b71d24..9430da3290a 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-91.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-91.c
@@ -7,13 +7,14 @@
 
 #define N 256
 
+/* Pick a value greater than the vector length.  */
 #if VECTOR_BITS > 128
 #define OFF (VECTOR_BITS * 5 / 32)
 #else
 #define OFF 20
 #endif
 
-extern int a[N+OFF];
+extern int a[N + OFF];
 
 /* The alignment of 'pa' is unknown. 
    Yet we do know that both the read access and write access have 
@@ -58,7 +59,7 @@ main3 ()
 
   for (i = 0; i < N; i++)
     {
-      a[i] = a[i+OFF];
+      a[i] = a[i + OFF];
     }
 
   return 0;
diff --git a/gcc/testsuite/gcc.dg/vect/vect-96.c b/gcc/testsuite/gcc.dg/vect/vect-96.c
index c72595f97b8..0cb935b9f16 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-96.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-96.c
@@ -48,7 +48,7 @@ int main (void)
    For targets that don't support unaligned loads, version for the store.  */
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 2 "vect" { target { { {! vect_no_align} && vector_alignment_reachable } && { ! vect_align_stack_vars } } } } } */
-/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { target { { {! vect_no_align} && vector_alignment_reachable } && vect_align_stack_vars } } } } */
+/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 2 "vect" { target { { {! vect_no_align} && vector_alignment_reachable } && { ! vect_align_stack_vars } } xfail { ! vect_unaligned_possible } } } } */
+/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { target { { {! vect_no_align} && vector_alignment_reachable } && vect_align_stack_vars } xfail { ! vect_unaligned_possible } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 1 "vect" { xfail { { vect_no_align } || { { ! vector_alignment_reachable} || vect_element_align } } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning." 1 "vect" { target { { vect_no_align && { ! vect_hw_misalign } } || { {! vector_alignment_reachable} && {! vect_element_align} } } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-multitypes-1.c b/gcc/testsuite/gcc.dg/vect/vect-multitypes-1.c
index 9f96601e4a7..378a5fe642a 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-multitypes-1.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-multitypes-1.c
@@ -90,5 +90,5 @@ int main (void)
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail { vect_no_align && { ! vect_hw_misalign } } } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 2 "vect" { xfail { { ! vect_unaligned_possible } || vect_sizes_32B_16B } } } } */
-/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 4 "vect" { xfail { { ! vect_unaligned_possible } || vect_sizes_32B_16B } } } } */
+/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 4 "vect" { target { vect_no_align && { { ! vect_hw_misalign } && vect_sizes_32B_16B } } }} } */
 
diff --git a/gcc/testsuite/gcc.dg/vect/vect-peel-3.c b/gcc/testsuite/gcc.dg/vect/vect-peel-3.c
index 231c13dfc28..d5c0cf10ce1 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-peel-3.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-peel-3.c
@@ -6,7 +6,7 @@
 
 #if VECTOR_BITS > 128
 #define NINTS (VECTOR_BITS / 32)
-#define EXTRA NINTS * 2
+#define EXTRA (NINTS * 2)
 #else
 #define NINTS 4
 #define EXTRA 10
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-s8a.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-s8a.c
index dc4f52019d5..ac674749b6f 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-s8a.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-s8a.c
@@ -1,4 +1,7 @@
 /* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target arm_v8_2a_dotprod_neon_hw { target { aarch64*-*-* || arm*-*-* } } } */
+/* { dg-additional-options "-march=armv8.2-a+dotprod" { target { aarch64*-*-* } } } */
+/* { dg-add-options arm_v8_2a_dotprod_neon }  */
 
 #include <stdarg.h>
 #include "tree-vect.h"
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-u8a.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-u8a.c
index f3cc6c78c25..d020f643bb8 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-u8a.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-dot-u8a.c
@@ -1,4 +1,7 @@
 /* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target arm_v8_2a_dotprod_neon_hw { target { aarch64*-*-* || arm*-*-* } } } */
+/* { dg-additional-options "-march=armv8.2-a+dotprod" { target { aarch64*-*-* } } } */
+/* { dg-add-options arm_v8_2a_dotprod_neon }  */
 
 #include <stdarg.h>
 #include "tree-vect.h"
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vdot-compile.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vdot-compile.c
new file mode 100644
index 00000000000..b7378adf8ee
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vdot-compile.c
@@ -0,0 +1,73 @@
+/* { dg-do compile { target { aarch64*-*-* } } } */
+/* { dg-additional-options "-O3 -march=armv8.2-a+dotprod" } */
+
+#include <arm_neon.h>
+
+/* Unsigned Dot Product instructions.  */
+
+uint32x2_t ufoo (uint32x2_t r, uint8x8_t x, uint8x8_t y)
+{
+  return vdot_u32 (r, x, y);
+}
+
+uint32x4_t ufooq (uint32x4_t r, uint8x16_t x, uint8x16_t y)
+{
+  return vdotq_u32 (r, x, y);
+}
+
+uint32x2_t ufoo_lane (uint32x2_t r, uint8x8_t x, uint8x8_t y)
+{
+  return vdot_lane_u32 (r, x, y, 0);
+}
+
+uint32x2_t ufoo_laneq (uint32x2_t r, uint8x8_t x, uint8x16_t y)
+{
+  return vdot_laneq_u32 (r, x, y, 0);
+}
+
+uint32x4_t ufooq_lane (uint32x4_t r, uint8x16_t x, uint8x8_t y)
+{
+  return vdotq_lane_u32 (r, x, y, 0);
+}
+
+uint32x4_t ufooq_laneq (uint32x4_t r, uint8x16_t x, uint8x16_t y)
+{
+  return vdotq_laneq_u32 (r, x, y, 0);
+}
+
+/* Signed Dot Product instructions.  */
+
+int32x2_t sfoo (int32x2_t r, int8x8_t x, int8x8_t y)
+{
+  return vdot_s32 (r, x, y);
+}
+
+int32x4_t sfooq (int32x4_t r, int8x16_t x, int8x16_t y)
+{
+  return vdotq_s32 (r, x, y);
+}
+
+int32x2_t sfoo_lane (int32x2_t r, int8x8_t x, int8x8_t y)
+{
+  return vdot_lane_s32 (r, x, y, 0);
+}
+
+int32x2_t sfoo_laneq (int32x2_t r, int8x8_t x, int8x16_t y)
+{
+  return vdot_laneq_s32 (r, x, y, 0);
+}
+
+int32x4_t sfooq_lane (int32x4_t r, int8x16_t x, int8x8_t y)
+{
+  return vdotq_lane_s32 (r, x, y, 0);
+}
+
+int32x4_t sfooq_laneq (int32x4_t r, int8x16_t x, int8x16_t y)
+{
+  return vdotq_laneq_s32 (r, x, y, 0);
+}
+
+/* { dg-final { scan-assembler-times {[us]dot\tv[0-9]+\.2s, v[0-9]+\.8b, v[0-9]+\.8b} 2 } } */
+/* { dg-final { scan-assembler-times {[us]dot\tv[0-9]+\.2s, v[0-9]+\.8b, v[0-9]+\.4b\[[0-9]+\]}  4 } } */
+/* { dg-final { scan-assembler-times {[us]dot\tv[0-9]+\.4s, v[0-9]+\.16b, v[0-9]+\.16b}  2 } } */
+/* { dg-final { scan-assembler-times {[us]dot\tv[0-9]+\.4s, v[0-9]+\.16b, v[0-9]+\.4b\[[0-9]+\]}  4 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vdot-exec.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vdot-exec.c
new file mode 100644
index 00000000000..3e7cd6c2fc2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vdot-exec.c
@@ -0,0 +1,81 @@
+/* { dg-skip-if "can't compile on arm." { arm*-*-* } } */
+/* { dg-do run { target { aarch64*-*-* } } } */
+/* { dg-additional-options "-O3 -march=armv8.2-a+dotprod" } */
+/* { dg-require-effective-target arm_v8_2a_dotprod_neon_hw } */
+
+#include <arm_neon.h>
+
+extern void abort();
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+# define ORDER(x, y) y
+#else
+# define ORDER(x, y) x - y
+#endif
+
+#define P(n1,n2) n1,n1,n1,n1,n2,n2,n2,n2
+#define ARR(nm, p, ty, ...) ty nm##_##p = { __VA_ARGS__ }
+#define TEST(t1, t2, t3, f, r1, r2, n1, n2) \
+	ARR(f, x, t1, r1);		    \
+	ARR(f, y, t2, r2);		    \
+	t3 f##_##r = {0};		    \
+	f##_##r = f (f##_##r, f##_##x, f##_##y);  \
+	if (f##_##r[0] != n1 || f##_##r[1] != n2)   \
+	  abort ();
+
+#define TEST_LANE(t1, t2, t3, f, r1, r2, n1, n2, n3, n4) \
+	ARR(f, x, t1, r1);		    \
+	ARR(f, y, t2, r2);		    \
+	t3 f##_##rx = {0};		    \
+	f##_##rx = f (f##_##rx, f##_##x, f##_##y, ORDER (1, 0));  \
+	if (f##_##rx[0] != n1 || f##_##rx[1] != n2)   \
+	  abort ();				    \
+	t3 f##_##rx1 = {0};			    \
+	f##_##rx1 = f (f##_##rx1, f##_##x, f##_##y, ORDER (1, 1));  \
+	if (f##_##rx1[0] != n3 || f##_##rx1[1] != n4)   \
+	  abort ();
+
+#define Px(n1,n2,n3,n4) P(n1,n2),P(n3,n4)
+#define TEST_LANEQ(t1, t2, t3, f, r1, r2, n1, n2, n3, n4, n5, n6, n7, n8) \
+	ARR(f, x, t1, r1);		    \
+	ARR(f, y, t2, r2);		    \
+	t3 f##_##rx = {0};		    \
+	f##_##rx = f (f##_##rx, f##_##x, f##_##y, ORDER (3, 0));  \
+	if (f##_##rx[0] != n1 || f##_##rx[1] != n2)   \
+	  abort ();				    \
+	t3 f##_##rx1 = {0};			    \
+	f##_##rx1 = f (f##_##rx1, f##_##x, f##_##y, ORDER (3, 1));  \
+	if (f##_##rx1[0] != n3 || f##_##rx1[1] != n4)   \
+	  abort (); \
+	t3 f##_##rx2 = {0};				    \
+	f##_##rx2 = f (f##_##rx2, f##_##x, f##_##y, ORDER (3, 2));  \
+	if (f##_##rx2[0] != n5 || f##_##rx2[1] != n6)   \
+	  abort ();				    \
+	t3 f##_##rx3 = {0};			    \
+	f##_##rx3 = f (f##_##rx3, f##_##x, f##_##y, ORDER (3, 3));  \
+	if (f##_##rx3[0] != n7 || f##_##rx3[1] != n8)   \
+	  abort ();
+
+int
+main()
+{
+  TEST (uint8x8_t, uint8x8_t, uint32x2_t, vdot_u32, P(1,2), P(2,3), 8, 24);
+  TEST (int8x8_t, int8x8_t, int32x2_t, vdot_s32, P(1,2), P(-2,-3), -8, -24);
+
+  TEST (uint8x16_t, uint8x16_t, uint32x4_t, vdotq_u32, P(1,2), P(2,3), 8, 24);
+  TEST (int8x16_t, int8x16_t, int32x4_t, vdotq_s32, P(1,2), P(-2,-3), -8, -24);
+
+  TEST_LANE (uint8x8_t, uint8x8_t, uint32x2_t, vdot_lane_u32, P(1,2), P(2,3), 8, 16, 12, 24);
+  TEST_LANE (int8x8_t, int8x8_t, int32x2_t, vdot_lane_s32, P(1,2), P(-2,-3), -8, -16, -12, -24);
+
+  TEST_LANE (uint8x16_t, uint8x8_t, uint32x4_t, vdotq_lane_u32, P(1,2), P(2,3), 8, 16, 12, 24);
+  TEST_LANE (int8x16_t, int8x8_t, int32x4_t, vdotq_lane_s32, P(1,2), P(-2,-3), -8, -16, -12, -24);
+
+  TEST_LANEQ (uint8x8_t, uint8x16_t, uint32x2_t, vdot_laneq_u32, P(1,2), Px(2,3,1,4), 8, 16, 12, 24, 4, 8, 16, 32);
+  TEST_LANEQ (int8x8_t, int8x16_t, int32x2_t, vdot_laneq_s32, P(1,2), Px(-2,-3,-1,-4), -8, -16, -12, -24, -4, -8, -16, -32);
+
+  TEST_LANEQ (uint8x16_t, uint8x16_t, uint32x4_t, vdotq_laneq_u32, Px(1,2,2,1), Px(2,3,1,4), 8, 16, 12, 24, 4, 8, 16, 32);
+  TEST_LANEQ (int8x16_t, int8x16_t, int32x4_t, vdotq_laneq_s32, Px(1,2,2,1), Px(-2,-3,-1,-4), -8, -16, -12, -24, -4, -8, -16, -32);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-qi.h b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-qi.h
new file mode 100644
index 00000000000..90b00aff95c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-qi.h
@@ -0,0 +1,15 @@
+TYPE char X[N] __attribute__ ((__aligned__(__BIGGEST_ALIGNMENT__)));
+TYPE char Y[N] __attribute__ ((__aligned__(__BIGGEST_ALIGNMENT__)));
+
+__attribute__ ((noinline)) int
+foo1(int len) {
+  int i;
+  TYPE int result = 0;
+  TYPE short prod;
+
+  for (i=0; i<len; i++) {
+    prod = X[i] * Y[i];
+    result += prod;
+  }
+  return result;
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-s8.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-s8.c
new file mode 100644
index 00000000000..57b5ef82f85
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-s8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile { target { aarch64*-*-* } } } */
+/* { dg-additional-options "-O3 -march=armv8.2-a+dotprod" } */
+
+#define N 64
+#define TYPE signed
+
+#include "vect-dot-qi.h"
+
+/* { dg-final { scan-assembler-times {sdot\tv[0-9]+\.4s, v[0-9]+\.16b, v[0-9]+\.16b} 4 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-u8.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-u8.c
new file mode 100644
index 00000000000..b2cef318500
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vect-dot-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile { target { aarch64*-*-* } } } */
+/* { dg-additional-options "-O3 -march=armv8.2-a+dotprod" } */
+
+#define N 64
+#define TYPE unsigned
+
+#include "vect-dot-qi.h"
+
+/* { dg-final { scan-assembler-times {udot\tv[0-9]+\.4s, v[0-9]+\.16b, v[0-9]+\.16b} 4 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/cmpelim_mult_uses_1.c b/gcc/testsuite/gcc.target/aarch64/cmpelim_mult_uses_1.c
new file mode 100644
index 00000000000..953c388037f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/cmpelim_mult_uses_1.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+/* X is both compared against zero and used.  Make sure we can still
+   generate an ADDS and avoid an explicit comparison against zero.  */
+
+int
+foo (int x, int y)
+{
+  x += y;
+  if (x != 0)
+    x = x + 2;
+  return x;
+}
+
+/* { dg-final { scan-assembler-times "adds\\tw\[0-9\]+, w\[0-9\]+, w\[0-9\]+" 1 } } */
+/* { dg-final { scan-assembler-not "cmp\\tw\[0-9\]+, 0" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fix_trunc1.c b/gcc/testsuite/gcc.target/aarch64/fix_trunc1.c
new file mode 100644
index 00000000000..0441458f635
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/fix_trunc1.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+float
+f1 (float x)
+{
+  int y = x;
+
+  return (float) y;
+}
+
+double
+f2 (double x)
+{
+  long y = x;
+
+  return (double) y;
+}
+
+/* { dg-final { scan-assembler "fcvtzs\\ts\[0-9\]+, s\[0-9\]+" } } */
+/* { dg-final { scan-assembler "scvtf\\ts\[0-9\]+, s\[0-9\]+" } } */
+/* { dg-final { scan-assembler "fcvtzs\\td\[0-9\]+, d\[0-9\]+" } } */
+/* { dg-final { scan-assembler "scvtf\\td\[0-9\]+, d\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/inline-lrint_2.c b/gcc/testsuite/gcc.target/aarch64/inline-lrint_2.c
index 6080e186d8f..bd0c73c8d34 100644
--- a/gcc/testsuite/gcc.target/aarch64/inline-lrint_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/inline-lrint_2.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target ilp32 } */
-/* { dg-options "-O3 -fno-math-errno" } */
+/* { dg-options "-O3 -fno-math-errno -fno-trapping-math" } */
 
 #include "lrint-matherr.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/ldp_stp_unaligned_2.c b/gcc/testsuite/gcc.target/aarch64/ldp_stp_unaligned_2.c
new file mode 100644
index 00000000000..1e46755a39a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ldp_stp_unaligned_2.c
@@ -0,0 +1,18 @@
+/* { dg-options "-O2 -fomit-frame-pointer" } */
+
+/* Check that we split unaligned LDP/STP into base and aligned offset.  */
+
+typedef struct
+{
+  int a, b, c, d, e;
+} S;
+
+void foo (S *);
+
+void test (int x)
+{
+  S s = { .a = x };
+  foo (&s);
+}
+
+/* { dg-final { scan-assembler-not "mov\tx\[0-9\]+, sp" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr78733.c b/gcc/testsuite/gcc.target/aarch64/pr78733.c
index ce462cedf9f..3cdb3ba7373 100644
--- a/gcc/testsuite/gcc.target/aarch64/pr78733.c
+++ b/gcc/testsuite/gcc.target/aarch64/pr78733.c
@@ -7,4 +7,5 @@ t (void)
   return (__int128)1 << 80;
 }
 
-/* { dg-final { scan-assembler "adr" } } */
+/* { dg-final { scan-assembler "\tmov\tx0, 0" } } */
+/* { dg-final { scan-assembler "\tmov\tx1, 65536" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr79041-2.c b/gcc/testsuite/gcc.target/aarch64/pr79041-2.c
index a889dfdd895..62856f10438 100644
--- a/gcc/testsuite/gcc.target/aarch64/pr79041-2.c
+++ b/gcc/testsuite/gcc.target/aarch64/pr79041-2.c
@@ -8,5 +8,6 @@ t (void)
   return (__int128)1 << 80;
 }
 
-/* { dg-final { scan-assembler "adr" } } */
+/* { dg-final { scan-assembler "\tmov\tx0, 0" } } */
+/* { dg-final { scan-assembler "\tmov\tx1, 65536" } } */
 /* { dg-final { scan-assembler-not "adrp" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr80295.c b/gcc/testsuite/gcc.target/aarch64/pr80295.c
new file mode 100644
index 00000000000..b3866d8d6a9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr80295.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+/* { dg-options "-mabi=ilp32" } */
+
+void f (void *b) 
+{ 
+  __builtin_update_setjmp_buf (b); 
+}
+
diff --git a/gcc/testsuite/gcc.target/aarch64/spellcheck_1.c b/gcc/testsuite/gcc.target/aarch64/spellcheck_1.c
index ccfe417e644..f57e0c54632 100644
--- a/gcc/testsuite/gcc.target/aarch64/spellcheck_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/spellcheck_1.c
@@ -4,6 +4,6 @@ __attribute__((target ("arch=armv8-a-typo"))) void
 foo ()
 {
   /* { dg-message "valid arguments are: \[^\n\r]*; did you mean 'armv8-a'?"  "" { target *-*-* } .-1 } */
-  /* { dg-error "unknown value 'armv8-a-typo' for 'arch' target attribute"  "" { target *-*-* } .-2 } */
-  /* { dg-error "target attribute 'arch=armv8-a-typo' is invalid"  "" { target *-*-* } .-3 } */
+  /* { dg-error "invalid name \\(\"armv8-a-typo\"\\) in 'target\\(\"arch=\"\\)' pragma or attribute"  "" { target *-*-* } .-2 } */
+  /* { dg-error "pragma or attribute 'target\\(\"arch=armv8-a-typo\"\\)' is not valid"  "" { target *-*-* } .-3 } */
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/spellcheck_2.c b/gcc/testsuite/gcc.target/aarch64/spellcheck_2.c
index 42ba51a7226..70096f89e0b 100644
--- a/gcc/testsuite/gcc.target/aarch64/spellcheck_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/spellcheck_2.c
@@ -3,7 +3,7 @@
 __attribute__((target ("cpu=cortex-a57-typo"))) void
 foo ()
 {
-  /* { dg-message "valid arguments are: \[^\n\r]*; did you mean 'cortex-a57?"  "" { target *-*-* } .-1 } */
-  /* { dg-error "unknown value 'cortex-a57-typo' for 'cpu' target attribute"  "" { target *-*-* } .-2 } */
-  /* { dg-error "target attribute 'cpu=cortex-a57-typo' is invalid"  "" { target *-*-* } .-3 } */
+  /* { dg-message "valid arguments are: \[^\n\r]*; did you mean 'cortex-a57'?"  "" { target *-*-* } .-1 } */
+  /* { dg-error "invalid name \\(\"cortex-a57-typo\"\\) in 'target\\(\"cpu=\"\\)' pragma or attribute"  "" { target *-*-* } .-2 } */
+  /* { dg-error "pragma or attribute 'target\\(\"cpu=cortex-a57-typo\"\\)' is not valid"  "" { target *-*-* } .-3 } */
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/spellcheck_3.c b/gcc/testsuite/gcc.target/aarch64/spellcheck_3.c
index 03d2bbf14a0..20dff2b6e45 100644
--- a/gcc/testsuite/gcc.target/aarch64/spellcheck_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/spellcheck_3.c
@@ -3,7 +3,7 @@
 __attribute__((target ("tune=cortex-a57-typo"))) void
 foo ()
 {
-  /* { dg-message "valid arguments are: \[^\n\r]*; did you mean 'cortex-a57?"  "" { target *-*-* } .-1 } */
-  /* { dg-error "unknown value 'cortex-a57-typo' for 'tune' target attribute"  "" { target *-*-* } .-2 } */
-  /* { dg-error "target attribute 'tune=cortex-a57-typo' is invalid"  "" { target *-*-* } .-3 } */
+  /* { dg-message "valid arguments are: \[^\n\r]*; did you mean 'cortex-a57'?"  "" { target *-*-* } .-1 } */
+  /* { dg-error "invalid name \\(\"cortex-a57-typo\"\\) in 'target\\(\"tune=\"\\)' pragma or attribute"  "" { target *-*-* } .-2 } */
+  /* { dg-error "pragma or attribute 'target\\(\"tune=cortex-a57-typo\"\\)' is not valid"  "" { target *-*-* } .-3 } */
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-12.c b/gcc/testsuite/gcc.target/aarch64/stack-check-12.c
new file mode 100644
index 00000000000..2ce38483b6b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/stack-check-12.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-clash-protection --param stack-clash-protection-guard-size=12" } */
+/* { dg-require-effective-target supports_stack_clash_protection } */
+
+extern void arf (unsigned long int *, unsigned long int *);
+void
+frob ()
+{
+  unsigned long int num[1000];
+  unsigned long int den[1000];
+  arf (den, num);
+}
+
+/* This verifies that the scheduler did not break the dependencies
+   by adjusting the offsets within the probe and that the scheduler
+   did not reorder around the stack probes.  */
+/* { dg-final { scan-assembler-times "sub\\tsp, sp, #4096\\n\\tstr\\txzr, .sp, 4088." 3 } } */
+
+
+
diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-13.c b/gcc/testsuite/gcc.target/aarch64/stack-check-13.c
new file mode 100644
index 00000000000..d8886835989
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/stack-check-13.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-clash-protection --param stack-clash-protection-guard-size=12" } */
+/* { dg-require-effective-target supports_stack_clash_protection } */
+
+#define ARG32(X) X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+#define ARG192(X) ARG32(X),ARG32(X),ARG32(X),ARG32(X),ARG32(X),ARG32(X)
+void out1(ARG192(__int128));
+int t1(int);
+
+int t3(int x)
+{
+  if (x < 1000)
+    return t1 (x) + 1;
+
+  out1 (ARG192(1));
+  return 0;
+}
+
+
+
+/* This test creates a large (> 1k) outgoing argument area that needs
+   to be probed.  We don't test the exact size of the space or the
+   exact offset to make the test a little less sensitive to trivial
+   output changes.  */
+/* { dg-final { scan-assembler-times "sub\\tsp, sp, #....\\n\\tstr\\txzr, \\\[sp" 1 } } */
+
+
+
diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-14.c b/gcc/testsuite/gcc.target/aarch64/stack-check-14.c
new file mode 100644
index 00000000000..59ffe01376d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/stack-check-14.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-clash-protection --param stack-clash-protection-guard-size=12" } */
+/* { dg-require-effective-target supports_stack_clash_protection } */
+
+int t1(int);
+
+int t2(int x)
+{
+  char *p = __builtin_alloca (4050);
+  x = t1 (x);
+  return p[x];
+}
+
+
+/* This test has a constant sized alloca that is smaller than the
+   probe interval.  But it actually requires two probes instead
+   of one because of the optimistic assumptions we made in the
+   aarch64 prologue code WRT probing state. 
+
+   The form can change quite a bit so we just check for two
+   probes without looking at the actual address.  */
+/* { dg-final { scan-assembler-times "str\\txzr," 2 } } */
+
+
+
diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-15.c b/gcc/testsuite/gcc.target/aarch64/stack-check-15.c
new file mode 100644
index 00000000000..e06db6dc2f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/stack-check-15.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-clash-protection --param stack-clash-protection-guard-size=12" } */
+/* { dg-require-effective-target supports_stack_clash_protection } */
+
+int t1(int);
+
+int t2(int x)
+{
+  char *p = __builtin_alloca (x);
+  x = t1 (x);
+  return p[x];
+}
+
+
+/* This test has a variable sized alloca.  It requires 3 probes.
+   One in the loop, one for the residual and at the end of the
+   alloca area. 
+
+   The form can change quite a bit so we just check for two
+   probes without looking at the actual address.  */
+/* { dg-final { scan-assembler-times "str\\txzr," 3 } } */
+
+
+
diff --git a/gcc/testsuite/gcc.target/aarch64/subs_compare_1.c b/gcc/testsuite/gcc.target/aarch64/subs_compare_1.c
index 95c8f696fee..2691250f79e 100644
--- a/gcc/testsuite/gcc.target/aarch64/subs_compare_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/subs_compare_1.c
@@ -11,5 +11,5 @@ foo (int a, int b)
     return 0;
 }
 
-/* { dg-final { scan-assembler-times "subs\\tw\[0-9\]+, w\[0-9\]+, w\[0-9\]+" 1 } } */
-/* { dg-final { scan-assembler-not "cmp\\tw\[0-9\]+, w\[0-9\]+" } } */
+/* { dg-final { scan-assembler-times "subs\\tw\[0-9\]+, w\[0-9\]+, w\[0-9\]+" 1 { xfail *-*-* } } } */
+/* { dg-final { scan-assembler-not "cmp\\tw\[0-9\]+, w\[0-9\]+" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/subs_compare_2.c b/gcc/testsuite/gcc.target/aarch64/subs_compare_2.c
index 60c6d9e5ccd..d343acc1195 100644
--- a/gcc/testsuite/gcc.target/aarch64/subs_compare_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/subs_compare_2.c
@@ -11,5 +11,5 @@ foo (int a, int b)
     return 0;
 }
 
-/* { dg-final { scan-assembler-times "subs\\tw\[0-9\]+, w\[0-9\]+, #4" 1 } } */
+/* { dg-final { scan-assembler-times "subs\\tw\[0-9\]+, w\[0-9\]+, #4" 1 { xfail *-*-* } } } */
 /* { dg-final { scan-assembler-not "cmp\\tw\[0-9\]+, w\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_arith_1.c b/gcc/testsuite/gcc.target/aarch64/sve_arith_1.c
index b3c4cb9d8a7..1a61d6a7f40 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_arith_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_arith_1.c
@@ -1,40 +1,41 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
+
+#include <stdint.h>
 
 #define DO_REGREG_OPS(TYPE, OP, NAME)				\
-void varith_##TYPE##_##NAME (TYPE* dst, TYPE* src, int count)	\
+void varith_##TYPE##_##NAME (TYPE *dst, TYPE *src, int count)	\
 {								\
   for (int i = 0; i < count; ++i)				\
     dst[i] = dst[i] OP src[i];					\
 }
 
-
 #define DO_IMMEDIATE_OPS(VALUE, TYPE, OP, NAME)		\
-void varithimm_##NAME##_##TYPE (TYPE* dst, int count)	\
+void varithimm_##NAME##_##TYPE (TYPE *dst, int count)	\
 {							\
   for (int i = 0; i < count; ++i)			\
     dst[i] = dst[i] OP VALUE;				\
 }
 
 #define DO_ARITH_OPS(TYPE, OP, NAME)			\
-DO_REGREG_OPS (TYPE, OP, NAME);				\
-DO_IMMEDIATE_OPS (0, TYPE, OP, NAME ## 0);		\
-DO_IMMEDIATE_OPS (5, TYPE, OP, NAME ## 5);		\
-DO_IMMEDIATE_OPS (255, TYPE, OP, NAME ## 255);		\
-DO_IMMEDIATE_OPS (256, TYPE, OP, NAME ## 256);		\
-DO_IMMEDIATE_OPS (257, TYPE, OP, NAME ## 257);		\
-DO_IMMEDIATE_OPS (65280, TYPE, OP, NAME ## 65280);	\
-DO_IMMEDIATE_OPS (65281, TYPE, OP, NAME ## 65281);	\
-DO_IMMEDIATE_OPS (-1, TYPE, OP, NAME ## minus1);
+  DO_REGREG_OPS (TYPE, OP, NAME);			\
+  DO_IMMEDIATE_OPS (0, TYPE, OP, NAME ## 0);		\
+  DO_IMMEDIATE_OPS (5, TYPE, OP, NAME ## 5);		\
+  DO_IMMEDIATE_OPS (255, TYPE, OP, NAME ## 255);	\
+  DO_IMMEDIATE_OPS (256, TYPE, OP, NAME ## 256);	\
+  DO_IMMEDIATE_OPS (257, TYPE, OP, NAME ## 257);	\
+  DO_IMMEDIATE_OPS (65280, TYPE, OP, NAME ## 65280);	\
+  DO_IMMEDIATE_OPS (65281, TYPE, OP, NAME ## 65281);	\
+  DO_IMMEDIATE_OPS (-1, TYPE, OP, NAME ## minus1);
 
-DO_ARITH_OPS (char, +, add)
-DO_ARITH_OPS (short, +, add)
-DO_ARITH_OPS (int, +, add)
-DO_ARITH_OPS (long, +, add)
-DO_ARITH_OPS (char, -, minus)
-DO_ARITH_OPS (short, -, minus)
-DO_ARITH_OPS (int, -, minus)
-DO_ARITH_OPS (long, -, minus)
+DO_ARITH_OPS (int8_t, +, add)
+DO_ARITH_OPS (int16_t, +, add)
+DO_ARITH_OPS (int32_t, +, add)
+DO_ARITH_OPS (int64_t, +, add)
+DO_ARITH_OPS (int8_t, -, minus)
+DO_ARITH_OPS (int16_t, -, minus)
+DO_ARITH_OPS (int32_t, -, minus)
+DO_ARITH_OPS (int64_t, -, minus)
 
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
@@ -47,20 +48,21 @@ DO_ARITH_OPS (long, -, minus)
 
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #1\n} 4 } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #5\n} 1 } } */
-/* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #255\n} } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #251\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #255\n} 4 } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #256\n} } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #257\n} } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #65280\n} } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #65281\n} } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #-1\n} } } */
-/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #1\n} 4 } } */
+/* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #1\n} } } */
 
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #1\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #5\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #255\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #256\n} 2 } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #257\n} } } */
-/* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #65280\n} } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #65280\n} 2 } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #65281\n} } } */
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #-1\n} } } */
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #1\n} 1 } } */
@@ -83,8 +85,8 @@ DO_ARITH_OPS (long, -, minus)
 /* { dg-final { scan-assembler-not {\tadd\tz[0-9]+\.d, z[0-9]+\.d, #-1\n} } } */
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.d, z[0-9]+\.d, #1\n} 1 } } */
 
-/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #1\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #5\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #1\n} } } */
+/* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #5\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #255\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #256\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #257\n} } } */
@@ -94,12 +96,11 @@ DO_ARITH_OPS (long, -, minus)
 
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #5\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #255\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #256\n} 2 } } */
+/* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #256\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #257\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #65280\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #65281\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #-1\n} } } */
-/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #1\n} 1 } } */
 
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.s, z[0-9]+\.s, #5\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.s, z[0-9]+\.s, #255\n} 1 } } */
@@ -118,4 +119,3 @@ DO_ARITH_OPS (long, -, minus)
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.d, z[0-9]+\.d, #65281\n} } } */
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.d, z[0-9]+\.d, #-1\n} } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.d, z[0-9]+\.d, #1\n} 1 } } */
-
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1.c b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1.c
index d97a501512b..86d3930e476 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1.c
@@ -1,17 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
-void vcvtf_32 (float *dst, signed int *src1, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+vcvtf_16 (_Float16 *dst, int16_t *src1, int size)
+{
+  for (int i = 0; i < size; i++)
+    dst[i] = (_Float16) src1[i];
+}
+
+void __attribute__ ((noinline, noclone))
+vcvtf_32 (float *dst, int32_t *src1, int size)
 {
   for (int i = 0; i < size; i++)
     dst[i] = (float) src1[i];
 }
 
-void vcvtf_64 (double *dst, signed long *src1, int size)
+void __attribute__ ((noinline, noclone))
+vcvtf_64 (double *dst, int64_t *src1, int size)
 {
   for (int i = 0; i < size; i++)
     dst[i] = (double) src1[i];
 }
 
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1_run.c
index b0aa05c055e..9b431ad0ed4 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_signed_1_run.c
@@ -1,47 +1,47 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_cvtf_signed_1.c"
 
 #define ARRAY_SIZE 47
 
-#define VAL1 ((i * 3) - (15 * 3))
-#define VAL2 ((i * 0xffdfffef) - (11 * 0xffdfffef))
+#define VAL1 (i ^ 3)
+#define VAL2 ((i * 3) - (15 * 3))
+#define VAL3 ((i * 0xffdfffef) - (11 * 0xffdfffef))
 
 int __attribute__ ((optimize (1)))
 main (void)
 {
-  static float array_destf[ARRAY_SIZE];
-  static double array_destd[ARRAY_SIZE];
+  static _Float16 array_dest16[ARRAY_SIZE];
+  static float array_dest32[ARRAY_SIZE];
+  static double array_dest64[ARRAY_SIZE];
 
-  signed int array_source_i[ARRAY_SIZE];
-  signed long array_source_l[ARRAY_SIZE];
+  int16_t array_source16[ARRAY_SIZE];
+  int32_t array_source32[ARRAY_SIZE];
+  int64_t array_source64[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
     {
-      array_source_i[i] = VAL1;
-      array_source_l[i] = VAL2;
+      array_source16[i] = VAL1;
+      array_source32[i] = VAL2;
+      array_source64[i] = VAL3;
+      asm volatile ("" ::: "memory");
     }
 
-  vcvtf_32 (array_destf, array_source_i, ARRAY_SIZE);
+  vcvtf_16 (array_dest16, array_source16, ARRAY_SIZE);
+  for (int i = 0; i < ARRAY_SIZE; i++)
+    if (array_dest16[i] != (_Float16) VAL1)
+      __builtin_abort ();
+
+  vcvtf_32 (array_dest32, array_source32, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_destf[i] != (float) VAL1)
-      {
-	fprintf (stderr,"%d: %f != %f\n", i, array_destf[i], (float) VAL1);
-	exit (1);
-      }
+    if (array_dest32[i] != (float) VAL2)
+      __builtin_abort ();
 
-  vcvtf_64 (array_destd, array_source_l, ARRAY_SIZE);
+  vcvtf_64 (array_dest64, array_source64, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_destd[i] != (double) VAL2)
-      {
-	fprintf (stderr,"%d: %lf != %f\n", i, array_destd[i], (double) VAL2);
-	exit (1);
-      }
+    if (array_dest64[i] != (double) VAL3)
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1.c b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1.c
index bd8cf6f6cf5..0605307d1e3 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1.c
@@ -1,17 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
-void vcvtf_32 (float *dst, unsigned int *src1, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+vcvtf_16 (_Float16 *dst, uint16_t *src1, int size)
+{
+  for (int i = 0; i < size; i++)
+    dst[i] = (_Float16) src1[i];
+}
+
+void __attribute__ ((noinline, noclone))
+vcvtf_32 (float *dst, uint32_t *src1, int size)
 {
   for (int i = 0; i < size; i++)
     dst[i] = (float) src1[i];
 }
 
-void vcvtf_64 (double *dst, unsigned long *src1, int size)
+void __attribute__ ((noinline, noclone))
+vcvtf_64 (double *dst, uint64_t *src1, int size)
 {
   for (int i = 0; i < size; i++)
     dst[i] = (double) src1[i];
 }
 
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1_run.c
index 5b9291ca2c2..a4434cbf478 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_cvtf_unsigned_1_run.c
@@ -1,47 +1,47 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_cvtf_unsigned_1.c"
 
 #define ARRAY_SIZE 65
 
-#define VAL1 (i * 9456)
-#define VAL2 (i * 0xfddff13f)
+#define VAL1 (i * 109)
+#define VAL2 (i * 9456)
+#define VAL3 (i * 0xfddff13f)
 
 int __attribute__ ((optimize (1)))
 main (void)
 {
-  static float array_destf[ARRAY_SIZE];
-  static double array_destd[ARRAY_SIZE];
+  static _Float16 array_dest16[ARRAY_SIZE];
+  static float array_dest32[ARRAY_SIZE];
+  static double array_dest64[ARRAY_SIZE];
 
-  unsigned int array_source_i[ARRAY_SIZE];
-  unsigned long array_source_l[ARRAY_SIZE];
+  uint16_t array_source16[ARRAY_SIZE];
+  uint32_t array_source32[ARRAY_SIZE];
+  uint64_t array_source64[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
     {
-      array_source_i[i] = VAL1;
-      array_source_l[i] = VAL2;
+      array_source16[i] = VAL1;
+      array_source32[i] = VAL2;
+      array_source64[i] = VAL3;
+      asm volatile ("" ::: "memory");
     }
 
-  vcvtf_32 (array_destf, array_source_i, ARRAY_SIZE);
+  vcvtf_16 (array_dest16, array_source16, ARRAY_SIZE);
+  for (int i = 0; i < ARRAY_SIZE; i++)
+    if (array_dest16[i] != (_Float16) VAL1)
+      __builtin_abort ();
+
+  vcvtf_32 (array_dest32, array_source32, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_destf[i] != (float) VAL1)
-      {
-	fprintf (stderr,"%d: %f != %f\n", i, array_destf[i], (float) VAL1);
-	exit (1);
-      }
+    if (array_dest32[i] != (float) VAL2)
+      __builtin_abort ();
 
-  vcvtf_64 (array_destd, array_source_l, ARRAY_SIZE);
+  vcvtf_64 (array_dest64, array_source64, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_destd[i] != (double) VAL2)
-      {
-	fprintf (stderr,"%d: %lf != %f\n", i, array_destd[i], (double) VAL2);
-	exit (1);
-      }
+    if (array_dest64[i] != (double) VAL3)
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1.C b/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1.c
index 1f7d8a4a9ba..9fed379607b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1.c
@@ -1,12 +1,14 @@
 /* { dg-do compile } */
-/* { dg-options "-std=c++11 -O3 -fno-inline -march=armv8-a+sve -fno-tree-loop-distribute-patterns" } */
+/* -fno-tree-loop-distribute-patterns prevents conversion to memset.  */
+/* { dg-options "-O3 -march=armv8-a+sve -fno-tree-loop-distribute-patterns" } */
 
 #include <stdint.h>
 
 #define NUM_ELEMS(TYPE) (1024 / sizeof (TYPE))
 
-#define DEF_SET_IMM(TYPE,IMM,SUFFIX)		\
-void set_##TYPE##SUFFIX (TYPE *__restrict__ a)	\
+#define DEF_SET_IMM(TYPE, IMM, SUFFIX)		\
+void __attribute__ ((noinline, noclone))	\
+set_##TYPE##_##SUFFIX (TYPE *a)			\
 {						\
   for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
     a[i] = IMM;					\
@@ -93,7 +95,7 @@ DEF_SET_IMM (int64_t, 0xFE00FE00FE00FE00LL, imm_FE00_pat)
 // shouldn't assert!
 DEF_SET_IMM (int32_t, 129, imm_m129)
 DEF_SET_IMM (int32_t, 32513, imm_32513)
-DEF_SET_IMM (int32_t, -32767, imm_m32767)
+DEF_SET_IMM (int32_t, -32763, imm_m32763)
 
 /* { dg-final { scan-assembler {\tmov\tz[0-9]+\.b, #-1\n} } } */
 
@@ -130,3 +132,7 @@ DEF_SET_IMM (int32_t, -32767, imm_m32767)
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.h, #-2\n} 2 } } */
 
 /* { dg-final { scan-assembler {\tmov\tz[0-9]+\.h, #-512\n} } } */
+
+/* { dg-final { scan-assembler-not {#129\n} } } */
+/* { dg-final { scan-assembler-not {#32513\n} } } */
+/* { dg-final { scan-assembler-not {#-32763\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1_run.C b/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1_run.c
index cbc16e8e2bb..237f44947ab 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1_run.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_dup_imm_1_run.c
@@ -1,23 +1,20 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-std=c++11 -O3 -fno-inline -march=armv8-a+sve -fno-tree-loop-distribute-patterns" } */
+/* { dg-options "-O3 -march=armv8-a+sve -fno-tree-loop-distribute-patterns" } */
 
-#include "sve_dup_imm_1.C"
+#include "sve_dup_imm_1.c"
 
-#include <stdlib.h>
-
-#define TEST_SET_IMM(TYPE,IMM,SUFFIX)		\
+#define TEST_SET_IMM(TYPE, IMM, SUFFIX)		\
   {						\
     TYPE v[NUM_ELEMS (TYPE)];			\
-    set_##TYPE##SUFFIX (v);			\
-    for (int i = 0; i < NUM_ELEMS (TYPE); i++ )	\
-      if (v[i] != IMM)				\
-        result++;				\
+    set_##TYPE##_##SUFFIX (v);			\
+    for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
+      if (v[i] != (TYPE) IMM)			\
+        __builtin_abort ();			\
   }
 
-int main (int argc, char **argv)
+int __attribute__ ((optimize (1)))
+main (int argc, char **argv)
 {
-  int result = 0;
-
   TEST_SET_IMM (int8_t,  0, imm_0)
   TEST_SET_IMM (int16_t, 0, imm_0)
   TEST_SET_IMM (int32_t, 0, imm_0)
@@ -62,15 +59,12 @@ int main (int argc, char **argv)
   TEST_SET_IMM (int32_t, 0x00010001, imm_0001_pat)
   TEST_SET_IMM (int64_t, 0x0001000100010001LL, imm_0001_pat)
 
-  TEST_SET_IMM (int16_t, int16_t (0xFEFE), imm_FE_pat)
-  TEST_SET_IMM (int32_t, int32_t (0xFEFEFEFE), imm_FE_pat)
-  TEST_SET_IMM (int64_t, int64_t (0xFEFEFEFEFEFEFEFE), imm_FE_pat)
-
-  TEST_SET_IMM (int32_t, int32_t (0xFE00FE00), imm_FE00_pat)
-  TEST_SET_IMM (int64_t, int64_t (0xFE00FE00FE00FE00), imm_FE00_pat)
+  TEST_SET_IMM (int16_t, 0xFEFE, imm_FE_pat)
+  TEST_SET_IMM (int32_t, 0xFEFEFEFE, imm_FE_pat)
+  TEST_SET_IMM (int64_t, 0xFEFEFEFEFEFEFEFE, imm_FE_pat)
 
-  if (result != 0)
-    abort ();
+  TEST_SET_IMM (int32_t, 0xFE00FE00, imm_FE00_pat)
+  TEST_SET_IMM (int64_t, 0xFE00FE00FE00FE00, imm_FE00_pat)
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_dup_lane_1.c b/gcc/testsuite/gcc.target/aarch64/sve_dup_lane_1.c
index d4de247b05e..ea977207226 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_dup_lane_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_dup_lane_1.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define MASK_2(X) X, X
 #define MASK_4(X) MASK_2 (X), MASK_2 (X)
@@ -44,7 +47,10 @@ typedef float v8sf __attribute__((vector_size (32)));
   T (v4df, 4, 3)				\
   T (v8sf, 8, 0)				\
   T (v8sf, 8, 5)				\
-  T (v8sf, 8, 7)
+  T (v8sf, 8, 7)				\
+  T (v16hf, 16, 0)				\
+  T (v16hf, 16, 6)				\
+  T (v16hf, 16, 15)				\
 
 TEST_ALL (DUP_LANE)
 
@@ -56,9 +62,9 @@ TEST_ALL (DUP_LANE)
 /* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[0\]} 2 } } */
 /* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[5\]} 2 } } */
 /* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[7\]} 2 } } */
-/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[0\]} 1 } } */
-/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[6\]} 1 } } */
-/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[15\]} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[0\]} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[6\]} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[15\]} 2 } } */
 /* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[0\]} 1 } } */
 /* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[19\]} 1 } } */
 /* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[31\]} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_ext_1.c b/gcc/testsuite/gcc.target/aarch64/sve_ext_1.c
index 3056c60eee7..1ec51aa2eaf 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_ext_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_ext_1.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define MASK_2(X) X, X + 1
 #define MASK_4(X) MASK_2 (X), MASK_2 (X + 2)
@@ -44,21 +47,24 @@ typedef float v8sf __attribute__((vector_size (32)));
   T (v4df, 4, 3)				\
   T (v8sf, 8, 1)				\
   T (v8sf, 8, 5)				\
-  T (v8sf, 8, 7)
+  T (v8sf, 8, 7)				\
+  T (v16hf, 16, 1)				\
+  T (v16hf, 16, 6)				\
+  T (v16hf, 16, 15)				\
 
 TEST_ALL (DUP_LANE)
 
 /* { dg-final { scan-assembler-not {\ttbl\t} } } */
 
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #1\n} 1 } } */
-/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #2\n} 1 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #2\n} 2 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #4\n} 2 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #8\n} 2 } } */
-/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #12\n} 1 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #12\n} 2 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #16\n} 2 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #19\n} 1 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #20\n} 2 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #24\n} 2 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #28\n} 2 } } */
-/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #30\n} 1 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #30\n} 2 } } */
 /* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #31\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_extract_1.c b/gcc/testsuite/gcc.target/aarch64/sve_extract_1.c
new file mode 100644
index 00000000000..1ba277ffa6d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_extract_1.c
@@ -0,0 +1,93 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
+
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
+typedef double v4df __attribute__((vector_size (32)));
+typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
+
+#define EXTRACT(ELT_TYPE, TYPE, INDEX)		\
+  ELT_TYPE permute_##TYPE##_##INDEX (void)	\
+  {						\
+    TYPE values;				\
+    asm ("" : "=w" (values));			\
+    return values[INDEX];			\
+  }
+
+#define TEST_ALL(T)				\
+  T (int64_t, v4di, 0)				\
+  T (int64_t, v4di, 1)				\
+  T (int64_t, v4di, 2)				\
+  T (int64_t, v4di, 3)				\
+  T (int32_t, v8si, 0)				\
+  T (int32_t, v8si, 1)				\
+  T (int32_t, v8si, 3)				\
+  T (int32_t, v8si, 4)				\
+  T (int32_t, v8si, 7)				\
+  T (int16_t, v16hi, 0)				\
+  T (int16_t, v16hi, 1)				\
+  T (int16_t, v16hi, 7)				\
+  T (int16_t, v16hi, 8)				\
+  T (int16_t, v16hi, 15)			\
+  T (int8_t, v32qi, 0)				\
+  T (int8_t, v32qi, 1)				\
+  T (int8_t, v32qi, 15)				\
+  T (int8_t, v32qi, 16)				\
+  T (int8_t, v32qi, 31)				\
+  T (double, v4df, 0)				\
+  T (double, v4df, 1)				\
+  T (double, v4df, 2)				\
+  T (double, v4df, 3)				\
+  T (float, v8sf, 0)				\
+  T (float, v8sf, 1)				\
+  T (float, v8sf, 3)				\
+  T (float, v8sf, 4)				\
+  T (float, v8sf, 7)				\
+  T (_Float16, v16hf, 0)			\
+  T (_Float16, v16hf, 1)			\
+  T (_Float16, v16hf, 7)			\
+  T (_Float16, v16hf, 8)			\
+  T (_Float16, v16hf, 15)
+
+TEST_ALL (EXTRACT)
+
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\td[0-9]+, v[0-9]+\.d\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\td[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.d, z[0-9]+\.d\[2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tx[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\ts[0-9]+, v[0-9]+\.s\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[4\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[0\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\th[0-9]+, v[0-9]+\.h\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[8\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[0\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[15\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[16\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.b\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_extract_2.c b/gcc/testsuite/gcc.target/aarch64/sve_extract_2.c
new file mode 100644
index 00000000000..b163f28ef28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_extract_2.c
@@ -0,0 +1,93 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=512 --save-temps" } */
+
+#include <stdint.h>
+
+typedef int64_t v8di __attribute__((vector_size (64)));
+typedef int32_t v16si __attribute__((vector_size (64)));
+typedef int16_t v32hi __attribute__((vector_size (64)));
+typedef int8_t v64qi __attribute__((vector_size (64)));
+typedef double v8df __attribute__((vector_size (64)));
+typedef float v16sf __attribute__((vector_size (64)));
+typedef _Float16 v32hf __attribute__((vector_size (64)));
+
+#define EXTRACT(ELT_TYPE, TYPE, INDEX)		\
+  ELT_TYPE permute_##TYPE##_##INDEX (void)	\
+  {						\
+    TYPE values;				\
+    asm ("" : "=w" (values));			\
+    return values[INDEX];			\
+  }
+
+#define TEST_ALL(T)				\
+  T (int64_t, v8di, 0)				\
+  T (int64_t, v8di, 1)				\
+  T (int64_t, v8di, 2)				\
+  T (int64_t, v8di, 7)				\
+  T (int32_t, v16si, 0)				\
+  T (int32_t, v16si, 1)				\
+  T (int32_t, v16si, 3)				\
+  T (int32_t, v16si, 4)				\
+  T (int32_t, v16si, 15)			\
+  T (int16_t, v32hi, 0)				\
+  T (int16_t, v32hi, 1)				\
+  T (int16_t, v32hi, 7)				\
+  T (int16_t, v32hi, 8)				\
+  T (int16_t, v32hi, 31)			\
+  T (int8_t, v64qi, 0)				\
+  T (int8_t, v64qi, 1)				\
+  T (int8_t, v64qi, 15)				\
+  T (int8_t, v64qi, 16)				\
+  T (int8_t, v64qi, 63)				\
+  T (double, v8df, 0)				\
+  T (double, v8df, 1)				\
+  T (double, v8df, 2)				\
+  T (double, v8df, 7)				\
+  T (float, v16sf, 0)				\
+  T (float, v16sf, 1)				\
+  T (float, v16sf, 3)				\
+  T (float, v16sf, 4)				\
+  T (float, v16sf, 15)				\
+  T (_Float16, v32hf, 0)			\
+  T (_Float16, v32hf, 1)			\
+  T (_Float16, v32hf, 7)			\
+  T (_Float16, v32hf, 8)			\
+  T (_Float16, v32hf, 31)
+
+TEST_ALL (EXTRACT)
+
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\td[0-9]+, v[0-9]+\.d\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\td[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.d, z[0-9]+\.d\[2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tx[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\ts[0-9]+, v[0-9]+\.s\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[4\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[0\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\th[0-9]+, v[0-9]+\.h\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[8\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[0\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[15\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[16\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.b\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_extract_3.c b/gcc/testsuite/gcc.target/aarch64/sve_extract_3.c
new file mode 100644
index 00000000000..87ac2351768
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_extract_3.c
@@ -0,0 +1,124 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=1024 --save-temps" } */
+
+#include <stdint.h>
+
+typedef int64_t v16di __attribute__((vector_size (128)));
+typedef int32_t v32si __attribute__((vector_size (128)));
+typedef int16_t v64hi __attribute__((vector_size (128)));
+typedef int8_t v128qi __attribute__((vector_size (128)));
+typedef double v16df __attribute__((vector_size (128)));
+typedef float v32sf __attribute__((vector_size (128)));
+typedef _Float16 v64hf __attribute__((vector_size (128)));
+
+#define EXTRACT(ELT_TYPE, TYPE, INDEX)		\
+  ELT_TYPE permute_##TYPE##_##INDEX (void)	\
+  {						\
+    TYPE values;				\
+    asm ("" : "=w" (values));			\
+    return values[INDEX];			\
+  }
+
+#define TEST_ALL(T)				\
+  T (int64_t, v16di, 0)				\
+  T (int64_t, v16di, 1)				\
+  T (int64_t, v16di, 2)				\
+  T (int64_t, v16di, 7)				\
+  T (int64_t, v16di, 8)				\
+  T (int64_t, v16di, 9)				\
+  T (int64_t, v16di, 15)			\
+  T (int32_t, v32si, 0)				\
+  T (int32_t, v32si, 1)				\
+  T (int32_t, v32si, 3)				\
+  T (int32_t, v32si, 4)				\
+  T (int32_t, v32si, 15)			\
+  T (int32_t, v32si, 16)			\
+  T (int32_t, v32si, 21)			\
+  T (int32_t, v32si, 31)			\
+  T (int16_t, v64hi, 0)				\
+  T (int16_t, v64hi, 1)				\
+  T (int16_t, v64hi, 7)				\
+  T (int16_t, v64hi, 8)				\
+  T (int16_t, v64hi, 31)			\
+  T (int16_t, v64hi, 32)			\
+  T (int16_t, v64hi, 47)			\
+  T (int16_t, v64hi, 63)			\
+  T (int8_t, v128qi, 0)				\
+  T (int8_t, v128qi, 1)				\
+  T (int8_t, v128qi, 15)			\
+  T (int8_t, v128qi, 16)			\
+  T (int8_t, v128qi, 63)			\
+  T (int8_t, v128qi, 64)			\
+  T (int8_t, v128qi, 100)			\
+  T (int8_t, v128qi, 127)			\
+  T (double, v16df, 0)				\
+  T (double, v16df, 1)				\
+  T (double, v16df, 2)				\
+  T (double, v16df, 7)				\
+  T (double, v16df, 8)				\
+  T (double, v16df, 9)				\
+  T (double, v16df, 15)				\
+  T (float, v32sf, 0)				\
+  T (float, v32sf, 1)				\
+  T (float, v32sf, 3)				\
+  T (float, v32sf, 4)				\
+  T (float, v32sf, 15)				\
+  T (float, v32sf, 16)				\
+  T (float, v32sf, 21)				\
+  T (float, v32sf, 31)				\
+  T (_Float16, v64hf, 0)			\
+  T (_Float16, v64hf, 1)			\
+  T (_Float16, v64hf, 7)			\
+  T (_Float16, v64hf, 8)			\
+  T (_Float16, v64hf, 31)			\
+  T (_Float16, v64hf, 32)			\
+  T (_Float16, v64hf, 47)			\
+  T (_Float16, v64hf, 63)
+
+TEST_ALL (EXTRACT)
+
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\td[0-9]+, v[0-9]+\.d\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\td[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.d, z[0-9]+\.d\[2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.d, z[0-9]+\.d\[7\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tx[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\ts[0-9]+, v[0-9]+\.s\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[4\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[15\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[0\]\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\th[0-9]+, v[0-9]+\.h\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[8\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[31\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[0\]\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[15\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[16\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[63\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.b\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #64\n} 7 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #72\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #84\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #94\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #100\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_extract_4.c b/gcc/testsuite/gcc.target/aarch64/sve_extract_4.c
new file mode 100644
index 00000000000..e61a2fa94e6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_extract_4.c
@@ -0,0 +1,135 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=2048 --save-temps" } */
+
+#include <stdint.h>
+
+typedef int64_t v32di __attribute__((vector_size (256)));
+typedef int32_t v64si __attribute__((vector_size (256)));
+typedef int16_t v128hi __attribute__((vector_size (256)));
+typedef int8_t v256qi __attribute__((vector_size (256)));
+typedef double v32df __attribute__((vector_size (256)));
+typedef float v64sf __attribute__((vector_size (256)));
+typedef _Float16 v128hf __attribute__((vector_size (256)));
+
+#define EXTRACT(ELT_TYPE, TYPE, INDEX)		\
+  ELT_TYPE permute_##TYPE##_##INDEX (void)	\
+  {						\
+    TYPE values;				\
+    asm ("" : "=w" (values));			\
+    return values[INDEX];			\
+  }
+
+#define TEST_ALL(T)				\
+  T (int64_t, v32di, 0)				\
+  T (int64_t, v32di, 1)				\
+  T (int64_t, v32di, 2)				\
+  T (int64_t, v32di, 7)				\
+  T (int64_t, v32di, 8)				\
+  T (int64_t, v32di, 9)				\
+  T (int64_t, v32di, 15)			\
+  T (int64_t, v32di, 31)			\
+  T (int32_t, v64si, 0)				\
+  T (int32_t, v64si, 1)				\
+  T (int32_t, v64si, 3)				\
+  T (int32_t, v64si, 4)				\
+  T (int32_t, v64si, 15)			\
+  T (int32_t, v64si, 16)			\
+  T (int32_t, v64si, 21)			\
+  T (int32_t, v64si, 31)			\
+  T (int32_t, v64si, 63)			\
+  T (int16_t, v128hi, 0)			\
+  T (int16_t, v128hi, 1)			\
+  T (int16_t, v128hi, 7)			\
+  T (int16_t, v128hi, 8)			\
+  T (int16_t, v128hi, 31)			\
+  T (int16_t, v128hi, 32)			\
+  T (int16_t, v128hi, 47)			\
+  T (int16_t, v128hi, 63)			\
+  T (int16_t, v128hi, 127)			\
+  T (int8_t, v256qi, 0)				\
+  T (int8_t, v256qi, 1)				\
+  T (int8_t, v256qi, 15)			\
+  T (int8_t, v256qi, 16)			\
+  T (int8_t, v256qi, 63)			\
+  T (int8_t, v256qi, 64)			\
+  T (int8_t, v256qi, 100)			\
+  T (int8_t, v256qi, 127)			\
+  T (int8_t, v256qi, 255)			\
+  T (double, v32df, 0)				\
+  T (double, v32df, 1)				\
+  T (double, v32df, 2)				\
+  T (double, v32df, 7)				\
+  T (double, v32df, 8)				\
+  T (double, v32df, 9)				\
+  T (double, v32df, 15)				\
+  T (double, v32df, 31)				\
+  T (float, v64sf, 0)				\
+  T (float, v64sf, 1)				\
+  T (float, v64sf, 3)				\
+  T (float, v64sf, 4)				\
+  T (float, v64sf, 15)				\
+  T (float, v64sf, 16)				\
+  T (float, v64sf, 21)				\
+  T (float, v64sf, 31)				\
+  T (float, v64sf, 63)				\
+  T (_Float16, v128hf, 0)			\
+  T (_Float16, v128hf, 1)			\
+  T (_Float16, v128hf, 7)			\
+  T (_Float16, v128hf, 8)			\
+  T (_Float16, v128hf, 31)			\
+  T (_Float16, v128hf, 32)			\
+  T (_Float16, v128hf, 47)			\
+  T (_Float16, v128hf, 63)			\
+  T (_Float16, v128hf, 127)
+
+TEST_ALL (EXTRACT)
+
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tx[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\td[0-9]+, v[0-9]+\.d\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\td[0-9]+, v[0-9]+\.d\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.d, z[0-9]+\.d\[2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.d, z[0-9]+\.d\[7\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tx[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[0\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\ts[0-9]+, v[0-9]+\.s\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\ts[0-9]+, v[0-9]+\.s\[3\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[4\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.s, z[0-9]+\.s\[15\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[0\]\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tdup\th[0-9]+, v[0-9]+\.h\[0\]\n} } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\th[0-9]+, v[0-9]+\.h\[7\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[8\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.h, z[0-9]+\.h\[31\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
+
+/* Also used to move the result of a non-Advanced SIMD extract.  */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[0\]\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[1\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumov\tw[0-9]+, v[0-9]+\.b\[15\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[16\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tdup\tz[0-9]+\.b, z[0-9]+\.b\[63\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlastb\tw[0-9]+, p[0-7], z[0-9]+\.b\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #64\n} 7 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #72\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #84\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #94\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #100\n} 1 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #120\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #124\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #126\n} 2 } } */
+/* { dg-final { scan-assembler-times {\text\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b, #127\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fabs_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fabs_1.c
index 61ec667363a..33e1db5d1df 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fabs_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fabs_1.c
@@ -9,9 +9,10 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
+DO_OPS (_Float16, fabsf)
 DO_OPS (float, fabsf)
 DO_OPS (double, fabs)
 
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1.c
index 8fd41db0a1f..7c5f6ddc996 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1.c
@@ -1,17 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
-void vfcvtz_32 (signed int *dst, float *src1, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+vfcvtz_16 (int16_t *dst, _Float16 *src1, int size)
+{
+  for (int i = 0; i < size; i++)
+    dst[i] = (int16_t) src1[i];
+}
+
+void __attribute__ ((noinline, noclone))
+vfcvtz_32 (int32_t *dst, float *src1, int size)
 {
   for (int i = 0; i < size; i++)
-    dst[i] = (signed int) src1[i];
+    dst[i] = (int32_t) src1[i];
 }
 
-void vfcvtz_64 (signed long *dst, double *src1, int size)
+void __attribute__ ((noinline, noclone))
+vfcvtz_64 (int64_t *dst, double *src1, int size)
 {
   for (int i = 0; i < size; i++)
-    dst[i] = (signed long) src1[i];
+    dst[i] = (int64_t) src1[i];
 }
 
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1_run.c
index 58ae7737a89..48968f8ce19 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_signed_1_run.c
@@ -1,47 +1,47 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O3 -march=armv8-a+sve" } */
 
 #include "sve_fcvtz_signed_1.c"
 
 #define ARRAY_SIZE 81
 
-#define VAL1 ((i * 237.86) - (29 * 237.86))
-#define VAL2 ((double) ((i * 0xf8dfef2f) - (11 * 0xf8dfef2f)))
+#define VAL1 ((i * 17) - 180)
+#define VAL2 ((i * 237.86) - (29 * 237.86))
+#define VAL3 ((double) ((i * 0xf8dfef2f) - (11 * 0xf8dfef2f)))
 
 int __attribute__ ((optimize (1)))
 main (void)
 {
-  static signed int array_desti[ARRAY_SIZE];
-  static signed long array_destl[ARRAY_SIZE];
+  static int16_t array_dest16[ARRAY_SIZE];
+  static int32_t array_dest32[ARRAY_SIZE];
+  static int64_t array_dest64[ARRAY_SIZE];
 
-  float array_source_f[ARRAY_SIZE];
-  double array_source_d[ARRAY_SIZE];
+  _Float16 array_source16[ARRAY_SIZE];
+  float array_source32[ARRAY_SIZE];
+  double array_source64[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
     {
-      array_source_f[i] = VAL1;
-      array_source_d[i] = VAL2;
+      array_source16[i] = VAL1;
+      array_source32[i] = VAL2;
+      array_source64[i] = VAL3;
+      asm volatile ("" ::: "memory");
     }
 
-  vfcvtz_32 (array_desti, array_source_f, ARRAY_SIZE);
+  vfcvtz_16 (array_dest16, array_source16, ARRAY_SIZE);
+  for (int i = 0; i < ARRAY_SIZE; i++)
+    if (array_dest16[i] != (int16_t) VAL1)
+      __builtin_abort ();
+
+  vfcvtz_32 (array_dest32, array_source32, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_desti[i] != (int) VAL1)
-      {
-	fprintf (stderr,"%d: %d != %d\n", i, array_desti[i], (int) VAL1);
-	exit (1);
-      }
+    if (array_dest32[i] != (int32_t) VAL2)
+      __builtin_abort ();
 
-  vfcvtz_64 (array_destl, array_source_d, ARRAY_SIZE);
+  vfcvtz_64 (array_dest64, array_source64, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_destl[i] != (long) VAL2)
-      {
-	fprintf (stderr,"%d: %ld != %ld\n", i, array_destl[i], (long) VAL2);
-	exit (1);
-      }
+    if (array_dest64[i] != (int64_t) VAL3)
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1.c
index b4dcd26cfd0..2691cf0bc17 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1.c
@@ -1,17 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
-void vfcvtz_32 (unsigned int *dst, float *src1, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+vfcvtz_16 (uint16_t *dst, _Float16 *src1, int size)
+{
+  for (int i = 0; i < size; i++)
+    dst[i] = (uint16_t) src1[i];
+}
+
+void __attribute__ ((noinline, noclone))
+vfcvtz_32 (uint32_t *dst, float *src1, int size)
 {
   for (int i = 0; i < size; i++)
-    dst[i] = (unsigned int) src1[i];
+    dst[i] = (uint32_t) src1[i];
 }
 
-void vfcvtz_64 (unsigned long *dst, double *src1, int size)
+void __attribute__ ((noinline, noclone))
+vfcvtz_64 (uint64_t *dst, double *src1, int size)
 {
   for (int i = 0; i < size; i++)
-    dst[i] = (unsigned long) src1[i];
+    dst[i] = (uint64_t) src1[i];
 }
 
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1_run.c
index e196d174c66..9c1be7c8a6f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fcvtz_unsigned_1_run.c
@@ -1,47 +1,47 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_fcvtz_unsigned_1.c"
 
 #define ARRAY_SIZE 75
 
-#define VAL1 (i * 2574.33)
-#define VAL2 ((double) (i * 0xff23efef))
+#define VAL1 (i * 19)
+#define VAL2 (i * 2574.33)
+#define VAL3 ((double) (i * 0xff23efef))
 
 int __attribute__ ((optimize (1)))
 main (void)
 {
-  static unsigned int array_desti[ARRAY_SIZE];
-  static unsigned long array_destl[ARRAY_SIZE];
+  static uint16_t array_dest16[ARRAY_SIZE];
+  static uint32_t array_dest32[ARRAY_SIZE];
+  static uint64_t array_dest64[ARRAY_SIZE];
 
-  float array_source_f[ARRAY_SIZE];
-  double array_source_d[ARRAY_SIZE];
+  _Float16 array_source16[ARRAY_SIZE];
+  float array_source32[ARRAY_SIZE];
+  double array_source64[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
     {
-      array_source_f[i] = VAL1;
-      array_source_d[i] = VAL2;
+      array_source16[i] = VAL1;
+      array_source32[i] = VAL2;
+      array_source64[i] = VAL3;
+      asm volatile ("" ::: "memory");
     }
 
-  vfcvtz_32 (array_desti, array_source_f, ARRAY_SIZE);
+  vfcvtz_16 (array_dest16, array_source16, ARRAY_SIZE);
+  for (int i = 0; i < ARRAY_SIZE; i++)
+    if (array_dest16[i] != (uint16_t) VAL1)
+      __builtin_abort ();
+
+  vfcvtz_32 (array_dest32, array_source32, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_desti[i] != (int) VAL1)
-      {
-	fprintf (stderr,"%d: %d != %d\n", i, array_desti[i], (int) VAL1);
-	exit (1);
-      }
+    if (array_dest32[i] != (uint32_t) VAL2)
+      __builtin_abort ();
 
-  vfcvtz_64 (array_destl, array_source_d, ARRAY_SIZE);
+  vfcvtz_64 (array_dest64, array_source64, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_destl[i] != (long) VAL2)
-      {
-	fprintf (stderr,"%d: %ld != %ld\n", i, array_destl[i], (long) VAL2);
-	exit (1);
-      }
+    if (array_dest64[i] != (uint64_t) VAL3)
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fdiv_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fdiv_1.c
index d0becaf25f1..b193726ea0a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fdiv_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fdiv_1.c
@@ -1,30 +1,41 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)                        \
-void vdiv##TYPE (TYPE *dst, TYPE src1)     \
-{                                          \
-  *dst = *dst / src1;                      \
-}                                          \
-void vdivr##TYPE (TYPE *_dst, TYPE _src1)  \
-{                                          \
-  register TYPE dst  asm("z0");            \
-  register TYPE src1 asm("z2");            \
-  register TYPE src2 asm("z4");            \
-  dst = *_dst;                             \
-  asm volatile ("" :: "w" (dst));          \
-  src1 = _src1;                            \
-  asm volatile ("" :: "w" (src1));         \
-  dst = src1 / dst;                        \
-  *_dst = dst;                             \
+#define DO_OP(TYPE)				\
+void vdiv_##TYPE (TYPE *x, TYPE y)		\
+{						\
+  register TYPE dst asm("z0");			\
+  register TYPE src asm("z2");			\
+  dst = *x;					\
+  src = y;					\
+  asm volatile ("" :: "w" (dst), "w" (src));	\
+  dst = dst / src;				\
+  asm volatile ("" :: "w" (dst));		\
+  *x = dst;					\
+}						\
+void vdivr_##TYPE (TYPE *x, TYPE y)		\
+{						\
+  register TYPE dst asm("z0");			\
+  register TYPE src asm("z2");			\
+  dst = *x;					\
+  src = y;					\
+  asm volatile ("" :: "w" (dst), "w" (src));	\
+  dst = src / dst;				\
+  asm volatile ("" :: "w" (dst));		\
+  *x = dst;					\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+
 /* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fdup_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fdup_1.c
index 9ed825b9d35..148e0f9bd89 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fdup_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fdup_1.c
@@ -1,22 +1,24 @@
-/* { dg-do compile } */
-/* { dg-options "-O3 -fno-inline -march=armv8-a+sve" } */
+/* { dg-do assemble } */
+/* -fno-tree-loop-distribute-patterns prevents conversion to memset.  */
+/* { dg-options "-O3 -march=armv8-a+sve -fno-tree-loop-distribute-patterns --save-temps" } */
 
 #include <stdint.h>
 
 #define NUM_ELEMS(TYPE) (1024 / sizeof (TYPE))
 
-#define DEF_SET_IMM(TYPE,IMM,SUFFIX)		\
-void set_##TYPE##SUFFIX (TYPE *restrict a)	\
+#define DEF_SET_IMM(TYPE, IMM, SUFFIX)		\
+void __attribute__ ((noinline, noclone))	\
+set_##TYPE##_##SUFFIX (TYPE *a)			\
 {						\
   for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
     a[i] = IMM;					\
 }
 
 #define DEF_SET_IMM_FP(IMM, SUFFIX) \
-DEF_SET_IMM (float, IMM, SUFFIX)    \
-DEF_SET_IMM (double, IMM, SUFFIX)
+  DEF_SET_IMM (float, IMM, SUFFIX)  \
+  DEF_SET_IMM (double, IMM, SUFFIX)
 
-//Valid
+/* Valid.  */
 DEF_SET_IMM_FP (1, imm1)
 DEF_SET_IMM_FP (0x1.1p0, imm1p0)
 DEF_SET_IMM_FP (0x1.fp0, immfp0)
@@ -25,8 +27,10 @@ DEF_SET_IMM_FP (0x1.1p-3, imm1pm3)
 DEF_SET_IMM_FP (0x1.fp4, immfp4)
 DEF_SET_IMM_FP (0x1.fp-3, immfpm3)
 
-//Invalid
+/* Should use MOV instead.  */
 DEF_SET_IMM_FP (0, imm0)
+
+/* Invalid.  */
 DEF_SET_IMM_FP (0x1.1fp0, imm1fp0)
 DEF_SET_IMM_FP (0x1.1p5, imm1p5)
 DEF_SET_IMM_FP (0x1.1p-4, imm1pm4)
@@ -43,6 +47,8 @@ DEF_SET_IMM_FP (0x1.1fp-4, imm1fpm4)
 /* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #3.1e\+1\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #2.421875e-1\n} 1 } } */
 
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.s, #0\n} 1 } } */
+
 /* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d,} 7 } } */
 
 /* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #1.0e\+0\n} 1 } } */
@@ -52,3 +58,5 @@ DEF_SET_IMM_FP (0x1.1fp-4, imm1fpm4)
 /* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #1.328125e-1\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #3.1e\+1\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #2.421875e-1\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #0\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fdup_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_fdup_1_run.c
index bcd5180bbc6..f4cb1a0bf71 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fdup_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fdup_1_run.c
@@ -1,28 +1,24 @@
 /* { dg-do run { target { aarch64_sve_hw } } } */
-/* { dg-options "-O3 -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O3 -march=armv8-a+sve -fno-tree-loop-distribute-patterns" } */
 
 #include "sve_fdup_1.c"
 
-#include <stdlib.h>
-
 #define TEST_SET_IMM(TYPE,IMM,SUFFIX)		\
   {						\
     TYPE v[NUM_ELEMS (TYPE)];			\
-    set_##TYPE##SUFFIX (v);			\
+    set_##TYPE##_##SUFFIX (v);			\
     for (int i = 0; i < NUM_ELEMS (TYPE); i++ )	\
       if (v[i] != IMM)				\
-        result++;				\
+	__builtin_abort ();			\
   }
 
 #define TEST_SET_IMM_FP(IMM, SUFFIX) \
-TEST_SET_IMM (float, IMM, SUFFIX)    \
-TEST_SET_IMM (double, IMM, SUFFIX)
-
+  TEST_SET_IMM (float, IMM, SUFFIX)  \
+  TEST_SET_IMM (double, IMM, SUFFIX)
 
-int main  (int argc, char **argv)
+int __attribute__ ((optimize (1)))
+main (int argc, char **argv)
 {
-  int result = 0;
-
   TEST_SET_IMM_FP (1, imm1)
   TEST_SET_IMM_FP (0x1.1p0, imm1p0)
   TEST_SET_IMM_FP (0x1.fp0, immfp0)
@@ -38,8 +34,5 @@ int main  (int argc, char **argv)
   TEST_SET_IMM_FP (0x1.1fp5, imm1fp5)
   TEST_SET_IMM_FP (0x1.1fp-4, imm1fpm4)
 
-  if (result != 0)
-    abort ();
-
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fmad_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fmad_1.c
index 75e39c4e3e4..2b1dbb087bc 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fmad_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fmad_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)                                   \
-void vmad##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
-  dst = (dst * src1) + src2;                          \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (dst * src1) + src2;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfmad\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmad\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fmla_1.c
index 657773eada1..d5e4df266bf 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fmla_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fmla_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)                                   \
-void vmla##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
-  dst = (src1 * src2) + dst;                          \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (src1 * src2) + dst;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfmla\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmla\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fmls_1.c
index 5aca3b145a9..c3f2c8a5823 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fmls_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fmls_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)                                   \
-void vmls##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
-  dst = (-src1 * src2) + dst;                         \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (-src1 * src2) + dst;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfmls\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmls\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fmsb_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fmsb_1.c
index 5f4143fc5da..30e1895c8d5 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fmsb_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fmsb_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)                                   \
-void vmsb##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
-  dst = (-dst * src1) + src2;                         \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (-dst * src1) + src2;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfmsb\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmsb\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fmul_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fmul_1.c
index f4fb574beac..3b648297963 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fmul_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fmul_1.c
@@ -1,31 +1,38 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
 #define DO_REGREG_OPS(TYPE, OP, NAME)				\
-void varith_##TYPE##_##NAME (TYPE* dst, TYPE* src, int count)	\
+void varith_##TYPE##_##NAME (TYPE *dst, TYPE *src, int count)	\
 {								\
   for (int i = 0; i < count; ++i)				\
     dst[i] = dst[i] OP src[i];					\
 }
 
 #define DO_IMMEDIATE_OPS(VALUE, TYPE, OP, NAME)		\
-void varithimm_##NAME##_##TYPE (TYPE* dst, int count)	\
+void varithimm_##NAME##_##TYPE (TYPE *dst, int count)	\
 {							\
   for (int i = 0; i < count; ++i)			\
-    dst[i] = dst[i] OP VALUE;				\
+    dst[i] = dst[i] OP (TYPE) VALUE;			\
 }
 
 #define DO_ARITH_OPS(TYPE, OP, NAME)				\
-DO_REGREG_OPS (TYPE, OP, NAME);					\
-DO_IMMEDIATE_OPS (0.5, TYPE, OP, NAME ## 0point5);		\
-DO_IMMEDIATE_OPS (2, TYPE, OP, NAME ## 2);			\
-DO_IMMEDIATE_OPS (5, TYPE, OP, NAME ## 5);			\
-DO_IMMEDIATE_OPS (-0.5, TYPE, OP, NAME ## minus0point5);	\
-DO_IMMEDIATE_OPS (-2, TYPE, OP, NAME ## minus2);
+  DO_REGREG_OPS (TYPE, OP, NAME);				\
+  DO_IMMEDIATE_OPS (0.5, TYPE, OP, NAME ## 0point5);		\
+  DO_IMMEDIATE_OPS (2, TYPE, OP, NAME ## 2);			\
+  DO_IMMEDIATE_OPS (5, TYPE, OP, NAME ## 5);			\
+  DO_IMMEDIATE_OPS (-0.5, TYPE, OP, NAME ## minus0point5);	\
+  DO_IMMEDIATE_OPS (-2, TYPE, OP, NAME ## minus2);
 
+DO_ARITH_OPS (_Float16, *, mul)
 DO_ARITH_OPS (float, *, mul)
 DO_ARITH_OPS (double, *, mul)
 
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-not   {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #2} } } */
+/* { dg-final { scan-assembler-not   {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #5} } } */
+/* { dg-final { scan-assembler-not   {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #-} } } */
+
 /* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
 /* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
 /* { dg-final { scan-assembler-not   {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #2} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fneg_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fneg_1.c
index ddb703bf875..7af81662fb9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fneg_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fneg_1.c
@@ -1,15 +1,17 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
 #define DO_OPS(TYPE)					\
-void vneg_##TYPE (TYPE* dst, TYPE* src, int count)	\
+void vneg_##TYPE (TYPE *dst, TYPE *src, int count)	\
 {							\
   for (int i = 0; i < count; ++i)			\
     dst[i] = -src[i];					\
 }
 
+DO_OPS (_Float16)
 DO_OPS (float)
 DO_OPS (double)
 
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fnmad_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fnmad_1.c
index 877261029db..84a95187314 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fnmad_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fnmad_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)					\
-void vfnmad##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)	\
-{							\
-  register TYPE dst  asm("z0");				\
-  register TYPE src1 asm("z2");				\
-  register TYPE src2 asm("z4");				\
-  dst = *_dst;						\
-  asm volatile ("" :: "w" (dst));			\
-  src1 = _src1;						\
-  asm volatile ("" :: "w" (src1));			\
-  src2 = _src2;						\
-  asm volatile ("" :: "w" (src2));			\
-  dst = (-src2) + (-dst * src1);			\
-  asm volatile ("" :: "w" (dst));			\
-  *_dst = dst;						\
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (-dst * src1) - src2;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfnmad\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfnmad\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fnmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fnmla_1.c
index 463c90550d6..dcc4811f1d8 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fnmla_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fnmla_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)					\
-void vfnmla##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)	\
-{							\
-  register TYPE dst  asm("z0");				\
-  register TYPE src1 asm("z2");				\
-  register TYPE src2 asm("z4");				\
-  dst = *_dst;						\
-  asm volatile ("" :: "w" (dst));			\
-  src1 = _src1;						\
-  asm volatile ("" :: "w" (src1));			\
-  src2 = _src2;						\
-  asm volatile ("" :: "w" (src2));			\
-  dst = (-dst) + (-src1 * src2);			\
-  asm volatile ("" :: "w" (dst));			\
-  *_dst = dst;						\
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (-src1 * src2) - dst;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfnmla\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfnmla\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fnmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fnmls_1.c
index 312600fea20..7a89399f4be 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fnmls_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fnmls_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)					\
-void vfnmls##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)	\
-{							\
-  register TYPE dst  asm("z0");				\
-  register TYPE src1 asm("z2");				\
-  register TYPE src2 asm("z4");				\
-  dst = *_dst;						\
-  asm volatile ("" :: "w" (dst));			\
-  src1 = _src1;						\
-  asm volatile ("" :: "w" (src1));			\
-  src2 = _src2;						\
-  asm volatile ("" :: "w" (src2));			\
-  dst = (-dst) + (src1 * src2);				\
-  asm volatile ("" :: "w" (dst));			\
-  *_dst = dst;						\
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (src1 * src2) - dst;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfnmls\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfnmls\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fnmsb_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fnmsb_1.c
index 71e36b0028f..6c95b0abc8e 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fnmsb_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fnmsb_1.c
@@ -1,28 +1,29 @@
 /* { dg-do assemble } */
 /* { dg-options " -O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
+typedef _Float16 v16hf __attribute__((vector_size(32)));
 typedef float v8sf __attribute__((vector_size(32)));
 typedef double v4df __attribute__((vector_size(32)));
 
-#define DO_OP(TYPE)					\
-void vfnmsb##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)	\
-{							\
-  register TYPE dst  asm("z0");				\
-  register TYPE src1 asm("z2");				\
-  register TYPE src2 asm("z4");				\
-  dst = *_dst;						\
-  asm volatile ("" :: "w" (dst));			\
-  src1 = _src1;						\
-  asm volatile ("" :: "w" (src1));			\
-  src2 = _src2;						\
-  asm volatile ("" :: "w" (src2));			\
-  dst = (-src2) + (dst * src1);				\
-  asm volatile ("" :: "w" (dst));			\
-  *_dst = dst;						\
+#define DO_OP(TYPE)						\
+void vmad##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (dst * src1) - src2;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
+DO_OP (v16hf)
 DO_OP (v8sf)
 DO_OP (v4df)
 
-/* { dg-final { scan-assembler-times {\tfnmsb\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfnmsb\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fp_arith_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fp_arith_1.c
index a4c09074d43..06fea806038 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fp_arith_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fp_arith_1.c
@@ -1,36 +1,51 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
 #define DO_REGREG_OPS(TYPE, OP, NAME)				\
-void varith_##TYPE##_##NAME (TYPE* dst, TYPE* src, int count)	\
+void varith_##TYPE##_##NAME (TYPE *dst, TYPE *src, int count)	\
 {								\
   for (int i = 0; i < count; ++i)				\
     dst[i] = dst[i] OP src[i];					\
 }
 
 #define DO_IMMEDIATE_OPS(VALUE, TYPE, OP, NAME)		\
-  void varithimm_##NAME##_##TYPE (TYPE* dst, int count)	\
+void varithimm_##NAME##_##TYPE (TYPE *dst, int count)	\
 {							\
   for (int i = 0; i < count; ++i)			\
-    dst[i] = dst[i] OP VALUE;				\
+    dst[i] = dst[i] OP (TYPE) VALUE;			\
 }
 
 #define DO_ARITH_OPS(TYPE, OP, NAME)				\
-DO_REGREG_OPS (TYPE, OP, NAME);					\
-DO_IMMEDIATE_OPS (1, TYPE, OP, NAME ## 1);			\
-DO_IMMEDIATE_OPS (0.5, TYPE, OP, NAME ## pointfive);		\
-DO_IMMEDIATE_OPS (2, TYPE, OP, NAME ## 2);			\
-DO_IMMEDIATE_OPS (2.5, TYPE, OP, NAME ## twopoint5);		\
-DO_IMMEDIATE_OPS (-0.5, TYPE, OP, NAME ## minuspointfive);	\
-DO_IMMEDIATE_OPS (-1, TYPE, OP, NAME ## minus1);
+  DO_REGREG_OPS (TYPE, OP, NAME);				\
+  DO_IMMEDIATE_OPS (1, TYPE, OP, NAME ## 1);			\
+  DO_IMMEDIATE_OPS (0.5, TYPE, OP, NAME ## pointfive);		\
+  DO_IMMEDIATE_OPS (2, TYPE, OP, NAME ## 2);			\
+  DO_IMMEDIATE_OPS (2.5, TYPE, OP, NAME ## twopoint5);		\
+  DO_IMMEDIATE_OPS (-0.5, TYPE, OP, NAME ## minuspointfive);	\
+  DO_IMMEDIATE_OPS (-1, TYPE, OP, NAME ## minus1);
 
+DO_ARITH_OPS (_Float16, +, add)
 DO_ARITH_OPS (float, +, add)
 DO_ARITH_OPS (double, +, add)
+
+DO_ARITH_OPS (_Float16, -, minus)
 DO_ARITH_OPS (float, -, minus)
 DO_ARITH_OPS (double, -, minus)
 
 /* No specific count because it's valid to use fadd or fsub for the
    out-of-range constants.  */
+/* { dg-final { scan-assembler {\tfadd\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-not   {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #2} } } */
+/* { dg-final { scan-assembler-not   {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #-} } } */
+
+/* { dg-final { scan-assembler {\tfsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-not   {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #2} } } */
+/* { dg-final { scan-assembler-not   {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #-} } } */
+
 /* { dg-final { scan-assembler {\tfadd\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} } } */
 /* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_frinta_1.c b/gcc/testsuite/gcc.target/aarch64/sve_frinta_1.c
index 37f26fc8203..bad2be4ed33 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_frinta_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_frinta_1.c
@@ -9,7 +9,6 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
 DO_OPS (float, roundf)
 DO_OPS (double, round)
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_frinti_1.c b/gcc/testsuite/gcc.target/aarch64/sve_frinti_1.c
index 9faf2a3f81b..4407fb56caa 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_frinti_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_frinti_1.c
@@ -9,7 +9,6 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
 DO_OPS (float, nearbyintf)
 DO_OPS (double, nearbyint)
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_frintm_1.c b/gcc/testsuite/gcc.target/aarch64/sve_frintm_1.c
index b59d21ff0c7..01bf65db343 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_frintm_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_frintm_1.c
@@ -9,7 +9,6 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
 DO_OPS (float, floorf)
 DO_OPS (double, floor)
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_frintp_1.c b/gcc/testsuite/gcc.target/aarch64/sve_frintp_1.c
index d9a55e3ade5..f8b2c08ac63 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_frintp_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_frintp_1.c
@@ -9,7 +9,6 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
 DO_OPS (float, ceilf)
 DO_OPS (double, ceil)
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_frintx_1.c b/gcc/testsuite/gcc.target/aarch64/sve_frintx_1.c
index 012d9cb9de5..a062295011a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_frintx_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_frintx_1.c
@@ -9,7 +9,6 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
 DO_OPS (float, rintf)
 DO_OPS (double, rint)
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_frintz_1.c b/gcc/testsuite/gcc.target/aarch64/sve_frintz_1.c
index 2ae8f0026a7..207814f5506 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_frintz_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_frintz_1.c
@@ -9,7 +9,6 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
 DO_OPS (float, truncf)
 DO_OPS (double, trunc)
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fsqrt_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fsqrt_1.c
index 224c6ccfe6f..55081c3bf4f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fsqrt_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fsqrt_1.c
@@ -9,7 +9,6 @@ vsqrt_##TYPE (TYPE *dst, TYPE *src, int count)	\
     dst[i] = __builtin_##OP (src[i]);		\
 }
 
-
 DO_OPS (float, sqrtf)
 DO_OPS (double, sqrt)
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_fsubr_1.c b/gcc/testsuite/gcc.target/aarch64/sve_fsubr_1.c
index e664bf38c29..b252ef059ce 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_fsubr_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_fsubr_1.c
@@ -1,23 +1,30 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
 #define DO_IMMEDIATE_OPS(VALUE, TYPE, NAME)			\
-void vsubrarithimm_##NAME##_##TYPE (TYPE* dst, int count)	\
+void vsubrarithimm_##NAME##_##TYPE (TYPE *dst, int count)	\
 {								\
   for (int i = 0; i < count; ++i)				\
-    dst[i] = VALUE - dst[i];					\
+    dst[i] = (TYPE) VALUE - dst[i];				\
 }
 
 #define DO_ARITH_OPS(TYPE)			\
-DO_IMMEDIATE_OPS (0, TYPE, 0);			\
-DO_IMMEDIATE_OPS (1, TYPE, 1);			\
-DO_IMMEDIATE_OPS (0.5, TYPE, 0point5);		\
-DO_IMMEDIATE_OPS (2, TYPE, 2);			\
-DO_IMMEDIATE_OPS (3.5, TYPE, 3point5);
+  DO_IMMEDIATE_OPS (0, TYPE, 0);		\
+  DO_IMMEDIATE_OPS (1, TYPE, 1);		\
+  DO_IMMEDIATE_OPS (0.5, TYPE, 0point5);	\
+  DO_IMMEDIATE_OPS (2, TYPE, 2);		\
+  DO_IMMEDIATE_OPS (3.5, TYPE, 3point5);
 
+DO_ARITH_OPS (_Float16)
 DO_ARITH_OPS (float)
 DO_ARITH_OPS (double)
 
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-not   {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #2} } } */
+/* { dg-final { scan-assembler-not   {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #3} } } */
+
 /* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_index_1.C b/gcc/testsuite/gcc.target/aarch64/sve_index_1.c
index b7ae2d19f1e..09e65cf0fc3 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_index_1.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_index_1.c
@@ -1,50 +1,57 @@
-/* { dg-do compile } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -fno-inline -march=armv8-a+sve -msve-vector-bits=256" } */
+/* { dg-do assemble } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
 #include <stdint.h>
 
 #define NUM_ELEMS(TYPE) (32 / sizeof (TYPE))
 
-#define DEF_LOOP(TYPE,BASE,STEP,SUFFIX)		\
-void loop_##TYPE##SUFFIX (TYPE *__restrict__ a)	\
+#define DEF_LOOP(TYPE, BASE, STEP, SUFFIX)	\
+void __attribute__ ((noinline, noclone))	\
+loop_##TYPE##_##SUFFIX (TYPE *a)		\
 {						\
-  for (TYPE i = 0; i < NUM_ELEMS (TYPE); ++i)	\
-    a[i] = TYPE (BASE) + TYPE (i * (STEP));	\
+  for (int i = 0; i < NUM_ELEMS (TYPE); ++i)	\
+    a[i] = (BASE) + i * (STEP);			\
 }
 
-#define DEF_LOOPS_ALL_UNSIGNED_TYPES(BASE,STEP,SUFFIX)	\
-DEF_LOOP (uint8_t,  BASE, STEP, SUFFIX)			\
-DEF_LOOP (uint16_t, BASE, STEP, SUFFIX)			\
-DEF_LOOP (uint32_t, BASE, STEP, SUFFIX)			\
-DEF_LOOP (uint64_t, BASE, STEP, SUFFIX)
+#define TEST_ALL_UNSIGNED_TYPES(T, BASE, STEP, SUFFIX)	\
+  T (uint8_t,  BASE, STEP, SUFFIX)			\
+  T (uint16_t, BASE, STEP, SUFFIX)			\
+  T (uint32_t, BASE, STEP, SUFFIX)			\
+  T (uint64_t, BASE, STEP, SUFFIX)
 
-#define DEF_LOOPS_ALL_SIGNED_TYPES(BASE,STEP,SUFFIX)	\
-DEF_LOOP (int8_t,  BASE, STEP, SUFFIX)			\
-DEF_LOOP (int16_t, BASE, STEP, SUFFIX)			\
-DEF_LOOP (int32_t, BASE, STEP, SUFFIX)			\
-DEF_LOOP (int64_t, BASE, STEP, SUFFIX)
+#define TEST_ALL_SIGNED_TYPES(T, BASE, STEP, SUFFIX)	\
+  T (int8_t,  BASE, STEP, SUFFIX)			\
+  T (int16_t, BASE, STEP, SUFFIX)			\
+  T (int32_t, BASE, STEP, SUFFIX)			\
+  T (int64_t, BASE, STEP, SUFFIX)
 
-/* Immediate Loops.  */
-DEF_LOOPS_ALL_UNSIGNED_TYPES (0, 1, b0s1)
-DEF_LOOPS_ALL_SIGNED_TYPES (0, 1, b0s1)
-DEF_LOOPS_ALL_UNSIGNED_TYPES (0, 15, b0s15)
-DEF_LOOPS_ALL_SIGNED_TYPES (0, 15, b0s15)
-DEF_LOOPS_ALL_SIGNED_TYPES (0, -1, b0sm1)
-DEF_LOOPS_ALL_SIGNED_TYPES (0, -16, b0sm16)
-DEF_LOOPS_ALL_SIGNED_TYPES (-16, 1, bm16s1)
-DEF_LOOPS_ALL_UNSIGNED_TYPES (15, 1, b15s1)
-DEF_LOOPS_ALL_SIGNED_TYPES (15, 1, b15s1)
+/* Immediate loops.  */
+#define TEST_IMMEDIATE(T)			\
+  TEST_ALL_UNSIGNED_TYPES (T, 0, 1, b0s1)	\
+  TEST_ALL_SIGNED_TYPES (T, 0, 1, b0s1)		\
+  TEST_ALL_UNSIGNED_TYPES (T, 0, 15, b0s15)	\
+  TEST_ALL_SIGNED_TYPES (T, 0, 15, b0s15)	\
+  TEST_ALL_SIGNED_TYPES (T, 0, -1, b0sm1)	\
+  TEST_ALL_SIGNED_TYPES (T, 0, -16, b0sm16)	\
+  TEST_ALL_SIGNED_TYPES (T, -16, 1, bm16s1)	\
+  TEST_ALL_UNSIGNED_TYPES (T, 15, 1, b15s1)	\
+  TEST_ALL_SIGNED_TYPES (T, 15, 1, b15s1)
 
-/* Non Immediate Loops.  */
-DEF_LOOPS_ALL_UNSIGNED_TYPES (0, 16, b0s16)
-DEF_LOOPS_ALL_SIGNED_TYPES (0, 16, b0s16)
-DEF_LOOPS_ALL_SIGNED_TYPES (0, -17, b0sm17)
-DEF_LOOPS_ALL_SIGNED_TYPES (-17, 1, bm17s1)
-DEF_LOOPS_ALL_UNSIGNED_TYPES (16, 1, b16s1)
-DEF_LOOPS_ALL_SIGNED_TYPES (16, 1, b16s1)
-DEF_LOOPS_ALL_UNSIGNED_TYPES (16, 16, b16s16)
-DEF_LOOPS_ALL_SIGNED_TYPES (16, 16, b16s16)
-DEF_LOOPS_ALL_SIGNED_TYPES (-17, -17, bm17sm17)
+/* Non-immediate loops.  */
+#define TEST_NONIMMEDIATE(T)			\
+  TEST_ALL_UNSIGNED_TYPES (T, 0, 16, b0s16)	\
+  TEST_ALL_SIGNED_TYPES (T, 0, 16, b0s16)	\
+  TEST_ALL_SIGNED_TYPES (T, 0, -17, b0sm17)	\
+  TEST_ALL_SIGNED_TYPES (T, -17, 1, bm17s1)	\
+  TEST_ALL_UNSIGNED_TYPES (T, 16, 1, b16s1)	\
+  TEST_ALL_SIGNED_TYPES (T, 16, 1, b16s1)	\
+  TEST_ALL_UNSIGNED_TYPES (T, 16, 16, b16s16)	\
+  TEST_ALL_SIGNED_TYPES (T, 16, 16, b16s16)	\
+  TEST_ALL_SIGNED_TYPES (T, -17, -17, bm17sm17)
+
+#define TEST_ALL(T) TEST_IMMEDIATE (T) TEST_NONIMMEDIATE (T)
+
+TEST_ALL (DEF_LOOP)
 
 /* { dg-final { scan-assembler-times {\tindex\tz[0-9]+\.b, #0, #1\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tindex\tz[0-9]+\.b, #0, #15\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_index_1_run.C b/gcc/testsuite/gcc.target/aarch64/sve_index_1_run.C
deleted file mode 100644
index 0698eaba6eb..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_index_1_run.C
+++ /dev/null
@@ -1,79 +0,0 @@
-/* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include "sve_index_1.C"
-
-#include <stdlib.h>
-#include <stdio.h>
-
-#define SUM_VECTOR(TYPE)			\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++ )	\
-  {						\
-    result += r_##TYPE[i];			\
-  }
-
-#define TEST_LOOPS_ALL_UNSIGNED_TYPES(SUFFIX)	\
-loop_uint8_t##SUFFIX (r_uint8_t);		\
-loop_uint16_t##SUFFIX (r_uint16_t);		\
-loop_uint32_t##SUFFIX (r_uint32_t);		\
-loop_uint64_t##SUFFIX (r_uint64_t);		\
-SUM_VECTOR (uint8_t);				\
-SUM_VECTOR (uint16_t);				\
-SUM_VECTOR (uint32_t);				\
-SUM_VECTOR (uint64_t);
-
-#define TEST_LOOPS_ALL_SIGNED_TYPES(SUFFIX)	\
-loop_int8_t##SUFFIX (r_int8_t);			\
-loop_int16_t##SUFFIX (r_int16_t);		\
-loop_int32_t##SUFFIX (r_int32_t);		\
-loop_int64_t##SUFFIX (r_int64_t);		\
-SUM_VECTOR (int8_t);				\
-SUM_VECTOR (int16_t);				\
-SUM_VECTOR (int32_t);				\
-SUM_VECTOR (int64_t);
-
-
-#define DEF_INIT_VECTOR(TYPE)			\
-  TYPE r_##TYPE[NUM_ELEMS (TYPE)];		\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
-    r_##TYPE[i] = 0;
-
-int main ()
-{
-  int result = 0;
-  DEF_INIT_VECTOR (int8_t)
-  DEF_INIT_VECTOR (int16_t)
-  DEF_INIT_VECTOR (int32_t)
-  DEF_INIT_VECTOR (int64_t)
-  DEF_INIT_VECTOR (uint8_t)
-  DEF_INIT_VECTOR (uint16_t)
-  DEF_INIT_VECTOR (uint32_t)
-  DEF_INIT_VECTOR (uint64_t)
-
-  TEST_LOOPS_ALL_UNSIGNED_TYPES (b0s1)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b0s1)
-  TEST_LOOPS_ALL_UNSIGNED_TYPES (b0s15)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b0s15)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b0sm1)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b0sm16)
-  TEST_LOOPS_ALL_SIGNED_TYPES (bm16s1)
-  TEST_LOOPS_ALL_UNSIGNED_TYPES (b15s1)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b15s1)
-
-  TEST_LOOPS_ALL_UNSIGNED_TYPES (b0s16)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b0s16)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b0sm17)
-  TEST_LOOPS_ALL_SIGNED_TYPES (bm17s1)
-  TEST_LOOPS_ALL_UNSIGNED_TYPES (b16s1)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b16s1)
-  TEST_LOOPS_ALL_UNSIGNED_TYPES (b16s16)
-  TEST_LOOPS_ALL_SIGNED_TYPES (b16s16)
-  TEST_LOOPS_ALL_SIGNED_TYPES (bm17sm17)
-
-  if (result != 24270)
-    {
-      fprintf (stderr, "result = %d\n", result);
-      abort ();
-    }
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_index_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_index_1_run.c
new file mode 100644
index 00000000000..7492ed3f756
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_index_1_run.c
@@ -0,0 +1,20 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve -msve-vector-bits=256" } */
+
+#include "sve_index_1.c"
+
+#define TEST_LOOP(TYPE, BASE, STEP, SUFFIX)	\
+  {						\
+    TYPE array[NUM_ELEMS (TYPE)] = {};		\
+    loop_##TYPE##_##SUFFIX (array);		\
+    for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
+      if (array[i] != (TYPE) (BASE + i * STEP))	\
+	__builtin_abort ();			\
+  }
+
+int __attribute__ ((optimize (1)))
+main ()
+{
+  TEST_ALL (TEST_LOOP)
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_indexoffsetlarge_1.c b/gcc/testsuite/gcc.target/aarch64/sve_indexoffsetlarge_1.c
deleted file mode 100644
index 4c9aab4aada..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_indexoffsetlarge_1.c
+++ /dev/null
@@ -1,31 +0,0 @@
-/* { dg-do assemble } */
-/* { dg-options "-std=c99 -ftree-vectorize -O2 -fno-inline -march=armv8-a+sve --save-temps" } */
-
-//Test sizes that are too big for an index register
-#define SIZE 4294967297
-
-#define INDEX_OFFSET_TEST(SIGNED, TYPE)\
-void set_##SIGNED##TYPE (SIGNED TYPE *out, SIGNED TYPE *in)\
-{\
-  unsigned long i;\
-  for (i = 0; i < SIZE; i++)\
-  {\
-    out[i] = in[i];\
-  }\
-}
-
-INDEX_OFFSET_TEST (signed, int)
-INDEX_OFFSET_TEST (unsigned, int)
-INDEX_OFFSET_TEST (signed, short)
-INDEX_OFFSET_TEST (unsigned, short)
-INDEX_OFFSET_TEST (signed, char)
-INDEX_OFFSET_TEST (unsigned, char)
-
-/* { dg-final { scan-assembler-not "ld1d\\tz\[0-9\]+.d, p\[0-9\]+/z, \\\[x\[0-9\]+, w\[0-9\]+, .xtw 3\\\]" } } */
-/* { dg-final { scan-assembler-not "st1d\\tz\[0-9\]+.d, p\[0-9\]+, \\\[x\[0-9\]+, w\[0-9\]+, .xtw 3\\\]" } } */
-/* { dg-final { scan-assembler-not "ld1w\\tz\[0-9\]+.s, p\[0-9\]+/z, \\\[x\[0-9\]+, w\[0-9\]+, .xtw 2\\\]" } } */
-/* { dg-final { scan-assembler-not "st1w\\tz\[0-9\]+.s, p\[0-9\]+, \\\[x\[0-9\]+, w\[0-9\]+, .xtw 2\\\]" } } */
-/* { dg-final { scan-assembler-not "ld1h\\tz\[0-9\]+.h, p\[0-9\]+/z, \\\[x\[0-9\]+, w\[0-9\]+, .xtw 1\\\]" } } */
-/* { dg-final { scan-assembler-not "st1h\\tz\[0-9\]+.h, p\[0-9\]+, \\\[x\[0-9\]+, w\[0-9\]+, .xtw 1\\\]" } } */
-/* { dg-final { scan-assembler-not "ld1b\\tz\[0-9\]+.b, p\[0-9\]+/z, \\\[x\[0-9\]+, w\[0-9\]+, .xtw\\\]" } } */
-/* { dg-final { scan-assembler-not "st1b\\tz\[0-9\]+.b, p\[0-9\]+, \\\[x\[0-9\]+, w\[0-9\]+, .xtw\\\]" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_infloop_1.c b/gcc/testsuite/gcc.target/aarch64/sve_infloop_1.c
deleted file mode 100644
index 11681c05409..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_infloop_1.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* { dg-do run { target { aarch64_sve_hw } } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve -fno-tree-loop-distribute-patterns" } */
-/* { dg-timeout 60 } */
-
-#include <stdint.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <limits.h>
-
-/* Make sure that in cases where
-    n <= TYPE_MAX && n > (TYPE_MAX + 1 - (sizeof (SVE_VEC) / sizeof (TYPE)))
-   that we don't iterate more times than we should due to overflow in the last
-   iteration of loop.  If n == TYPE_MAX we could spin forever.  */
-
-#define SIMPLE_LOOP(TYPE)			\
-TYPE foo_##TYPE (TYPE n, TYPE * __restrict__ a)	\
-{						\
-  TYPE i;					\
-  TYPE v = 0;					\
-  for (i = 0; i < n; i++)			\
-    v += a[i];					\
-  return v;					\
-}
-
-SIMPLE_LOOP (uint8_t)
-SIMPLE_LOOP (uint16_t)
-
-/* Minimum architected SVE vector = 128 bits, i.e. 16 bytes. Just choose
-   something that meets the critera shown above.  */
-#define N_uint8_t  (UCHAR_MAX - 1)
-#define N_uint16_t (USHRT_MAX - 1)
-
-#define N_MAX 1024
-#define DEF_VAR(TYPE)						\
-  TYPE *a_##TYPE = (TYPE *) malloc (N_##TYPE * sizeof (TYPE));	\
-  for (i = 0; i < N_##TYPE; i++)				\
-    a_##TYPE[i] = 1;						\
-  TYPE r_##TYPE;
-
-#define TEST_SIMPLE_LOOP(TYPE) r_##TYPE = foo_##TYPE (N_##TYPE, a_##TYPE);
-
-#define VERIFY(TYPE)							\
-  if (r_##TYPE != N_##TYPE)						\
-    {									\
-      fprintf (stderr, "r_" #TYPE " = %ld\n", (uint64_t) r_##TYPE);	\
-      abort ();								\
-    }
-
-int main ()
-{
-  int i;
-  DEF_VAR (uint8_t)
-  DEF_VAR (uint16_t)
-
-  /* We only test 8 and 16 bit as others take too long.  */
-  TEST_SIMPLE_LOOP (uint8_t)
-  TEST_SIMPLE_LOOP (uint16_t)
-
-  VERIFY (uint8_t)
-  VERIFY (uint16_t)
-
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_ld1r_1.c b/gcc/testsuite/gcc.target/aarch64/sve_ld1r_1.c
new file mode 100644
index 00000000000..314c2b89624
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_ld1r_1.c
@@ -0,0 +1,53 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
+
+#include <stdint.h>
+
+#define DUP4(X) X, X, X, X
+#define DUP8(X) DUP4 (X), DUP4 (X)
+#define DUP16(X) DUP8 (X), DUP8 (X)
+#define DUP32(X) DUP16 (X), DUP16 (X)
+
+typedef uint8_t vuint8_t __attribute__ ((vector_size (32)));
+typedef uint16_t vuint16_t __attribute__ ((vector_size (32)));
+typedef uint32_t vuint32_t __attribute__ ((vector_size (32)));
+typedef uint64_t vuint64_t __attribute__ ((vector_size (32)));
+
+#define TEST(TYPE, NAME, INIT)					\
+  void								\
+  NAME##_##TYPE (TYPE *dest, __typeof__(dest[0][0]) *ptr)	\
+  {								\
+    TYPE x = { INIT };						\
+    *dest = x;							\
+  }
+
+#define TEST_GROUP(TYPE, NAME, DUP)		\
+  TEST (TYPE, NAME_##m1, DUP (ptr[-1]))		\
+  TEST (TYPE, NAME_##0, DUP (ptr[0]))		\
+  TEST (TYPE, NAME_##63, DUP (ptr[63]))		\
+  TEST (TYPE, NAME_##64, DUP (ptr[64]))
+
+TEST_GROUP (vuint8_t, t8, DUP32)
+TEST_GROUP (vuint16_t, t16, DUP16)
+TEST_GROUP (vuint32_t, t16, DUP8)
+TEST_GROUP (vuint64_t, t16, DUP4)
+
+/* { dg-final { scan-assembler-not {\tld1rb\tz[0-9]+\.b, p[0-7]/z, \[x1, -1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rb\tz[0-9]+\.b, p[0-7]/z, \[x1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rb\tz[0-9]+\.b, p[0-7]/z, \[x1, 63\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1rb\tz[0-9]+\.b, p[0-7]/z, \[x1, 64\]\n} } } */
+
+/* { dg-final { scan-assembler-not {\tld1rh\tz[0-9]+\.h, p[0-7]/z, \[x1, -1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rh\tz[0-9]+\.h, p[0-7]/z, \[x1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rh\tz[0-9]+\.h, p[0-7]/z, \[x1, 126\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1rh\tz[0-9]+\.h, p[0-7]/z, \[x1, 128\]\n} } } */
+
+/* { dg-final { scan-assembler-not {\tld1rw\tz[0-9]+\.s, p[0-7]/z, \[x1, -1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rw\tz[0-9]+\.s, p[0-7]/z, \[x1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rw\tz[0-9]+\.s, p[0-7]/z, \[x1, 252\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1rw\tz[0-9]+\.s, p[0-7]/z, \[x1, 256\]\n} } } */
+
+/* { dg-final { scan-assembler-not {\tld1rd\tz[0-9]+\.d, p[0-7]/z, \[x1, -1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rd\tz[0-9]+\.d, p[0-7]/z, \[x1\]\n} } } */
+/* { dg-final { scan-assembler {\tld1rd\tz[0-9]+\.d, p[0-7]/z, \[x1, 504\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1rd\tz[0-9]+\.d, p[0-7]/z, \[x1, 512\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_ld1r_2.C b/gcc/testsuite/gcc.target/aarch64/sve_ld1r_2.C
deleted file mode 100644
index d209b48d249..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_ld1r_2.C
+++ /dev/null
@@ -1,51 +0,0 @@
-/* { dg-do compile } */
-/* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256" } */
-
-#define DUP4(X) X, X, X, X
-#define DUP8(X) DUP4 (X), DUP4 (X)
-#define DUP16(X) DUP8 (X), DUP8 (X)
-#define DUP32(X) DUP16 (X), DUP16 (X)
-
-typedef unsigned char vuint8_t __attribute__ ((vector_size (32)));
-typedef unsigned short vuint16_t __attribute__ ((vector_size (32)));
-typedef unsigned int vuint32_t __attribute__ ((vector_size (32)));
-typedef unsigned long vuint64_t __attribute__ ((vector_size (32)));
-
-#define TEST(TYPE, NAME, INIT)				\
-  void							\
-  NAME (TYPE *dest, __typeof__(dest[0][0]) *ptr)	\
-  {							\
-    TYPE x = { INIT };					\
-    *dest = x;						\
-  }
-
-#define TEST_GROUP(TYPE, NAME, DUP)		\
-  TEST (TYPE, NAME_##m1, DUP (ptr[-1]))		\
-  TEST (TYPE, NAME_##0, DUP (ptr[0]))		\
-  TEST (TYPE, NAME_##63, DUP (ptr[63]))		\
-  TEST (TYPE, NAME_##64, DUP (ptr[64]))
-
-TEST_GROUP (vuint8_t, t8, DUP32)
-TEST_GROUP (vuint16_t, t16, DUP16)
-TEST_GROUP (vuint32_t, t16, DUP8)
-TEST_GROUP (vuint64_t, t16, DUP4)
-
-/* { dg-final { scan-assembler-not {\tld1rb\tz[0-9]*.b, p[0-7]/z, \[x1, -1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rb\tz[0-9]*.b, p[0-7]/z, \[x1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rb\tz[0-9]*.b, p[0-7]/z, \[x1, 63\]\n} } } */
-/* { dg-final { scan-assembler-not {\tld1rb\tz[0-9]*.b, p[0-7]/z, \[x1, 64\]\n} } } */
-
-/* { dg-final { scan-assembler-not {\tld1rh\tz[0-9]*.h, p[0-7]/z, \[x1, -1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rh\tz[0-9]*.h, p[0-7]/z, \[x1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rh\tz[0-9]*.h, p[0-7]/z, \[x1, 126\]\n} } } */
-/* { dg-final { scan-assembler-not {\tld1rh\tz[0-9]*.h, p[0-7]/z, \[x1, 128\]\n} } } */
-
-/* { dg-final { scan-assembler-not {\tld1rw\tz[0-9]*.s, p[0-7]/z, \[x1, -1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rw\tz[0-9]*.s, p[0-7]/z, \[x1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rw\tz[0-9]*.s, p[0-7]/z, \[x1, 252\]\n} } } */
-/* { dg-final { scan-assembler-not {\tld1rw\tz[0-9]*.s, p[0-7]/z, \[x1, 256\]\n} } } */
-
-/* { dg-final { scan-assembler-not {\tld1rd\tz[0-9]*.d, p[0-7]/z, \[x1, -1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rd\tz[0-9]*.d, p[0-7]/z, \[x1\]\n} } } */
-/* { dg-final { scan-assembler {\tld1rd\tz[0-9]*.d, p[0-7]/z, \[x1, 504\]\n} } } */
-/* { dg-final { scan-assembler-not {\tld1rd\tz[0-9]*.d, p[0-7]/z, \[x1, 512\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_load_const_offset_1.c b/gcc/testsuite/gcc.target/aarch64/sve_load_const_offset_1.c
index 3bbae95f332..0bc757907cf 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_load_const_offset_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_load_const_offset_1.c
@@ -1,10 +1,12 @@
 /* { dg-do assemble } */
-/* { dg-options "-O -march=armv8-a+sve -save-temps -msve-vector-bits=256" } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef long v4di __attribute__ ((vector_size (32)));
-typedef int v8si __attribute__ ((vector_size (32)));
-typedef short v16hi __attribute__ ((vector_size (32)));
-typedef char v32qi __attribute__ ((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__ ((vector_size (32)));
+typedef int32_t v8si __attribute__ ((vector_size (32)));
+typedef int16_t v16hi __attribute__ ((vector_size (32)));
+typedef int8_t v32qi __attribute__ ((vector_size (32)));
 
 #define TEST_TYPE(TYPE)						\
   void sve_load_##TYPE##_neg9 (TYPE *a)				\
@@ -52,26 +54,26 @@ TEST_TYPE (v32qi)
 /* { dg-final { scan-assembler-times {\tadd\tx[0-9]+, x0, 16\n} 4 } } */
 /* { dg-final { scan-assembler-times {\tadd\tx[0-9]+, x0, 256\n} 4 } } */
 
-/* { dg-final { scan-assembler-not {\tld1d\tz0.d, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
-/* { dg-final { scan-assembler-times {\tld1d\tz0.d, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tld1d\tz0.d, p[0-7]/z, \[x0\]\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tld1d\tz0.d, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-not {\tld1d\tz0.d, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1d\tz0\.d, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz0\.d, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz0\.d, p[0-7]/z, \[x0\]\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz0\.d, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tld1d\tz0\.d, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
 
-/* { dg-final { scan-assembler-not {\tld1w\tz0.s, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
-/* { dg-final { scan-assembler-times {\tld1w\tz0.s, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tld1w\tz0.s, p[0-7]/z, \[x0\]\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tld1w\tz0.s, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-not {\tld1w\tz0.s, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1w\tz0\.s, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz0\.s, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz0\.s, p[0-7]/z, \[x0\]\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz0\.s, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tld1w\tz0\.s, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
 
-/* { dg-final { scan-assembler-not {\tld1h\tz0.h, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
-/* { dg-final { scan-assembler-times {\tld1h\tz0.h, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tld1h\tz0.h, p[0-7]/z, \[x0\]\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tld1h\tz0.h, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-not {\tld1h\tz0.h, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1h\tz0\.h, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz0\.h, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz0\.h, p[0-7]/z, \[x0\]\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz0\.h, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tld1h\tz0\.h, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
 
-/* { dg-final { scan-assembler-not {\tld1b\tz0.b, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
-/* { dg-final { scan-assembler-times {\tld1b\tz0.b, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tld1b\tz0.b, p[0-7]/z, \[x0\]\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tld1b\tz0.b, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
-/* { dg-final { scan-assembler-not {\tld1b\tz0.b, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-not {\tld1b\tz0\.b, p[0-7]/z, \[x0, #-9, mul vl\]\n} } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz0\.b, p[0-7]/z, \[x0, #-8, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz0\.b, p[0-7]/z, \[x0\]\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz0\.b, p[0-7]/z, \[x0, #7, mul vl\]\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tld1b\tz0\.b, p[0-7]/z, \[x0, #8, mul vl\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_load_scalar_offset_1.c b/gcc/testsuite/gcc.target/aarch64/sve_load_scalar_offset_1.c
index 77a3ef82f62..9163702db1d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_load_scalar_offset_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_load_scalar_offset_1.c
@@ -1,68 +1,70 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef long v4di __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef char v32qi __attribute__((vector_size(32)));
+#include <stdint.h>
 
-void sve_load_64_u_lsl (unsigned long *a)
+typedef int64_t v4di __attribute__ ((vector_size (32)));
+typedef int32_t v8si __attribute__ ((vector_size (32)));
+typedef int16_t v16hi __attribute__ ((vector_size (32)));
+typedef int8_t v32qi __attribute__ ((vector_size (32)));
+
+void sve_load_64_u_lsl (uint64_t *a)
 {
   register unsigned long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v4di *)&a[i]));
 }
 
-void sve_load_64_s_lsl (signed long *a)
+void sve_load_64_s_lsl (int64_t *a)
 {
   register long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v4di *)&a[i]));
 }
 
-void sve_load_32_u_lsl (unsigned int *a)
+void sve_load_32_u_lsl (uint32_t *a)
 {
   register unsigned long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v8si *)&a[i]));
 }
 
-void sve_load_32_s_lsl (signed int *a)
+void sve_load_32_s_lsl (int32_t *a)
 {
   register long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v8si *)&a[i]));
 }
 
-void sve_load_16_z_lsl (unsigned short *a)
+void sve_load_16_z_lsl (uint16_t *a)
 {
   register unsigned long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v16hi *)&a[i]));
 }
 
-void sve_load_16_s_lsl (signed short *a)
+void sve_load_16_s_lsl (int16_t *a)
 {
   register long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v16hi *)&a[i]));
 }
 
-void sve_load_8_z (unsigned char *a)
+void sve_load_8_z (uint8_t *a)
 {
   register unsigned long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v32qi *)&a[i]));
 }
 
-void sve_load_8_s (signed char *a)
+void sve_load_8_s (int8_t *a)
 {
   register long i asm("x1");
   asm volatile ("" : "=r" (i));
   asm volatile ("" :: "w" (*(v32qi *)&a[i]));
 }
 
-/* { dg-final { scan-assembler-times {\tld1d\tz0.d, p[0-7]/z, \[x0, x1, lsl 3\]\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tld1w\tz0.s, p[0-7]/z, \[x0, x1, lsl 2\]\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tld1h\tz0.h, p[0-7]/z, \[x0, x1, lsl 1\]\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tld1b\tz0.b, p[0-7]/z, \[x0, x1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz0\.d, p[0-7]/z, \[x0, x1, lsl 3\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz0\.s, p[0-7]/z, \[x0, x1, lsl 2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz0\.h, p[0-7]/z, \[x0, x1, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz0\.b, p[0-7]/z, \[x0, x1\]\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_load_scalar_offset_2.c b/gcc/testsuite/gcc.target/aarch64/sve_load_scalar_offset_2.c
deleted file mode 100644
index 7a36cce95cd..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_load_scalar_offset_2.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* { dg-do assemble } */
-/* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
-
-typedef long v4di __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef char v32qi __attribute__((vector_size(32)));
-
-void sve_load_64_u_lsl (unsigned long *a)
-{
-  register unsigned long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v4di *)&a[i]));
-}
-
-void sve_load_64_s_lsl (signed long *a)
-{
-  register long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v4di *)&a[i]));
-}
-
-void sve_load_32_u_lsl (unsigned int *a)
-{
-  register unsigned long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v8si *)&a[i]));
-}
-
-void sve_load_32_s_lsl (signed int *a)
-{
-  register long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v8si *)&a[i]));
-}
-
-void sve_load_16_z_lsl (unsigned short *a)
-{
-  register unsigned long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v16hi *)&a[i]));
-}
-
-void sve_load_16_s_lsl (signed short *a)
-{
-  register long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v16hi *)&a[i]));
-}
-
-void sve_load_8_z (unsigned char *a)
-{
-  register unsigned long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v32qi *)&a[i]));
-}
-
-void sve_load_8_s (signed char *a)
-{
-  register long i asm("x1");
-  asm volatile ("" : "=r" (i));
-  asm volatile ("" :: "w" (*(v32qi *)&a[i]));
-}
-
-/* { dg-final { scan-assembler-times "ld1d\\tz0.d, p\[0-9\]+/z, \\\[x0, x1, lsl 3\\\]" 2 } } */
-/* { dg-final { scan-assembler-times "ld1w\\tz0.s, p\[0-9\]+/z, \\\[x0, x1, lsl 2\\\]" 2 } } */
-/* { dg-final { scan-assembler-times "ld1h\\tz0.h, p\[0-9\]+/z, \\\[x0, x1, lsl 1\\\]" 2 } } */
-/* { dg-final { scan-assembler-times "ld1b\\tz0.b, p\[0-9\]+/z, \\\[x0, x1\\\]" 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_logical_1.c b/gcc/testsuite/gcc.target/aarch64/sve_logical_1.c
index 02f92b95733..aa39adf85f8 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_logical_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_logical_1.c
@@ -1,82 +1,82 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
-#define DO_CONSTANT(VALUE, TYPE, OP, NAME)				\
-void vlogical_imm_##NAME##_##TYPE (TYPE* dst, unsigned long count)	\
-{									\
-  for (int i = 0; i < count; i++)					\
-    dst[i] = dst[i] OP VALUE;						\
+#define DO_CONSTANT(VALUE, TYPE, OP, NAME)			\
+void vlogical_imm_##NAME##_##TYPE (TYPE *dst, int count)	\
+{								\
+  for (int i = 0; i < count; i++)				\
+    dst[i] = dst[i] OP VALUE;					\
 }
 
 #define DO_LOGICAL_OPS_BRIEF(TYPE, OP, NAME)	\
-DO_CONSTANT (1, TYPE, OP, NAME ## 1)		\
-DO_CONSTANT (2, TYPE, OP, NAME ## 2)		\
-DO_CONSTANT (5, TYPE, OP, NAME ## 5)		\
-DO_CONSTANT (6, TYPE, OP, NAME ## 6)		\
-DO_CONSTANT (8, TYPE, OP, NAME ## 8)		\
-DO_CONSTANT (9, TYPE, OP, NAME ## 9)		\
-DO_CONSTANT (-1, TYPE, OP, NAME ## minus1)	\
-DO_CONSTANT (-2, TYPE, OP, NAME ## minus2)	\
-DO_CONSTANT (-5, TYPE, OP, NAME ## minus5)	\
-DO_CONSTANT (-6, TYPE, OP, NAME ## minus6)
+  DO_CONSTANT (1, TYPE, OP, NAME ## 1)		\
+  DO_CONSTANT (2, TYPE, OP, NAME ## 2)		\
+  DO_CONSTANT (5, TYPE, OP, NAME ## 5)		\
+  DO_CONSTANT (6, TYPE, OP, NAME ## 6)		\
+  DO_CONSTANT (8, TYPE, OP, NAME ## 8)		\
+  DO_CONSTANT (9, TYPE, OP, NAME ## 9)		\
+  DO_CONSTANT (-1, TYPE, OP, NAME ## minus1)	\
+  DO_CONSTANT (-2, TYPE, OP, NAME ## minus2)	\
+  DO_CONSTANT (-5, TYPE, OP, NAME ## minus5)	\
+  DO_CONSTANT (-6, TYPE, OP, NAME ## minus6)
 
-#define DO_LOGICAL_OPS(TYPE, OP, NAME)			\
-DO_CONSTANT (1, TYPE, OP, NAME ## 1)			\
-DO_CONSTANT (2, TYPE, OP, NAME ## 2)			\
-DO_CONSTANT (3, TYPE, OP, NAME ## 3)			\
-DO_CONSTANT (4, TYPE, OP, NAME ## 4)			\
-DO_CONSTANT (5, TYPE, OP, NAME ## 5)			\
-DO_CONSTANT (6, TYPE, OP, NAME ## 6)			\
-DO_CONSTANT (7, TYPE, OP, NAME ## 7)			\
-DO_CONSTANT (8, TYPE, OP, NAME ## 8)			\
-DO_CONSTANT (9, TYPE, OP, NAME ## 9)			\
-DO_CONSTANT (10, TYPE, OP, NAME ## 10)			\
-DO_CONSTANT (11, TYPE, OP, NAME ## 11)			\
-DO_CONSTANT (12, TYPE, OP, NAME ## 12)			\
-DO_CONSTANT (13, TYPE, OP, NAME ## 13)			\
-DO_CONSTANT (14, TYPE, OP, NAME ## 14)			\
-DO_CONSTANT (15, TYPE, OP, NAME ## 15)			\
-DO_CONSTANT (16, TYPE, OP, NAME ## 16)			\
-DO_CONSTANT (17, TYPE, OP, NAME ## 17)			\
-DO_CONSTANT (18, TYPE, OP, NAME ## 18)			\
-DO_CONSTANT (19, TYPE, OP, NAME ## 19)			\
-DO_CONSTANT (20, TYPE, OP, NAME ## 20)			\
-DO_CONSTANT (21, TYPE, OP, NAME ## 21)			\
-DO_CONSTANT (22, TYPE, OP, NAME ## 22)			\
-DO_CONSTANT (23, TYPE, OP, NAME ## 23)			\
-DO_CONSTANT (24, TYPE, OP, NAME ## 24)			\
-DO_CONSTANT (25, TYPE, OP, NAME ## 25)			\
-DO_CONSTANT (26, TYPE, OP, NAME ## 26)			\
-DO_CONSTANT (27, TYPE, OP, NAME ## 27)			\
-DO_CONSTANT (28, TYPE, OP, NAME ## 28)			\
-DO_CONSTANT (29, TYPE, OP, NAME ## 29)			\
-DO_CONSTANT (30, TYPE, OP, NAME ## 30)			\
-DO_CONSTANT (31, TYPE, OP, NAME ## 31)			\
-DO_CONSTANT (32, TYPE, OP, NAME ## 32)			\
-DO_CONSTANT (33, TYPE, OP, NAME ## 33)			\
-DO_CONSTANT (34, TYPE, OP, NAME ## 34)			\
-DO_CONSTANT (35, TYPE, OP, NAME ## 35)			\
-DO_CONSTANT (252, TYPE, OP, NAME ## 252)		\
-DO_CONSTANT (253, TYPE, OP, NAME ## 253)		\
-DO_CONSTANT (254, TYPE, OP, NAME ## 254)		\
-DO_CONSTANT (255, TYPE, OP, NAME ## 255)		\
-DO_CONSTANT (256, TYPE, OP, NAME ## 256)		\
-DO_CONSTANT (257, TYPE, OP, NAME ## 257)		\
-DO_CONSTANT (65535, TYPE, OP, NAME ## 65535)		\
-DO_CONSTANT (65536, TYPE, OP, NAME ## 65536)		\
-DO_CONSTANT (65537, TYPE, OP, NAME ## 65537)		\
-DO_CONSTANT (2147483646, TYPE, OP, NAME ## 2147483646)	\
-DO_CONSTANT (2147483647, TYPE, OP, NAME ## 2147483647)	\
-DO_CONSTANT (2147483648, TYPE, OP, NAME ## 2147483648)	\
-DO_CONSTANT (-1, TYPE, OP, NAME ## minus1)		\
-DO_CONSTANT (-2, TYPE, OP, NAME ## minus2)		\
-DO_CONSTANT (-3, TYPE, OP, NAME ## minus3)		\
-DO_CONSTANT (-4, TYPE, OP, NAME ## minus4)		\
-DO_CONSTANT (-5, TYPE, OP, NAME ## minus5)		\
-DO_CONSTANT (-6, TYPE, OP, NAME ## minus6)		\
-DO_CONSTANT (-7, TYPE, OP, NAME ## minus7)		\
-DO_CONSTANT (-8, TYPE, OP, NAME ## minus8)		\
-DO_CONSTANT (-9, TYPE, OP, NAME ## minus9)
+#define DO_LOGICAL_OPS(TYPE, OP, NAME)				\
+  DO_CONSTANT (1, TYPE, OP, NAME ## 1)				\
+  DO_CONSTANT (2, TYPE, OP, NAME ## 2)				\
+  DO_CONSTANT (3, TYPE, OP, NAME ## 3)				\
+  DO_CONSTANT (4, TYPE, OP, NAME ## 4)				\
+  DO_CONSTANT (5, TYPE, OP, NAME ## 5)				\
+  DO_CONSTANT (6, TYPE, OP, NAME ## 6)				\
+  DO_CONSTANT (7, TYPE, OP, NAME ## 7)				\
+  DO_CONSTANT (8, TYPE, OP, NAME ## 8)				\
+  DO_CONSTANT (9, TYPE, OP, NAME ## 9)				\
+  DO_CONSTANT (10, TYPE, OP, NAME ## 10)			\
+  DO_CONSTANT (11, TYPE, OP, NAME ## 11)			\
+  DO_CONSTANT (12, TYPE, OP, NAME ## 12)			\
+  DO_CONSTANT (13, TYPE, OP, NAME ## 13)			\
+  DO_CONSTANT (14, TYPE, OP, NAME ## 14)			\
+  DO_CONSTANT (15, TYPE, OP, NAME ## 15)			\
+  DO_CONSTANT (16, TYPE, OP, NAME ## 16)			\
+  DO_CONSTANT (17, TYPE, OP, NAME ## 17)			\
+  DO_CONSTANT (18, TYPE, OP, NAME ## 18)			\
+  DO_CONSTANT (19, TYPE, OP, NAME ## 19)			\
+  DO_CONSTANT (20, TYPE, OP, NAME ## 20)			\
+  DO_CONSTANT (21, TYPE, OP, NAME ## 21)			\
+  DO_CONSTANT (22, TYPE, OP, NAME ## 22)			\
+  DO_CONSTANT (23, TYPE, OP, NAME ## 23)			\
+  DO_CONSTANT (24, TYPE, OP, NAME ## 24)			\
+  DO_CONSTANT (25, TYPE, OP, NAME ## 25)			\
+  DO_CONSTANT (26, TYPE, OP, NAME ## 26)			\
+  DO_CONSTANT (27, TYPE, OP, NAME ## 27)			\
+  DO_CONSTANT (28, TYPE, OP, NAME ## 28)			\
+  DO_CONSTANT (29, TYPE, OP, NAME ## 29)			\
+  DO_CONSTANT (30, TYPE, OP, NAME ## 30)			\
+  DO_CONSTANT (31, TYPE, OP, NAME ## 31)			\
+  DO_CONSTANT (32, TYPE, OP, NAME ## 32)			\
+  DO_CONSTANT (33, TYPE, OP, NAME ## 33)			\
+  DO_CONSTANT (34, TYPE, OP, NAME ## 34)			\
+  DO_CONSTANT (35, TYPE, OP, NAME ## 35)			\
+  DO_CONSTANT (252, TYPE, OP, NAME ## 252)			\
+  DO_CONSTANT (253, TYPE, OP, NAME ## 253)			\
+  DO_CONSTANT (254, TYPE, OP, NAME ## 254)			\
+  DO_CONSTANT (255, TYPE, OP, NAME ## 255)			\
+  DO_CONSTANT (256, TYPE, OP, NAME ## 256)			\
+  DO_CONSTANT (257, TYPE, OP, NAME ## 257)			\
+  DO_CONSTANT (65535, TYPE, OP, NAME ## 65535)			\
+  DO_CONSTANT (65536, TYPE, OP, NAME ## 65536)			\
+  DO_CONSTANT (65537, TYPE, OP, NAME ## 65537)			\
+  DO_CONSTANT (2147483646, TYPE, OP, NAME ## 2147483646)	\
+  DO_CONSTANT (2147483647, TYPE, OP, NAME ## 2147483647)	\
+  DO_CONSTANT (2147483648, TYPE, OP, NAME ## 2147483648)	\
+  DO_CONSTANT (-1, TYPE, OP, NAME ## minus1)			\
+  DO_CONSTANT (-2, TYPE, OP, NAME ## minus2)			\
+  DO_CONSTANT (-3, TYPE, OP, NAME ## minus3)			\
+  DO_CONSTANT (-4, TYPE, OP, NAME ## minus4)			\
+  DO_CONSTANT (-5, TYPE, OP, NAME ## minus5)			\
+  DO_CONSTANT (-6, TYPE, OP, NAME ## minus6)			\
+  DO_CONSTANT (-7, TYPE, OP, NAME ## minus7)			\
+  DO_CONSTANT (-8, TYPE, OP, NAME ## minus8)			\
+  DO_CONSTANT (-9, TYPE, OP, NAME ## minus9)
 
 DO_LOGICAL_OPS_BRIEF (char, &, and)
 DO_LOGICAL_OPS_BRIEF (long, &, and)
@@ -215,8 +215,7 @@ DO_LOGICAL_OPS (int, ^, xor)
 /* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.s, z[0-9]+\.s, #0xfffffff8\n} 1 } } */
 /* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.s, z[0-9]+\.s, #0xfffffff7\n} 1 } } */
 
-/* No specific number because this also doubles as a move.  */
-/* { dg-final { scan-assembler {\torr\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 22 } } */
 
 /* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.s, z[0-9]+\.s, #0x1\n} 1 } } */
 /* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.s, z[0-9]+\.s, #0x2\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1.c b/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1.c
index 73d78bdc8be..5546cefe686 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1.c
@@ -1,13 +1,13 @@
 /* { dg-do compile } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve" } */
+/* { dg-options "-O3 -march=armv8-a+sve" } */
 
-__attribute__((noinline, noclone))
-void vadd (int *dst, int *op1, int *op2, int count)
+void __attribute__((noinline, noclone))
+vadd (int *dst, int *op1, int *op2, int count)
 {
   for (int i = 0; i < count; ++i)
     dst[i] = op1[i] + op2[i];
 }
 
-/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+.s, p[0-7]/z,} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+.s, p[0-7],} 1 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z,} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7],} 1 } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1_run.c
index 6f06ce6e8a6..c7d0352e273 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_loop_add_1_run.c
@@ -1,11 +1,12 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve" } */
+/* { dg-options "-O3 -march=armv8-a+sve" } */
 
 #include "sve_loop_add_1.c"
 
 #define ELEMS 10
 
-int main (void)
+int __attribute__ ((optimize (1)))
+main (void)
 {
   int in1[ELEMS] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
   int in2[ELEMS] = { 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
@@ -16,7 +17,7 @@ int main (void)
 
   for (int i = 0; i < ELEMS; ++i)
     if (out[i] != check[i])
-      return 1;
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_loop_add_5.c b/gcc/testsuite/gcc.target/aarch64/sve_loop_add_5.c
index 30891703a63..a27bde6f9da 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_loop_add_5.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_loop_add_5.c
@@ -12,8 +12,8 @@
 /* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7]+, \[x[0-9]+, x[0-9]+\]} 8 } } */
 
 /* The induction vector is invariant for steps of -16 and 16.  */
-/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #} 3 } } */
-/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #} 3 } } */
+/* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, #} } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, #} 6 } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 8 } } */
 
 /* { dg-final { scan-assembler-times {\tindex\tz[0-9]+\.h, w[0-9]+, #-16\n} 1 { xfail *-*-* } } } */
@@ -25,8 +25,8 @@
 /* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7]+, \[x[0-9]+, x[0-9]+, lsl 1\]} 8 } } */
 
 /* The (-)17 * 16 is out of range.  */
-/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #} 3 } } */
-/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #} 3 } } */
+/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.h, z[0-9]+\.h, #} 2 } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, #} 4 } } */
 /* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 10 } } */
 
 /* { dg-final { scan-assembler-times {\tindex\tz[0-9]+\.s, w[0-9]+, #-16\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_mad_1.c b/gcc/testsuite/gcc.target/aarch64/sve_mad_1.c
index 6da2e115782..ccb20b4191f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_mad_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_mad_1.c
@@ -1,34 +1,34 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef char v32qi __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef long v4di __attribute__((vector_size(32)));
+#include <stdint.h>
 
-#define DO_OP(TYPE)                                   \
-void vmla##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
-  dst = (dst * src1) + src2;                          \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+typedef int8_t v32qi __attribute__((vector_size(32)));
+typedef int16_t v16hi __attribute__((vector_size(32)));
+typedef int32_t v8si __attribute__((vector_size(32)));
+typedef int64_t v4di __attribute__((vector_size(32)));
+
+#define DO_OP(TYPE)						\
+void vmla_##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (dst * src1) + src2;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
-DO_OP(v32qi)
-DO_OP(v16hi)
-DO_OP(v8si)
-DO_OP(v4di)
+DO_OP (v32qi)
+DO_OP (v16hi)
+DO_OP (v8si)
+DO_OP (v4di)
 
-/* { dg-final { scan-assembler-times {\tmad\tz0.b, p[0-7]/m, z2.b, z4.b} 1 } } */
-/* { dg-final { scan-assembler-times {\tmad\tz0.h, p[0-7]/m, z2.h, z4.h} 1 } } */
-/* { dg-final { scan-assembler-times {\tmad\tz0.s, p[0-7]/m, z2.s, z4.s} 1 } } */
-/* { dg-final { scan-assembler-times {\tmad\tz0.d, p[0-7]/m, z2.d, z4.d} 1 } } */
+/* { dg-final { scan-assembler-times {\tmad\tz0\.b, p[0-7]/m, z2\.b, z4\.b} 1 } } */
+/* { dg-final { scan-assembler-times {\tmad\tz0\.h, p[0-7]/m, z2\.h, z4\.h} 1 } } */
+/* { dg-final { scan-assembler-times {\tmad\tz0\.s, p[0-7]/m, z2\.s, z4\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tmad\tz0\.d, p[0-7]/m, z2\.d, z4\.d} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1.C b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1.c
index f42eb6e9edf..733ffd1b765 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1.c
@@ -1,39 +1,45 @@
 /* { dg-do compile } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -ffast-math -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
 
 #include <stdint.h>
 
 #define NUM_ELEMS(TYPE) (320 / sizeof (TYPE))
 
-#define DEF_MAXMIN(TYPE,NAME,CMP_OP)					\
-void fun_##NAME##TYPE (TYPE *__restrict__ r, TYPE *__restrict__ a,	\
-                       TYPE *__restrict__ b)				\
-{									\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++)				\
-    r[i] = a[i] CMP_OP b[i] ? a[i] : b[i];				\
+#define DEF_MAXMIN(TYPE, NAME, CMP_OP)				\
+void __attribute__ ((noinline, noclone))			\
+fun_##NAME##_##TYPE (TYPE *restrict r, TYPE *restrict a,	\
+		     TYPE *restrict b)				\
+{								\
+  for (int i = 0; i < NUM_ELEMS (TYPE); i++)			\
+    r[i] = a[i] CMP_OP b[i] ? a[i] : b[i];			\
 }
 
-DEF_MAXMIN (int8_t, max, >)
-DEF_MAXMIN (int16_t, max, >)
-DEF_MAXMIN (int32_t, max, >)
-DEF_MAXMIN (int64_t, max, >)
-DEF_MAXMIN (uint8_t, max, >)
-DEF_MAXMIN (uint16_t, max, >)
-DEF_MAXMIN (uint32_t, max, >)
-DEF_MAXMIN (uint64_t, max, >)
-DEF_MAXMIN (float, max, >)
-DEF_MAXMIN (double, max, >)
-
-DEF_MAXMIN (int8_t, min, <)
-DEF_MAXMIN (int16_t, min, <)
-DEF_MAXMIN (int32_t, min, <)
-DEF_MAXMIN (int64_t, min, <)
-DEF_MAXMIN (uint8_t, min, <)
-DEF_MAXMIN (uint16_t, min, <)
-DEF_MAXMIN (uint32_t, min, <)
-DEF_MAXMIN (uint64_t, min, <)
-DEF_MAXMIN (float, min, <)
-DEF_MAXMIN (double, min, <)
+#define TEST_ALL(T)			\
+  T (int8_t, max, >)			\
+  T (int16_t, max, >)			\
+  T (int32_t, max, >)			\
+  T (int64_t, max, >)			\
+  T (uint8_t, max, >)			\
+  T (uint16_t, max, >)			\
+  T (uint32_t, max, >)			\
+  T (uint64_t, max, >)			\
+  T (_Float16, max, >)			\
+  T (float, max, >)			\
+  T (double, max, >)			\
+					\
+  T (int8_t, min, <)			\
+  T (int16_t, min, <)			\
+  T (int32_t, min, <)			\
+  T (int64_t, min, <)			\
+  T (uint8_t, min, <)			\
+  T (uint16_t, min, <)			\
+  T (uint32_t, min, <)			\
+  T (uint64_t, min, <)			\
+  T (_Float16, min, <)			\
+  T (float, min, <)			\
+  T (double, min, <)
+
+TEST_ALL (DEF_MAXMIN)
 
 /* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
@@ -45,6 +51,7 @@ DEF_MAXMIN (double, min, <)
 /* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
 
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
 
@@ -58,5 +65,6 @@ DEF_MAXMIN (double, min, <)
 /* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
 
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1_run.C b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1_run.C
deleted file mode 100644
index 37dc9a4cdec..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1_run.C
+++ /dev/null
@@ -1,88 +0,0 @@
-/* { dg-do run { target { aarch64_sve_hw } } } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -ffast-math -fno-inline -march=armv8-a+sve" } */
-
-#include "sve_maxmin_1.C"
-
-#include <stdlib.h>
-#include <stdio.h>
-
-#define DEF_INIT_VECTOR(TYPE)				\
-  TYPE a_##TYPE[NUM_ELEMS (TYPE)];			\
-  TYPE b_##TYPE[NUM_ELEMS (TYPE)];			\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++ )		\
-  {							\
-    a_##TYPE[i] = ((i * 2) % 3) * (i & 1 ? 1 : -1);	\
-    b_##TYPE[i] = (1 + (i % 4)) * (i & 1 ? -1 : 1);	\
-  }
-
-#define TEST_MAX(RES,TYPE)			\
-{						\
-  TYPE r_##TYPE[NUM_ELEMS (TYPE)];		\
-  fun_max##TYPE (r_##TYPE, a_##TYPE, b_##TYPE);	\
-  TYPE tmp = 0;					\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
-    tmp += r_##TYPE[i];				\
-  (RES) += tmp;					\
-}
-
-#define TEST_MIN(RES,TYPE)			\
-{						\
-  TYPE r_##TYPE[NUM_ELEMS (TYPE)];		\
-  fun_max##TYPE (r_##TYPE, a_##TYPE, b_##TYPE);	\
-  TYPE tmp = 0;					\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
-    tmp += r_##TYPE[i];				\
-  (RES) += tmp;					\
-}
-
-int main ()
-{
-  int result = 0;
-  double resultF = 0.0;
-  DEF_INIT_VECTOR (int8_t)
-  DEF_INIT_VECTOR (int16_t)
-  DEF_INIT_VECTOR (int32_t)
-  DEF_INIT_VECTOR (int64_t)
-  DEF_INIT_VECTOR (uint8_t)
-  DEF_INIT_VECTOR (uint16_t)
-  DEF_INIT_VECTOR (uint32_t)
-  DEF_INIT_VECTOR (uint64_t)
-  DEF_INIT_VECTOR (float)
-  DEF_INIT_VECTOR (double)
-
-  TEST_MIN (result, int8_t)
-  TEST_MIN (result, int16_t)
-  TEST_MIN (result, int32_t)
-  TEST_MIN (result, int64_t)
-  TEST_MIN (result, uint8_t)
-  TEST_MIN (result, uint16_t)
-  TEST_MIN (result, uint32_t)
-  TEST_MIN (result, uint64_t)
-  TEST_MIN (resultF, float)
-  TEST_MIN (resultF, double)
-
-  TEST_MAX (result, int8_t)
-  TEST_MAX (result, int16_t)
-  TEST_MAX (result, int32_t)
-  TEST_MAX (result, int64_t)
-  TEST_MAX (result, uint8_t)
-  TEST_MAX (result, uint16_t)
-  TEST_MAX (result, uint32_t)
-  TEST_MAX (result, uint64_t)
-  TEST_MAX (resultF, float)
-  TEST_MAX (resultF, double)
-
-  if (result != 131400)
-    {
-      fprintf (stderr, "result = %d\n", result);
-      abort ();
-    }
-
-  if (resultF != 362)
-    {
-      fprintf (stderr, "resultF = %1.16lf\n", resultF);
-      abort ();
-    }
-
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1_run.c
new file mode 100644
index 00000000000..d3130bff8fe
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_1_run.c
@@ -0,0 +1,27 @@
+/* { dg-do run { target { aarch64_sve_hw } } } */
+/* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
+
+#include "sve_maxmin_1.c"
+
+#define TEST_LOOP(TYPE, NAME, CMP_OP)			\
+  {							\
+    TYPE a[NUM_ELEMS (TYPE)];				\
+    TYPE b[NUM_ELEMS (TYPE)];				\
+    TYPE r[NUM_ELEMS (TYPE)];				\
+    for (int i = 0; i < NUM_ELEMS (TYPE); i++)		\
+      {							\
+	a[i] = ((i * 2) % 3) * (i & 1 ? 1 : -1);	\
+	b[i] = (1 + (i % 4)) * (i & 1 ? -1 : 1);	\
+	asm volatile ("" ::: "memory");			\
+      }							\
+    fun_##NAME##_##TYPE (r, a, b);			\
+    for (int i = 0; i < NUM_ELEMS (TYPE); i++)		\
+      if (r[i] != (a[i] CMP_OP b[i] ? a[i] : b[i]))	\
+	__builtin_abort ();				\
+  }
+
+int main ()
+{
+  TEST_ALL (TEST_LOOP)
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1.C b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1.c
index 8a5ce725bf1..27561d19694 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1.c
@@ -1,23 +1,27 @@
 /* { dg-do compile } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include <math.h>
 
 #define NUM_ELEMS(TYPE) (320 / sizeof (TYPE))
 
-#define DEF_MAXMIN(TYPE,FUN)					\
-void test_##FUN (TYPE *__restrict__ r, TYPE *__restrict__ a,	\
-                 TYPE *__restrict__ b)				\
+#define DEF_MAXMIN(TYPE, FUN)					\
+void __attribute__ ((noinline, noclone))			\
+test_##FUN##_##TYPE (TYPE *restrict r, TYPE *restrict a,	\
+		     TYPE *restrict b)				\
 {								\
   for (int i = 0; i < NUM_ELEMS (TYPE); i++)			\
     r[i] = FUN (a[i], b[i]);					\
 }
 
-DEF_MAXMIN (float, fmaxf)
-DEF_MAXMIN (double, fmax)
+#define TEST_ALL(T)				\
+  T (float, fmaxf)				\
+  T (double, fmax)				\
+						\
+  T (float, fminf)				\
+  T (double, fmin)
 
-DEF_MAXMIN (float, fminf)
-DEF_MAXMIN (double, fmin)
+TEST_ALL (DEF_MAXMIN)
 
 /* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1_run.C b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1_run.C
deleted file mode 100644
index 06c868638e9..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1_run.C
+++ /dev/null
@@ -1,56 +0,0 @@
-/* { dg-do run { target { aarch64_sve_hw } } } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include "sve_maxmin_strict_1.C"
-
-#define DEF_INIT_VECTOR(TYPE)				\
-  TYPE a_##TYPE[NUM_ELEMS (TYPE)];			\
-  TYPE b_##TYPE[NUM_ELEMS (TYPE)];			\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++ )		\
-  {							\
-    a_##TYPE[i] = ((i * 2) % 3) * (i & 1 ? 1 : -1);	\
-    b_##TYPE[i] = (1 + (i % 4)) * (i & 1 ? -1 : 1);	\
-  }
-
-#define TEST_MAX(RES,FUN,TYPE)			\
-{						\
-  TYPE r_##TYPE[NUM_ELEMS (TYPE)];		\
-  test_##FUN (r_##TYPE, a_##TYPE, b_##TYPE);	\
-  TYPE tmp = 0;					\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
-    tmp += r_##TYPE[i];				\
-  (RES) += tmp;					\
-}
-
-#define TEST_MIN(RES,FUN,TYPE)			\
-{						\
-  TYPE r_##TYPE[NUM_ELEMS (TYPE)];		\
-  test_##FUN (r_##TYPE, a_##TYPE, b_##TYPE);	\
-  TYPE tmp = 0;					\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
-    tmp += r_##TYPE[i];				\
-  (RES) += tmp;					\
-}
-
-int main ()
-{
-  double resultF = 0.0;
-  DEF_INIT_VECTOR (float)
-  DEF_INIT_VECTOR (double)
-
-  TEST_MIN (resultF, fminf, float)
-  TEST_MIN (resultF, fmin, double)
-
-  TEST_MAX (resultF, fmaxf, float)
-  TEST_MAX (resultF, fmax, double)
-
-  if (resultF != -57)
-    {
-      fprintf (stderr, "resultF = %1.16lf\n", resultF);
-      abort ();
-    }
-
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1_run.c
new file mode 100644
index 00000000000..2b869c62a5d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_maxmin_strict_1_run.c
@@ -0,0 +1,27 @@
+/* { dg-do run { target { aarch64_sve_hw } } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
+
+#include "sve_maxmin_strict_1.c"
+
+#define TEST_LOOP(TYPE, FUN)				\
+  {							\
+    TYPE a[NUM_ELEMS (TYPE)];				\
+    TYPE b[NUM_ELEMS (TYPE)];				\
+    TYPE r[NUM_ELEMS (TYPE)];				\
+    for (int i = 0; i < NUM_ELEMS (TYPE); i++)		\
+      {							\
+	a[i] = ((i * 2) % 3) * (i & 1 ? 1 : -1);	\
+	b[i] = (1 + (i % 4)) * (i & 1 ? -1 : 1);	\
+	asm volatile ("" ::: "memory");			\
+      }							\
+    test_##FUN##_##TYPE (r, a, b);			\
+    for (int i = 0; i < NUM_ELEMS (TYPE); i++)		\
+      if (r[i] != FUN (a[i], b[i]))			\
+	__builtin_abort ();				\
+  }
+
+int main ()
+{
+  TEST_ALL (TEST_LOOP)
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_mla_1.c b/gcc/testsuite/gcc.target/aarch64/sve_mla_1.c
index 329cba68ffb..a4d705e38ba 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_mla_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_mla_1.c
@@ -1,26 +1,26 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef char v32qi __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef long v4di __attribute__((vector_size(32)));
+#include <stdint.h>
 
-#define DO_OP(TYPE)                                   \
-void vmla##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
-  dst = (src1 * src2) + dst;                          \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+typedef int8_t v32qi __attribute__((vector_size(32)));
+typedef int16_t v16hi __attribute__((vector_size(32)));
+typedef int32_t v8si __attribute__((vector_size(32)));
+typedef int64_t v4di __attribute__((vector_size(32)));
+
+#define DO_OP(TYPE)						\
+void vmla_##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = (src1 * src2) + dst;					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
 DO_OP (v32qi)
@@ -28,7 +28,7 @@ DO_OP (v16hi)
 DO_OP (v8si)
 DO_OP (v4di)
 
-/* { dg-final { scan-assembler-times {\tmla\tz0.b, p[0-7]/m, z2.b, z4.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmla\tz0.h, p[0-7]/m, z2.h, z4.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmla\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmla\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmla\tz0\.b, p[0-7]/m, z2\.b, z4\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmla\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmla\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmla\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_mls_1.c b/gcc/testsuite/gcc.target/aarch64/sve_mls_1.c
index abcfc5f40e9..b7cc1dba087 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_mls_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_mls_1.c
@@ -1,26 +1,26 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef char v32qi __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef long v4di __attribute__((vector_size(32)));
+#include <stdint.h>
 
-#define DO_OP(TYPE)                                   \
-void vmls##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
-  dst = dst - (src1 * src2);                          \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+typedef int8_t v32qi __attribute__((vector_size(32)));
+typedef int16_t v16hi __attribute__((vector_size(32)));
+typedef int32_t v8si __attribute__((vector_size(32)));
+typedef int64_t v4di __attribute__((vector_size(32)));
+
+#define DO_OP(TYPE)						\
+void vmla_##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
+  dst = dst - (src1 * src2);					\
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
 DO_OP (v32qi)
@@ -28,7 +28,7 @@ DO_OP (v16hi)
 DO_OP (v8si)
 DO_OP (v4di)
 
-/* { dg-final { scan-assembler-times {\tmls\tz0.b, p[0-7]/m, z2.b, z4.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmls\tz0.h, p[0-7]/m, z2.h, z4.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmls\tz0.s, p[0-7]/m, z2.s, z4.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmls\tz0.d, p[0-7]/m, z2.d, z4.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmls\tz0\.b, p[0-7]/m, z2\.b, z4\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmls\tz0\.h, p[0-7]/m, z2\.h, z4\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmls\tz0\.s, p[0-7]/m, z2\.s, z4\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmls\tz0\.d, p[0-7]/m, z2\.d, z4\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_mov_rr_1.c b/gcc/testsuite/gcc.target/aarch64/sve_mov_rr_1.c
index d5a3a38442b..a38375af017 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_mov_rr_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_mov_rr_1.c
@@ -11,4 +11,4 @@ void sve_copy_rr (void)
   asm volatile ("#foo" :: "w" (y));
 }
 
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+.d, z[0-9]+.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_msb_1.c b/gcc/testsuite/gcc.target/aarch64/sve_msb_1.c
index 132740b0866..fc05837a920 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_msb_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_msb_1.c
@@ -1,34 +1,34 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef char v32qi __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef long v4di __attribute__((vector_size(32)));
+#include <stdint.h>
 
-#define DO_OP(TYPE)                                   \
-void vmla##TYPE (TYPE *_dst, TYPE _src1, TYPE _src2)  \
-{                                                     \
-  register TYPE dst  asm("z0");                       \
-  register TYPE src1 asm("z2");                       \
-  register TYPE src2 asm("z4");                       \
-  dst = *_dst;                                        \
-  asm volatile ("" :: "w" (dst));                     \
-  src1 = _src1;                                       \
-  asm volatile ("" :: "w" (src1));                    \
-  src2 = _src2;                                       \
-  asm volatile ("" :: "w" (src2));                    \
+typedef int8_t v32qi __attribute__((vector_size(32)));
+typedef int16_t v16hi __attribute__((vector_size(32)));
+typedef int32_t v8si __attribute__((vector_size(32)));
+typedef int64_t v4di __attribute__((vector_size(32)));
+
+#define DO_OP(TYPE)						\
+void vmla_##TYPE (TYPE *x, TYPE y, TYPE z)			\
+{								\
+  register TYPE dst  asm("z0");					\
+  register TYPE src1 asm("z2");					\
+  register TYPE src2 asm("z4");					\
+  dst = *x;							\
+  src1 = y;							\
+  src2 = z;							\
+  asm volatile ("" :: "w" (dst), "w" (src1), "w" (src2));	\
   dst = src2 - (dst * src1);                          \
-  asm volatile ("" :: "w" (dst));                     \
-  *_dst = dst;                                        \
+  asm volatile ("" :: "w" (dst));				\
+  *x = dst;							\
 }
 
-DO_OP(v32qi)
-DO_OP(v16hi)
-DO_OP(v8si)
-DO_OP(v4di)
+DO_OP (v32qi)
+DO_OP (v16hi)
+DO_OP (v8si)
+DO_OP (v4di)
 
-/* { dg-final { scan-assembler-times {\tmsb\tz0.b, p[0-7]/m, z2.b, z4.b} 1 } } */
-/* { dg-final { scan-assembler-times {\tmsb\tz0.h, p[0-7]/m, z2.h, z4.h} 1 } } */
-/* { dg-final { scan-assembler-times {\tmsb\tz0.s, p[0-7]/m, z2.s, z4.s} 1 } } */
-/* { dg-final { scan-assembler-times {\tmsb\tz0.d, p[0-7]/m, z2.d, z4.d} 1 } } */
+/* { dg-final { scan-assembler-times {\tmsb\tz0\.b, p[0-7]/m, z2\.b, z4\.b} 1 } } */
+/* { dg-final { scan-assembler-times {\tmsb\tz0\.h, p[0-7]/m, z2\.h, z4\.h} 1 } } */
+/* { dg-final { scan-assembler-times {\tmsb\tz0\.s, p[0-7]/m, z2\.s, z4\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tmsb\tz0\.d, p[0-7]/m, z2\.d, z4\.d} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_mul_1.c b/gcc/testsuite/gcc.target/aarch64/sve_mul_1.c
index ae6f8688c58..2b1cd4a7a93 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_mul_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_mul_1.c
@@ -1,34 +1,36 @@
 /* { dg-do assemble } */
-/* { dg-options {-std=c99 -O3 -march=armv8-a+sve --save-temps} } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
+
+#include <stdint.h>
 
 #define DO_REGREG_OPS(TYPE, OP, NAME)				\
-void varith_##TYPE##_##NAME (TYPE* dst, TYPE* src, int count)	\
+void varith_##TYPE##_##NAME (TYPE *dst, TYPE *src, int count)	\
 {								\
   for (int i = 0; i < count; ++i)				\
     dst[i] = dst[i] OP src[i];					\
 }
 
 #define DO_IMMEDIATE_OPS(VALUE, TYPE, OP, NAME)		\
-void varithimm_##NAME##_##TYPE (TYPE* dst, int count)	\
+void varithimm_##NAME##_##TYPE (TYPE *dst, int count)	\
 {							\
   for (int i = 0; i < count; ++i)			\
     dst[i] = dst[i] OP VALUE;				\
 }
 
 #define DO_ARITH_OPS(TYPE, OP, NAME)			\
-DO_REGREG_OPS (TYPE, OP, NAME);				\
-DO_IMMEDIATE_OPS (0, TYPE, OP, NAME ## 0);		\
-DO_IMMEDIATE_OPS (86, TYPE, OP, NAME ## 86);		\
-DO_IMMEDIATE_OPS (109, TYPE, OP, NAME ## 109);		\
-DO_IMMEDIATE_OPS (141, TYPE, OP, NAME ## 141);		\
-DO_IMMEDIATE_OPS (-1, TYPE, OP, NAME ## minus1);	\
-DO_IMMEDIATE_OPS (-110, TYPE, OP, NAME ## minus110);	\
-DO_IMMEDIATE_OPS (-141, TYPE, OP, NAME ## minus141);
-
-DO_ARITH_OPS (char, *, mul)
-DO_ARITH_OPS (short, *, mul)
-DO_ARITH_OPS (int, *, mul)
-DO_ARITH_OPS (long, *, mul)
+  DO_REGREG_OPS (TYPE, OP, NAME);			\
+  DO_IMMEDIATE_OPS (0, TYPE, OP, NAME ## 0);		\
+  DO_IMMEDIATE_OPS (86, TYPE, OP, NAME ## 86);		\
+  DO_IMMEDIATE_OPS (109, TYPE, OP, NAME ## 109);	\
+  DO_IMMEDIATE_OPS (141, TYPE, OP, NAME ## 141);	\
+  DO_IMMEDIATE_OPS (-1, TYPE, OP, NAME ## minus1);	\
+  DO_IMMEDIATE_OPS (-110, TYPE, OP, NAME ## minus110);	\
+  DO_IMMEDIATE_OPS (-141, TYPE, OP, NAME ## minus141);
+
+DO_ARITH_OPS (int8_t, *, mul)
+DO_ARITH_OPS (int16_t, *, mul)
+DO_ARITH_OPS (int32_t, *, mul)
+DO_ARITH_OPS (int64_t, *, mul)
 
 /* { dg-final { scan-assembler-times {\tmul\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmul\tz[0-9]+\.b, z[0-9]+\.b, #86\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_neg_1.c b/gcc/testsuite/gcc.target/aarch64/sve_neg_1.c
index 8e5e8e58b07..b463c2c0580 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_neg_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_neg_1.c
@@ -1,17 +1,21 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
+
+#include <stdint.h>
 
 #define DO_OPS(TYPE)					\
-void vneg_##TYPE (TYPE* dst, TYPE* src, int count)	\
+void vneg_##TYPE (TYPE *dst, TYPE *src, int count)	\
 {							\
   for (int i = 0; i < count; ++i)			\
     dst[i] = -src[i];					\
 }
 
-DO_OPS (char)
-DO_OPS (int)
-DO_OPS (long)
+DO_OPS (int8_t)
+DO_OPS (int16_t)
+DO_OPS (int32_t)
+DO_OPS (int64_t)
 
 /* { dg-final { scan-assembler-times {\tneg\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tneg\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1.c b/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1.c
index 8f54a2a3143..3871451bc1d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1.c
@@ -1,25 +1,30 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
 
-#define DO_VNLOGICAL(TYPE)						\
-void __attribute__ ((weak))						\
-vnlogical_not_##TYPE (TYPE *dst, unsigned long count)			\
-{									\
-  for (int i = 0; i < count; i++)					\
-    dst[i] = ~dst[i];							\
-}									\
-									\
-void __attribute__ ((weak))						\
-vnlogical_bic_##TYPE (TYPE *dst, TYPE *src, unsigned long count)	\
-{									\
-  for (int i = 0; i < count; i++)					\
-    dst[i] = dst[i] & ~src[i];						\
+#include <stdint.h>
+
+#define DO_VNLOGICAL(TYPE)				\
+void __attribute__ ((noinline, noclone))		\
+vnlogical_not_##TYPE (TYPE *dst, int count)		\
+{							\
+  for (int i = 0; i < count; i++)			\
+    dst[i] = ~dst[i];					\
+}							\
+							\
+void __attribute__ ((noinline, noclone))		\
+vnlogical_bic_##TYPE (TYPE *dst, TYPE *src, int count)	\
+{							\
+  for (int i = 0; i < count; i++)			\
+    dst[i] = dst[i] & ~src[i];				\
 }
 
-DO_VNLOGICAL (char)
-DO_VNLOGICAL (short)
-DO_VNLOGICAL (int)
-DO_VNLOGICAL (long)
+#define TEST_ALL(T)				\
+  T (int8_t)					\
+  T (int16_t)					\
+  T (int32_t)					\
+  T (int64_t)
+
+TEST_ALL (DO_VNLOGICAL)
 
 /* { dg-final { scan-assembler-times {\tnot\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tnot\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1_run.c
index ca3f47134fa..905d44b8265 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_nlogical_1_run.c
@@ -9,7 +9,10 @@
   {							\
     TYPE dst[N], src[N];				\
     for (int i = 0; i < N; ++i)				\
-      dst[i] = i ^ 42;					\
+      {							\
+	dst[i] = i ^ 42;				\
+	asm volatile ("" ::: "memory");			\
+      }							\
     vnlogical_not_##TYPE (dst, N);			\
     for (int i = 0; i < N; ++i)				\
       if (dst[i] != (TYPE) ~(i ^ 42))			\
@@ -18,6 +21,7 @@
       {							\
 	dst[i] = i ^ 42;				\
 	src[i] = i % 5;					\
+	asm volatile ("" ::: "memory");			\
       }							\
     vnlogical_bic_##TYPE (dst, src, N);			\
     for (int i = 0; i < N; ++i)				\
@@ -25,12 +29,9 @@
 	__builtin_abort ();				\
   }
 
-int
+int __attribute__ ((optimize (1)))
 main (void)
 {
-  TEST_VNLOGICAL (char);
-  TEST_VNLOGICAL (short);
-  TEST_VNLOGICAL (int);
-  TEST_VNLOGICAL (long);
+  TEST_ALL (TEST_VNLOGICAL)
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_1.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_1.c
index 12fa945a794..723b4e3433b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_1.c
@@ -1,20 +1,25 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-#define PACK(TYPED, TYPES, SIGN)				\
-void pack_##TYPED##_##TYPES##_##SIGN (SIGN TYPED *d,		\
-				      SIGN TYPES *s, int size)	\
-{								\
-  for (int i = 0; i < size; i++)				\
-    d[i] = s[i] + 1;						\
+#include <stdint.h>
+
+#define PACK(TYPED, TYPES)				\
+void __attribute__ ((noinline, noclone))		\
+pack_##TYPED##_##TYPES (TYPED *d, TYPES *s, int size)	\
+{							\
+  for (int i = 0; i < size; i++)			\
+    d[i] = s[i] + 1;					\
 }
 
-PACK (int, long, signed)			\
-PACK (short, int, signed)			\
-PACK (char, short, signed)			\
-PACK (int, long, unsigned)			\
-PACK (short, int, unsigned)			\
-PACK (char, short, unsigned)
+#define TEST_ALL(T)				\
+  T (int32_t, int64_t)				\
+  T (int16_t, int32_t)				\
+  T (int8_t, int16_t)				\
+  T (uint32_t, uint64_t)			\
+  T (uint16_t, uint32_t)			\
+  T (uint8_t, uint16_t)
+
+TEST_ALL (PACK)
 
 /* { dg-final { scan-assembler-times {\tuzp1\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuzp1\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_1_run.c
index 208889f86b8..cb7876cb135 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_1_run.c
@@ -1,46 +1,28 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_pack_1.c"
 
 #define ARRAY_SIZE 57
 
-#define RUN_AND_CHECK_LOOP(TYPED, TYPES, VALUED, VALUES)		\
-{									\
-  int value = 0;							\
-  TYPED arrayd[ARRAY_SIZE];						\
-  TYPES arrays[ARRAY_SIZE];						\
-  memset (arrayd, 67, ARRAY_SIZE * sizeof (TYPED));			\
-  memset (arrays, VALUES, ARRAY_SIZE * sizeof (TYPES));			\
-  pack_##TYPED##_##TYPES##_signed (arrayd, arrays, ARRAY_SIZE);		\
-  for (int i = 0; i < ARRAY_SIZE; i++)					\
-    if (arrayd[i] != VALUED)						\
-      {									\
-        fprintf (stderr,"%d: %d != %d\n", i, arrayd[i], VALUED);	\
-        exit (1);							\
-      }									\
-  memset (arrayd, 74, ARRAY_SIZE*sizeof (TYPED));			\
-  pack_##TYPED##_##TYPES##_unsigned (arrayd, arrays, ARRAY_SIZE);	\
-  for (int i = 0; i < ARRAY_SIZE; i++)					\
-    if (arrayd[i] != VALUED)						\
-      {									\
-        fprintf (stderr,"%d: %d != %d\n", i, arrayd[i], VALUED);	\
-        exit (1);							\
-      }									\
-}
+#define TEST_LOOP(TYPED, TYPES)					\
+  {								\
+    TYPED arrayd[ARRAY_SIZE];					\
+    TYPES arrays[ARRAY_SIZE];					\
+    for (int i = 0; i < ARRAY_SIZE; i++)			\
+      {								\
+	arrays[i] = (i - 10) * 3;				\
+	asm volatile ("" ::: "memory");				\
+      }								\
+    pack_##TYPED##_##TYPES (arrayd, arrays, ARRAY_SIZE);	\
+    for (int i = 0; i < ARRAY_SIZE; i++)			\
+      if (arrayd[i] != (TYPED) ((TYPES) ((i - 10) * 3) + 1))	\
+	__builtin_abort ();					\
+  }
 
-int main (void)
+int __attribute__ ((optimize (1)))
+main (void)
 {
-  int total = 5;
-  RUN_AND_CHECK_LOOP (char, short, total + 1, total);
-  total = (total << 8) + 5;
-  RUN_AND_CHECK_LOOP (short, int, total + 1, total);
-  total = (total << 8) + 5;
-  total = (total << 8) + 5;
-  RUN_AND_CHECK_LOOP (int, long, total + 1, total);
+  TEST_ALL (TEST_LOOP)
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1.c
index 2d1918cc2cd..a99d227e4c8 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1.c
@@ -1,7 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-void pack_int_double_plus_3 (signed int *d, double *s, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+pack_int_double_plus_3 (int32_t *d, double *s, int size)
 {
   for (int i = 0; i < size; i++)
     d[i] = s[i] + 3;
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1_run.c
index 11d85fc8eb0..2a45bb5b1e8 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_signed_1_run.c
@@ -1,9 +1,5 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_pack_fcvt_signed_1.c"
 
@@ -14,19 +10,19 @@
 int __attribute__ ((optimize (1)))
 main (void)
 {
-  static signed int array_dest[ARRAY_SIZE];
+  static int32_t array_dest[ARRAY_SIZE];
   double array_source[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
-    array_source[i] = VAL1;
+    {
+      array_source[i] = VAL1;
+      asm volatile ("" ::: "memory");
+    }
 
   pack_int_double_plus_3 (array_dest, array_source, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_dest[i] != (int) VAL1 + 3)
-      {
-	fprintf (stderr,"%d: %d != %d\n", i, array_dest[i], (int) VAL1 + 3);
-	exit (1);
-      }
+    if (array_dest[i] != (int32_t) VAL1 + 3)
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1.c
index f7692989a71..a039d6fdd66 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1.c
@@ -1,7 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-void pack_int_double_plus_7 (unsigned int *d, double *s, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+pack_int_double_plus_7 (uint32_t *d, double *s, int size)
 {
   for (int i = 0; i < size; i++)
     d[i] = s[i] + 7;
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1_run.c
index 196b6de358a..8a1e72485ad 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_fcvt_unsigned_1_run.c
@@ -1,9 +1,5 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_pack_fcvt_unsigned_1.c"
 
@@ -14,19 +10,19 @@
 int __attribute__ ((optimize (1)))
 main (void)
 {
-  static unsigned int array_dest[ARRAY_SIZE];
+  static uint32_t array_dest[ARRAY_SIZE];
   double array_source[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
-    array_source[i] = VAL1;
+    {
+      array_source[i] = VAL1;
+      asm volatile ("" ::: "memory");
+    }
 
   pack_int_double_plus_7 (array_dest, array_source, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_dest[i] != (int) VAL1 + 7)
-      {
-	fprintf (stderr,"%d: %d != %d\n", i, array_dest[i], (int) VAL1 + 7);
-	exit (1);
-      }
+    if (array_dest[i] != (uint32_t) VAL1 + 7)
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1.c
index 7faf7652e75..746154e530d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1.c
@@ -1,7 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-void pack_float_plus_1point1 (float *d, double *s, int size)
+void __attribute__ ((noinline, noclone))
+pack_float_plus_1point1 (float *d, double *s, int size)
 {
   for (int i = 0; i < size; i++)
     d[i] = s[i] + 1.1;
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1_run.c
index 85a7eca9173..91e8a699f0b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_pack_float_1_run.c
@@ -1,9 +1,5 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_pack_float_1.c"
 
@@ -18,16 +14,15 @@ main (void)
   double array_source[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
-    array_source[i] = VAL1;
+    {
+      array_source[i] = VAL1;
+      asm volatile ("" ::: "memory");
+    }
 
   pack_float_plus_1point1 (array_dest, array_source, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
     if (array_dest[i] != (float) (VAL1 + 1.1))
-      {
-	fprintf (stderr, "%d: %f != %f\n", i, array_dest[i],
-		 (float) (VAL1 + 1.1));
-	exit (1);
-      }
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_popcount_1.c b/gcc/testsuite/gcc.target/aarch64/sve_popcount_1.c
index 0e640dab810..c3bb2756b2a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_popcount_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_popcount_1.c
@@ -1,16 +1,17 @@
 /* { dg-do assemble } */
 /* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve --save-temps" } */
 
-void
-popcount_32 (unsigned int *restrict dst, unsigned int *restrict src, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+popcount_32 (unsigned int *restrict dst, uint32_t *restrict src, int size)
 {
   for (int i = 0; i < size; ++i)
     dst[i] = __builtin_popcount (src[i]);
 }
 
-void
-popcount_64 (unsigned int *restrict dst, unsigned long *restrict src,
-	     int size)
+void __attribute__ ((noinline, noclone))
+popcount_64 (unsigned int *restrict dst, uint64_t *restrict src, int size)
 {
   for (int i = 0; i < size; ++i)
     dst[i] = __builtin_popcountl (src[i]);
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_popcount_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_popcount_1_run.c
index 9ef47bcbf2c..6be828fa81a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_popcount_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_popcount_1_run.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_popcount_1.c"
 
@@ -16,24 +16,31 @@ unsigned int data[] = {
   0x0, 0
 };
 
-int
+int __attribute__ ((optimize (1)))
 main (void)
 {
   unsigned int count = sizeof (data) / sizeof (data[0]) / 2;
 
-  unsigned int in32[count], out32[count];
+  uint32_t in32[count];
+  unsigned int out32[count];
   for (unsigned int i = 0; i < count; ++i)
-    in32[i] = data[i * 2];
+    {
+      in32[i] = data[i * 2];
+      asm volatile ("" ::: "memory");
+    }
   popcount_32 (out32, in32, count);
   for (unsigned int i = 0; i < count; ++i)
     if (out32[i] != data[i * 2 + 1])
       abort ();
 
   count /= 2;
-  unsigned long in64[count];
+  uint64_t in64[count];
   unsigned int out64[count];
   for (unsigned int i = 0; i < count; ++i)
-    in64[i] = ((unsigned long) data[i * 4] << 32) | data[i * 4 + 2];
+    {
+      in64[i] = ((uint64_t) data[i * 4] << 32) | data[i * 4 + 2];
+      asm volatile ("" ::: "memory");
+    }
   popcount_64 (out64, in64, count);
   for (unsigned int i = 0; i < count; ++i)
     if (out64[i] != data[i * 4 + 1] + data[i * 4 + 3])
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_1.C b/gcc/testsuite/gcc.target/aarch64/sve_reduc_1.c
index da3b5fa1963..4c26e78fae8 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_reduc_1.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_reduc_1.c
@@ -1,10 +1,11 @@
 /* { dg-do compile } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -ffast-math -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
 
 #include <stdint.h>
 
 #define DEF_REDUC_PLUS(TYPE)			\
-TYPE reduc_plus_##TYPE (TYPE *a, int n)		\
+TYPE __attribute__ ((noinline, noclone))	\
+reduc_plus_##TYPE (TYPE *a, int n)		\
 {						\
   TYPE r = 0;					\
   for (int i = 0; i < n; ++i)			\
@@ -12,19 +13,24 @@ TYPE reduc_plus_##TYPE (TYPE *a, int n)		\
   return r;					\
 }
 
-DEF_REDUC_PLUS (int8_t)
-DEF_REDUC_PLUS (int16_t)
-DEF_REDUC_PLUS (int32_t)
-DEF_REDUC_PLUS (int64_t)
-DEF_REDUC_PLUS (uint8_t)
-DEF_REDUC_PLUS (uint16_t)
-DEF_REDUC_PLUS (uint32_t)
-DEF_REDUC_PLUS (uint64_t)
-DEF_REDUC_PLUS (float)
-DEF_REDUC_PLUS (double)
-
-#define DEF_REDUC_MAXMIN(TYPE,NAME,CMP_OP)	\
-TYPE reduc_##NAME##TYPE (TYPE *a, int n)	\
+#define TEST_PLUS(T)				\
+  T (int8_t)					\
+  T (int16_t)					\
+  T (int32_t)					\
+  T (int64_t)					\
+  T (uint8_t)					\
+  T (uint16_t)					\
+  T (uint32_t)					\
+  T (uint64_t)					\
+  T (_Float16)					\
+  T (float)					\
+  T (double)
+
+TEST_PLUS (DEF_REDUC_PLUS)
+
+#define DEF_REDUC_MAXMIN(TYPE, NAME, CMP_OP)	\
+TYPE __attribute__ ((noinline, noclone))	\
+reduc_##NAME##_##TYPE (TYPE *a, int n)		\
 {						\
   TYPE r = 13;					\
   for (int i = 0; i < n; ++i)			\
@@ -32,30 +38,36 @@ TYPE reduc_##NAME##TYPE (TYPE *a, int n)	\
   return r;					\
 }
 
-DEF_REDUC_MAXMIN (int8_t, max, >)
-DEF_REDUC_MAXMIN (int16_t, max, >)
-DEF_REDUC_MAXMIN (int32_t, max, >)
-DEF_REDUC_MAXMIN (int64_t, max, >)
-DEF_REDUC_MAXMIN (uint8_t, max, >)
-DEF_REDUC_MAXMIN (uint16_t, max, >)
-DEF_REDUC_MAXMIN (uint32_t, max, >)
-DEF_REDUC_MAXMIN (uint64_t, max, >)
-DEF_REDUC_MAXMIN (float, max, >)
-DEF_REDUC_MAXMIN (double, max, >)
-
-DEF_REDUC_MAXMIN (int8_t, min, <)
-DEF_REDUC_MAXMIN (int16_t, min, <)
-DEF_REDUC_MAXMIN (int32_t, min, <)
-DEF_REDUC_MAXMIN (int64_t, min, <)
-DEF_REDUC_MAXMIN (uint8_t, min, <)
-DEF_REDUC_MAXMIN (uint16_t, min, <)
-DEF_REDUC_MAXMIN (uint32_t, min, <)
-DEF_REDUC_MAXMIN (uint64_t, min, <)
-DEF_REDUC_MAXMIN (float, min, <)
-DEF_REDUC_MAXMIN (double, min, <)
-
-#define DEF_REDUC_BITWISE(TYPE,NAME,BIT_OP)	\
-TYPE reduc_##NAME##TYPE (TYPE *a, int n)	\
+#define TEST_MAXMIN(T)				\
+  T (int8_t, max, >)				\
+  T (int16_t, max, >)				\
+  T (int32_t, max, >)				\
+  T (int64_t, max, >)				\
+  T (uint8_t, max, >)				\
+  T (uint16_t, max, >)				\
+  T (uint32_t, max, >)				\
+  T (uint64_t, max, >)				\
+  T (_Float16, max, >)				\
+  T (float, max, >)				\
+  T (double, max, >)				\
+						\
+  T (int8_t, min, <)				\
+  T (int16_t, min, <)				\
+  T (int32_t, min, <)				\
+  T (int64_t, min, <)				\
+  T (uint8_t, min, <)				\
+  T (uint16_t, min, <)				\
+  T (uint32_t, min, <)				\
+  T (uint64_t, min, <)				\
+  T (_Float16, min, <)				\
+  T (float, min, <)				\
+  T (double, min, <)
+
+TEST_MAXMIN (DEF_REDUC_MAXMIN)
+
+#define DEF_REDUC_BITWISE(TYPE, NAME, BIT_OP)	\
+TYPE __attribute__ ((noinline, noclone))	\
+reduc_##NAME##_##TYPE (TYPE *a, int n)		\
 {						\
   TYPE r = 13;					\
   for (int i = 0; i < n; ++i)			\
@@ -63,80 +75,93 @@ TYPE reduc_##NAME##TYPE (TYPE *a, int n)	\
   return r;					\
 }
 
-DEF_REDUC_BITWISE (int8_t, and, &=)
-DEF_REDUC_BITWISE (int16_t, and, &=)
-DEF_REDUC_BITWISE (int32_t, and, &=)
-DEF_REDUC_BITWISE (int64_t, and, &=)
-DEF_REDUC_BITWISE (uint8_t, and, &=)
-DEF_REDUC_BITWISE (uint16_t, and, &=)
-DEF_REDUC_BITWISE (uint32_t, and, &=)
-DEF_REDUC_BITWISE (uint64_t, and, &=)
-
-DEF_REDUC_BITWISE (int8_t, ior, |=)
-DEF_REDUC_BITWISE (int16_t, ior, |=)
-DEF_REDUC_BITWISE (int32_t, ior, |=)
-DEF_REDUC_BITWISE (int64_t, ior, |=)
-DEF_REDUC_BITWISE (uint8_t, ior, |=)
-DEF_REDUC_BITWISE (uint16_t, ior, |=)
-DEF_REDUC_BITWISE (uint32_t, ior, |=)
-DEF_REDUC_BITWISE (uint64_t, ior, |=)
-
-DEF_REDUC_BITWISE (int8_t, xor, ^=)
-DEF_REDUC_BITWISE (int16_t, xor, ^=)
-DEF_REDUC_BITWISE (int32_t, xor, ^=)
-DEF_REDUC_BITWISE (int64_t, xor, ^=)
-DEF_REDUC_BITWISE (uint8_t, xor, ^=)
-DEF_REDUC_BITWISE (uint16_t, xor, ^=)
-DEF_REDUC_BITWISE (uint32_t, xor, ^=)
-DEF_REDUC_BITWISE (uint64_t, xor, ^=)
-
-/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 2 } } */
-
-/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 1 } } */
-
-/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 1 } } */
-
-/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 1 } } */
-
-/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 1 } } */
-
-/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 1 } } */
-
-/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 2 } } */
-
-/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 2 } } */
-
-/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.b, p[0-7]/m, z[0-9]\.b, z[0-9]\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.h, p[0-7]/m, z[0-9]\.h, z[0-9]\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.s, p[0-7]/m, z[0-9]\.s, z[0-9]\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.d, p[0-7]/m, z[0-9]\.d, z[0-9]\.d\n} 2 } } */
+#define TEST_BITWISE(T)				\
+  T (int8_t, and, &=)				\
+  T (int16_t, and, &=)				\
+  T (int32_t, and, &=)				\
+  T (int64_t, and, &=)				\
+  T (uint8_t, and, &=)				\
+  T (uint16_t, and, &=)				\
+  T (uint32_t, and, &=)				\
+  T (uint64_t, and, &=)				\
+						\
+  T (int8_t, ior, |=)				\
+  T (int16_t, ior, |=)				\
+  T (int32_t, ior, |=)				\
+  T (int64_t, ior, |=)				\
+  T (uint8_t, ior, |=)				\
+  T (uint16_t, ior, |=)				\
+  T (uint32_t, ior, |=)				\
+  T (uint64_t, ior, |=)				\
+						\
+  T (int8_t, xor, ^=)				\
+  T (int16_t, xor, ^=)				\
+  T (int32_t, xor, ^=)				\
+  T (int64_t, xor, ^=)				\
+  T (uint8_t, xor, ^=)				\
+  T (uint16_t, xor, ^=)				\
+  T (uint32_t, xor, ^=)				\
+  T (uint64_t, xor, ^=)
+
+TEST_BITWISE (DEF_REDUC_BITWISE)
+
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tadd\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsmin\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsmax\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumin\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tumax\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tand\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\torr\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\teor\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
 
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.b\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfaddv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfaddv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
 
@@ -148,6 +173,7 @@ DEF_REDUC_BITWISE (uint64_t, xor, ^=)
 /* { dg-final { scan-assembler-times {\tumaxv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tumaxv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tumaxv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnmv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
 
@@ -159,6 +185,7 @@ DEF_REDUC_BITWISE (uint64_t, xor, ^=)
 /* { dg-final { scan-assembler-times {\tuminv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuminv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuminv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnmv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_1_run.C b/gcc/testsuite/gcc.target/aarch64/sve_reduc_1_run.C
deleted file mode 100644
index 17c978de7f7..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_reduc_1_run.C
+++ /dev/null
@@ -1,117 +0,0 @@
-/* { dg-do run { target { aarch64_sve_hw } } } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -ffast-math -fno-inline -march=armv8-a+sve" } */
-
-#include "sve_reduc_1.C"
-
-#include <stdlib.h>
-#include <stdio.h>
-
-#define NUM_ELEMS(TYPE) (73 + sizeof (TYPE))
-
-#define DEF_INIT_VECTOR(TYPE)				\
-  TYPE r_##TYPE[NUM_ELEMS (TYPE) + 1];			\
-  for (int i = 0; i < NUM_ELEMS (TYPE) + 1; i++)	\
-    r_##TYPE[i] = (i * 2) * (i & 1 ? 1 : -1);
-
-#define TEST_REDUC_PLUS(RES,TYPE) \
-  (RES) += reduc_plus_##TYPE (r_##TYPE, NUM_ELEMS (TYPE));
-#define TEST_REDUC_MAX(RES,TYPE) \
-  (RES) += reduc_max##TYPE (r_##TYPE, NUM_ELEMS (TYPE));
-#define TEST_REDUC_MIN(RES,TYPE) \
-  (RES) += reduc_min##TYPE (r_##TYPE, NUM_ELEMS (TYPE));
-#define TEST_REDUC_AND(RES,TYPE) \
-  (RES) += reduc_and##TYPE (r_##TYPE, NUM_ELEMS (TYPE));
-#define TEST_REDUC_IOR(RES,TYPE) \
-  (RES) += reduc_ior##TYPE (r_##TYPE, NUM_ELEMS (TYPE));
-#define TEST_REDUC_XOR(RES,TYPE) \
-  (RES) += reduc_xor##TYPE (r_##TYPE, NUM_ELEMS (TYPE));
-
-int main ()
-{
-  int result = 0;
-  double resultF = 0.0;
-  DEF_INIT_VECTOR (int8_t)
-  DEF_INIT_VECTOR (int16_t)
-  DEF_INIT_VECTOR (int32_t)
-  DEF_INIT_VECTOR (int64_t)
-  DEF_INIT_VECTOR (uint8_t)
-  DEF_INIT_VECTOR (uint16_t)
-  DEF_INIT_VECTOR (uint32_t)
-  DEF_INIT_VECTOR (uint64_t)
-  DEF_INIT_VECTOR (float)
-  DEF_INIT_VECTOR (double)
-
-  TEST_REDUC_PLUS (result, int8_t)
-  TEST_REDUC_PLUS (result, int16_t)
-  TEST_REDUC_PLUS (result, int32_t)
-  TEST_REDUC_PLUS (result, int64_t)
-  TEST_REDUC_PLUS (result, uint8_t)
-  TEST_REDUC_PLUS (result, uint16_t)
-  TEST_REDUC_PLUS (result, uint32_t)
-  TEST_REDUC_PLUS (result, uint64_t)
-  TEST_REDUC_PLUS (resultF, float)
-  TEST_REDUC_PLUS (resultF, double)
-
-  TEST_REDUC_MIN (result, int8_t)
-  TEST_REDUC_MIN (result, int16_t)
-  TEST_REDUC_MIN (result, int32_t)
-  TEST_REDUC_MIN (result, int64_t)
-  TEST_REDUC_MIN (result, uint8_t)
-  TEST_REDUC_MIN (result, uint16_t)
-  TEST_REDUC_MIN (result, uint32_t)
-  TEST_REDUC_MIN (result, uint64_t)
-  TEST_REDUC_MIN (resultF, float)
-  TEST_REDUC_MIN (resultF, double)
-
-  TEST_REDUC_MAX (result, int8_t)
-  TEST_REDUC_MAX (result, int16_t)
-  TEST_REDUC_MAX (result, int32_t)
-  TEST_REDUC_MAX (result, int64_t)
-  TEST_REDUC_MAX (result, uint8_t)
-  TEST_REDUC_MAX (result, uint16_t)
-  TEST_REDUC_MAX (result, uint32_t)
-  TEST_REDUC_MAX (result, uint64_t)
-  TEST_REDUC_MAX (resultF, float)
-  TEST_REDUC_MAX (resultF, double)
-
-  TEST_REDUC_AND (result, int8_t)
-  TEST_REDUC_AND (result, int16_t)
-  TEST_REDUC_AND (result, int32_t)
-  TEST_REDUC_AND (result, int64_t)
-  TEST_REDUC_AND (result, uint8_t)
-  TEST_REDUC_AND (result, uint16_t)
-  TEST_REDUC_AND (result, uint32_t)
-  TEST_REDUC_AND (result, uint64_t)
-
-  TEST_REDUC_IOR (result, int8_t)
-  TEST_REDUC_IOR (result, int16_t)
-  TEST_REDUC_IOR (result, int32_t)
-  TEST_REDUC_IOR (result, int64_t)
-  TEST_REDUC_IOR (result, uint8_t)
-  TEST_REDUC_IOR (result, uint16_t)
-  TEST_REDUC_IOR (result, uint32_t)
-  TEST_REDUC_IOR (result, uint64_t)
-
-  TEST_REDUC_XOR (result, int8_t)
-  TEST_REDUC_XOR (result, int16_t)
-  TEST_REDUC_XOR (result, int32_t)
-  TEST_REDUC_XOR (result, int64_t)
-  TEST_REDUC_XOR (result, uint8_t)
-  TEST_REDUC_XOR (result, uint16_t)
-  TEST_REDUC_XOR (result, uint32_t)
-  TEST_REDUC_XOR (result, uint64_t)
-
-  if (result != 262400)
-    {
-      fprintf (stderr, "result = %d\n", result);
-      abort ();
-    }
-
-  if (resultF != -160)
-    {
-      fprintf (stderr, "resultF = %1.16lf\n", resultF);
-      abort ();
-    }
-
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_reduc_1_run.c
new file mode 100644
index 00000000000..9f4afbcf3a7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_reduc_1_run.c
@@ -0,0 +1,56 @@
+/* { dg-do run { target { aarch64_sve_hw } } } */
+/* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
+
+#include "sve_reduc_1.c"
+
+#define NUM_ELEMS(TYPE) (73 + sizeof (TYPE))
+
+#define INIT_VECTOR(TYPE)				\
+  TYPE a[NUM_ELEMS (TYPE) + 1];				\
+  for (int i = 0; i < NUM_ELEMS (TYPE) + 1; i++)	\
+    {							\
+      a[i] = ((i * 2) * (i & 1 ? 1 : -1) | 3);		\
+      asm volatile ("" ::: "memory");			\
+    }
+
+#define TEST_REDUC_PLUS(TYPE)				\
+  {							\
+    INIT_VECTOR (TYPE);					\
+    TYPE r1 = reduc_plus_##TYPE (a, NUM_ELEMS (TYPE));	\
+    volatile TYPE r2 = 0;				\
+    for (int i = 0; i < NUM_ELEMS (TYPE); ++i)		\
+      r2 += a[i];					\
+    if (r1 != r2)					\
+      __builtin_abort ();				\
+  }
+
+#define TEST_REDUC_MAXMIN(TYPE, NAME, CMP_OP)			\
+  {								\
+    INIT_VECTOR (TYPE);						\
+    TYPE r1 = reduc_##NAME##_##TYPE (a, NUM_ELEMS (TYPE));	\
+    volatile TYPE r2 = 13;					\
+    for (int i = 0; i < NUM_ELEMS (TYPE); ++i)			\
+      r2 = a[i] CMP_OP r2 ? a[i] : r2;				\
+    if (r1 != r2)						\
+      __builtin_abort ();					\
+  }
+
+#define TEST_REDUC_BITWISE(TYPE, NAME, BIT_OP)			\
+  {								\
+    INIT_VECTOR (TYPE);						\
+    TYPE r1 = reduc_##NAME##_##TYPE (a, NUM_ELEMS (TYPE));	\
+    volatile TYPE r2 = 13;					\
+    for (int i = 0; i < NUM_ELEMS (TYPE); ++i)			\
+      r2 BIT_OP a[i];						\
+    if (r1 != r2)						\
+      __builtin_abort ();					\
+  }
+
+int main ()
+{
+  TEST_PLUS (TEST_REDUC_PLUS)
+  TEST_MAXMIN (TEST_REDUC_MAXMIN)
+  TEST_BITWISE (TEST_REDUC_BITWISE)
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_2.C b/gcc/testsuite/gcc.target/aarch64/sve_reduc_2.c
index 6ac37570164..669306549d3 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_reduc_2.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_reduc_2.c
@@ -1,109 +1,126 @@
 /* { dg-do compile } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -ffast-math -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
 
 #include <stdint.h>
 
 #define NUM_ELEMS(TYPE) (1024 / sizeof (TYPE))
 
-#define DEF_REDUC_PLUS(TYPE)						\
-void reduc_plus_##TYPE (TYPE (*__restrict__ a)[NUM_ELEMS (TYPE)],	\
-			TYPE *__restrict__ r, int n)			\
-{									\
-  for (int i = 0; i < n; i++)						\
-    {									\
-      r[i] = 0;								\
-      for (int j = 0; j < NUM_ELEMS (TYPE); j++)			\
-        r[i] += a[i][j];						\
-    }									\
+#define DEF_REDUC_PLUS(TYPE)					\
+void __attribute__ ((noinline, noclone))			\
+reduc_plus_##TYPE (TYPE (*restrict a)[NUM_ELEMS (TYPE)],	\
+		   TYPE *restrict r, int n)			\
+{								\
+  for (int i = 0; i < n; i++)					\
+    {								\
+      r[i] = 0;							\
+      for (int j = 0; j < NUM_ELEMS (TYPE); j++)		\
+        r[i] += a[i][j];					\
+    }								\
 }
 
-DEF_REDUC_PLUS (int8_t)
-DEF_REDUC_PLUS (int16_t)
-DEF_REDUC_PLUS (int32_t)
-DEF_REDUC_PLUS (int64_t)
-DEF_REDUC_PLUS (uint8_t)
-DEF_REDUC_PLUS (uint16_t)
-DEF_REDUC_PLUS (uint32_t)
-DEF_REDUC_PLUS (uint64_t)
-DEF_REDUC_PLUS (float)
-DEF_REDUC_PLUS (double)
-
-#define DEF_REDUC_MAXMIN(TYPE,NAME,CMP_OP)				\
-void reduc_##NAME##TYPE (TYPE (*__restrict__ a)[NUM_ELEMS (TYPE)],	\
-			 TYPE *__restrict__ r, int n)			\
-{									\
-  for (int i = 0; i < n; i++)						\
-    {									\
-      r[i] = a[i][0];							\
-      for (int j = 0; j < NUM_ELEMS (TYPE); j++)			\
-        r[i] = a[i][j] CMP_OP r[i] ? a[i][j] : r[i];			\
-    }									\
+#define TEST_PLUS(T)				\
+  T (int8_t)					\
+  T (int16_t)					\
+  T (int32_t)					\
+  T (int64_t)					\
+  T (uint8_t)					\
+  T (uint16_t)					\
+  T (uint32_t)					\
+  T (uint64_t)					\
+  T (_Float16)					\
+  T (float)					\
+  T (double)
+
+TEST_PLUS (DEF_REDUC_PLUS)
+
+#define DEF_REDUC_MAXMIN(TYPE, NAME, CMP_OP)			\
+void __attribute__ ((noinline, noclone))			\
+reduc_##NAME##_##TYPE (TYPE (*restrict a)[NUM_ELEMS (TYPE)],	\
+		       TYPE *restrict r, int n)			\
+{								\
+  for (int i = 0; i < n; i++)					\
+    {								\
+      r[i] = a[i][0];						\
+      for (int j = 0; j < NUM_ELEMS (TYPE); j++)		\
+        r[i] = a[i][j] CMP_OP r[i] ? a[i][j] : r[i];		\
+    }								\
 }
 
-DEF_REDUC_MAXMIN (int8_t, max, >)
-DEF_REDUC_MAXMIN (int16_t, max, >)
-DEF_REDUC_MAXMIN (int32_t, max, >)
-DEF_REDUC_MAXMIN (int64_t, max, >)
-DEF_REDUC_MAXMIN (uint8_t, max, >)
-DEF_REDUC_MAXMIN (uint16_t, max, >)
-DEF_REDUC_MAXMIN (uint32_t, max, >)
-DEF_REDUC_MAXMIN (uint64_t, max, >)
-DEF_REDUC_MAXMIN (float, max, >)
-DEF_REDUC_MAXMIN (double, max, >)
-
-DEF_REDUC_MAXMIN (int8_t, min, <)
-DEF_REDUC_MAXMIN (int16_t, min, <)
-DEF_REDUC_MAXMIN (int32_t, min, <)
-DEF_REDUC_MAXMIN (int64_t, min, <)
-DEF_REDUC_MAXMIN (uint8_t, min, <)
-DEF_REDUC_MAXMIN (uint16_t, min, <)
-DEF_REDUC_MAXMIN (uint32_t, min, <)
-DEF_REDUC_MAXMIN (uint64_t, min, <)
-DEF_REDUC_MAXMIN (float, min, <)
-DEF_REDUC_MAXMIN (double, min, <)
-
-#define DEF_REDUC_BITWISE(TYPE,NAME,BIT_OP)\
-void reduc_##NAME##TYPE (TYPE (*__restrict__ a)[NUM_ELEMS(TYPE)], TYPE *__restrict__ r, int n)\
-{\
-  for (int i = 0; i < n; i++)\
-    {\
-      r[i] = a[i][0];\
-      for (int j = 0; j < NUM_ELEMS(TYPE); j++)\
-        r[i] BIT_OP a[i][j];\
-    }\
-}\
-
-DEF_REDUC_BITWISE (int8_t, and, &=)
-DEF_REDUC_BITWISE (int16_t, and, &=)
-DEF_REDUC_BITWISE (int32_t, and, &=)
-DEF_REDUC_BITWISE (int64_t, and, &=)
-DEF_REDUC_BITWISE (uint8_t, and, &=)
-DEF_REDUC_BITWISE (uint16_t, and, &=)
-DEF_REDUC_BITWISE (uint32_t, and, &=)
-DEF_REDUC_BITWISE (uint64_t, and, &=)
-
-DEF_REDUC_BITWISE (int8_t, ior, |=)
-DEF_REDUC_BITWISE (int16_t, ior, |=)
-DEF_REDUC_BITWISE (int32_t, ior, |=)
-DEF_REDUC_BITWISE (int64_t, ior, |=)
-DEF_REDUC_BITWISE (uint8_t, ior, |=)
-DEF_REDUC_BITWISE (uint16_t, ior, |=)
-DEF_REDUC_BITWISE (uint32_t, ior, |=)
-DEF_REDUC_BITWISE (uint64_t, ior, |=)
-
-DEF_REDUC_BITWISE (int8_t, xor, ^=)
-DEF_REDUC_BITWISE (int16_t, xor, ^=)
-DEF_REDUC_BITWISE (int32_t, xor, ^=)
-DEF_REDUC_BITWISE (int64_t, xor, ^=)
-DEF_REDUC_BITWISE (uint8_t, xor, ^=)
-DEF_REDUC_BITWISE (uint16_t, xor, ^=)
-DEF_REDUC_BITWISE (uint32_t, xor, ^=)
-DEF_REDUC_BITWISE (uint64_t, xor, ^=)
+#define TEST_MAXMIN(T)				\
+  T (int8_t, max, >)				\
+  T (int16_t, max, >)				\
+  T (int32_t, max, >)				\
+  T (int64_t, max, >)				\
+  T (uint8_t, max, >)				\
+  T (uint16_t, max, >)				\
+  T (uint32_t, max, >)				\
+  T (uint64_t, max, >)				\
+  T (_Float16, max, >)				\
+  T (float, max, >)				\
+  T (double, max, >)				\
+						\
+  T (int8_t, min, <)				\
+  T (int16_t, min, <)				\
+  T (int32_t, min, <)				\
+  T (int64_t, min, <)				\
+  T (uint8_t, min, <)				\
+  T (uint16_t, min, <)				\
+  T (uint32_t, min, <)				\
+  T (uint64_t, min, <)				\
+  T (_Float16, min, <)				\
+  T (float, min, <)				\
+  T (double, min, <)
+
+TEST_MAXMIN (DEF_REDUC_MAXMIN)
+
+#define DEF_REDUC_BITWISE(TYPE,NAME,BIT_OP)			\
+void __attribute__ ((noinline, noclone))			\
+reduc_##NAME##TYPE (TYPE (*restrict a)[NUM_ELEMS(TYPE)],	\
+		    TYPE *restrict r, int n)			\
+{								\
+  for (int i = 0; i < n; i++)					\
+    {								\
+      r[i] = a[i][0];						\
+      for (int j = 0; j < NUM_ELEMS(TYPE); j++)			\
+        r[i] BIT_OP a[i][j];					\
+    }								\
+}
+
+#define TEST_BITWISE(T)				\
+  T (int8_t, and, &=)				\
+  T (int16_t, and, &=)				\
+  T (int32_t, and, &=)				\
+  T (int64_t, and, &=)				\
+  T (uint8_t, and, &=)				\
+  T (uint16_t, and, &=)				\
+  T (uint32_t, and, &=)				\
+  T (uint64_t, and, &=)				\
+						\
+  T (int8_t, ior, |=)				\
+  T (int16_t, ior, |=)				\
+  T (int32_t, ior, |=)				\
+  T (int64_t, ior, |=)				\
+  T (uint8_t, ior, |=)				\
+  T (uint16_t, ior, |=)				\
+  T (uint32_t, ior, |=)				\
+  T (uint64_t, ior, |=)				\
+						\
+  T (int8_t, xor, ^=)				\
+  T (int16_t, xor, ^=)				\
+  T (int32_t, xor, ^=)				\
+  T (int64_t, xor, ^=)				\
+  T (uint8_t, xor, ^=)				\
+  T (uint16_t, xor, ^=)				\
+  T (uint32_t, xor, ^=)				\
+  T (uint64_t, xor, ^=)
+
+TEST_BITWISE (DEF_REDUC_BITWISE)
 
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.b\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfaddv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfaddv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
 
@@ -115,6 +132,7 @@ DEF_REDUC_BITWISE (uint64_t, xor, ^=)
 /* { dg-final { scan-assembler-times {\tumaxv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tumaxv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tumaxv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnmv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
 
@@ -126,6 +144,7 @@ DEF_REDUC_BITWISE (uint64_t, xor, ^=)
 /* { dg-final { scan-assembler-times {\tuminv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuminv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tuminv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnmv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_2_run.C b/gcc/testsuite/gcc.target/aarch64/sve_reduc_2_run.C
deleted file mode 100644
index 6f170fb0de6..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_reduc_2_run.C
+++ /dev/null
@@ -1,135 +0,0 @@
-/* { dg-do run { target { aarch64_sve_hw } } } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -ffast-math -fno-inline -march=armv8-a+sve" } */
-
-#include "sve_reduc_2.C"
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <math.h>
-
-#define NROWS 5
-
-#define DEF_INIT_VECTOR(TYPE)					\
-  TYPE mat_##TYPE[NROWS][NUM_ELEMS (TYPE)];			\
-  TYPE r_##TYPE[NROWS];						\
-  for (int i = 0; i < NROWS; i++)				\
-    for (int j = 0; j < NUM_ELEMS (TYPE); j++ )			\
-      mat_##TYPE[i][j] = i + (j * 2) * (j & 1 ? 1 : -1);
-
-#define TEST_REDUC_PLUS(TYPE) reduc_plus_##TYPE (mat_##TYPE, r_##TYPE, NROWS);
-#define TEST_REDUC_MAX(TYPE)  reduc_max##TYPE   (mat_##TYPE, r_##TYPE, NROWS);
-#define TEST_REDUC_MIN(TYPE)  reduc_min##TYPE   (mat_##TYPE, r_##TYPE, NROWS);
-#define TEST_REDUC_AND(TYPE)  reduc_and##TYPE   (mat_##TYPE, r_##TYPE, NROWS);
-#define TEST_REDUC_IOR(TYPE)  reduc_ior##TYPE   (mat_##TYPE, r_##TYPE, NROWS);
-#define TEST_REDUC_XOR(TYPE)  reduc_xor##TYPE   (mat_##TYPE, r_##TYPE, NROWS);
-
-#define SUM_VECTOR(RES, TYPE)\
-  for (int i = 0; i < NROWS; i++)\
-    (RES) += r_##TYPE[i];
-
-#define SUM_INT_RESULT(RES)\
-  SUM_VECTOR (RES, int8_t);\
-  SUM_VECTOR (RES, int16_t);\
-  SUM_VECTOR (RES, int32_t);\
-  SUM_VECTOR (RES, int64_t);\
-  SUM_VECTOR (RES, uint8_t);\
-  SUM_VECTOR (RES, uint16_t);\
-  SUM_VECTOR (RES, uint32_t);\
-  SUM_VECTOR (RES, uint64_t);\
-
-#define SUM_FLOAT_RESULT(RES)\
-  SUM_VECTOR (RES, float);\
-  SUM_VECTOR (RES, double);\
-
-int main ()
-{
-  int result = 0;
-  double resultF = 0.0;
-  DEF_INIT_VECTOR (int8_t)
-  DEF_INIT_VECTOR (int16_t)
-  DEF_INIT_VECTOR (int32_t)
-  DEF_INIT_VECTOR (int64_t)
-  DEF_INIT_VECTOR (uint8_t)
-  DEF_INIT_VECTOR (uint16_t)
-  DEF_INIT_VECTOR (uint32_t)
-  DEF_INIT_VECTOR (uint64_t)
-  DEF_INIT_VECTOR (float)
-  DEF_INIT_VECTOR (double)
-
-  TEST_REDUC_PLUS (int8_t)
-  TEST_REDUC_PLUS (int16_t)
-  TEST_REDUC_PLUS (int32_t)
-  TEST_REDUC_PLUS (int64_t)
-  TEST_REDUC_PLUS (uint8_t)
-  TEST_REDUC_PLUS (uint16_t)
-  TEST_REDUC_PLUS (uint32_t)
-  TEST_REDUC_PLUS (uint64_t)
-  TEST_REDUC_PLUS (float)
-  TEST_REDUC_PLUS (double)
-
-  SUM_INT_RESULT (result);
-  SUM_FLOAT_RESULT (resultF);
-
-  TEST_REDUC_MIN (int8_t)
-  TEST_REDUC_MIN (int16_t)
-  TEST_REDUC_MIN (int32_t)
-  TEST_REDUC_MIN (int64_t)
-  TEST_REDUC_MIN (uint8_t)
-  TEST_REDUC_MIN (uint16_t)
-  TEST_REDUC_MIN (uint32_t)
-  TEST_REDUC_MIN (uint64_t)
-  TEST_REDUC_MIN (float)
-  TEST_REDUC_MIN (double)
-
-  TEST_REDUC_MAX (int8_t)
-  TEST_REDUC_MAX (int16_t)
-  TEST_REDUC_MAX (int32_t)
-  TEST_REDUC_MAX (int64_t)
-  TEST_REDUC_MAX (uint8_t)
-  TEST_REDUC_MAX (uint16_t)
-  TEST_REDUC_MAX (uint32_t)
-  TEST_REDUC_MAX (uint64_t)
-  TEST_REDUC_MAX (float)
-  TEST_REDUC_MAX (double)
-
-  TEST_REDUC_AND (int8_t)
-  TEST_REDUC_AND (int16_t)
-  TEST_REDUC_AND (int32_t)
-  TEST_REDUC_AND (int64_t)
-  TEST_REDUC_AND (uint8_t)
-  TEST_REDUC_AND (uint16_t)
-  TEST_REDUC_AND (uint32_t)
-  TEST_REDUC_AND (uint64_t)
-
-  TEST_REDUC_IOR (int8_t)
-  TEST_REDUC_IOR (int16_t)
-  TEST_REDUC_IOR (int32_t)
-  TEST_REDUC_IOR (int64_t)
-  TEST_REDUC_IOR (uint8_t)
-  TEST_REDUC_IOR (uint16_t)
-  TEST_REDUC_IOR (uint32_t)
-  TEST_REDUC_IOR (uint64_t)
-
-  TEST_REDUC_XOR (int8_t)
-  TEST_REDUC_XOR (int16_t)
-  TEST_REDUC_XOR (int32_t)
-  TEST_REDUC_XOR (int64_t)
-  TEST_REDUC_XOR (uint8_t)
-  TEST_REDUC_XOR (uint16_t)
-  TEST_REDUC_XOR (uint32_t)
-  TEST_REDUC_XOR (uint64_t)
-
-  if (result != 26880)
-    {
-      fprintf (stderr, "result = %d\n", result);
-      abort ();
-    }
-
-  if (resultF != double (5760))
-    {
-      fprintf (stderr, "resultF = %1.16lf\n", resultF);
-      abort ();
-    }
-
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_2_run.c b/gcc/testsuite/gcc.target/aarch64/sve_reduc_2_run.c
new file mode 100644
index 00000000000..041db66c8cf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_reduc_2_run.c
@@ -0,0 +1,79 @@
+/* { dg-do run { target { aarch64_sve_hw } } } */
+/* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
+
+#include "sve_reduc_2.c"
+
+#define NROWS 53
+
+/* -ffast-math fuzz for PLUS.  */
+#define CMP__Float16(X, Y) ((X) >= (Y) * 0.875 && (X) <= (Y) * 1.125)
+#define CMP_float(X, Y) ((X) == (Y))
+#define CMP_double(X, Y) ((X) == (Y))
+#define CMP_int8_t(X, Y) ((X) == (Y))
+#define CMP_int16_t(X, Y) ((X) == (Y))
+#define CMP_int32_t(X, Y) ((X) == (Y))
+#define CMP_int64_t(X, Y) ((X) == (Y))
+#define CMP_uint8_t(X, Y) ((X) == (Y))
+#define CMP_uint16_t(X, Y) ((X) == (Y))
+#define CMP_uint32_t(X, Y) ((X) == (Y))
+#define CMP_uint64_t(X, Y) ((X) == (Y))
+
+#define INIT_MATRIX(TYPE)				\
+  TYPE mat[NROWS][NUM_ELEMS (TYPE)];			\
+  TYPE r[NROWS];					\
+  for (int i = 0; i < NROWS; i++)			\
+    for (int j = 0; j < NUM_ELEMS (TYPE); j++)		\
+      {							\
+	mat[i][j] = i + (j * 2) * (j & 1 ? 1 : -1);	\
+	asm volatile ("" ::: "memory");			\
+      }
+
+#define TEST_REDUC_PLUS(TYPE)				\
+  {							\
+    INIT_MATRIX (TYPE);					\
+    reduc_plus_##TYPE (mat, r, NROWS);			\
+    for (int i = 0; i < NROWS; i++)			\
+      {							\
+	volatile TYPE r2 = 0;				\
+	for (int j = 0; j < NUM_ELEMS (TYPE); ++j)	\
+	  r2 += mat[i][j];				\
+	if (!CMP_##TYPE (r[i], r2))			\
+	  __builtin_abort ();				\
+      }							\
+    }
+
+#define TEST_REDUC_MAXMIN(TYPE, NAME, CMP_OP)		\
+  {							\
+    INIT_MATRIX (TYPE);					\
+    reduc_##NAME##_##TYPE (mat, r, NROWS);		\
+    for (int i = 0; i < NROWS; i++)			\
+      {							\
+	volatile TYPE r2 = mat[i][0];			\
+	for (int j = 0; j < NUM_ELEMS (TYPE); ++j)	\
+	  r2 = mat[i][j] CMP_OP r2 ? mat[i][j] : r2;	\
+	if (r[i] != r2)					\
+	  __builtin_abort ();				\
+      }							\
+    }
+
+#define TEST_REDUC_BITWISE(TYPE, NAME, BIT_OP)		\
+  {							\
+    INIT_MATRIX (TYPE);					\
+    reduc_##NAME##_##TYPE (mat, r, NROWS);		\
+    for (int i = 0; i < NROWS; i++)			\
+      {							\
+	volatile TYPE r2 = mat[i][0];			\
+	for (int j = 0; j < NUM_ELEMS (TYPE); ++j)	\
+	  r2 BIT_OP mat[i][j];				\
+	if (r[i] != r2)					\
+	  __builtin_abort ();				\
+      }							\
+    }
+
+int main ()
+{
+  TEST_PLUS (TEST_REDUC_PLUS)
+  TEST_MAXMIN (TEST_REDUC_MAXMIN)
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_3.c b/gcc/testsuite/gcc.target/aarch64/sve_reduc_3.c
index 9e997adedca..7daf3ae130e 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_reduc_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_reduc_3.c
@@ -1,18 +1,52 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
 
-double
-f (double *restrict a, double *restrict b, int *lookup)
-{
-  double res = 0.0;
-  for (int i = 0; i < 512; ++i)
-    res += a[lookup[i]] * b[i];
-  return res;
+#include <stdint.h>
+
+#define NUM_ELEMS(TYPE) (32 / sizeof (TYPE))
+
+#define REDUC_PTR(DSTTYPE, SRCTYPE)				\
+void reduc_ptr_##DSTTYPE##_##SRCTYPE (DSTTYPE *restrict sum,	\
+				      SRCTYPE *restrict array,	\
+				      int count)		\
+{								\
+  *sum = 0;							\
+  for (int i = 0; i < count; ++i)				\
+    *sum += array[i];						\
 }
 
-/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+.d, p[0-7]/m, } 2 } } */
-/* Check that the vector instructions are the only instructions.  */
-/* { dg-final { scan-assembler-times {\tfmla\t} 2 } } */
-/* { dg-final { scan-assembler-not {\tfadd\t} } } */
-/* { dg-final { scan-assembler-times {\tfaddv\td0,} 1 } } */
-/* { dg-final { scan-assembler-not {\tsel\t} } } */
+REDUC_PTR (int8_t, int8_t)
+REDUC_PTR (int16_t, int16_t)
+
+REDUC_PTR (int32_t, int32_t)
+REDUC_PTR (int64_t, int64_t)
+
+REDUC_PTR (_Float16, _Float16)
+REDUC_PTR (float, float)
+REDUC_PTR (double, double)
+
+/* Widening reductions.  */
+REDUC_PTR (int32_t, int8_t)
+REDUC_PTR (int32_t, int16_t)
+
+REDUC_PTR (int64_t, int8_t)
+REDUC_PTR (int64_t, int16_t)
+REDUC_PTR (int64_t, int32_t)
+
+REDUC_PTR (float, _Float16)
+REDUC_PTR (double, float)
+
+/* Float<>Int conversions */
+REDUC_PTR (_Float16, int16_t)
+REDUC_PTR (float, int32_t)
+REDUC_PTR (double, int64_t)
+
+REDUC_PTR (int16_t, _Float16)
+REDUC_PTR (int32_t, float)
+REDUC_PTR (int64_t, double)
+
+/* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfaddv\th[0-9]+, p[0-7], z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfaddv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_reduc_4.c b/gcc/testsuite/gcc.target/aarch64/sve_reduc_4.c
index 2ba09b14851..9e997adedca 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_reduc_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_reduc_4.c
@@ -1,47 +1,18 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -ftree-vectorize -ffast-math -fno-inline -march=armv8-a+sve" } */
-
-#include <stdint.h>
-
-#define NUM_ELEMS(TYPE) (32 / sizeof (TYPE))
-
-#define REDUC_PTR(DSTTYPE, SRCTYPE)				\
-void reduc_ptr_##DSTTYPE##_##SRCTYPE (DSTTYPE *restrict sum,	\
-				     SRCTYPE *restrict array,	\
-				     int count)			\
-{								\
-  *sum = 0;							\
-  for (int i = 0; i < count; ++i)				\
-    *sum += array[i];						\
+/* { dg-options "-O2 -ftree-vectorize -ffast-math -march=armv8-a+sve" } */
+
+double
+f (double *restrict a, double *restrict b, int *lookup)
+{
+  double res = 0.0;
+  for (int i = 0; i < 512; ++i)
+    res += a[lookup[i]] * b[i];
+  return res;
 }
 
-REDUC_PTR (int8_t, int8_t)
-REDUC_PTR (int16_t, int16_t)
-
-REDUC_PTR (int32_t, int32_t)
-REDUC_PTR (int64_t, int64_t)
-
-REDUC_PTR (float, float)
-REDUC_PTR (double, double)
-
-/* Widening reductions.  */
-REDUC_PTR (int32_t, int8_t)
-REDUC_PTR (int32_t, int16_t)
-
-REDUC_PTR (int64_t, int8_t)
-REDUC_PTR (int64_t, int16_t)
-REDUC_PTR (int64_t, int32_t)
-
-REDUC_PTR (double, float)
-
-/* Float<>Int conversions */
-REDUC_PTR (float, int32_t)
-REDUC_PTR (double, int64_t)
-
-REDUC_PTR (int32_t, float)
-REDUC_PTR (int64_t, double)
-
-/* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.s\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tuaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tfaddv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tfaddv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+.d, p[0-7]/m, } 2 } } */
+/* Check that the vector instructions are the only instructions.  */
+/* { dg-final { scan-assembler-times {\tfmla\t} 2 } } */
+/* { dg-final { scan-assembler-not {\tfadd\t} } } */
+/* { dg-final { scan-assembler-times {\tfaddv\td0,} 1 } } */
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_revb_1.c b/gcc/testsuite/gcc.target/aarch64/sve_revb_1.c
index 2b8c6e523ca..9307200fb05 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_revb_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_revb_1.c
@@ -1,7 +1,9 @@
 /* { dg-do assemble } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int8_t v32qi __attribute__((vector_size (32)));
 
 #define MASK_2(X, Y) (X) ^ (Y), (X + 1) ^ (Y)
 #define MASK_4(X, Y) MASK_2 (X, Y), MASK_2 (X + 2, Y)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_revh_1.c b/gcc/testsuite/gcc.target/aarch64/sve_revh_1.c
index aaa08dc03e2..fb238373c4e 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_revh_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_revh_1.c
@@ -1,7 +1,10 @@
 /* { dg-do assemble } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef unsigned short v16hi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef uint16_t v16hi __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define MASK_2(X, Y) (X) ^ (Y), (X + 1) ^ (Y)
 #define MASK_4(X, Y) MASK_2 (X, Y), MASK_2 (X + 2, Y)
@@ -21,11 +24,13 @@ typedef unsigned short v16hi __attribute__((vector_size (32)));
 
 #define TEST_ALL(T)				\
   T (v16hi, 16, 2)				\
-  T (v16hi, 16, 4)
+  T (v16hi, 16, 4)				\
+  T (v16hf, 16, 2)				\
+  T (v16hf, 16, 4)
 
 TEST_ALL (PERMUTE)
 
 /* { dg-final { scan-assembler-not {\ttbl\t} } } */
 
-/* { dg-final { scan-assembler-times {\trevh\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d} 1 } } */
-/* { dg-final { scan-assembler-times {\trevh\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\trevh\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\trevh\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_revw_1.c b/gcc/testsuite/gcc.target/aarch64/sve_revw_1.c
index ac7ef0ef267..4834e2c2b01 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_revw_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_revw_1.c
@@ -1,7 +1,9 @@
 /* { dg-do assemble } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef unsigned int v8si __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef uint32_t v8si __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
 
 #define MASK_2(X, Y) (X) ^ (Y), (X + 1) ^ (Y)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_shift_1.c b/gcc/testsuite/gcc.target/aarch64/sve_shift_1.c
index 24aa47488ef..b19cd7a3161 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_shift_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_shift_1.c
@@ -1,77 +1,84 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O2 -ftree-vectorize -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve --save-temps" } */
 
-#define DO_REG_OPS(TYPE)						\
-void ashiftr_##TYPE (signed TYPE* dst, signed TYPE src, int count)	\
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] >> src;						\
-}									\
-void lshiftr_##TYPE (unsigned TYPE* dst, unsigned TYPE src, int count)	\
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] >> src;						\
-}									\
-void lshiftl_##TYPE (unsigned TYPE* dst, unsigned TYPE src, int count)	\
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] << src;						\
-}									\
-void vashiftr_##TYPE (signed TYPE* dst, signed TYPE* src, int count)	\
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] >> src[i];						\
-}									\
-void vlshiftr_##TYPE (unsigned TYPE* dst, unsigned TYPE* src, int count) \
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] >> src[i];						\
-}									\
-void vlshiftl_##TYPE (unsigned TYPE* dst, unsigned TYPE* src, int count) \
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] << src[i];						\
+#include <stdint.h>
+
+#define DO_REG_OPS(TYPE)					\
+void ashiftr_##TYPE (TYPE *dst, TYPE src, int count)		\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] >> src;					\
+}								\
+void lshiftr_##TYPE (u##TYPE *dst, u##TYPE src, int count)	\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] >> src;					\
+}								\
+void lshiftl_##TYPE (u##TYPE *dst, u##TYPE src, int count)	\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] << src;					\
+}								\
+void vashiftr_##TYPE (TYPE *dst, TYPE *src, int count)		\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] >> src[i];					\
+}								\
+void vlshiftr_##TYPE (u##TYPE *dst, u##TYPE *src, int count)	\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] >> src[i];					\
+}								\
+void vlshiftl_##TYPE (u##TYPE *dst, u##TYPE *src, int count)	\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] << src[i];					\
 }
 
-#define DO_IMMEDIATE_OPS(VALUE, TYPE, NAME)				\
-void vashiftr_imm_##NAME##_##TYPE (signed TYPE* dst, int count)		\
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] >> VALUE;						\
-}									\
-void vlshiftr_imm_##NAME##_##TYPE (unsigned TYPE* dst, int count)	\
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] >> VALUE;						\
-}									\
-void vlshiftl_imm_##NAME##_##TYPE (unsigned TYPE* dst, int count)	\
-{									\
-  for (int i = 0; i < count; ++i)					\
-    dst[i] = dst[i] << VALUE;						\
+#define DO_IMMEDIATE_OPS(VALUE, TYPE, NAME)			\
+void vashiftr_imm_##NAME##_##TYPE (TYPE *dst, int count)	\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] >> VALUE;					\
+}								\
+void vlshiftr_imm_##NAME##_##TYPE (u##TYPE *dst, int count)	\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] >> VALUE;					\
+}								\
+void vlshiftl_imm_##NAME##_##TYPE (u##TYPE *dst, int count)	\
+{								\
+  for (int i = 0; i < count; ++i)				\
+    dst[i] = dst[i] << VALUE;					\
 }
 
-DO_REG_OPS (int);
+DO_REG_OPS (int32_t);
+DO_REG_OPS (int64_t);
 
-DO_IMMEDIATE_OPS (0, char, 0);
-DO_IMMEDIATE_OPS (5, char, 5);
-DO_IMMEDIATE_OPS (7, char, 7);
+DO_IMMEDIATE_OPS (0, int8_t, 0);
+DO_IMMEDIATE_OPS (5, int8_t, 5);
+DO_IMMEDIATE_OPS (7, int8_t, 7);
 
-DO_IMMEDIATE_OPS (0, short, 0);
-DO_IMMEDIATE_OPS (5, short, 5);
-DO_IMMEDIATE_OPS (15, short, 15);
+DO_IMMEDIATE_OPS (0, int16_t, 0);
+DO_IMMEDIATE_OPS (5, int16_t, 5);
+DO_IMMEDIATE_OPS (15, int16_t, 15);
 
-DO_IMMEDIATE_OPS (0, int, 0);
-DO_IMMEDIATE_OPS (5, int, 5);
-DO_IMMEDIATE_OPS (31, int, 31);
+DO_IMMEDIATE_OPS (0, int32_t, 0);
+DO_IMMEDIATE_OPS (5, int32_t, 5);
+DO_IMMEDIATE_OPS (31, int32_t, 31);
 
-DO_IMMEDIATE_OPS (0, long, 0);
-DO_IMMEDIATE_OPS (5, long, 5);
-DO_IMMEDIATE_OPS (63, long, 63);
+DO_IMMEDIATE_OPS (0, int64_t, 0);
+DO_IMMEDIATE_OPS (5, int64_t, 5);
+DO_IMMEDIATE_OPS (63, int64_t, 63);
 
 /* { dg-final { scan-assembler-times {\tasr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tlsr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tlsl\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
 
+/* { dg-final { scan-assembler-times {\tasr\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlsr\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+
 /* { dg-final { scan-assembler-times {\tasr\tz[0-9]+\.b, z[0-9]+\.b, #5\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tlsr\tz[0-9]+\.b, z[0-9]+\.b, #5\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tlsl\tz[0-9]+\.b, z[0-9]+\.b, #5\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_single_1.c b/gcc/testsuite/gcc.target/aarch64/sve_single_1.c
index e50d7064858..f7aeed06907 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_single_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_single_1.c
@@ -5,25 +5,28 @@
 #define N 32
 #endif
 
-#define TEST_LOOP(NAME, TYPE, VALUE)		\
+#include <stdint.h>
+
+#define TEST_LOOP(TYPE, VALUE)			\
   void						\
-  NAME (TYPE *data)				\
+  test_##TYPE (TYPE *data)			\
   {						\
     _Pragma ("omp simd")			\
     for (int i = 0; i < N / sizeof (TYPE); ++i)	\
       data[i] = VALUE;				\
   }
 
-TEST_LOOP (uc, unsigned char, 1)
-TEST_LOOP (sc, signed char, 2)
-TEST_LOOP (us, unsigned short, 3)
-TEST_LOOP (ss, signed short, 4)
-TEST_LOOP (ui, unsigned int, 5)
-TEST_LOOP (si, signed int, 6)
-TEST_LOOP (ul, unsigned long, 7)
-TEST_LOOP (sl, signed long, 8)
-TEST_LOOP (f, float, 1.0f)
-TEST_LOOP (d, double, 2.0)
+TEST_LOOP (uint8_t, 1)
+TEST_LOOP (int8_t, 2)
+TEST_LOOP (uint16_t, 3)
+TEST_LOOP (int16_t, 4)
+TEST_LOOP (uint32_t, 5)
+TEST_LOOP (int32_t, 6)
+TEST_LOOP (uint64_t, 7)
+TEST_LOOP (int64_t, 8)
+TEST_LOOP (_Float16, 1.0f)
+TEST_LOOP (float, 2.0f)
+TEST_LOOP (double, 3.0)
 
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.b, #1\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.b, #2\n} 1 } } */
@@ -33,16 +36,17 @@ TEST_LOOP (d, double, 2.0)
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.s, #6\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #7\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #8\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #1\.0e\+0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.h, #15360\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #3\.0e\+0\n} 1 } } */
 
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl32\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl16\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl16\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl8\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d, vl4\n} 3 } } */
 
 /* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b,} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d,} 3 } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_single_2.c b/gcc/testsuite/gcc.target/aarch64/sve_single_2.c
index e167782323b..7daea6262d6 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_single_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_single_2.c
@@ -12,16 +12,17 @@
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.s, #6\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #7\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #8\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #1\.0e\+0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.h, #15360\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #3\.0e\+0\n} 1 } } */
 
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl64\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl32\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl32\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl16\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d, vl8\n} 3 } } */
 
 /* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b,} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d,} 3 } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_single_3.c b/gcc/testsuite/gcc.target/aarch64/sve_single_3.c
index 8967586bf50..e779d6c50d9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_single_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_single_3.c
@@ -12,16 +12,17 @@
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.s, #6\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #7\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #8\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #1\.0e\+0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.h, #15360\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #3\.0e\+0\n} 1 } } */
 
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl128\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl64\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl64\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl32\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d, vl16\n} 3 } } */
 
 /* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b,} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d,} 3 } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_single_4.c b/gcc/testsuite/gcc.target/aarch64/sve_single_4.c
index 99e2284b164..7c8b3015551 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_single_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_single_4.c
@@ -12,16 +12,17 @@
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.s, #6\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #7\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, #8\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #1\.0e\+0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.h, #15360\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.s, #2\.0e\+0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmov\tz[0-9]+\.d, #3\.0e\+0\n} 1 } } */
 
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl256\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl128\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl128\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl64\n} 3 } } */
 /* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d, vl32\n} 3 } } */
 
 /* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b,} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s,} 3 } } */
 /* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d,} 3 } } */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_store_scalar_offset_1.c b/gcc/testsuite/gcc.target/aarch64/sve_store_scalar_offset_1.c
index 3fa4f187fa4..3e7367cd9fa 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_store_scalar_offset_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_store_scalar_offset_1.c
@@ -1,53 +1,55 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
-typedef long v4di __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef char v32qi __attribute__((vector_size(32)));
+#include <stdint.h>
 
-void sve_store_64_z_lsl (unsigned long *a, unsigned long i)
+typedef int64_t v4di __attribute__((vector_size(32)));
+typedef int32_t v8si __attribute__((vector_size(32)));
+typedef int16_t v16hi __attribute__((vector_size(32)));
+typedef int8_t v32qi __attribute__((vector_size(32)));
+
+void sve_store_64_z_lsl (uint64_t *a, unsigned long i)
 {
   asm volatile ("" : "=w" (*(v4di *) &a[i]));
 }
 
-void sve_store_64_s_lsl (signed long *a, signed long i)
+void sve_store_64_s_lsl (int64_t *a, signed long i)
 {
   asm volatile ("" : "=w" (*(v4di *) &a[i]));
 }
 
-void sve_store_32_z_lsl (unsigned int *a, unsigned long i)
+void sve_store_32_z_lsl (uint32_t *a, unsigned long i)
 {
   asm volatile ("" : "=w" (*(v8si *) &a[i]));
 }
 
-void sve_store_32_s_lsl (signed int *a, signed long i)
+void sve_store_32_s_lsl (int32_t *a, signed long i)
 {
   asm volatile ("" : "=w" (*(v8si *) &a[i]));
 }
 
-void sve_store_16_z_lsl (unsigned short *a, unsigned long i)
+void sve_store_16_z_lsl (uint16_t *a, unsigned long i)
 {
   asm volatile ("" : "=w" (*(v16hi *) &a[i]));
 }
 
-void sve_store_16_s_lsl (signed short *a, signed long i)
+void sve_store_16_s_lsl (int16_t *a, signed long i)
 {
   asm volatile ("" : "=w" (*(v16hi *) &a[i]));
 }
 
 /* ??? The other argument order leads to a redundant move.  */
-void sve_store_8_z (unsigned long i, unsigned char *a)
+void sve_store_8_z (unsigned long i, uint8_t *a)
 {
   asm volatile ("" : "=w" (*(v32qi *) &a[i]));
 }
 
-void sve_store_8_s (signed long i, signed char *a)
+void sve_store_8_s (signed long i, int8_t *a)
 {
   asm volatile ("" : "=w" (*(v32qi *) &a[i]));
 }
 
-/* { dg-final { scan-assembler-times {\tst1d\tz0.d, p[0-7], \[x0, x1, lsl 3\]\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1w\tz0.s, p[0-7], \[x0, x1, lsl 2\]\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1h\tz0.h, p[0-7], \[x0, x1, lsl 1\]\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tst1b\tz0.b, p[0-7], \[x1, x0\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz0\.d, p[0-7], \[x0, x1, lsl 3\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz0\.s, p[0-7], \[x0, x1, lsl 2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz0\.h, p[0-7], \[x0, x1, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz0\.b, p[0-7], \[x1, x0\]\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_store_scalar_offset_2.c b/gcc/testsuite/gcc.target/aarch64/sve_store_scalar_offset_2.c
deleted file mode 100644
index 586e9726396..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_store_scalar_offset_2.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/* { dg-do assemble } */
-/* { dg-options "-O3 -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
-
-typedef long v4di __attribute__((vector_size(32)));
-typedef int v8si __attribute__((vector_size(32)));
-typedef short v16hi __attribute__((vector_size(32)));
-typedef char v32qi __attribute__((vector_size(32)));
-
-void sve_store_64_z_lsl (unsigned long *a, unsigned long i)
-{
-  asm volatile ("" : "=w" (*(v4di *)&a[i]));
-}
-
-void sve_store_64_s_lsl (signed long *a, signed long i)
-{
-  asm volatile ("" : "=w" (*(v4di *)&a[i]));
-}
-
-void sve_store_32_z_lsl (unsigned int *a, unsigned long i)
-{
-  asm volatile ("" : "=w" (*(v8si *)&a[i]));
-}
-
-void sve_store_32_s_lsl (signed int *a, signed long i)
-{
-  asm volatile ("" : "=w" (*(v8si *)&a[i]));
-}
-
-void sve_store_16_z_lsl (unsigned short *a, unsigned long i)
-{
-  asm volatile ("" : "=w" (*(v16hi *)&a[i]));
-}
-
-void sve_store_16_s_lsl (signed short *a, signed long i)
-{
-  asm volatile ("" : "=w" (*(v16hi *)&a[i]));
-}
-
-/* ??? The other argument order leads to a redundant move.  */
-void sve_store_8_z (unsigned long i, unsigned char *a)
-{
-  asm volatile ("" : "=w" (*(v32qi *)&a[i]));
-}
-
-void sve_store_8_s (signed long i, signed char *a)
-{
-  asm volatile ("" : "=w" (*(v32qi *)&a[i]));
-}
-
-/* { dg-final { scan-assembler-times "st1d\\tz0.d, p\[0-9\]+, \\\[x0, x1, lsl 3\\\]" 2 } } */
-/* { dg-final { scan-assembler-times "st1w\\tz0.s, p\[0-9\]+, \\\[x0, x1, lsl 2\\\]" 2 } } */
-/* { dg-final { scan-assembler-times "st1h\\tz0.h, p\[0-9\]+, \\\[x0, x1, lsl 1\\\]" 2 } } */
-/* { dg-final { scan-assembler-times "st1b\\tz0.b, p\[0-9\]+, \\\[x1, x0\\\]" 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_subr_1.c b/gcc/testsuite/gcc.target/aarch64/sve_subr_1.c
index de4dbe8c6cc..1d8dc76719d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_subr_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_subr_1.c
@@ -1,28 +1,32 @@
 /* { dg-do assemble } */
-/* { dg-options "-std=c99 -O3 -march=armv8-a+sve --save-temps" } */
+/* { dg-options "-O3 -march=armv8-a+sve --save-temps" } */
+
+#include <stdint.h>
 
 #define DO_IMMEDIATE_OPS(VALUE, TYPE, NAME)			\
-void vsubrarithimm_##NAME##_##TYPE (TYPE* dst, int count)	\
+void vsubr_arithimm_##NAME##_##TYPE (TYPE *dst, int count)	\
 {								\
   for (int i = 0; i < count; ++i)				\
     dst[i] = VALUE - dst[i];					\
 }
 
 #define DO_ARITH_OPS(TYPE)			\
-DO_IMMEDIATE_OPS (0, TYPE, 0);			\
-DO_IMMEDIATE_OPS (5, TYPE, 5);			\
-DO_IMMEDIATE_OPS (255, TYPE, 255);		\
-DO_IMMEDIATE_OPS (256, TYPE, 256);		\
-DO_IMMEDIATE_OPS (257, TYPE, 257);		\
-DO_IMMEDIATE_OPS (65280, TYPE, 65280);		\
-DO_IMMEDIATE_OPS (65281, TYPE, 65281);		\
-DO_IMMEDIATE_OPS (-1, TYPE, minus1);
-
-DO_ARITH_OPS (char)
-DO_ARITH_OPS (int)
-DO_ARITH_OPS (long)
+  DO_IMMEDIATE_OPS (0, TYPE, 0);		\
+  DO_IMMEDIATE_OPS (5, TYPE, 5);		\
+  DO_IMMEDIATE_OPS (255, TYPE, 255);		\
+  DO_IMMEDIATE_OPS (256, TYPE, 256);		\
+  DO_IMMEDIATE_OPS (257, TYPE, 257);		\
+  DO_IMMEDIATE_OPS (65280, TYPE, 65280);	\
+  DO_IMMEDIATE_OPS (65281, TYPE, 65281);	\
+  DO_IMMEDIATE_OPS (-1, TYPE, minus1);
+
+DO_ARITH_OPS (int8_t)
+DO_ARITH_OPS (int16_t)
+DO_ARITH_OPS (int32_t)
+DO_ARITH_OPS (int64_t)
 
 /* { dg-final { scan-assembler-not {\tsub\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tsub\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
 
@@ -35,6 +39,14 @@ DO_ARITH_OPS (long)
 /* { dg-final { scan-assembler-not   {\tsubr\tz[0-9]+\.b, z[0-9]+\.b, #65281\n} } } */
 /* { dg-final { scan-assembler-not   {\tsubr\tz[0-9]+\.b, z[0-9]+\.b, #-1\n} } } */
 
+/* { dg-final { scan-assembler-times {\tsubr\tz[0-9]+\.h, z[0-9]+\.h, #5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsubr\tz[0-9]+\.h, z[0-9]+\.h, #255\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsubr\tz[0-9]+\.h, z[0-9]+\.h, #256\n} 1 } } */
+/* { dg-final { scan-assembler-not   {\tsubr\tz[0-9]+\.h, z[0-9]+\.h, #257\n} } } */
+/* { dg-final { scan-assembler-times {\tsubr\tz[0-9]+\.h, z[0-9]+\.h, #65280\n} 1 } } */
+/* { dg-final { scan-assembler-not   {\tsubr\tz[0-9]+\.h, z[0-9]+\.h, #65281\n} } } */
+/* { dg-final { scan-assembler-not   {\tsubr\tz[0-9]+\.h, z[0-9]+\.h, #-1\n} } } */
+
 /* { dg-final { scan-assembler-times {\tsubr\tz[0-9]+\.s, z[0-9]+\.s, #5\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tsubr\tz[0-9]+\.s, z[0-9]+\.s, #255\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tsubr\tz[0-9]+\.s, z[0-9]+\.s, #256\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_trn1_1.c b/gcc/testsuite/gcc.target/aarch64/sve_trn1_1.c
index c82f30e9578..0c7b887d232 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_trn1_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_trn1_1.c
@@ -5,12 +5,15 @@
 #define BIAS 0
 #endif
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define MASK_2(X, Y) X, Y + X
 #define MASK_4(X, Y) MASK_2 (X, Y), MASK_2 (X + 2, Y)
@@ -37,7 +40,8 @@ typedef float v8sf __attribute__((vector_size (32)));
   T (v16hi, 16)					\
   T (v32qi, 32)					\
   T (v4df, 4)					\
-  T (v8sf, 8)
+  T (v8sf, 8)					\
+  T (v16hf, 16)
 
 TEST_ALL (PERMUTE)
 
@@ -45,5 +49,5 @@ TEST_ALL (PERMUTE)
 
 /* { dg-final { scan-assembler-times {\ttrn1\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d} 2 } } */
 /* { dg-final { scan-assembler-times {\ttrn1\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s} 2 } } */
-/* { dg-final { scan-assembler-times {\ttrn1\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 1 } } */
+/* { dg-final { scan-assembler-times {\ttrn1\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 2 } } */
 /* { dg-final { scan-assembler-times {\ttrn1\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_trn2_1.c b/gcc/testsuite/gcc.target/aarch64/sve_trn2_1.c
index a4b3ea40a21..6654781bbd5 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_trn2_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_trn2_1.c
@@ -8,5 +8,5 @@
 
 /* { dg-final { scan-assembler-times {\ttrn2\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d} 2 } } */
 /* { dg-final { scan-assembler-times {\ttrn2\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s} 2 } } */
-/* { dg-final { scan-assembler-times {\ttrn2\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 1 } } */
+/* { dg-final { scan-assembler-times {\ttrn2\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 2 } } */
 /* { dg-final { scan-assembler-times {\ttrn2\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1.c
index de010318fd1..c415c4bf5d1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1.c
@@ -1,7 +1,10 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
 
-void unpack_double_int_plus8 (double *d, signed int *s, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+unpack_double_int_plus8 (double *d, int32_t *s, int size)
 {
   for (int i = 0; i < size; i++)
     d[i] = s[i] + 8;
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1_run.c
index 083f1b346d3..f8d9cc2b2ca 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_signed_1_run.c
@@ -1,9 +1,5 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_unpack_fcvt_signed_1.c"
 
@@ -15,19 +11,18 @@ int __attribute__ ((optimize (1)))
 main (void)
 {
   double array_dest[ARRAY_SIZE];
-  signed int array_source[ARRAY_SIZE];
+  int32_t array_source[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
-    array_source[i] = VAL1;
+    {
+      array_source[i] = VAL1;
+      asm volatile ("" ::: "memory");
+    }
 
   unpack_double_int_plus8 (array_dest, array_source, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
-    if (array_dest[i] != (float) (VAL1 + 8))
-      {
-	fprintf (stderr,"%d: %f != %f\n", i, array_dest[i],
-		 (float) (VAL1 + 8));
-	exit (1);
-      }
+    if (array_dest[i] != (double) (VAL1 + 8))
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1.c
index cc1b5e576f4..fb9fe810cf9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1.c
@@ -1,7 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-void unpack_double_int_plus9 (double *d, unsigned int *s, int size)
+#include <stdint.h>
+
+void __attribute__ ((noinline, noclone))
+unpack_double_int_plus9 (double *d, uint32_t *s, int size)
 {
   for (int i = 0; i < size; i++)
     d[i] = (double) (s[i] + 9);
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1_run.c
index 1c31e18c410..93788a342ce 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_fcvt_unsigned_1_run.c
@@ -1,9 +1,5 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include "sve_unpack_fcvt_unsigned_1.c"
 
@@ -15,19 +11,18 @@ int __attribute__ ((optimize (1)))
 main (void)
 {
   double array_dest[ARRAY_SIZE];
-  unsigned int array_source[ARRAY_SIZE];
+  uint32_t array_source[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
-    array_source[i] = VAL1;
+    {
+      array_source[i] = VAL1;
+      asm volatile ("" ::: "memory");
+    }
 
   unpack_double_int_plus9 (array_dest, array_source, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
     if (array_dest[i] != (double) (VAL1 + 9))
-      {
-	fprintf (stderr,"%d: %lf != %lf\n", i, array_dest[i],
-		 (double) (VAL1 + 9));
-	exit (1);
-      }
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1.c
index 86bd60918e2..73c7a815e36 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1.c
@@ -1,7 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-void unpack_float_plus_7point9 (double *d, float *s, int size)
+void __attribute__ ((noinline, noclone))
+unpack_float_plus_7point9 (double *d, float *s, int size)
 {
   for (int i = 0; i < size; i++)
     d[i] = s[i] + 7.9;
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1_run.c
index 4e280dd10f9..2a645b33d4b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_float_1_run.c
@@ -1,10 +1,6 @@
 /* { dg-do run { target aarch64_sve_hw } } */
 /* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
 
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-
 #include "sve_unpack_float_1.c"
 
 #define ARRAY_SIZE 199
@@ -18,16 +14,15 @@ main (void)
   float array_source[ARRAY_SIZE];
 
   for (int i = 0; i < ARRAY_SIZE; i++)
-    array_source[i] = VAL1;
+    {
+      array_source[i] = VAL1;
+      asm volatile ("" ::: "memory");
+    }
 
   unpack_float_plus_7point9 (array_dest, array_source, ARRAY_SIZE);
   for (int i = 0; i < ARRAY_SIZE; i++)
     if (array_dest[i] != (double) (VAL1 + 7.9))
-      {
-	fprintf (stderr,"%d: %f != %f\n", i, array_dest[i],
-		 (double) (VAL1 + 7.9));
-	exit (1);
-      }
+      __builtin_abort ();
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1.c
index e4d0393b047..4d345cf81e9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1.c
@@ -1,20 +1,25 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
 
-#define UNPACK(TYPED, TYPES, SIGND)					\
-void unpack_##TYPED##_##TYPES##_##SIGND (SIGND TYPED *d, signed TYPES *s, \
-					 int size)			\
-{									\
-   for (int i = 0; i < size; i++)					\
-     d[i] = s[i] + 1;							\
+#include <stdint.h>
+
+#define UNPACK(TYPED, TYPES)				\
+void __attribute__ ((noinline, noclone))		\
+unpack_##TYPED##_##TYPES (TYPED *d, TYPES *s, int size)	\
+{							\
+  for (int i = 0; i < size; i++)			\
+    d[i] = s[i] + 1;					\
 }
 
-UNPACK (long, int, signed)
-UNPACK (int, short, signed)
-UNPACK (short, char, signed)
-UNPACK (long, int, unsigned)
-UNPACK (int, short, unsigned)
-UNPACK (short, char, unsigned)
+#define TEST_ALL(T)			\
+  T (int64_t, int32_t)			\
+  T (int32_t, int16_t)			\
+  T (int16_t, int8_t)			\
+  T (uint64_t, int32_t)			\
+  T (uint32_t, int16_t)			\
+  T (uint16_t, int8_t)
+
+TEST_ALL (UNPACK)
 
 /* { dg-final { scan-assembler-times {\tsunpkhi\tz[0-9]+\.d, z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tsunpkhi\tz[0-9]+\.s, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1_run.c
index b63aa5b7d1d..d183408d124 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_signed_1_run.c
@@ -1,46 +1,28 @@
 /* { dg-do run { target aarch64_sve_hw } } */
 /* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
 
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-
 #include "sve_unpack_signed_1.c"
 
 #define ARRAY_SIZE 33
 
-#define RUN_AND_CHECK_LOOP(TYPED, TYPES, VALUED, VALUES)		\
-{									\
-  int value = 0;							\
-  TYPED arrayd[ARRAY_SIZE];						\
-  TYPES arrays[ARRAY_SIZE];						\
-  memset (arrayd, 67, ARRAY_SIZE * sizeof (TYPED));			\
-  memset (arrays, VALUES, ARRAY_SIZE * sizeof (TYPES));			\
-  unpack_##TYPED##_##TYPES##_signed (arrayd, arrays, ARRAY_SIZE);	\
-  for (int i = 0; i < ARRAY_SIZE; i++)					\
-    if (arrayd[i] != VALUED)						\
-      {									\
-	fprintf (stderr,"%d: %d != %d\n", i, arrayd[i], VALUED);	\
-	exit (1);							\
-      }									\
-  memset (arrayd, 74, ARRAY_SIZE * sizeof (TYPED));			\
-  unpack_##TYPED##_##TYPES##_unsigned (arrayd, arrays, ARRAY_SIZE);	\
-  for (int i = 0; i < ARRAY_SIZE; i++)					\
-    if (arrayd[i] != VALUED)						\
-      {									\
-	fprintf (stderr,"%d: %d != %d\n", i, arrayd[i], VALUED);	\
-	exit (1);							\
-      }									\
-}
+#define TEST_LOOP(TYPED, TYPES)					\
+  {								\
+    TYPED arrayd[ARRAY_SIZE];					\
+    TYPES arrays[ARRAY_SIZE];					\
+    for (int i = 0; i < ARRAY_SIZE; i++)			\
+      {								\
+	arrays[i] = (i - 10) * 3;				\
+	asm volatile ("" ::: "memory");				\
+      }								\
+    unpack_##TYPED##_##TYPES (arrayd, arrays, ARRAY_SIZE);	\
+    for (int i = 0; i < ARRAY_SIZE; i++)			\
+      if (arrayd[i] != (TYPED) ((TYPES) ((i - 10) * 3) + 1))	\
+	__builtin_abort ();					\
+  }
 
-int main (void)
+int __attribute__ ((optimize (1)))
+main (void)
 {
-  int total = 5;
-  RUN_AND_CHECK_LOOP (short, char, total+1, total);
-  total = (total << 8) + 5;
-  RUN_AND_CHECK_LOOP (int, short, total+1, total);
-  total = (total << 8) + 5;
-  total = (total << 8) + 5;
-  RUN_AND_CHECK_LOOP (long, int, total+1, total);
+  TEST_ALL (TEST_LOOP)
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1.c
index 94192d977f4..fa8de963264 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1.c
@@ -1,20 +1,25 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
 
-#define UNPACK(TYPED, TYPES, SIGND)					\
-void unpack_##TYPED##_##TYPES##_##SIGND (SIGND TYPED *d, unsigned TYPES *s, \
-					 int size)			\
-{									\
-  for (int i = 0; i < size; i++)					\
-    d[i] = s[i] + 1;							\
+#include <stdint.h>
+
+#define UNPACK(TYPED, TYPES)				\
+void __attribute__ ((noinline, noclone))		\
+unpack_##TYPED##_##TYPES (TYPED *d, TYPES *s, int size)	\
+{							\
+  for (int i = 0; i < size; i++)			\
+    d[i] = s[i] + 1;					\
 }
 
-UNPACK (long, int, signed)			\
-UNPACK (int, short, signed)			\
-UNPACK (short, char, signed)			\
-UNPACK (long, int, unsigned)			\
-UNPACK (int, short, unsigned)			\
-UNPACK (short, char, unsigned)
+#define TEST_ALL(T)			\
+  T (int64_t, uint32_t)			\
+  T (int32_t, uint16_t)			\
+  T (int16_t, uint8_t)			\
+  T (uint64_t, uint32_t)		\
+  T (uint32_t, uint16_t)		\
+  T (uint16_t, uint8_t)
+
+TEST_ALL (UNPACK)
 
 /* { dg-final { scan-assembler-times {\tuunpkhi\tz[0-9]+\.d, z[0-9]+\.s\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuunpkhi\tz[0-9]+\.s, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1_run.c
index 33f5f939c84..3fa66220f17 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_unpack_unsigned_1_run.c
@@ -1,46 +1,28 @@
 /* { dg-do run { target aarch64_sve_hw } } */
 /* { dg-options "-O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
 
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-
 #include "sve_unpack_unsigned_1.c"
 
 #define ARRAY_SIZE 85
 
-#define RUN_AND_CHECK_LOOP(TYPED, TYPES, VALUED, VALUES)		\
-{									\
-  int value = 0;							\
-  TYPED arrayd[ARRAY_SIZE];						\
-  TYPES arrays[ARRAY_SIZE];						\
-  memset (arrayd, 67, ARRAY_SIZE * sizeof (TYPED));			\
-  memset (arrays, VALUES, ARRAY_SIZE * sizeof (TYPES));			\
-  unpack_##TYPED##_##TYPES##_signed (arrayd, arrays, ARRAY_SIZE);	\
-  for (int i = 0; i < ARRAY_SIZE; i++)					\
-    if (arrayd[i] != VALUED)						\
-      {									\
-	fprintf (stderr,"%d: %d != %d\n", i, arrayd[i], VALUED);	\
-	exit (1);							\
-      }									\
-  memset (arrayd, 74, ARRAY_SIZE * sizeof (TYPED));			\
-  unpack_##TYPED##_##TYPES##_unsigned (arrayd, arrays, ARRAY_SIZE);	\
-  for (int i = 0; i < ARRAY_SIZE; i++)					\
-    if (arrayd[i] != VALUED)						\
-      {									\
-	fprintf (stderr,"%d: %d != %d\n", i, arrayd[i], VALUED);	\
-	exit (1);							\
-      }									\
-}
+#define TEST_LOOP(TYPED, TYPES)					\
+  {								\
+    TYPED arrayd[ARRAY_SIZE];					\
+    TYPES arrays[ARRAY_SIZE];					\
+    for (int i = 0; i < ARRAY_SIZE; i++)			\
+      {								\
+	arrays[i] = (i - 10) * 3;				\
+	asm volatile ("" ::: "memory");				\
+      }								\
+    unpack_##TYPED##_##TYPES (arrayd, arrays, ARRAY_SIZE);	\
+    for (int i = 0; i < ARRAY_SIZE; i++)			\
+      if (arrayd[i] != (TYPED) ((TYPES) ((i - 10) * 3) + 1))	\
+	__builtin_abort ();					\
+  }
 
-int main (void)
+int __attribute__ ((optimize (1)))
+main (void)
 {
-  int total = 5;
-  RUN_AND_CHECK_LOOP (short, char, total + 1, total);
-  total = (total << 8) + 5;
-  RUN_AND_CHECK_LOOP (int, short, total + 1, total);
-  total = (total << 8) + 5;
-  total = (total << 8) + 5;
-  RUN_AND_CHECK_LOOP (long, int, total + 1, total);
+  TEST_ALL (TEST_LOOP)
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1.c b/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1.c
index 22fc84f066c..aaa4fdccbf0 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define UZP1(TYPE, MASK)				\
 TYPE uzp1_##TYPE (TYPE values1, TYPE values2)		\
@@ -25,6 +28,8 @@ UZP1 (v32qi, ((v32qi) { 0, 2, 4, 6, 8, 10, 12, 14,
 			48, 50, 52, 54, 56, 58, 60, 62 }));
 UZP1 (v4df,  ((v4di) { 0, 2, 4, 6 }));
 UZP1 (v8sf,  ((v8si) { 0, 2, 4, 6, 8, 10, 12, 14 }));
+UZP1 (v16hf, ((v16hi) { 0, 2, 4, 6, 8, 10, 12, 14,
+			16, 18, 20, 22, 24, 26, 28, 30 }));
 
 /* { dg-final { scan-assembler-not {\ttbl\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} } } */
 /* { dg-final { scan-assembler-not {\ttbl\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} } } */
@@ -33,5 +38,5 @@ UZP1 (v8sf,  ((v8si) { 0, 2, 4, 6, 8, 10, 12, 14 }));
 
 /* { dg-final { scan-assembler-times {\tuzp1\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuzp1\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tuzp1\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tuzp1\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuzp1\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1_run.c
index 338670c03af..d35dad0ffca 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_uzp1_1_run.c
@@ -2,7 +2,6 @@
 /* { dg-options "-O -march=armv8-a+sve" } */
 
 #include "sve_uzp1_1.c"
-extern void abort (void);
 
 #define TEST_UZP1(TYPE, EXPECTED_RESULT, VALUES1, VALUES2)		\
 {									\
@@ -12,7 +11,7 @@ extern void abort (void);
   TYPE dest;								\
   dest = uzp1_##TYPE (values1, values2);				\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
 }
 
 int main (void)
@@ -53,5 +52,12 @@ int main (void)
 	     ((v8sf) { 3.0, 5.0, 7.0, 9.0, 33.0, 35.0, 37.0, 39.0 }),
 	     ((v8sf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0 }),
 	     ((v8sf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0 }));
+  TEST_UZP1 (v16hf,
+	     ((v16hf) { 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0, 17.0,
+			33.0, 35.0, 37.0, 39.0, 41.0, 43.0, 45.0, 47.0 }),
+	     ((v16hf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
+			11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0 }),
+	     ((v16hf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0,
+			41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0 }));
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1.c b/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1.c
index 39c8ff43368..1bb84d80eb0 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define UZP2(TYPE, MASK)				\
 TYPE uzp2_##TYPE (TYPE values1, TYPE values2)		\
@@ -24,6 +27,8 @@ UZP2 (v32qi, ((v32qi) { 1, 3, 5, 7, 9, 11, 13, 15,
 			49, 51, 53, 55, 57, 59, 61, 63 }));
 UZP2 (v4df,  ((v4di) { 1, 3, 5, 7 }));
 UZP2 (v8sf,  ((v8si) { 1, 3, 5, 7, 9, 11, 13, 15 }));
+UZP2 (v16hf, ((v16hi) { 1, 3, 5, 7, 9, 11, 13, 15,
+			17, 19, 21, 23, 25, 27, 29, 31 }));
 
 /* { dg-final { scan-assembler-not {\ttbl\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} } } */
 /* { dg-final { scan-assembler-not {\ttbl\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} } } */
@@ -32,5 +37,5 @@ UZP2 (v8sf,  ((v8si) { 1, 3, 5, 7, 9, 11, 13, 15 }));
 
 /* { dg-final { scan-assembler-times {\tuzp2\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuzp2\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tuzp2\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tuzp2\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
 /* { dg-final { scan-assembler-times {\tuzp2\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1_run.c
index b9b8cccfafe..d7a241c1258 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_uzp2_1_run.c
@@ -2,7 +2,6 @@
 /* { dg-options "-O -march=armv8-a+sve" } */
 
 #include "sve_uzp2_1.c"
-extern void abort (void);
 
 #define TEST_UZP2(TYPE, EXPECTED_RESULT, VALUES1, VALUES2)		\
 {									\
@@ -12,7 +11,7 @@ extern void abort (void);
   TYPE dest;								\
   dest = uzp2_##TYPE (values1, values2);				\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
 }
 
 int main (void)
@@ -53,5 +52,12 @@ int main (void)
 	     ((v8sf) { 4.0, 6.0, 8.0, 10.0, 34.0, 36.0, 38.0, 40.0 }),
 	     ((v8sf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0 }),
 	     ((v8sf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0 }));
+  TEST_UZP2 (v16hf,
+	     ((v16hf) { 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0,
+			34.0, 36.0, 38.0, 40.0, 42.0, 44.0, 46.0, 48.0 }),
+	     ((v16hf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
+			11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0 }),
+	     ((v16hf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0,
+			41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0 }));
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_1.C b/gcc/testsuite/gcc.target/aarch64/sve_vcond_1.C
index 48ad92d0ab7..9be09546c80 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_1.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_1.C
@@ -1,5 +1,5 @@
-/* { dg-do compile { target { ! *-*-* } } } */
-/* { dg-options "-std=c++11 -O3 -fno-inline -march=armv8-a+sve" } */
+/* { dg-do assemble } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256 --save-temps" } */
 
 #include <stdint.h>
 
@@ -13,231 +13,233 @@ typedef uint16_t v16hu __attribute__((vector_size(32)));
 typedef uint32_t v8su __attribute__((vector_size(32)));
 typedef uint64_t v4du __attribute__((vector_size(32)));
 
-#define NUM_ELEMS(TYPE) (sizeof (r_##TYPE) / sizeof (r_##TYPE[0]))
-
-#define DEF_VCOND(TYPE,COND,SUFFIX)				\
-TYPE vcond_##TYPE##SUFFIX (TYPE x, TYPE y, TYPE a, TYPE b)	\
+#define DEF_VCOND_VAR(TYPE, COND, SUFFIX)			\
+TYPE vcond_##TYPE##_##SUFFIX (TYPE x, TYPE y, TYPE a, TYPE b)	\
 {								\
   TYPE r;							\
   r = a COND b ? x : y;						\
   return r;							\
 }
 
-#define DEF_VCOND_IMM(TYPE,COND,IMM,SUFFIX)		\
-TYPE vcond_imm_##TYPE##SUFFIX (TYPE x, TYPE y, TYPE a)	\
-{							\
-  TYPE r;						\
-  r = a COND IMM ? x : y;				\
-  return r;						\
+#define DEF_VCOND_IMM(TYPE, COND, IMM, SUFFIX)			\
+TYPE vcond_imm_##TYPE##_##SUFFIX (TYPE x, TYPE y, TYPE a)	\
+{								\
+  TYPE r;							\
+  r = a COND IMM ? x : y;					\
+  return r;							\
 }
 
-#define DEF_VCOND_SIGNED_ALL(COND,SUFFIX)	\
-DEF_VCOND (v32qi,COND,SUFFIX)			\
-DEF_VCOND (v16hi,COND,SUFFIX)			\
-DEF_VCOND (v8si,COND,SUFFIX)			\
-DEF_VCOND (v4di,COND,SUFFIX)
-
-#define DEF_VCOND_UNSIGNED_ALL(COND,SUFFIX)	\
-DEF_VCOND (v32qu,COND,SUFFIX)			\
-DEF_VCOND (v16hu,COND,SUFFIX)			\
-DEF_VCOND (v8su,COND,SUFFIX)			\
-DEF_VCOND (v4du,COND,SUFFIX)
-
-#define DEF_VCOND_ALL(COND,SUFFIX)		\
-DEF_VCOND_SIGNED_ALL (COND,SUFFIX)		\
-DEF_VCOND_UNSIGNED_ALL (COND,SUFFIX)
-
-#define DEF_VCOND_IMM_SIGNED_ALL(COND,IMM,SUFFIX)	\
-DEF_VCOND_IMM (v32qi,COND,IMM,SUFFIX)			\
-DEF_VCOND_IMM (v16hi,COND,IMM,SUFFIX)			\
-DEF_VCOND_IMM (v8si,COND,IMM,SUFFIX)			\
-DEF_VCOND_IMM (v4di,COND,IMM,SUFFIX)
-
-#define DEF_VCOND_IMM_UNSIGNED_ALL(COND,IMM,SUFFIX)	\
-DEF_VCOND_IMM (v32qu,COND,IMM,SUFFIX)			\
-DEF_VCOND_IMM (v16hu,COND,IMM,SUFFIX)			\
-DEF_VCOND_IMM (v8su,COND,IMM,SUFFIX)			\
-DEF_VCOND_IMM (v4du,COND,IMM,SUFFIX)
-
-#define DEF_VCOND_IMM_ALL(COND,IMM,SUFFIX)	\
-DEF_VCOND_IMM_SIGNED_ALL (COND,IMM,SUFFIX)	\
-DEF_VCOND_IMM_UNSIGNED_ALL (COND,IMM,SUFFIX)
-
-DEF_VCOND_ALL (>, _gt)
-DEF_VCOND_ALL (<, _lt)
-DEF_VCOND_ALL (>=, _ge)
-DEF_VCOND_ALL (<=, _le)
-DEF_VCOND_ALL (==, _eq)
-DEF_VCOND_ALL (!=, _ne)
-
-/* == Expect immediates to make it into the encoding == */
-
-DEF_VCOND_IMM_ALL (>, 5, _gt)
-DEF_VCOND_IMM_ALL (<, 5, _lt)
-DEF_VCOND_IMM_ALL (>=, 5, _ge)
-DEF_VCOND_IMM_ALL (<=, 5, _le)
-DEF_VCOND_IMM_ALL (==, 5, _eq)
-DEF_VCOND_IMM_ALL (!=, 5, _ne)
-
-DEF_VCOND_IMM_SIGNED_ALL (>, 15, _gt2)
-DEF_VCOND_IMM_SIGNED_ALL (<, 15, _lt2)
-DEF_VCOND_IMM_SIGNED_ALL (>=, 15, _ge2)
-DEF_VCOND_IMM_SIGNED_ALL (<=, 15, _le2)
-DEF_VCOND_IMM_SIGNED_ALL (==, 15, _eq2)
-DEF_VCOND_IMM_SIGNED_ALL (!=, 15, _ne2)
-
-DEF_VCOND_IMM_SIGNED_ALL (>, -16, _gt3)
-DEF_VCOND_IMM_SIGNED_ALL (<, -16, _lt3)
-DEF_VCOND_IMM_SIGNED_ALL (>=, -16, _ge3)
-DEF_VCOND_IMM_SIGNED_ALL (<=, -16, _le3)
-DEF_VCOND_IMM_SIGNED_ALL (==, -16, _eq3)
-DEF_VCOND_IMM_SIGNED_ALL (!=, -16, _ne3)
-
-DEF_VCOND_IMM_UNSIGNED_ALL (>, 0, _gt4)
-/* Testing if an unsigned value >= 0 or < 0 is pointless as it will
-   get folded away by the compiler.  */
-DEF_VCOND_IMM_UNSIGNED_ALL (<=, 0, _le4)
-
-DEF_VCOND_IMM_UNSIGNED_ALL (>, 31, _gt5)
-DEF_VCOND_IMM_UNSIGNED_ALL (<, 31, _lt5)
-DEF_VCOND_IMM_UNSIGNED_ALL (>=, 31, _ge5)
-DEF_VCOND_IMM_UNSIGNED_ALL (<=, 31, _le5)
-
-/* Expect immediates to NOT make it into the encoding, and instead be
-   forced into a register.  == */
-DEF_VCOND_IMM_ALL (>, 32, _gt6)
-DEF_VCOND_IMM_ALL (<, 32, _lt6)
-DEF_VCOND_IMM_ALL (>=, 32, _ge6)
-DEF_VCOND_IMM_ALL (<=, 32, _le6)
-DEF_VCOND_IMM_ALL (==, 32, _eq6)
-DEF_VCOND_IMM_ALL (!=, 32, _ne6)
-
-/* { dg-final { scan-assembler {\tsel\tz[0-9]+.b, p[0-7], z[0-9]+\.b, z[0-9]+\.b\n} } } */
-/* { dg-final { scan-assembler {\tsel\tz[0-9]+.h, p[0-7], z[0-9]+\.h, z[0-9]+\.h\n} } } */
-/* { dg-final { scan-assembler {\tsel\tz[0-9]+.s, p[0-7], z[0-9]+\.s, z[0-9]+\.s\n} } } */
-/* { dg-final { scan-assembler {\tsel\tz[0-9]+.d, p[0-7], z[0-9]+\.d, z[0-9]+\.d\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
-
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
-
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
-
-
-
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
-
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
-
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpgt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
-
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmplt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpge\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
-
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmple\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpeq\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
-/* { dg-final { scan-assembler {\tcmpne\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
-
-
-
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #0\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #0\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #0\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #0\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #0\n} } } */
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #0\n} } } */
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #0\n} } } */
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #0\n} } } */
-
-
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmphi\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
-
-/* { dg-final { scan-assembler {\tcmplo\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmplo\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmplo\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmplo\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
-
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmphs\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
-
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
-/* { dg-final { scan-assembler {\tcmpls\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
+#define TEST_COND_VAR_SIGNED_ALL(T, COND, SUFFIX)	\
+  T (v32qi, COND, SUFFIX)				\
+  T (v16hi, COND, SUFFIX)				\
+  T (v8si, COND, SUFFIX)				\
+  T (v4di, COND, SUFFIX)
+
+#define TEST_COND_VAR_UNSIGNED_ALL(T, COND, SUFFIX)	\
+  T (v32qu, COND, SUFFIX)				\
+  T (v16hu, COND, SUFFIX)				\
+  T (v8su, COND, SUFFIX)				\
+  T (v4du, COND, SUFFIX)
+
+#define TEST_COND_VAR_ALL(T, COND, SUFFIX)		\
+  TEST_COND_VAR_SIGNED_ALL (T, COND, SUFFIX)		\
+  TEST_COND_VAR_UNSIGNED_ALL (T, COND, SUFFIX)
+
+#define TEST_VAR_ALL(T)				\
+  TEST_COND_VAR_ALL (T, >, gt)			\
+  TEST_COND_VAR_ALL (T, <, lt)			\
+  TEST_COND_VAR_ALL (T, >=, ge)			\
+  TEST_COND_VAR_ALL (T, <=, le)			\
+  TEST_COND_VAR_ALL (T, ==, eq)			\
+  TEST_COND_VAR_ALL (T, !=, ne)
+
+#define TEST_COND_IMM_SIGNED_ALL(T, COND, IMM, SUFFIX)	\
+  T (v32qi, COND, IMM, SUFFIX)				\
+  T (v16hi, COND, IMM, SUFFIX)				\
+  T (v8si, COND, IMM, SUFFIX)				\
+  T (v4di, COND, IMM, SUFFIX)
+
+#define TEST_COND_IMM_UNSIGNED_ALL(T, COND, IMM, SUFFIX)	\
+  T (v32qu, COND, IMM, SUFFIX)					\
+  T (v16hu, COND, IMM, SUFFIX)					\
+  T (v8su, COND, IMM, SUFFIX)					\
+  T (v4du, COND, IMM, SUFFIX)
+
+#define TEST_COND_IMM_ALL(T, COND, IMM, SUFFIX)		\
+  TEST_COND_IMM_SIGNED_ALL (T, COND, IMM, SUFFIX)	\
+  TEST_COND_IMM_UNSIGNED_ALL (T, COND, IMM, SUFFIX)
+
+#define TEST_IMM_ALL(T)							\
+  /* Expect immediates to make it into the encoding.  */		\
+  TEST_COND_IMM_ALL (T, >, 5, gt)					\
+  TEST_COND_IMM_ALL (T, <, 5, lt)					\
+  TEST_COND_IMM_ALL (T, >=, 5, ge)					\
+  TEST_COND_IMM_ALL (T, <=, 5, le)					\
+  TEST_COND_IMM_ALL (T, ==, 5, eq)					\
+  TEST_COND_IMM_ALL (T, !=, 5, ne)					\
+									\
+  TEST_COND_IMM_SIGNED_ALL (T, >, 15, gt2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <, 15, lt2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, >=, 15, ge2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <=, 15, le2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, ==, 15, eq2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, !=, 15, ne2)				\
+									\
+  TEST_COND_IMM_SIGNED_ALL (T, >, -16, gt3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <, -16, lt3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, >=, -16, ge3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <=, -16, le3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, ==, -16, eq3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, !=, -16, ne3)				\
+									\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >, 0, gt4)				\
+  /* Testing if an unsigned value >= 0 or < 0 is pointless as it will	\
+     get folded away by the compiler.  */				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <=, 0, le4)				\
+									\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >, 31, gt5)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <, 31, lt5)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >=, 31, ge5)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <=, 31, le5)				\
+									\
+  /* Expect immediates to NOT make it into the encoding, and instead be	\
+     forced into a register.  */					\
+  TEST_COND_IMM_ALL (T, >, 32, gt6)					\
+  TEST_COND_IMM_ALL (T, <, 32, lt6)					\
+  TEST_COND_IMM_ALL (T, >=, 32, ge6)					\
+  TEST_COND_IMM_ALL (T, <=, 32, le6)					\
+  TEST_COND_IMM_ALL (T, ==, 32, eq6)					\
+  TEST_COND_IMM_ALL (T, !=, 32, ne6)
+
+TEST_VAR_ALL (DEF_VCOND_VAR)
+TEST_IMM_ALL (DEF_VCOND_IMM)
+
+/* { dg-final { scan-assembler {\tsel\tz[0-9]+\.b, p[0-7], z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler {\tsel\tz[0-9]+\.h, p[0-7], z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler {\tsel\tz[0-9]+\.s, p[0-7], z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler {\tsel\tz[0-9]+\.d, p[0-7], z[0-9]+\.d, z[0-9]+\.d\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+
+
+
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
+
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
+
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
+
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmplt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpge\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
+
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmple\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpeq\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} } } */
+/* { dg-final { scan-assembler {\tcmpne\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} } } */
+
+
+
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #0\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #0\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #0\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #0\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #0\n} } } */
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #0\n} } } */
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #0\n} } } */
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #0\n} } } */
+
+
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmphi\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
+
+/* { dg-final { scan-assembler {\tcmplo\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmplo\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmplo\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmplo\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
+
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmphs\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
+
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #31\n} } } */
+/* { dg-final { scan-assembler {\tcmpls\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #31\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_1_run.C b/gcc/testsuite/gcc.target/aarch64/sve_vcond_1_run.C
index e2b1c62a667..42e09d94393 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_1_run.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_1_run.C
@@ -1,100 +1,46 @@
-/* { dg-do run { target { ! *-*-* } } } */
-/* { dg-options "-std=c++11 -O3 -fno-inline -march=armv8-a+sve" } */
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O -march=armv8-a+sve" } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" { target aarch64_sve256_hw } } */
 
 #include "sve_vcond_1.C"
 
-#include <stdlib.h>
+#define NUM_ELEMS(X) (sizeof (X) / sizeof (X[0]))
 
-#define TEST_VCOND(TYPE,COND,SUFFIX)			\
+#define TEST_VCOND_VAR(TYPE, COND, SUFFIX)		\
 {							\
-  TYPE x = { 1 }, y = { 2 }, a = { 3 }, b = { 4 };	\
-  r_##TYPE += vcond_##TYPE##SUFFIX (x, y, a, b);	\
+  TYPE x, y, a, b;					\
+  for (int i = 0; i < NUM_ELEMS (x); ++i)		\
+    {							\
+      a[i] = i - 2;					\
+      b[i] = NUM_ELEMS (x) - 2 - i;			\
+      x[i] = i * 2;					\
+      y[i] = -i * 3;					\
+    }							\
+  TYPE r = vcond_##TYPE##_##SUFFIX (x, y, a, b);	\
+  for (int i = 0; i < NUM_ELEMS (x); ++i)		\
+    if (r[i] != (a[i] COND b[i] ? x[i] : y[i]))		\
+      __builtin_abort ();				\
 }
 
-#define TEST_VCOND_IMM(TYPE,COND,IMM,SUFFIX)		\
+#define TEST_VCOND_IMM(TYPE, COND, IMM, SUFFIX)		\
 {							\
-  TYPE x = { 1 }, y = { 2 }, a = { 3 };			\
-  r_##TYPE += vcond_imm_##TYPE##SUFFIX (x, y, a);	\
+  TYPE x, y, a;						\
+  for (int i = 0; i < NUM_ELEMS (x); ++i)		\
+    {							\
+      a[i] = IMM - 2 + i;				\
+      x[i] = i * 2;					\
+      y[i] = -i * 3;					\
+    }							\
+  TYPE r = vcond_imm_##TYPE##_##SUFFIX (x, y, a);	\
+  for (int i = 0; i < NUM_ELEMS (x); ++i)		\
+    if (r[i] != (a[i] COND IMM ? x[i] : y[i]))		\
+      __builtin_abort ();				\
 }
 
 
-#define TEST_VCOND_SIGNED_ALL(COND, SUFFIX)	\
-TEST_VCOND (v32qi, COND, SUFFIX)		\
-TEST_VCOND (v16hi, COND, SUFFIX)		\
-TEST_VCOND (v8si, COND, SUFFIX)			\
-TEST_VCOND (v4di, COND, SUFFIX)
-
-#define TEST_VCOND_UNSIGNED_ALL(COND, SUFFIX)	\
-TEST_VCOND (v32qu, COND, SUFFIX)		\
-TEST_VCOND (v16hu, COND, SUFFIX)		\
-TEST_VCOND (v8su, COND, SUFFIX)			\
-TEST_VCOND (v4du, COND, SUFFIX)
-
-#define TEST_VCOND_ALL(COND, SUFFIX)		\
-TEST_VCOND_SIGNED_ALL (COND, SUFFIX)		\
-TEST_VCOND_UNSIGNED_ALL(COND, SUFFIX)
-
-#define TEST_VCOND_IMM_SIGNED_ALL(COND, IMM, SUFFIX)	\
-TEST_VCOND_IMM (v32qi, COND, IMM, SUFFIX)		\
-TEST_VCOND_IMM (v16hi, COND, IMM, SUFFIX)		\
-TEST_VCOND_IMM (v8si, COND, IMM, SUFFIX)		\
-TEST_VCOND_IMM (v4di, COND, IMM, SUFFIX)
-
-#define TEST_VCOND_IMM_UNSIGNED_ALL(COND, IMM, SUFFIX)	\
-TEST_VCOND_IMM (v32qu, COND, IMM, SUFFIX)		\
-TEST_VCOND_IMM (v16hu, COND, IMM, SUFFIX)		\
-TEST_VCOND_IMM (v8su, COND, IMM, SUFFIX)		\
-TEST_VCOND_IMM (v4du, COND, IMM, SUFFIX)
-
-#define TEST_VCOND_IMM_ALL(COND,IMM,SUFFIX)	\
-TEST_VCOND_IMM_SIGNED_ALL (COND,IMM,SUFFIX)	\
-TEST_VCOND_IMM_UNSIGNED_ALL (COND,IMM,SUFFIX)
-
-#define DEF_INIT_VECTOR(TYPE)			\
-  TYPE r_##TYPE;				\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++ )	\
-    r_##TYPE[i] = i * 3;
-
-#define SUM_VECTOR(VAL,TYPE)			\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++ )	\
-    VAL += r_##TYPE[i];
-
 int main (int argc, char **argv)
 {
-  int result = 0;
-  DEF_INIT_VECTOR (v32qi)
-  DEF_INIT_VECTOR (v16hi)
-  DEF_INIT_VECTOR (v8si)
-  DEF_INIT_VECTOR (v4di)
-  DEF_INIT_VECTOR (v32qu)
-  DEF_INIT_VECTOR (v16hu)
-  DEF_INIT_VECTOR (v8su)
-  DEF_INIT_VECTOR (v4du)
-
-  TEST_VCOND_ALL (>, _gt)
-  TEST_VCOND_ALL (<, _lt)
-  TEST_VCOND_ALL (>=, _ge)
-  TEST_VCOND_ALL (<=, _le)
-  TEST_VCOND_ALL (==, _eq)
-  TEST_VCOND_ALL (!=, _ne)
-
-  TEST_VCOND_IMM_ALL (>, 5, _gt)
-  TEST_VCOND_IMM_ALL (<, 5, _lt)
-  TEST_VCOND_IMM_ALL (>=, 5, _ge)
-  TEST_VCOND_IMM_ALL (<=, 5, _le)
-  TEST_VCOND_IMM_ALL (==, 5, _eq)
-  TEST_VCOND_IMM_ALL (!=, 5, _ne)
-
-  SUM_VECTOR (result, v32qi)
-  SUM_VECTOR (result, v16hi)
-  SUM_VECTOR (result, v8si)
-  SUM_VECTOR (result, v4di)
-  SUM_VECTOR (result, v32qu)
-  SUM_VECTOR (result, v16hu)
-  SUM_VECTOR (result, v8su)
-  SUM_VECTOR (result, v4du)
-
-  if (result != 4044)
-    abort ();
+  TEST_VAR_ALL (TEST_VCOND_VAR)
+  TEST_IMM_ALL (TEST_VCOND_IMM)
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_2.C b/gcc/testsuite/gcc.target/aarch64/sve_vcond_2.C
deleted file mode 100644
index 80299d7e4b8..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_2.C
+++ /dev/null
@@ -1,310 +0,0 @@
-/* { dg-do compile } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -march=armv8-a+sve -fno-inline -fno-ipa-icf" } */
-
-#include <stdint.h>
-
-#define NUM_ELEMS(TYPE) (32 / sizeof (TYPE))
-
-#define DEF_VCOND(DATA_TYPE, CMP_TYPE, COND, SUFFIX)			\
-  void vcond_##CMP_TYPE##SUFFIX (DATA_TYPE *__restrict__ r,		\
-				 DATA_TYPE *__restrict__ a,		\
-				 DATA_TYPE *__restrict__ b,		\
-				 CMP_TYPE *__restrict__ x,		\
-				 CMP_TYPE *__restrict__ y,		\
-				 int n)					\
-  {									\
-    for (int i = 0; i < n; i++)						\
-      {									\
-	CMP_TYPE yval = y[i], xval = x[i];				\
-	DATA_TYPE aval = a[i], bval = b[i];				\
-	r[i] = xval COND yval ? aval : bval;				\
-      }									\
-  }
-
-#define DEF_VCOND_IMM(DATA_TYPE, CMP_TYPE, COND, IMM, SUFFIX)	\
-  void vcond_imm_##CMP_TYPE##SUFFIX (DATA_TYPE *__restrict__ r,	\
-				     DATA_TYPE *__restrict__ a,	\
-				     DATA_TYPE *__restrict__ b,	\
-				     CMP_TYPE *__restrict__ x,	\
-				     int n)			\
-  {								\
-    for (int i = 0; i < n; i++)					\
-      {								\
-	CMP_TYPE xval = x[i];					\
-	DATA_TYPE aval = a[i], bval = b[i];			\
-	r[i] = xval COND (CMP_TYPE) IMM ? aval : bval;		\
-      }								\
-  }
-
-#define DEF_VCOND_SIGNED_ALL(COND, SUFFIX)			\
-  DEF_VCOND (int8_t, int8_t, COND, SUFFIX)			\
-  DEF_VCOND (int16_t, int16_t, COND, SUFFIX)			\
-  DEF_VCOND (int32_t, int32_t, COND, SUFFIX)			\
-  DEF_VCOND (int64_t, int64_t, COND, SUFFIX)			\
-  DEF_VCOND (float, int32_t, COND, SUFFIX##_float)		\
-  DEF_VCOND (double, int64_t, COND, SUFFIX##_double)
-
-#define DEF_VCOND_UNSIGNED_ALL(COND, SUFFIX)			\
-  DEF_VCOND (uint8_t, uint8_t, COND, SUFFIX)			\
-  DEF_VCOND (uint16_t, uint16_t, COND, SUFFIX)			\
-  DEF_VCOND (uint32_t, uint32_t, COND, SUFFIX)			\
-  DEF_VCOND (uint64_t, uint64_t, COND, SUFFIX)			\
-  DEF_VCOND (float, uint32_t, COND, SUFFIX##_float)		\
-  DEF_VCOND (double, uint64_t, COND, SUFFIX##_double)
-
-#define DEF_VCOND_ALL(COND, SUFFIX)		\
-  DEF_VCOND_SIGNED_ALL (COND, SUFFIX)		\
-  DEF_VCOND_UNSIGNED_ALL (COND, SUFFIX)
-
-#define DEF_VCOND_IMM_SIGNED_ALL(COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (int8_t, int8_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (int16_t, int16_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (int32_t, int32_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (int64_t, int64_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (float, int32_t, COND, IMM, SUFFIX##_float)	\
-  DEF_VCOND_IMM (double, int64_t, COND, IMM, SUFFIX##_double)
-
-#define DEF_VCOND_IMM_UNSIGNED_ALL(COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (uint8_t, uint8_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (uint16_t, uint16_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (uint32_t, uint32_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (uint64_t, uint64_t, COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM (float, uint32_t, COND, IMM, SUFFIX##_float)	\
-  DEF_VCOND_IMM (double, uint64_t, COND, IMM, SUFFIX##_double)
-
-#define DEF_VCOND_IMM_ALL(COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM_SIGNED_ALL (COND, IMM, SUFFIX)		\
-  DEF_VCOND_IMM_UNSIGNED_ALL (COND, IMM, SUFFIX)
-
-DEF_VCOND_ALL (>, _gt)
-DEF_VCOND_ALL (<, _lt)
-DEF_VCOND_ALL (>=, _ge)
-DEF_VCOND_ALL (<=, _le)
-DEF_VCOND_ALL (==, _eq)
-DEF_VCOND_ALL (!=, _ne)
-
-/* == Expect immediates to make it into the encoding == */
-
-DEF_VCOND_IMM_ALL (>, 5, _gt)
-DEF_VCOND_IMM_ALL (<, 5, _lt)
-DEF_VCOND_IMM_ALL (>=, 5, _ge)
-DEF_VCOND_IMM_ALL (<=, 5, _le)
-DEF_VCOND_IMM_ALL (==, 5, _eq)
-DEF_VCOND_IMM_ALL (!=, 5, _ne)
-
-DEF_VCOND_IMM_SIGNED_ALL (>, 15, _gt2)
-DEF_VCOND_IMM_SIGNED_ALL (<, 15, _lt2)
-DEF_VCOND_IMM_SIGNED_ALL (>=, 15, _ge2)
-DEF_VCOND_IMM_SIGNED_ALL (<=, 15, _le2)
-DEF_VCOND_IMM_ALL (==, 15, _eq2)
-DEF_VCOND_IMM_ALL (!=, 15, _ne2)
-
-DEF_VCOND_IMM_SIGNED_ALL (>, 16, _gt3)
-DEF_VCOND_IMM_SIGNED_ALL (<, 16, _lt3)
-DEF_VCOND_IMM_SIGNED_ALL (>=, 16, _ge3)
-DEF_VCOND_IMM_SIGNED_ALL (<=, 16, _le3)
-DEF_VCOND_IMM_ALL (==, 16, _eq3)
-DEF_VCOND_IMM_ALL (!=, 16, _ne3)
-
-DEF_VCOND_IMM_SIGNED_ALL (>, -16, _gt4)
-DEF_VCOND_IMM_SIGNED_ALL (<, -16, _lt4)
-DEF_VCOND_IMM_SIGNED_ALL (>=, -16, _ge4)
-DEF_VCOND_IMM_SIGNED_ALL (<=, -16, _le4)
-DEF_VCOND_IMM_ALL (==, -16, _eq4)
-DEF_VCOND_IMM_ALL (!=, -16, _ne4)
-
-DEF_VCOND_IMM_SIGNED_ALL (>, -17, _gt5)
-DEF_VCOND_IMM_SIGNED_ALL (<, -17, _lt5)
-DEF_VCOND_IMM_SIGNED_ALL (>=, -17, _ge5)
-DEF_VCOND_IMM_SIGNED_ALL (<=, -17, _le5)
-DEF_VCOND_IMM_ALL (==, -17, _eq5)
-DEF_VCOND_IMM_ALL (!=, -17, _ne5)
-
-DEF_VCOND_IMM_UNSIGNED_ALL (>, 0, _gt6)
-/* Testing if an unsigned value >= 0 or < 0 is pointless as it will get
-   folded away by the compiler.  */
-DEF_VCOND_IMM_UNSIGNED_ALL (<=, 0, _le6)
-
-DEF_VCOND_IMM_UNSIGNED_ALL (>, 127, _gt7)
-DEF_VCOND_IMM_UNSIGNED_ALL (<, 127, _lt7)
-DEF_VCOND_IMM_UNSIGNED_ALL (>=, 127, _ge7)
-DEF_VCOND_IMM_UNSIGNED_ALL (<=, 127, _le7)
-
-/* == Expect immediates to NOT make it into the encoding, and instead be
-      forced into a register.  == */
-DEF_VCOND_IMM_UNSIGNED_ALL (>, 128, _gt8)
-DEF_VCOND_IMM_UNSIGNED_ALL (<, 128, _lt8)
-DEF_VCOND_IMM_UNSIGNED_ALL (>=, 128, _ge8)
-DEF_VCOND_IMM_UNSIGNED_ALL (<=, 128, _le8)
-
-/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+.b, p[0-7], z[0-9]+\.b, z[0-9]+\.b\n} 66 } } */
-/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+.h, p[0-7], z[0-9]+\.h, z[0-9]+\.h\n} 66 } } */
-/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+.s, p[0-7], z[0-9]+\.s, z[0-9]+\.s\n} 132 } } */
-/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+.d, p[0-7], z[0-9]+\.d, z[0-9]+\.d\n} 132 } } */
-
-/* There are two signed ordered register comparisons for each of .b and .h,
-   one for a variable comparison and one for one of the two out-of-range
-   constant comparisons.  The other out-of-ranger constant comparison can
-   be adjusted to an in-range value by inverting the handling of equality.
-
-   The same pattern appears twice for each .s and .d, once for integer data
-   and once for floating-point data.  */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
-
-/* Out-of-range >= is converted to in-range >.  */
-/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
-
-/* Out-of-range < is converted to in-range <=.  */
-/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
-
-/* 6 for .b and .h: {signed, unsigned\n} x {variable, too high, too low\n}.  */
-/* 12 for .s and .d: the above 6 repeated for integer and floating-point
-   data.  */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 6 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 12 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 12 } } */
-
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 6 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 12 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 12 } } */
-
-/* Also used for >= 16. */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
-
-/* gcc converts "a < 15" into "a <= 14".  */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #14\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #14\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #14\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #14\n} 2 } } */
-
-/* gcc converts "a >= 15" into "a > 14".  */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #14\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #14\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #14\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #14\n} 2 } } */
-
-/* Also used for < 16.  */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmple\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
-
-/* Appears once for each signedness.  */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
-
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
-
-/* gcc converts "a > -16" into "a >= -15".  */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-15\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-15\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-15\n} 2 } } */
-
-/* Also used for <= -17.  */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
-
-/* Also used for > -17.  */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-16\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
-
-/* gcc converts "a <= -16" into "a < -15".  */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #-15\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #-15\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #-15\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #-15\n} 2 } } */
-
-/* gcc converts "a > 0" into "a != 0".  */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #0\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #0\n} 2 } } */
-
-/* gcc converts "a <= 0" into "a == 0".  */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #0\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #0\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #0\n} 2 } } */
-
-/* Also used for >= 128.  */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #127\n} 2 { xfail *-*-* } } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #127\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #127\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #127\n} 4 } } */
-
-/* gcc converts "a < 127" into "a <= 126".  */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #126\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #126\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #126\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #126\n} 2 } } */
-
-/* gcc converts "a >= 127" into "a > 126".  */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #126\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #126\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #126\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #126\n} 2 } } */
-
-/* Also used for < 128.  */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].b, p[0-7]/z, z[0-9]+\.b, #127\n} 2 { xfail *-*-* } } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].h, p[0-7]/z, z[0-9]+\.h, #127\n} 2 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].s, p[0-7]/z, z[0-9]+\.s, #127\n} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7].d, p[0-7]/z, z[0-9]+\.d, #127\n} 4 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_2.c b/gcc/testsuite/gcc.target/aarch64/sve_vcond_2.c
new file mode 100644
index 00000000000..0c67f8147c6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_2.c
@@ -0,0 +1,318 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
+
+#include <stdint.h>
+
+#define DEF_VCOND_VAR(DATA_TYPE, CMP_TYPE, COND, SUFFIX)	\
+  void __attribute__ ((noinline, noclone))			\
+  vcond_var_##CMP_TYPE##_##SUFFIX (DATA_TYPE *__restrict__ r,	\
+				   DATA_TYPE *__restrict__ x,	\
+				   DATA_TYPE *__restrict__ y,	\
+				   CMP_TYPE *__restrict__ a,	\
+				   CMP_TYPE *__restrict__ b,	\
+				   int n)			\
+  {								\
+    for (int i = 0; i < n; i++)					\
+      {								\
+	DATA_TYPE xval = x[i], yval = y[i];			\
+	CMP_TYPE aval = a[i], bval = b[i];			\
+	r[i] = aval COND bval ? xval : yval;			\
+      }								\
+  }
+
+#define DEF_VCOND_IMM(DATA_TYPE, CMP_TYPE, COND, IMM, SUFFIX)	\
+  void __attribute__ ((noinline, noclone))			\
+  vcond_imm_##CMP_TYPE##_##SUFFIX (DATA_TYPE *__restrict__ r,	\
+				   DATA_TYPE *__restrict__ x,	\
+				   DATA_TYPE *__restrict__ y,	\
+				   CMP_TYPE *__restrict__ a,	\
+				   int n)			\
+  {								\
+    for (int i = 0; i < n; i++)					\
+      {								\
+	DATA_TYPE xval = x[i], yval = y[i];			\
+	CMP_TYPE aval = a[i];					\
+	r[i] = aval COND (CMP_TYPE) IMM ? xval : yval;		\
+      }								\
+  }
+
+#define TEST_COND_VAR_SIGNED_ALL(T, COND, SUFFIX)	\
+  T (int8_t, int8_t, COND, SUFFIX)			\
+  T (int16_t, int16_t, COND, SUFFIX)			\
+  T (int32_t, int32_t, COND, SUFFIX)			\
+  T (int64_t, int64_t, COND, SUFFIX)			\
+  T (_Float16, int16_t, COND, SUFFIX##_float16)		\
+  T (float, int32_t, COND, SUFFIX##_float)		\
+  T (double, int64_t, COND, SUFFIX##_double)
+
+#define TEST_COND_VAR_UNSIGNED_ALL(T, COND, SUFFIX)	\
+  T (uint8_t, uint8_t, COND, SUFFIX)			\
+  T (uint16_t, uint16_t, COND, SUFFIX)			\
+  T (uint32_t, uint32_t, COND, SUFFIX)			\
+  T (uint64_t, uint64_t, COND, SUFFIX)			\
+  T (_Float16, uint16_t, COND, SUFFIX##_float16)	\
+  T (float, uint32_t, COND, SUFFIX##_float)		\
+  T (double, uint64_t, COND, SUFFIX##_double)
+
+#define TEST_COND_VAR_ALL(T, COND, SUFFIX)	\
+  TEST_COND_VAR_SIGNED_ALL (T, COND, SUFFIX)	\
+  TEST_COND_VAR_UNSIGNED_ALL (T, COND, SUFFIX)
+
+#define TEST_VAR_ALL(T)				\
+  TEST_COND_VAR_ALL (T, >, _gt)			\
+  TEST_COND_VAR_ALL (T, <, _lt)			\
+  TEST_COND_VAR_ALL (T, >=, _ge)		\
+  TEST_COND_VAR_ALL (T, <=, _le)		\
+  TEST_COND_VAR_ALL (T, ==, _eq)		\
+  TEST_COND_VAR_ALL (T, !=, _ne)
+
+#define TEST_COND_IMM_SIGNED_ALL(T, COND, IMM, SUFFIX)	\
+  T (int8_t, int8_t, COND, IMM, SUFFIX)			\
+  T (int16_t, int16_t, COND, IMM, SUFFIX)		\
+  T (int32_t, int32_t, COND, IMM, SUFFIX)		\
+  T (int64_t, int64_t, COND, IMM, SUFFIX)		\
+  T (_Float16, int16_t, COND, IMM, SUFFIX##_float16)	\
+  T (float, int32_t, COND, IMM, SUFFIX##_float)		\
+  T (double, int64_t, COND, IMM, SUFFIX##_double)
+
+#define TEST_COND_IMM_UNSIGNED_ALL(T, COND, IMM, SUFFIX)	\
+  T (uint8_t, uint8_t, COND, IMM, SUFFIX)			\
+  T (uint16_t, uint16_t, COND, IMM, SUFFIX)			\
+  T (uint32_t, uint32_t, COND, IMM, SUFFIX)			\
+  T (uint64_t, uint64_t, COND, IMM, SUFFIX)			\
+  T (_Float16, uint16_t, COND, IMM, SUFFIX##_float16)		\
+  T (float, uint32_t, COND, IMM, SUFFIX##_float)		\
+  T (double, uint64_t, COND, IMM, SUFFIX##_double)
+
+#define TEST_COND_IMM_ALL(T, COND, IMM, SUFFIX)		\
+  TEST_COND_IMM_SIGNED_ALL (T, COND, IMM, SUFFIX)	\
+  TEST_COND_IMM_UNSIGNED_ALL (T, COND, IMM, SUFFIX)
+
+#define TEST_IMM_ALL(T)							\
+  /* Expect immediates to make it into the encoding.  */		\
+  TEST_COND_IMM_ALL (T, >, 5, _gt)					\
+  TEST_COND_IMM_ALL (T, <, 5, _lt)					\
+  TEST_COND_IMM_ALL (T, >=, 5, _ge)					\
+  TEST_COND_IMM_ALL (T, <=, 5, _le)					\
+  TEST_COND_IMM_ALL (T, ==, 5, _eq)					\
+  TEST_COND_IMM_ALL (T, !=, 5, _ne)					\
+									\
+  TEST_COND_IMM_SIGNED_ALL (T, >, 15, _gt2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <, 15, _lt2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, >=, 15, _ge2)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <=, 15, _le2)				\
+  TEST_COND_IMM_ALL (T, ==, 15, _eq2)					\
+  TEST_COND_IMM_ALL (T, !=, 15, _ne2)					\
+									\
+  TEST_COND_IMM_SIGNED_ALL (T, >, 16, _gt3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <, 16, _lt3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, >=, 16, _ge3)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <=, 16, _le3)				\
+  TEST_COND_IMM_ALL (T, ==, 16, _eq3)					\
+  TEST_COND_IMM_ALL (T, !=, 16, _ne3)					\
+									\
+  TEST_COND_IMM_SIGNED_ALL (T, >, -16, _gt4)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <, -16, _lt4)				\
+  TEST_COND_IMM_SIGNED_ALL (T, >=, -16, _ge4)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <=, -16, _le4)				\
+  TEST_COND_IMM_ALL (T, ==, -16, _eq4)					\
+  TEST_COND_IMM_ALL (T, !=, -16, _ne4)					\
+									\
+  TEST_COND_IMM_SIGNED_ALL (T, >, -17, _gt5)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <, -17, _lt5)				\
+  TEST_COND_IMM_SIGNED_ALL (T, >=, -17, _ge5)				\
+  TEST_COND_IMM_SIGNED_ALL (T, <=, -17, _le5)				\
+  TEST_COND_IMM_ALL (T, ==, -17, _eq5)					\
+  TEST_COND_IMM_ALL (T, !=, -17, _ne5)					\
+									\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >, 0, _gt6)				\
+  /* Testing if an unsigned value >= 0 or < 0 is pointless as it will	\
+     get folded away by the compiler.  */				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <=, 0, _le6)				\
+									\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >, 127, _gt7)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <, 127, _lt7)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >=, 127, _ge7)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <=, 127, _le7)				\
+									\
+  /* Expect immediates to NOT make it into the encoding, and instead be \
+     forced into a register.  */					\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >, 128, _gt8)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <, 128, _lt8)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, >=, 128, _ge8)				\
+  TEST_COND_IMM_UNSIGNED_ALL (T, <=, 128, _le8)
+
+TEST_VAR_ALL (DEF_VCOND_VAR)
+TEST_IMM_ALL (DEF_VCOND_IMM)
+
+/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+\.b, p[0-7], z[0-9]+\.b, z[0-9]+\.b\n} 66 } } */
+/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+\.h, p[0-7], z[0-9]+\.h, z[0-9]+\.h\n} 132 } } */
+/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+\.s, p[0-7], z[0-9]+\.s, z[0-9]+\.s\n} 132 } } */
+/* { dg-final { scan-assembler-times {\tsel\tz[0-9]+\.d, p[0-7], z[0-9]+\.d, z[0-9]+\.d\n} 132 } } */
+
+/* There are two signed ordered register comparisons for .b, one for a
+   variable comparison and one for one of the two out-of-range constant
+   comparisons.  The other out-of-ranger constant comparison can be
+   adjusted to an in-range value by inverting the handling of equality.
+
+   The same pattern appears twice for .h, .s and .d, once for integer data
+   and once for floating-point data.  */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
+
+/* Out-of-range >= is converted to in-range >.  */
+/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmphs\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+
+/* Out-of-range < is converted to in-range <=.  */
+/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmplo\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
+
+/* 6 for .b: {signed, unsigned\n} x {variable, too high, too low}.  */
+/* 12 for .h,.s and .d: the above 6 repeated for integer and floating-point
+   data.  */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 12 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 12 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, z[0-9]+\.b\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 12 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 12 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, z[0-9]+\.d\n} 12 } } */
+
+/* Also used for >= 16. */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
+
+/* gcc converts "a < 15" into "a <= 14".  */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #14\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #14\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #14\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #14\n} 2 } } */
+
+/* gcc converts "a >= 15" into "a > 14".  */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #14\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #14\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #14\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #14\n} 2 } } */
+
+/* Also used for < 16.  */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
+
+/* Appears once for each signedness.  */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #15\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #15\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
+
+/* gcc converts "a > -16" into "a >= -15".  */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-15\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-15\n} 2 } } */
+
+/* Also used for <= -17.  */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
+
+/* Also used for > -17.  */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-16\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-16\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-16\n} 4 } } */
+
+/* gcc converts "a <= -16" into "a < -15".  */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #-15\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #-15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #-15\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #-15\n} 2 } } */
+
+/* gcc converts "a > 0" into "a != 0".  */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #0\n} 2 } } */
+
+/* gcc converts "a <= 0" into "a == 0".  */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #0\n} 2 } } */
+
+/* Also used for >= 128.  */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #127\n} 2 { xfail *-*-* } } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #127\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #127\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #127\n} 4 } } */
+
+/* gcc converts "a < 127" into "a <= 126".  */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #126\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #126\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #126\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #126\n} 2 } } */
+
+/* gcc converts "a >= 127" into "a > 126".  */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #126\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #126\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #126\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #126\n} 2 } } */
+
+/* Also used for < 128.  */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.b, p[0-7]/z, z[0-9]+\.b, #127\n} 2 { xfail *-*-* } } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.h, p[0-7]/z, z[0-9]+\.h, #127\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.s, p[0-7]/z, z[0-9]+\.s, #127\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\tp[0-7]\.d, p[0-7]/z, z[0-9]+\.d, #127\n} 4 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_2_run.C b/gcc/testsuite/gcc.target/aarch64/sve_vcond_2_run.C
deleted file mode 100644
index b3c54b74fde..00000000000
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_2_run.C
+++ /dev/null
@@ -1,118 +0,0 @@
-/* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -march=armv8-a+sve -fno-inline" } */
-
-#include "sve_vcond_2.C"
-
-#include <stdlib.h>
-
-#define TEST_VCOND(DATA_TYPE, CMP_TYPE, COND, SUFFIX)		\
-{								\
-  const int n = 32 / sizeof (DATA_TYPE);			\
-  CMP_TYPE x[n], y[n];						\
-  DATA_TYPE a[n], b[n];						\
-  for (int i = 0; i < n; ++i)					\
-    {								\
-      x[i] = i;							\
-      y[i] = (i & 1) + 5;					\
-      a[i] = 6 * i;						\
-      b[i] = 4 + i;						\
-    }								\
-  vcond_##CMP_TYPE##SUFFIX (r_##DATA_TYPE, a, b, x, y, n);	\
-  for (int i = 0; i < n; ++i)					\
-    if (r_##DATA_TYPE[i] != (x[i] COND y[i] ? a[i] : b[i]))	\
-      abort ();							\
-}
-
-#define TEST_VCOND_IMM(DATA_TYPE, CMP_TYPE, COND, IMM, SUFFIX)	\
-{								\
-  const int n = 32 / sizeof (DATA_TYPE);			\
-  CMP_TYPE x[n];						\
-  DATA_TYPE a[n], b[n];						\
-  for (int i = 0; i < n; ++i)					\
-    {								\
-      x[i] = i - 1;						\
-      a[i] = 5 * i + IMM;					\
-      b[i] = 7 + i - IMM * 2;					\
-    }								\
-  vcond_imm_##CMP_TYPE##SUFFIX (r_##DATA_TYPE, a, b, x, n);	\
-  for (int i = 0; i < n; ++i)					\
-    if (r_##DATA_TYPE[i] != (x[i] COND IMM ? a[i] : b[i]))	\
-      abort ();							\
-}
-
-#define TEST_VCOND_SIGNED_ALL(COND, SUFFIX)		\
-  TEST_VCOND (int8_t, int8_t, COND, SUFFIX)		\
-  TEST_VCOND (int16_t, int16_t, COND, SUFFIX)		\
-  TEST_VCOND (int32_t, int32_t, COND, SUFFIX)		\
-  TEST_VCOND (int64_t, int64_t, COND, SUFFIX)		\
-  TEST_VCOND (float, int32_t, COND, SUFFIX##_float)	\
-  TEST_VCOND (double, int64_t, COND, SUFFIX##_double)
-
-#define TEST_VCOND_UNSIGNED_ALL(COND, SUFFIX)		\
-  TEST_VCOND (uint8_t, uint8_t, COND, SUFFIX)		\
-  TEST_VCOND (uint16_t, uint16_t, COND, SUFFIX)		\
-  TEST_VCOND (uint32_t, uint32_t, COND, SUFFIX)		\
-  TEST_VCOND (uint64_t, uint64_t, COND, SUFFIX)		\
-  TEST_VCOND (float, uint32_t, COND, SUFFIX##_float)	\
-  TEST_VCOND (double, uint64_t, COND, SUFFIX##_double)
-
-#define TEST_VCOND_ALL(COND, SUFFIX)		\
-  TEST_VCOND_SIGNED_ALL (COND, SUFFIX)		\
-  TEST_VCOND_UNSIGNED_ALL (COND, SUFFIX)
-
-#define TEST_VCOND_IMM_SIGNED_ALL(COND, IMM, SUFFIX)		\
-  TEST_VCOND_IMM (int8_t, int8_t, COND, IMM, SUFFIX)		\
-  TEST_VCOND_IMM (int16_t, int16_t, COND, IMM, SUFFIX)		\
-  TEST_VCOND_IMM (int32_t, int32_t, COND, IMM, SUFFIX)		\
-  TEST_VCOND_IMM (int64_t, int64_t, COND, IMM, SUFFIX)		\
-  TEST_VCOND_IMM (float, int32_t, COND, IMM, SUFFIX##_float)	\
-  TEST_VCOND_IMM (double, int64_t, COND, IMM, SUFFIX##_double)
-
-#define TEST_VCOND_IMM_UNSIGNED_ALL(COND, IMM, SUFFIX)		\
-  TEST_VCOND_IMM (uint8_t, uint8_t, COND, IMM, SUFFIX)		\
-  TEST_VCOND_IMM (uint16_t, uint16_t, COND, IMM, SUFFIX)	\
-  TEST_VCOND_IMM (uint32_t, uint32_t, COND, IMM, SUFFIX)	\
-  TEST_VCOND_IMM (uint64_t, uint64_t, COND, IMM, SUFFIX)	\
-  TEST_VCOND_IMM (float, uint32_t, COND, IMM, SUFFIX##_float)	\
-  TEST_VCOND_IMM (double, uint64_t, COND, IMM, SUFFIX##_double)
-
-#define TEST_VCOND_IMM_ALL(COND, IMM, SUFFIX)	\
-  TEST_VCOND_IMM_SIGNED_ALL (COND, IMM, SUFFIX)	\
-  TEST_VCOND_IMM_UNSIGNED_ALL (COND, IMM, SUFFIX)
-
-#define DEF_INIT_VECTOR(TYPE)			\
-  TYPE r_##TYPE[NUM_ELEMS(TYPE)];		\
-  for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
-    r_##TYPE[i] = i * 3;
-
-int __attribute__ ((optimize (1)))
-main (int argc, char **argv)
-{
-  int result = 0;
-  DEF_INIT_VECTOR (int8_t)
-  DEF_INIT_VECTOR (int16_t)
-  DEF_INIT_VECTOR (int32_t)
-  DEF_INIT_VECTOR (int64_t)
-  DEF_INIT_VECTOR (uint8_t)
-  DEF_INIT_VECTOR (uint16_t)
-  DEF_INIT_VECTOR (uint32_t)
-  DEF_INIT_VECTOR (uint64_t)
-  DEF_INIT_VECTOR (float)
-  DEF_INIT_VECTOR (double)
-
-  TEST_VCOND_ALL (>, _gt)
-  TEST_VCOND_ALL (<, _lt)
-  TEST_VCOND_ALL (>=, _ge)
-  TEST_VCOND_ALL (<=, _le)
-  TEST_VCOND_ALL (==, _eq)
-  TEST_VCOND_ALL (!=, _ne)
-
-  TEST_VCOND_IMM_ALL (>, 5, _gt)
-  TEST_VCOND_IMM_ALL (<, 5, _lt)
-  TEST_VCOND_IMM_ALL (>=, 5, _ge)
-  TEST_VCOND_IMM_ALL (<=, 5, _le)
-  TEST_VCOND_IMM_ALL (==, 5, _eq)
-  TEST_VCOND_IMM_ALL (!=, 5, _ne)
-
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_2_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vcond_2_run.c
new file mode 100644
index 00000000000..4cdb5bb9e43
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_2_run.c
@@ -0,0 +1,49 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
+
+#include "sve_vcond_2.c"
+
+#define N 97
+
+#define TEST_VCOND_VAR(DATA_TYPE, CMP_TYPE, COND, SUFFIX)	\
+{								\
+  DATA_TYPE x[N], y[N], r[N];					\
+  CMP_TYPE a[N], b[N];						\
+  for (int i = 0; i < N; ++i)					\
+    {								\
+      x[i] = i;							\
+      y[i] = (i & 1) + 5;					\
+      a[i] = i - N / 3;						\
+      b[i] = N - N / 3 - i;					\
+      asm volatile ("" ::: "memory");				\
+    }								\
+  vcond_var_##CMP_TYPE##_##SUFFIX (r, x, y, a, b, N);		\
+  for (int i = 0; i < N; ++i)					\
+    if (r[i] != (a[i] COND b[i] ? x[i] : y[i]))			\
+      __builtin_abort ();					\
+}
+
+#define TEST_VCOND_IMM(DATA_TYPE, CMP_TYPE, COND, IMM, SUFFIX)	\
+{								\
+  DATA_TYPE x[N], y[N], r[N];					\
+  CMP_TYPE a[N];						\
+  for (int i = 0; i < N; ++i)					\
+    {								\
+      x[i] = i;							\
+      y[i] = (i & 1) + 5;					\
+      a[i] = IMM - N / 3 + i;					\
+      asm volatile ("" ::: "memory");				\
+    }								\
+  vcond_imm_##CMP_TYPE##_##SUFFIX (r, x, y, a, N);		\
+  for (int i = 0; i < N; ++i)					\
+    if (r[i] != (a[i] COND (CMP_TYPE) IMM ? x[i] : y[i]))	\
+      __builtin_abort ();					\
+}
+
+int __attribute__ ((optimize (1)))
+main (int argc, char **argv)
+{
+  TEST_VAR_ALL (TEST_VCOND_VAR)
+  TEST_IMM_ALL (TEST_VCOND_IMM)
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_3.C b/gcc/testsuite/gcc.target/aarch64/sve_vcond_3.c
index 68c033b1a7d..9750bd07fda 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_3.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_3.c
@@ -5,20 +5,18 @@
 
 #define DEF_SEL_IMM(TYPE, SUFFIX, IMM)					\
 void									\
-sel_##TYPE##_##SUFFIX (TYPE *__restrict__ a, TYPE *__restrict__ b,	\
-		       int n)						\
+sel_##TYPE##_##SUFFIX (TYPE *restrict a, TYPE *restrict b, int n)	\
 {									\
   for (int i = 0; i < n; i++)						\
     a[i] = b[i] != 0 ? IMM : 0;						\
 }
 
-#define DEF_SEL_VAR(TYPE)					\
-void								\
-sel_##TYPE##_var (TYPE *__restrict__ a, TYPE *__restrict__ b,	\
-		  TYPE val, int n)				\
-{								\
-  for (int i = 0; i < n; i++)					\
-    a[i] = b[i] != 0 ? val : 0;					\
+#define DEF_SEL_VAR(TYPE)						\
+void									\
+sel_##TYPE##_var (TYPE *restrict a, TYPE *restrict b, TYPE val, int n)	\
+{									\
+  for (int i = 0; i < n; i++)						\
+    a[i] = b[i] != 0 ? val : 0;						\
 }
 
 #define TEST_TYPE8(TYPE)			\
@@ -54,17 +52,17 @@ TEST_TYPE16 (int16_t)
 TEST_TYPE32 (int32_t)
 TEST_TYPE32 (int64_t)
 
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.b, p[0-7]/z, #-128\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.b, p[0-7]/z, #-127\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.b, p[0-7]/z, #2\n} 1 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.b, p[0-7]/z, #127\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.b, p[0-7]/z, #-128\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.b, p[0-7]/z, #-127\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.b, p[0-7]/z, #2\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.b, p[0-7]/z, #127\n} 1 } } */
 
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #-32768\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #-32512\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #-256\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #-128\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #-127\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #2\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #127\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #256\n} 3 } } */
-/* { dg-final { scan-assembler-times {\tmov\tz[0-9]*\.[hsd], p[0-7]/z, #32512\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #-32768\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #-32512\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #-256\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #-128\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #-127\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #2\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #127\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #256\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.[hsd], p[0-7]/z, #32512\n} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_4_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vcond_4_run.c
index 36c43e9f1e8..e8d06bb9f17 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_4_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_4_run.c
@@ -8,8 +8,6 @@
 
 #include <fenv.h>
 
-extern void abort (void) __attribute__ ((noreturn));
-
 #include "sve_vcond_4.c"
 
 #define N 401
@@ -33,6 +31,7 @@ extern void abort (void) __attribute__ ((noreturn));
 	  b[i] = i * 0.1;						\
 	else								\
 	  b[i] = i;							\
+	asm volatile ("" ::: "memory");					\
       }									\
     feclearexcept (FE_ALL_EXCEPT);					\
     test_##TYPE1##_##TYPE2##_##CMP##_var (dest1, src, 11, a, b, N);	\
@@ -40,15 +39,15 @@ extern void abort (void) __attribute__ ((noreturn));
     test_##TYPE1##_##TYPE2##_##CMP##_sel (dest3, 33, 44, a, 9, N);	\
     if (TEST_EXCEPTIONS							\
 	&& !fetestexcept (FE_INVALID) != !(EXPECT_INVALID))		\
-      abort ();								\
+      __builtin_abort ();						\
     for (int i = 0; i < N; ++i)						\
       {									\
 	if (dest1[i] != (CMP (a[i], b[i]) ? src[i] : 11))		\
-	  abort ();							\
+	  __builtin_abort ();						\
 	if (dest2[i] != (CMP (a[i], 0) ? src[i] : 22))			\
-	  abort ();							\
+	  __builtin_abort ();						\
 	if (dest3[i] != (CMP (a[i], 9) ? 33 : 44))			\
-	  abort ();							\
+	  __builtin_abort ();						\
       }									\
   }
 
@@ -64,7 +63,7 @@ extern void abort (void) __attribute__ ((noreturn));
   RUN_LOOP (uint64_t, double, CMP, EXPECT_INVALID) \
   RUN_LOOP (double, double, CMP, EXPECT_INVALID)
 
-int __attribute__ ((optimize (1, "no-tree-vectorize")))
+int __attribute__ ((optimize (1)))
 main (void)
 {
   RUN_CMP (eq, 0)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_6.c b/gcc/testsuite/gcc.target/aarch64/sve_vcond_6.c
index 8ab040ef51e..74336050d8d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_6.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_6.c
@@ -25,6 +25,7 @@
   }
 
 #define TEST_BINOP(T, BINOP) \
+  T (_Float16, BINOP) \
   T (float, BINOP) \
   T (double, BINOP)
 
@@ -40,11 +41,11 @@
 TEST_ALL (LOOP)
 
 /* Currently we don't manage to remove ANDs from the other loops.  */
-/* { dg-final { scan-assembler-times {\tand\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 2 { xfail *-*-* } } } */
+/* { dg-final { scan-assembler-times {\tand\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 3 { xfail *-*-* } } } */
 /* { dg-final { scan-assembler {\tand\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} } } */
-/* { dg-final { scan-assembler-times {\torr\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 2 } } */
-/* { dg-final { scan-assembler-times {\teor\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 2 } } */
-/* { dg-final { scan-assembler-times {\tnand\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 2 } } */
-/* { dg-final { scan-assembler-times {\tnor\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 2 } } */
-/* { dg-final { scan-assembler-times {\tbic\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 2 } } */
-/* { dg-final { scan-assembler-times {\torn\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 2 } } */
+/* { dg-final { scan-assembler-times {\torr\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\teor\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\tnand\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\tnor\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\tbic\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\torn\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vcond_6_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vcond_6_run.c
index ff8ad90da9f..edad9b8272d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vcond_6_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vcond_6_run.c
@@ -1,8 +1,6 @@
 /* { dg-do run { target aarch64_sve_hw } } */
 /* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-extern void abort (void) __attribute__ ((noreturn));
-
 #include "sve_vcond_6.c"
 
 #define N 401
@@ -17,6 +15,7 @@ extern void abort (void) __attribute__ ((noreturn));
 	b[i] = i % 7 < 4 ? __builtin_nan("") : i;			\
 	c[i] = i % 9 < 5 ? __builtin_nan("") : i;			\
 	d[i] = i % 11 < 6 ? __builtin_nan("") : i;			\
+	asm volatile ("" ::: "memory");					\
       }									\
     test_##TYPE##_##BINOP (dest, src, a, b, c, d, 100, N);		\
     for (int i = 0; i < N; ++i)						\
@@ -24,11 +23,11 @@ extern void abort (void) __attribute__ ((noreturn));
 	int res = BINOP (__builtin_isunordered (a[i], b[i]),		\
 			 __builtin_isunordered (c[i], d[i]));		\
 	if (dest[i] != (res ? src[i] : 100.0))				\
-	  abort ();							\
+	  __builtin_abort ();						\
       }									\
   }
 
-int __attribute__ ((optimize (1, "no-tree-vectorize")))
+int __attribute__ ((optimize (1)))
 main (void)
 {
   TEST_ALL (RUN_LOOP)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1.C b/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1.c
index d6194dcbf8f..95f19f7f786 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1.c
@@ -1,25 +1,30 @@
 /* { dg-do compile } */
-/* { dg-options "-std=c++11 -O2 -ftree-vectorize -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
 #include <stdint.h>
 
 #define NUM_ELEMS(TYPE) (128 / sizeof (TYPE))
 
 #define DUP_FN(TYPE)				\
-void dup_##TYPE (TYPE *r, TYPE v)		\
+void __attribute__ ((noinline, noclone))	\
+dup_##TYPE (TYPE *r, TYPE v)			\
 {						\
   for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
     r[i] = v;					\
 }
 
+DUP_FN (int8_t)
 DUP_FN (int16_t)
 DUP_FN (int32_t)
 DUP_FN (int64_t)
+DUP_FN (_Float16)
 DUP_FN (float)
 DUP_FN (double)
 
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.b, w[0-9]+\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.h, w[0-9]+\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.s, w[0-9]+\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, x[0-9]+\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.h, h[0-9]+\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.s, s[0-9]+\n} 1 } } */
 /* { dg-final { scan-assembler-times {\tmov\tz[0-9]+\.d, d[0-9]+\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1_run.C b/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1_run.c
index 579327ef81d..ba7eb44be70 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1_run.C
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_init_1_run.c
@@ -1,23 +1,26 @@
 /* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-std=c++11 -O3 -fno-inline -march=armv8-a+sve" } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve" } */
 
-#include "sve_vec_init_1.C"
-
-#include <stdlib.h>
+#include "sve_vec_init_1.c"
 
 #define TEST_INIT_VECTOR(TYPE, VAL)		\
-  TYPE r_##TYPE[NUM_ELEMS (TYPE)];		\
-  dup_##TYPE (r_##TYPE, VAL);			\
+  {						\
+  TYPE r[NUM_ELEMS (TYPE)];			\
+  dup_##TYPE (r, VAL);				\
   for (int i = 0; i < NUM_ELEMS (TYPE); i++)	\
-    if (r_##TYPE[i] != VAL)			\
-      abort ();
+    if (r[i] != VAL)				\
+      __builtin_abort ();			\
+  }
 
-int main (void)
+int __attribute__ ((optimize (1)))
+main (void)
 {
+  TEST_INIT_VECTOR (int8_t, 0x2a);
   TEST_INIT_VECTOR (int16_t, 0x3976);
   TEST_INIT_VECTOR (int32_t, 0x31232976);
   TEST_INIT_VECTOR (int64_t, 0x9489363731232976LL);
 
+  TEST_INIT_VECTOR (_Float16, -0x1.fp10);
   TEST_INIT_VECTOR (float, -0x1.fe02p10);
   TEST_INIT_VECTOR (double, 0x1.fe02eeeee1p10);
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1.c
index 214c0c3930f..ae8542f2c75 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1.c
@@ -1,15 +1,19 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define VEC_PERM(TYPE, MASKTYPE)					\
-TYPE vec_perm_##TYPE (TYPE values1, TYPE values2, MASKTYPE mask)	\
+TYPE __attribute__ ((noinline, noclone)) 				\
+vec_perm_##TYPE (TYPE values1, TYPE values2, MASKTYPE mask)		\
 {									\
   return __builtin_shuffle (values1, values2, mask);			\
 }
@@ -20,8 +24,9 @@ VEC_PERM (v16hi, v16hi);
 VEC_PERM (v32qi, v32qi);
 VEC_PERM (v4df, v4di);
 VEC_PERM (v8sf, v8si);
+VEC_PERM (v16hf, v16hi);
 
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_overrange_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_overrange_run.c
index 630e30867cb..6ab82250d4c 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_overrange_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_overrange_run.c
@@ -1,8 +1,8 @@
 /* { dg-do run { target aarch64_sve_hw } } */
 /* { dg-options "-O -march=armv8-a+sve" } */
+/* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" { target aarch64_sve256_hw } } */
 
 #include "sve_vec_perm_1.c"
-extern void abort (void);
 
 #define TEST_VEC_PERM(TYPE, MASK_TYPE, EXPECTED_RESULT,			\
 		      VALUES1, VALUES2, MASK)				\
@@ -14,7 +14,7 @@ extern void abort (void);
   TYPE dest;								\
   dest = vec_perm_##TYPE (values1, values2, mask);			\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
 }
 
 int main (void)
@@ -92,5 +92,20 @@ int main (void)
 			   15 + (16 * 4), 7 + (16 * 4),
 			   6 + (16 * 3), 5 + (16 * 2),
 			   4 + (16 * 1), 10 + (16 * 0) }));
+  TEST_VEC_PERM (v16hf, v16hi,
+		 ((v16hf) { 12.0, 16.0, 18.0, 10.0, 42.0, 43.0, 44.0, 34.0,
+			    7.0, 48.0, 3.0, 35.0, 9.0, 8.0, 7.0, 13.0 }),
+		 ((v16hf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
+			    11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0 }),
+		 ((v16hf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0,
+			    41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0 }),
+		 ((v16hi) { 9 + (32 * 2), 13 + (32 * 2),
+			    15 + (32 * 8), 7 + (32 * 9),
+			    25 + (32 * 4), 26 + (32 * 3),
+			    27 + (32 * 1), 17 + (32 * 2),
+			    4 + (32 * 6), 31 + (32 * 7),
+			    0 + (32 * 8), 18 + (32 * 9),
+			    6 + (32 * 6), 5 + (32 * 7),
+			    4 + (32 * 2), 10 + (32 * 2) }));
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_run.c
index ce8cc79728a..4d46ff02192 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_1_run.c
@@ -3,7 +3,6 @@
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" { target aarch64_sve256_hw } } */
 
 #include "sve_vec_perm_1.c"
-extern void abort (void);
 
 #define TEST_VEC_PERM(TYPE, MASK_TYPE, EXPECTED_RESULT,			\
 		      VALUES1, VALUES2, MASK)				\
@@ -15,7 +14,7 @@ extern void abort (void);
   TYPE dest;								\
   dest = vec_perm_##TYPE (values1, values2, mask);			\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
 }
 
 int main (void)
@@ -67,5 +66,14 @@ int main (void)
 		 ((v8sf) { 33.2, 34.2, 35.2, 36.2,
 			   37.2, 38.2, 39.2, 40.2 }),
 		 ((v8si) { 9, 13, 15, 7, 6, 5, 4, 10 }));
+  TEST_VEC_PERM (v16hf, v16hi,
+		 ((v16hf) { 12.0, 16.0, 18.0, 10.0, 42.0, 43.0, 44.0, 34.0,
+			    7.0, 48.0, 3.0, 35.0, 9.0, 8.0, 7.0, 13.0 }),
+		 ((v16hf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
+			    11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0 }),
+		 ((v16hf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0,
+			    41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0 }),
+		 ((v16hi) { 9, 13, 15, 7, 25, 26, 27, 17,
+			    4, 31, 0, 18, 6, 5, 4, 10 }));
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1.c
index d26d0902165..e76b3bc5abb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1.c
@@ -1,15 +1,19 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define VEC_PERM_CONST(TYPE, MASK)			\
-TYPE vec_perm_##TYPE (TYPE values1, TYPE values2)	\
+TYPE __attribute__ ((noinline, noclone)) 		\
+vec_perm_##TYPE (TYPE values1, TYPE values2)		\
 {							\
   return __builtin_shuffle (values1, values2, MASK);	\
 }
@@ -24,8 +28,10 @@ VEC_PERM_CONST (v32qi, ((v32qi) { 13, 31, 11, 2, 48, 28, 3, 4,
 				  2, 57, 22, 11, 6, 16, 18, 21 }));
 VEC_PERM_CONST (v4df,  ((v4di) { 7, 3, 2, 1 }));
 VEC_PERM_CONST (v8sf,  ((v8si) { 1, 9, 13, 11, 2, 5, 4, 2 }));
+VEC_PERM_CONST (v16hf, ((v16hi) { 8, 27, 5, 4, 21, 12, 13, 0,
+				  22, 1, 8, 9, 3, 24, 15, 1 }));
 
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_overrun.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_overrun.c
index 8507cb46fb9..b4f82091f7c 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_overrun.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_overrun.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define VEC_PERM_CONST_OVERRUN(TYPE, MASK)			\
 TYPE vec_perm_overrun_##TYPE (TYPE values1, TYPE values2)	\
@@ -50,8 +53,16 @@ VEC_PERM_CONST_OVERRUN (v8sf,  ((v8si) { 1 + (16 * 1), 9 + (16 * 2),
 					 13 + (16 * 2), 11 + (16 * 3),
 					 2 + (16 * 2), 5 + (16 * 2),
 					 4 + (16 * 4), 2 + (16 * 3) }));
+VEC_PERM_CONST_OVERRUN (v16hf, ((v16hi) { 8 + (32 * 3), 27 + (32 * 1),
+					  5 + (32 * 3), 4 + (32 * 3),
+					  21 + (32 * 1), 12 + (32 * 3),
+					  13 + (32 * 3), 0 + (32 * 1),
+					  22 + (32 * 2), 1 + (32 * 2),
+					  8 + (32 * 2), 9 + (32 * 1),
+					  3 + (32 * 2), 24 + (32 * 2),
+					  15 + (32 * 1), 1 + (32 * 1) }));
 
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 4 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 4 } } */
-/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_run.c
index 7edda5398e2..7324c1da0a4 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_1_run.c
@@ -4,7 +4,6 @@
 
 #include "sve_vec_perm_const_1.c"
 #include "sve_vec_perm_const_1_overrun.c"
-extern void abort (void);
 
 #define TEST_VEC_PERM(TYPE, EXPECTED_RESULT, VALUES1, VALUES2)		\
 {									\
@@ -14,11 +13,11 @@ extern void abort (void);
   TYPE dest;								\
   dest = vec_perm_##TYPE (values1, values2);				\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
   TYPE dest2;								\
   dest2 = vec_perm_overrun_##TYPE (values1, values2);			\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
 }
 
 int main (void)
@@ -60,5 +59,12 @@ int main (void)
 		 ((v8sf) { 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5, 10.5 }),
 		 ((v8sf) { 33.5, 34.5, 35.5, 36.5,
 			   37.5, 38.5, 39.5, 40.5 }));
+  TEST_VEC_PERM (v16hf,
+		 ((v16hf) { 11.0, 44.0, 8.0, 7.0, 38.0, 15.0, 16.0, 3.0,
+			    39.0, 4.0, 11.0, 12.0, 6.0, 41.0, 18.0, 4.0 }),
+		 ((v16hf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
+			    12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0 }),
+		 ((v16hf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0,
+			    41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0 }));
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1.c
index c1e12faa850..a4efb4fea79 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define VEC_PERM_SINGLE(TYPE, MASK)			\
 TYPE vec_perm_##TYPE (TYPE values1, TYPE values2)	\
@@ -24,8 +27,10 @@ VEC_PERM_SINGLE (v32qi, ((v32qi) { 13, 21, 11, 2, 8, 28, 3, 4,
 				   2, 7, 22, 11, 6, 16, 18, 21 }));
 VEC_PERM_SINGLE (v4df,  ((v4di) { 3, 3, 1, 1 }));
 VEC_PERM_SINGLE (v8sf,  ((v8si) { 4, 5, 6, 0, 2, 7, 4, 2 }));
+VEC_PERM_SINGLE (v16hf, ((v16hi) { 8, 7, 5, 4, 11, 12, 13, 0,
+				   1, 1, 8, 9, 3, 14, 15, 1 }));
 
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1_run.c
index 2aa08f59590..fbae30c8d1c 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_const_single_1_run.c
@@ -3,7 +3,6 @@
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" { target aarch64_sve256_hw } } */
 
 #include "sve_vec_perm_const_single_1.c"
-extern void abort (void);
 
 #define TEST_VEC_PERM(TYPE, EXPECTED_RESULT, VALUES1, VALUES2)		\
 {									\
@@ -13,7 +12,7 @@ extern void abort (void);
   TYPE dest;								\
   dest = vec_perm_##TYPE (values1, values2);				\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
 }
 
 int main (void)
@@ -55,5 +54,12 @@ int main (void)
 		 ((v8sf) { 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5, 10.5 }),
 		 ((v8sf) { 33.5, 34.5, 35.5, 36.5,
 			   37.5, 38.5, 39.5, 40.5 }));
+  TEST_VEC_PERM (v16hf,
+		 ((v16hf) { 11.0, 10.0, 8.0, 7.0, 14.0, 15.0, 16.0, 3.0,
+			    4.0, 4.0, 11.0, 12.0, 6.0, 17.0, 18.0, 4.0 }),
+		 ((v16hf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
+			    11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0 }),
+		 ((v16hf) { 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0,
+			    41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0 }));
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1.c
index 54c3a3068b0..a82b57dc378 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
 /* { dg-options "-O -march=armv8-a+sve -msve-vector-bits=256" } */
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define VEC_PERM(TYPE, MASKTYPE)			\
 TYPE vec_perm_##TYPE (TYPE values, MASKTYPE mask)	\
@@ -14,14 +17,15 @@ TYPE vec_perm_##TYPE (TYPE values, MASKTYPE mask)	\
   return __builtin_shuffle (values, mask);		\
 }
 
-VEC_PERM (v4di, v4di);				\
-VEC_PERM (v8si, v8si);				\
-VEC_PERM (v16hi, v16hi);			\
-VEC_PERM (v32qi, v32qi);			\
-VEC_PERM (v4df, v4di);				\
-VEC_PERM (v8sf, v8si);
+VEC_PERM (v4di, v4di)
+VEC_PERM (v8si, v8si)
+VEC_PERM (v16hi, v16hi)
+VEC_PERM (v32qi, v32qi)
+VEC_PERM (v4df, v4di)
+VEC_PERM (v8sf, v8si)
+VEC_PERM (v16hf, v16hi)
 
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 2 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
-/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
 /* { dg-final { scan-assembler-times {\ttbl\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1_run.c
index 6caa1f95cfd..539c99d4f61 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_vec_perm_single_1_run.c
@@ -5,7 +5,7 @@
 #include "sve_vec_perm_single_1.c"
 extern void abort (void);
 
-#define TEST_VEC_PERM(TYPE, MASK_TYPE, EXPECTED_RESULT,	VALUES, MASK)	\
+#define TEST_VEC_PERM(TYPE, MASK_TYPE, EXPECTED_RESULT, VALUES, MASK)	\
 {									\
   TYPE expected_result = EXPECTED_RESULT;				\
   TYPE values = VALUES;							\
@@ -13,7 +13,7 @@ extern void abort (void);
   TYPE dest;								\
   dest = vec_perm_##TYPE (values, mask);				\
   if (__builtin_memcmp (&dest, &expected_result, sizeof (TYPE)) != 0)	\
-    abort ();								\
+    __builtin_abort ();							\
 }
 
 int main (void)
@@ -54,5 +54,12 @@ int main (void)
 		 ((v8sf) { 4.2, 8.2, 10.2, 10.2, 9.2, 8.2, 7.2, 5.2 }),
 		 ((v8sf) { 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 9.2, 10.2 }),
 		 ((v8si) { 9, 13, 15, 7, 6, 5, 4, 10 }));
+  TEST_VEC_PERM (v16hf, v16hi,
+		 ((v16hf) { 12.0, 16.0, 18.0, 10.0, 12.0, 13.0, 14.0, 4.0,
+			    7.0, 18.0, 3.0, 5.0, 9.0, 8.0, 7.0, 13.0 }),
+		 ((v16hf) { 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
+			    11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0 }),
+		 ((v16hi) { 9, 13, 15, 7, 25, 26, 27, 17,
+			    4, 31, 0, 18, 6, 5, 4, 10 }));
   return 0;
 }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_zip1_1.c b/gcc/testsuite/gcc.target/aarch64/sve_zip1_1.c
index 509dddcb100..918313f62bd 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_zip1_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_zip1_1.c
@@ -5,12 +5,15 @@
 #define BIAS 0
 #endif
 
-typedef long v4di __attribute__((vector_size (32)));
-typedef int v8si __attribute__((vector_size (32)));
-typedef short v16hi __attribute__((vector_size (32)));
-typedef char v32qi __attribute__((vector_size (32)));
+#include <stdint.h>
+
+typedef int64_t v4di __attribute__((vector_size (32)));
+typedef int32_t v8si __attribute__((vector_size (32)));
+typedef int16_t v16hi __attribute__((vector_size (32)));
+typedef int8_t v32qi __attribute__((vector_size (32)));
 typedef double v4df __attribute__((vector_size (32)));
 typedef float v8sf __attribute__((vector_size (32)));
+typedef _Float16 v16hf __attribute__((vector_size (32)));
 
 #define MASK_2(X, Y) X, Y + X
 #define MASK_4(X, Y) MASK_2 (X, Y), MASK_2 (X + 1, Y)
@@ -38,7 +41,8 @@ typedef float v8sf __attribute__((vector_size (32)));
   T (v16hi, 16)					\
   T (v32qi, 32)					\
   T (v4df, 4)					\
-  T (v8sf, 8)
+  T (v8sf, 8)					\
+  T (v16hf, 16)
 
 TEST_ALL (PERMUTE)
 
@@ -46,5 +50,5 @@ TEST_ALL (PERMUTE)
 
 /* { dg-final { scan-assembler-times {\tzip1\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d} 2 } } */
 /* { dg-final { scan-assembler-times {\tzip1\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s} 2 } } */
-/* { dg-final { scan-assembler-times {\tzip1\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 1 } } */
+/* { dg-final { scan-assembler-times {\tzip1\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 2 } } */
 /* { dg-final { scan-assembler-times {\tzip1\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve_zip2_1.c b/gcc/testsuite/gcc.target/aarch64/sve_zip2_1.c
index 360ffab7d3e..40a899bc40a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve_zip2_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve_zip2_1.c
@@ -8,5 +8,5 @@
 
 /* { dg-final { scan-assembler-times {\tzip2\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d} 2 } } */
 /* { dg-final { scan-assembler-times {\tzip2\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s} 2 } } */
-/* { dg-final { scan-assembler-times {\tzip2\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 1 } } */
+/* { dg-final { scan-assembler-times {\tzip2\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h} 2 } } */
 /* { dg-final { scan-assembler-times {\tzip2\tz[0-9]+\.b, z[0-9]+\.b, z[0-9]+\.b} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/target_attr_11.c b/gcc/testsuite/gcc.target/aarch64/target_attr_11.c
index 7cfb826fc44..a3df438206b 100644
--- a/gcc/testsuite/gcc.target/aarch64/target_attr_11.c
+++ b/gcc/testsuite/gcc.target/aarch64/target_attr_11.c
@@ -10,4 +10,4 @@ foo (int a)
 }
 
 /* { dg-error "does not allow a negated form" "" { target *-*-* } 0 } */
-/* { dg-error "is invalid" "" { target *-*-* } 0 } */
+/* { dg-error "is not valid" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.target/aarch64/target_attr_12.c b/gcc/testsuite/gcc.target/aarch64/target_attr_12.c
index 39cb9964003..8a3a25bfed7 100644
--- a/gcc/testsuite/gcc.target/aarch64/target_attr_12.c
+++ b/gcc/testsuite/gcc.target/aarch64/target_attr_12.c
@@ -10,4 +10,4 @@ foo (int a)
 }
 
 /* { dg-error "does not accept an argument" "" { target *-*-* } 0 } */
-/* { dg-error "is invalid" "" { target *-*-* } 0 } */
+/* { dg-error "is not valid" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.target/aarch64/target_attr_17.c b/gcc/testsuite/gcc.target/aarch64/target_attr_17.c
index 483cc6d4a1d..2a7a7511bea 100644
--- a/gcc/testsuite/gcc.target/aarch64/target_attr_17.c
+++ b/gcc/testsuite/gcc.target/aarch64/target_attr_17.c
@@ -5,4 +5,4 @@ foo (int a)
   return a + 5;
 }
 
-/* { dg-error "target attribute.*is invalid" "" { target *-*-* } 0 } */
\ No newline at end of file
+/* { dg-error "attribute 'target\\(\"invalid-attr-string\"\\)' is not valid" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-vcvt.c b/gcc/testsuite/gcc.target/aarch64/vect-vcvt.c
index a1422d7090b..436399c6195 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-vcvt.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-vcvt.c
@@ -56,13 +56,13 @@ TEST (SUFFIX, q, 32, 4, u,u,s)			\
 TEST (SUFFIX, q, 64, 2, u,u,d)			\
 
 BUILD_VARIANTS ( )
-/* { dg-final { scan-assembler "fcvtzs\\tw\[0-9\]+, s\[0-9\]+" } } */
-/* { dg-final { scan-assembler "fcvtzs\\tx\[0-9\]+, d\[0-9\]+" } } */
+/* { dg-final { scan-assembler "fcvtzs\\t(w|s)\[0-9\]+, s\[0-9\]+" } } */
+/* { dg-final { scan-assembler "fcvtzs\\t(x|d)\[0-9\]+, d\[0-9\]+" } } */
 /* { dg-final { scan-assembler "fcvtzs\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s" } } */
 /* { dg-final { scan-assembler "fcvtzs\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s" } } */
 /* { dg-final { scan-assembler "fcvtzs\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d" } } */
-/* { dg-final { scan-assembler "fcvtzu\\tw\[0-9\]+, s\[0-9\]+" } } */
-/* { dg-final { scan-assembler "fcvtzu\\tx\[0-9\]+, d\[0-9\]+" } } */
+/* { dg-final { scan-assembler "fcvtzu\\t(w|s)\[0-9\]+, s\[0-9\]+" } } */
+/* { dg-final { scan-assembler "fcvtzu\\t(x|d)\[0-9\]+, d\[0-9\]+" } } */
 /* { dg-final { scan-assembler "fcvtzu\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s" } } */
 /* { dg-final { scan-assembler "fcvtzu\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s" } } */
 /* { dg-final { scan-assembler "fcvtzu\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d" } } */
diff --git a/gcc/testsuite/gcc.target/alpha/sqrt.c b/gcc/testsuite/gcc.target/alpha/sqrt.c
new file mode 100644
index 00000000000..a3c8b243ae4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/alpha/sqrt.c
@@ -0,0 +1,25 @@
+/* glibc bug, https://sourceware.org/ml/libc-alpha/2017-04/msg00256.html
+   When using software completions, we have to prevent assembler to match
+   input and output operands of sqrtt/sqrtf insn.  Fixed in glibc 2.26.  */
+/* { dg-do run } */
+/* { dg-options "-O2 -fno-builtin-sqrt -mieee" } */
+
+double sqrt (double);
+
+static double
+float64frombits (unsigned long b)
+{
+  union { unsigned long __b; double __d; } u = { .__b = b };
+  return u.__d;
+}
+
+int
+main (void)
+{
+  double a = float64frombits (2);
+
+  if (sqrt (a) != 3.1434555694052576e-162)
+    __builtin_abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/arc/loop-1.c b/gcc/testsuite/gcc.target/arc/loop-1.c
new file mode 100755
index 00000000000..274bb4623c9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arc/loop-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+/* Check how we handle empty body loops.  */
+
+int a;
+void fn1(void) {
+  int i;
+  for (; i < 8; i++) {
+    double A[a];
+  }
+}
diff --git a/gcc/testsuite/gcc.target/arm/peep-ldrd-1.c b/gcc/testsuite/gcc.target/arm/peep-ldrd-1.c
index eb2b86ee7b6..d49eff6b87e 100644
--- a/gcc/testsuite/gcc.target/arm/peep-ldrd-1.c
+++ b/gcc/testsuite/gcc.target/arm/peep-ldrd-1.c
@@ -8,4 +8,4 @@ int foo(int a, int b, int* p, int *q)
   *p = a;
   return a;
 }
-/* { dg-final { scan-assembler "ldrd" } } */
+/* { dg-final { scan-assembler "ldrd\\t" } } */
diff --git a/gcc/testsuite/gcc.target/arm/peep-ldrd-2.c b/gcc/testsuite/gcc.target/arm/peep-ldrd-2.c
new file mode 100644
index 00000000000..6822c2b1454
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/peep-ldrd-2.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target arm_prefer_ldrd_strd } */
+/* { dg-options "-O2 -mno-unaligned-access" }  */
+int foo(int a, int b, int* p, int *q)
+{
+  a = p[2] + p[3];
+  *q = a;
+  *p = a;
+  return a;
+}
+/* { dg-final { scan-assembler-not "ldrd\\t" } } */
diff --git a/gcc/testsuite/gcc.target/arm/peep-strd-1.c b/gcc/testsuite/gcc.target/arm/peep-strd-1.c
index bd330769599..fe1beac7229 100644
--- a/gcc/testsuite/gcc.target/arm/peep-strd-1.c
+++ b/gcc/testsuite/gcc.target/arm/peep-strd-1.c
@@ -6,4 +6,4 @@ void foo(int a, int b, int* p)
   p[2] = a;
   p[3] = b;
 }
-/* { dg-final { scan-assembler "strd" } } */
+/* { dg-final { scan-assembler "strd\\t" } } */
diff --git a/gcc/testsuite/gcc.target/arm/peep-strd-2.c b/gcc/testsuite/gcc.target/arm/peep-strd-2.c
new file mode 100644
index 00000000000..bfc5ebe9eec
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/peep-strd-2.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target arm_prefer_ldrd_strd } */
+/* { dg-options "-O2 -mno-unaligned-access" }  */
+void foo(int a, int b, int* p)
+{
+  p[2] = a;
+  p[3] = b;
+}
+/* { dg-final { scan-assembler-not "strd\\t" } } */
diff --git a/gcc/testsuite/gcc.target/arm/require-pic-register-loc.c b/gcc/testsuite/gcc.target/arm/require-pic-register-loc.c
index bd85e8640c2..268e9e42667 100644
--- a/gcc/testsuite/gcc.target/arm/require-pic-register-loc.c
+++ b/gcc/testsuite/gcc.target/arm/require-pic-register-loc.c
@@ -18,12 +18,12 @@ main (int argc)        /* line 9.  */
   return 0;
 }
 
-/* { dg-final { scan-assembler-not "\.loc 1 7 0" } } */
-/* { dg-final { scan-assembler-not "\.loc 1 8 0" } } */
-/* { dg-final { scan-assembler-not "\.loc 1 9 0" } } */
+/* { dg-final { scan-assembler-not "\.loc 1 7 \[0-9\]\+" } } */
+/* { dg-final { scan-assembler-not "\.loc 1 8 \[0-9\]\+" } } */
+/* { dg-final { scan-assembler-not "\.loc 1 9 \[0-9\]\+" } } */
 
 /* The loc at the start of the prologue.  */
-/* { dg-final { scan-assembler-times "\.loc 1 10 0" 1 } } */
+/* { dg-final { scan-assembler-times "\.loc 1 10 \[0-9\]\+" 1 } } */
 
 /* The loc at the end of the prologue, with the first user line.  */
-/* { dg-final { scan-assembler-times "\.loc 1 11 0" 1 } } */
+/* { dg-final { scan-assembler-times "\.loc 1 11 \[0-9\]\+" 1 } } */
diff --git a/gcc/testsuite/gcc.target/arm/simd/vdot-exec.c b/gcc/testsuite/gcc.target/arm/simd/vdot-exec.c
new file mode 100644
index 00000000000..054f4703394
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/simd/vdot-exec.c
@@ -0,0 +1,55 @@
+/* { dg-do run } */
+/* { dg-additional-options "-O3" } */
+/* { dg-require-effective-target arm_v8_2a_dotprod_neon_hw } */
+/* { dg-add-options arm_v8_2a_dotprod_neon }  */
+
+#include <arm_neon.h>
+
+extern void abort();
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+# define ORDER(x, y) y
+#else
+# define ORDER(x, y) x - y
+#endif
+
+#define P(n1,n2) n1,n1,n1,n1,n2,n2,n2,n2
+#define ARR(nm, p, ty, ...) ty nm##_##p = { __VA_ARGS__ }
+#define TEST(t1, t2, t3, f, r1, r2, n1, n2) \
+	ARR(f, x, t1, r1);		    \
+	ARR(f, y, t2, r2);		    \
+	t3 f##_##r = {0};		    \
+	f##_##r = f (f##_##r, f##_##x, f##_##y);  \
+	if (f##_##r[0] != n1 || f##_##r[1] != n2)   \
+	  abort ();
+
+#define TEST_LANE(t1, t2, t3, f, r1, r2, n1, n2, n3, n4) \
+	ARR(f, x, t1, r1);		    \
+	ARR(f, y, t2, r2);		    \
+	t3 f##_##rx = {0};		    \
+	f##_##rx = f (f##_##rx, f##_##x, f##_##y, ORDER (1, 0));  \
+	if (f##_##rx[0] != n1 || f##_##rx[1] != n2)   \
+	  abort ();				    \
+	t3 f##_##rx1 = {0};			    \
+	f##_##rx1 =  f (f##_##rx1, f##_##x, f##_##y, ORDER (1, 1));  \
+	if (f##_##rx1[0] != n3 || f##_##rx1[1] != n4)   \
+	  abort (); \
+
+int
+main()
+{
+  TEST (uint8x8_t, uint8x8_t, uint32x2_t, vdot_u32, P(1,2), P(2,3), 8, 24);
+  TEST (int8x8_t, int8x8_t, int32x2_t, vdot_s32, P(1,2), P(-2,-3), -8, -24);
+
+  TEST (uint8x16_t, uint8x16_t, uint32x4_t, vdotq_u32, P(1,2), P(2,3), 8, 24);
+  TEST (int8x16_t, int8x16_t, int32x4_t, vdotq_s32, P(1,2), P(-2,-3), -8, -24);
+
+  TEST_LANE (uint8x8_t, uint8x8_t, uint32x2_t, vdot_lane_u32, P(1,2), P(2,3), 8, 16, 12, 24);
+
+  TEST_LANE (int8x8_t, int8x8_t, int32x2_t, vdot_lane_s32, P(1,2), P(-2,-3), -8, -16, -12, -24);
+
+  TEST_LANE (uint8x16_t, uint8x8_t, uint32x4_t, vdotq_lane_u32, P(1,2), P(2,3), 8, 16, 12, 24);
+  TEST_LANE (int8x16_t, int8x8_t, int32x4_t, vdotq_lane_s32, P(1,2), P(-2,-3), -8, -16, -12, -24);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/387-ficom-1.c b/gcc/testsuite/gcc.target/i386/387-ficom-1.c
index 8c73ddcb2da..325698854a9 100644
--- a/gcc/testsuite/gcc.target/i386/387-ficom-1.c
+++ b/gcc/testsuite/gcc.target/i386/387-ficom-1.c
@@ -37,5 +37,5 @@ int test_ld_i (int x)
   return (long double)i != x;
 }
 
-/* { dg-final { scan-assembler-times "ficomps" 3 } } */
+/* { dg-final { scan-assembler-times "ficomp\[s\t\]" 3 } } */
 /* { dg-final { scan-assembler-times "ficompl" 3 } } */
diff --git a/gcc/testsuite/gcc.target/i386/387-ficom-2.c b/gcc/testsuite/gcc.target/i386/387-ficom-2.c
index 4190ebaae71..d5283684f8e 100644
--- a/gcc/testsuite/gcc.target/i386/387-ficom-2.c
+++ b/gcc/testsuite/gcc.target/i386/387-ficom-2.c
@@ -5,5 +5,5 @@
 
 #include "387-ficom-1.c"
 
-/* { dg-final { scan-assembler-times "ficomps" 3 } } */
+/* { dg-final { scan-assembler-times "ficomp\[s\t\]" 3 } } */
 /* { dg-final { scan-assembler-times "ficompl" 3 } } */
diff --git a/gcc/testsuite/gcc.target/i386/attr-nocf-check-1a.c b/gcc/testsuite/gcc.target/i386/attr-nocf-check-1a.c
new file mode 100644
index 00000000000..9549e697658
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/attr-nocf-check-1a.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection -mcet" } */
+
+int func (int) __attribute__ ((nocf_check));
+int (*fptr) (int) __attribute__ ((nocf_check));
+typedef void (*nocf_check_t) (void) __attribute__ ((nocf_check));
+
+int
+foo1 (int arg)
+{
+  return func (arg) + fptr (arg);
+}
+
+void
+foo2 (void (*foo) (void))
+{
+  void (*func) (void) __attribute__((nocf_check)) = foo; /* { dg-warning "incompatible pointer type" "" { target c } } */
+							 /* { dg-error "invalid conversion" "" { target c++ } .-1 } */
+  func ();
+}
+
+void
+foo3 (nocf_check_t foo)
+{
+  foo ();
+}
+
+void
+foo4 (void (*foo) (void) __attribute__((nocf_check)))
+{
+  foo ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/attr-nocf-check-3a.c b/gcc/testsuite/gcc.target/i386/attr-nocf-check-3a.c
new file mode 100644
index 00000000000..1a833012409
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/attr-nocf-check-3a.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection -mcet" } */
+
+int  foo (void) __attribute__ ((nocf_check));
+void (*foo1) (void) __attribute__((nocf_check));
+void (*foo2) (void);
+
+int __attribute__ ((nocf_check))
+foo (void) /* The function's address is not tracked.  */
+{
+  /* This call site is not tracked for
+     control-flow instrumentation.  */
+  (*foo1)();
+
+  foo1 = foo2; /* { dg-warning "incompatible pointer type" "" { target c } } */
+	       /* { dg-error "invalid conversion" "" { target c++ } .-1 } */
+  /* This call site is still not tracked for
+     control-flow instrumentation.  */
+  (*foo1)();
+
+  /* This call site is tracked for
+     control-flow instrumentation.  */
+  (*foo2)();
+
+  foo2 = foo1; /* { dg-warning "incompatible pointer type" "" { target c } } */
+	       /* { dg-error "invalid conversion" "" { target c++ } .-1 } */
+  /* This call site is still tracked for
+     control-flow instrumentation.  */
+  (*foo2)();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx-1.c b/gcc/testsuite/gcc.target/i386/avx-1.c
index 085ba81a672..46238265ae6 100644
--- a/gcc/testsuite/gcc.target/i386/avx-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -Werror-implicit-function-declaration -march=k8 -m3dnow -mavx -mavx2 -maes -mpclmul" } */
+/* { dg-options "-O2 -Werror-implicit-function-declaration -march=k8 -m3dnow -mavx -mavx2 -maes -mpclmul -mgfni" } */
 /* { dg-add-options bind_pic_locally } */
 
 #include <mm_malloc.h>
@@ -412,8 +412,8 @@
 /* avx512dqintrin.h */
 #define __builtin_ia32_kshiftliqi(A, B) __builtin_ia32_kshiftliqi(A, 8)
 #define __builtin_ia32_kshiftriqi(A, B) __builtin_ia32_kshiftriqi(A, 8)
-#define __builtin_ia32_reducess(A, B, F) __builtin_ia32_reducess(A, B, 1)
-#define __builtin_ia32_reducesd(A, B, F) __builtin_ia32_reducesd(A, B, 1)
+#define __builtin_ia32_reducess_mask(A, B, F, W, U) __builtin_ia32_reducess_mask(A, B, 1, W, U)
+#define __builtin_ia32_reducesd_mask(A, B, F, W, U) __builtin_ia32_reducesd_mask(A, B, 1, W, U)
 #define __builtin_ia32_reduceps512_mask(A, E, C, D) __builtin_ia32_reduceps512_mask(A, 1, C, D)
 #define __builtin_ia32_reducepd512_mask(A, E, C, D) __builtin_ia32_reducepd512_mask(A, 1, C, D)
 #define __builtin_ia32_rangess128_round(A, B, I, F) __builtin_ia32_rangess128_round(A, B, 1, 8)
@@ -603,6 +603,16 @@
 #define __builtin_ia32_extracti64x2_256_mask(A, E, C, D) __builtin_ia32_extracti64x2_256_mask(A, 1, C, D)
 #define __builtin_ia32_extractf64x2_256_mask(A, E, C, D) __builtin_ia32_extractf64x2_256_mask(A, 1, C, D)
 
+/* gfniintrin.h */
+#define __builtin_ia32_vgf2p8affineinvqb_v16qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v16qi(A, B, 1) 
+#define __builtin_ia32_vgf2p8affineinvqb_v32qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v32qi(A, B, 1)
+#define __builtin_ia32_vgf2p8affineinvqb_v64qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v64qi(A, B, 1)
+#define __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(A, B, 1, D, E) 
+#define __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(A, B, 1, D, E) 
+#define __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(A, B, 1, D, E) 
+
+
+
 #include <wmmintrin.h>
 #include <immintrin.h>
 #include <mm3dnow.h>
diff --git a/gcc/testsuite/gcc.target/i386/avx-pr82370.c b/gcc/testsuite/gcc.target/i386/avx-pr82370.c
new file mode 100644
index 00000000000..4dc8a5bdaaf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx-pr82370.c
@@ -0,0 +1,65 @@
+/* PR target/82370 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx -mno-avx2 -masm=att" } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 3 } } */
+
+typedef short int v32hi __attribute__((vector_size (64)));
+typedef short int v16hi __attribute__((vector_size (32)));
+typedef short int v8hi __attribute__((vector_size (16)));
+typedef int v16si __attribute__((vector_size (64)));
+typedef int v8si __attribute__((vector_size (32)));
+typedef int v4si __attribute__((vector_size (16)));
+typedef long long int v8di __attribute__((vector_size (64)));
+typedef long long int v4di __attribute__((vector_size (32)));
+typedef long long int v2di __attribute__((vector_size (16)));
+typedef unsigned short int v32uhi __attribute__((vector_size (64)));
+typedef unsigned short int v16uhi __attribute__((vector_size (32)));
+typedef unsigned short int v8uhi __attribute__((vector_size (16)));
+typedef unsigned int v16usi __attribute__((vector_size (64)));
+typedef unsigned int v8usi __attribute__((vector_size (32)));
+typedef unsigned int v4usi __attribute__((vector_size (16)));
+typedef unsigned long long int v8udi __attribute__((vector_size (64)));
+typedef unsigned long long int v4udi __attribute__((vector_size (32)));
+typedef unsigned long long int v2udi __attribute__((vector_size (16)));
+
+#ifdef __AVX512F__
+v32hi f1 (v32hi *x) { return *x >> 3; }
+v32uhi f2 (v32uhi *x) { return *x >> 5; }
+v32uhi f3 (v32uhi *x) { return *x << 7; }
+#endif
+v16hi f4 (v16hi *x) { return *x >> 3; }
+v16uhi f5 (v16uhi *x) { return *x >> 5; }
+v16uhi f6 (v16uhi *x) { return *x << 7; }
+v8hi f7 (v8hi *x) { return *x >> 3; }
+v8uhi f8 (v8uhi *x) { return *x >> 5; }
+v8uhi f9 (v8uhi *x) { return *x << 7; }
+#ifdef __AVX512F__
+v16si f10 (v16si *x) { return *x >> 3; }
+v16usi f11 (v16usi *x) { return *x >> 5; }
+v16usi f12 (v16usi *x) { return *x << 7; }
+#endif
+v8si f13 (v8si *x) { return *x >> 3; }
+v8usi f14 (v8usi *x) { return *x >> 5; }
+v8usi f15 (v8usi *x) { return *x << 7; }
+v4si f16 (v4si *x) { return *x >> 3; }
+v4usi f17 (v4usi *x) { return *x >> 5; }
+v4usi f18 (v4usi *x) { return *x << 7; }
+#ifdef __AVX512F__
+v8di f19 (v8di *x) { return *x >> 3; }
+v8udi f20 (v8udi *x) { return *x >> 5; }
+v8udi f21 (v8udi *x) { return *x << 7; }
+#endif
+v4di f22 (v4di *x) { return *x >> 3; }
+v4udi f23 (v4udi *x) { return *x >> 5; }
+v4udi f24 (v4udi *x) { return *x << 7; }
+v2di f25 (v2di *x) { return *x >> 3; }
+v2udi f26 (v2udi *x) { return *x >> 5; }
+v2udi f27 (v2udi *x) { return *x << 7; }
diff --git a/gcc/testsuite/gcc.target/i386/avx2-pr82370.c b/gcc/testsuite/gcc.target/i386/avx2-pr82370.c
new file mode 100644
index 00000000000..6609ebb504a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx2-pr82370.c
@@ -0,0 +1,23 @@
+/* PR target/82370 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx2 -mno-avx512f -masm=att" } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+
+#include "avx-pr82370.c"
diff --git a/gcc/testsuite/gcc.target/i386/avx512-check.h b/gcc/testsuite/gcc.target/i386/avx512-check.h
index 9693fa46721..9390c1ab9ea 100644
--- a/gcc/testsuite/gcc.target/i386/avx512-check.h
+++ b/gcc/testsuite/gcc.target/i386/avx512-check.h
@@ -75,6 +75,9 @@ main ()
 #ifdef AVX512VPOPCNTDQ
       && (ecx & bit_AVX512VPOPCNTDQ)
 #endif
+#ifdef GFNI
+      && (ecx & bit_GFNI)
+#endif
       && avx512f_os_support ())
     {
       DO_TEST ();
diff --git a/gcc/testsuite/gcc.target/i386/avx512bw-pr82370.c b/gcc/testsuite/gcc.target/i386/avx512bw-pr82370.c
new file mode 100644
index 00000000000..174f499a885
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512bw-pr82370.c
@@ -0,0 +1,33 @@
+/* PR target/82370 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512bw -mno-avx512vl -masm=att" } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vps\[lr]\[la]\[dwq]\[ \t]\+\\\$\[357], %zmm\[0-9]\+, %zmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+
+#include "avx-pr82370.c"
diff --git a/gcc/testsuite/gcc.target/i386/avx512bw-vpermt2w-1.c b/gcc/testsuite/gcc.target/i386/avx512bw-vpermt2w-1.c
index be8737ec785..a734cb600ce 100644
--- a/gcc/testsuite/gcc.target/i386/avx512bw-vpermt2w-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512bw-vpermt2w-1.c
@@ -1,14 +1,14 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512bw -mavx512vl -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } *
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2w\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } *
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2w\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512dq-vreducesd-1.c b/gcc/testsuite/gcc.target/i386/avx512dq-vreducesd-1.c
index b7549fada36..b8f24a0ccbd 100644
--- a/gcc/testsuite/gcc.target/i386/avx512dq-vreducesd-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512dq-vreducesd-1.c
@@ -2,13 +2,24 @@
 /* { dg-options "-mavx512dq -O2" } */
 /* { dg-final { scan-assembler-times "vreducesd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
 
+/* { dg-final { scan-assembler-times "vreducesd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vreducesd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vreducesd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+
+
 #include <immintrin.h>
 
+#define IMM 123
+
 volatile __m128d x1, x2;
 volatile __mmask8 m;
 
 void extern
 avx512dq_test (void)
 {
-  x1 = _mm_reduce_sd (x1, x2, 123);
+  x1 = _mm_reduce_sd (x1, x2, IMM);
+
+  x1 = _mm_mask_reduce_sd(x1, m, x1, x2, IMM);
+
+  x1 = _mm_maskz_reduce_sd(m, x1, x2, IMM);
 }
diff --git a/gcc/testsuite/gcc.target/i386/avx512dq-vreducesd-2.c b/gcc/testsuite/gcc.target/i386/avx512dq-vreducesd-2.c
new file mode 100644
index 00000000000..93e18271cbd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512dq-vreducesd-2.c
@@ -0,0 +1,66 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512dq" } */
+/* { dg-require-effective-target avx512dq } */
+
+#define AVX512DQ
+#include "avx512f-helper.h"
+#include <string.h>
+
+#define SIZE (AVX512F_LEN / 64)
+#include "avx512f-mask-type.h"
+
+#define IMM 0x23
+
+void
+CALC (double *r, double *s)
+{
+  int i;
+
+  memcpy (&r[1], &s[1], sizeof(double));
+
+  for (i = 0; i < 1; i++)
+    {
+      double tmp = (int) (4 * s[i]) / 4.0;
+      r[i] = s[i] - tmp;
+    }
+}
+
+void
+TEST (void)
+{
+  union128d res1, res2, res3;
+  union128d s1, s2, src;
+  double res_ref[2];
+  MASK_TYPE mask = MASK_VALUE;
+  int j;
+
+  for (j = 0; j < 2; j++)
+    {
+      s1.a[j] = j / 123.456;
+      s2.a[j] = j / 123.456;
+      res_ref[j] = j / 123.456;
+      res1.a[j] = DEFAULT_VALUE;
+      res2.a[j] = DEFAULT_VALUE;
+      res3.a[j] = DEFAULT_VALUE;
+    }
+
+  res1.x = _mm_reduce_sd (s1.x, s2.x, IMM);
+  res2.x = _mm_mask_reduce_sd (s1.x, mask, s1.x, s2.x, IMM);
+  res3.x = _mm_maskz_reduce_sd (mask, s1.x, s2.x, IMM);
+
+  CALC (res_ref, s2.a);
+
+  if (check_union128d (res1, res_ref))
+    abort ();
+ 
+  MASK_MERGE (d) (res_ref, mask, 1);
+
+  if (check_union128d (res2, res_ref))
+    abort ();
+
+  MASK_ZERO (d) (res_ref, mask, 1);
+
+  if (check_union128d (res3, res_ref))
+    abort ();
+
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512dq-vreducess-1.c b/gcc/testsuite/gcc.target/i386/avx512dq-vreducess-1.c
index 2a6afe9643b..804074e2ba6 100644
--- a/gcc/testsuite/gcc.target/i386/avx512dq-vreducess-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512dq-vreducess-1.c
@@ -2,13 +2,23 @@
 /* { dg-options "-mavx512dq -O2" } */
 /* { dg-final { scan-assembler-times "vreducess\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
 
+/* { dg-final { scan-assembler-times "vreducess\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vreducess\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vreducess\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+
 #include <immintrin.h>
 
+#define IMM 123
+
 volatile __m128 x1, x2;
 volatile __mmask8 m;
 
 void extern
 avx512dq_test (void)
 {
-  x1 = _mm_reduce_ss (x1, x2, 123);
+  x1 = _mm_reduce_ss (x1, x2, IMM);
+
+  x1 = _mm_mask_reduce_ss (x1, m, x1, x2, IMM);
+ 
+  x1 = _mm_maskz_reduce_ss (m, x1, x2, IMM);
 }
diff --git a/gcc/testsuite/gcc.target/i386/avx512dq-vreducess-2.c b/gcc/testsuite/gcc.target/i386/avx512dq-vreducess-2.c
new file mode 100644
index 00000000000..8558c3b3468
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512dq-vreducess-2.c
@@ -0,0 +1,68 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512dq" } */
+/* { dg-require-effective-target avx512dq } */
+
+#define AVX512DQ
+#include "avx512f-helper.h"
+#include <string.h>
+
+#define SIZE (AVX512F_LEN / 64)
+#include "avx512f-mask-type.h"
+
+#define IMM 0x23
+
+void
+CALC (float *r, float *s)
+{
+  int i;
+
+  memcpy (&r[1], &s[1], 2 * sizeof(float));
+
+  for (i = 0; i < 2; i++)
+    {
+      float tmp = (int) (4 * s[i]) / 4.0;
+      r[i] = s[i] - tmp;
+    }
+}
+
+void
+TEST (void)
+{
+  printf("\nsize = %d\n\n", SIZE);
+
+  union128 res1, res2, res3;
+  union128 s1, s2, src;
+  float res_ref[4];
+  MASK_TYPE mask = MASK_VALUE;
+  int j;
+
+  for (j = 0; j < 4; j++)
+    {
+      s1.a[j] = j / 123.456;
+      s2.a[j] = j / 123.456;
+      res_ref[j] = j / 123.456;
+      res1.a[j] = DEFAULT_VALUE;
+      res2.a[j] = DEFAULT_VALUE;
+      res3.a[j] = DEFAULT_VALUE;
+    }
+
+  res1.x = _mm_reduce_ss (s1.x, s2.x, IMM);
+  res2.x = _mm_mask_reduce_ss (s1.x, mask, s1.x, s2.x, IMM);
+  res3.x = _mm_maskz_reduce_ss (mask, s1.x, s2.x, IMM);
+
+  CALC (res_ref, s2.a);
+
+  if (check_union128 (res1, res_ref))
+    abort ();
+ 
+  MASK_MERGE () (res_ref, mask, 1);
+
+  if (check_union128 (res2, res_ref))
+    abort ();
+
+  MASK_ZERO () (res_ref, mask, 1);
+
+  if (check_union128 (res3, res_ref))
+    abort ();
+
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-gf2p8affineinvqb-2.c b/gcc/testsuite/gcc.target/i386/avx512f-gf2p8affineinvqb-2.c
new file mode 100644
index 00000000000..af4839f4434
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-gf2p8affineinvqb-2.c
@@ -0,0 +1,74 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512f -mgfni -mavx512bw" } */
+/* { dg-require-effective-target avx512f } */
+/* { dg-require-effective-target gfni } */
+
+#define AVX512F
+
+#define GFNI
+#include "avx512f-helper.h"
+
+#define SIZE (AVX512F_LEN / 8)
+
+#include "avx512f-mask-type.h"
+#include <x86intrin.h>
+
+static void
+CALC (unsigned char *r, unsigned char *s1, unsigned char *s2, unsigned char imm)
+{
+  for (int a = 0; a < SIZE/8; a++)
+    {
+      for (int val = 0; val < 8; val++)
+        {
+          unsigned char result = 0;
+          for (int bit = 0; bit < 8; bit++)
+          {
+            unsigned char temp = s1[a*8 + val] & s2[a*8 + bit];
+            unsigned char parity = __popcntd(temp);
+            if (parity % 2)
+              result |= (1 << (8 - bit - 1));
+          }
+          r[a*8 + val] = result ^ imm; 
+        }
+    }
+}
+
+void
+TEST (void)
+{
+  int i;
+  UNION_TYPE (AVX512F_LEN, i_b) res1, res2, res3, src1, src2;
+  MASK_TYPE mask = MASK_VALUE;
+  char res_ref[SIZE];
+  unsigned char imm = 0;
+
+  for (i = 0; i < SIZE; i++)
+    {
+      src1.a[i] = i %2 ; // gfni inverse of 1 and 0 are 1 and 0
+      src2.a[i] = 1;
+    }
+
+  for (i = 0; i < SIZE; i++)
+    {
+      res1.a[i] = DEFAULT_VALUE;
+      res2.a[i] = DEFAULT_VALUE;
+      res3.a[i] = DEFAULT_VALUE;
+    }
+
+  CALC (res_ref, src1.a, src2.a, imm);
+
+  res1.x = INTRINSIC (_gf2p8affineinv_epi64_epi8) (src1.x, src2.x, imm);
+  res2.x = INTRINSIC (_mask_gf2p8affineinv_epi64_epi8) (res2.x, mask, src1.x, src2.x, imm);
+  res3.x = INTRINSIC (_maskz_gf2p8affineinv_epi64_epi8) (mask, src1.x, src2.x, imm);
+
+  if (UNION_CHECK (AVX512F_LEN, i_b) (res1, res_ref))
+    abort ();
+
+  MASK_MERGE (i_b) (res_ref, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_b) (res2, res_ref))
+    abort ();
+
+  MASK_ZERO (i_b) (res_ref, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_b) (res3, res_ref))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-pr82370.c b/gcc/testsuite/gcc.target/i386/avx512f-pr82370.c
new file mode 100644
index 00000000000..20ad8dccd29
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-pr82370.c
@@ -0,0 +1,33 @@
+/* PR target/82370 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512f -mno-avx512bw -mno-avx512vl -masm=att" } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vps\[lr]\[la]\[dwq]\[ \t]\+\\\$\[357], %zmm\[0-9]\+, %zmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 0 } } */
+
+#include "avx-pr82370.c"
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-1.c
index 4b53e379acc..d3c30fcedb9 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-1.c
@@ -1,7 +1,7 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -mavx512f" } */
-/* { dg-final { scan-assembler-times "vcmppd\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\](?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vcmppd\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\]\{%k\[0-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vcmppd\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\](?:\n|\[ \\t\]+#)" 9 } } */
+/* { dg-final { scan-assembler-times "vcmppd\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\]\{%k\[0-7\]\}(?:\n|\[ \\t\]+#)" 9 } } */
 /* { dg-final { scan-assembler-times "vcmppd\[ \\t\]+\[^\{\n\]*\{sae\}\[^\n\]*%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\](?:\n|\[ \\t\]+#)" 1 } } */
 /* { dg-final { scan-assembler-times "vcmppd\[ \\t\]+\[^\{\n\]*\{sae\}\[^\n\]*%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\]\{%k\[0-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
 
@@ -17,4 +17,29 @@ avx512f_test (void)
   m = _mm512_mask_cmp_pd_mask (m, x, x, _CMP_FALSE_OQ);
   m = _mm512_cmp_round_pd_mask (x, x, _CMP_FALSE_OQ, _MM_FROUND_NO_EXC);
   m = _mm512_mask_cmp_round_pd_mask (m, x, x, _CMP_FALSE_OQ, _MM_FROUND_NO_EXC);
+
+  m = _mm512_cmpeq_pd_mask (x, x);
+  m = _mm512_mask_cmpeq_pd_mask (m, x, x);
+
+  m = _mm512_cmplt_pd_mask (x, x);
+  m = _mm512_mask_cmplt_pd_mask (m, x, x);
+
+  m = _mm512_cmple_pd_mask (x, x);
+  m = _mm512_mask_cmple_pd_mask (m, x, x);
+
+  m = _mm512_cmpunord_pd_mask (x, x);
+  m = _mm512_mask_cmpunord_pd_mask (m, x, x);
+
+  m = _mm512_cmpneq_pd_mask (x, x);
+  m = _mm512_mask_cmpneq_pd_mask (m, x, x);
+
+  m = _mm512_cmpnlt_pd_mask (x, x);
+  m = _mm512_mask_cmpnlt_pd_mask (m, x, x);
+
+  m = _mm512_cmpnle_pd_mask (x, x);
+  m = _mm512_mask_cmpnle_pd_mask (m, x, x);
+
+  m = _mm512_cmpord_pd_mask (x, x);
+  m = _mm512_mask_cmpord_pd_mask (m, x, x);
 }
+
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-2.c b/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-2.c
index 52e226d9f15..cee11971399 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-2.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vcmppd-2.c
@@ -11,58 +11,69 @@
 #define SIZE (AVX512F_LEN / 64)
 #include "avx512f-mask-type.h"
 
+#undef SUF
+#undef SSIZE
+#undef GEN_CMP
+#undef CHECK_CMP
+
 #if AVX512F_LEN == 512
-#define CMP(imm, rel)					\
-    dst_ref = 0;					\
-    for (i = 0; i < 8; i++)				\
-    {							\
-      dst_ref = (((int) rel) << i) | dst_ref;		\
-    }							\
-    source1.x = _mm512_loadu_pd(s1);			\
-    source2.x = _mm512_loadu_pd(s2);			\
-    dst1 = _mm512_cmp_pd_mask(source1.x, source2.x, imm);\
-    dst2 = _mm512_mask_cmp_pd_mask(mask, source1.x, source2.x, imm);\
-    if (dst_ref != dst1) abort();			\
-    if ((dst_ref & mask) != dst2) abort();
+#define SUF(fun) _mm512##fun
+#define SSIZE 8
+
+#define GEN_CMP(type)				\
+    {						\
+    dst3 = _mm512_cmp##type##_pd_mask(source1.x, source2.x);\
+    dst4 = _mm512_mask_cmp##type##_pd_mask(mask, source1.x, source2.x);\
+    if (dst3 != dst1) abort();			\
+    if (dst4 != dst2) abort();			\
+    }
+
+#define CHECK_CMP(imm)				\
+    if (imm == _CMP_EQ_OQ) GEN_CMP(eq)		\
+    if (imm == _CMP_LT_OS) GEN_CMP(lt)		\
+    if (imm == _CMP_LE_OS) GEN_CMP(le)		\
+    if (imm == _CMP_UNORD_Q) GEN_CMP(unord)	\
+    if (imm == _CMP_NEQ_UQ) GEN_CMP(neq)	\
+    if (imm == _CMP_NLT_US) GEN_CMP(nlt)	\
+    if (imm == _CMP_NLE_US) GEN_CMP(nle)	\
+    if (imm == _CMP_ORD_Q) GEN_CMP(ord)	
+
 #endif
 
 #if AVX512F_LEN == 256
-#undef CMP
-#define CMP(imm, rel)					\
-    dst_ref = 0;					\
-    for (i = 0; i < 4; i++)				\
-    {							\
-      dst_ref = (((int) rel) << i) | dst_ref;		\
-    }							\
-    source1.x = _mm256_loadu_pd(s1);			\
-    source2.x = _mm256_loadu_pd(s2);			\
-    dst1 = _mm256_cmp_pd_mask(source1.x, source2.x, imm);\
-    dst2 = _mm256_mask_cmp_pd_mask(mask, source1.x, source2.x, imm);\
-    if (dst_ref != dst1) abort();			\
-    if ((dst_ref & mask) != dst2) abort();
+#define SUF(fun) _mm256##fun
+#define SSIZE 4
+#define GEN_CMP(type)
+#define CHECK_CMP(imm)
 #endif
 
 #if AVX512F_LEN == 128
+#define SUF(fun) _mm##fun
+#define SSIZE 2
+#define GEN_CMP(type)
+#define CHECK_CMP(imm)
+#endif
+
 #undef CMP
 #define CMP(imm, rel)					\
     dst_ref = 0;					\
-    for (i = 0; i < 2; i++)				\
+    for (i = 0; i < SSIZE; i++)				\
     {							\
       dst_ref = (((int) rel) << i) | dst_ref;		\
     }							\
-    source1.x = _mm_loadu_pd(s1);			\
-    source2.x = _mm_loadu_pd(s2);			\
-    dst1 = _mm_cmp_pd_mask(source1.x, source2.x, imm);\
-    dst2 = _mm_mask_cmp_pd_mask(mask, source1.x, source2.x, imm);\
+    source1.x = SUF(_loadu_pd)(s1);			\
+    source2.x = SUF(_loadu_pd)(s2);			\
+    dst1 = SUF(_cmp_pd_mask)(source1.x, source2.x, imm);\
+    dst2 = SUF(_mask_cmp_pd_mask)(mask, source1.x, source2.x, imm);\
     if (dst_ref != dst1) abort();			\
-    if ((dst_ref & mask) != dst2) abort();
-#endif
+    if ((dst_ref & mask) != dst2) abort();		\
+    CHECK_CMP(imm)
 
 void
 TEST ()
 {
     UNION_TYPE (AVX512F_LEN, d) source1, source2;
-    MASK_TYPE dst1, dst2, dst_ref;
+    MASK_TYPE dst1, dst2, dst3, dst4, dst_ref;
     MASK_TYPE mask = MASK_VALUE;
     int i;
     double s1[8]={2134.3343, 6678.346, 453.345635, 54646.464,
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-1.c
index 9812915a4e7..27be36070ef 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-1.c
@@ -1,7 +1,7 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -mavx512f" } */
-/* { dg-final { scan-assembler-times "vcmpps\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\](?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vcmpps\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\]\{%k\[0-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vcmpps\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\](?:\n|\[ \\t\]+#)" 9 } } */
+/* { dg-final { scan-assembler-times "vcmpps\[ \\t\]+\[^\{\n\]*\[^\}\]%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\]\{%k\[0-7\]\}(?:\n|\[ \\t\]+#)" 9 } } */
 /* { dg-final { scan-assembler-times "vcmpps\[ \\t\]+\[^\{\n\]*\{sae\}\[^\n\]*%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\](?:\n|\[ \\t\]+#)" 1 } } */
 /* { dg-final { scan-assembler-times "vcmpps\[ \\t\]+\[^\{\n\]*\{sae\}\[^\n\]*%zmm\[0-9\]+\[^\n^k\]*%k\[1-7\]\{%k\[0-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
 
@@ -17,4 +17,28 @@ avx512f_test (void)
   m = _mm512_mask_cmp_ps_mask (m, x, x, _CMP_FALSE_OQ);
   m = _mm512_cmp_round_ps_mask (x, x, _CMP_FALSE_OQ, _MM_FROUND_NO_EXC);
   m = _mm512_mask_cmp_round_ps_mask (m, x, x, _CMP_FALSE_OQ, _MM_FROUND_NO_EXC);
+
+  m = _mm512_cmpeq_ps_mask (x, x);
+  m = _mm512_mask_cmpeq_ps_mask (m, x, x);
+
+  m = _mm512_cmplt_ps_mask (x, x);
+  m = _mm512_mask_cmplt_ps_mask (m, x, x);
+
+  m = _mm512_cmple_ps_mask (x, x);
+  m = _mm512_mask_cmple_ps_mask (m, x, x);
+
+  m = _mm512_cmpunord_ps_mask (x, x);
+  m = _mm512_mask_cmpunord_ps_mask (m, x, x);
+
+  m = _mm512_cmpneq_ps_mask (x, x);
+  m = _mm512_mask_cmpneq_ps_mask (m, x, x);
+
+  m = _mm512_cmpnlt_ps_mask (x, x);
+  m = _mm512_mask_cmpnlt_ps_mask (m, x, x);
+
+  m = _mm512_cmpnle_ps_mask (x, x);
+  m = _mm512_mask_cmpnle_ps_mask (m, x, x);
+
+  m = _mm512_cmpord_ps_mask (x, x);
+  m = _mm512_mask_cmpord_ps_mask (m, x, x);
 }
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-2.c b/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-2.c
index 2ffa2ed16b7..22e368f723e 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-2.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vcmpps-2.c
@@ -11,59 +11,69 @@
 #define SIZE (AVX512F_LEN / 32)
 #include "avx512f-mask-type.h"
 
+#undef SUF
+#undef SSIZE
+#undef GEN_CMP
+#undef CHECK_CMP
+
 #if AVX512F_LEN == 512
-#undef CMP
-#define CMP(imm, rel)					\
-    dst_ref = 0;					\
-    for (i = 0; i < 16; i++)				\
-    {							\
-      dst_ref = (((int) rel) << i) | dst_ref;		\
-    }							\
-    source1.x = _mm512_loadu_ps(s1);			\
-    source2.x = _mm512_loadu_ps(s2);			\
-    dst1 = _mm512_cmp_ps_mask(source1.x, source2.x, imm);\
-    dst2 = _mm512_mask_cmp_ps_mask(mask, source1.x, source2.x, imm);\
-    if (dst_ref != dst1) abort();			\
-    if ((dst_ref & mask) != dst2) abort();
+#define SUF(fun) _mm512##fun
+#define SSIZE 16
+
+#define GEN_CMP(type)				\
+    {						\
+    dst3 = _mm512_cmp##type##_ps_mask(source1.x, source2.x);\
+    dst4 = _mm512_mask_cmp##type##_ps_mask(mask, source1.x, source2.x);\
+    if (dst3 != dst1) abort();			\
+    if (dst4 != dst2) abort();			\
+    }
+
+#define CHECK_CMP(imm)				\
+    if (imm == _CMP_EQ_OQ) GEN_CMP(eq)		\
+    if (imm == _CMP_LT_OS) GEN_CMP(lt)		\
+    if (imm == _CMP_LE_OS) GEN_CMP(le)		\
+    if (imm == _CMP_UNORD_Q) GEN_CMP(unord)	\
+    if (imm == _CMP_NEQ_UQ) GEN_CMP(neq)	\
+    if (imm == _CMP_NLT_US) GEN_CMP(nlt)	\
+    if (imm == _CMP_NLE_US) GEN_CMP(nle)	\
+    if (imm == _CMP_ORD_Q) GEN_CMP(ord)	
+
 #endif
 
 #if AVX512F_LEN == 256
-#undef CMP
-#define CMP(imm, rel)					\
-    dst_ref = 0;					\
-    for (i = 0; i < 8; i++)				\
-    {							\
-      dst_ref = (((int) rel) << i) | dst_ref;		\
-    }							\
-    source1.x = _mm256_loadu_ps(s1);			\
-    source2.x = _mm256_loadu_ps(s2);			\
-    dst1 = _mm256_cmp_ps_mask(source1.x, source2.x, imm);\
-    dst2 = _mm256_mask_cmp_ps_mask(mask, source1.x, source2.x, imm);\
-    if (dst_ref != dst1) abort();			\
-    if ((dst_ref & mask) != dst2) abort();
+#define SUF(fun) _mm256##fun
+#define SSIZE 8
+#define GEN_CMP(type)
+#define CHECK_CMP(imm)
 #endif
 
 #if AVX512F_LEN == 128
+#define SUF(fun) _mm##fun
+#define SSIZE 4
+#define GEN_CMP(type)
+#define CHECK_CMP(imm)
+#endif
+
 #undef CMP
 #define CMP(imm, rel)					\
     dst_ref = 0;					\
-    for (i = 0; i < 4; i++)				\
+    for (i = 0; i < SSIZE; i++)				\
     {							\
       dst_ref = (((int) rel) << i) | dst_ref;		\
     }							\
-    source1.x = _mm_loadu_ps(s1);			\
-    source2.x = _mm_loadu_ps(s2);			\
-    dst1 = _mm_cmp_ps_mask(source1.x, source2.x, imm);\
-    dst2 = _mm_mask_cmp_ps_mask(mask, source1.x, source2.x, imm);\
+    source1.x = SUF(_loadu_ps)(s1);			\
+    source2.x = SUF(_loadu_ps)(s2);			\
+    dst1 = SUF(_cmp_ps_mask)(source1.x, source2.x, imm);\
+    dst2 = SUF(_mask_cmp_ps_mask)(mask, source1.x, source2.x, imm);\
     if (dst_ref != dst1) abort();			\
-    if ((dst_ref & mask) != dst2) abort();
-#endif
+    if ((dst_ref & mask) != dst2) abort();		\
+    CHECK_CMP(imm)
 
 void
 TEST ()
 {
     UNION_TYPE (AVX512F_LEN,) source1, source2;
-    MASK_TYPE dst1, dst2, dst_ref;
+    MASK_TYPE dst1, dst2, dst3, dst4, dst_ref;
     MASK_TYPE mask = MASK_VALUE;
     int i;
     float s1[16] = {2134.3343, 6678.346, 453.345635, 54646.464,
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2d-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2d-1.c
index ceb1bd3bf0c..919cd217c98 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2d-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2d-1.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512f -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2pd-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2pd-1.c
index 2a4955b0f34..c021efb3192 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2pd-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2pd-1.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512f -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
 /* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2pd\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2ps-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2ps-1.c
index dadc6d70530..ffe177bf320 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2ps-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2ps-1.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512f -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2ps\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
 /* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2ps\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2q-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2q-1.c
index 9c6e989b9dd..74bb4ed037c 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-vpermt2q-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpermt2q-1.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512f -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512vbmi-vpermt2b-1.c b/gcc/testsuite/gcc.target/i386/avx512vbmi-vpermt2b-1.c
index f1c31cc56b0..24a0b9e3fce 100644
--- a/gcc/testsuite/gcc.target/i386/avx512vbmi-vpermt2b-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512vbmi-vpermt2b-1.c
@@ -1,14 +1,14 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512vbmi -mavx512vl -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%zmm\[0-9\]+" 3 } } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%ymm\[0-9\]+" 3 } } *
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%xmm\[0-9\]+" 3 } } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\[^\{\]" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\[^\{\]" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\[^\{\]" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2b\[ \\t\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%zmm\[0-9\]+" 3 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%ymm\[0-9\]+" 3 } } *
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%xmm\[0-9\]+" 3 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\[^\{\]" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\[^\{\]" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\[^\{\]" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2b\[ \\t\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-gf2p8affineinvqb-2.c b/gcc/testsuite/gcc.target/i386/avx512vl-gf2p8affineinvqb-2.c
new file mode 100644
index 00000000000..fa545263041
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-gf2p8affineinvqb-2.c
@@ -0,0 +1,17 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512bw -mavx512vl -mgfni" } */
+/* { dg-require-effective-target avx512vl } */
+/* { dg-require-effective-target avx512bw } */
+/* { dg-require-effective-target gfni } */
+
+#define AVX512VL
+#define AVX512F_LEN 256
+#define AVX512F_LEN_HALF 128
+#include "avx512f-gf2p8affineinvqb-2.c"
+
+#undef AVX512F_LEN
+#undef AVX512F_LEN_HALF
+
+#define AVX512F_LEN 128
+#define AVX512F_LEN_HALF 128
+#include "avx512f-gf2p8affineinvqb-2.c"
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-pr82370.c b/gcc/testsuite/gcc.target/i386/avx512vl-pr82370.c
new file mode 100644
index 00000000000..486ece5c2ef
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-pr82370.c
@@ -0,0 +1,31 @@
+/* PR target/82370 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512vl -mno-avx512bw -masm=att" } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %xmm\[0-9]\+, %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, %ymm\[0-9]\+, %ymm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, %ymm\[0-9]\+, %ymm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, %ymm\[0-9]\+, %ymm\[0-9]\+" 3 } } */
+/* { dg-final { scan-assembler-times "vps\[lr]\[la]\[dq]\[ \t]\+\\\$\[357], %\[xyz]mm\[0-9]\+, %\[xyz]mm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vps\[lr]\[la]w\[ \t]\+\\\$\[357], \\(%\[a-z0-9,]*\\), %\[xyz]mm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+
+#include "avx-pr82370.c"
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2d-1.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2d-1.c
index 3a6de905cb6..218650c6cc4 100644
--- a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2d-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2d-1.c
@@ -1,11 +1,11 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512vl -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2d\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2d\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2pd-1.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2pd-1.c
index 5dd0734bca4..64bd30e40c3 100644
--- a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2pd-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2pd-1.c
@@ -1,11 +1,11 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512vl -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2pd\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2pd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
 /* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
 /* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2pd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2pd\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2pd\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2ps-1.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2ps-1.c
index 0d7e37bb548..7af2dea6f9a 100644
--- a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2ps-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2ps-1.c
@@ -1,11 +1,11 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512vl -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2ps\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2ps\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
 /* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
 /* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2ps\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2ps\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2ps\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2q-1.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2q-1.c
index 475aa6dbd04..0cbd8b5b2a3 100644
--- a/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2q-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpermt2q-1.c
@@ -1,11 +1,11 @@
 /* { dg-do compile } */
 /* { dg-options "-mavx512vl -O2" } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
-/* { dg-final { scan-assembler-times "vpermt2q\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[ti]2q\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
 
 #include <immintrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/avx512vlbw-pr82370.c b/gcc/testsuite/gcc.target/i386/avx512vlbw-pr82370.c
new file mode 100644
index 00000000000..6809b4dda60
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512vlbw-pr82370.c
@@ -0,0 +1,33 @@
+/* PR target/82370 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512vl -mavx512bw -masm=att" } */
+/* { dg-final { scan-assembler-times "vps\[lr]\[la]\[dwq]\[ \t]\+\\\$\[357], %\[xyz]mm\[0-9]\+, %\[xyz]mm\[0-9]\+" 0 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslld\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllq\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsllw\[ \t]\+\\\$7, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrad\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraq\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsraw\[ \t]\+\\\$3, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrld\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrlw\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+
+#include "avx-pr82370.c"
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-10.c b/gcc/testsuite/gcc.target/i386/cet-intrin-10.c
new file mode 100644
index 00000000000..695dc5edc34
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-10.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mcet" } */
+/* { dg-final { scan-assembler-times "clrssbsy" 1 } } */
+
+#include <immintrin.h>
+
+void f2 (void *__B)
+{
+  _clrssbsy (__B);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-3.c b/gcc/testsuite/gcc.target/i386/cet-intrin-3.c
new file mode 100644
index 00000000000..bcd7203fdb4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-3.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 2 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 4 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler "rdsspd|incsspd\[ \t]+(%|)eax" { target ia32 } } } */
+/* { dg-final { scan-assembler "rdssp\[dq]\[ \t]+(%|)\[re]ax" { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler "incssp\[dq]\[ \t]+(%|)\[re]di" { target { ! ia32 } } } } */
+
+#include <immintrin.h>
+
+unsigned int f1 ()
+{
+  unsigned int x = 0;
+  return _rdsspd (x);
+}
+
+void f3 (unsigned int _a)
+{
+  _incsspd (_a);
+}
+
+#ifdef __x86_64__
+unsigned long long f2 ()
+{
+  unsigned long long x = 0;
+  return _rdsspq (x);
+}
+
+void f4 (unsigned int _a)
+{
+  _incsspq (_a);
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-4.c b/gcc/testsuite/gcc.target/i386/cet-intrin-4.c
new file mode 100644
index 00000000000..76ec160543f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-4.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mshstk" } */
+/* { dg-final { scan-assembler "rdsspd|incsspd\[ \t]+(%|)eax" { target ia32 } } } */
+/* { dg-final { scan-assembler "rdssp\[dq]\[ \t]+(%|)\[re]ax"  { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler "incssp\[dq]\[ \t]+(%|)\[re]di" { target { ! ia32 } } } } */
+
+#include <immintrin.h>
+
+unsigned int f1 ()
+{
+  unsigned int x = 0;
+  return _rdsspd (x);
+}
+
+void f3 (unsigned int _a)
+{
+  _incsspd (_a);
+}
+
+#ifdef __x86_64__
+unsigned long long f2 ()
+{
+  unsigned long long x = 0;
+  return _rdsspq (x);
+}
+
+void f4 (unsigned int _a)
+{
+  _incsspq (_a);
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-5.c b/gcc/testsuite/gcc.target/i386/cet-intrin-5.c
new file mode 100644
index 00000000000..8a1b637905c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-5.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mcet" } */
+/* { dg-final { scan-assembler-times "saveprevssp" 1 } } */
+
+#include <immintrin.h>
+
+void f2 (void)
+{
+  _saveprevssp ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-6.c b/gcc/testsuite/gcc.target/i386/cet-intrin-6.c
new file mode 100644
index 00000000000..dfa6d20ca26
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-6.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mcet" } */
+/* { dg-final { scan-assembler-times "rstorssp" 1 } } */
+
+#include <immintrin.h>
+
+void f2 (void *__B)
+{
+  _rstorssp (__B);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-7.c b/gcc/testsuite/gcc.target/i386/cet-intrin-7.c
new file mode 100644
index 00000000000..ecd1825a303
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-7.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mcet" } */
+/* { dg-final { scan-assembler-times "wrssd" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "wrss\[d|q]" 2 { target lp64 } } } */
+
+#include <immintrin.h>
+
+void f1 (unsigned int __A, void *__B)
+{
+  _wrssd (__A, __B);
+}
+
+#ifdef __x86_64__
+void f2 (unsigned long long __A, void *__B)
+{
+  _wrssq (__A, __B);
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-8.c b/gcc/testsuite/gcc.target/i386/cet-intrin-8.c
new file mode 100644
index 00000000000..2188876cca5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-8.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mcet" } */
+/* { dg-final { scan-assembler-times "wrussd" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "wruss\[d|q]" 2 { target lp64 } } } */
+
+#include <immintrin.h>
+
+void f1 (unsigned int __A, void *__B)
+{
+  _wrussd (__A, __B);
+}
+
+#ifdef __x86_64__
+void f2 (unsigned long long __A, void *__B)
+{
+  _wrussq (__A, __B);
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/i386/cet-intrin-9.c b/gcc/testsuite/gcc.target/i386/cet-intrin-9.c
new file mode 100644
index 00000000000..569931a9492
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-intrin-9.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mcet" } */
+/* { dg-final { scan-assembler-times "setssbsy" 1 } } */
+
+#include <immintrin.h>
+
+void f2 (void)
+{
+  _setssbsy ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-label-2.c b/gcc/testsuite/gcc.target/i386/cet-label-2.c
new file mode 100644
index 00000000000..c7f79819079
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-label-2.c
@@ -0,0 +1,24 @@
+/* Verify that CET works.  */
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 3 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 3 { target { ! ia32 } } } } */
+
+__attribute__ ((noinline, noclone))
+static int
+func (int arg)
+{
+  static void *array[] = { &&foo, &&bar };
+
+  goto *array[arg];
+foo:
+  return arg*111;
+bar:
+  return arg*777;
+}
+
+int
+foo (int arg)
+{
+  return func (arg);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-label.c b/gcc/testsuite/gcc.target/i386/cet-label.c
new file mode 100644
index 00000000000..8fb8d420349
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-label.c
@@ -0,0 +1,16 @@
+/* Verify that CET works.  */
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 3 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 3 { target { ! ia32 } } } } */
+
+int func (int arg)
+{
+  static void *array[] = { &&foo, &&bar };
+
+  goto *array[arg];
+foo:
+  return arg*111;
+bar:
+  return arg*777;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-1a.c b/gcc/testsuite/gcc.target/i386/cet-notrack-1a.c
new file mode 100644
index 00000000000..ab0bd3ba9b2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-1a.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O0 -fcf-protection=none -mno-cet" } */
+/* { dg-final { scan-assembler-not "endbr" } } */
+/* { dg-final { scan-assembler-not "notrack call\[ \t]+" } } */
+
+int func (int a) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored. Use -fcf-protection option to enable it" } */
+int (*fptr) (int a) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored. Use -fcf-protection option to enable it" } */
+
+int foo (int arg)
+{
+  int a, b;
+  a = func (arg);
+  b = (*fptr) (arg);
+  return a+b;
+}
+
+int __attribute__ ((nocf_check))
+func (int arg)
+{ /* { dg-warning "'nocf_check' attribute ignored. Use -fcf-protection option to enable it" } */
+  int (*fptrl) (int a) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored. Use -fcf-protection option to enable it" } */
+  return arg*(*fptrl)(arg);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-1b.c b/gcc/testsuite/gcc.target/i386/cet-notrack-1b.c
new file mode 100644
index 00000000000..6faf88fdf04
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-1b.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+" 2 } } */
+
+int func (int a) __attribute__ ((nocf_check));
+int (*fptr) (int a) __attribute__ ((nocf_check));
+
+int foo (int arg)
+{
+int a, b;
+  a = func (arg);
+  b = (*fptr) (arg);
+  return a+b;
+}
+
+int __attribute__ ((nocf_check))
+func (int arg)
+{
+int (*fptrl) (int a) __attribute__ ((nocf_check));
+  return arg*(*fptrl)(arg);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-2a.c b/gcc/testsuite/gcc.target/i386/cet-notrack-2a.c
new file mode 100644
index 00000000000..6f441e49edf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-2a.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+" 1 } } */
+
+void
+bar (void (*foo) (void))
+{
+  void (*func) (void) __attribute__((nocf_check)) = foo; /* { dg-warning "incompatible pointer type" } */
+  func ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-2b.c b/gcc/testsuite/gcc.target/i386/cet-notrack-2b.c
new file mode 100644
index 00000000000..0df46450e88
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-2b.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "notrack jmp\[ \t]+" 1 } } */
+
+void
+bar (void (*foo) (void))
+{
+  void (*func) (void) __attribute__((nocf_check)) = foo; /* { dg-warning "incompatible pointer type" } */
+  func ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-3.c b/gcc/testsuite/gcc.target/i386/cet-notrack-3.c
new file mode 100644
index 00000000000..5e124c7f95c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-3.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+" 1 } } */
+
+typedef void (*func_t) (void) __attribute__((nocf_check));
+extern func_t func;
+
+void
+bar (void)
+{
+  func ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-4a.c b/gcc/testsuite/gcc.target/i386/cet-notrack-4a.c
new file mode 100644
index 00000000000..34cfd9098c2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-4a.c
@@ -0,0 +1,6 @@
+/* { dg-do compile } */
+/* { dg-options "-fcf-protection=none -mno-cet" } */
+
+int var1 __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
+int *var2 __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
+void (**var3) (void) __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-4b.c b/gcc/testsuite/gcc.target/i386/cet-notrack-4b.c
new file mode 100644
index 00000000000..6065ef69c25
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-4b.c
@@ -0,0 +1,6 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+
+int var1 __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
+int *var2 __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
+void (**var3) (void) __attribute__((nocf_check)); /* { dg-warning "'nocf_check' attribute only applies to function types" } */
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-5a.c b/gcc/testsuite/gcc.target/i386/cet-notrack-5a.c
new file mode 100644
index 00000000000..d23968e58d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-5a.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-not "\tcall\[ \t]+" } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+" 1 } } */
+
+int (*fptr) (int) __attribute__ ((nocf_check));
+
+int
+foo (int arg)
+{
+  int a;
+  a = (*fptr) (arg); /* notrack call.  */
+  return arg+a;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-5b.c b/gcc/testsuite/gcc.target/i386/cet-notrack-5b.c
new file mode 100644
index 00000000000..42d9d07b19d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-5b.c
@@ -0,0 +1,21 @@
+/* Check the attribute do not proparate through assignment.  */
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "\tcall\[ \t]+" 1 } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+" 1 } } */
+
+int (*fptr) (int) __attribute__ ((nocf_check));
+int (*fptr1) (int);
+
+int
+foo (int arg)
+{
+  int a;
+  a = (*fptr) (arg); /* non-checked call.  */
+  arg += a;
+  fptr1 = fptr; /* { dg-warning "incompatible pointer type" } */ 
+  a = (*fptr1) (arg); /* checked call.  */
+  return arg+a;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-6a.c b/gcc/testsuite/gcc.target/i386/cet-notrack-6a.c
new file mode 100644
index 00000000000..e0fb4f90aaf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-6a.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "\t(?:call|jmp)\[ \t]+.*foo" 1 } } */
+/* { dg-final { scan-assembler-not "notrack call\[ \t]+" } } */
+
+int foo (int arg);
+
+int func (int arg)
+{
+  int (*fptrl) (int a) __attribute__ ((nocf_check)) = foo; /* { dg-warning "incompatible pointer type" } */
+
+  return (*fptrl)(arg);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-6b.c b/gcc/testsuite/gcc.target/i386/cet-notrack-6b.c
new file mode 100644
index 00000000000..1c47c9f7d20
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-6b.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O0 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-not "\tcall\[ \t]+" } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+" 1 } } */
+
+int foo (int arg);
+
+int func (int arg)
+{
+  int (*fptrl) (int a) __attribute__ ((nocf_check)) = foo; /* { dg-warning "incompatible pointer type" } */
+
+  return (*fptrl)(arg);  /* notrack call.  */
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-7.c b/gcc/testsuite/gcc.target/i386/cet-notrack-7.c
new file mode 100644
index 00000000000..f2e31d0258a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-7.c
@@ -0,0 +1,15 @@
+/* Check the notrack prefix is not generated for direct call.  */
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "notrack call\[ \t]+.*foo" 0 } } */
+/* { dg-final { scan-assembler-times "\tcall\[ \t]+.*foo" 1 } } */
+
+extern void foo (void) __attribute__((nocf_check));
+
+void
+bar (void)
+{
+  foo ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-icf-1.c b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-1.c
new file mode 100644
index 00000000000..7987d53d305
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-1.c
@@ -0,0 +1,31 @@
+/* Verify nocf_check functions are not ICF optimized.  */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-not "endbr" } } */
+/* { dg-final { scan-assembler-not "fn3:" } } */
+/* { dg-final { scan-assembler "set\[ \t]+fn2,fn1" } } */
+/* { dg-final { scan-assembler "set\[ \t]+fn3,fn1" } } */
+
+static __attribute__((noinline)) int
+fn1 (int x)
+{
+  return x + 12;
+}
+
+static __attribute__((noinline)) int
+fn2 (int x)
+{
+  return x + 12;
+}
+
+static __attribute__((noinline, nocf_check)) int
+fn3 (int x)
+{ /* { dg-warning "'nocf_check' attribute ignored. Use -fcf-protection option to enable it" } */
+  return x + 12;
+}
+
+int
+fn4 (int x)
+{
+  return fn1 (x) + fn2 (x) + fn3 (x);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-icf-2.c b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-2.c
new file mode 100644
index 00000000000..db0b0a44237
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-2.c
@@ -0,0 +1,30 @@
+/* Verify nocf_check functions are not ICF optimized.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler "endbr" } } */
+/* { dg-final { scan-assembler "fn3:" } } */
+/* { dg-final { scan-assembler "set\[ \t]+fn2,fn1" } } */
+
+static __attribute__((noinline)) int
+fn1 (int x)
+{
+  return x + 12;
+}
+
+static __attribute__((noinline)) int
+fn2 (int x)
+{
+  return x + 12;
+}
+
+static __attribute__((noinline, nocf_check)) int
+fn3 (int x)
+{
+  return x + 12;
+}
+
+int
+fn4 (int x)
+{
+  return fn1 (x) + fn2 (x) + fn3 (x);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-icf-3.c b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-3.c
new file mode 100644
index 00000000000..07c4a6b61ef
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-3.c
@@ -0,0 +1,36 @@
+/* Verify nocf_check function calls are not ICF optimized.  */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-not "endbr" } } */
+/* { dg-final { scan-assembler-not "fn2:" } } */
+/* { dg-final { scan-assembler "set\[ \t]+fn2,fn1" } } */
+/* { dg-final { scan-assembler "set\[ \t]+fn3,fn1" } } */
+
+int (*foo)(int);
+
+typedef int (*type1_t) (int) __attribute__ ((nocf_check)); /* { dg-warning "'nocf_check' attribute ignored. Use -fcf-protection option to enable it" } */
+typedef int (*type2_t) (int);
+
+static __attribute__((noinline)) int
+fn1 (int x)
+{
+  return ((type2_t)foo)(x + 12);
+}
+
+static __attribute__((noinline)) int
+fn2 (int x)
+{
+  return ((type1_t)foo)(x + 12);
+}
+
+static __attribute__((noinline)) int
+fn3 (int x)
+{
+  return ((type2_t)foo)(x + 12);
+}
+
+int
+fn4 (int x)
+{
+  return fn1 (x) + fn2 (x) + fn3 (x);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-notrack-icf-4.c b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-4.c
new file mode 100644
index 00000000000..e4e96aaf0dc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-notrack-icf-4.c
@@ -0,0 +1,35 @@
+/* Verify nocf_check function calls are not ICF optimized.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler "endbr" } } */
+/* { dg-final { scan-assembler "fn2:" } } */
+/* { dg-final { scan-assembler "set\[ \t]+fn3,fn1" } } */
+
+int (*foo)(int);
+
+typedef int (*type1_t) (int) __attribute__ ((nocf_check));
+typedef int (*type2_t) (int);
+
+static __attribute__((noinline)) int
+fn1 (int x)
+{
+  return ((type2_t)foo)(x + 12);
+}
+
+static __attribute__((noinline)) int
+fn2 (int x)
+{
+  return ((type1_t)foo)(x + 12);
+}
+
+static __attribute__((noinline)) int
+fn3 (int x)
+{
+  return ((type2_t)foo)(x + 12);
+}
+
+int
+fn4 (int x)
+{
+  return fn1 (x) + fn2 (x) + fn3 (x);
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-property-1.c b/gcc/testsuite/gcc.target/i386/cet-property-1.c
new file mode 100644
index 00000000000..df243efc574
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-property-1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-fcf-protection -mcet" } */
+/* { dg-final { scan-assembler ".note.gnu.property" } } */
+
+extern void foo (void);
+
+void
+bar (void)
+{
+  foo ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-property-2.c b/gcc/testsuite/gcc.target/i386/cet-property-2.c
new file mode 100644
index 00000000000..5a87dab92f1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-property-2.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-mcet" } */
+/* { dg-final { scan-assembler-not ".note.gnu.property" } } */
+
+extern void foo (void);
+
+void
+bar (void)
+{
+  foo ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-rdssp-1.c b/gcc/testsuite/gcc.target/i386/cet-rdssp-1.c
new file mode 100644
index 00000000000..fb50ff43504
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-rdssp-1.c
@@ -0,0 +1,39 @@
+/* { dg-do run { target cet } } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+
+void _exit(int status) __attribute__ ((__noreturn__));
+
+#ifdef __x86_64__
+# define incssp(x) __builtin_ia32_incsspq (x)
+# define rdssp(x) __builtin_ia32_rdsspq (x)
+#else
+# define incssp(x) __builtin_ia32_incsspd (x)
+# define rdssp(x) __builtin_ia32_rdsspd (x)
+#endif
+
+static void
+__attribute__ ((noinline, noclone))
+test (unsigned long frames)
+{
+  unsigned long ssp = 0;
+  ssp = rdssp (ssp);
+  if (ssp != 0)
+    {
+      unsigned long tmp = frames;
+      while (tmp > 255)
+	{
+	  incssp (tmp);
+	  tmp -= 255;
+	}
+      incssp (tmp);
+    }
+  /* We must call _exit since shadow stack is incorrect now.  */
+  _exit (0);
+}
+
+int
+main ()
+{
+  test (1);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-sjlj-1.c b/gcc/testsuite/gcc.target/i386/cet-sjlj-1.c
new file mode 100644
index 00000000000..374d12aa745
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-sjlj-1.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 4 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 4 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "rdssp\[dq]" 2 } } */
+/* { dg-final { scan-assembler-times "incssp\[dq]" 1 } } */
+
+/* Based on gcc.dg/setjmp-3.c.  */
+
+void *buf[5];
+
+extern void abort (void);
+
+void raise0(void)
+{
+  __builtin_longjmp (buf, 1);
+}
+
+int execute(int cmd)
+{
+  int last = 0;
+
+  if (__builtin_setjmp (buf) == 0)
+    while (1)
+      {
+	last = 1;
+	raise0 ();
+      }
+
+  if (last == 0)
+    return 0;
+  else
+    return cmd;
+}
+
+int main(void)
+{
+  if (execute (1) == 0)
+    abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-sjlj-2.c b/gcc/testsuite/gcc.target/i386/cet-sjlj-2.c
new file mode 100644
index 00000000000..c97094a19c9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-sjlj-2.c
@@ -0,0 +1,4 @@
+/* { dg-do run { target cet } } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+
+#include "cet-sjlj-1.c"
diff --git a/gcc/testsuite/gcc.target/i386/cet-sjlj-3.c b/gcc/testsuite/gcc.target/i386/cet-sjlj-3.c
new file mode 100644
index 00000000000..585f4d7ae89
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-sjlj-3.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 4 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 4 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "call	_?setjmp" 1 } } */
+/* { dg-final { scan-assembler-times "call	longjmp" 1 } } */
+
+#include <stdio.h>
+#include <setjmp.h>
+
+jmp_buf buf;
+int bar (int);
+
+int
+foo (int i)
+{
+  int j = i * 11;
+
+  if (!setjmp (buf))
+    {
+      j += 33;
+      printf ("After setjmp: j = %d\n", j);
+      bar (j);
+    }
+
+  return j + i;
+}
+
+int
+bar (int i)
+{
+int j = i;
+
+  j -= 111;
+  printf ("In longjmp: j = %d\n", j);
+  longjmp (buf, 1);
+
+  return j;
+}
+
+int
+main ()
+{
+  foo (10);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-sjlj-4.c b/gcc/testsuite/gcc.target/i386/cet-sjlj-4.c
new file mode 100644
index 00000000000..d41406fde1f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-sjlj-4.c
@@ -0,0 +1,45 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 3 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 3 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "rdssp\[dq]" 2 } } */
+/* { dg-final { scan-assembler-times "incssp\[dq]" 1 } } */
+
+/* Based on gcc.dg/setjmp-3.c.  */
+
+void *buf[5];
+
+extern void abort (void);
+
+void
+raise0 (void)
+{
+  __builtin_longjmp (buf, 1);
+}
+
+__attribute__ ((noinline, noclone))
+static int
+execute (int cmd)
+{
+  int last = 0;
+
+  if (__builtin_setjmp (buf) == 0)
+    while (1)
+      {
+	last = 1;
+	raise0 ();
+      }
+
+  if (last == 0)
+    return 0;
+  else
+    return cmd;
+}
+
+int main(void)
+{
+  if (execute (1) == 0)
+    abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-sjlj-5.c b/gcc/testsuite/gcc.target/i386/cet-sjlj-5.c
new file mode 100644
index 00000000000..8e54b4bfec8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-sjlj-5.c
@@ -0,0 +1,48 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 2 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 2 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "call	_?setjmp" 1 } } */
+/* { dg-final { scan-assembler-times "call	longjmp" 1 } } */
+
+#include <stdio.h>
+#include <setjmp.h>
+
+jmp_buf buf;
+static int bar (int);
+
+__attribute__ ((noinline, noclone))
+static int
+foo (int i)
+{
+  int j = i * 11;
+
+  if (!setjmp (buf))
+    {
+      j += 33;
+      printf ("After setjmp: j = %d\n", j);
+      bar (j);
+    }
+
+  return j + i;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+bar (int i)
+{
+ int j = i;
+
+  j -= 111;
+  printf ("In longjmp: j = %d\n", j);
+  longjmp (buf, 1);
+
+  return j;
+}
+
+int
+main ()
+{
+  foo (10);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-switch-1.c b/gcc/testsuite/gcc.target/i386/cet-switch-1.c
new file mode 100644
index 00000000000..7a75857fcb1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-switch-1.c
@@ -0,0 +1,26 @@
+/* Verify that CET works.  */
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times "endbr32" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 1 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "notrack jmp\[ \t]+\[*]" 1 } } */
+
+void func2 (int);
+
+int func1 (int arg)
+{
+  switch (arg)
+  {
+    case 1: func2 (arg*100);
+    case 2: func2 (arg*300);
+    case 5: func2 (arg*500);
+    case 8: func2 (arg*700);
+    case 7: func2 (arg*900);
+    case -1: func2 (arg*-100);
+    case -2: func2 (arg*-300);
+    case -5: func2 (arg*-500);
+    case -7: func2 (arg*-700);
+    case -9: func2 (arg*-900);
+  }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-switch-2.c b/gcc/testsuite/gcc.target/i386/cet-switch-2.c
new file mode 100644
index 00000000000..e620b837a3c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-switch-2.c
@@ -0,0 +1,26 @@
+/* Verify that CET works.  */
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet -mcet-switch" } */
+/* { dg-final { scan-assembler-times "endbr32" 12 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 12 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "\[ \t]+jmp\[ \t]+\[*]" 1 } } */
+
+void func2 (int);
+
+int func1 (int arg)
+{
+  switch (arg)
+  {
+    case 1: func2 (arg*100);
+    case 2: func2 (arg*300);
+    case 5: func2 (arg*500);
+    case 8: func2 (arg*700);
+    case 7: func2 (arg*900);
+    case -1: func2 (arg*-100);
+    case -2: func2 (arg*-300);
+    case -5: func2 (arg*-500);
+    case -7: func2 (arg*-700);
+    case -9: func2 (arg*-900);
+  }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/cet-switch-3.c b/gcc/testsuite/gcc.target/i386/cet-switch-3.c
new file mode 100644
index 00000000000..9b1b4369582
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/cet-switch-3.c
@@ -0,0 +1,34 @@
+/* Verify that CET works.  */
+/* { dg-do compile } */
+/* { dg-options "-O -fcf-protection -mcet -mcet-switch" } */
+/* { dg-final { scan-assembler-times "endbr32" 12 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "endbr64" 12 { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "\[ \t]+jmp\[ \t]+\[*]" 1 } } */
+
+void func2 (int);
+
+__attribute__ ((noinline, noclone))
+static int
+func1 (int arg)
+{
+  switch (arg)
+  {
+    case 1: func2 (arg*100);
+    case 2: func2 (arg*300);
+    case 5: func2 (arg*500);
+    case 8: func2 (arg*700);
+    case 7: func2 (arg*900);
+    case -1: func2 (arg*-100);
+    case -2: func2 (arg*-300);
+    case -5: func2 (arg*-500);
+    case -7: func2 (arg*-700);
+    case -9: func2 (arg*-900);
+  }
+  return 0;
+}
+
+int
+foo (int arg)
+{
+  return func1 (arg);
+}
diff --git a/gcc/testsuite/gcc.target/i386/gfni-1.c b/gcc/testsuite/gcc.target/i386/gfni-1.c
new file mode 100644
index 00000000000..5e22c9eae92
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/gfni-1.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-mgfni -mavx512bw -mavx512f -O2" } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%zmm\[0-9\]+\[^\\n\\r]*%zmm\[0-9\]+\[^\\n\\r\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%zmm\[0-9\]+\[^\\n\\r]*%zmm\[0-9\]+\[^\\n\\r\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+
+#include <x86intrin.h>
+
+volatile __m512i x1, x2;
+volatile __mmask64 m64;
+ 
+void extern
+avx512vl_test (void)
+{
+    x1 = _mm512_gf2p8affineinv_epi64_epi8(x1, x2, 3);
+    x1 = _mm512_mask_gf2p8affineinv_epi64_epi8(x1, m64, x2, x1, 3);
+    x1 = _mm512_maskz_gf2p8affineinv_epi64_epi8(m64, x1, x2, 3);
+}
diff --git a/gcc/testsuite/gcc.target/i386/gfni-2.c b/gcc/testsuite/gcc.target/i386/gfni-2.c
new file mode 100644
index 00000000000..4d1f151aa40
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/gfni-2.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-mgfni -mavx512bw -mavx512vl -O2" } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%ymm\[0-9\]+\[^\n\r]*%ymm\[0-9\]+\[^\n\r]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%ymm\[0-9\]+\[^\\n\\r]*%ymm\[0-9\]+\[^\\n\\r\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%ymm\[0-9\]+\[^\\n\\r]*%ymm\[0-9\]+\[^\\n\\r\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%xmm\[0-9\]+\[^\n\r]*%xmm\[0-9\]+\[^\n\r]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%xmm\[0-9\]+\[^\\n\\r]*%xmm\[0-9\]+\[^\\n\\r\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%xmm\[0-9\]+\[^\\n\\r]*%xmm\[0-9\]+\[^\\n\\r\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)" 1 } } */
+
+#include <x86intrin.h>
+
+int *p;
+volatile __m256i x3, x4;
+volatile __m128i x5, x6;
+volatile __mmask32 m32;
+volatile __mmask16 m16;
+ 
+void extern
+avx512vl_test (void)
+{
+    x3 = _mm256_gf2p8affineinv_epi64_epi8(x3, x4, 3);
+    x3 = _mm256_mask_gf2p8affineinv_epi64_epi8(x3, m32, x4, x3, 3);
+    x3 = _mm256_maskz_gf2p8affineinv_epi64_epi8(m32, x3, x4, 3);
+    x5 = _mm_gf2p8affineinv_epi64_epi8(x5, x6, 3);
+    x5 = _mm_mask_gf2p8affineinv_epi64_epi8(x5, m16, x6, x5, 3);
+    x5 = _mm_maskz_gf2p8affineinv_epi64_epi8(m16, x5, x6, 3);
+}
diff --git a/gcc/testsuite/gcc.target/i386/gfni-3.c b/gcc/testsuite/gcc.target/i386/gfni-3.c
new file mode 100644
index 00000000000..de5f80b1124
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/gfni-3.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-mgfni -mavx -O2" } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%ymm\[0-9\]+\[^\n\r]*%ymm\[0-9\]+\[^\n\r]*%ymm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
+/* { dg-final { scan-assembler-times "vgf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%xmm\[0-9\]+\[^\n\r]*%xmm\[0-9\]+\[^\n\r]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
+
+#include <x86intrin.h>
+
+int *p;
+volatile __m256i x3, x4;
+volatile __m128i x5, x6;
+ 
+void extern
+avx512vl_test (void)
+{
+    x3 = _mm256_gf2p8affineinv_epi64_epi8(x3, x4, 3);
+    x5 = _mm_gf2p8affineinv_epi64_epi8(x5, x6, 3);
+}
diff --git a/gcc/testsuite/gcc.target/i386/gfni-4.c b/gcc/testsuite/gcc.target/i386/gfni-4.c
new file mode 100644
index 00000000000..1532716191e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/gfni-4.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-mgfni -O2" } */
+/* { dg-final { scan-assembler-times "gf2p8affineinvqb\[ \\t\]+\[^\{\n\]*\\\$3\[^\n\r]*%xmm\[0-9\]+\[^\n\r]*%xmm\[0-9\]+(?:\n|\[ \\t\]+#)" 1 } } */
+
+#include <x86intrin.h>
+
+int *p;
+volatile __m128i x5, x6;
+ 
+void extern
+avx512vl_test (void)
+{
+    x5 = _mm_gf2p8affineinv_epi64_epi8(x5, x6, 3);
+}
diff --git a/gcc/testsuite/gcc.target/i386/i386.exp b/gcc/testsuite/gcc.target/i386/i386.exp
index eae253192ad..b2bdbfdc06b 100644
--- a/gcc/testsuite/gcc.target/i386/i386.exp
+++ b/gcc/testsuite/gcc.target/i386/i386.exp
@@ -421,6 +421,21 @@ proc check_effective_target_avx512vpopcntdq { } {
     } "-mavx512vpopcntdq" ]
 }
 
+# Return 1 if gfni instructions can be compiled.
+proc check_effective_target_gfni { } {
+    return [check_no_compiler_messages gfni object {
+        typedef char __v16qi __attribute__ ((__vector_size__ (16)));
+
+        __v16qi
+        _mm_gf2p8affineinv_epi64_epi8 (__v16qi __A, __v16qi __B, const int __C)
+        {
+            return (__v16qi) __builtin_ia32_vgf2p8affineinvqb_v16qi ((__v16qi) __A,
+								     (__v16qi) __B,
+								      0);
+        }
+    } "-mgfni" ]
+}
+
 # If a testcase doesn't have special options, use these.
 global DEFAULT_CFLAGS
 if ![info exists DEFAULT_CFLAGS] then {
diff --git a/gcc/testsuite/gcc.target/i386/naked-1.c b/gcc/testsuite/gcc.target/i386/naked-1.c
index cf62bb1114f..07bb10edd8f 100644
--- a/gcc/testsuite/gcc.target/i386/naked-1.c
+++ b/gcc/testsuite/gcc.target/i386/naked-1.c
@@ -10,5 +10,5 @@ foo (void)
   __asm__ ("# naked");
 }
 /* { dg-final { scan-assembler "# naked" } } */
-/* { dg-final { scan-assembler "ud2" } } */
-/* { dg-final { scan-assembler-not "ret" } } */
+/* { dg-final { scan-assembler "(?n)^\\s*ud2$" } } */
+/* { dg-final { scan-assembler-not "(?n)^\\s*ret$" } } */
diff --git a/gcc/testsuite/gcc.target/i386/naked-2.c b/gcc/testsuite/gcc.target/i386/naked-2.c
index adcd7121541..2da8b81a8cb 100644
--- a/gcc/testsuite/gcc.target/i386/naked-2.c
+++ b/gcc/testsuite/gcc.target/i386/naked-2.c
@@ -10,5 +10,5 @@ foo (void)
   __asm__ ("# naked");
 }
 /* { dg-final { scan-assembler "# naked" } } */
-/* { dg-final { scan-assembler-not "push" } } */
-/* { dg-final { scan-assembler-not "pop" } } */
+/* { dg-final { scan-assembler-not "(?n)^\\s*push" } } */
+/* { dg-final { scan-assembler-not "(?n)^\\s*pop" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr61403.c b/gcc/testsuite/gcc.target/i386/pr61403.c
index 0a89f56753f..38ba4a1b1ec 100644
--- a/gcc/testsuite/gcc.target/i386/pr61403.c
+++ b/gcc/testsuite/gcc.target/i386/pr61403.c
@@ -23,4 +23,4 @@ norm (struct XYZ *in, struct XYZ *out, int size)
     }
 }
 
-/* { dg-final { scan-assembler "blend" } } */
+/* { dg-final { scan-assembler "rsqrtps" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr70021.c b/gcc/testsuite/gcc.target/i386/pr70021.c
index de6da345119..6562c0f2bd0 100644
--- a/gcc/testsuite/gcc.target/i386/pr70021.c
+++ b/gcc/testsuite/gcc.target/i386/pr70021.c
@@ -1,7 +1,7 @@
 /* PR target/70021 */
 /* { dg-do run } */
 /* { dg-require-effective-target avx2 } */
-/* { dg-options "-O2 -ftree-vectorize -mavx2 -fdump-tree-vect-details" } */
+/* { dg-options "-O2 -ftree-vectorize -mavx2 -fdump-tree-vect-details -mtune=skylake" } */
 
 #include "avx2-check.h"
 
diff --git a/gcc/testsuite/gcc.target/i386/pr70263-2.c b/gcc/testsuite/gcc.target/i386/pr70263-2.c
index 18ebbf05fb7..19f79fd0e36 100644
--- a/gcc/testsuite/gcc.target/i386/pr70263-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr70263-2.c
@@ -4,20 +4,13 @@
 /* { dg-final { scan-rtl-dump "Adding REG_EQUIV to insn \[0-9\]+ for source of insn \[0-9\]+" "ira" } } */
 
 typedef float XFtype __attribute__ ((mode (XF)));
-typedef _Complex float XCtype __attribute__ ((mode (XC)));
-XCtype
-__mulxc3 (XFtype a, XFtype b, XFtype c, XFtype d)
+
+void bar (XFtype);
+
+void
+foo (XFtype a, XFtype c)
 {
-  XFtype ac, bd, ad, bc, x, y;
-  ac = a * c;
-__asm__ ("": "=m" (ac):"m" (ac));
-  if (x != x)
-    {
-      _Bool recalc = 0;
-      if (((!(!(((ac) - (ac)) != ((ac) - (ac)))))))
-	recalc = 1;
-      if (recalc)
-	x = __builtin_huge_vall () * (a * c - b * d);
-    }
-  return x;
+  XFtype ac = a * c;
+
+  bar (ac);
 }
diff --git a/gcc/testsuite/gcc.target/i386/pr79683.c b/gcc/testsuite/gcc.target/i386/pr79683.c
index cbd43fd2af0..9e28d85fc89 100644
--- a/gcc/testsuite/gcc.target/i386/pr79683.c
+++ b/gcc/testsuite/gcc.target/i386/pr79683.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O3 -msse2" } */
+/* { dg-options "-O3 -msse2 -fvect-cost-model=unlimited" } */
 
 struct s {
     __INT64_TYPE__ a;
diff --git a/gcc/testsuite/gcc.target/i386/pr81706.c b/gcc/testsuite/gcc.target/i386/pr81706.c
new file mode 100644
index 00000000000..333fd159770
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr81706.c
@@ -0,0 +1,32 @@
+/* PR libstdc++/81706 */
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mno-avx512f" } */
+/* { dg-final { scan-assembler "call\[^\n\r]_ZGVdN4v_cos" } } */
+/* { dg-final { scan-assembler "call\[^\n\r]_ZGVdN4v_sin" } } */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern double cos (double) __attribute__ ((nothrow, leaf, simd ("notinbranch")));
+extern double sin (double) __attribute__ ((nothrow, leaf, simd ("notinbranch")));
+#ifdef __cplusplus
+}
+#endif
+double p[1024] = { 1.0 };
+double q[1024] = { 1.0 };
+
+void
+foo (void)
+{
+  int i;
+  for (i = 0; i < 1024; i++)
+    p[i] = cos (q[i]);
+}
+
+void
+bar (void)
+{
+  int i;
+  for (i = 0; i < 1024; i++)
+    p[i] = __builtin_sin (q[i]);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82002-1.c b/gcc/testsuite/gcc.target/i386/pr82002-1.c
new file mode 100644
index 00000000000..86678a01992
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82002-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-Ofast -mstackrealign -mabi=ms" } */
+
+void a (char *);
+void
+b ()
+{
+  char c[10000000000];
+  c[1099511627776] = 'b';
+  a (c);
+  a (c);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82002-2a.c b/gcc/testsuite/gcc.target/i386/pr82002-2a.c
new file mode 100644
index 00000000000..bc85080ba8e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82002-2a.c
@@ -0,0 +1,14 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-Ofast -mstackrealign -mabi=ms" } */
+/* { dg-xfail-if "" { *-*-* }  } */
+/* { dg-xfail-run-if "" { *-*-* }  } */
+
+void __attribute__((sysv_abi)) a (char *);
+void
+b ()
+{
+  char c[10000000000];
+  c[1099511627776] = 'b';
+  a (c);
+  a (c);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82002-2b.c b/gcc/testsuite/gcc.target/i386/pr82002-2b.c
new file mode 100644
index 00000000000..10e44cd7b1d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82002-2b.c
@@ -0,0 +1,14 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-Ofast -mstackrealign -mabi=ms -mcall-ms2sysv-xlogues" } */
+/* { dg-xfail-if "" { *-*-* }  } */
+/* { dg-xfail-run-if "" { *-*-* }  } */
+
+void __attribute__((sysv_abi)) a (char *);
+void
+b ()
+{
+  char c[10000000000];
+  c[1099511627776] = 'b';
+  a (c);
+  a (c);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82196-1.c b/gcc/testsuite/gcc.target/i386/pr82196-1.c
index 541d975480d..ff108132bb5 100644
--- a/gcc/testsuite/gcc.target/i386/pr82196-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr82196-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile { target lp64 } } */
-/* { dg-options "-msse -mcall-ms2sysv-xlogues -O2" } */
+/* { dg-options "-mno-avx -msse -mcall-ms2sysv-xlogues -O2" } */
 /* { dg-final { scan-assembler "call.*__sse_savms64f?_12" } } */
 /* { dg-final { scan-assembler "jmp.*__sse_resms64f?x_12" } } */
 
diff --git a/gcc/testsuite/gcc.target/i386/pr82370.c b/gcc/testsuite/gcc.target/i386/pr82370.c
new file mode 100644
index 00000000000..cc4d9b6f255
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82370.c
@@ -0,0 +1,18 @@
+/* PR target/82370 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512vl -mavx512bw -masm=att" } */
+/* { dg-final { scan-assembler-times "vpslldq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrldq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %xmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslldq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrldq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %ymm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpslldq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+/* { dg-final { scan-assembler-times "vpsrldq\[ \t]\+\\\$5, \\(%\[a-z0-9,]*\\), %zmm\[0-9]\+" 1 } } */
+
+#include <x86intrin.h>
+
+__m512i f1 (__m512i *x) { return _mm512_bslli_epi128 (*x, 5); }
+__m512i f2 (__m512i *x) { return _mm512_bsrli_epi128 (*x, 5); }
+__m256i f3 (__m256i *x) { return _mm256_bslli_epi128 (*x, 5); }
+__m256i f4 (__m256i *x) { return _mm256_bsrli_epi128 (*x, 5); }
+__m128i f5 (__m128i *x) { return _mm_bslli_si128 (*x, 5); }
+__m128i f6 (__m128i *x) { return _mm_bsrli_si128 (*x, 5); }
diff --git a/gcc/testsuite/gcc.target/i386/pr82460-1.c b/gcc/testsuite/gcc.target/i386/pr82460-1.c
new file mode 100644
index 00000000000..6529c4a9b9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82460-1.c
@@ -0,0 +1,30 @@
+/* PR target/82460 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512vbmi" } */
+/* { dg-final { scan-assembler-not {\mvmovd} } } */
+
+#include <x86intrin.h>
+
+__m512i
+f1 (__m512i x, __m512i y, char *z)
+{
+  return _mm512_permutex2var_epi32 (y, x, _mm512_loadu_si512 (z));
+}
+
+__m512i
+f2 (__m512i x, __m512i y, char *z)
+{
+  return _mm512_permutex2var_epi32 (x, y, _mm512_loadu_si512 (z));
+}
+
+__m512i
+f3 (__m512i x, __m512i y, __m512i z)
+{
+  return _mm512_permutex2var_epi8 (y, x, z);
+}
+
+__m512i
+f4 (__m512i x, __m512i y, __m512i z)
+{
+  return _mm512_permutex2var_epi8 (x, y, z);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82460-2.c b/gcc/testsuite/gcc.target/i386/pr82460-2.c
new file mode 100644
index 00000000000..4d965216b59
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82460-2.c
@@ -0,0 +1,17 @@
+/* PR target/82460 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -mavx512vbmi -mno-prefer-avx256" } */
+/* We want to reuse the permutation mask in the loop, so use vpermt2b rather
+   than vpermi2b.  */
+/* { dg-final { scan-assembler-not {\mvpermi2b\M} } } */
+/* { dg-final { scan-assembler {\mvpermt2b\M} } } */
+
+void
+foo (unsigned char *__restrict__ x, const unsigned short *__restrict__ y,
+     unsigned long z)
+{
+  unsigned char *w = x + z;
+  do
+    *x++ = *y++ >> 8;
+  while (x < w);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82499-1.c b/gcc/testsuite/gcc.target/i386/pr82499-1.c
new file mode 100644
index 00000000000..3aba62a466f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82499-1.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* The pic register save adds unavoidable stack pointer references.  */
+/* { dg-skip-if "" { ia32 && { ! nonpic } } } */
+/* These options are selected to ensure 1 word needs to be allocated
+   on the stack to maintain alignment for the call.  This should be
+   transformed to push+pop.  We also want to force unwind info updates.  */
+/* { dg-options "-Os -fomit-frame-pointer -fasynchronous-unwind-tables" } */
+/* { dg-additional-options "-mpreferred-stack-boundary=3" { target ia32 } } */
+/* { dg-additional-options "-mpreferred-stack-boundary=4" { target { ! ia32 } } } */
+/* ms_abi has reserved stack-region.  */
+/* { dg-skip-if "" { x86_64-*-mingw* } } */
+
+extern void g (void);
+int
+f (void)
+{
+  g ();
+  return 42;
+}
+
+/* { dg-final { scan-assembler-not "(sub|add)(l|q)\[\\t \]*\\$\[0-9\]*,\[\\t \]*%\[re\]?sp" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr82499-2.c b/gcc/testsuite/gcc.target/i386/pr82499-2.c
new file mode 100644
index 00000000000..dde4d657e1a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82499-2.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* The pic register save adds unavoidable stack pointer references.  */
+/* { dg-skip-if "" { ia32 && { ! nonpic } } } */
+/* These options are selected to ensure 1 word needs to be allocated
+   on the stack to maintain alignment for the call.  This should be
+   transformed to push+pop.  We also want to force unwind info updates.  */
+/* { dg-options "-Os -fomit-frame-pointer -fasynchronous-unwind-tables" } */
+/* { dg-additional-options "-mpreferred-stack-boundary=3" { target ia32 } } */
+/* { dg-additional-options "-mpreferred-stack-boundary=4 -mno-red-zone" { target { ! ia32 } } } */
+/* ms_abi has reserved stack-region.  */
+/* { dg-skip-if "" { x86_64-*-mingw* } } */
+
+extern void g (void);
+int
+f (void)
+{
+  g ();
+  return 42;
+}
+
+/* { dg-final { scan-assembler-not "(sub|add)(l|q)\[\\t \]*\\$\[0-9\]*,\[\\t \]*%\[re\]?sp" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr82499-3.c b/gcc/testsuite/gcc.target/i386/pr82499-3.c
new file mode 100644
index 00000000000..b55a860fcca
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82499-3.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* The pic register save adds unavoidable stack pointer references.  */
+/* { dg-skip-if "" { ia32 && { ! nonpic } } } */
+/* These options are selected to ensure 1 word needs to be allocated
+   on the stack to maintain alignment for the call.  This should be
+   transformed to push+pop.  We also want to force unwind info updates.  */
+/* { dg-options "-O2 -mtune-ctrl=single_push,single_pop -fomit-frame-pointer -fasynchronous-unwind-tables" } */
+/* { dg-additional-options "-mpreferred-stack-boundary=3" { target ia32 } } */
+/* { dg-additional-options "-mpreferred-stack-boundary=4" { target { ! ia32 } } } */
+/* ms_abi has reserved stack-region.  */
+/* { dg-skip-if "" { x86_64-*-mingw* } } */
+
+extern void g (void);
+int
+f (void)
+{
+  g ();
+  return 42;
+}
+
+/* { dg-final { scan-assembler-not "(sub|add)(l|q)\[\\t \]*\\$\[0-9\]*,\[\\t \]*%\[re\]?sp" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr82556.c b/gcc/testsuite/gcc.target/i386/pr82556.c
new file mode 100644
index 00000000000..409a301af30
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82556.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-strict-aliasing -fwrapv -fexcess-precision=standard" } */
+extern int foo();
+typedef struct {
+  char id;
+  unsigned char fork_flags;
+  short data_length;
+} Header;
+int a;
+void X() {
+  do {
+    char* b;
+    Header c;
+    if (a)
+      c.fork_flags |= 1;
+    __builtin_memcpy(b, &c, __builtin_offsetof(Header, data_length));
+    b += foo();
+  } while (1);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82580.c b/gcc/testsuite/gcc.target/i386/pr82580.c
new file mode 100644
index 00000000000..965dfeec28d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82580.c
@@ -0,0 +1,39 @@
+/* PR target/82580 */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#ifdef __SIZEOF_INT128__
+typedef unsigned __int128 U;
+typedef signed __int128 S;
+#else
+typedef unsigned long long U;
+typedef signed long long S;
+#endif
+void bar (void);
+int f0 (U x, U y) { return x == y; }
+int f1 (U x, U y) { return x != y; }
+int f2 (U x, U y) { return x > y; }
+int f3 (U x, U y) { return x >= y; }
+int f4 (U x, U y) { return x < y; }
+int f5 (U x, U y) { return x <= y; }
+int f6 (S x, S y) { return x == y; }
+int f7 (S x, S y) { return x != y; }
+int f8 (S x, S y) { return x > y; }
+int f9 (S x, S y) { return x >= y; }
+int f10 (S x, S y) { return x < y; }
+int f11 (S x, S y) { return x <= y; }
+void f12 (U x, U y) { if (x == y) bar (); }
+void f13 (U x, U y) { if (x != y) bar (); }
+void f14 (U x, U y) { if (x > y) bar (); }
+void f15 (U x, U y) { if (x >= y) bar (); }
+void f16 (U x, U y) { if (x < y) bar (); }
+void f17 (U x, U y) { if (x <= y) bar (); }
+void f18 (S x, S y) { if (x == y) bar (); }
+void f19 (S x, S y) { if (x != y) bar (); }
+void f20 (S x, S y) { if (x > y) bar (); }
+void f21 (S x, S y) { if (x >= y) bar (); }
+void f22 (S x, S y) { if (x < y) bar (); }
+void f23 (S x, S y) { if (x <= y) bar (); }
+
+/* { dg-final { scan-assembler-times {\msbb} 16 } } */
+/* { dg-final { scan-assembler-not {\mmovzb} { target lp64 } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr82618.c b/gcc/testsuite/gcc.target/i386/pr82618.c
new file mode 100644
index 00000000000..f6e3589c808
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82618.c
@@ -0,0 +1,18 @@
+/* PR target/82618 */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#ifdef __SIZEOF_INT128__
+typedef unsigned __int128 U;
+typedef unsigned long long H;
+#else
+typedef unsigned long long U;
+typedef unsigned int H;
+#endif
+
+H f0 (U x, U y)
+{
+  return (x - y) >> (__CHAR_BIT__ * sizeof (H));
+}
+
+/* { dg-final { scan-assembler {\mcmp} } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr82628.c b/gcc/testsuite/gcc.target/i386/pr82628.c
new file mode 100644
index 00000000000..d7135220485
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82628.c
@@ -0,0 +1,34 @@
+/* { dg-do run { target ia32 } } */
+/* { dg-options "-Os" } */
+
+void
+__attribute__ ((noipa))
+foo (const char *x)
+{
+  asm volatile ("" : "+g" (x) : : "memory");
+  if (x)
+    __builtin_abort ();
+}
+
+int a, b = 1;
+
+int
+main ()
+{
+  while (1)
+    {
+      unsigned long long d = 18446744073709551615UL;
+      while (1)
+	{
+	  int e = b;
+	  while (d < 2)
+	    foo ("0");
+	  if (a)
+	    d++;
+	  if (b)
+	    break;
+	}
+      break;
+    }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82659-1.c b/gcc/testsuite/gcc.target/i386/pr82659-1.c
new file mode 100644
index 00000000000..485771d0f38
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82659-1.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times {\mendbr} 1 } } */
+
+extern int x;
+
+static void
+__attribute__ ((noinline, noclone))
+test (int i)
+{
+  x = i;
+}
+
+void
+bar (int i)
+{
+  test (i);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82659-2.c b/gcc/testsuite/gcc.target/i386/pr82659-2.c
new file mode 100644
index 00000000000..7afffa440aa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82659-2.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+extern int x;
+
+void
+test (int i)
+{
+  x = i;
+}
+
+void
+bar (int i)
+{
+  test (i);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82659-3.c b/gcc/testsuite/gcc.target/i386/pr82659-3.c
new file mode 100644
index 00000000000..5f97b314092
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82659-3.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+extern int x;
+
+static void
+__attribute__ ((noinline, noclone))
+test (int i)
+{
+  x = i;
+}
+
+extern __typeof (test) foo __attribute__ ((alias ("test")));
+
+void
+bar (int i)
+{
+  test (i);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82659-4.c b/gcc/testsuite/gcc.target/i386/pr82659-4.c
new file mode 100644
index 00000000000..c3cacaccbef
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82659-4.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+static void
+test (void)
+{
+}
+
+void *
+bar (void)
+{
+  return test;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82659-5.c b/gcc/testsuite/gcc.target/i386/pr82659-5.c
new file mode 100644
index 00000000000..95413671d5c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82659-5.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times {\mendbr} 1 } } */
+
+static void
+test (void)
+{
+}
+
+void (*test_p) (void) = test;
diff --git a/gcc/testsuite/gcc.target/i386/pr82659-6.c b/gcc/testsuite/gcc.target/i386/pr82659-6.c
new file mode 100644
index 00000000000..51fc1a9f5c9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82659-6.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection -mcet" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+extern int x;
+
+ __attribute__ ((visibility ("hidden")))
+void
+test (int i)
+{
+  x = i;
+}
+
+void
+bar (int i)
+{
+  test (i);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr82662.c b/gcc/testsuite/gcc.target/i386/pr82662.c
new file mode 100644
index 00000000000..8a9332b5c5b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82662.c
@@ -0,0 +1,26 @@
+/* PR target/82580 */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#ifdef __SIZEOF_INT128__
+typedef unsigned __int128 U;
+typedef signed __int128 S;
+#else
+typedef unsigned long long U;
+typedef signed long long S;
+#endif
+void bar (void);
+int f0 (U x, U y) { return x == y; }
+int f1 (U x, U y) { return x != y; }
+int f2 (U x, U y) { return x > y; }
+int f3 (U x, U y) { return x >= y; }
+int f4 (U x, U y) { return x < y; }
+int f5 (U x, U y) { return x <= y; }
+int f6 (S x, S y) { return x == y; }
+int f7 (S x, S y) { return x != y; }
+int f8 (S x, S y) { return x > y; }
+int f9 (S x, S y) { return x >= y; }
+int f10 (S x, S y) { return x < y; }
+int f11 (S x, S y) { return x <= y; }
+
+/* { dg-final { scan-assembler-times {\mset} 12 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr82673.c b/gcc/testsuite/gcc.target/i386/pr82673.c
new file mode 100644
index 00000000000..50eb5a3bcfc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82673.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O0 -fno-omit-frame-pointer -fvar-tracking-assignments" } */
+
+register long *B asm ("ebp");
+
+long y = 20;
+
+void
+bar (void) /* { dg-error "frame pointer required, but reserved" } */
+{
+  B = &y;
+} /* { dg-error "bp cannot be used in asm here" } */
diff --git a/gcc/testsuite/gcc.target/i386/pr82795.c b/gcc/testsuite/gcc.target/i386/pr82795.c
new file mode 100644
index 00000000000..9e7fec74699
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr82795.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2" } */
+
+void
+sj (int qh, int rn, int *by)
+{
+  for (;;)
+    if (qh != 0)
+      {
+	int dc;
+
+	for (dc = 0; dc < 17; ++dc)
+	  {
+	    int nn;
+
+	    nn = (rn != 0) ? qh : dc;
+	    if (nn != 0)
+	      qh = nn;
+	    else
+	      qh = (qh != 0) ? *by : dc;
+	  }
+      }
+}
diff --git a/gcc/testsuite/gcc.target/i386/sse-12.c b/gcc/testsuite/gcc.target/i386/sse-12.c
index b98b8b60aa7..82f5d3c653b 100644
--- a/gcc/testsuite/gcc.target/i386/sse-12.c
+++ b/gcc/testsuite/gcc.target/i386/sse-12.c
@@ -1,9 +1,9 @@
 /* Test that {,x,e,p,t,s,w,a,b,i}mmintrin.h, mm3dnow.h, fma4intrin.h,
    xopintrin.h, abmintrin.h, bmiintrin.h, tbmintrin.h, lwpintrin.h,
-   popcntintrin.h and mm_malloc.h are usable
+   popcntintrin.h gfniintrin.h and mm_malloc.h are usable
    with -O -std=c89 -pedantic-errors.  */
 /* { dg-do compile } */
-/* { dg-options "-O -std=c89 -pedantic-errors -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mavx512ifma -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid" } */
+/* { dg-options "-O -std=c89 -pedantic-errors -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mavx512ifma -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid -mgfni" } */
 
 #include <x86intrin.h>
 
diff --git a/gcc/testsuite/gcc.target/i386/sse-13.c b/gcc/testsuite/gcc.target/i386/sse-13.c
index c5c43b12611..c35ec9a47cb 100644
--- a/gcc/testsuite/gcc.target/i386/sse-13.c
+++ b/gcc/testsuite/gcc.target/i386/sse-13.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -Werror-implicit-function-declaration -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512vl -mavx512dq -mavx512bw -mavx512vbmi -mavx512ifma -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid" } */
+/* { dg-options "-O2 -Werror-implicit-function-declaration -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512vl -mavx512dq -mavx512bw -mavx512vbmi -mavx512ifma -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid -mgfni" } */
 /* { dg-add-options bind_pic_locally } */
 
 #include <mm_malloc.h>
@@ -429,8 +429,8 @@
 /* avx512dqintrin.h */
 #define __builtin_ia32_kshiftliqi(A, B) __builtin_ia32_kshiftliqi(A, 8)
 #define __builtin_ia32_kshiftriqi(A, B) __builtin_ia32_kshiftriqi(A, 8)
-#define __builtin_ia32_reducess(A, B, F) __builtin_ia32_reducess(A, B, 1)
-#define __builtin_ia32_reducesd(A, B, F) __builtin_ia32_reducesd(A, B, 1)
+#define __builtin_ia32_reducess_mask(A, B, F, W, U) __builtin_ia32_reducess_mask(A, B, 1, W, U)
+#define __builtin_ia32_reducesd_mask(A, B, F, W, U) __builtin_ia32_reducesd_mask(A, B, 1, W, U)
 #define __builtin_ia32_reduceps512_mask(A, E, C, D) __builtin_ia32_reduceps512_mask(A, 1, C, D)
 #define __builtin_ia32_reducepd512_mask(A, E, C, D) __builtin_ia32_reducepd512_mask(A, 1, C, D)
 #define __builtin_ia32_rangess128_round(A, B, I, F) __builtin_ia32_rangess128_round(A, B, 1, 8)
@@ -620,4 +620,12 @@
 #define __builtin_ia32_extracti64x2_256_mask(A, E, C, D) __builtin_ia32_extracti64x2_256_mask(A, 1, C, D)
 #define __builtin_ia32_extractf64x2_256_mask(A, E, C, D) __builtin_ia32_extractf64x2_256_mask(A, 1, C, D)
 
+/* gfniintrin.h */
+#define __builtin_ia32_vgf2p8affineinvqb_v16qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v16qi(A, B, 1) 
+#define __builtin_ia32_vgf2p8affineinvqb_v32qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v32qi(A, B, 1)
+#define __builtin_ia32_vgf2p8affineinvqb_v64qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v64qi(A, B, 1)
+#define __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(A, B, 1, D, E) 
+#define __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(A, B, 1, D, E) 
+#define __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(A, B, 1, D, E) 
+
 #include <x86intrin.h>
diff --git a/gcc/testsuite/gcc.target/i386/sse-14.c b/gcc/testsuite/gcc.target/i386/sse-14.c
index c2a19b3ccef..388026f927a 100644
--- a/gcc/testsuite/gcc.target/i386/sse-14.c
+++ b/gcc/testsuite/gcc.target/i386/sse-14.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O0 -Werror-implicit-function-declaration -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512dq -mavx512bw -mavx512vl -mavx512ifma -mavx512vbmi -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid" } */
+/* { dg-options "-O0 -Werror-implicit-function-declaration -march=k8 -msse4a -m3dnow -mavx -mavx2 -mfma4 -mxop -maes -mpclmul -mpopcnt -mabm -mlzcnt -mbmi -mbmi2 -mtbm -mlwp -mfsgsbase -mrdrnd -mf16c -mfma -mrtm -mrdseed -mprfchw -madx -mfxsr -mxsaveopt -mavx512f -mavx512er -mavx512cd -mavx512pf -msha -mprefetchwt1 -mxsavec -mxsaves -mclflushopt -mavx512dq -mavx512bw -mavx512vl -mavx512ifma -mavx512vbmi -mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq -mclwb -mmwaitx -mclzero -mpku -msgx -mrdpid -mgfni" } */
 /* { dg-add-options bind_pic_locally } */
 
 #include <mm_malloc.h>
@@ -7,8 +7,8 @@
 /* Test that the intrinsics compile without optimization.  All of them are
    defined as inline functions in {,x,e,p,t,s,w,a,b,i}mmintrin.h, mm3dnow.h,
    fma4intrin.h, xopintrin.h, abmintrin.h, bmiintrin.h, tbmintrin.h, 
-   lwpintrin.h, fmaintrin.h and mm_malloc.h that reference the proper 
-   builtin functions.
+   lwpintrin.h, fmaintrin.h gfniintrin.h and mm_malloc.h that reference
+   the proper builtin functions.
 
    Defining away "extern" and "__inline" results in all of them being compiled
    as proper functions.  */
@@ -684,3 +684,8 @@ test_1 ( __bextri_u32, unsigned int, unsigned int, 1)
 #ifdef __x86_64__
 test_1 ( __bextri_u64, unsigned long long, unsigned long long, 1)
 #endif
+
+/* gfniintrin.h */
+test_2 (_mm_gf2p8affineinv_epi64_epi8, __m128i, __m128i, __m128i, 1)
+test_2 (_mm256_gf2p8affineinv_epi64_epi8, __m256i, __m256i, __m256i, 1)
+test_2 (_mm512_gf2p8affineinv_epi64_epi8, __m512i, __m512i, __m512i, 1)
diff --git a/gcc/testsuite/gcc.target/i386/sse-22.c b/gcc/testsuite/gcc.target/i386/sse-22.c
index cd8945be1cb..3e64e2915ec 100644
--- a/gcc/testsuite/gcc.target/i386/sse-22.c
+++ b/gcc/testsuite/gcc.target/i386/sse-22.c
@@ -101,7 +101,7 @@
 
 
 #ifndef DIFFERENT_PRAGMAS
-#pragma GCC target ("sse4a,3dnow,avx,avx2,fma4,xop,aes,pclmul,popcnt,abm,lzcnt,bmi,bmi2,tbm,lwp,fsgsbase,rdrnd,f16c,rtm,rdseed,prfchw,adx,fxsr,xsaveopt,avx512f,avx512er,avx512cd,avx512pf,sha,prefetchwt1,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512ifma,avx5124fmaps,avx5124vnniw,avx512vpopcntdq")
+#pragma GCC target ("sse4a,3dnow,avx,avx2,fma4,xop,aes,pclmul,popcnt,abm,lzcnt,bmi,bmi2,tbm,lwp,fsgsbase,rdrnd,f16c,rtm,rdseed,prfchw,adx,fxsr,xsaveopt,avx512f,avx512er,avx512cd,avx512pf,sha,prefetchwt1,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512ifma,avx5124fmaps,avx5124vnniw,avx512vpopcntdq,gfni")
 #endif
 
 /* Following intrinsics require immediate arguments.  They
@@ -218,7 +218,7 @@ test_4 (_mm_cmpestrz, int, __m128i, int, __m128i, int, 1)
 
 /* immintrin.h (AVX/AVX2/RDRND/FSGSBASE/F16C/RTM/AVX512F/SHA) */
 #ifdef DIFFERENT_PRAGMAS
-#pragma GCC target ("avx,avx2,rdrnd,fsgsbase,f16c,rtm,avx512f,avx512er,avx512cd,avx512pf,sha,avx512vl,avx512bw,avx512dq,avx512ifma,avx512vbmi,avx5124fmaps,avx5124vnniw,avx512vpopcntdq")
+#pragma GCC target ("avx,avx2,rdrnd,fsgsbase,f16c,rtm,avx512f,avx512er,avx512cd,avx512pf,sha,avx512vl,avx512bw,avx512dq,avx512ifma,avx512vbmi,avx5124fmaps,avx5124vnniw,avx512vpopcntdq,gfni")
 #endif
 #include <immintrin.h>
 test_1 (_cvtss_sh, unsigned short, float, 1)
@@ -695,6 +695,11 @@ test_2 (_mm_rsqrt28_round_ss, __m128, __m128, __m128, 8)
 /* shaintrin.h */
 test_2 (_mm_sha1rnds4_epu32, __m128i, __m128i, __m128i, 1)
 
+/* gfniintrin.h */
+test_2 (_mm_gf2p8affineinv_epi64_epi8, __m128i, __m128i, __m128i, 1)
+test_2 (_mm256_gf2p8affineinv_epi64_epi8, __m256i, __m256i, __m256i, 1)
+test_2 (_mm512_gf2p8affineinv_epi64_epi8, __m512i, __m512i, __m512i, 1)
+
 /* wmmintrin.h (AES/PCLMUL).  */
 #ifdef DIFFERENT_PRAGMAS
 #pragma GCC target ("aes,pclmul")
diff --git a/gcc/testsuite/gcc.target/i386/sse-23.c b/gcc/testsuite/gcc.target/i386/sse-23.c
index fc339a51e63..911258fa042 100644
--- a/gcc/testsuite/gcc.target/i386/sse-23.c
+++ b/gcc/testsuite/gcc.target/i386/sse-23.c
@@ -428,8 +428,8 @@
 /* avx512dqintrin.h */
 #define __builtin_ia32_kshiftliqi(A, B) __builtin_ia32_kshiftliqi(A, 8)
 #define __builtin_ia32_kshiftriqi(A, B) __builtin_ia32_kshiftriqi(A, 8)
-#define __builtin_ia32_reducess(A, B, F) __builtin_ia32_reducess(A, B, 1)
-#define __builtin_ia32_reducesd(A, B, F) __builtin_ia32_reducesd(A, B, 1)
+#define __builtin_ia32_reducess_mask(A, B, F, W, U) __builtin_ia32_reducess_mask(A, B, 1, W, U)
+#define __builtin_ia32_reducesd_mask(A, B, F, W, U) __builtin_ia32_reducesd_mask(A, B, 1, W, U)
 #define __builtin_ia32_reduceps512_mask(A, E, C, D) __builtin_ia32_reduceps512_mask(A, 1, C, D)
 #define __builtin_ia32_reducepd512_mask(A, E, C, D) __builtin_ia32_reducepd512_mask(A, 1, C, D)
 #define __builtin_ia32_rangess128_round(A, B, I, F) __builtin_ia32_rangess128_round(A, B, 1, 8)
@@ -619,6 +619,14 @@
 #define __builtin_ia32_extracti64x2_256_mask(A, E, C, D) __builtin_ia32_extracti64x2_256_mask(A, 1, C, D)
 #define __builtin_ia32_extractf64x2_256_mask(A, E, C, D) __builtin_ia32_extractf64x2_256_mask(A, 1, C, D)
 
-#pragma GCC target ("sse4a,3dnow,avx,avx2,fma4,xop,aes,pclmul,popcnt,abm,lzcnt,bmi,bmi2,tbm,lwp,fsgsbase,rdrnd,f16c,fma,rtm,rdseed,prfchw,adx,fxsr,xsaveopt,avx512f,avx512er,avx512cd,avx512pf,sha,prefetchwt1,xsavec,xsaves,clflushopt,avx512bw,avx512dq,avx512vl,avx512vbmi,avx512ifma,avx5124fmaps,avx5124vnniw,avx512vpopcntdq,clwb,mwaitx,clzero,pku,sgx,rdpid")
+/* gfniintrin.h */
+#define __builtin_ia32_vgf2p8affineinvqb_v16qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v16qi(A, B, 1) 
+#define __builtin_ia32_vgf2p8affineinvqb_v32qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v32qi(A, B, 1)
+#define __builtin_ia32_vgf2p8affineinvqb_v64qi(A, B, C) __builtin_ia32_vgf2p8affineinvqb_v64qi(A, B, 1)
+#define __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v16qi_mask(A, B, 1, D, E) 
+#define __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v32qi_mask(A, B, 1, D, E) 
+#define __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(A, B, C, D, E) __builtin_ia32_vgf2p8affineinvqb_v64qi_mask(A, B, 1, D, E) 
+
+#pragma GCC target ("sse4a,3dnow,avx,avx2,fma4,xop,aes,pclmul,popcnt,abm,lzcnt,bmi,bmi2,tbm,lwp,fsgsbase,rdrnd,f16c,fma,rtm,rdseed,prfchw,adx,fxsr,xsaveopt,avx512f,avx512er,avx512cd,avx512pf,sha,prefetchwt1,xsavec,xsaves,clflushopt,avx512bw,avx512dq,avx512vl,avx512vbmi,avx512ifma,avx5124fmaps,avx5124vnniw,avx512vpopcntdq,clwb,mwaitx,clzero,pku,sgx,rdpid,gfni")
 
 #include <x86intrin.h>
diff --git a/gcc/testsuite/gcc.target/i386/stack-check-12.c b/gcc/testsuite/gcc.target/i386/stack-check-12.c
new file mode 100644
index 00000000000..cb69bb08086
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/stack-check-12.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-clash-protection -mtune=generic" } */
+/* { dg-require-effective-target supports_stack_clash_protection } */
+
+__attribute__ ((noreturn)) void exit (int);
+
+__attribute__ ((noreturn)) void
+f (void)
+{
+  asm volatile ("nop" ::: "edi");
+  exit (1);
+}
+
+/* { dg-final { scan-assembler-not "or\[ql\]" } } */
+/* { dg-final { scan-assembler "pushl	%esi" { target ia32 } } } */
+/* { dg-final { scan-assembler "popl	%esi" { target ia32 } } }*/
+/* { dg-final { scan-assembler "pushq	%rax" { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler "popq	%rax" { target { ! ia32 } } } }*/
+
diff --git a/gcc/testsuite/gcc.target/i386/vect-pack-trunc-2.c b/gcc/testsuite/gcc.target/i386/vect-pack-trunc-2.c
index f3d899c1134..3503deaa9d9 100644
--- a/gcc/testsuite/gcc.target/i386/vect-pack-trunc-2.c
+++ b/gcc/testsuite/gcc.target/i386/vect-pack-trunc-2.c
@@ -25,4 +25,4 @@ avx512bw_test ()
       abort ();
 }
 
-/* { dg-final { scan-assembler-times "vpermi2w\[ \\t\]+\[^\n\]*%zmm" 1 } } */
+/* { dg-final { scan-assembler-times "vperm\[it]2w\[ \\t\]+\[^\n\]*%zmm" 1 } } */
diff --git a/gcc/testsuite/gcc.target/mips/msa.c b/gcc/testsuite/gcc.target/mips/msa.c
index 6b35e21bfd3..cdd5ca28dac 100644
--- a/gcc/testsuite/gcc.target/mips/msa.c
+++ b/gcc/testsuite/gcc.target/mips/msa.c
@@ -1,6 +1,6 @@
 /* Test MIPS MSA ASE instructions */
 /* { dg-do compile } */
-/* { dg-options "-mfp64 -mhard-float -mmsa -fexpensive-optimizations" } */
+/* { dg-options "-mfp64 -mhard-float -mmsa -fexpensive-optimizations -fcommon" } */
 /* { dg-skip-if "madd and msub need combine" { *-*-* } { "-O0" } { "" } } */
 
 /* { dg-final { scan-assembler-times "\t.comm\tv16i8_\\d+,16,16" 3 } } */
diff --git a/gcc/testsuite/gcc.target/nios2/cdx-branch.c b/gcc/testsuite/gcc.target/nios2/cdx-branch.c
index 3b984f2712a..3a9c459cec3 100644
--- a/gcc/testsuite/gcc.target/nios2/cdx-branch.c
+++ b/gcc/testsuite/gcc.target/nios2/cdx-branch.c
@@ -23,7 +23,7 @@ extern int i (int);
 extern int j (int);
 extern int k (int);
 
-int h (int a)
+int h (int a, int b)
 {
   int x;
 
@@ -31,7 +31,7 @@ int h (int a)
      an unconditional branch from one branch of the "if" to
      the return statement.  We compile this testcase with -Os to
      avoid insertion of a duplicate epilogue in place of the branch.  */
-  if (a == 1)
+  if (a == b)
     x = i (37);
   else
     x = j (42);
diff --git a/gcc/testsuite/gcc.target/nios2/gpopt-gprel-sec.c b/gcc/testsuite/gcc.target/nios2/gpopt-gprel-sec.c
new file mode 100644
index 00000000000..1083fe6e6ab
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/gpopt-gprel-sec.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O -mgpopt=local -mgprel-sec=\\.frog.+" } */
+
+extern int a __attribute__ ((section (".frog1")));
+static volatile int b __attribute__ ((section (".frog2"))) = 1;
+extern int c __attribute__ ((section (".data")));
+static volatile int d __attribute__ ((section (".data"))) = 2;
+
+extern int e;
+static volatile int f = 3;
+
+volatile int g __attribute__ ((weak)) = 4;
+
+extern int h[100];
+static int i[100];
+static int j[100] __attribute__ ((section (".sdata")));
+
+typedef int (*ftype) (int);
+extern int foo (int);
+
+extern int bar (int, int*, int*, int*, ftype);
+
+int baz (void)
+{
+  return bar (a + b + c + d + e + f + g, h, i, j, foo);
+}
+
+/* { dg-final { scan-assembler "%gprel\\(a\\)" } } */
+/* { dg-final { scan-assembler "%gprel\\(b\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(c\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(d\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(e\\)" } } */
+/* { dg-final { scan-assembler "%gprel\\(f\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(g\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(h\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(i\\)" } } */
+/* { dg-final { scan-assembler "%gprel\\(j\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(foo\\)" } } */
diff --git a/gcc/testsuite/gcc.target/nios2/gpopt-r0rel-sec.c b/gcc/testsuite/gcc.target/nios2/gpopt-r0rel-sec.c
new file mode 100644
index 00000000000..5fda9e9a381
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/gpopt-r0rel-sec.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O -mgpopt=local -mr0rel-sec=\\.frog.+" } */
+
+extern int a __attribute__ ((section (".frog1")));
+static volatile int b __attribute__ ((section (".frog2"))) = 1;
+extern int c __attribute__ ((section (".data")));
+static volatile int d __attribute__ ((section (".data"))) = 2;
+
+extern int e;
+static volatile int f = 3;
+
+volatile int g __attribute__ ((weak)) = 4;
+
+extern int h[100];
+static int i[100];
+static int j[100] __attribute__ ((section (".sdata")));
+
+typedef int (*ftype) (int);
+extern int foo (int);
+
+extern int bar (int, int*, int*, int*, ftype);
+
+int baz (void)
+{
+  return bar (a + b + c + d + e + f + g, h, i, j, foo);
+}
+
+/* { dg-final { scan-assembler "%lo\\(a\\)\\(r0\\)" } } */
+/* { dg-final { scan-assembler "%lo\\(b\\)\\(r0\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(c\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(d\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(e\\)" } } */
+/* { dg-final { scan-assembler "%gprel\\(f\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(g\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(h\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(i\\)" } } */
+/* { dg-final { scan-assembler "%gprel\\(j\\)" } } */
+/* { dg-final { scan-assembler-not "%gprel\\(foo\\)" } } */
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-bypass.c b/gcc/testsuite/gcc.target/nios2/lo-addr-bypass.c
new file mode 100644
index 00000000000..24e6cfd4cc0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-bypass.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=r2 -mbypass-cache" } */
+/* { dg-final { scan-assembler-times "addi\tr., r., %lo" 12 } } */
+/* { dg-final { scan-assembler-not "ldw\t" } } */
+/* { dg-final { scan-assembler-not "stw\t" } } */
+/* { dg-final { scan-assembler-not "ldwio\tr., %lo" } } */
+/* { dg-final { scan-assembler-not "stwio\tr., %lo" } } */
+
+/* Check that we do not generate %lo addresses with R2 ldstio instructions.
+   %lo requires a 16-bit relocation and on R2 these instructions only have a
+   12-bit register offset.  */
+#define TYPE int
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern TYPE S1;
+extern TYPE S2[];
+
+extern struct ss S3;
+extern struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-char.c b/gcc/testsuite/gcc.target/nios2/lo-addr-char.c
new file mode 100644
index 00000000000..dd992458323
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-char.c
@@ -0,0 +1,60 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-times "addi\tr., r., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "ldbu\tr., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "ldb\tr., %lo" 16 } } */
+/* { dg-final { scan-assembler-times "stb\tr., %lo" 4 } } */
+
+/* Check that various address forms involving a symbolic constant
+   with a possible constant offset and/or index register are optimized
+   to generate a %lo relocation in the load/store instructions instead
+   of a plain register indirect addressing mode.  */
+/* Note: get* uses ldhu but ext* uses ldh since TYPE is signed.  */
+
+#define TYPE signed char
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern TYPE S1;
+extern TYPE S2[];
+
+extern struct ss S3;
+extern struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
+int extw1 (void) { return (int)(S1); }
+int extw2 (int i) { return (int)(S2[i]); }
+int extw3 (void) { return (int)(S3.x2); }
+int extw4 (int i) { return (int)(S4[i].x2); }
+unsigned int extwu1 (void) { return (unsigned int)(S1); }
+unsigned int extwu2 (int i) { return (unsigned int)(S2[i]); }
+unsigned int extwu3 (void) { return (unsigned int)(S3.x2); }
+unsigned int extwu4 (int i) { return (unsigned int)(S4[i].x2); }
+
+short exth1 (void) { return (short)(S1); }
+short exth2 (int i) { return (short)(S2[i]); }
+short exth3 (void) { return (short)(S3.x2); }
+short exth4 (int i) { return (short)(S4[i].x2); }
+unsigned short exthu1 (void) { return (unsigned short)(S1); }
+unsigned short exthu2 (int i) { return (unsigned short)(S2[i]); }
+unsigned short exthu3 (void) { return (unsigned short)(S3.x2); }
+unsigned short exthu4 (int i) { return (unsigned short)(S4[i].x2); }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-int.c b/gcc/testsuite/gcc.target/nios2/lo-addr-int.c
new file mode 100644
index 00000000000..9a6f779d383
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-int.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-times "addi\tr., r., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "ldw\tr., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "stw\tr., %lo" 4 } } */
+
+/* Check that various address forms involving a symbolic constant
+   with a possible constant offset and/or index register are optimized
+   to generate a %lo relocation in the load/store instructions instead
+   of a plain register indirect addressing mode.  */
+
+#define TYPE int
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern TYPE S1;
+extern TYPE S2[];
+
+extern struct ss S3;
+extern struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-pic.c b/gcc/testsuite/gcc.target/nios2/lo-addr-pic.c
new file mode 100644
index 00000000000..bcd623785bd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-pic.c
@@ -0,0 +1,38 @@
+/* { dg-do compile { target nios2-*-linux-gnu } } */
+/* { dg-options "-O2 -fpic" } */
+/* { dg-final { scan-assembler-not "ldw\tr., %lo" } } */
+/* { dg-final { scan-assembler-not "stw\tr., %lo" } } */
+
+/* Check that address transformations for symbolic constants do NOT
+   apply to code compiled with -fPIC, which requires references to
+   go through the GOT pointer (r22) instead.  */
+
+#define TYPE int
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern TYPE S1;
+extern TYPE S2[];
+
+extern struct ss S3;
+extern struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-short.c b/gcc/testsuite/gcc.target/nios2/lo-addr-short.c
new file mode 100644
index 00000000000..792ec227291
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-short.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-times "addi\tr., r., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "ldhu\tr., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "ldh\tr., %lo" 8 } } */
+/* { dg-final { scan-assembler-times "sth\tr., %lo" 4 } } */
+
+/* Check that various address forms involving a symbolic constant
+   with a possible constant offset and/or index register are optimized
+   to generate a %lo relocation in the load/store instructions instead
+   of a plain register indirect addressing mode.  */
+/* Note: get* uses ldhu but ext* uses ldh since TYPE is signed.  */
+
+#define TYPE short
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern TYPE S1;
+extern TYPE S2[];
+
+extern struct ss S3;
+extern struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
+int extw1 (void) { return (int)(S1); }
+int extw2 (int i) { return (int)(S2[i]); }
+int extw3 (void) { return (int)(S3.x2); }
+int extw4 (int i) { return (int)(S4[i].x2); }
+unsigned int extwu1 (void) { return (unsigned int)(S1); }
+unsigned int extwu2 (int i) { return (unsigned int)(S2[i]); }
+unsigned int extwu3 (void) { return (unsigned int)(S3.x2); }
+unsigned int extwu4 (int i) { return (unsigned int)(S4[i].x2); }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-tls.c b/gcc/testsuite/gcc.target/nios2/lo-addr-tls.c
new file mode 100644
index 00000000000..d56fbc2ed81
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-tls.c
@@ -0,0 +1,38 @@
+/* { dg-require-effective-target tls } */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-not "ldw\tr., %lo" } } */
+/* { dg-final { scan-assembler-not "stw\tr., %lo" } } */
+
+/* Check that address transformations for symbolic constants do NOT
+   apply to TLS variables.  */
+
+#define TYPE int
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern __thread TYPE S1;
+extern __thread TYPE S2[];
+
+extern __thread struct ss S3;
+extern __thread struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-uchar.c b/gcc/testsuite/gcc.target/nios2/lo-addr-uchar.c
new file mode 100644
index 00000000000..e9733afde4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-uchar.c
@@ -0,0 +1,58 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-times "addi\tr., r., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "ldbu\tr., %lo" 20 } } */
+/* { dg-final { scan-assembler-times "stb\tr., %lo" 4 } } */
+
+/* Check that various address forms involving a symbolic constant
+   with a possible constant offset and/or index register are optimized
+   to generate a %lo relocation in the load/store instructions instead
+   of a plain register indirect addressing mode.  */
+
+#define TYPE unsigned char
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern TYPE S1;
+extern TYPE S2[];
+
+extern struct ss S3;
+extern struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
+int extw1 (void) { return (int)(S1); }
+int extw2 (int i) { return (int)(S2[i]); }
+int extw3 (void) { return (int)(S3.x2); }
+int extw4 (int i) { return (int)(S4[i].x2); }
+unsigned int extwu1 (void) { return (unsigned int)(S1); }
+unsigned int extwu2 (int i) { return (unsigned int)(S2[i]); }
+unsigned int extwu3 (void) { return (unsigned int)(S3.x2); }
+unsigned int extwu4 (int i) { return (unsigned int)(S4[i].x2); }
+
+short exth1 (void) { return (short)(S1); }
+short exth2 (int i) { return (short)(S2[i]); }
+short exth3 (void) { return (short)(S3.x2); }
+short exth4 (int i) { return (short)(S4[i].x2); }
+unsigned short exthu1 (void) { return (unsigned short)(S1); }
+unsigned short exthu2 (int i) { return (unsigned short)(S2[i]); }
+unsigned short exthu3 (void) { return (unsigned short)(S3.x2); }
+unsigned short exthu4 (int i) { return (unsigned short)(S4[i].x2); }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-ushort.c b/gcc/testsuite/gcc.target/nios2/lo-addr-ushort.c
new file mode 100644
index 00000000000..4a19c13bf2c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-ushort.c
@@ -0,0 +1,49 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { scan-assembler-times "addi\tr., r., %lo" 4 } } */
+/* { dg-final { scan-assembler-times "ldhu\tr., %lo" 12 } } */
+/* { dg-final { scan-assembler-times "sth\tr., %lo" 4 } } */
+
+/* Check that various address forms involving a symbolic constant
+   with a possible constant offset and/or index register are optimized
+   to generate a %lo relocation in the load/store instructions instead
+   of a plain register indirect addressing mode.  */
+
+#define TYPE unsigned short
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern TYPE S1;
+extern TYPE S2[];
+
+extern struct ss S3;
+extern struct ss S4[];
+
+TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
+int extw1 (void) { return (int)(S1); }
+int extw2 (int i) { return (int)(S2[i]); }
+int extw3 (void) { return (int)(S3.x2); }
+int extw4 (int i) { return (int)(S4[i].x2); }
+unsigned int extwu1 (void) { return (unsigned int)(S1); }
+unsigned int extwu2 (int i) { return (unsigned int)(S2[i]); }
+unsigned int extwu3 (void) { return (unsigned int)(S3.x2); }
+unsigned int extwu4 (int i) { return (unsigned int)(S4[i].x2); }
+
diff --git a/gcc/testsuite/gcc.target/nios2/lo-addr-volatile.c b/gcc/testsuite/gcc.target/nios2/lo-addr-volatile.c
new file mode 100644
index 00000000000..40a8be429bf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/lo-addr-volatile.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=r2 -mno-cache-volatile" } */
+/* { dg-final { scan-assembler-times "addi\tr., r., %lo" 12 } } */
+/* { dg-final { scan-assembler-not "ldw\t" } } */
+/* { dg-final { scan-assembler-not "stw\t" } } */
+/* { dg-final { scan-assembler-not "ldwio\tr., %lo" } } */
+/* { dg-final { scan-assembler-not "stwio\tr., %lo" } } */
+
+/* Check that we do not generate %lo addresses with R2 ldstio instructions.
+   %lo requires a 16-bit relocation and on R2 these instructions only have a
+   12-bit register offset.  */
+
+#define TYPE int
+
+struct ss
+{
+  TYPE x1,x2;
+};
+
+extern volatile TYPE S1;
+extern volatile TYPE S2[];
+
+extern volatile struct ss S3;
+extern volatile struct ss S4[];
+
+volatile TYPE *addr1 (void) { return &S1; }
+TYPE get1 (void) { return S1; }
+void set1 (TYPE value) { S1 = value; }
+
+volatile TYPE *addr2 (int i) { return &(S2[i]); }
+TYPE get2 (int i) { return S2[i]; }
+void set2 (int i, TYPE value) { S2[i] = value; }
+
+volatile TYPE *addr3 (void) { return &(S3.x2); }
+TYPE get3 (void) { return S3.x2; }
+void set3 (TYPE value) { S3.x2 = value; }
+
+volatile TYPE *addr4 (int i) { return &(S4[i].x2); }
+TYPE get4 (int i) { return S4[i].x2; }
+void set4 (int i, TYPE value) { S4[i].x2 = value; }
+
diff --git a/gcc/testsuite/gcc.target/powerpc/float128-fma2.c b/gcc/testsuite/gcc.target/powerpc/float128-fma2.c
deleted file mode 100644
index e5f15aa2de9..00000000000
--- a/gcc/testsuite/gcc.target/powerpc/float128-fma2.c
+++ /dev/null
@@ -1,9 +0,0 @@
-/* { dg-do compile { target { powerpc*-*-* && lp64 } } } */
-/* { dg-require-effective-target powerpc_p9vector_ok } */
-/* { dg-options "-mpower9-vector -mno-float128-hardware -O2" } */
-
-__float128
-xfma (__float128 a, __float128 b, __float128 c)
-{
-  return __builtin_fmaf128 (a, b, c); /* { dg-error "ISA 3.0 IEEE 128-bit" } */
-}
diff --git a/gcc/testsuite/gcc.target/powerpc/float128-hw.c b/gcc/testsuite/gcc.target/powerpc/float128-hw.c
index 68e4c27aa58..929c6ddabe0 100644
--- a/gcc/testsuite/gcc.target/powerpc/float128-hw.c
+++ b/gcc/testsuite/gcc.target/powerpc/float128-hw.c
@@ -2,16 +2,58 @@
 /* { dg-require-effective-target powerpc_p9vector_ok } */
 /* { dg-options "-mpower9-vector -O2" } */
 
-__float128 f128_add (__float128 a, __float128 b) { return a+b; }
-__float128 f128_sub (__float128 a, __float128 b) { return a-b; }
-__float128 f128_mul (__float128 a, __float128 b) { return a*b; }
-__float128 f128_div (__float128 a, __float128 b) { return a/b; }
-__float128 f128_fma (__float128 a, __float128 b, __float128 c) { return (a*b)+c; }
-long f128_cmove (__float128 a, __float128 b, long c, long d) { return (a == b) ? c : d; }
+#ifndef TYPE
+#define TYPE _Float128
+#endif
+
+/* Test the code generation of the various _Float128 operations.  */
+TYPE f128_add (TYPE a, TYPE b) { return a+b; }
+TYPE f128_sub (TYPE a, TYPE b) { return a-b; }
+TYPE f128_mul (TYPE a, TYPE b) { return a*b; }
+TYPE f128_div (TYPE a, TYPE b) { return a/b; }
+TYPE f128_fma (TYPE a, TYPE b, TYPE c) { return (a*b)+c; }
+TYPE f128_fms (TYPE a, TYPE b, TYPE c) { return (a*b)-c; }
+TYPE f128_nfma (TYPE a, TYPE b, TYPE c) { return -((a*b)+c); }
+TYPE f128_nfms (TYPE a, TYPE b, TYPE c) { return -((a*b)-c); }
+TYPE f128_neg (TYPE a) { return -a; }
+
+long f128_cmove (TYPE a, TYPE b, long c, long d) { return (a == b) ? c : d; }
+
+double f128_to_double (TYPE a) { return (double)a; }
+float f128_to_float (TYPE a) { return (float)a; }
+long f128_to_long (TYPE a) { return (long)a; }
+unsigned long f128_to_ulong (TYPE a) { return (unsigned long)a; }
+int f128_to_int (TYPE a) { return (int)a; }
+unsigned int f128_to_uint (TYPE a) { return (unsigned int)a; }
+
+TYPE double_to_f128 (double a) { return (TYPE)a; }
+TYPE float_to_f128 (float a) { return (TYPE)a; }
+TYPE long_to_f128 (long a) { return (TYPE)a; }
+TYPE ulong_to_f128 (unsigned long a) { return (TYPE)a; }
+TYPE int_to_f128 (int a) { return (TYPE)a; }
+TYPE uint_to_f128 (unsigned int a) { return (TYPE)a; }
+
+/* { dg-final { scan-assembler     {\mmfvsrd\M}    } } */
+/* { dg-final { scan-assembler     {\mmfvsrwz\M}   } } */
+/* { dg-final { scan-assembler     {\mmtvsrd\M}    } } */
+/* { dg-final { scan-assembler     {\mmtvsrwa\M}   } } */
+/* { dg-final { scan-assembler 	   {\mxscmpuqp\M}  } } */
+/* { dg-final { scan-assembler 	   {\mxscvdpqp\M}  } } */
+/* { dg-final { scan-assembler 	   {\mxscvqpdp\M}  } } */
+/* { dg-final { scan-assembler 	   {\mxscvqpdpo\M} } } */
+/* { dg-final { scan-assembler 	   {\mxscvqpsdz\M} } } */
+/* { dg-final { scan-assembler 	   {\mxscvqpswz\M} } } */
+/* { dg-final { scan-assembler 	   {\mxscvqpudz\M} } } */
+/* { dg-final { scan-assembler 	   {\mxscvqpuwz\M} } } */
+/* { dg-final { scan-assembler 	   {\mxscvsdqp\M}  } } */
+/* { dg-final { scan-assembler 	   {\mxscvudqp\M}  } } */
+/* { dg-final { scan-assembler 	   {\mxsdivqp\M}   } } */
+/* { dg-final { scan-assembler 	   {\mxsmaddqp\M}  } } */
+/* { dg-final { scan-assembler 	   {\mxsmsubqp\M}  } } */
+/* { dg-final { scan-assembler 	   {\mxsmulqp\M}   } } */
+/* { dg-final { scan-assembler 	   {\mxsnegqp\M}   } } */
+/* { dg-final { scan-assembler 	   {\mxsnmaddqp\M} } } */
+/* { dg-final { scan-assembler 	   {\mxsnmsubqp\M} } } */
+/* { dg-final { scan-assembler 	   {\mxssubqp\M}   } } */
+/* { dg-final { scan-assembler-not {\mbl\M}        } } */
 
-/* { dg-final { scan-assembler "xsaddqp"  } } */
-/* { dg-final { scan-assembler "xssubqp"  } } */
-/* { dg-final { scan-assembler "xsmulqp"  } } */
-/* { dg-final { scan-assembler "xsdivqp"  } } */
-/* { dg-final { scan-assembler "xsmaddqp" } } */
-/* { dg-final { scan-assembler "xscmpuqp" } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/float128-hw2.c b/gcc/testsuite/gcc.target/powerpc/float128-hw2.c
new file mode 100644
index 00000000000..f144360da3c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/float128-hw2.c
@@ -0,0 +1,60 @@
+/* { dg-do compile { target { powerpc*-*-* && lp64 } } } */
+/* { dg-require-effective-target powerpc_p9vector_ok } */
+/* { dg-options "-mpower9-vector -O2 -ffast-math -std=gnu11" } */
+
+/* Test to make sure the compiler handles the standard _Float128 functions that
+   have hardware support in ISA 3.0/power9.  */
+
+#define __STDC_WANT_IEC_60559_TYPES_EXT__ 1
+
+#ifndef __FP_FAST_FMAF128
+#error "__FP_FAST_FMAF128 should be defined."
+#endif
+
+extern _Float128 copysignf128 (_Float128, _Float128);
+extern _Float128 sqrtf128 (_Float128);
+extern _Float128 fmaf128 (_Float128, _Float128, _Float128);
+
+_Float128
+do_copysign (_Float128 a, _Float128 b)
+{
+  return copysignf128 (a, b);
+}
+
+_Float128
+do_sqrt (_Float128 a)
+{
+  return sqrtf128 (a);
+}
+
+_Float128
+do_fma (_Float128 a, _Float128 b, _Float128 c)
+{
+  return fmaf128 (a, b, c);
+}
+
+_Float128
+do_fms (_Float128 a, _Float128 b, _Float128 c)
+{
+  return fmaf128 (a, b, -c);
+}
+
+_Float128
+do_nfma (_Float128 a, _Float128 b, _Float128 c)
+{
+  return -fmaf128 (a, b, c);
+}
+
+_Float128
+do_nfms (_Float128 a, _Float128 b, _Float128 c)
+{
+  return -fmaf128 (a, b, -c);
+}
+
+/* { dg-final { scan-assembler     {\mxscpsgnqp\M} } } */
+/* { dg-final { scan-assembler     {\mxssqrtqp\M}  } } */
+/* { dg-final { scan-assembler     {\mxsmaddqp\M}  } } */
+/* { dg-final { scan-assembler     {\mxsmsubqp\M}  } } */
+/* { dg-final { scan-assembler     {\mxsnmaddqp\M} } } */
+/* { dg-final { scan-assembler     {\mxsnmsubqp\M} } } */
+/* { dg-final { scan-assembler-not {\mbl\M}        } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/float128-hw3.c b/gcc/testsuite/gcc.target/powerpc/float128-hw3.c
new file mode 100644
index 00000000000..e63099dde08
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/float128-hw3.c
@@ -0,0 +1,56 @@
+/* { dg-do compile { target { powerpc*-*-* && lp64 } } } */
+/* { dg-require-effective-target powerpc_p9vector_ok } */
+/* { dg-options "-mpower9-vector -O2 -ffast-math -std=c11" } */
+
+/* Test to make sure the compiler calls the external function instead of doing
+   the built-in processing for _Float128 functions that have hardware support
+   in ISA 3.0/power9 if are in strict standards mode, where the <func>f128 name
+   is not a synonym for __builtin_<func>f128.  */
+
+extern _Float128 copysignf128 (_Float128, _Float128);
+extern _Float128 sqrtf128 (_Float128);
+extern _Float128 fmaf128 (_Float128, _Float128, _Float128);
+
+_Float128
+do_copysign (_Float128 a, _Float128 b)
+{
+  return copysignf128 (a, b);
+}
+
+_Float128
+do_sqrt (_Float128 a)
+{
+  return sqrtf128 (a);
+}
+
+_Float128
+do_fma (_Float128 a, _Float128 b, _Float128 c)
+{
+  return fmaf128 (a, b, c);
+}
+
+_Float128
+do_fms (_Float128 a, _Float128 b, _Float128 c)
+{
+  return fmaf128 (a, b, -c);
+}
+
+_Float128
+do_nfma (_Float128 a, _Float128 b, _Float128 c)
+{
+  return -fmaf128 (a, b, c);
+}
+
+_Float128
+do_nfms (_Float128 a, _Float128 b, _Float128 c)
+{
+  return -fmaf128 (a, b, -c);
+}
+
+/* { dg-final { scan-assembler-not   {\mxscpsgnqp\M} } } */
+/* { dg-final { scan-assembler-not   {\mxssqrtqp\M}  } } */
+/* { dg-final { scan-assembler-not   {\mxsmaddqp\M}  } } */
+/* { dg-final { scan-assembler-not   {\mxsmsubqp\M}  } } */
+/* { dg-final { scan-assembler-not   {\mxsnmaddqp\M} } } */
+/* { dg-final { scan-assembler-not   {\mxsnmsubqp\M} } } */
+/* { dg-final { scan-assembler-times {\mbl\M} 6      } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/float128-sqrt2.c b/gcc/testsuite/gcc.target/powerpc/float128-sqrt2.c
deleted file mode 100644
index 94527ebbd98..00000000000
--- a/gcc/testsuite/gcc.target/powerpc/float128-sqrt2.c
+++ /dev/null
@@ -1,9 +0,0 @@
-/* { dg-do compile { target { powerpc*-*-* && lp64 } } } */
-/* { dg-require-effective-target powerpc_p9vector_ok } */
-/* { dg-options "-mpower9-vector -mno-float128-hardware -O2" } */
-
-__float128
-xsqrt (__float128 a)
-{
-  return __builtin_sqrtf128 (a); /* { dg-error "ISA 3.0 IEEE 128-bit" } */
-}
diff --git a/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-char.c b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-char.c
new file mode 100644
index 00000000000..19ea3d3184a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-char.c
@@ -0,0 +1,19 @@
+/* Verify that overloaded built-ins for vec_neg with char
+   inputs produce the right code.  */
+
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_altivec_ok } */
+/* { dg-options "-maltivec -O2" } */
+
+#include <altivec.h>
+
+vector signed char
+test2 (vector signed char x)
+{
+  return vec_neg (x);
+}
+
+/* { dg-final { scan-assembler-times "xxspltib|vspltisw|vxor" 1 } } */
+/* { dg-final { scan-assembler-times "vsububm" 1 } } */
+/* { dg-final { scan-assembler-times "vmaxsb" 0 } } */
+
diff --git a/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-floatdouble.c b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-floatdouble.c
new file mode 100644
index 00000000000..79ad92465a2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-floatdouble.c
@@ -0,0 +1,23 @@
+/* Verify that overloaded built-ins for vec_neg with float and
+   double inputs for VSX produce the right code.  */
+
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_vsx_ok } */
+/* { dg-options "-mvsx -O2" } */
+
+#include <altivec.h>
+
+vector float
+test1 (vector float x)
+{
+  return vec_neg (x);
+}
+
+vector double
+test2 (vector double x)
+{
+  return vec_neg (x);
+}
+
+/* { dg-final { scan-assembler-times "xvnegsp" 1 } } */
+/* { dg-final { scan-assembler-times "xvnegdp" 1 } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-int.c b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-int.c
new file mode 100644
index 00000000000..d6ca1283bc9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-int.c
@@ -0,0 +1,18 @@
+/* Verify that overloaded built-ins for vec_neg with int
+   inputs produce the right code.  */
+
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_altivec_ok } */
+/* { dg-options "-maltivec -O2" } */
+
+#include <altivec.h>
+
+vector signed int
+test1 (vector signed int x)
+{
+  return vec_neg (x);
+}
+
+/* { dg-final { scan-assembler-times "xxspltib|vspltisw|vxor" 1 } } */
+/* { dg-final { scan-assembler-times "vsubuwm" 1 } } */
+/* { dg-final { scan-assembler-times "vmaxsw" 0 } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-longlong.c b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-longlong.c
new file mode 100644
index 00000000000..48f71788648
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-longlong.c
@@ -0,0 +1,18 @@
+/* Verify that overloaded built-ins for vec_neg with long long
+   inputs produce the right code.  */
+
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_p8vector_ok } */
+/* { dg-options "-mpower8-vector -O2" } */
+
+#include <altivec.h>
+
+vector signed long long
+test3 (vector signed long long x)
+{
+  return vec_neg (x);
+}
+
+/* { dg-final { scan-assembler-times "xxspltib|vspltisw" 1 } } */
+/* { dg-final { scan-assembler-times "vsubudm" 1 } } */
+/* { dg-final { scan-assembler-times "vmaxsd" 0 } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-short.c b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-short.c
new file mode 100644
index 00000000000..997a9d48617
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/fold-vec-neg-short.c
@@ -0,0 +1,18 @@
+/* Verify that overloaded built-ins for vec_neg with short
+   inputs produce the right code.  */
+
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_altivec_ok } */
+/* { dg-options "-maltivec -O2" } */
+
+#include <altivec.h>
+
+vector signed short
+test3 (vector signed short x)
+{
+  return vec_neg (x);
+}
+
+/* { dg-final { scan-assembler-times "xxspltib|vspltisw|vxor" 1 } } */
+/* { dg-final { scan-assembler-times "vsubuhm" 1 } } */
+/* { dg-final { scan-assembler-times "vmaxsh" 0 } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/fold-vec-perm-longlong.c b/gcc/testsuite/gcc.target/powerpc/fold-vec-perm-longlong.c
index 7f3e57447f9..1333d882e0e 100644
--- a/gcc/testsuite/gcc.target/powerpc/fold-vec-perm-longlong.c
+++ b/gcc/testsuite/gcc.target/powerpc/fold-vec-perm-longlong.c
@@ -16,7 +16,7 @@ testbl (vector bool long long vbl2, vector bool long long vbl3,
 }
 
 vector signed long long
-testsl (vector signed long vsl2, vector signed long vsl3,
+testsl (vector signed long long vsl2, vector signed long long vsl3,
 	vector unsigned char vuc)
 {
   return vec_perm (vsl2, vsl3, vuc);
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-addpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-addpd-1.c
new file mode 100644
index 00000000000..1dffbbe3ce7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-addpd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_addpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_add_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] + s2.a[0];
+  e[1] = s1.a[1] + s2.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-addsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-addsd-1.c
new file mode 100644
index 00000000000..12c414d2d93
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-addsd-1.c
@@ -0,0 +1,54 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+
+#include <stdint.h>
+#include <stdio.h>
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_addsd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_add_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+
+  e[0] = s1.a[0] + s2.a[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_addsd_1; check_union128d failed\n");
+      printf ("\t [%f,%f] + [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-andnpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-andnpd-1.c
new file mode 100644
index 00000000000..89e52aa1265
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-andnpd-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_andnpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_andnot_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  long long source1[2]={34545, 95567};
+  long long source2[2]={674, 57897};
+  long long e[2];
+   
+  s1.x = _mm_loadu_pd ((double *)source1);
+  s2.x = _mm_loadu_pd ((double *)source2);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = (~source1[0]) & source2[0];
+  e[1] = (~source1[1]) & source2[1];
+
+  if (check_union128d (u, (double *)e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-andpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-andpd-1.c
new file mode 100644
index 00000000000..d23099b0e9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-andpd-1.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_andpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_and_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  
+  union
+  {
+    double d[2];
+    long long ll[2];
+  }source1, source2, e;
+   
+  s1.x = _mm_set_pd (34545, 95567);
+  s2.x = _mm_set_pd (674, 57897);
+
+  _mm_storeu_pd (source1.d, s1.x);
+  _mm_storeu_pd (source2.d, s2.x);
+
+  u.x = test (s1.x, s2.x); 
+   
+  e.ll[0] = source1.ll[0] & source2.ll[0];
+  e.ll[1] = source1.ll[1] & source2.ll[1];
+
+  if (check_union128d (u, e.d))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-check.h b/gcc/testsuite/gcc.target/powerpc/sse2-check.h
new file mode 100644
index 00000000000..beb1b7d24f4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-check.h
@@ -0,0 +1,52 @@
+#include <stdlib.h>
+
+/* Define this to enable the combination of VSX vector double and
+   SSE2 data types.  */
+#define __VSX_SSE2__ 1
+
+#include "m128-check.h"
+
+/* define DEBUG replace abort with printf on error.  */
+//#define DEBUG 1
+
+#if 1
+
+#define TEST sse2_test
+
+static void sse2_test (void);
+
+static void
+__attribute__ ((noinline))
+do_test (void)
+{
+  sse2_test ();
+}
+
+int
+main ()
+  {
+#ifdef __BUILTIN_CPU_SUPPORTS__
+    /* Most SSE2 (vector double) intrinsic operations require VSX
+       instructions, but some operations may need only VMX
+       instructions.  This also true for SSE2 scalar doubles as they
+       imply that "other half" of the vector remains unchanged or set
+       to zeros.  The VSX scalar operations leave ther "other half"
+       undefined, and require additional merge operations.
+       Some conversions (to/from integer) need the  direct register
+       transfer instructions from POWER8 for best performance.
+       So we test for arch_2_07.  */
+    if ( __builtin_cpu_supports ("arch_2_07") )
+      {
+	do_test ();
+#ifdef DEBUG
+	printf ("PASSED\n");
+#endif
+      }
+#ifdef DEBUG
+    else
+    printf ("SKIPPED\n");
+#endif
+#endif /* __BUILTIN_CPU_SUPPORTS__ */
+    return 0;
+  }
+#endif
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cmppd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cmppd-1.c
new file mode 100644
index 00000000000..af9df4d3209
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cmppd-1.c
@@ -0,0 +1,76 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cmp_pd_1
+#endif
+
+#include <emmintrin.h>
+#include <math.h>
+
+double ps1[] = {2134.3343, 6678.346};
+double ps2[] = {41124.234, 6678.346};
+long long pdd[] =  {1, 2}, pd[2];
+union{long long l[2]; double d[2];} pe;
+
+void pd_check(char *id, __m128d dst)
+{
+    __v2di dest = (__v2di)dst;
+
+    if(checkVl(pd, pe.l, 2))
+    {
+        printf("mm_cmp%s_pd FAILED\n", id);
+	printf("dst [%lld, %lld], e.l[%lld, %lld]\n", 
+		dest[0], dest[1], pe.l[0], pe.l[1]);
+    }
+}
+
+#define CMP(cmp, rel0, rel1)					\
+    pe.l[0] = rel0 ? -1 : 0;	                        \
+    pe.l[1] = rel1 ? -1 : 0;	                        \
+    dest = _mm_loadu_pd((double*)pdd);	      		\
+    source1 = _mm_loadu_pd(ps1);				\
+    source2 = _mm_loadu_pd(ps2);				\
+    dest = _mm_cmp##cmp##_pd(source1, source2);		\
+    _mm_storeu_pd((double*) pd, dest);			\
+    pd_check("" #cmp "", dest);
+
+static void
+TEST ()
+{
+    __m128d source1, source2, dest;
+
+    CMP(eq, !isunordered(ps1[0], ps2[0]) && ps1[0] == ps2[0],
+    		!isunordered(ps1[1], ps2[1]) && ps1[1] == ps2[1]);
+    CMP(lt, !isunordered(ps1[0], ps2[0]) && ps1[0] < ps2[0],
+    		!isunordered(ps1[1], ps2[1]) && ps1[1] < ps2[1]);
+    CMP(le, !isunordered(ps1[0], ps2[0]) && ps1[0] <= ps2[0],
+    		!isunordered(ps1[1], ps2[1]) && ps1[1] <= ps2[1]);
+    CMP(unord, isunordered(ps1[0], ps2[0]),
+    		isunordered(ps1[1], ps2[1]));
+    CMP(neq, isunordered(ps1[0], ps2[0]) || ps1[0] != ps2[0],
+    		isunordered(ps1[1], ps2[1]) || ps1[1] != ps2[01]);
+    CMP(nlt, isunordered(ps1[0], ps2[0]) || ps1[0] >= ps2[0],
+    		isunordered(ps1[1], ps2[1]) || ps1[1] >= ps2[1]);
+    CMP(nle, isunordered(ps1[0], ps2[0]) || ps1[0] > ps2[0],
+    		isunordered(ps1[1], ps2[1]) || ps1[1] > ps2[1]);
+    CMP(ord, !isunordered(ps1[0], ps2[0]),
+    		!isunordered(ps1[1], ps2[1]));
+
+    CMP(ge, isunordered(ps1[0], ps2[0]) || ps1[0] >= ps2[0],
+    		isunordered(ps1[1], ps2[1]) || ps1[1] >= ps2[1]);
+    CMP(gt, isunordered(ps1[0], ps2[0]) || ps1[0] > ps2[0],
+    		isunordered(ps1[1], ps2[1]) || ps1[1] > ps2[1]);
+    CMP(nge, !isunordered(ps1[0], ps2[0]) && ps1[0] < ps2[0],
+    		!isunordered(ps1[1], ps2[1]) && ps1[1] < ps2[1]);
+    CMP(ngt, !isunordered(ps1[0], ps2[0]) && ps1[0] <= ps2[0],
+    		!isunordered(ps1[1], ps2[1]) && ps1[1] <= ps2[1]);
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cmpsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cmpsd-1.c
new file mode 100644
index 00000000000..331923c53d3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cmpsd-1.c
@@ -0,0 +1,65 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cmp_sd_1
+#endif
+
+#include <emmintrin.h>
+#include <math.h>
+
+double s1[] = {2134.3343, 6678.346};
+double s2[] = {41124.234, 6678.346};
+long long dd[] =  {1, 2}, d[2];
+union{long long l[2]; double d[2];} e;
+
+void check(char *id, __m128d dst)
+{
+   __v2di dest = (__v2di)dst;
+
+   if(checkVl(d, e.l, 2))
+    {
+      printf("mm_cmp%s_sd FAILED\n", id);
+      printf("dst [%lld, %lld], e.l[%lld]\n",
+             dest[0], dest[1], e.l[0]);
+    }
+}
+
+#define CMP(cmp, rel)					\
+    e.l[0] = rel ? -1 : 0;	                        \
+    dest = _mm_loadu_pd((double*)dd);	      		\
+    source1 = _mm_loadu_pd(s1);				\
+    source2 = _mm_loadu_pd(s2);				\
+    dest = _mm_cmp##cmp##_sd(source1, source2);	\
+    _mm_storeu_pd((double*) d, dest);			\
+    check("" #cmp "", dest);
+
+static void
+TEST ()
+{
+    __m128d source1, source2, dest;
+
+    e.d[1] = s1[1];
+
+    CMP(eq, !isunordered(s1[0], s2[0]) && s1[0] == s2[0]);
+    CMP(lt, !isunordered(s1[0], s2[0]) && s1[0] < s2[0]);
+    CMP(le, !isunordered(s1[0], s2[0]) && s1[0] <= s2[0]);
+    CMP(unord, isunordered(s1[0], s2[0]));
+    CMP(neq, isunordered(s1[0], s2[0]) || s1[0] != s2[0]);
+    CMP(nlt, isunordered(s1[0], s2[0]) || s1[0] >= s2[0]);
+    CMP(nle, isunordered(s1[0], s2[0]) || s1[0] > s2[0]);
+    CMP(ord, !isunordered(s1[0], s2[0]));
+
+    CMP(ge, isunordered(s1[0], s2[0]) || s1[0] >= s2[0]);
+    CMP(gt, isunordered(s1[0], s2[0]) || s1[0] > s2[0]);
+    CMP(nge, !isunordered(s1[0], s2[0]) && s1[0] < s2[0]);
+    CMP(ngt, !isunordered(s1[0], s2[0]) && s1[0] <= s2[0]);
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-comisd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-1.c
new file mode 100644
index 00000000000..7bed4b41f5b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-1.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_comi_sd_1
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_comieq_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,2344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] == s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-comisd-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-2.c
new file mode 100644
index 00000000000..6a8b45d3102
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-2.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_comi_sd_2
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_comilt_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,2344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] < s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-comisd-3.c b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-3.c
new file mode 100644
index 00000000000..2ed5e4ee06f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-3.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_comi_sd_3
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_comile_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,2344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] <= s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-comisd-4.c b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-4.c
new file mode 100644
index 00000000000..2a3b5b8465f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-4.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_comi_sd_4
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_comigt_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,12344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] > s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-comisd-5.c b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-5.c
new file mode 100644
index 00000000000..59139cb0a9b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-5.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_comi_sd_5
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_comige_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,2344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] >= s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-comisd-6.c b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-6.c
new file mode 100644
index 00000000000..e904e2bc7fa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-comisd-6.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_comi_sd_6
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_comineq_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,2344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] != s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtdq2pd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtdq2pd-1.c
new file mode 100644
index 00000000000..0c9ee3a351d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtdq2pd-1.c
@@ -0,0 +1,55 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtepi32_pd
+#endif
+
+#include <emmintrin.h>
+#ifdef _ARCH_PWR8
+static __m128d
+__attribute__((noinline, unused))
+test (__m128i p)
+{
+  return _mm_cvtepi32_pd (p); 
+}
+#endif
+
+static void
+TEST (void)
+{
+#ifdef _ARCH_PWR8
+  union128d u;
+  union128i_d s;
+  double e[2];
+
+  s.x = _mm_set_epi32 (123, 321, 456, 987);
+
+  u.x = test (s.x);
+
+  e[0] = (double)s.a[0]; 
+  e[1] = (double)s.a[1]; 
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_cvtepi32_pd; check_union128d failed\n");
+      printf ("\t [%d,%d, %d, %d] -> [%f,%f]\n",
+    		  s.a[0], s.a[1], s.a[2], s.a[3],
+			  u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n",
+			  e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtdq2ps-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtdq2ps-1.c
new file mode 100644
index 00000000000..50dec1bf5a9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtdq2ps-1.c
@@ -0,0 +1,43 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtepi32_ps
+#endif
+
+#include <emmintrin.h>
+
+static __m128
+__attribute__((noinline, unused))
+test (__m128i p)
+{
+  return _mm_cvtepi32_ps (p); 
+}
+
+static void
+TEST (void)
+{
+  union128 u;
+  union128i_d s;
+  float e[4];
+
+  s.x = _mm_set_epi32 (123, 321, 456, 987);
+
+  u.x = test (s.x);
+
+  e[0] = (float)s.a[0]; 
+  e[1] = (float)s.a[1]; 
+  e[2] = (float)s.a[2]; 
+  e[3] = (float)s.a[3]; 
+
+  if (check_union128 (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtpd2dq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtpd2dq-1.c
new file mode 100644
index 00000000000..ecbcbe99b53
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtpd2dq-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtpd_epi32
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  return _mm_cvtpd_epi32 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u;
+  union128d s;
+  int e[4] = {0};
+
+  s.x = _mm_set_pd (2.78, 7777768.82);
+
+  u.x = test (s.x);
+
+  e[0] = (int)(s.a[0] + 0.5); 
+  e[1] = (int)(s.a[1] + 0.5); 
+
+  if (check_union128i_d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_cvtpd_epi32; check_union128i_d failed\n");
+      printf ("\t [%f,%f] -> [%d,%d,%d,%d]\n", s.a[0], s.a[1], u.a[0], u.a[1],
+	      u.a[2], u.a[3]);
+      printf ("\t expect [%d,%d,%d,%d]\n", e[0], e[1], e[2], e[3]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtpd2ps-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtpd2ps-1.c
new file mode 100644
index 00000000000..7c9c01dc33a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtpd2ps-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtpd_ps
+#endif
+
+#include <emmintrin.h>
+
+static __m128
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  return _mm_cvtpd_ps (p); 
+}
+
+static void
+TEST (void)
+{
+  union128 u;
+  union128d s;
+  float e[4] = { 0.0 };
+
+  s.x = _mm_set_pd (123.321, 456.987);
+
+  u.x = test (s.x);
+
+  e[0] = (float)s.a[0]; 
+  e[1] = (float)s.a[1]; 
+
+  if (check_union128 (u, e))
+#if DEBUG
+  {
+    printf ("sse2_test_cvtpd_ps; check_union128 failed\n");
+      printf ("\t [%f,%f] -> [%f,%f,%f,%f]\n", s.a[0], s.a[1], u.a[0], u.a[1],
+	      u.a[2], u.a[3]);
+      printf ("\t expect [%f,%f,%f,%f]\n", e[0], e[1], e[2], e[3]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtps2dq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtps2dq-1.c
new file mode 100644
index 00000000000..36a94ff88f9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtps2dq-1.c
@@ -0,0 +1,52 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtps2dq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128 p)
+{
+  return _mm_cvtps_epi32 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u;
+  union128 s;
+  int e[4] = {0};
+
+  s.x = _mm_set_ps (2.78, 7777768.82, 2.331, 3.456);
+
+  u.x = test (s.x);
+
+  e[0] = (int)(s.a[0] + 0.5); 
+  e[1] = (int)(s.a[1] + 0.5); 
+  e[2] = (int)(s.a[2] + 0.5); 
+  e[3] = (int)(s.a[3] + 0.5); 
+
+  if (check_union128i_d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_cvtps2dq_1; check_union128i_d failed\n");
+      printf ("\t [%f,%f,%f,%f] -> [%d,%d,%d,%d]\n", s.a[0], s.a[1], s.a[2],
+	      s.a[3], u.a[0], u.a[1], u.a[2], u.a[3]);
+      printf ("\t expect [%d,%d,%d,%d]\n", e[0], e[1], e[2], e[3]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtps2pd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtps2pd-1.c
new file mode 100644
index 00000000000..de85ac407dc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtps2pd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtps2pd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128 p)
+{
+  return _mm_cvtps_pd (p); 
+}
+
+static void
+TEST (void)
+{
+  union128d u;
+  union128 s;
+  double e[2];
+
+  s.x = _mm_set_ps (2.78, 7777768.82, 2.331, 3.456);
+
+  u.x = test (s.x);
+
+  e[0] = (double)s.a[0]; 
+  e[1] = (double)s.a[1]; 
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_cvtps2pd_1; check_union128d failed\n");
+      printf ("\t cvt\t [%f,%f,%f,%f] -> [%f,%f]\n", s.a[0], s.a[1], s.a[2],
+	      s.a[3], u.a[0], u.a[1]);
+      printf ("\t expect\t [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2si-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2si-1.c
new file mode 100644
index 00000000000..77a1ad5af4c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2si-1.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtsd2si_1
+#endif
+
+#include <emmintrin.h>
+
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  return _mm_cvtsd_si32 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128d s;
+  int e;
+  int d;
+
+  s.x = _mm_set_pd (123.321, 456.987);
+
+  d = test (s.x);
+
+  e = (int)(s.a[0] + 0.5);
+
+  if (d != e)
+#if DEBUG
+  {
+      printf ("sse2_test_cvtsd2si_1; failed\n");
+      printf ("\t [%f,%f] -> [%d]\n", s.a[0], s.a[1], d);
+      printf ("\t expect [%d]\n", e);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2si-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2si-2.c
new file mode 100644
index 00000000000..a36e0e90fb6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2si-2.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtsd2si_2
+#endif
+
+#include <emmintrin.h>
+
+static long long
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  return _mm_cvtsd_si64 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128d s;
+  long long e;
+  long long d;
+
+  s.x = _mm_set_pd (829496729501.4, 429496729501.4);
+
+  d = test (s.x);
+
+  e = (long long)(s.a[0] + 0.5);
+
+  if (d != e)
+#if DEBUG
+  {
+      printf ("sse2_test_cvtsd2si_2; failed\n");
+      printf ("\t [%f,%f] -> [%ld]\n", s.a[0], s.a[1], d);
+      printf ("\t expect [%ld]\n", e);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2ss-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2ss-1.c
new file mode 100644
index 00000000000..33274cfa73d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsd2ss-1.c
@@ -0,0 +1,53 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtsd2ss_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128 
+__attribute__((noinline, unused))
+test (__m128 p1, __m128d p2)
+{
+  return _mm_cvtsd_ss (p1, p2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1;
+  union128 u, s2;
+  double source1[2] = {123.345, 67.3321};
+  float  e[4] = {5633.098, 93.21, 3.34, 4555.2};
+
+  s1.x = _mm_loadu_pd (source1);
+  s2.x = _mm_loadu_ps (e);
+
+  __asm("" : "+v"(s1.x), "+v"(s2.x));
+  u.x = test(s2.x, s1.x);
+
+  e[0] = (float)source1[0];
+
+  if (check_union128(u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_cvtsd2ss_1; check_union128 failed\n");
+      printf ("\t [%f,%f,%f,%f],[%f,%f]\n", s2.a[0], s2.a[1], s2.a[2], s2.a[3],
+    		  s1.a[0], s1.a[1]);
+      printf ("\t -> \t[%f,%f,%f,%f]\n", u.a[0], u.a[1], u.a[2], u.a[3]);
+      printf ("\texpect\t[%f,%f,%f,%f]\n", e[0], e[1], e[2], e[3]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtsi2sd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsi2sd-1.c
new file mode 100644
index 00000000000..5465945e8b5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsi2sd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtsi2sd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d 
+__attribute__((noinline, unused))
+test (__m128d p, int b)
+{
+  __asm("" : "+v"(p), "+r"(b));
+  return _mm_cvtsi32_sd (p, b); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s;
+  int b = 128;
+  double e[2];
+
+  s.x = _mm_set_pd (123.321, 456.987);
+
+  u.x = test (s.x, b);
+  e[0] = (double)b;
+  e[1] = s.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtsi2sd-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsi2sd-2.c
new file mode 100644
index 00000000000..cd8f0840702
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtsi2sd-2.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtsi2sd_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128d 
+__attribute__((noinline, unused))
+test (__m128d p, long long b)
+{
+  __asm("" : "+v"(p), "+r"(b));
+  return _mm_cvtsi64_sd (p, b); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s;
+  long long b = 42949672951333LL;
+  double e[2];
+
+  s.x = _mm_set_pd (123.321, 456.987);
+
+  u.x = test (s.x, b);
+  e[0] = (double)b;
+  e[1] = s.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvtss2sd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvtss2sd-1.c
new file mode 100644
index 00000000000..d93bae68d8b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvtss2sd-1.c
@@ -0,0 +1,52 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvtss2sd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d 
+__attribute__((noinline, unused))
+test (__m128d a, __m128 b)
+{
+  return _mm_cvtss_sd (a, b); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1;
+  union128 s2;
+  double e[2];
+
+  s1.x = _mm_set_pd (123.321, 456.987);
+  s2.x = _mm_set_ps (123.321, 456.987, 666.45, 231.987);
+
+  u.x = test (s1.x, s2.x);
+
+  e[0] = (double)s2.a[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_cvtss2sd_1; check_union128d failed\n");
+      printf ("\t [%f,%f], [%f,%f,%f,%f]\n", s1.a[0], s1.a[1], s2.a[0], s2.a[1],
+	      s2.a[2], s2.a[3]);
+      printf ("\t -> \t[%f,%f]\n", u.a[0], u.a[1]);
+      printf ("\texpect\t[%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvttpd2dq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvttpd2dq-1.c
new file mode 100644
index 00000000000..baa7d3baa75
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvttpd2dq-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvttpd_epi32
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  return _mm_cvttpd_epi32 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128d s;
+  union128i_d u;
+  int e[4] = {0};
+
+  s.x = _mm_set_pd (123.321, 456.987);
+
+  u.x = test (s.x);
+
+  e[0] = (int)s.a[0]; 
+  e[1] = (int)s.a[1]; 
+
+  if (check_union128i_d (u, e))
+#if DEBUG
+  {
+	  printf ("sse2_test_cvttpd_epi32; check_union128i_d failed\n");
+      printf ("\t [%f,%f] -> [%d,%d,%d,%d]\n", s.a[0], s.a[1], u.a[0], u.a[1],
+	      u.a[2], u.a[3]);
+      printf ("\t expect [%d,%d,%d,%d]\n", e[0], e[1], e[2], e[3]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvttps2dq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvttps2dq-1.c
new file mode 100644
index 00000000000..88427d8c6f2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvttps2dq-1.c
@@ -0,0 +1,43 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvttps2dq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128 p)
+{
+  return _mm_cvttps_epi32 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128 s;
+  union128i_d u;
+  int e[4] = {0};
+
+  s.x = _mm_set_ps (123.321, 456.987, 33.56, 7765.321);
+
+  u.x = test (s.x);
+
+  e[0] = (int)s.a[0]; 
+  e[1] = (int)s.a[1]; 
+  e[2] = (int)s.a[2]; 
+  e[3] = (int)s.a[3]; 
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvttsd2si-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvttsd2si-1.c
new file mode 100644
index 00000000000..2dc96d1eaca
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvttsd2si-1.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvttsd2si_1
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  __asm("" : "+v"(p));
+  return _mm_cvttsd_si32 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128d s;
+  int e;
+  int d;
+
+  s.x = _mm_set_pd (123.321, 456.987);
+
+  d = test (s.x);
+  e = (int)(s.a[0]);
+
+  if (d != e)
+#if DEBUG
+  {
+      printf ("sse2_test_cvttsd2si_1; failed\n");
+      printf ("\t [%f,%f] -> [%d]\n", s.a[0], s.a[1], d);
+      printf ("\t expect [%d]\n", e);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-cvttsd2si-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-cvttsd2si-2.c
new file mode 100644
index 00000000000..cd6fa926b4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-cvttsd2si-2.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_cvttsd2si_2
+#endif
+
+#include <emmintrin.h>
+
+static long long
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  __asm("" : "+v"(p));
+  return _mm_cvttsd_si64 (p); 
+}
+
+static void
+TEST (void)
+{
+  union128d s;
+  long long e;
+  long long d;
+
+  s.x = _mm_set_pd (123.321, 42949672339501.4);
+
+  d = test (s.x);
+  e = (long long)(s.a[0]);
+
+  if (d != e)
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-divpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-divpd-1.c
new file mode 100644
index 00000000000..e4a3bdaa54f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-divpd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_divpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_div_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] / s2.a[0];
+  e[1] = s1.a[1] / s2.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+    printf ("sse2_test_divpd_1; check_union128d failed\n");
+    printf ("\t [%f,%f] * [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+    printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-divsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-divsd-1.c
new file mode 100644
index 00000000000..197151e9a18
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-divsd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_divsd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_div_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x);
+   
+  e[0] = s1.a[0] / s2.a[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+    printf ("sse2_test_divsd_1; check_union128d failed\n");
+    printf ("\t [%f,%f] / [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+    printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-maxpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-maxpd-1.c
new file mode 100644
index 00000000000..4462a2ee011
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-maxpd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_maxpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_max_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] > s2.a[0] ? s1.a[0]:s2.a[0];
+  e[1] = s1.a[1] > s2.a[1] ? s1.a[1]:s2.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-maxsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-maxsd-1.c
new file mode 100644
index 00000000000..e17628950fc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-maxsd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_maxsd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_max_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] > s2.a[0] ? s1.a[0]:s2.a[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_maxsd_3; check_union128d failed\n");
+      printf ("\t [%f,%f] + [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-minpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-minpd-1.c
new file mode 100644
index 00000000000..f4d1960bf78
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-minpd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_minpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_min_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] < s2.a[0] ? s1.a[0]:s2.a[0];
+  e[1] = s1.a[1] < s2.a[1] ? s1.a[1]:s2.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-minsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-minsd-1.c
new file mode 100644
index 00000000000..4b3087bc403
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-minsd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_minsd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_min_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] < s2.a[0] ? s1.a[0]:s2.a[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_minsd_3; check_union128d failed\n");
+      printf ("\t [%f,%f] + [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-mmx.c b/gcc/testsuite/gcc.target/powerpc/sse2-mmx.c
new file mode 100644
index 00000000000..115d83a4283
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-mmx.c
@@ -0,0 +1,82 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#include "sse2-check.h"
+
+#ifndef TEST
+#define TEST sse2_test_mmx_1
+#endif
+
+#include <mmintrin.h>
+
+#define N 4
+
+unsigned long long a[N], b[N], result[N];
+
+unsigned long long check_data[N] =
+  { 0x101010101010100full,
+    0x1010101010101010ull,
+    0x1010101010101010ull,
+    0x1010101010101010ull };
+
+__m64
+unsigned_add3 (const __m64 * a, const __m64 * b,
+	       __m64 * result, unsigned int count)
+{
+  __m64 _a, _b, one, sum, carry, onesCarry;
+
+  unsigned int i;
+
+  carry = _mm_setzero_si64 ();
+
+  one = _mm_cmpeq_pi8 (carry, carry);
+  one = _mm_sub_si64 (carry, one);
+
+  for (i = 0; i < count; i++)
+    {
+      _a = a[i];
+      _b = b[i];
+
+      sum = _mm_add_si64 (_a, _b);
+      sum = _mm_add_si64 (sum, carry);
+
+      result[i] = sum;
+
+      onesCarry = _mm_and_si64 (_mm_xor_si64 (_a, _b), carry);
+      onesCarry = _mm_or_si64 (_mm_and_si64 (_a, _b), onesCarry);
+      onesCarry = _mm_and_si64 (onesCarry, one);
+
+      _a = _mm_srli_si64 (_a, 1);
+      _b = _mm_srli_si64 (_b, 1);
+
+      carry = _mm_add_si64 (_mm_add_si64 (_a, _b), onesCarry);
+      carry = _mm_srli_si64 (carry, 63);
+    }
+
+  return carry;
+}
+
+void __attribute__((noinline))
+TEST (void)
+{
+  unsigned long long carry;
+  int i;
+
+  /* Really long numbers.  */
+  a[3] = a[2] = a[1] = a[0] = 0xd3d3d3d3d3d3d3d3ull;
+  b[3] = b[2] = b[1] = b[0] = 0x3c3c3c3c3c3c3c3cull;
+
+  carry = (unsigned long long) unsigned_add3
+    ((__m64 *)a, (__m64 *)b, (__m64 *)result, N);
+
+  _mm_empty ();
+
+  if (carry != 1)
+    abort ();
+
+  for (i = 0; i < N; i++)
+    if (result [i] != check_data[i])
+      abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movhpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-movhpd-1.c
new file mode 100644
index 00000000000..9b7c2ec3a92
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movhpd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movhpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, double *p)
+{
+  __asm("" : "+v"(s1), "+b"(p));
+  return _mm_loadh_pd (s1, p); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1;
+  double s2[2] = {41124.234,2344.2354};
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  u.x = test (s1.x, s2); 
+
+  e[0] = s1.a[0];
+  e[1] = s2[0];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movhpd-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-movhpd-2.c
new file mode 100644
index 00000000000..b5eb08657cd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movhpd-2.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movhpd_2
+#endif
+
+#include <emmintrin.h>
+
+static void
+__attribute__((noinline, unused))
+test (double *p, __m128d a)
+{
+  __asm("" : "+v"(a), "+b"(p));
+  return _mm_storeh_pd (p, a); 
+}
+
+static void
+TEST (void)
+{
+  union128d s;
+  double d[1];
+  double e[1];
+   
+  s.x = _mm_set_pd (2134.3343,1234.635654);
+  test (d, s.x);
+
+  e[0] = s.a[1];
+
+  if (e[0] != d[0])
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movlpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-movlpd-1.c
new file mode 100644
index 00000000000..fec05bca99a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movlpd-1.c
@@ -0,0 +1,43 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movlpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d a, double *e)
+{
+  __asm("" : "+v"(a), "+b"(e));
+  return _mm_loadl_pd (a, e); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1;
+  double d[2] = {2134.3343,1234.635654};
+  double e[2];
+
+  s1.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = _mm_loadu_pd (d);
+
+  u.x = test (s1.x, d);  
+
+  e[0] = d[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movlpd-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-movlpd-2.c
new file mode 100644
index 00000000000..6974d3be646
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movlpd-2.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movlpd_2
+#endif
+
+#include <emmintrin.h>
+
+static void
+__attribute__((noinline, unused))
+test (double *e, __m128d a)
+{
+  __asm("" : "+v"(a), "+b"(e));
+  return _mm_storel_pd (e, a); 
+}
+
+static void
+TEST (void)
+{
+  union128d u;
+  double e[2];
+
+  u.x = _mm_set_pd (41124.234,2344.2354);
+
+  test (e, u.x);  
+
+  e[1] = u.a[1];
+  
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movmskpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-movmskpd-1.c
new file mode 100644
index 00000000000..dda519dc762
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movmskpd-1.c
@@ -0,0 +1,61 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movmskpd_1
+#endif
+
+#include <emmintrin.h>
+
+#ifdef _ARCH_PWR8
+static int
+__attribute__((noinline, unused))
+test (__m128d p)
+{
+  __asm("" : "+v"(p));
+  return _mm_movemask_pd (p); 
+}
+#endif
+
+static void
+TEST (void)
+{
+#ifdef _ARCH_PWR8
+  double source[2] = {1.234, -2234.23};
+  union128d s1;
+  int d;
+  int e;
+
+  s1.x = _mm_loadu_pd (source);
+
+  d = test (s1.x);
+
+  e = 0;
+  if (source[0] < 0)
+    e |= 1;
+  
+  if (source[1] < 0)
+    e |= 1 << 1;
+
+  if (checkVi (&d, &e, 1))
+#if DEBUG
+  {
+    printf ("sse2_test_movmskpd_1; check_union128d failed\n");
+    printf ("\t [%f,%f] -> [%d]\n",
+    		  s1.a[0], s1.a[1], d);
+    printf ("\t expect [%d]\n",
+			  e);
+  }
+#else
+    abort ();
+#endif
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-movq-1.c
new file mode 100644
index 00000000000..6b65e15b09c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movq-1.c
@@ -0,0 +1,47 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i b)
+{
+  __asm("" : "+v"(b));
+  return _mm_move_epi64 (b); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q u, s1;
+  long long e[2] = { 0 };
+
+  s1.x = _mm_set_epi64x(12876, 3376590);
+  u.x = test (s1.x);
+  e[0] = s1.a[0];
+
+  if (check_union128i_q (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_movq_1; check_union128i_q failed\n");
+      printf ("\t move_epi64 ([%llx, %llx]) -> [%llx, %llx]\n", s1.a[0],
+	      s1.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%llx, %llx]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movq-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-movq-2.c
new file mode 100644
index 00000000000..e742157e9ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movq-2.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movq_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (long long b)
+{
+  __asm("" : "+r" (b));
+  return _mm_cvtsi64_si128 (b); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q u;
+  long long b = 4294967295133LL;
+  long long e[2] = {0};
+
+  u.x = test (b);
+
+  e[0] = b;
+
+  if (check_union128i_q (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movq-3.c b/gcc/testsuite/gcc.target/powerpc/sse2-movq-3.c
new file mode 100644
index 00000000000..ea80e2375d8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movq-3.c
@@ -0,0 +1,36 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movq_3
+#endif
+
+#include <emmintrin.h>
+
+static long long
+__attribute__((noinline, unused))
+test (__m128i b)
+{
+  __asm("" : "+v"(b));
+  return _mm_cvtsi128_si64 (b); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q u;
+  long long e;
+
+  u.x = _mm_set_epi64x (4294967295133LL, 3844294967295133LL);
+  e = test (u.x);
+  if (e != u.a[0])
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-movsd-1.c
new file mode 100644
index 00000000000..fe471ed1aa2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movsd-1.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movsd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (double *p)
+{
+  return _mm_load_sd (p); 
+}
+
+static void
+TEST (void)
+{
+  union128d u;
+  double d[2] = {128.023, 3345.1234};
+  double e[2];
+
+  u.x = _mm_loadu_pd (e);
+  u.x = test (d);
+
+  e[0] = d[0];
+  e[1] = 0.0;
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movsd-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-movsd-2.c
new file mode 100644
index 00000000000..2c1e35538f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movsd-2.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movsd_2
+#endif
+
+#include <emmintrin.h>
+
+static void 
+__attribute__((noinline, unused))
+test (double *p, __m128d a)
+{
+  _mm_store_sd (p, a); 
+}
+
+static void
+TEST (void)
+{
+  union128d u;
+  double d[1];
+  double e[1];
+
+  u.x = _mm_set_pd (128.023, 3345.1234);
+  test (d, u.x);
+
+  e[0] = u.a[0];
+
+  if (checkVd (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-movsd-3.c b/gcc/testsuite/gcc.target/powerpc/sse2-movsd-3.c
new file mode 100644
index 00000000000..57a5c23ae1c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-movsd-3.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_movsd_3
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d a, __m128d b)
+{
+  __asm("" : "+v"(a), "+v"(b));
+  return _mm_move_sd (a, b);
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2] = { 256.046, 3345.1234 };
+
+  s1.x = _mm_setr_pd (128.023, 3345.1234);
+  s2.x = _mm_setr_pd (256.046, 4533.1234);
+  __asm("" : "+v"(s1.x), "+v"(s2.x));
+  u.x = test (s1.x, s2.x);
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_movsd_3; check_union128d failed\n");
+      printf ("\t [%f,%f], [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-mulpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-mulpd-1.c
new file mode 100644
index 00000000000..b19f3b86123
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-mulpd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_mulpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_mul_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x);
+   
+  e[0] = s1.a[0] * s2.a[0];
+  e[1] = s1.a[1] * s2.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_mul_pd_1; check_union128d failed\n");
+      printf ("\t [%f,%f] * [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-mulsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-mulsd-1.c
new file mode 100644
index 00000000000..8206d263459
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-mulsd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_mulsd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_mul_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] * s2.a[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_mul_sd_1; check_union128d failed\n");
+      printf ("\t [%f,%f] * [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-orpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-orpd-1.c
new file mode 100644
index 00000000000..f3d9ab8f458
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-orpd-1.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_orpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_or_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+
+  union
+  {
+     double d[2];
+     long long ll[2];
+  }d1, d2, e;
+
+  s1.x = _mm_set_pd (1234, 44386);
+  s2.x = _mm_set_pd (5198, 23098);
+
+  _mm_storeu_pd (d1.d, s1.x);
+  _mm_storeu_pd (d2.d, s2.x);
+
+  u.x = test (s1.x, s2.x);
+  
+  e.ll[0] = d1.ll[0] | d2.ll[0];
+  e.ll[1] = d1.ll[1] | d2.ll[1];
+
+  if (check_union128d (u, e.d))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-packssdw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-packssdw-1.c
new file mode 100644
index 00000000000..d67d47c4360
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-packssdw-1.c
@@ -0,0 +1,73 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_packssdw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_packs_epi32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d s1, s2;
+  union128i_w u;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi32 (2134, -128, 655366, 9999);
+  s2.x = _mm_set_epi32 (41124, 234, 2, -800900);
+  u.x = test (s1.x, s2.x); 
+
+  for (i = 0; i < 4; i++)
+    {
+      if (s1.a[i] > 32767)
+        e[i] = 32767;
+      else if (s1.a[i] < -32768)
+        e[i] = -32768;
+      else
+        e[i] = s1.a[i];
+    }
+  
+  for (i = 0; i < 4; i++)
+   {
+      if (s2.a[i] > 32767)
+        e[i+4] = 32767;
+      else if (s2.a[i] < -32768)
+        e[i+4] = -32768;
+      else
+        e[i+4] = s2.a[i];
+    }
+
+  if (check_union128i_w (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_packssdw_1; check_union128i_w failed\n");
+      printf (
+	  "\t ([%x,%x,%x,%x], [%x,%x,%x,%x]) -> [%x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  s1.a[0], s1.a[1], s1.a[2], s1.a[3], s2.a[0], s2.a[1], s2.a[2],
+	  s2.a[3], u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6],
+	  u.a[7]);
+      printf ("\t expect [%x,%x,%x,%x, %x,%x,%x,%x]\n", e[0], e[1], e[2], e[3],
+			  e[4], e[5], e[6], e[7]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-packsswb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-packsswb-1.c
new file mode 100644
index 00000000000..3043688bfd4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-packsswb-1.c
@@ -0,0 +1,78 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_packsswb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_packs_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w s1, s2;
+  union128i_b u;
+  char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi16 (2134, -128, 1234, 6354, 1002, 3004, 4050, 9999);
+  s2.x = _mm_set_epi16 (41124, 234, 2344, 2354, 607, 1, 2, -8009);
+  u.x = test (s1.x, s2.x); 
+
+  for (i = 0; i < 8; i++)
+    {
+      if (s1.a[i] > 127)
+        e[i] = 127;
+      else if (s1.a[i] < -128)
+        e[i] = -128;
+      else
+        e[i] = s1.a[i];
+    }
+  
+  for (i = 0; i < 8; i++)
+   {
+      if (s2.a[i] > 127)
+        e[i+8] = 127;
+      else if (s2.a[i] < -128)
+        e[i+8] = -128;
+      else
+        e[i+8] = s2.a[i];
+    }
+
+  if (check_union128i_b (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_packsswb_1; check_union128i_w failed\n");
+      printf ("\t ([%x,%x,%x,%x, %x,%x,%x,%x], [%x,%x,%x,%x, %x,%x,%x,%x])\n",
+	      s1.a[0], s1.a[1], s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6],
+	      s1.a[7], s2.a[0], s2.a[1], s2.a[2], s2.a[3], s2.a[4], s2.a[5],
+	      s2.a[6], s2.a[7]);
+      printf ("\t\t -> [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	      u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7],
+	      u.a[8], u.a[9], u.a[10], u.a[11], u.a[12], u.a[13], u.a[14],
+	      u.a[15]);
+      printf (
+	  "\t expect [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  e[0], e[1], e[2], e[3], e[4], e[5], e[6], e[7], e[8], e[9], e[10],
+	  e[11], e[12], e[13], e[14], e[15]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-packuswb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-packuswb-1.c
new file mode 100644
index 00000000000..825003d6103
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-packuswb-1.c
@@ -0,0 +1,69 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_packuswb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_packus_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w s1, s2;
+  union128i_ub u;
+  unsigned char e[16];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (1, 2, 3, 4, -5, -6, -7, -8);
+  s2.x = _mm_set_epi16 (-9, -10, -11, -12, 13, 14, 15, 16);
+  u.x = test (s1.x, s2.x); 
+
+  for (i=0; i<8; i++)
+    {
+      tmp = s1.a[i]<0 ? 0 : s1.a[i];
+      tmp = tmp>255 ? 255 : tmp;
+      e[i] = tmp;
+
+      tmp = s2.a[i]<0 ? 0 : s2.a[i];
+      tmp = tmp>255 ? 255 : tmp;
+      e[i+8] = tmp;
+    }
+
+  if (check_union128i_ub (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_packuswb_1; check_union128i_w failed\n");
+      printf ("\t ([%x,%x,%x,%x, %x,%x,%x,%x], [%x,%x,%x,%x, %x,%x,%x,%x])\n",
+	      s1.a[0], s1.a[1], s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6],
+	      s1.a[7], s2.a[0], s2.a[1], s2.a[2], s2.a[3], s2.a[4], s2.a[5],
+	      s2.a[6], s2.a[7]);
+      printf ("\t\t -> [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	      u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7],
+	      u.a[8], u.a[9], u.a[10], u.a[11], u.a[12], u.a[13], u.a[14],
+	      u.a[15]);
+      printf (
+	  "\t expect [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  e[0], e[1], e[2], e[3], e[4], e[5], e[6], e[7], e[8], e[9], e[10],
+	  e[11], e[12], e[13], e[14], e[15]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddb-1.c
new file mode 100644
index 00000000000..766e2ecfde7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddb-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_add_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,-80,-40,-100,-15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, -100, -34, -78, -39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+     e[i] = s1.a[i] + s2.a[i];
+
+  if (check_union128i_b (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddd-1.c
new file mode 100644
index 00000000000..8c5796b282f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddd-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_add_epi32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s1, s2;
+  int e[4];
+  int i;
+   
+  s1.x = _mm_set_epi32 (30,90,-80,-40);
+  s2.x = _mm_set_epi32 (76, -100, -34, -78);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 4; i++)
+     e[i] = s1.a[i] + s2.a[i];
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddq-1.c
new file mode 100644
index 00000000000..67a85a089d3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddq-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_add_epi64 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q u, s1, s2;
+  long long e[2];
+  int i;
+   
+  s1.x = _mm_set_epi64x (90,-80);
+  s2.x = _mm_set_epi64x (76, -100);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 2; i++)
+     e[i] = s1.a[i] + s2.a[i];
+
+  if (check_union128i_q (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddsb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddsb-1.c
new file mode 100644
index 00000000000..cb6f37ad229
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddsb-1.c
@@ -0,0 +1,74 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddsb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_adds_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,-80,-40,-100,-15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, -100, -34, -78, -39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+    {
+      tmp = (signed char)s1.a[i] + (signed char)s2.a[i];
+
+      if (tmp > 127)
+        tmp = 127;
+      if (tmp < -128)
+        tmp = -128;
+
+      e[i] = tmp;
+    }
+
+  if (check_union128i_b (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_paddsb_1; check_union128i_b failed\n");
+      printf (
+	  "\tadds\t([%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x],\n",
+	      s1.a[0], s1.a[1], s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6],
+	      s1.a[7], s1.a[8], s1.a[9], s1.a[10], s1.a[11], s1.a[12], s1.a[13],
+	      s1.a[14], s1.a[15]);
+      printf ("\t\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x])\n",
+	      s2.a[0], s2.a[1], s2.a[2], s2.a[3], s2.a[4], s2.a[5], s2.a[6],
+	      s2.a[7], s2.a[8], s2.a[9], s2.a[10], s2.a[11], s2.a[12], s2.a[13],
+	      s2.a[14], s2.a[15]);
+      printf ("\t ->\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	      u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7],
+	      u.a[8], u.a[9], u.a[10], u.a[11], u.a[12], u.a[13], u.a[14],
+	      u.a[15]);
+      printf (
+	  "\texpect\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  e[0], e[1], e[2], e[3], e[4], e[5], e[6], e[7], e[8], e[9], e[10],
+	  e[11], e[12], e[13], e[14], e[15]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddsw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddsw-1.c
new file mode 100644
index 00000000000..82ce0a4b62a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddsw-1.c
@@ -0,0 +1,65 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddsw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_adds_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,-80,-40,-100,-15);
+  s2.x = _mm_set_epi16 (11, 98, 76, -100, -34, -78, -39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      tmp = s1.a[i] + s2.a[i];
+      
+      if (tmp > 32767)
+        tmp = 32767;
+      if (tmp < -32768)
+        tmp = -32768;
+      
+      e[i] = tmp;
+    }
+
+  if (check_union128i_w (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_paddsw_1; check_union128i_w failed\n");
+      printf ("\tadds\t([%x,%x,%x,%x, %x,%x,%x,%x],\n", s1.a[0], s1.a[1],
+	      s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6], s1.a[7]);
+      printf ("\t\t [%x,%x,%x,%x, %x,%x,%x,%x])\n", s2.a[0], s2.a[1], s2.a[2],
+	      s2.a[3], s2.a[4], s2.a[5], s2.a[6], s2.a[7]);
+      printf ("\t ->\t [%x,%x,%x,%x, %x,%x,%x,%x]\n", u.a[0], u.a[1], u.a[2],
+	      u.a[3], u.a[4], u.a[5], u.a[6], u.a[7]);
+      printf ("\texpect\t [%x,%x,%x,%x, %x,%x,%x,%x]\n", e[0], e[1], e[2], e[3],
+	      e[4], e[5], e[6], e[7]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddusb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddusb-1.c
new file mode 100644
index 00000000000..df3d8b230ee
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddusb-1.c
@@ -0,0 +1,74 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddusb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_adds_epu8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16] = {0};
+  int i, tmp;
+   
+  s1.x = _mm_set_epi8 (30, 2, 3, 4, 10, 20, 30, 90, 80, 40, 100, 15, 98, 25, 98, 7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, 100, 34, 78, 39, 6,  3,  4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+    {
+      tmp = (unsigned char)s1.a[i] + (unsigned char)s2.a[i];
+
+      if (tmp > 255)
+        tmp = -1;
+      if (tmp < 0)
+        tmp = 0; 
+
+      e[i] = tmp; 
+    }
+
+  if (check_union128i_b (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_paddusb_1; check_union128i_b failed\n");
+      printf (
+	  "\tadds\t([%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x],\n",
+	      s1.a[0], s1.a[1], s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6],
+	      s1.a[7], s1.a[8], s1.a[9], s1.a[10], s1.a[11], s1.a[12], s1.a[13],
+	      s1.a[14], s1.a[15]);
+      printf ("\t\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x])\n",
+	      s2.a[0], s2.a[1], s2.a[2], s2.a[3], s2.a[4], s2.a[5], s2.a[6],
+	      s2.a[7], s2.a[8], s2.a[9], s2.a[10], s2.a[11], s2.a[12], s2.a[13],
+	      s2.a[14], s2.a[15]);
+      printf ("\t ->\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	      u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7],
+	      u.a[8], u.a[9], u.a[10], u.a[11], u.a[12], u.a[13], u.a[14],
+	      u.a[15]);
+      printf (
+	  "\texpect\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  e[0], e[1], e[2], e[3], e[4], e[5], e[6], e[7], e[8], e[9], e[10],
+	  e[11], e[12], e[13], e[14], e[15]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddusw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddusw-1.c
new file mode 100644
index 00000000000..0bc707446b5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddusw-1.c
@@ -0,0 +1,52 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddusw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_adds_epu16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,80,40,100,15);
+  s2.x = _mm_set_epi16 (11, 98, 76, 100, 34, 78, 39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      tmp = s1.a[i] + s2.a[i];
+      
+      if (tmp > 65535)
+        tmp = -1;
+
+      if (tmp < 0)
+        tmp = 0;
+      
+      e[i] = tmp;
+    }
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-paddw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-paddw-1.c
new file mode 100644
index 00000000000..d91351efa92
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-paddw-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_paddw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_add_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,-80,-40,-100,-15);
+  s2.x = _mm_set_epi16 (11, 98, 76, -100, -34, -78, -39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+     e[i] = s1.a[i] + s2.a[i];
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pavgb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pavgb-1.c
new file mode 100644
index 00000000000..5d489a3ce08
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pavgb-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pavgb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_avg_epu8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_ub u, s1, s2;
+  unsigned char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,80,40,100,15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, 100, 34, 78, 39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+     e[i] = (s1.a[i] + s2.a[i]+1)>>1;
+
+  if (check_union128i_ub (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pavgw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pavgw-1.c
new file mode 100644
index 00000000000..995cb8fa488
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pavgw-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pavgw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_avg_epu16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_uw u, s1, s2;
+  unsigned short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,80,40,100,15);
+  s2.x = _mm_set_epi16 (11, 98, 76, 100, 34, 78, 39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+     e[i] = (s1.a[i] + s2.a[i]+1)>>1;
+
+  if (check_union128i_uw (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqb-1.c
new file mode 100644
index 00000000000..3d0e4c60e33
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqb-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pcmpeqb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_cmpeq_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,80,40,100,15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, 100, 34, 78, 39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+     e[i] = (s1.a[i] == s2.a[i]) ? -1:0;
+
+  if (check_union128i_b (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqd-1.c
new file mode 100644
index 00000000000..4af7deccae8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pcmpeqd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_cmpeq_epi32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s1, s2;
+  int e[4];
+  int i;
+   
+  s1.x = _mm_set_epi32 (98, 25, 98,7);
+  s2.x = _mm_set_epi32 (88, 44, 33, 229);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 4; i++)
+     e[i] = (s1.a[i] == s2.a[i]) ? -1:0;
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqw-1.c
new file mode 100644
index 00000000000..fca6b099e96
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpeqw-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pcmpeqw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_cmpeq_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (20,30,90,80,40,100,15,98);
+  s2.x = _mm_set_epi16 (34, 78, 39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+     e[i] = (s1.a[i] == s2.a[i]) ? -1:0;
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtb-1.c
new file mode 100644
index 00000000000..10fbc5cd912
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtb-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pcmpgtb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_cmpgt_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,80,40,100,15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, 100, 34, 78, 39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+     e[i] = (s1.a[i] > s2.a[i]) ? -1:0;
+
+  if (check_union128i_b (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtd-1.c
new file mode 100644
index 00000000000..bc046d6944b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pcmpgtd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_cmpgt_epi32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s1, s2;
+  int e[4];
+  int i;
+   
+  s1.x = _mm_set_epi32 (98, 25, 98,7);
+  s2.x = _mm_set_epi32 (88, 44, 33, 229);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 4; i++)
+     e[i] = (s1.a[i] > s2.a[i]) ? -1:0;
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtw-1.c
new file mode 100644
index 00000000000..19b82cbd7d4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pcmpgtw-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pcmpgtw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_cmpgt_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (20,30,90,80,40,100,15,98);
+  s2.x = _mm_set_epi16 (34, 78, 39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+     e[i] = (s1.a[i] > s2.a[i]) ? -1:0;
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pextrw.c b/gcc/testsuite/gcc.target/powerpc/sse2-pextrw.c
new file mode 100644
index 00000000000..2bb812e411f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pextrw.c
@@ -0,0 +1,65 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pextrw_1
+#endif
+
+#include <emmintrin.h>
+
+#define msk0   0
+#define msk1   1
+#define msk2   2
+#define msk3   3
+#define msk4   4
+#define msk5   5
+#define msk6   6
+#define msk7   7
+
+static void
+TEST (void)
+{
+  union
+    {
+      __m128i x;
+      int i[4];
+      short s[8];
+    } val1;
+  int res[8], masks[8];
+  int i;
+
+  val1.i[0] = 0x04030201;
+  val1.i[1] = 0x08070605;
+  val1.i[2] = 0x0C0B0A09;
+  val1.i[3] = 0x100F0E0D;
+
+  res[0] = _mm_extract_epi16 (val1.x, msk0);
+  res[1] = _mm_extract_epi16 (val1.x, msk1);
+  res[2] = _mm_extract_epi16 (val1.x, msk2);
+  res[3] = _mm_extract_epi16 (val1.x, msk3);
+  res[4] = _mm_extract_epi16 (val1.x, msk4);
+  res[5] = _mm_extract_epi16 (val1.x, msk5);
+  res[6] = _mm_extract_epi16 (val1.x, msk6);
+  res[7] = _mm_extract_epi16 (val1.x, msk7);
+
+  masks[0] = msk0;
+  masks[1] = msk1;
+  masks[2] = msk2;
+  masks[3] = msk3;
+  masks[4] = msk4;
+  masks[5] = msk5;
+  masks[6] = msk6;
+  masks[7] = msk7;
+
+  for (i = 0; i < 8; i++)
+    if (res[i] != val1.s [masks[i]])
+      abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pinsrw.c b/gcc/testsuite/gcc.target/powerpc/sse2-pinsrw.c
new file mode 100644
index 00000000000..2fd5b0bd625
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pinsrw.c
@@ -0,0 +1,87 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pinsrw_1
+#endif
+
+#include <emmintrin.h>
+#include <string.h>
+
+#define msk0 0x00
+#define msk1 0x01
+#define msk2 0x02
+#define msk3 0x03
+#define msk4 0x04
+#define msk5 0x05
+#define msk6 0x06
+#define msk7 0x07
+
+static void
+TEST (void)
+{
+  union
+    {
+      __m128i x;
+      unsigned int i[4];
+      unsigned short s[8];
+    } res [8], val, tmp;
+  int masks[8];
+  unsigned short ins[4] = { 3, 4, 5, 6 };
+  int i;
+
+  val.i[0] = 0x35251505;
+  val.i[1] = 0x75655545;
+  val.i[2] = 0xB5A59585;
+  val.i[3] = 0xF5E5D5C5;
+
+  /* Check pinsrw imm8, r32, xmm.  */
+  res[0].x = _mm_insert_epi16 (val.x, ins[0], msk0);
+  res[1].x = _mm_insert_epi16 (val.x, ins[0], msk1);
+  res[2].x = _mm_insert_epi16 (val.x, ins[0], msk2);
+  res[3].x = _mm_insert_epi16 (val.x, ins[0], msk3);
+  res[4].x = _mm_insert_epi16 (val.x, ins[0], msk4);
+  res[5].x = _mm_insert_epi16 (val.x, ins[0], msk5);
+  res[6].x = _mm_insert_epi16 (val.x, ins[0], msk6);
+  res[7].x = _mm_insert_epi16 (val.x, ins[0], msk7);
+
+  masks[0] = msk0;
+  masks[1] = msk1;
+  masks[2] = msk2;
+  masks[3] = msk3;
+  masks[4] = msk4;
+  masks[5] = msk5;
+  masks[6] = msk6;
+  masks[7] = msk7;
+
+  for (i = 0; i < 8; i++)
+    {
+      tmp.x = val.x;
+      tmp.s[masks[i]] = ins[0];
+      if (memcmp (&tmp, &res[i], sizeof (tmp)))
+	abort ();
+    }
+    
+  /* Check pinsrw imm8, m16, xmm.  */
+  for (i = 0; i < 8; i++)
+    {
+      res[i].x = _mm_insert_epi16 (val.x, ins[i % 2], msk0);
+      masks[i] = msk0;
+    }
+
+  for (i = 0; i < 8; i++)
+    {
+      tmp.x = val.x;
+      tmp.s[masks[i]] = ins[i % 2];
+      if (memcmp (&tmp, &res[i], sizeof (tmp)))
+	abort ();
+    }
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmaddwd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmaddwd-1.c
new file mode 100644
index 00000000000..9595598ac63
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmaddwd-1.c
@@ -0,0 +1,43 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmaddwd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_madd_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w  s1, s2;
+  union128i_d u;
+  int e[4]; 
+  int i; 
+ 
+  s1.x = _mm_set_epi16 (2134,3343,1234,6354, 1, 3, 4, 5);
+  s2.x = _mm_set_epi16 (41124,234,2344,2354,9, -1, -8, -10);
+  u.x = test (s1.x, s2.x); 
+
+  for (i = 0; i < 4; i++)
+    e[i] = (s1.a[i*2] * s2.a[i*2])+(s1.a[(i*2) + 1] * s2.a[(i*2) + 1]);   
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmaxsw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmaxsw-1.c
new file mode 100644
index 00000000000..c1765aeecc6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmaxsw-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmaxsw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_max_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (1,2,3,4,5,6,7,8);
+  s2.x = _mm_set_epi16 (8,7,6,5,4,3,2,1);
+  u.x = test (s1.x, s2.x);
+
+  for (i=0; i<8; i++)
+    e[i] = s1.a[i]>s2.a[i]?s1.a[i]:s2.a[i]; 
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmaxub-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmaxub-1.c
new file mode 100644
index 00000000000..500d02e2e33
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmaxub-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmaxub_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_max_epu8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_ub u, s1, s2;
+  unsigned char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16);
+  s2.x = _mm_set_epi8 (16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1);
+  u.x = test (s1.x, s2.x);
+
+  for (i=0; i<16; i++)
+    e[i] = s1.a[i]>s2.a[i]?s1.a[i]:s2.a[i]; 
+
+  if (check_union128i_ub (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pminsw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pminsw-1.c
new file mode 100644
index 00000000000..5af2280a722
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pminsw-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pminsw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_min_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (1,2,3,4,5,6,7,8);
+  s2.x = _mm_set_epi16 (8,7,6,5,4,3,2,1);
+  u.x = test (s1.x, s2.x);
+
+  for (i=0; i<8; i++)
+    e[i] = s1.a[i]<s2.a[i]?s1.a[i]:s2.a[i]; 
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pminub-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pminub-1.c
new file mode 100644
index 00000000000..1eeca208d94
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pminub-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pminub_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_min_epu8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_ub u, s1, s2;
+  unsigned char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16);
+  s2.x = _mm_set_epi8 (16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1);
+  u.x = test (s1.x, s2.x);
+
+  for (i=0; i<16; i++)
+    e[i] = s1.a[i]<s2.a[i]?s1.a[i]:s2.a[i]; 
+
+  if (check_union128i_ub (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmovmskb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmovmskb-1.c
new file mode 100644
index 00000000000..37d72c231ea
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmovmskb-1.c
@@ -0,0 +1,57 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmovmskb_1
+#endif
+
+#include <emmintrin.h>
+
+#ifdef _ARCH_PWR8
+static int
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_movemask_epi8 (s1); 
+}
+#endif
+
+static void
+TEST (void)
+{
+#ifdef _ARCH_PWR8
+  union128i_b s1;
+  int i, u, e=0;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,-80,-40,-100,-15,98, 25, 98,7);
+
+  __asm("" : "+v"(s1.x));
+  u = test (s1.x); 
+  
+  for (i = 0; i < 16; i++)
+    if (s1.a[i] & (1<<7))
+      e = e | (1<<i);
+
+  if (checkVi (&u, &e, 1))
+#if DEBUG
+    {
+      printf ("sse2_test_pmovmskb_1; checkVi failed\n");
+      printf ("\t ([%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x], -> %x)\n",
+	      s1.a[0], s1.a[1], s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6],
+	      s1.a[7], s1.a[8], s1.a[9], s1.a[10], s1.a[11], s1.a[12], s1.a[13],
+	      s1.a[14], s1.a[15], u);
+      printf ("\t expect %x\n", e);
+    }
+#else
+    abort ();
+#endif
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmulhuw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmulhuw-1.c
new file mode 100644
index 00000000000..3635a867dbc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmulhuw-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmulhuw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_mulhi_epu16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_uw u, s1, s2;
+  unsigned short e[8];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (10,2067,3033,90,80,40,1000,15);
+  s2.x = _mm_set_epi16 (11, 9834, 7444, 10222, 34, 7833, 39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      tmp = s1.a[i] * s2.a[i];
+    
+      e[i] = (tmp & 0xffff0000)>>16;
+    }
+
+  if (check_union128i_uw (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmulhw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmulhw-1.c
new file mode 100644
index 00000000000..1255c03b051
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmulhw-1.c
@@ -0,0 +1,60 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmulhw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_mulhi_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (10,2067,-3033,90,80,40,-1000,15);
+  s2.x = _mm_set_epi16 (11, 9834, 7444, -10222, 34, -7833, 39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      tmp = s1.a[i] * s2.a[i];
+    
+      e[i] = (tmp & 0xffff0000)>>16;
+    }
+
+  if (check_union128i_w (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_pmulhw_1; check_union128i_w failed\n");
+      printf ("\tmulhi\t([%x,%x,%x,%x, %x,%x,%x,%x],\n", s1.a[0], s1.a[1],
+	      s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6], s1.a[7]);
+      printf ("\t\t [%x,%x,%x,%x, %x,%x,%x,%x])\n", s2.a[0], s2.a[1], s2.a[2],
+	      s2.a[3], s2.a[4], s2.a[5], s2.a[6], s2.a[7]);
+      printf ("\t ->\t [%x,%x,%x,%x, %x,%x,%x,%x]\n", u.a[0], u.a[1], u.a[2],
+	      u.a[3], u.a[4], u.a[5], u.a[6], u.a[7]);
+      printf ("\texpect\t [%x,%x,%x,%x, %x,%x,%x,%x]\n", e[0], e[1], e[2], e[3],
+	      e[4], e[5], e[6], e[7]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmullw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmullw-1.c
new file mode 100644
index 00000000000..3dca01ab3a8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmullw-1.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#define NO_WARN_X86_INTRINSICS 1
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmullw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_mullo_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (10,2067,-3033,90,80,40,-1000,15);
+  s2.x = _mm_set_epi16 (11, 9834, 7444, -10222, 34, -7833, 39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      tmp = s1.a[i] * s2.a[i];
+    
+      e[i] = tmp;
+    }
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pmuludq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pmuludq-1.c
new file mode 100644
index 00000000000..54fb06fc9fd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pmuludq-1.c
@@ -0,0 +1,53 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pmuludq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_mul_epu32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d s1, s2;
+  union128i_q u;
+  long long e[2];
+   
+  s1.x = _mm_set_epi32 (10,2067,3033,905);
+  s2.x = _mm_set_epi32 (11, 9834, 7444, 10222);
+  __asm("" : "+v"(s1.x), "+v"(s2.x));
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] * s2.a[0];
+  e[1] = s1.a[2] * s2.a[2];
+
+  if (check_union128i_q (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_pmuludq_1; check_union128i_q failed\n");
+      printf ("\t ([%x,%x,%x,%x], [%x,%x,%x,%x], -> [%llx, %llx])\n", s1.a[0],
+	      s1.a[1], s1.a[2], s1.a[3], s2.a[0], s2.a[1], s2.a[2], s2.a[3],
+	      u.a[0], u.a[1]);
+      printf ("\t expect [%llx, %llx]\n", e[0], e[1]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psadbw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psadbw-1.c
new file mode 100644
index 00000000000..0d6d6a340b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psadbw-1.c
@@ -0,0 +1,69 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psadbw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  return _mm_sad_epu8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_ub s1, s2;
+  union128i_w u;
+  short e[8] = { 0 };
+  unsigned char tmp[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16);
+  s2.x = _mm_set_epi8 (16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1);
+  u.x = test (s1.x, s2.x);
+
+  for (i = 0; i < 16; i++)
+    tmp [i] = __builtin_abs (s1.a[i] - s2.a[i]);
+
+  for (i = 0; i < 8; i++)
+    e[0] += tmp[i];
+
+  for (i = 8; i < 16; i++)
+    e[4] += tmp[i]; 
+
+
+  if (check_union128i_w (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_psadbw_1; check_union128i_w failed\n");
+      printf (
+	  "\tadds\t([%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x],\n",
+	      s1.a[0], s1.a[1], s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6],
+	      s1.a[7], s1.a[8], s1.a[9], s1.a[10], s1.a[11], s1.a[12], s1.a[13],
+	      s1.a[14], s1.a[15]);
+      printf ("\t\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x])\n",
+	      s2.a[0], s2.a[1], s2.a[2], s2.a[3], s2.a[4], s2.a[5], s2.a[6],
+	      s2.a[7], s2.a[8], s2.a[9], s2.a[10], s2.a[11], s2.a[12], s2.a[13],
+	      s2.a[14], s2.a[15]);
+      printf ("\t ->\t [%x,%x,%x,%x, %x,%x,%x,%x]\n", u.a[0], u.a[1], u.a[2],
+	      u.a[3], u.a[4], u.a[5], u.a[6], u.a[7]);
+      printf ("\texpect\t [%x,%x,%x,%x, %x,%x,%x,%x]\n", e[0], e[1], e[2], e[3],
+	      e[4], e[5], e[6], e[7]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pshufd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pshufd-1.c
new file mode 100644
index 00000000000..6c195fb4557
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pshufd-1.c
@@ -0,0 +1,51 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pshufd_1
+#endif
+
+#define N 0xec
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_shuffle_epi32 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s1;
+  int e[4] = { 0 };
+  int i;
+   
+  s1.x = _mm_set_epi32 (16,15,14,13);
+  u.x = test (s1.x);
+
+  for (i = 0; i < 4; i++)
+    e[i] = s1.a[((N & (0x3<<(2*i)))>>(2*i))];
+
+  if (check_union128i_d(u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_pshufd_1; check_union128i_d failed\n");
+      printf ("\t ([%x,%x,%x,%x]) -> [%x,%x,%x,%x]\n", s1.a[0], s1.a[1],
+	      s1.a[2], s1.a[3], u.a[0], u.a[1], u.a[2], u.a[3]);
+      printf ("\t expect [%x,%x,%x,%x]\n", e[0], e[1], e[2], e[3]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pshufhw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pshufhw-1.c
new file mode 100644
index 00000000000..a9230217830
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pshufhw-1.c
@@ -0,0 +1,65 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pshufhw_1
+#endif
+
+#define N 0xec
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_shufflehi_epi16 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q s1;
+  union128i_w u;
+  short  e[8] = { 0 };
+  int i;
+  int m1[4] = { 0x3, 0x3<<2, 0x3<<4, 0x3<<6 };
+  int m2[4];
+  
+  s1.x = _mm_set_epi64x (0xabcde,0xef58a234);
+  u.x = test (s1.x);
+
+  for (i = 0; i < 4; i++)
+    e[i] = (s1.a[0]>>(16 * i)) & 0xffff;
+
+  for (i = 0; i < 4; i++)
+    m2[i] = (N & m1[i])>>(2*i);
+
+  for (i = 0; i < 4; i++)
+    e[i+4] = (s1.a[1] >> (16 * m2[i])) & 0xffff;
+
+  if (check_union128i_w(u, e))
+#if DEBUG
+  {
+    union128i_w s;
+    s.x = s1.x;
+    printf ("sse2_test_pshufhw_1; check_union128i_w failed\n");
+    printf ("\t ([%hx,%hx,%hx,%hx, %hx,%hx,%hx,%hx])\n", s.a[0], s.a[1],
+	      s.a[2], s.a[3], s.a[4], s.a[5], s.a[6], s.a[7]);
+    printf ("\t\t -> [%hx,%hx,%hx,%hx, %hx,%hx,%hx,%hx]\n", u.a[0], u.a[1],
+	      u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7]);
+    printf ("\t expect [%hx,%hx,%hx,%hx, %hx,%hx,%hx,%hx]\n", e[0], e[1],
+	      e[2], e[3], e[4], e[5], e[6], e[7]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pshuflw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pshuflw-1.c
new file mode 100644
index 00000000000..e662cec6e19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pshuflw-1.c
@@ -0,0 +1,65 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pshuflw_1
+#endif
+
+#define N 0xec
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_shufflelo_epi16 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q s1;
+  union128i_w u;
+  short  e[8] = { 0 };
+  int i;
+  int m1[4] = { 0x3, 0x3<<2, 0x3<<4, 0x3<<6 };
+  int m2[4];
+  
+  s1.x = _mm_set_epi64x (0xabcde,0xef58a234);
+  u.x = test (s1.x);
+
+  for (i = 0; i < 4; i++)
+    e[i+4] = (s1.a[1]>>(16 * i)) & 0xffff;
+
+  for (i = 0; i < 4; i++)
+    m2[i] = (N & m1[i])>>(2*i);
+
+  for (i = 0; i < 4; i++)
+    e[i] = (s1.a[0] >> (16 * m2[i])) & 0xffff;
+
+  if (check_union128i_w(u, e))
+#if DEBUG
+  {
+    union128i_w s;
+    s.x = s1.x;
+    printf ("sse2_test_pshuflw_1; check_union128i_w failed\n");
+    printf ("\t ([%hx,%hx,%hx,%hx, %hx,%hx,%hx,%hx])\n", s.a[0], s.a[1],
+	      s.a[2], s.a[3], s.a[4], s.a[5], s.a[6], s.a[7]);
+    printf ("\t\t -> [%hx,%hx,%hx,%hx, %hx,%hx,%hx,%hx]\n", u.a[0], u.a[1],
+	      u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7]);
+    printf ("\t expect [%hx,%hx,%hx,%hx, %hx,%hx,%hx,%hx]\n", e[0], e[1],
+	      e[2], e[3], e[4], e[5], e[6], e[7]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pslld-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pslld-1.c
new file mode 100644
index 00000000000..90aac103fc2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pslld-1.c
@@ -0,0 +1,44 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pslld_1
+#endif
+
+#define N 0xf
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_slli_epi32 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s;
+  int e[4] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi32 (1, -2, 3, 4);
+
+  u.x = test (s.x);
+
+  if (N < 32)
+    for (i = 0; i < 4; i++)
+      e[i] = s.a[i] << N; 
+
+  if (check_union128i_d (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pslld-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-pslld-2.c
new file mode 100644
index 00000000000..2b4c5e797ff
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pslld-2.c
@@ -0,0 +1,55 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pslld_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i c)
+{
+  return _mm_sll_epi32 (s1, c); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s;
+  union128i_q c;
+  int e[4] = { 0 };
+  int i;
+ 
+  s.x = _mm_set_epi32 (2, -3, 0x7000, 0x9000);
+  c.x = _mm_set_epi64x (12, 23);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 32)
+    for (i = 0; i < 4; i++)
+      e[i] = s.a[i] << c.a[0]; 
+
+  if (check_union128i_d (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_pslld_2; check_union128i_d failed\n");
+      printf ("\tsll\t([%x,%x,%x,%x], [%llx,%llx]\n", s.a[0], s.a[1], s.a[2],
+	      s.a[3], c.a[0], c.a[1]);
+      printf ("\t ->\t [%x,%x,%x,%x]\n", u.a[0], u.a[1], u.a[2], u.a[3]);
+      printf ("\texpect\t [%x,%x,%x,%x]\n", e[0], e[1], e[2], e[3]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-pslldq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-pslldq-1.c
new file mode 100644
index 00000000000..f4bd6002fe5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-pslldq-1.c
@@ -0,0 +1,65 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_pslldq_1
+#endif
+
+#define N 0x5
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_slli_si128 (s1, N); 
+}
+
+static void 
+TEST (void)
+{
+  union128i_b u, s;
+  char src[16] =
+    { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 };
+  char e[16] =
+    { 0 };
+  int i;
+
+  s.x = _mm_loadu_si128 ((__m128i *) src);
+
+  u.x = test (s.x);
+
+  for (i = 0; i < 16 - N; i++)
+    e[i + N] = src[i];
+
+  if (check_union128i_b (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_pslldq_1; check_union128i_b failed\n");
+
+      printf ("\t s ([%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x])\n",
+	      s.a[0], s.a[1], s.a[2], s.a[3], s.a[4], s.a[5], s.a[6], s.a[7],
+	      s.a[8], s.a[9], s.a[10], s.a[11], s.a[12], s.a[13], s.a[14],
+	      s.a[15]);
+      printf (
+	  "\t u ->\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7],
+	  u.a[8], u.a[9], u.a[10], u.a[11], u.a[12], u.a[13], u.a[14], u.a[15]);
+      printf (
+	  "\t expect\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  e[0], e[1], e[2], e[3], e[4], e[5], e[6], e[7], e[8], e[9], e[10],
+	  e[11], e[12], e[13], e[14], e[15]);
+    }
+#else
+  abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psllq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psllq-1.c
new file mode 100644
index 00000000000..06904c50217
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psllq-1.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psllq_1
+#endif
+
+#define N 60
+
+#include <emmintrin.h>
+
+#ifdef _ARCH_PWR8
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_slli_epi64 (s1, N); 
+}
+#endif
+
+static void
+TEST (void)
+{
+#ifdef _ARCH_PWR8
+  union128i_q u, s;
+  long long e[2] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi64x (-1, 0xf);
+
+  u.x = test (s.x);
+
+  if (N < 64)
+    for (i = 0; i < 2; i++)
+      e[i] = s.a[i] << N; 
+
+  if (check_union128i_q (u, e))
+    abort (); 
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psllq-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-psllq-2.c
new file mode 100644
index 00000000000..5eb7bc39a60
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psllq-2.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psllq_2
+#endif
+
+#include <emmintrin.h>
+
+#ifdef _ARCH_PWR8
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i c)
+{
+  return _mm_sll_epi64 (s1, c); 
+}
+#endif
+
+static void
+TEST (void)
+{
+#ifdef _ARCH_PWR8
+  union128i_q u, s, c;
+  long long e[2] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi64x (-1, 0xf);
+  c.x = _mm_set_epi64x (60,50);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 64)
+    for (i = 0; i < 2; i++)
+      e[i] = s.a[i] << c.a[0]; 
+
+  if (check_union128i_q (u, e))
+    abort (); 
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psllw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psllw-1.c
new file mode 100644
index 00000000000..f744bb244cb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psllw-1.c
@@ -0,0 +1,44 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psllw_1
+#endif
+
+#define N 0xb
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_slli_epi16 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s;
+  short e[8] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi16 (1, 2, 3, 4, 5, 6, 0x7000, 0x9000);
+
+  u.x = test (s.x);
+
+  if (N < 16)
+    for (i = 0; i < 8; i++)
+      e[i] = s.a[i] << N; 
+
+  if (check_union128i_w (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psllw-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-psllw-2.c
new file mode 100644
index 00000000000..1335e2bb249
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psllw-2.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psllw_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i c)
+{
+  return _mm_sll_epi16 (s1, c); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s;
+  union128i_q c;
+  short e[8] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi16 (1, 2, 3, 4, 5, 6, 0x7000, 0x9000);
+  c.x = _mm_set_epi64x (12, 13);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 16)
+    for (i = 0; i < 8; i++)
+      e[i] = s.a[i] << c.a[0]; 
+
+  if (check_union128i_w (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrad-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrad-1.c
new file mode 100644
index 00000000000..03c40a11d07
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrad-1.c
@@ -0,0 +1,44 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrad_1
+#endif
+
+#define N 0xf
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_srai_epi32 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s;
+  int e[4] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi32 (1, -2, 3, 4);
+
+  u.x = test (s.x);
+
+  if (N < 32)
+    for (i = 0; i < 4; i++)
+      e[i] = s.a[i] >> N; 
+
+  if (check_union128i_d (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrad-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrad-2.c
new file mode 100644
index 00000000000..387f383f3db
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrad-2.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrad_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i count)
+{
+  return _mm_sra_epi32 (s1, count); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s;
+  union128i_q c;
+  int e[4] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi32 (1, -2, 3, 4);
+  c.x = _mm_set_epi64x (16, 29);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 32)
+    for (i = 0; i < 4; i++)
+      e[i] = s.a[i] >> c.a[0]; 
+
+  if (check_union128i_d (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psraw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psraw-1.c
new file mode 100644
index 00000000000..23a423a4762
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psraw-1.c
@@ -0,0 +1,44 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psraw_1
+#endif
+
+#define N 0xb
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_srai_epi16 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s;
+  short e[8] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi16 (1, -2, 3, 4, -5, 6, 0x7000, 0x9000);
+
+  u.x = test (s.x);
+
+  if (N < 16)
+    for (i = 0; i < 8; i++)
+      e[i] = s.a[i] >> N; 
+
+  if (check_union128i_w (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psraw-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-psraw-2.c
new file mode 100644
index 00000000000..b41a6d3157a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psraw-2.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psraw_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i c)
+{
+  return _mm_sra_epi16 (s1, c); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s;
+  union128i_q c;
+  short e[8] = {0};
+  int i;
+ 
+  s.x = _mm_set_epi16 (1, -2, 3, 4, 5, 6, -0x7000, 0x9000);
+  c.x = _mm_set_epi64x (12, 13);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 16)
+    for (i = 0; i < 8; i++)
+      e[i] = s.a[i] >> c.a[0]; 
+
+  if (check_union128i_w (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrld-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrld-1.c
new file mode 100644
index 00000000000..e96cf0a3ca8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrld-1.c
@@ -0,0 +1,57 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrld_1
+#endif
+
+#define N 0xf
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_srli_epi32 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s;
+  int e[4] = { 0 };
+  unsigned int tmp;
+  int i;
+ 
+  s.x = _mm_set_epi32 (1, -2, 3, 4);
+
+  u.x = test (s.x);
+
+  if (N < 32)
+    for (i = 0; i < 4; i++)
+      {
+        tmp  = s.a[i];
+        e[i] = tmp >> N; 
+      }
+
+  if (check_union128i_d (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_psrld_1; check_union128i_d failed\n");
+      printf ("\tsrl\t([%x,%x,%x,%x],%d\n", s.a[0], s.a[1], s.a[2], s.a[3], N);
+      printf ("\t ->\t [%x,%x,%x,%x]\n", u.a[0], u.a[1], u.a[2], u.a[3]);
+      printf ("\texpect\t [%x,%x,%x,%x]\n", e[0], e[1], e[2], e[3]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrld-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrld-2.c
new file mode 100644
index 00000000000..6192e2a2d59
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrld-2.c
@@ -0,0 +1,59 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrld_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i c)
+{
+  return _mm_srl_epi32 (s1, c); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s;
+  union128i_q c;
+  int e[4] = { 0 };
+  unsigned int tmp;
+  int i;
+ 
+  s.x = _mm_set_epi32 (2, -3, 0x7000, 0x9000);
+  c.x = _mm_set_epi64x (12, 23);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 32)
+    for (i = 0; i < 4; i++)
+      {
+        tmp = s.a[i];
+        e[i] = tmp >> c.a[0];
+      } 
+
+  if (check_union128i_d (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_psrld_2; check_union128i_d failed\n");
+      printf ("\tsrld\t([%x,%x,%x,%x], [%llx,%llx]\n", s.a[0], s.a[1], s.a[2],
+	      s.a[3], c.a[0], c.a[1]);
+      printf ("\t ->\t [%x,%x,%x,%x]\n", u.a[0], u.a[1], u.a[2], u.a[3]);
+      printf ("\texpect\t [%x,%x,%x,%x]\n", e[0], e[1], e[2], e[3]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrldq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrldq-1.c
new file mode 100644
index 00000000000..5b74cae7f90
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrldq-1.c
@@ -0,0 +1,62 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrldq_1
+#endif
+
+#define N 0x5
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_srli_si128 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s;
+  char src[16] = { 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 };
+  char e[16] = { 0 };
+  int i;
+   
+  s.x = _mm_loadu_si128 ((__m128i *)src);
+
+  u.x = test (s.x);
+
+  for (i = 0; i < 16-N; i++)
+    e[i] = src[i+N];
+
+  if (check_union128i_b (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_psrldq_1; check_union128i_b failed\n");
+      printf ("\tsrl\t([%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x],\n",
+	      s.a[0], s.a[1], s.a[2], s.a[3], s.a[4], s.a[5], s.a[6], s.a[7],
+	      s.a[8], s.a[9], s.a[10], s.a[11], s.a[12], s.a[13], s.a[14],
+	      s.a[15]);
+      printf ("\t ->\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	      u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7],
+	      u.a[8], u.a[9], u.a[10], u.a[11], u.a[12], u.a[13], u.a[14],
+	      u.a[15]);
+      printf (
+	  "\texpect\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  e[0], e[1], e[2], e[3], e[4], e[5], e[6], e[7], e[8], e[9], e[10],
+	  e[11], e[12], e[13], e[14], e[15]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrlq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrlq-1.c
new file mode 100644
index 00000000000..9b13f0be7fe
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrlq-1.c
@@ -0,0 +1,51 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrlq_1
+#endif
+
+#define N 60
+
+#include <emmintrin.h>
+
+#ifdef _ARCH_PWR8
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_srli_epi64 (s1, N); 
+}
+#endif
+
+static void
+TEST (void)
+{
+#ifdef _ARCH_PWR8
+  union128i_q u, s;
+  long long e[2] = {0};
+  unsigned long long tmp;
+  int i;
+ 
+  s.x = _mm_set_epi64x (-1, 0xf);
+
+  u.x = test (s.x);
+
+  if (N < 64)
+    for (i = 0; i < 2; i++) {
+      tmp = s.a[i]; 
+      e[i] = tmp >> N;
+    }
+
+  if (check_union128i_q (u, e))
+    abort (); 
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrlq-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrlq-2.c
new file mode 100644
index 00000000000..168c77f99ec
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrlq-2.c
@@ -0,0 +1,51 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrlq_2
+#endif
+
+#include <emmintrin.h>
+
+#ifdef _ARCH_PWR8
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i c)
+{
+  return _mm_srl_epi64 (s1, c); 
+}
+#endif
+
+static void
+TEST (void)
+{
+#ifdef _ARCH_PWR8
+  union128i_q u, s, c;
+  long long e[2] = {0};
+  unsigned long long tmp;
+  int i;
+ 
+  s.x = _mm_set_epi64x (-1, 0xf);
+  c.x = _mm_set_epi64x (60,50);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 64)
+    for (i = 0; i < 2; i++){
+      tmp = s.a[i];
+      e[i] =tmp >> c.a[0];
+    } 
+
+  if (check_union128i_q (u, e))
+    abort (); 
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrlw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrlw-1.c
new file mode 100644
index 00000000000..6f0a856fc7a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrlw-1.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrlw_1
+#endif
+
+#define N 0xb
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1)
+{
+  return _mm_srli_epi16 (s1, N); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s;
+  short e[8] = {0};
+  unsigned short tmp;
+  int i;
+ 
+  s.x = _mm_set_epi16 (1, -2, 3, -4, 5, 6, 0x7000, 0x9000);
+
+  u.x = test (s.x);
+
+  if (N < 16)
+    for (i = 0; i < 8; i++)
+      {
+        tmp = s.a[i];
+        e[i] = tmp >> N;
+      }
+
+  if (check_union128i_w (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psrlw-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-psrlw-2.c
new file mode 100644
index 00000000000..9457b49a8cb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psrlw-2.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psrlw_2
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i c)
+{
+  return _mm_srl_epi16 (s1, c); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s;
+  union128i_q c;
+  short e[8] = {0};
+  unsigned short tmp;
+  int i;
+ 
+  s.x = _mm_set_epi16 (1, -2, 3, 4, 5, 6, -0x7000, 0x9000);
+  c.x = _mm_set_epi64x (12, 13);
+
+  __asm("" : "+v"(s.x), "+v"(c.x));
+  u.x = test (s.x, c.x);
+
+  if (c.a[0] < 16)
+    for (i = 0; i < 8; i++)
+      {
+        tmp = s.a[i];
+        e[i] = tmp >> c.a[0];
+      }
+
+  if (check_union128i_w (u, e))
+    abort (); 
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubb-1.c
new file mode 100644
index 00000000000..a0d99ddada1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubb-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_sub_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,-80,-40,-100,-15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, -100, -34, -78, -39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+     e[i] = s1.a[i] - s2.a[i];
+
+  if (check_union128i_b (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubd-1.c
new file mode 100644
index 00000000000..624ae2de5be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubd-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_sub_epi32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s1, s2;
+  int e[4];
+  int i;
+   
+  s1.x = _mm_set_epi32 (30,90,-80,-40);
+  s2.x = _mm_set_epi32 (76, -100, -34, -78);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 4; i++)
+     e[i] = s1.a[i] - s2.a[i];
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubq-1.c
new file mode 100644
index 00000000000..426ccb7dcd3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubq-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_sub_epi64 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q u, s1, s2;
+  long long e[2];
+  int i;
+   
+  s1.x = _mm_set_epi64x (90,-80);
+  s2.x = _mm_set_epi64x (76, -100);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 2; i++)
+     e[i] = s1.a[i] - s2.a[i];
+
+  if (check_union128i_q (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubsb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubsb-1.c
new file mode 100644
index 00000000000..be02da5a34b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubsb-1.c
@@ -0,0 +1,51 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubsb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_subs_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,-80,-40,-100,-15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, -100, -34, -78, -39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+    {
+      tmp = (signed char)s1.a[i] - (signed char)s2.a[i];
+
+      if (tmp > 127)
+        tmp = 127;
+      if (tmp < -128)
+        tmp = -128;
+
+      e[i] = tmp;
+    }
+
+  if (check_union128i_b (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubsw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubsw-1.c
new file mode 100644
index 00000000000..afed3c86914
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubsw-1.c
@@ -0,0 +1,51 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubsw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_subs_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,-80,-40,-100,-15);
+  s2.x = _mm_set_epi16 (11, 98, 76, -100, -34, -78, -39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      tmp = s1.a[i] - s2.a[i];
+      
+      if (tmp > 32767)
+        tmp = 32767;
+      if (tmp < -32768)
+        tmp = -32768;
+      
+      e[i] = tmp;
+    }
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubusb-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubusb-1.c
new file mode 100644
index 00000000000..e5f128b6c38
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubusb-1.c
@@ -0,0 +1,74 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubusb_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_subs_epu8 (s1, s2);
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16] = { 0 };
+  int i, tmp;
+   
+  s1.x = _mm_set_epi8 (30, 2, 3, 4, 10, 20, 30, 90, 80, 40, 100, 15, 98, 25, 98, 7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, 100, 34, 78, 39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 16; i++)
+    {
+      tmp = (unsigned char)s1.a[i] - (unsigned char)s2.a[i];
+
+      if (tmp > 255)
+        tmp = -1;
+      if (tmp < 0)
+        tmp = 0; 
+
+      e[i] = tmp; 
+    }
+
+  if (check_union128i_b (u, e))
+#if DEBUG
+    {
+      printf ("sse2_test_psubusb_1; check_union128i_b failed\n");
+      printf (
+	  "\tadds\t([%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x],\n",
+	      s1.a[0], s1.a[1], s1.a[2], s1.a[3], s1.a[4], s1.a[5], s1.a[6],
+	      s1.a[7], s1.a[8], s1.a[9], s1.a[10], s1.a[11], s1.a[12], s1.a[13],
+	      s1.a[14], s1.a[15]);
+      printf ("\t\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x])\n",
+	      s2.a[0], s2.a[1], s2.a[2], s2.a[3], s2.a[4], s2.a[5], s2.a[6],
+	      s2.a[7], s2.a[8], s2.a[9], s2.a[10], s2.a[11], s2.a[12], s2.a[13],
+	      s2.a[14], s2.a[15]);
+      printf ("\t ->\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	      u.a[0], u.a[1], u.a[2], u.a[3], u.a[4], u.a[5], u.a[6], u.a[7],
+	      u.a[8], u.a[9], u.a[10], u.a[11], u.a[12], u.a[13], u.a[14],
+	      u.a[15]);
+      printf (
+	  "\texpect\t [%x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x, %x,%x,%x,%x]\n",
+	  e[0], e[1], e[2], e[3], e[4], e[5], e[6], e[7], e[8], e[9], e[10],
+	  e[11], e[12], e[13], e[14], e[15]);
+    }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubusw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubusw-1.c
new file mode 100644
index 00000000000..11ddca627e7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubusw-1.c
@@ -0,0 +1,52 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubusw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_subs_epu16 (s1, s2);
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i, tmp;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,80,40,100,15);
+  s2.x = _mm_set_epi16 (11, 98, 76, 100, 34, 78, 39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      tmp = (unsigned short)s1.a[i] - (unsigned short)s2.a[i];
+      
+      if (tmp > 65535)
+        tmp = -1;
+
+      if (tmp < 0)
+        tmp = 0;
+      
+      e[i] = tmp;
+    }
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-psubw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-psubw-1.c
new file mode 100644
index 00000000000..04570a2bc2a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-psubw-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_psubw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_sub_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,-80,-40,-100,-15);
+  s2.x = _mm_set_epi16 (11, 98, 76, -100, -34, -78, -39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+     e[i] = s1.a[i] - s2.a[i];
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpckhbw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhbw-1.c
new file mode 100644
index 00000000000..b666c58e276
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhbw-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpckhbw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpackhi_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,-80,-40,-100,-15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, -100, -34, -78, -39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      e[2*i] = s1.a[8+i];
+      e[2*i + 1] = s2.a[8+i];
+    }
+
+  if (check_union128i_b (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpckhdq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhdq-1.c
new file mode 100644
index 00000000000..c4866198110
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhdq-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpckhdq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpackhi_epi32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s1, s2;
+  int e[4];
+  int i;
+   
+  s1.x = _mm_set_epi32 (10,20,-80,-40);
+  s2.x = _mm_set_epi32 (11, -34, -78, -39);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 2; i++)
+    {
+      e[2*i] = s1.a[2+i];
+      e[2*i+1] = s2.a[2+i];
+    }
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpckhqdq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhqdq-1.c
new file mode 100644
index 00000000000..849f23f5433
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhqdq-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpckhqdq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpackhi_epi64 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q u, s1, s2;
+  long long  e[2];
+   
+  s1.x = _mm_set_epi64x (10,-40);
+  s2.x = _mm_set_epi64x (1134, -7839);
+  u.x = test (s1.x, s2.x); 
+  
+  e[0] = s1.a[1];
+  e[1] = s2.a[1];
+
+  if (check_union128i_q (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpckhwd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhwd-1.c
new file mode 100644
index 00000000000..7077ecc5a2f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpckhwd-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpckhwd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpackhi_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,-80,-40,-100,-15);
+  s2.x = _mm_set_epi16 (11, 98, 76, -100, -34, -78, -39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 4; i++)
+    {
+      e[2*i] = s1.a[4+i];
+      e[2*i+1] = s2.a[4+i];
+    }
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpcklbw-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpcklbw-1.c
new file mode 100644
index 00000000000..e1ee1aee35d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpcklbw-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpcklbw_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpacklo_epi8 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_b u, s1, s2;
+  char e[16];
+  int i;
+   
+  s1.x = _mm_set_epi8 (1,2,3,4,10,20,30,90,-80,-40,-100,-15,98, 25, 98,7);
+  s2.x = _mm_set_epi8 (88, 44, 33, 22, 11, 98, 76, -100, -34, -78, -39, 6, 3, 4, 5, 119);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 8; i++)
+    {
+      e[2*i] = s1.a[i];
+      e[2*i + 1] = s2.a[i];
+    }
+
+  if (check_union128i_b (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpckldq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpckldq-1.c
new file mode 100644
index 00000000000..a47f72dda80
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpckldq-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpckldq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpacklo_epi32 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_d u, s1, s2;
+  int e[4];
+  int i;
+   
+  s1.x = _mm_set_epi32 (10,20,-80,-40);
+  s2.x = _mm_set_epi32 (11, -34, -78, -39);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 2; i++)
+    {
+      e[2*i] = s1.a[i];
+      e[2*i+1] = s2.a[i];
+    }
+
+  if (check_union128i_d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpcklqdq-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpcklqdq-1.c
new file mode 100644
index 00000000000..a45f636bdf5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpcklqdq-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpcklqdq_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpacklo_epi64 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_q u, s1, s2;
+  long long  e[2];
+   
+  s1.x = _mm_set_epi64x (10,-40);
+  s2.x = _mm_set_epi64x (1134, -7839);
+  u.x = test (s1.x, s2.x); 
+  
+  e[0] = s1.a[0];
+  e[1] = s2.a[0];
+
+  if (check_union128i_q (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-punpcklwd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-punpcklwd-1.c
new file mode 100644
index 00000000000..5afd9799df7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-punpcklwd-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_punpcklwd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128i
+__attribute__((noinline, unused))
+test (__m128i s1, __m128i s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpacklo_epi16 (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128i_w u, s1, s2;
+  short e[8];
+  int i;
+   
+  s1.x = _mm_set_epi16 (10,20,30,90,-80,-40,-100,-15);
+  s2.x = _mm_set_epi16 (11, 98, 76, -100, -34, -78, -39, 14);
+  u.x = test (s1.x, s2.x); 
+   
+  for (i = 0; i < 4; i++)
+    {
+      e[2*i] = s1.a[i];
+      e[2*i+1] = s2.a[i];
+    }
+
+  if (check_union128i_w (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-shufpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-shufpd-1.c
new file mode 100644
index 00000000000..e81c818fa2d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-shufpd-1.c
@@ -0,0 +1,42 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_shufpd_1
+#endif
+
+#define N 0xab
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_shuffle_pd (s1, s2, N); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2] = {0.0};
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (453.345635,54646.464356);
+  u.x = test (s1.x, s2.x);
+
+  e[0] = (N & (1 << 0)) ? s1.a[1] : s1.a[0];
+  e[1] = (N & (1 << 1)) ? s2.a[1] : s2.a[0];
+
+  if (check_union128d(u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-sqrtpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-sqrtpd-1.c
new file mode 100644
index 00000000000..fa0b1fee8be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-sqrtpd-1.c
@@ -0,0 +1,54 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_sqrt_pd_1
+#endif
+
+#include <emmintrin.h>
+#include <math.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1)
+{
+  return _mm_sqrt_pd (s1); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1;
+  __m128d bogus = { 123.0, 456.0 };
+  double e[2];
+  int i;
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  u.x = test (s1.x); 
+
+  for (i = 0; i < 2; i++)
+    {
+      __m128d tmp = _mm_load_sd (&s1.a[i]);
+      tmp = _mm_sqrt_sd (bogus, tmp);
+      _mm_store_sd (&e[i], tmp);
+    }
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_sqrt_pd_1; check_union128d failed\n");
+      printf ("\t [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-subpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-subpd-1.c
new file mode 100644
index 00000000000..6428dc92e31
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-subpd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_subpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_sub_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] - s2.a[0];
+  e[1] = s1.a[1] - s2.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-subsd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-subsd-1.c
new file mode 100644
index 00000000000..c5afa3ab02c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-subsd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_subsd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_sub_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0] - s2.a[0];
+  e[1] = s1.a[1];
+
+  if (check_union128d (u, e))
+#if DEBUG
+  {
+      printf ("sse2_test_subsd_1; check_union128d failed\n");
+      printf ("\t [%f,%f] - [%f,%f] -> [%f,%f]\n", s1.a[0], s1.a[1], s2.a[0],
+	      s2.a[1], u.a[0], u.a[1]);
+      printf ("\t expect [%f,%f]\n", e[0], e[1]);
+  }
+#else
+    abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-1.c
new file mode 100644
index 00000000000..a68ed519ca8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-1.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_ucomisd_1
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_ucomieq_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,2344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] == s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-2.c b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-2.c
new file mode 100644
index 00000000000..b3f00c82632
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-2.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_ucomisd_2
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_ucomilt_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,12344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] < s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-3.c b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-3.c
new file mode 100644
index 00000000000..2bfcf84357d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-3.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_ucomisd_3
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_ucomile_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1] = {0};
+  int e[1] = {0};
+ 
+  s1.x = _mm_set_pd (2134.3343,12344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] <= s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-4.c b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-4.c
new file mode 100644
index 00000000000..42243b29c5f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-4.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_ucomisd_4
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_ucomigt_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,12344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] > s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-5.c b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-5.c
new file mode 100644
index 00000000000..1fc2a2b3c08
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-5.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_ucomisd_5
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_ucomige_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,12344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] >= s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-6.c b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-6.c
new file mode 100644
index 00000000000..5ce8d453527
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-ucomisd-6.c
@@ -0,0 +1,39 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_ucomisd_6
+#endif
+
+#include <emmintrin.h>
+
+static int 
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_ucomineq_sd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d s1, s2;
+  int d[1];
+  int e[1];
+ 
+  s1.x = _mm_set_pd (2134.3343,12344.2354);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  d[0] = test (s1.x, s2.x); 
+  e[0] = s1.a[0] != s2.a[0];
+
+  if (checkVi (d, e, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-unpckhpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-unpckhpd-1.c
new file mode 100644
index 00000000000..f1547861c21
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-unpckhpd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_unpckhpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpackhi_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[1];
+  e[1] = s2.a[1];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-unpcklpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-unpcklpd-1.c
new file mode 100644
index 00000000000..5c1ad1e18ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-unpcklpd-1.c
@@ -0,0 +1,41 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_unpcklpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  __asm("" : "+v"(s1), "+v"(s2));
+  return _mm_unpacklo_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union128d u, s1, s2;
+  double e[2];
+   
+  s1.x = _mm_set_pd (2134.3343,1234.635654);
+  s2.x = _mm_set_pd (41124.234,2344.2354);
+  u.x = test (s1.x, s2.x); 
+   
+  e[0] = s1.a[0];
+  e[1] = s2.a[0];
+
+  if (check_union128d (u, e))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/sse2-xorpd-1.c b/gcc/testsuite/gcc.target/powerpc/sse2-xorpd-1.c
new file mode 100644
index 00000000000..d1c04bfc8f8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/sse2-xorpd-1.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -mpower8-vector -Wno-psabi" } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-require-effective-target p8vector_hw } */
+
+#ifndef CHECK_H
+#define CHECK_H "sse2-check.h"
+#endif
+
+#include CHECK_H
+
+#ifndef TEST
+#define TEST sse2_test_xorpd_1
+#endif
+
+#include <emmintrin.h>
+
+static __m128d
+__attribute__((noinline, unused))
+test (__m128d s1, __m128d s2)
+{
+  return _mm_xor_pd (s1, s2); 
+}
+
+static void
+TEST (void)
+{
+  union
+    {
+      double d[2];
+      long long l[2];
+    }source1, source2, e;
+
+  union128d u, s1, s2;
+  int i; 
+   
+  s1.x = _mm_set_pd (11.1321456, 2.287332);
+  s2.x = _mm_set_pd (3.37768, 4.43222234);
+
+  _mm_storeu_pd (source1.d, s1.x);
+  _mm_storeu_pd (source2.d, s2.x);
+
+  u.x = test (s1.x, s2.x); 
+ 
+  for (i = 0; i < 2; i++)
+    e.l[i] = source1.l[i] ^ source2.l[i];
+
+  if (check_union128d (u, e.d))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/zvector/vec-cmp-2.c b/gcc/testsuite/gcc.target/s390/zvector/vec-cmp-2.c
index 0711f9c0531..1e63defa063 100644
--- a/gcc/testsuite/gcc.target/s390/zvector/vec-cmp-2.c
+++ b/gcc/testsuite/gcc.target/s390/zvector/vec-cmp-2.c
@@ -7,197 +7,197 @@
 
 #include <vecintrin.h>
 
-extern void foo (void);
+int g = 1;
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_eq_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_all_eq (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_eq_double:\n\tvfcedbs\t%v\[0-9\]*,%v24,%v26\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_ne_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_all_ne (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_ne_double:\n\tvfcedbs\t%v\[0-9\]*,%v24,%v26\n\tjle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_gt_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_all_gt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_gt_double:\n\tvfchdbs\t%v\[0-9\]*,%v24,%v26\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_lt_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_all_lt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_lt_double:\n\tvfchdbs\t%v\[0-9\]*,%v26,%v24\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_ge_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_all_ge (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_ge_double:\n\tvfchedbs\t%v\[0-9\]*,%v24,%v26\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_le_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_all_le (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_le_double:\n\tvfchedbs\t%v\[0-9\]*,%v26,%v24\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_eq_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_any_eq (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_eq_double:\n\tvfcedbs\t%v\[0-9\]*,%v24,%v26\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_ne_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_any_ne (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_ne_double:\n\tvfcedbs\t%v\[0-9\]*,%v24,%v26\n\tje 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_gt_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_any_gt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_gt_double:\n\tvfchdbs\t%v\[0-9\]*,%v24,%v26\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_lt_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_any_lt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_lt_double:\n\tvfchdbs\t%v\[0-9\]*,%v26,%v24\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_ge_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_any_ge (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_ge_double:\n\tvfchedbs\t%v\[0-9\]*,%v24,%v26\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_le_double (vector double a, vector double b)
 {
   if (__builtin_expect (vec_any_le (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_le_double:\n\tvfchedbs\t%v\[0-9\]*,%v26,%v24\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_eq_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_all_eq (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_eq_int:\n\tvceqfs\t%v\[0-9\]*,%v24,%v26\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_ne_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_all_ne (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_ne_int:\n\tvceqfs\t%v\[0-9\]*,%v24,%v26\n\tjle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_gt_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_all_gt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_gt_int:\n\tvchfs\t%v\[0-9\]*,%v24,%v26\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_lt_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_all_lt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_lt_int:\n\tvchfs\t%v\[0-9\]*,%v26,%v24\n\tjne 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_ge_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_all_ge (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_ge_int:\n\tvchfs\t%v\[0-9\]*,%v26,%v24\n\tjle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 all_le_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_all_le (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times all_le_int:\n\tvchfs\t%v\[0-9\]*,%v24,%v26\n\tjle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_eq_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_any_eq (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_eq_int:\n\tvceqfs\t%v\[0-9\]*,%v24,%v26\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_ne_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_any_ne (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_ne_int:\n\tvceqfs\t%v\[0-9\]*,%v24,%v26\n\tje 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_gt_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_any_gt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_gt_int:\n\tvchfs\t%v\[0-9\]*,%v24,%v26\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_lt_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_any_lt (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_lt_int:\n\tvchfs\t%v\[0-9\]*,%v26,%v24\n\tjnle 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_ge_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_any_ge (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_ge_int:\n\tvchfs\t%v\[0-9\]*,%v26,%v24\n\tje 1 } } */
 
-int __attribute__((noinline,noclone))
+void __attribute__((noinline,noclone))
 any_le_int (vector int a, vector int b)
 {
   if (__builtin_expect (vec_any_le (a, b), 1))
-    foo ();
+    g = 2;
 }
 /* { dg-final { scan-assembler-times any_le_int:\n\tvchfs\t%v\[0-9\]*,%v24,%v26\n\tje 1 } } */
 
diff --git a/gcc/testsuite/gfortran.dg/allocate_error_7.f90 b/gcc/testsuite/gfortran.dg/allocate_error_7.f90
new file mode 100644
index 00000000000..f1c8bc3db64
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/allocate_error_7.f90
@@ -0,0 +1,12 @@
+! { dg-do compile }
+!
+! Code contributed by Gerhard Steinmetz
+!
+program pr82620
+   type t(a)
+      integer, len :: a
+   end type
+   type(t(:)), allocatable :: x, y
+   allocate(t(4) :: x)
+   allocate)t(7) :: y)     ! { dg-error "Syntax error in ALLOCATE" }
+end program pr82620
diff --git a/gcc/testsuite/gfortran.dg/array_constructor_51.f90 b/gcc/testsuite/gfortran.dg/array_constructor_51.f90
new file mode 100644
index 00000000000..4c3cdf71fcf
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/array_constructor_51.f90
@@ -0,0 +1,20 @@
+! { dg-do compile }
+! { dg-additional-options "-ffrontend-optimize -fdump-tree-original" }
+! PR 82567 - long compile times caused by large constant constructors
+! multiplied by variables
+
+  SUBROUTINE sub()
+  IMPLICIT NONE
+  
+  INTEGER, PARAMETER :: n = 1000
+  REAL, ALLOCATABLE :: x(:)
+  REAL :: xc, h
+  INTEGER :: i
+ 
+  ALLOCATE( x(n) )
+  xc = 100.
+  h = xc/n
+  x = h*[(i,i=1,n)]
+  
+end
+! { dg-final { scan-tree-dump-times "__var" 0 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/assumed_size_2.f90 b/gcc/testsuite/gfortran.dg/assumed_size_2.f90
new file mode 100644
index 00000000000..e9a1185b527
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/assumed_size_2.f90
@@ -0,0 +1,4 @@
+! { dg-do compile }
+subroutine foo(a)
+  dimension  a(*,*) ! { dg-error "Bad specification for assumed size array" }
+end
diff --git a/gcc/testsuite/gfortran.dg/class_63.f90 b/gcc/testsuite/gfortran.dg/class_63.f90
new file mode 100644
index 00000000000..cf99bcf9cb2
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/class_63.f90
@@ -0,0 +1,80 @@
+! { dg-do run }
+!
+! Tests the fix for PR81758, in which the vpointer for 'ptr' in
+! function 'pointer_value' would be set to the vtable of the component
+! 'container' rather than that of the component 'vec_elem'. In this test
+! case it is ensured that there is a single typebound procedure for both
+! types, so that different values are returned. In the original problem
+! completely different procedures were involved so that a segfault resulted.
+!
+! Reduced from the original code of Dimitry Liakh  <liakhdi@ornl.gov> by
+!                                   Paul Thomas  <pault@gcc.gnu.org>
+!
+module types
+  type, public:: gfc_container_t
+  contains
+    procedure, public:: get_value => ContTypeGetValue
+  end type gfc_container_t
+
+  !Element of a container:
+  type, public:: gfc_cont_elem_t
+    integer :: value_p
+  contains
+    procedure, public:: get_value => ContElemGetValue
+  end type gfc_cont_elem_t
+
+  !Vector element:
+  type, extends(gfc_cont_elem_t), public:: vector_elem_t
+  end type vector_elem_t
+
+  !Vector:
+  type, extends(gfc_container_t), public:: vector_t
+    type(vector_elem_t), allocatable, private :: vec_elem
+  end type vector_t
+
+  type, public :: vector_iter_t
+    class(vector_t), pointer, private :: container => NULL()
+  contains
+    procedure, public:: get_vector_value => vector_Value
+    procedure, public:: get_pointer_value => pointer_value
+  end type
+
+contains
+  integer function ContElemGetValue (this)
+    class(gfc_cont_elem_t) :: this
+    ContElemGetValue = this%value_p
+  end function
+
+  integer function ContTypeGetValue (this)
+    class(gfc_container_t) :: this
+    ContTypeGetValue = 0
+  end function
+
+  integer function vector_Value (this)
+    class(vector_iter_t) :: this
+    vector_value = this%container%vec_elem%get_value()
+  end function
+
+  integer function pointer_value (this)
+    class(vector_iter_t), target :: this
+    class(gfc_cont_elem_t), pointer :: ptr
+    ptr => this%container%vec_elem
+    pointer_value = ptr%get_value()
+  end function
+
+  subroutine factory (arg)
+    class (vector_iter_t), pointer :: arg
+    allocate (vector_iter_t :: arg)
+    allocate (vector_t :: arg%container)
+    allocate (arg%container%vec_elem)
+    arg%container%vec_elem%value_p = 99
+  end subroutine
+end module
+
+  use types
+  class (vector_iter_t), pointer :: x
+
+  call factory (x)
+  if (x%get_vector_value() .ne. 99) call abort
+  if (x%get_pointer_value() .ne. 99) call abort
+end
diff --git a/gcc/testsuite/gfortran.dg/class_64.f90 b/gcc/testsuite/gfortran.dg/class_64.f90
new file mode 100644
index 00000000000..059ebaa8a01
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/class_64.f90
@@ -0,0 +1,38 @@
+! { dg-do compile }
+! { dg-options "-fdump-tree-original" }
+!
+! Test the fix for PR80850 in which the _len field was not being
+! set for 'arg' in the call to 'foo'.
+!
+  type :: mytype
+    integer :: i
+  end type
+  class (mytype), pointer :: c
+
+  allocate (c, source = mytype (99_8))
+
+  call foo(c)
+  call bar(c)
+
+  deallocate (c)
+
+contains
+
+  subroutine foo (arg)
+    class(*) :: arg
+    select type (arg)
+      type is (mytype)
+        if (arg%i .ne. 99_8) call abort
+    end select
+  end subroutine
+
+  subroutine bar (arg)
+    class(mytype) :: arg
+    select type (arg)
+      type is (mytype)
+        if (arg%i .ne. 99_8) call abort
+    end select
+  end subroutine
+
+end
+! { dg-final { scan-tree-dump-times "arg.*._len" 1 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/dec_structure_22.f90 b/gcc/testsuite/gfortran.dg/dec_structure_22.f90
new file mode 100644
index 00000000000..ddbee02602a
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/dec_structure_22.f90
@@ -0,0 +1,38 @@
+      ! { dg-do run }
+      ! { dg-options "-fdec-structure" }
+      !
+      ! PR fortran/82511
+      !
+      ! Verify that structure variables with UNION components
+      ! are accepted in an I/O-list READ.
+      !
+      implicit none
+
+      structure /s/
+        union
+          map
+            character(16) :: c16_1
+          end map
+          map
+            character(16) :: c16_2
+          end map
+        end union
+      end structure
+
+      record /s/ r
+      character(32) :: instr = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%^"
+
+      r.c16_1 = '                '
+      r.c16_2 = '                '
+      ! The record r shall be treated as if its components are listed:
+      ! read(...) r.c16_1, r.c16_2
+      ! This shall correspond to the formatted read of A16,A16
+      read(instr, '(A16,A16)') r
+
+      ! r.c16_1 and r.c16_2 are in a union, thus share the same memory
+      ! and the first 16 bytes of instr are overwritten
+      if ( r.c16_1 .ne. instr(17:32) .or. r.c16_2 .ne. instr(17:32) ) then
+        call abort()
+      endif
+
+      end
diff --git a/gcc/testsuite/gfortran.dg/derived_init_4.f90 b/gcc/testsuite/gfortran.dg/derived_init_4.f90
new file mode 100644
index 00000000000..114975150aa
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/derived_init_4.f90
@@ -0,0 +1,60 @@
+! { dg-do run }
+!
+! Test the fix for PR81048, where in the second call to 'g2' the
+! default initialization was "forgotten". 'g1', 'g1a' and 'g3' check
+! that this does not occur for scalars and explicit results.
+!
+! Contributed by David Smith  <dm577216smith@gmail.com>
+!
+program test
+   type f
+       integer :: f = -1
+   end type
+   type(f) :: a, b(3)
+   type(f), allocatable :: ans
+   b = g2(a)
+   b = g2(a)
+   ans = g1(a)
+   if (ans%f .ne. -1) call abort
+   ans = g1(a)
+   if (ans%f .ne. -1) call abort
+   ans = g1a(a)
+   if (ans%f .ne. -1) call abort
+   ans = g1a(a)
+   if (ans%f .ne. -1) call abort
+   b = g3(a)
+   b = g3(a)
+contains
+   function g3(a) result(res)
+      type(f) :: a, res(3)
+      do j = 1, 3
+         if (res(j)%f == -1) then
+             res(j)%f = a%f - 1
+         else
+             call abort
+         endif
+      enddo
+   end function g3
+
+   function g2(a)
+      type(f) :: a, g2(3)
+      do j = 1, 3
+         if (g2(j)%f == -1) then
+             g2(j)%f = a%f - 1
+         else
+             call abort
+         endif
+      enddo
+   end function g2
+
+   function g1(a)
+     type(f) :: g1, a
+     if (g1%f .ne. -1 ) call abort
+   end function
+
+   function g1a(a) result(res)
+     type(f) :: res, a
+     if (res%f .ne. -1 ) call abort
+   end function
+end program test
+
diff --git a/gcc/testsuite/gfortran.dg/dtio_13.f90 b/gcc/testsuite/gfortran.dg/dtio_13.f90
index 9b907201afc..131af05c847 100644
--- a/gcc/testsuite/gfortran.dg/dtio_13.f90
+++ b/gcc/testsuite/gfortran.dg/dtio_13.f90
@@ -136,9 +136,7 @@ program test
    character(3) :: a, b
    class(t) :: chairman ! { dg-error "must be dummy, allocatable or pointer" }
    open (unit=71, file='myunformatted_data.dat', form='unformatted')
-! The following error is spurious and is eliminated if previous error is corrected.
-! TODO Although better than an ICE, fix me.
-   read (71) a, chairman, b ! { dg-error "cannot be polymorphic" }
+   read (71) a, chairman, b 
    close (unit=71)
 end
 
diff --git a/gcc/testsuite/gfortran.dg/equiv_pure.f90 b/gcc/testsuite/gfortran.dg/equiv_pure.f90
new file mode 100644
index 00000000000..5b0ce419d2a
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/equiv_pure.f90
@@ -0,0 +1,52 @@
+! { dg-do compile }
+! PR fortran/82796
+! Code contributed by ripero84 at gmail dot com 
+module eq
+   implicit none
+   integer :: n1, n2
+   integer, dimension(2) :: a
+   equivalence (a(1), n1)
+   equivalence (a(2), n2)
+   common /a/ a
+end module eq
+
+module m
+   use eq
+   implicit none
+   type, public :: t
+     integer :: i
+   end type t
+end module m
+
+module p
+   implicit none
+   contains
+   pure integer function d(h)
+     use m
+     implicit none
+     integer, intent(in) :: h
+     d = h
+   end function
+end module p
+
+module q
+   implicit none
+   contains
+   pure integer function d(h)
+     use m, only : t
+     implicit none
+     integer, intent(in) :: h
+     d = h
+   end function
+end module q
+
+module r
+   implicit none
+   contains
+   pure integer function d(h)
+     use m, only : a          ! { dg-error "cannot be an EQUIVALENCE object" }
+     implicit none
+     integer, intent(in) :: h
+     d = h
+   end function
+end module r
diff --git a/gcc/testsuite/gfortran.dg/execute_command_line_3.f90 b/gcc/testsuite/gfortran.dg/execute_command_line_3.f90
index 87d73d1b50f..c1790d801f3 100644
--- a/gcc/testsuite/gfortran.dg/execute_command_line_3.f90
+++ b/gcc/testsuite/gfortran.dg/execute_command_line_3.f90
@@ -15,10 +15,9 @@ character(len=:), allocatable :: command
    if (j /= 3 .or. msg /= "Invalid command line" ) call abort
    msg = ''
    call execute_command_line(command , wait=.false., exitstat=i,            cmdmsg=msg )
-   print *,msg
-   if (msg /= '') call abort
-   call execute_command_line(command ,               exitstat=i, cmdstat=j             )
    if (j /= 3) call abort
    call execute_command_line(command , wait=.false., exitstat=i                        )
+   if (msg /= '') call abort
+   call execute_command_line(command ,               exitstat=i, cmdstat=j             )
 
 end program boom
diff --git a/gcc/testsuite/gfortran.dg/gomp/pr82568.f90 b/gcc/testsuite/gfortran.dg/gomp/pr82568.f90
new file mode 100644
index 00000000000..303278ca58f
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/pr82568.f90
@@ -0,0 +1,75 @@
+! PR fortran/82568
+
+MODULE PR82568_MOD
+  INTEGER :: N
+END MODULE
+PROGRAM PR82568
+  INTEGER :: I, L
+  !$OMP PARALLEL DO
+  DO I=1,2
+    BLOCK
+      USE PR82568_MOD
+      INTEGER :: J
+      DO J=1,2
+        PRINT*,I,J
+      END DO
+      DO K=1,2
+        PRINT*,I,K
+      END DO
+      DO L=1,2
+        PRINT*,I,L
+      END DO
+      DO N=1,2
+        PRINT*,I,N
+      END DO
+    END BLOCK
+    DO M=1,2
+      PRINT*,I,M
+    END DO
+  END DO
+  !$OMP TASK
+  DO I=1,2
+    BLOCK
+      USE PR82568_MOD
+      INTEGER :: J
+      DO J=1,2
+        PRINT*,I,J
+      END DO
+      DO K=1,2
+        PRINT*,I,K
+      END DO
+      DO L=1,2
+        PRINT*,I,L
+      END DO
+      DO N=1,2
+        PRINT*,I,N
+      END DO
+    END BLOCK
+    DO M=1,2
+      PRINT*,I,M
+    END DO
+  END DO
+  !$OMP END TASK
+  !$OMP TASKLOOP
+  DO I=1,2
+    BLOCK
+      USE PR82568_MOD
+      INTEGER :: J
+      DO J=1,2
+        PRINT*,I,J
+      END DO
+      DO K=1,2
+        PRINT*,I,K
+      END DO
+      DO L=1,2
+        PRINT*,I,L
+      END DO
+      DO N=1,2
+        PRINT*,I,N
+      END DO
+    END BLOCK
+    DO M=1,2
+      PRINT*,I,M
+    END DO
+  END DO
+END PROGRAM PR82568
diff --git a/gcc/testsuite/gfortran.dg/graphite/pr82672.f90 b/gcc/testsuite/gfortran.dg/graphite/pr82672.f90
new file mode 100644
index 00000000000..77a1c706218
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/graphite/pr82672.f90
@@ -0,0 +1,33 @@
+! { dg-do compile }
+! { dg-options "-O2 -floop-nest-optimize" }
+
+  character(len=20,kind=4) :: s4
+  character(len=20,kind=1) :: s1
+
+  s1 = "foo\u0000"
+  s1 = "foo\u00ff"
+  s1 = "foo\u0100"
+  s1 = "foo\u0101"
+  s1 = "foo\U00000101"
+
+  s1 = 4_"foo bar"
+  s1 = 4_"foo\u00ff"
+  s1 = 4_"foo\u0101"
+  s1 = 4_"foo\u1101"
+  s1 = 4_"foo\UFFFFFFFF"
+
+  s4 = "foo\u0000"
+  s4 = "foo\u00ff"
+  s4 = "foo\u0100"
+  s4 = "foo\U00000100"
+
+  s4 = 4_"foo bar"
+  s4 = 4_"\xFF\x96"
+  s4 = 4_"\x00\x96"
+  s4 = 4_"foo\u00ff"
+  s4 = 4_"foo\u0101"
+  s4 = 4_"foo\u1101"
+  s4 = 4_"foo\Uab98EF56"
+  s4 = 4_"foo\UFFFFFFFF"
+
+end
diff --git a/gcc/testsuite/gfortran.dg/illegal_char.f90 b/gcc/testsuite/gfortran.dg/illegal_char.f90
new file mode 100644
index 00000000000..597c7b98ddd
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/illegal_char.f90
@@ -0,0 +1,6 @@
+! { dg-do compile }
+! PR 82372 - show hexcode of illegal, non-printable characters
+program main
+  tmp =È   1.0 ! { dg-error "Invalid character 0xC8" }
+  print *,tmp
+end
diff --git a/gcc/testsuite/gfortran.dg/implied_do_io_1.f90 b/gcc/testsuite/gfortran.dg/implied_do_io_1.f90
index e4a6d6b37b3..aef36af13eb 100644
--- a/gcc/testsuite/gfortran.dg/implied_do_io_1.f90
+++ b/gcc/testsuite/gfortran.dg/implied_do_io_1.f90
@@ -56,4 +56,4 @@ program main
 1000 format (A2,100I4)
 end program main
 
-! { dg-final { scan-tree-dump-times "while" 7 "original" } }
+! { dg-final { scan-tree-dump-times "(?n)^\\s*while \\(1\\)$" 7 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/large_real_kind_2.F90 b/gcc/testsuite/gfortran.dg/large_real_kind_2.F90
index 7ed4c30e0d5..486c8c00361 100644
--- a/gcc/testsuite/gfortran.dg/large_real_kind_2.F90
+++ b/gcc/testsuite/gfortran.dg/large_real_kind_2.F90
@@ -1,6 +1,5 @@
 ! { dg-do run }
 ! { dg-require-effective-target fortran_large_real }
-! { dg-xfail-if "" { "*-*-freebsd*" } }
 
 ! Testing library calls on large real kinds (larger than kind=8)
   implicit none
diff --git a/gcc/testsuite/gfortran.dg/matmul_const.f90 b/gcc/testsuite/gfortran.dg/matmul_const.f90
new file mode 100644
index 00000000000..35dce322774
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/matmul_const.f90
@@ -0,0 +1,10 @@
+! { dg-do  run }
+! { dg-additional-options "-fno-frontend-optimize -fdump-tree-original" }
+program main
+  integer, parameter :: A(3,2) = reshape([1,2,3,4,5,6],[3,2])
+  integer, parameter :: B(2,3) = reshape([1,1,1,1,1,1],[2,3])
+  character (len=30) :: line
+  write (unit=line,fmt='(9i3)') matmul(A,B)
+  if (line /= '  5  7  9  5  7  9  5  7  9') call abort
+end program main
+!  dg-final { scan-tree-dump-times "matmul_i4" 0 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/pdt_16.f03 b/gcc/testsuite/gfortran.dg/pdt_16.f03
new file mode 100644
index 00000000000..067d87d660d
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pdt_16.f03
@@ -0,0 +1,21 @@
+! { dg-do compile }
+!
+! Test the fix for all three errors in PR82586
+!
+! Contributed by G Steinmetz  <gscfq@t-online.de>
+!
+module m
+   type t(a)                 ! { dg-error "does not have a component" }
+   end type
+end
+
+program p
+   type t(a                  ! { dg-error "Expected parameter list" }
+      integer, kind :: a
+      real(a) :: x
+   end type
+   type u(a, a)              ! { dg-error "Duplicate name" }
+      integer, kind :: a     ! { dg-error "already declared" }
+      integer, len :: a      ! { dg-error "already declared" }
+   end type
+end
diff --git a/gcc/testsuite/gfortran.dg/pdt_17.f03 b/gcc/testsuite/gfortran.dg/pdt_17.f03
new file mode 100644
index 00000000000..1b0a30dca4c
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pdt_17.f03
@@ -0,0 +1,11 @@
+! { dg-do compile }
+!
+! Test the fix for PR82587
+!
+! Contributed by G Steinmetz  <gscfq@t-online.de>
+!
+program p
+   type t(a)                   ! { dg-error "does not have a component" }
+      integer(kind=t()) :: x   ! { dg-error "used before it is defined" }
+   end type
+end
diff --git a/gcc/testsuite/gfortran.dg/pdt_18.f03 b/gcc/testsuite/gfortran.dg/pdt_18.f03
new file mode 100644
index 00000000000..896a727eaae
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pdt_18.f03
@@ -0,0 +1,19 @@
+! { dg-do compile }
+!
+! Test the fix for PR82589
+!
+! Contributed by G Steinmetz  <gscfq@t-online.de>
+!
+module m
+   type t(a)
+      integer, KIND, private :: a        ! { dg-error "attribute conflicts with" }
+      integer, KIND, allocatable :: a    ! { dg-error "attribute conflicts with" }
+      integer, KIND, POINTER :: a        ! { dg-error "attribute conflicts with" }
+      integer, KIND, dimension(2) :: a   ! { dg-error "attribute conflicts with" }
+      integer, len, private :: a         ! { dg-error "attribute conflicts with" }
+      integer, len, allocatable :: a     ! { dg-error "attribute conflicts with" }
+      integer, len, POINTER :: a         ! { dg-error "attribute conflicts with" }
+      integer, len, dimension(2) :: a    ! { dg-error "attribute conflicts with" }
+      integer, kind :: a
+   end type
+end
diff --git a/gcc/testsuite/gfortran.dg/pdt_4.f03 b/gcc/testsuite/gfortran.dg/pdt_4.f03
index 13c00af79f1..15cb6417ca7 100644
--- a/gcc/testsuite/gfortran.dg/pdt_4.f03
+++ b/gcc/testsuite/gfortran.dg/pdt_4.f03
@@ -26,7 +26,7 @@ end module
   integer, kind :: bad_kind    ! { dg-error "not allowed outside a TYPE definition" }
   integer, len :: bad_len      ! { dg-error "not allowed outside a TYPE definition" }
 
-  type :: bad_pdt (a,b, c, d)
+  type :: bad_pdt (a,b, c, d)  ! { dg-error "does not have a component" }
     real, kind :: a            ! { dg-error "must be INTEGER" }
     INTEGER(8), kind :: b      ! { dg-error "be default integer kind" }
     real, LEN :: c             ! { dg-error "must be INTEGER" }
diff --git a/gcc/testsuite/gfortran.dg/pdt_8.f03 b/gcc/testsuite/gfortran.dg/pdt_8.f03
index d5e393e5e0c..aeec407fb4b 100644
--- a/gcc/testsuite/gfortran.dg/pdt_8.f03
+++ b/gcc/testsuite/gfortran.dg/pdt_8.f03
@@ -15,9 +15,10 @@ type :: t(i,a,x)         ! { dg-error "does not|has neither" }
   real, kind :: x        ! { dg-error "must be INTEGER" }
 end type
 
-type :: t1(k,y)          ! { dg-error "not declared as a component of the type" }
+type :: t1(k,y)          ! { dg-error "does not have a component" }
   integer, kind :: k
 end type
 
-type(t1(4,4)) :: z
+! This is a knock-on from the previous error
+type(t1(4,4)) :: z       ! { dg-error "Invalid character in name" }
 end
diff --git a/gcc/testsuite/gfortran.dg/pr81735.f90 b/gcc/testsuite/gfortran.dg/pr81735.f90
new file mode 100644
index 00000000000..6aae203aa0f
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pr81735.f90
@@ -0,0 +1,25 @@
+! { dg-do compile }
+! { dg-options "-fdump-tree-original" }
+!
+! Contributed by Danila  <flashmozzg@gmail.com>
+!
+program fooprog
+    implicit none
+    type FooType
+        integer, allocatable :: x
+    end type FooType
+
+    type(FooType), pointer :: bar
+
+    bar => foo()
+
+contains
+    function foo() result(res)
+        type(FooType), pointer :: res
+
+        character(:), allocatable :: rt
+        rt = ""
+        res => null()
+    end function foo
+end program fooprog
+! { dg-final { scan-tree-dump-times "__builtin_free" 1 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/spellcheck-operator.f90 b/gcc/testsuite/gfortran.dg/spellcheck-operator.f90
new file mode 100644
index 00000000000..810a770698b
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/spellcheck-operator.f90
@@ -0,0 +1,30 @@
+! { dg-do compile }
+! test levenshtein based spelling suggestions
+
+module mymod1
+  implicit none
+  contains
+    function something_good (iarg1)
+      integer :: something_good
+      integer, intent(in) :: iarg1
+      something_good = iarg1 + 42
+    end function something_good
+end module mymod1
+
+program spellchekc
+  use mymod1
+  implicit none
+
+  interface operator (.mywrong.)
+    module procedure something_wring ! { dg-error "Procedure .something_wring. in operator interface .mywrong. at .1. is neither function nor subroutine; did you mean .something_good.\\?|User operator procedure .something_wring. at .1. must be a FUNCTION" }
+  end interface
+
+  interface operator (.mygood.)
+    module procedure something_good
+  end interface
+
+  integer :: i, j, added
+  i = 0
+  j = 0
+  added = .mygoof. j ! { dg-error "Unknown operator .mygoof. at .1.; did you mean .mygood.\\?" }
+end program spellchekc
diff --git a/gcc/testsuite/gfortran.dg/spellcheck-parameter.f90 b/gcc/testsuite/gfortran.dg/spellcheck-parameter.f90
new file mode 100644
index 00000000000..715c5abcce7
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/spellcheck-parameter.f90
@@ -0,0 +1,15 @@
+! { dg-do compile }
+! Contributed by Joost VandeVondele
+! test levenshtein based spelling suggestions for keyword arguments
+
+module test
+contains
+  subroutine mysub(iarg1)
+    integer :: iarg1
+  end subroutine
+end module
+
+use test
+call mysub(iarg=1) ! { dg-error "Keyword argument .iarg. at .1. is not in the procedure; did you mean .iarg1.\\?" }
+
+end
diff --git a/gcc/testsuite/gfortran.dg/spellcheck-procedure_1.f90 b/gcc/testsuite/gfortran.dg/spellcheck-procedure_1.f90
new file mode 100644
index 00000000000..3b7f7169468
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/spellcheck-procedure_1.f90
@@ -0,0 +1,41 @@
+! { dg-do compile }
+! test levenshtein based spelling suggestions
+
+module mymod1
+  implicit none
+  contains
+    function something_else (iarg1)
+      integer :: something_else
+      integer, intent(in) :: iarg1
+      something_else = iarg1 + 42
+    end function something_else
+    function add_fourtytwo (iarg1)
+      integer :: add_fourtytwo
+      integer, intent(in) :: iarg1
+      add_fourtytwo = iarg1 + 42
+    end function add_fourtytwo
+end module mymod1
+
+function myadd(iarg1, iarg2)
+  implicit none
+  integer :: myadd
+  integer, intent(in) :: iarg1, iarg2
+  myadd = iarg1 + iarg2
+end function myadd
+
+program spellchekc
+  use mymod1, something_good => something_else
+  implicit none
+
+  integer :: myadd, i, j, myvar
+  i = 0
+  j = 0
+
+  j = something_goof(j) ! { dg-error "no IMPLICIT type; did you mean .something_good.\\?" }
+  j = myaddd(i, j) ! { dg-error "no IMPLICIT type; did you mean .myadd.\\?" }
+  if (j /= 42) call abort
+  j = add_fourtytow(i, j) ! { dg-error "no IMPLICIT type; did you mean .add_fourtytwo.\\?" }
+  myval = myadd(i, j) ! { dg-error "no IMPLICIT type; did you mean .myvar.\\?" }
+  if (j /= 42 * 2) call abort
+
+end program spellchekc
diff --git a/gcc/testsuite/gfortran.dg/spellcheck-procedure_2.f90 b/gcc/testsuite/gfortran.dg/spellcheck-procedure_2.f90
new file mode 100644
index 00000000000..a6ea5f9f280
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/spellcheck-procedure_2.f90
@@ -0,0 +1,35 @@
+! { dg-do compile }
+! test levenshtein based spelling suggestions
+
+
+program spellchekc
+  implicit none (external) ! { dg-warning "GNU Extension: IMPORT NONE with spec list" }
+
+  interface
+    subroutine bark_unless_zero(iarg)
+      implicit none
+      integer, intent(in) :: iarg
+    end subroutine bark_unless_zero
+  end interface
+
+  integer :: i
+  i = 0
+
+  if (i /= 1) call abort
+  call bark_unless_0(i) ! { dg-error "not explicitly declared; did you mean .bark_unless_zero.\\?" }
+!  call complain_about_0(i) ! { -dg-error "not explicitly declared; did you mean .complain_about_zero.\\?" }
+
+contains
+! We cannot reliably see this ATM, would need an unambiguous bit somewhere
+  subroutine complain_about_zero(iarg)
+    integer, intent(in) :: iarg
+    if (iarg /= 0) call abort
+  end subroutine complain_about_zero
+
+end program spellchekc
+
+subroutine bark_unless_zero(iarg)
+  implicit none
+  integer, intent(in) :: iarg
+  if (iarg /= 0) call abort
+end subroutine bark_unless_zero
diff --git a/gcc/testsuite/gfortran.dg/spellcheck-structure.f90 b/gcc/testsuite/gfortran.dg/spellcheck-structure.f90
new file mode 100644
index 00000000000..929e05f2151
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/spellcheck-structure.f90
@@ -0,0 +1,35 @@
+! { dg-do compile }
+! test levenshtein based spelling suggestions
+implicit none
+
+!!!!!!!!!!!!!! structure tests !!!!!!!!!!!!!!
+type type1
+   real :: radius
+   integer :: i
+end type type1
+
+type type2
+  integer :: myint
+  type(type1) :: mytype
+end type type2
+
+type type3
+  type(type2) :: type_2
+end type type3
+type type4
+  type(type3) :: type_3
+end type type4
+
+type(type1) :: t1
+t1%radiuz = .0 ! { dg-error ".radiuz. at .1. is not a member of the .type1. structure; did you mean .radius.\\?" }
+t1%x = .0 ! { dg-error ".x. at .1. is not a member of the .type1. structure" }
+type(type2) :: t2
+t2%mytape%radius = .0 ! { dg-error ".mytape. at .1. is not a member of the .type2. structure; did you mean .mytype.\\?" }
+t2%mytype%radious = .0 ! { dg-error ".radious. at .1. is not a member of the .type1. structure; did you mean .radius.\\?" }
+type(type4) :: t4
+t4%type_3%type_2%mytype%radium = 88.0 ! { dg-error ".radium. at .1. is not a member of the .type1. structure; did you mean .radius.\\?" }
+
+!!!!!!!!!!!!!! symbol tests !!!!!!!!!!!!!!
+integer :: iarg1
+iarg2 = 1 ! { dg-error "Symbol .iarg2. at .1. has no IMPLICIT type; did you mean .iarg1.\\?" }
+end
diff --git a/gcc/testsuite/gfortran.dg/submodule_30.f08 b/gcc/testsuite/gfortran.dg/submodule_30.f08
new file mode 100644
index 00000000000..25dcbebe656
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/submodule_30.f08
@@ -0,0 +1,42 @@
+! { dg-do run }
+!
+! Test the fix for PR82550 in which the reference to 'p' in 'foo'
+! was not being correctly handled.
+!
+! Contributed by Reinhold Bader  <Bader@lrz.de>
+!
+module m_subm_18_pos
+  implicit none
+  integer :: i = 0
+  interface
+    module subroutine foo(fun_ptr)
+      procedure(p), pointer, intent(out) :: fun_ptr
+    end subroutine
+  end interface
+contains
+  subroutine p()
+    i = 1
+  end subroutine p
+end module m_subm_18_pos
+submodule (m_subm_18_pos) subm_18_pos
+    implicit none
+contains
+    module subroutine foo(fun_ptr)
+      procedure(p), pointer, intent(out) :: fun_ptr
+      fun_ptr => p
+    end subroutine
+end submodule
+program p_18_pos
+  use m_subm_18_pos
+  implicit none
+  procedure(), pointer :: x
+  call foo(x)
+  call x()
+  if (i == 1) then
+     write(*,*) 'OK'
+  else
+     write(*,*) 'FAIL'
+     call abort
+  end if
+end program p_18_pos
+
diff --git a/gcc/testsuite/gnat.dg/default_pkg_actual.adb b/gcc/testsuite/gnat.dg/default_pkg_actual.adb
new file mode 100644
index 00000000000..d10ae0c152b
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/default_pkg_actual.adb
@@ -0,0 +1,32 @@
+--  { dg-do compile }
+
+procedure Default_Pkg_Actual is
+
+   generic
+   package As is
+   end As;
+
+   generic
+      type T is private;
+      with package A0 is new As;
+   package Bs is
+   end Bs;
+
+   generic
+      with package Xa is new As;
+   package Xs is
+      package Xb is new Bs(T => Integer, A0 => Xa);
+   end Xs;
+
+   generic
+      with package Yb is new Bs(T => Integer, others => <>);
+   package Ys is
+   end Ys;
+
+   package A is new As;
+   package X is new Xs(Xa => A);
+   package Y is new Ys(Yb => X.Xb);
+
+begin
+   null;
+end;
diff --git a/gcc/testsuite/gnat.dg/default_pkg_actual2.adb b/gcc/testsuite/gnat.dg/default_pkg_actual2.adb
new file mode 100644
index 00000000000..7ab614a0994
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/default_pkg_actual2.adb
@@ -0,0 +1,27 @@
+--  { dg-do compile }
+
+procedure Default_Pkg_Actual2 is
+
+   generic
+   package P1 is
+   end;
+
+   generic
+      with package FP1a is new P1;
+      with package FP1b is new P1;
+   package P2 is
+   end;
+
+   generic
+      with package FP2 is new P2 (FP1a => <>,  FP1b => <>);
+   package P3 is
+   end;
+
+   package NP1a is new P1;
+   package NP1b is new P1;
+   package NP2  is new P2 (NP1a, NP1b);
+   package NP4  is new P3 (NP2);
+
+begin
+   null;
+end;
diff --git a/gcc/testsuite/gnat.dg/dimensions.adb b/gcc/testsuite/gnat.dg/dimensions.adb
new file mode 100644
index 00000000000..86fc6eef670
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/dimensions.adb
@@ -0,0 +1,5 @@
+--  { dg-do compile }
+
+package body Dimensions is
+   procedure Dummy is null;
+end Dimensions;
diff --git a/gcc/testsuite/gnat.dg/dimensions.ads b/gcc/testsuite/gnat.dg/dimensions.ads
new file mode 100644
index 00000000000..54bab081470
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/dimensions.ads
@@ -0,0 +1,29 @@
+package Dimensions is
+
+   type Mks_Int_Type is new Integer
+     with
+      Dimension_System => (
+        (Unit_Name => Meter,    Unit_Symbol => 'm',   Dim_Symbol => 'L'),
+        (Unit_Name => Kilogram, Unit_Symbol => "kg",  Dim_Symbol => 'M'),
+        (Unit_Name => Second,   Unit_Symbol => 's',   Dim_Symbol => 'T'),
+        (Unit_Name => Ampere,   Unit_Symbol => 'A',   Dim_Symbol => 'I'),
+        (Unit_Name => Kelvin,   Unit_Symbol => 'K',   Dim_Symbol => '@'),
+        (Unit_Name => Mole,     Unit_Symbol => "mol", Dim_Symbol => 'N'),
+        (Unit_Name => Candela,  Unit_Symbol => "cd",  Dim_Symbol => 'J'));
+
+   subtype Int_Length is Mks_Int_Type
+     with
+      Dimension => (Symbol => 'm',
+        Meter  => 1,
+        others => 0);
+
+   subtype Int_Speed is Mks_Int_Type
+     with
+      Dimension => (
+        Meter  =>  1,
+        Second => -1,
+        others =>  0);
+
+   procedure Dummy;
+
+end Dimensions;
diff --git a/gcc/testsuite/gnat.dg/opt68.adb b/gcc/testsuite/gnat.dg/opt68.adb
new file mode 100644
index 00000000000..caf6b713996
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/opt68.adb
@@ -0,0 +1,53 @@
+-- { dg-do compile }
+-- { dg-options "-O3" }
+
+with Ada.Unchecked_Deallocation;
+
+package body Opt68 is
+
+  procedure Free
+    is new Ada.Unchecked_Deallocation (Queue_Element, A_Queue_Element);
+
+  procedure Copy (dest : in out Queue; src : Queue) is
+    d, s, pd, ps, t : A_Queue_Element;
+  begin
+    if src.sz /= 0 then
+      d := dest.front;
+      s := src.front;
+      while d /= null and s /= null loop
+        d.value := s.value;
+        pd := d;
+        ps := s;
+        d  := d.next;
+        s  := s.next;
+      end loop;
+      if src.sz = dest.sz then
+        return;
+      elsif s = null then
+        while d /= null loop
+          t := d.next;
+          Free (d);
+          d := t;
+        end loop;
+        dest.back      := pd;
+        dest.back.next := null;
+      else
+        if pd = null then
+          dest.front       := new Queue_Element;
+          dest.front.value := s.value;
+          s                := s.next;
+          pd               := dest.front;
+        end if;
+        while s /= null loop
+          pd.next       := new Queue_Element;
+          pd.next.value := s.value;
+          pd            := pd.next;
+          s             := s.next;
+        end loop;
+        dest.back := pd;
+      end if;
+      dest.sz := src.sz;
+    end if;
+  end;
+
+end Opt68;
diff --git a/gcc/testsuite/gnat.dg/opt68.ads b/gcc/testsuite/gnat.dg/opt68.ads
new file mode 100644
index 00000000000..25e28a50d7b
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/opt68.ads
@@ -0,0 +1,26 @@
+with Ada.Finalization;
+
+package Opt68 is
+
+  type Cont is new Ada.Finalization.Controlled with null record;
+
+  type Element is record
+    C : Cont;
+  end record;
+
+  type Queue_Element;
+  type A_Queue_Element is access Queue_Element;
+  type Queue_Element is record
+    Value : Element;
+    Next  : A_Queue_Element;
+  end record;
+
+  type Queue is limited record
+    Sz    : Natural;
+    Front : A_Queue_Element;
+    Back  : A_Queue_Element;
+  end record;
+
+  procedure Copy (dest : in out Queue; src : Queue);
+
+end Opt68;
diff --git a/gcc/testsuite/gnat.dg/remote_call_iface.adb b/gcc/testsuite/gnat.dg/remote_call_iface.adb
new file mode 100644
index 00000000000..6816ad95a65
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/remote_call_iface.adb
@@ -0,0 +1,7 @@
+--  { dg-do compile }
+
+package body Remote_Call_Iface is
+   procedure Proc is begin null; end;
+begin
+   Proc;
+end Remote_Call_Iface;
diff --git a/gcc/testsuite/gnat.dg/remote_call_iface.ads b/gcc/testsuite/gnat.dg/remote_call_iface.ads
new file mode 100644
index 00000000000..ce12fef88ca
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/remote_call_iface.ads
@@ -0,0 +1,5 @@
+generic
+package Remote_Call_Iface is
+   pragma Remote_Call_Interface;
+   procedure Proc;
+end Remote_Call_Iface;
diff --git a/gcc/testsuite/gnat.dg/specs/discr_private.ads b/gcc/testsuite/gnat.dg/specs/discr2.ads
index 0ddfbd137ff..f7ece058812 100644
--- a/gcc/testsuite/gnat.dg/specs/discr_private.ads
+++ b/gcc/testsuite/gnat.dg/specs/discr2.ads
@@ -1,7 +1,7 @@
 -- { dg-do compile }
 -- { dg-options "-gnatws" }
 
-package Discr_Private is
+package Discr2 is
 
    package Dec is
       type T_DECIMAL (Prec : Integer := 1) is private;
@@ -47,4 +47,4 @@ package Discr_Private is
        end case;
    end record;
 
-end Discr_Private;
+end Discr2;
diff --git a/gcc/testsuite/gnat.dg/specs/discr_record_constant.ads b/gcc/testsuite/gnat.dg/specs/discr3.ads
index f43b1386909..bcb996b7386 100644
--- a/gcc/testsuite/gnat.dg/specs/discr_record_constant.ads
+++ b/gcc/testsuite/gnat.dg/specs/discr3.ads
@@ -2,7 +2,7 @@
 
 pragma Restrictions (No_Implicit_Heap_Allocations);
 
-package Discr_Record_Constant is
+package Discr3 is
 
    type T (Big : Boolean := False) is record
       case Big is
@@ -19,4 +19,4 @@ package Discr_Record_Constant is
     Con : constant T := D;    --  Violation of restriction
     Ter : constant T := Con;  --  Violation of restriction
 
-end Discr_Record_Constant;
+end Discr3;
diff --git a/gcc/testsuite/gnat.dg/specs/discr4.ads b/gcc/testsuite/gnat.dg/specs/discr4.ads
new file mode 100644
index 00000000000..a7fc25b9d66
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/specs/discr4.ads
@@ -0,0 +1,23 @@
+-- { dg-do compile }
+-- { dg-options "-O" }
+
+with Discr4_Pkg; use Discr4_Pkg;
+
+package Discr4 is
+
+   type Data is record
+      Val : Rec;
+      Set : Boolean;
+   end record;
+
+   type Pair is record
+      Lower, Upper : Data;
+   end record;
+
+   function Build (L, U : Rec) return Pair is ((L, True), (U, False));
+
+   C1 : constant Pair := Build (Rec_One, Rec_Three);
+
+   C2 : constant Pair := Build (Get (0), Rec_Three);
+
+end Discr4;
diff --git a/gcc/testsuite/gnat.dg/specs/discr4_pkg.ads b/gcc/testsuite/gnat.dg/specs/discr4_pkg.ads
new file mode 100644
index 00000000000..231a8fb77e8
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/specs/discr4_pkg.ads
@@ -0,0 +1,27 @@
+package Discr4_Pkg is
+
+   type Enum is (One, Two, Three);
+
+   type Rec is private;
+
+   Rec_One : constant Rec;
+   Rec_Three  : constant Rec;
+
+   function Get (Value : Integer) return Rec;
+
+private
+
+   type Rec (D : Enum := Two) is record
+      case D is
+         when One => null;
+         when Two => Value : Integer;
+         when Three => null;
+      end case;
+   end record;
+
+   Rec_One   : constant Rec := (D => One);
+   Rec_Three : constant Rec := (D => Three);
+
+   function Get (Value : Integer) return Rec is (Two, Value);
+
+end Discr4_Pkg;
diff --git a/gcc/testsuite/gnat.dg/stack_usage4.adb b/gcc/testsuite/gnat.dg/stack_usage4.adb
new file mode 100644
index 00000000000..24cd1a75bf0
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/stack_usage4.adb
@@ -0,0 +1,11 @@
+-- { dg-do compile }
+-- { dg-options "-Wstack-usage=512" }
+
+with Stack_Usage4_Pkg; use Stack_Usage4_Pkg;
+
+procedure Stack_Usage4 is
+   BS : Bounded_String := Get;
+   S : String := BS.Data (BS.Data'First .. BS.Len);
+begin
+   null;
+end;
diff --git a/gcc/testsuite/gnat.dg/stack_usage4_pkg.ads b/gcc/testsuite/gnat.dg/stack_usage4_pkg.ads
new file mode 100644
index 00000000000..9bad62776cd
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/stack_usage4_pkg.ads
@@ -0,0 +1,12 @@
+package Stack_Usage4_Pkg is
+
+   subtype Name_Index_Type is Natural range 1 .. 63;
+
+   type Bounded_String is record
+      Len  : Name_Index_Type;
+      Data : String (Name_Index_Type'Range);
+   end record;
+
+   function Get return Bounded_String;
+
+end Stack_Usage4_Pkg;
diff --git a/gcc/testsuite/gnat.dg/sync_iface_call.adb b/gcc/testsuite/gnat.dg/sync_iface_call.adb
new file mode 100644
index 00000000000..1603981892e
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/sync_iface_call.adb
@@ -0,0 +1,34 @@
+--  { dg-do compile }
+
+with Sync_Iface_Call_Pkg;
+with Sync_Iface_Call_Pkg2;
+
+procedure Sync_Iface_Call is
+
+   Impl : access Sync_Iface_Call_Pkg.IFace'Class :=
+       new Sync_Iface_Call_Pkg2.Impl;
+   Val : aliased Integer := 10;
+begin
+   select
+      Impl.Do_Stuff (Val);
+   or
+      delay 10.0;
+   end select;
+   select
+      Impl.Do_Stuff_Access (Val'Access);
+   or
+      delay 10.0;
+   end select;
+
+   select
+      Impl.Do_Stuff_2 (Val);
+   or
+      delay 10.0;
+   end select;
+
+   select
+      Impl.Do_Stuff_2_Access (Val'Access);
+   or
+      delay 10.0;
+   end select;
+end Sync_Iface_Call;
diff --git a/gcc/testsuite/gnat.dg/sync_iface_call_pkg.ads b/gcc/testsuite/gnat.dg/sync_iface_call_pkg.ads
new file mode 100644
index 00000000000..e392c024c79
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/sync_iface_call_pkg.ads
@@ -0,0 +1,21 @@
+package Sync_Iface_Call_Pkg is
+
+   type IFace is synchronized interface;
+
+   procedure Do_Stuff
+     (This  : in out IFace;
+      Value : in Integer) is null;
+
+   procedure Do_Stuff_Access
+     (This  : in out IFace;
+      Value : not null access Integer) is null;
+
+   procedure Do_Stuff_2
+     (This  : not null access IFace;
+      Value : in Integer) is null;
+
+   procedure Do_Stuff_2_Access
+     (This  : not null access IFace;
+      Value : not null access Integer) is null;
+
+end Sync_Iface_Call_Pkg;
diff --git a/gcc/testsuite/gnat.dg/sync_iface_call_pkg2.adb b/gcc/testsuite/gnat.dg/sync_iface_call_pkg2.adb
new file mode 100644
index 00000000000..b3c221e5b1a
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/sync_iface_call_pkg2.adb
@@ -0,0 +1,8 @@
+package body Sync_Iface_Call_Pkg2 is
+
+   task body Impl is
+   begin
+      null;
+   end Impl;
+
+end Sync_Iface_Call_Pkg2;
diff --git a/gcc/testsuite/gnat.dg/sync_iface_call_pkg2.ads b/gcc/testsuite/gnat.dg/sync_iface_call_pkg2.ads
new file mode 100644
index 00000000000..ca21b1d6d08
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/sync_iface_call_pkg2.ads
@@ -0,0 +1,7 @@
+with Sync_Iface_Call_Pkg;
+
+package Sync_Iface_Call_Pkg2 is
+
+   task type Impl is new Sync_Iface_Call_Pkg.IFace with end;
+
+end Sync_Iface_Call_Pkg2;
diff --git a/gcc/testsuite/jit.dg/jit.exp b/gcc/testsuite/jit.dg/jit.exp
index 39e37c2da82..869d9f693a0 100644
--- a/gcc/testsuite/jit.dg/jit.exp
+++ b/gcc/testsuite/jit.dg/jit.exp
@@ -580,6 +580,15 @@ proc jit-dg-test { prog do_what extra_tool_flags } {
 	verbose "$name is not meant to generate a reproducer"
     }
 
+    # Normally we would return $comp_output and $output_file to the
+    # caller, which would delete $output_file, the generated executable.
+    # If we need to debug, it's handy to be able to suppress this behavior,
+    # keeping the executable around.
+    set preserve_executables [info exists env(PRESERVE_EXECUTABLES)]
+    if $preserve_executables {
+	set output_file ""
+    }
+    
     return [list $comp_output $output_file]
 }
 
diff --git a/gcc/testsuite/lib/gcc-dg.exp b/gcc/testsuite/lib/gcc-dg.exp
index cb5d1843c92..d8f9b7bd2bb 100644
--- a/gcc/testsuite/lib/gcc-dg.exp
+++ b/gcc/testsuite/lib/gcc-dg.exp
@@ -560,7 +560,7 @@ proc gcc-dg-debug-runtest { target_compile trivial opt_opts testcases } {
 
     if ![info exists DEBUG_TORTURE_OPTIONS] {
 	set DEBUG_TORTURE_OPTIONS ""
-	foreach type {-gdwarf-2 -gstabs -gstabs+ -gxcoff -gxcoff+ -gcoff} {
+	foreach type {-gdwarf-2 -gstabs -gstabs+ -gxcoff -gxcoff+} {
 	    set comp_output [$target_compile \
 		    "$srcdir/$subdir/$trivial" "trivial.S" assembly \
 		    "additional_flags=$type"]
diff --git a/gcc/testsuite/lib/gcov.exp b/gcc/testsuite/lib/gcov.exp
index 632d50667a7..ede01e70212 100644
--- a/gcc/testsuite/lib/gcov.exp
+++ b/gcc/testsuite/lib/gcov.exp
@@ -59,7 +59,7 @@ proc verify-lines { testname testcase file } {
     while { [gets $fd line] >= 0 } {
         # We want to match both "-" and "#####" as count as well as numbers,
         # since we want to detect lines that shouldn't be marked as covered.
-	if [regexp "^ *(\[^:]*): *(\[0-9\\-#]+):.*count\\((\[0-9\\-#=]+)\\)(.*)" \
+	if [regexp "^ *(\[^:]*): *(\[0-9\\-#]+):.*count\\((\[0-9\\-#=\\.kMGTPEZY]+)\\)(.*)" \
 		"$line" all is n shouldbe rest] {
 	    if [regexp "^ *{(.*)}" $rest all xfailed] {
 		switch [dg-process-target $xfailed] {
@@ -108,7 +108,7 @@ proc verify-intermediate { testname testcase file } {
 	if [regexp "^function:(\[0-9\]+),(\[0-9\]+),.*" $line] {
 	    incr function
 	}
-	if [regexp "^lcount:(\[0-9\]+),(\[0-9\]+)" $line] {
+	if [regexp "^lcount:(\[0-9\]+),(\[0-9\]+),(\[01\])" $line] {
 	    incr lcount
 	}
 	if [regexp "^branch:(\[0-9\]+),(taken|nottaken|notexec)" $line] {
diff --git a/gcc/testsuite/lib/gfortran-dg.exp b/gcc/testsuite/lib/gfortran-dg.exp
index 27b2a69b9e2..6f190092f28 100644
--- a/gcc/testsuite/lib/gfortran-dg.exp
+++ b/gcc/testsuite/lib/gfortran-dg.exp
@@ -162,7 +162,7 @@ proc gfortran-dg-debug-runtest { target_compile trivial opt_opts testcases } {
 
     if ![info exists DEBUG_TORTURE_OPTIONS] {
        set DEBUG_TORTURE_OPTIONS ""
-       set type_list [list "-gstabs" "-gstabs+" "-gxcoff" "-gxcoff+" "-gcoff" "-gdwarf-2" ]
+       set type_list [list "-gstabs" "-gstabs+" "-gxcoff" "-gxcoff+" "-gdwarf-2" ]
        foreach type $type_list {
            set comp_output [$target_compile \
                    "$srcdir/$subdir/$trivial" "trivial.S" assembly \
diff --git a/gcc/testsuite/lib/scanasm.exp b/gcc/testsuite/lib/scanasm.exp
index bab23e8e165..a66bb282531 100644
--- a/gcc/testsuite/lib/scanasm.exp
+++ b/gcc/testsuite/lib/scanasm.exp
@@ -231,6 +231,7 @@ proc scan-assembler-times { args } {
 
     set testcase [testname-for-summary]
     set pattern [lindex $args 0]
+    set times [lindex $args 1]
     set pp_pattern [make_pattern_printable $pattern]
 
     # This must match the rule in gcc-dg.exp.
@@ -239,7 +240,7 @@ proc scan-assembler-times { args } {
     set files [glob -nocomplain $output_file]
     if { $files == "" } {
 	verbose -log "$testcase: output file does not exist"
-	unresolved "$testcase scan-assembler-times $pp_pattern [lindex $args 1]"
+	unresolved "$testcase scan-assembler-times $pp_pattern $times"
 	return
     }
 
@@ -247,10 +248,11 @@ proc scan-assembler-times { args } {
     set text [read $fd]
     close $fd
 
-    if { [llength [regexp -inline -all -- $pattern $text]] == [lindex $args 1]} {
-	pass "$testcase scan-assembler-times $pp_pattern [lindex $args 1]"
+    set result_count [llength [regexp -inline -all -- $pattern $text]]
+    if {$result_count == $times} {
+	pass "$testcase scan-assembler-times $pp_pattern $times"
     } else {
-	fail "$testcase scan-assembler-times $pp_pattern [lindex $args 1]"
+	fail "$testcase scan-assembler-times $pp_pattern $times (found $result_count times)"
     }
 }
 
@@ -482,16 +484,16 @@ proc dg-function-on-line { args } {
     }
 
     if { [istarget hppa*-*-*] } {
-	set pattern [format {\t;[^:]+:%d\n(\t[^\t]+\n)+%s:\n\t.PROC} \
+	set pattern [format {\t;[^:]+:%d(:[0-9]+)?\n(\t[^\t]+\n)+%s:\n\t.PROC} \
                      $line $symbol]
     } elseif { [istarget mips*-*-*] } {
-	set pattern [format {\t\.loc [0-9]+ %d 0( [^\n]*)?\n(\t.cfi_startproc[^\t]*\n)*\t\.set\t(no)?mips16\n\t(\.set\t(no)?micromips\n\t)?\.ent\t%s\n\t\.type\t%s, @function\n%s:\n} \
+	set pattern [format {\t\.loc [0-9]+ %d [0-9]+( [^\n]*)?\n(\t.cfi_startproc[^\t]*\n)*\t\.set\t(no)?mips16\n\t(\.set\t(no)?micromips\n\t)?\.ent\t%s\n\t\.type\t%s, @function\n%s:\n} \
 		     $line $symbol $symbol $symbol]
     } elseif { [istarget microblaze*-*-*] } {
-        set pattern [format {:%d\n\$.*:\n\t\.ent\t%s\n\t\.type\t%s, @function\n%s:\n} \
+        set pattern [format {:%d(:[0-9]+)?\n\$.*:\n\t\.ent\t%s\n\t\.type\t%s, @function\n%s:\n} \
                      $line $symbol $symbol $symbol]
     } else {
-	set pattern [format {%s:[^\t]*(\t.(fnstart|frame|mask|file)[^\t]*)*\t[^:]+:%d\n} \
+	set pattern [format {%s:[^\t]*(\t.(fnstart|frame|mask|file)[^\t]*)*\t[^:]+:%d(:[0-9]+)?\n} \
                      $symbol $line]
     }
 
diff --git a/gcc/testsuite/lib/scandump.exp b/gcc/testsuite/lib/scandump.exp
index 2e6eebfaf33..4a64ac6e05d 100644
--- a/gcc/testsuite/lib/scandump.exp
+++ b/gcc/testsuite/lib/scandump.exp
@@ -86,6 +86,7 @@ proc scan-dump-times { args } {
     }
 
     set testcase [testname-for-summary]
+    set times [lindex $args 2]
     set suf [dump-suffix [lindex $args 3]]
     set printable_pattern [make_pattern_printable [lindex $args 1]]
     set testname "$testcase scan-[lindex $args 0]-dump-times $suf \"$printable_pattern\" [lindex $args 2]"
@@ -101,10 +102,11 @@ proc scan-dump-times { args } {
     set text [read $fd]
     close $fd
 
-    if { [llength [regexp -inline -all -- [lindex $args 1] $text]] == [lindex $args 2]} {
+    set result_count [llength [regexp -inline -all -- [lindex $args 1] $text]]
+    if {$result_count == $times} {
         pass "$testname"
     } else {
-        fail "$testname"
+        fail "$testname (found $result_count times)"
     }
 }
 
diff --git a/gcc/testsuite/lib/scanlang.exp b/gcc/testsuite/lib/scanlang.exp
index 796214385c8..729d3069c2a 100644
--- a/gcc/testsuite/lib/scanlang.exp
+++ b/gcc/testsuite/lib/scanlang.exp
@@ -28,11 +28,11 @@ load_lib scandump.exp
 proc scan-lang-dump { args } {
 
     if { [llength $args] < 2 } {
-	error "scan-tree-dump: too few arguments"
+	error "scan-lang-dump: too few arguments"
 	return
     }
     if { [llength $args] > 3 } {
-	error "scan-tree-dump: too many arguments"
+	error "scan-lang-dump: too many arguments"
 	return
     }
     if { [llength $args] >= 3 } {
diff --git a/gcc/testsuite/lib/target-supports-dg.exp b/gcc/testsuite/lib/target-supports-dg.exp
index d50d8b07ada..6080421fa9e 100644
--- a/gcc/testsuite/lib/target-supports-dg.exp
+++ b/gcc/testsuite/lib/target-supports-dg.exp
@@ -180,6 +180,21 @@ proc dg-require-iconv { args } {
     }
 }
 
+# If this target does not have sufficient stack size, skip this test.
+
+proc dg-require-stack-size { args } {
+    if { ![is-effective-target stack_size] } {
+	return
+    }
+    
+    set stack_size [dg-effective-target-value stack_size]
+    set required [expr [lindex $args 1]]
+    if { $stack_size < $required } {
+	upvar dg-do-what dg-do-what
+        set dg-do-what [list [lindex ${dg-do-what} 0] "N" "P"]
+    }
+}
+
 # If this target does not support named sections skip this test.
 
 proc dg-require-named-sections { args } {
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 5fbdb740ac6..b2096723426 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -548,7 +548,8 @@ proc check_effective_target_keeps_null_pointer_checks { } {
     if [target_info exists keeps_null_pointer_checks] {
       return 1
     }
-    if { [istarget avr-*-*] } {
+    if { [istarget avr-*-*]
+	 || [istarget msp430-*-*] } {
 	return 1;   
     }
     return 0
@@ -3296,7 +3297,8 @@ proc check_effective_target_vect_peeling_profitable { } {
     } else {
 	set et_vect_peeling_profitable_saved($et_index) 1
         if { ([istarget s390*-*-*]
-	      && [check_effective_target_s390_vx]) } {
+	      && [check_effective_target_s390_vx])
+	     || [check_effective_target_vect_element_align_preferred] } {
 	    set et_vect_peeling_profitable_saved($et_index) 0
         }
     }
@@ -3367,12 +3369,8 @@ proc check_effective_target_aarch64_sve { } {
     }]
 }
 
-# If targetting AArch64 SVE, return the size in bits of an SVE vector,
-# or -1 if the size is variable.  Return 0 if not targetting AArch64 SVE.
+# Return the size in bits of an SVE vector, or 0 if the size is variable.
 proc aarch64_sve_bits { } {
-    if { ![check_effective_target_aarch64_sve] } {
-	return 0
-    }
     return [check_cached_effective_target aarch64_sve_bits {
 	global tool
 
@@ -3388,25 +3386,6 @@ proc aarch64_sve_bits { } {
     }]
 }
 
-# Return true if targetting AArch64 SVE and if the target system's
-# vectors have exactly BITS bits.
-proc aarch64_sve_hw_bits { bits } {
-    if { ![check_effective_target_aarch64_sve_hw] } {
-	return 0
-    }
-    return [check_runtime aarch64_sve${bits}_hw [subst {
-	int
-	main (void)
-	{
-	  int res;
-	  asm volatile ("cntd %0" : "=r" (res));
-	  if (res * 64 != $bits)
-	    __builtin_abort ();
-	  return 0;
-	}
-    }]]
-}
-
 # Return 1 if this is a compiler supporting ARC atomic operations
 proc check_effective_target_arc_atomic { } {
     return [check_no_compiler_messages arc_atomic assembly {
@@ -4332,18 +4311,45 @@ proc check_effective_target_arm_neon_hw { } {
     } [add_options_for_arm_neon ""]]
 }
 
+# Return true if this is an AArch64 target that can run SVE code.
+
 proc check_effective_target_aarch64_sve_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+	return 0
+    }
     return [check_runtime aarch64_sve_hw_available {
 	int
 	main (void)
 	{
-	  unsigned long res;
 	  asm volatile ("ptrue p0.b");
 	  return 0;
 	}
     }]
 }
 
+# Return true if this is an AArch64 target that can run SVE code and
+# if its SVE vectors have exactly BITS bits.
+
+proc aarch64_sve_hw_bits { bits } {
+    if { ![check_effective_target_aarch64_sve_hw] } {
+	return 0
+    }
+    return [check_runtime aarch64_sve${bits}_hw [subst {
+	int
+	main (void)
+	{
+	  int res;
+	  asm volatile ("cntd %0" : "=r" (res));
+	  if (res * 64 != $bits)
+	    __builtin_abort ();
+	  return 0;
+	}
+    }]]
+}
+
+# Return true if this is an AArch64 target that can run SVE code and
+# if its SVE vectors have exactly 256 bits.
+
 proc check_effective_target_aarch64_sve256_hw { } {
     return [aarch64_sve_hw_bits 256]
 }
@@ -4470,6 +4476,48 @@ proc check_effective_target_arm_v8_2a_fp16_neon_ok { } {
 		check_effective_target_arm_v8_2a_fp16_neon_ok_nocache]
 }
 
+# Return 1 if the target supports ARMv8.2 Adv.SIMD Dot Product
+# instructions, 0 otherwise.  The test is valid for ARM and for AArch64.
+# Record the command line options needed.
+
+proc check_effective_target_arm_v8_2a_dotprod_neon_ok_nocache { } {
+    global et_arm_v8_2a_dotprod_neon_flags
+    set et_arm_v8_2a_dotprod_neon_flags ""
+
+    if { ![istarget arm*-*-*] && ![istarget aarch64*-*-*] } {
+        return 0;
+    }
+
+    # Iterate through sets of options to find the compiler flags that
+    # need to be added to the -march option.
+    foreach flags {"" "-mfloat-abi=softfp -mfpu=neon-fp-armv8" "-mfloat-abi=hard -mfpu=neon-fp-armv8"} {
+        if { [check_no_compiler_messages_nocache \
+                  arm_v8_2a_dotprod_neon_ok object {
+            #if !defined (__ARM_FEATURE_DOTPROD)
+            #error "__ARM_FEATURE_DOTPROD not defined"
+            #endif
+        } "$flags -march=armv8.2-a+dotprod"] } {
+            set et_arm_v8_2a_dotprod_neon_flags "$flags -march=armv8.2-a+dotprod"
+            return 1
+        }
+    }
+
+    return 0;
+}
+
+proc check_effective_target_arm_v8_2a_dotprod_neon_ok { } {
+    return [check_cached_effective_target arm_v8_2a_dotprod_neon_ok \
+                check_effective_target_arm_v8_2a_dotprod_neon_ok_nocache]
+}
+
+proc add_options_for_arm_v8_2a_dotprod_neon { flags } {
+    if { ! [check_effective_target_arm_v8_2a_dotprod_neon_ok] } {
+        return "$flags"
+    }
+    global et_arm_v8_2a_dotprod_neon_flags
+    return "$flags $et_arm_v8_2a_dotprod_neon_flags"
+}
+
 # Return 1 if the target supports executing ARMv8 NEON instructions, 0
 # otherwise.
 
@@ -4607,6 +4655,42 @@ proc check_effective_target_arm_v8_2a_fp16_neon_hw { } {
     } [add_options_for_arm_v8_2a_fp16_neon ""]]
 }
 
+# Return 1 if the target supports executing AdvSIMD instructions from ARMv8.2
+# with the Dot Product extension, 0 otherwise.  The test is valid for ARM and for
+# AArch64.
+
+proc check_effective_target_arm_v8_2a_dotprod_neon_hw { } {
+    if { ![check_effective_target_arm_v8_2a_dotprod_neon_ok] } {
+        return 0;
+    }
+    return [check_runtime arm_v8_2a_dotprod_neon_hw_available {
+        #include "arm_neon.h"
+        int
+        main (void)
+        {
+
+	  uint32x2_t results = {0,0};
+	  uint8x8_t a = {1,1,1,1,2,2,2,2};
+	  uint8x8_t b = {2,2,2,2,3,3,3,3};
+
+          #ifdef __ARM_ARCH_ISA_A64
+          asm ("udot %0.2s, %1.8b, %2.8b"
+               : "=w"(results)
+               : "w"(a), "w"(b)
+               : /* No clobbers.  */);
+
+	  #else
+          asm ("vudot.u8 %P0, %P1, %P2"
+               : "=w"(results)
+               : "w"(a), "w"(b)
+               : /* No clobbers.  */);
+          #endif
+
+          return (results[0] == 8 && results[1] == 24) ? 1 : 0;
+        }
+    } [add_options_for_arm_v8_2a_dotprod_neon ""]]
+}
+
 # Return 1 if this is a ARM target with NEON enabled.
 
 proc check_effective_target_arm_neon { } {
@@ -5544,11 +5628,6 @@ proc check_effective_target_vect_perm { } {
     return $et_vect_perm_saved($et_index)
 }
 
-proc check_effective_target_vect_any_perm { } {
-    return [expr { [check_effective_target_vect_perm]
-		   || [istarget aarch64*-*-*] }]
-}
-
 # Return 1 if, for some VF:
 #
 # - the target's default vector size is VF * ELEMENT_BITS bits
@@ -5558,22 +5637,42 @@ proc check_effective_target_vect_any_perm { } {
 #      int<ELEMENT_BITS>_t s1[COUNT][COUNT * VF], s2[COUNT * VF];
 #      for (int i = 0; i < COUNT; ++i)
 #        for (int j = 0; j < COUNT * VF; ++j)
-#          s1[i][j] = s2[J - j % COUNT + i % COUNT]
+#          s1[i][j] = s2[j - j % COUNT + i]
 #
 #   using only a single 2-vector permute for each vector in s1.
 #
 # E.g. for COUNT == 3 and vector length 4, the two arrays would be:
 #
-#  s2    | a0 a1 a2 a3 | b0 b1 b2 b3 | c0 c1 c2 c3
-#  ------+-------------+-------------+------------
-#  s1[0] | a0 a0 a0 a3 | a3 a3 b2 b2 | b2 c1 c1 c1
-#  s1[1] | a1 a1 a1 b0 | b0 b0 b3 b3 | b3 c2 c2 c3
-#  s1[2] | a2 a2 a2 b1 | b1 b1 c0 c0 | c0 c3 c3 c3
+#    s2    | a0 a1 a2 a3 | b0 b1 b2 b3 | c0 c1 c2 c3
+#    ------+-------------+-------------+------------
+#    s1[0] | a0 a0 a0 a3 | a3 a3 b2 b2 | b2 c1 c1 c1
+#    s1[1] | a1 a1 a1 b0 | b0 b0 b3 b3 | b3 c2 c2 c2
+#    s1[2] | a2 a2 a2 b1 | b1 b1 c0 c0 | c0 c3 c3 c3
 #
 # Each s1 permute requires only two of a, b and c.
 #
-# In general, this is possible for a VF if VF <= COUNT or if
-# (VF - gcd (VF, COUNT)) is a multiple of COUNT.
+# The distance between the start of vector n in s1[0] and the start
+# of vector n in s2 is:
+#
+#    A = (n * VF) % COUNT
+#
+# The corresponding value for the end of vector n is:
+#
+#    B = (n * VF + VF - 1) % COUNT
+#
+# Subtracting i from each value gives the corresponding difference
+# for s1[i].  The condition being tested by this function is false
+# iff A - i > 0 and B - i < 0 for some i and n, such that the first
+# element for s1[i] comes from vector n - 1 of s2 and the last element
+# comes from vector n + 1 of s2.  The condition is therefore true iff
+# A <= B for all n.  This is turn means the condition is true iff:
+#
+#    (n * VF) % COUNT + (VF - 1) % COUNT < COUNT
+#
+# for all n.  COUNT - (n * VF) % COUNT is bounded by gcd (VF, COUNT),
+# and will be that value for at least one n in [0, COUNT), so we want:
+#
+#    (VF - 1) % COUNT < gcd (VF, COUNT)
 
 proc vect_perm_supported { count element_bits } {
     set vector_bits [lindex [available_vector_sizes] 0]
@@ -5581,10 +5680,16 @@ proc vect_perm_supported { count element_bits } {
 	return 0
     }
     set vf [expr { $vector_bits / $element_bits }]
-    # Since VF is a power of 2, gcd (VF, COUNT) == (COUNT & -COUNT)
-    # when COUNT < VF.
-    return [expr { $vf <= $count
-		   || $vf % $count == ($count & -$count) }]
+
+    # Compute gcd (VF, COUNT).
+    set gcd $vf
+    set temp1 $count
+    while { $temp1 > 0 } {
+	set temp2 [expr { $gcd % $temp1 }]
+	set gcd $temp1
+	set temp1 $temp2
+    }
+    return [expr { ($vf - 1) % $count < $gcd }]
 }
 
 # Return 1 if the target supports SLP permutation of 3 vectors when each
@@ -6005,6 +6110,8 @@ proc check_effective_target_vect_sdot_qi { } {
     } else {
 	set et_vect_sdot_qi_saved($et_index) 0
 	if { [istarget ia64-*-*]
+	     || [istarget aarch64*-*-*]
+	     || [istarget arm*-*-*]
 	     || ([istarget mips*-*-*]
 		 && [et-is-effective-target mips_msa]) } {
             set et_vect_udot_qi_saved 1
@@ -6029,6 +6136,8 @@ proc check_effective_target_vect_udot_qi { } {
     } else {
 	set et_vect_udot_qi_saved($et_index) 0
         if { [istarget powerpc*-*-*]
+	     || [istarget aarch64*-*-*]
+	     || [istarget arm*-*-*]
 	     || [istarget ia64-*-*]
 	     || ([istarget mips*-*-*]
 		 && [et-is-effective-target mips_msa]) } {
@@ -6354,9 +6463,8 @@ proc check_effective_target_vect_align_stack_vars { } {
 
 proc check_effective_target_vector_alignment_reachable { } {
     set et_vector_alignment_reachable 0
-    if { ![check_effective_target_vect_element_align_preferred]
-	 && ([check_effective_target_vect_aligned_arrays]
-	     || [check_effective_target_natural_alignment_32]) } {
+    if { [check_effective_target_vect_aligned_arrays]
+	 || [check_effective_target_natural_alignment_32] } {
 	set et_vector_alignment_reachable 1
     }
     verbose "check_effective_target_vector_alignment_reachable:\
@@ -6368,9 +6476,8 @@ proc check_effective_target_vector_alignment_reachable { } {
 
 proc check_effective_target_vector_alignment_reachable_for_64bit { } {
     set et_vector_alignment_reachable_for_64bit 0
-    if { ![check_effective_target_vect_element_align_preferred]
-	 && ([check_effective_target_vect_aligned_arrays] 
-	     || [check_effective_target_natural_alignment_64]) } {
+    if { [check_effective_target_vect_aligned_arrays] 
+	 || [check_effective_target_natural_alignment_64] } {
 	set et_vector_alignment_reachable_for_64bit 1
     }
     verbose "check_effective_target_vector_alignment_reachable_for_64bit:\
@@ -6407,7 +6514,7 @@ proc check_effective_target_vect_element_align { } {
 proc check_effective_target_vect_unaligned_possible { } {
     return [expr { ![check_effective_target_vect_element_align_preferred]
 		   && (![check_effective_target_vect_no_align]
-		       || [check_effective_target vect_hw_misalign]) }]
+		       || [check_effective_target_vect_hw_misalign]) }]
 }
 
 # Return 1 if the target supports vector LOAD_LANES operations, 0 otherwise.
@@ -6697,19 +6804,6 @@ foreach N {2 3 4 8} {
 # Return the list of vector sizes (in bits) that each target supports.
 # A vector length of "0" indicates variable-length vectors.
 
-proc check_effective_target_vect_multiple_sizes { } {
-    global et_vect_multiple_sizes_saved
-    global et_index
-
-    set et_vect_multiple_sizes_saved($et_index) 0
-    if { [istarget aarch64*-*-*]
-	 || [is-effective-target arm_neon]
-	 || (([istarget i?86-*-*] || [istarget x86_64-*-*])
-	     && ([check_avx_available] && ![check_prefer_avx128])) } {
-	set et_vect_multiple_sizes_saved($et_index) 1
-    }
-}
-
 proc available_vector_sizes { } {
     set result {}
     if { [istarget aarch64*-*-*] } {
@@ -6732,28 +6826,22 @@ proc available_vector_sizes { } {
     return $result
 }
 
-# Return true if variable-length vectors are supported.
-
-proc check_effective_target_vect_variable_length { } {
-    return [expr { [lindex [available_vector_sizes] 0] == 0 }]
-}
-
-# Return true if exactly 3 distinct vector sizes are supported.
+# Return 1 if the target supports multiple vector sizes
 
-proc check_effective_target_vect_3_sizes { } {
-    return [expr { [llength [available_vector_sizes]] == 3 }]
+proc check_effective_target_vect_multiple_sizes { } {
+    return [expr { [llength [available_vector_sizes]] > 1 }]
 }
 
-# Return true if exactly 2 distinct vector sizes are supported.
+# Return true if variable-length vectors are supported.
 
-proc check_effective_target_vect_2_sizes { } {
-    return [expr { [llength [available_vector_sizes]] == 2 }]
+proc check_effective_target_vect_variable_length { } {
+    return [expr { [lindex [available_vector_sizes] 0] == 0 }]
 }
 
-# Return true if exactly 1 distinct vector size is supported.
+# Return 1 if the target supports vectors of 64 bits.
 
-proc check_effective_target_vect_1_size { } {
-    return [expr { [llength [available_vector_sizes]] == 1 }]
+proc check_effective_target_vect64 { } {
+    return [expr { [lsearch -exact [available_vector_sizes] 64] >= 0 }]
 }
 
 # Return 1 if the target supports vectors of 256 bits.
@@ -6762,12 +6850,6 @@ proc check_effective_target_vect256 { } {
     return [expr { [lsearch -exact [available_vector_sizes] 256] >= 0 }]
 }
 
-# Return 1 if the target supports vectors of 64 bits.
-
-proc check_effective_target_vect64 { } {
-    return [expr { [lsearch -exact [available_vector_sizes] 64] >= 0 }]
-}
-
 # Return 1 if the target supports vector copysignf calls.
 
 proc check_effective_target_vect_call_copysignf { } {
@@ -8552,7 +8634,7 @@ proc check_effective_target_aarch64_tiny { } {
 # Create functions to check that the AArch64 assembler supports the
 # various architecture extensions via the .arch_extension pseudo-op.
 
-foreach { aarch64_ext } { "fp" "simd" "crypto" "crc" "lse"} {
+foreach { aarch64_ext } { "fp" "simd" "crypto" "crc" "lse" "dotprod"} {
     eval [string map [list FUNC $aarch64_ext] {
 	proc check_effective_target_aarch64_asm_FUNC_ok { } {
 	  if { [istarget aarch64*-*-*] } {
@@ -9111,14 +9193,9 @@ proc check_effective_target_autoincdec { } {
 # 
 proc check_effective_target_supports_stack_clash_protection { } {
 
-   # Temporary until the target bits are fully ACK'd.
-#  if { [istarget aarch*-*-*] } {
-#	return 1
-#  }
-
     if { [istarget x86_64-*-*] || [istarget i?86-*-*] 
 	  || [istarget powerpc*-*-*] || [istarget rs6000*-*-*]
-	  || [istarget s390*-*-*] } {
+	  || [istarget aarch64*-**] || [istarget s390*-*-*] } {
 	return 1
     }
   return 0
@@ -9185,3 +9262,16 @@ proc check_effective_target_callee_realigns_stack { } {
   }
   return 0
 }
+
+# Return 1 if CET instructions can be compiled.
+proc check_effective_target_cet { } {
+    if { !([istarget i?86-*-*] || [istarget x86_64-*-*]) } {
+	return 0
+    }
+    return [check_no_compiler_messages cet object {
+	void foo (void)
+	{
+	  asm ("setssbsy");
+	}
+    } "-O2" ]
+}
diff --git a/gcc/toplev.c b/gcc/toplev.c
index 48d580c3ab0..e5292d4b314 100644
--- a/gcc/toplev.c
+++ b/gcc/toplev.c
@@ -88,8 +88,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "dbxout.h"
 #endif
 
-#include "sdbout.h"
-
 #ifdef XCOFF_DEBUGGING_INFO
 #include "xcoffout.h"		/* Needed for external data declarations. */
 #endif
@@ -958,7 +956,7 @@ output_stack_usage (void)
   stack_usage_kind = STATIC;
 
   /* Add the maximum amount of space pushed onto the stack.  */
-  if (maybe_nonzero (current_function_pushed_stack_size))
+  if (may_ne (current_function_pushed_stack_size, 0))
     {
       HOST_WIDE_INT extra;
       if (current_function_pushed_stack_size.is_constant (&extra))
@@ -1290,6 +1288,32 @@ process_options (void)
 	   "-floop-parallelize-all)");
 #endif
 
+  if (flag_cf_protection != CF_NONE
+      && !(flag_cf_protection & CF_SET))
+    {
+      if (flag_cf_protection == CF_FULL)
+	{
+	  error_at (UNKNOWN_LOCATION,
+		    "%<-fcf-protection=full%> is not supported for this "
+		    "target");
+	  flag_cf_protection = CF_NONE;
+	}
+      if (flag_cf_protection == CF_BRANCH)
+	{
+	  error_at (UNKNOWN_LOCATION,
+		    "%<-fcf-protection=branch%> is not supported for this "
+		    "target");
+	  flag_cf_protection = CF_NONE;
+	}
+      if (flag_cf_protection == CF_RETURN)
+	{
+	  error_at (UNKNOWN_LOCATION,
+		    "%<-fcf-protection=return%> is not supported for this "
+		    "target");
+	  flag_cf_protection = CF_NONE;
+	}
+    }
+
   if (flag_check_pointer_bounds)
     {
       if (targetm.chkp_bound_mode () == VOIDmode)
@@ -1454,8 +1478,6 @@ process_options (void)
   else if (write_symbols == XCOFF_DEBUG)
     debug_hooks = &xcoff_debug_hooks;
 #endif
-  else if (SDB_DEBUGGING_INFO && write_symbols == SDB_DEBUG)
-    debug_hooks = &sdb_debug_hooks;
 #ifdef DWARF2_DEBUGGING_INFO
   else if (write_symbols == DWARF2_DEBUG)
     debug_hooks = &dwarf2_debug_hooks;
diff --git a/gcc/tracer.c b/gcc/tracer.c
index dd071c1650c..58caf13b0de 100644
--- a/gcc/tracer.c
+++ b/gcc/tracer.c
@@ -132,9 +132,9 @@ count_insns (basic_block bb)
 static bool
 better_p (const_edge e1, const_edge e2)
 {
-  if (e1->count.initialized_p () && e2->count.initialized_p ()
-      && !(e1->count == e2->count))
-    return e1->count > e2->count;
+  if (e1->count ().initialized_p () && e2->count ().initialized_p ()
+      && ((e1->count () > e2->count ()) || (e1->count () < e2->count  ())))
+    return e1->count () > e2->count ();
   if (EDGE_FREQUENCY (e1) != EDGE_FREQUENCY (e2))
     return EDGE_FREQUENCY (e1) > EDGE_FREQUENCY (e2);
   /* This is needed to avoid changes in the decision after
@@ -179,7 +179,7 @@ find_best_predecessor (basic_block bb)
   if (!best || ignore_bb_p (best->src))
     return NULL;
   if (EDGE_FREQUENCY (best) * REG_BR_PROB_BASE
-      < bb->frequency * branch_ratio_cutoff)
+      < bb->count.to_frequency (cfun) * branch_ratio_cutoff)
     return NULL;
   return best;
 }
@@ -194,7 +194,7 @@ find_trace (basic_block bb, basic_block *trace)
   edge e;
 
   if (dump_file)
-    fprintf (dump_file, "Trace seed %i [%i]", bb->index, bb->frequency);
+    fprintf (dump_file, "Trace seed %i [%i]", bb->index, bb->count.to_frequency (cfun));
 
   while ((e = find_best_predecessor (bb)) != NULL)
     {
@@ -203,11 +203,11 @@ find_trace (basic_block bb, basic_block *trace)
 	  || find_best_successor (bb2) != e)
 	break;
       if (dump_file)
-	fprintf (dump_file, ",%i [%i]", bb->index, bb->frequency);
+	fprintf (dump_file, ",%i [%i]", bb->index, bb->count.to_frequency (cfun));
       bb = bb2;
     }
   if (dump_file)
-    fprintf (dump_file, " forward %i [%i]", bb->index, bb->frequency);
+    fprintf (dump_file, " forward %i [%i]", bb->index, bb->count.to_frequency (cfun));
   trace[i++] = bb;
 
   /* Follow the trace in forward direction.  */
@@ -218,7 +218,7 @@ find_trace (basic_block bb, basic_block *trace)
 	  || find_best_predecessor (bb) != e)
 	break;
       if (dump_file)
-	fprintf (dump_file, ",%i [%i]", bb->index, bb->frequency);
+	fprintf (dump_file, ",%i [%i]", bb->index, bb->count.to_frequency (cfun));
       trace[i++] = bb;
     }
   if (dump_file)
@@ -282,11 +282,11 @@ tail_duplicate (void)
     {
       int n = count_insns (bb);
       if (!ignore_bb_p (bb))
-	blocks[bb->index] = heap.insert (-bb->frequency, bb);
+	blocks[bb->index] = heap.insert (-bb->count.to_frequency (cfun), bb);
 
       counts [bb->index] = n;
       ninsns += n;
-      weighted_insns += n * bb->frequency;
+      weighted_insns += n * bb->count.to_frequency (cfun);
     }
 
   if (profile_info && profile_status_for_fn (cfun) == PROFILE_READ)
@@ -314,7 +314,7 @@ tail_duplicate (void)
       n = find_trace (bb, trace);
 
       bb = trace[0];
-      traced_insns += bb->frequency * counts [bb->index];
+      traced_insns += bb->count.to_frequency (cfun) * counts [bb->index];
       if (blocks[bb->index])
 	{
 	  heap.delete_node (blocks[bb->index]);
@@ -330,7 +330,7 @@ tail_duplicate (void)
 	      heap.delete_node (blocks[bb2->index]);
 	      blocks[bb2->index] = NULL;
 	    }
-	  traced_insns += bb2->frequency * counts [bb2->index];
+	  traced_insns += bb2->count.to_frequency (cfun) * counts [bb2->index];
 	  if (EDGE_COUNT (bb2->preds) > 1
 	      && can_duplicate_block_p (bb2)
 	      /* We have the tendency to duplicate the loop header
@@ -345,11 +345,11 @@ tail_duplicate (void)
 	      /* Reconsider the original copy of block we've duplicated.
 	         Removing the most common predecessor may make it to be
 	         head.  */
-	      blocks[bb2->index] = heap.insert (-bb2->frequency, bb2);
+	      blocks[bb2->index] = heap.insert (-bb2->count.to_frequency (cfun), bb2);
 
 	      if (dump_file)
 		fprintf (dump_file, "Duplicated %i as %i [%i]\n",
-			 bb2->index, copy->index, copy->frequency);
+			 bb2->index, copy->index, copy->count.to_frequency (cfun));
 
 	      bb2 = copy;
 	      changed = true;
diff --git a/gcc/trans-mem.c b/gcc/trans-mem.c
index 40b53681186..ef5655aa61a 100644
--- a/gcc/trans-mem.c
+++ b/gcc/trans-mem.c
@@ -2932,17 +2932,13 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED)
       edge ef = make_edge (test_bb, join_bb, EDGE_FALSE_VALUE);
       redirect_edge_pred (fallthru_edge, join_bb);
 
-      join_bb->frequency = test_bb->frequency = transaction_bb->frequency;
       join_bb->count = test_bb->count = transaction_bb->count;
 
       ei->probability = profile_probability::always ();
       et->probability = profile_probability::likely ();
       ef->probability = profile_probability::unlikely ();
-      et->count = test_bb->count.apply_probability (et->probability);
-      ef->count = test_bb->count.apply_probability (ef->probability);
 
-      code_bb->count = et->count;
-      code_bb->frequency = EDGE_FREQUENCY (et);
+      code_bb->count = et->count ();
 
       transaction_bb = join_bb;
     }
@@ -2966,7 +2962,6 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED)
       gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
 
       edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU);
-      test_bb->frequency = transaction_bb->frequency;
       test_bb->count = transaction_bb->count;
       ei->probability = profile_probability::always ();
 
@@ -2975,15 +2970,11 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED)
       redirect_edge_pred (fallthru_edge, test_bb);
       fallthru_edge->flags = EDGE_FALSE_VALUE;
       fallthru_edge->probability = profile_probability::very_likely ();
-      fallthru_edge->count = test_bb->count.apply_probability
-				(fallthru_edge->probability);
 
       // Abort/over edge.
       redirect_edge_pred (abort_edge, test_bb);
       abort_edge->flags = EDGE_TRUE_VALUE;
       abort_edge->probability = profile_probability::unlikely ();
-      abort_edge->count = test_bb->count.apply_probability
-				(abort_edge->probability);
 
       transaction_bb = test_bb;
     }
@@ -3011,8 +3002,7 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED)
       // out of the fallthru edge.
       edge e = make_edge (transaction_bb, test_bb, fallthru_edge->flags);
       e->probability = fallthru_edge->probability;
-      test_bb->count = e->count = fallthru_edge->count;
-      test_bb->frequency = EDGE_FREQUENCY (e);
+      test_bb->count = fallthru_edge->count ();
 
       // Now update the edges to the inst/uninist implementations.
       // For now assume that the paths are equally likely.  When using HTM,
@@ -3022,14 +3012,10 @@ expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED)
       redirect_edge_pred (inst_edge, test_bb);
       inst_edge->flags = EDGE_FALSE_VALUE;
       inst_edge->probability = profile_probability::even ();
-      inst_edge->count
-	= test_bb->count.apply_probability (inst_edge->probability);
 
       redirect_edge_pred (uninst_edge, test_bb);
       uninst_edge->flags = EDGE_TRUE_VALUE;
       uninst_edge->probability = profile_probability::even ();
-      uninst_edge->count
-	= test_bb->count.apply_probability (uninst_edge->probability);
     }
 
   // If we have no previous special cases, and we have PHIs at the beginning
@@ -3214,10 +3200,7 @@ split_bb_make_tm_edge (gimple *stmt, basic_block dest_bb,
     }
   edge e = make_edge (bb, dest_bb, EDGE_ABNORMAL);
   if (e)
-    {
-      e->probability = profile_probability::guessed_never ();
-      e->count = profile_count::guessed_zero ();
-    }
+    e->probability = profile_probability::guessed_never ();
 
   // Record the need for the edge for the benefit of the rtl passes.
   if (cfun->gimple_df->tm_restart == NULL)
diff --git a/gcc/tree-affine.c b/gcc/tree-affine.c
index 092b1e017af..5e7aef1113a 100644
--- a/gcc/tree-affine.c
+++ b/gcc/tree-affine.c
@@ -815,16 +815,16 @@ wide_int_constant_multiple_p (const poly_widest_int &val,
 {
   poly_widest_int rem, cst;
 
-  if (known_zero (val))
+  if (must_eq (val, 0))
     {
-      if (*mult_set && maybe_nonzero (*mult))
+      if (*mult_set && may_ne (*mult, 0))
 	return false;
       *mult_set = true;
       *mult = 0;
       return true;
     }
 
-  if (maybe_zero (div))
+  if (may_eq (div, 0))
     return false;
 
   if (!multiple_p (val, div, &cst))
@@ -848,7 +848,7 @@ aff_combination_constant_multiple_p (aff_tree *val, aff_tree *div,
   bool mult_set = false;
   unsigned i;
 
-  if (val->n == 0 && known_zero (val->offset))
+  if (val->n == 0 && must_eq (val->offset, 0))
     {
       *mult = 0;
       return true;
diff --git a/gcc/tree-affine.h b/gcc/tree-affine.h
index c08d4e5fc6b..0acf47410a7 100644
--- a/gcc/tree-affine.h
+++ b/gcc/tree-affine.h
@@ -102,7 +102,7 @@ aff_combination_zero_p (aff_tree *aff)
   if (!aff)
     return true;
 
-  if (aff->n == 0 && known_zero (aff->offset))
+  if (aff->n == 0 && must_eq (aff->offset, 0))
     return true;
 
   return false;
@@ -121,7 +121,7 @@ inline bool
 aff_combination_singleton_var_p (aff_tree *aff)
 {
   return (aff->n == 1
-	  && known_zero (aff->offset)
+	  && must_eq (aff->offset, 0)
 	  && (aff->elts[0].coef == 1 || aff->elts[0].coef == -1));
 }
 #endif /* GCC_TREE_AFFINE_H */
diff --git a/gcc/tree-call-cdce.c b/gcc/tree-call-cdce.c
index 1578350c0c6..02c89cce62f 100644
--- a/gcc/tree-call-cdce.c
+++ b/gcc/tree-call-cdce.c
@@ -314,6 +314,7 @@ can_test_argument_range (gcall *call)
     CASE_FLT_FN (BUILT_IN_POW10):
     /* Sqrt.  */
     CASE_FLT_FN (BUILT_IN_SQRT):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_SQRT):
       return check_builtin_call (call);
     /* Special one: two argument pow.  */
     case BUILT_IN_POW:
@@ -342,6 +343,7 @@ edom_only_function (gcall *call)
     CASE_FLT_FN (BUILT_IN_SIGNIFICAND):
     CASE_FLT_FN (BUILT_IN_SIN):
     CASE_FLT_FN (BUILT_IN_SQRT):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_SQRT):
     CASE_FLT_FN (BUILT_IN_FMOD):
     CASE_FLT_FN (BUILT_IN_REMAINDER):
       return true;
@@ -703,6 +705,7 @@ get_no_error_domain (enum built_in_function fnc)
                          308, true, false);
     /* sqrt: [0, +inf)  */
     CASE_FLT_FN (BUILT_IN_SQRT):
+    CASE_FLT_FN_FLOATN_NX (BUILT_IN_SQRT):
       return get_domain (0, true, true,
                          0, false, false);
     default:
@@ -903,7 +906,6 @@ shrink_wrap_one_built_in_call_with_conds (gcall *bi_call, vec <gimple *> conds,
      Here we take the second approach because it's slightly simpler
      and because it's easy to see that it doesn't lose profile counts.  */
   bi_call_bb->count = profile_count::zero ();
-  bi_call_bb->frequency = 0;
   while (!edges.is_empty ())
     {
       edge_pair e = edges.pop ();
@@ -913,23 +915,13 @@ shrink_wrap_one_built_in_call_with_conds (gcall *bi_call, vec <gimple *> conds,
       gcc_assert (src_bb == nocall_edge->src);
 
       call_edge->probability = profile_probability::very_unlikely ();
-      call_edge->count
-	 = src_bb->count.apply_probability (call_edge->probability);
       nocall_edge->probability = profile_probability::always ()
 				 - call_edge->probability;
-      nocall_edge->count = src_bb->count - call_edge->count;
 
-      unsigned int call_frequency
-	 = call_edge->probability.apply (src_bb->frequency);
-
-      bi_call_bb->count += call_edge->count;
-      bi_call_bb->frequency += call_frequency;
+      bi_call_bb->count += call_edge->count ();
 
       if (nocall_edge->dest != join_tgt_bb)
-	{
-	  nocall_edge->dest->count = nocall_edge->count;
-	  nocall_edge->dest->frequency = src_bb->frequency - call_frequency;
-	}
+	nocall_edge->dest->count = src_bb->count - bi_call_bb->count;
     }
 
   if (dom_info_available_p (CDI_DOMINATORS))
diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c
index 53978fbafa1..105e5a1dde7 100644
--- a/gcc/tree-cfg.c
+++ b/gcc/tree-cfg.c
@@ -1062,8 +1062,8 @@ gimple_find_sub_bbs (gimple_seq seq, gimple_stmt_iterator *gsi)
       edge_iterator ei;
       FOR_EACH_EDGE (e, ei, bb->preds)
 	{
-	  if (e->count.initialized_p ())
-	    cnt += e->count;
+	  if (e->count ().initialized_p ())
+	    cnt += e->count ();
 	  else
 	    all = false;
 	  freq += EDGE_FREQUENCY (e);
@@ -1071,9 +1071,6 @@ gimple_find_sub_bbs (gimple_seq seq, gimple_stmt_iterator *gsi)
       tree_guess_outgoing_edge_probabilities (bb);
       if (all || profile_status_for_fn (cfun) == PROFILE_READ)
         bb->count = cnt;
-      bb->frequency = freq;
-      FOR_EACH_EDGE (e, ei, bb->succs)
-	e->count = bb->count.apply_probability (e->probability);
 
       bb = bb->next_bb;
     }
@@ -2083,7 +2080,6 @@ gimple_merge_blocks (basic_block a, basic_block b)
   if (a->loop_father == b->loop_father)
     {
       a->count = a->count.merge (b->count);
-      a->frequency = MAX (a->frequency, b->frequency);
     }
 
   /* Merge the sequences.  */
@@ -2842,8 +2838,7 @@ gimple_split_edge (edge edge_in)
   after_bb = split_edge_bb_loc (edge_in);
 
   new_bb = create_empty_bb (after_bb);
-  new_bb->frequency = EDGE_FREQUENCY (edge_in);
-  new_bb->count = edge_in->count;
+  new_bb->count = edge_in->count ();
 
   e = redirect_edge_and_branch (edge_in, new_bb);
   gcc_assert (e == edge_in);
@@ -6342,9 +6337,8 @@ gimple_duplicate_sese_region (edge entry, edge exit,
   bool free_region_copy = false, copying_header = false;
   struct loop *loop = entry->dest->loop_father;
   edge exit_copy;
-  vec<basic_block> doms;
+  vec<basic_block> doms = vNULL;
   edge redirected;
-  int total_freq = 0, entry_freq = 0;
   profile_count total_count = profile_count::uninitialized ();
   profile_count entry_count = profile_count::uninitialized ();
 
@@ -6406,27 +6400,16 @@ gimple_duplicate_sese_region (edge entry, edge exit,
   if (entry->dest->count.initialized_p ())
     {
       total_count = entry->dest->count;
-      entry_count = entry->count;
+      entry_count = entry->count ();
       /* Fix up corner cases, to avoid division by zero or creation of negative
 	 frequencies.  */
       if (entry_count > total_count)
 	entry_count = total_count;
     }
-  if (!(total_count > 0) || !(entry_count > 0))
-    {
-      total_freq = entry->dest->frequency;
-      entry_freq = EDGE_FREQUENCY (entry);
-      /* Fix up corner cases, to avoid division by zero or creation of negative
-	 frequencies.  */
-      if (total_freq == 0)
-	total_freq = 1;
-      else if (entry_freq > total_freq)
-	entry_freq = total_freq;
-    }
 
   copy_bbs (region, n_region, region_copy, &exit, 1, &exit_copy, loop,
 	    split_edge_bb_loc (entry), update_dominance);
-  if (total_count > 0 && entry_count > 0)
+  if (total_count.initialized_p () && entry_count.initialized_p ())
     {
       scale_bbs_frequencies_profile_count (region, n_region,
 				           total_count - entry_count,
@@ -6434,12 +6417,6 @@ gimple_duplicate_sese_region (edge entry, edge exit,
       scale_bbs_frequencies_profile_count (region_copy, n_region, entry_count,
 				           total_count);
     }
-  else
-    {
-      scale_bbs_frequencies_int (region, n_region, total_freq - entry_freq,
-				 total_freq);
-      scale_bbs_frequencies_int (region_copy, n_region, entry_freq, total_freq);
-    }
 
   if (copying_header)
     {
@@ -6528,7 +6505,6 @@ gimple_duplicate_sese_tail (edge entry, edge exit,
   struct loop *orig_loop = entry->dest->loop_father;
   basic_block switch_bb, entry_bb, nentry_bb;
   vec<basic_block> doms;
-  int total_freq = 0, exit_freq = 0;
   profile_count total_count = profile_count::uninitialized (),
 		exit_count = profile_count::uninitialized ();
   edge exits[2], nexits[2], e;
@@ -6573,30 +6549,16 @@ gimple_duplicate_sese_tail (edge entry, edge exit,
      inside.  */
   doms = get_dominated_by_region (CDI_DOMINATORS, region, n_region);
 
-  if (exit->src->count > 0)
-    {
-      total_count = exit->src->count;
-      exit_count = exit->count;
-      /* Fix up corner cases, to avoid division by zero or creation of negative
-	 frequencies.  */
-      if (exit_count > total_count)
-	exit_count = total_count;
-    }
-  else
-    {
-      total_freq = exit->src->frequency;
-      exit_freq = EDGE_FREQUENCY (exit);
-      /* Fix up corner cases, to avoid division by zero or creation of negative
-	 frequencies.  */
-      if (total_freq == 0)
-	total_freq = 1;
-      if (exit_freq > total_freq)
-	exit_freq = total_freq;
-    }
+  total_count = exit->src->count;
+  exit_count = exit->count ();
+  /* Fix up corner cases, to avoid division by zero or creation of negative
+     frequencies.  */
+  if (exit_count > total_count)
+    exit_count = total_count;
 
   copy_bbs (region, n_region, region_copy, exits, 2, nexits, orig_loop,
 	    split_edge_bb_loc (exit), true);
-  if (total_count.initialized_p ())
+  if (total_count.initialized_p () && exit_count.initialized_p ())
     {
       scale_bbs_frequencies_profile_count (region, n_region,
 				           total_count - exit_count,
@@ -6604,12 +6566,6 @@ gimple_duplicate_sese_tail (edge entry, edge exit,
       scale_bbs_frequencies_profile_count (region_copy, n_region, exit_count,
 				           total_count);
     }
-  else
-    {
-      scale_bbs_frequencies_int (region, n_region, total_freq - exit_freq,
-				 total_freq);
-      scale_bbs_frequencies_int (region_copy, n_region, exit_freq, total_freq);
-    }
 
   /* Create the switch block, and put the exit condition to it.  */
   entry_bb = entry->dest;
@@ -6631,10 +6587,8 @@ gimple_duplicate_sese_tail (edge entry, edge exit,
   sorig = single_succ_edge (switch_bb);
   sorig->flags = exits[1]->flags;
   sorig->probability = exits[1]->probability;
-  sorig->count = exits[1]->count;
   snew = make_edge (switch_bb, nentry_bb, exits[0]->flags);
   snew->probability = exits[0]->probability;
-  snew->count = exits[1]->count;
   
 
   /* Register the new edge from SWITCH_BB in loop exit lists.  */
@@ -7652,9 +7606,15 @@ move_sese_region_to_fn (struct function *dest_cfun, basic_block entry_bb,
      FIXME, this is silly.  The CFG ought to become a parameter to
      these helpers.  */
   push_cfun (dest_cfun);
-  make_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), entry_bb, EDGE_FALLTHRU);
+  ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = entry_bb->count;
+  make_single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), entry_bb, EDGE_FALLTHRU);
   if (exit_bb)
-    make_edge (exit_bb,  EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
+    {
+      make_single_succ_edge (exit_bb,  EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
+      EXIT_BLOCK_PTR_FOR_FN (cfun)->count = exit_bb->count;
+    }
+  else
+    EXIT_BLOCK_PTR_FOR_FN (cfun)->count = profile_count::zero ();
   pop_cfun ();
 
   /* Back in the original function, the SESE region has disappeared,
@@ -8369,7 +8329,6 @@ gimple_flow_call_edges_add (sbitmap blocks)
 		    }
 		  e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE);
 		  e->probability = profile_probability::guessed_never ();
-		  e->count = profile_count::guessed_zero ();
 		}
 	      gsi_prev (&gsi);
 	    }
@@ -8730,7 +8689,7 @@ gimple_account_profile_record (basic_block bb, int after_pass,
       else if (profile_status_for_fn (cfun) == PROFILE_GUESSED)
 	record->time[after_pass]
 	  += estimate_num_insns (gsi_stmt (i),
-				 &eni_time_weights) * bb->frequency;
+				 &eni_time_weights) * bb->count.to_frequency (cfun);
     }
 }
 
@@ -8881,14 +8840,11 @@ insert_cond_bb (basic_block bb, gimple *stmt, gimple *cond,
   new_bb = create_empty_bb (bb);
   edge e = make_edge (bb, new_bb, EDGE_TRUE_VALUE);
   e->probability = prob;
-  e->count = bb->count.apply_probability (prob);
-  new_bb->count = e->count;
-  new_bb->frequency = prob.apply (bb->frequency);
+  new_bb->count = e->count ();
   make_single_succ_edge (new_bb, fall->dest, EDGE_FALLTHRU);
 
   /* Fix edge for split bb.  */
   fall->flags = EDGE_FALSE_VALUE;
-  fall->count -= e->count;
   fall->probability -= e->probability;
 
   /* Update dominance info.  */
@@ -9118,13 +9074,29 @@ pass_warn_function_return::execute (function *fun)
       && EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (fun)->preds) > 0)
     {
       location = UNKNOWN_LOCATION;
-      FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (fun)->preds)
+      for (ei = ei_start (EXIT_BLOCK_PTR_FOR_FN (fun)->preds);
+	   (e = ei_safe_edge (ei)); )
 	{
 	  last = last_stmt (e->src);
 	  if ((gimple_code (last) == GIMPLE_RETURN
 	       || gimple_call_builtin_p (last, BUILT_IN_RETURN))
-	      && (location = gimple_location (last)) != UNKNOWN_LOCATION)
+	      && location == UNKNOWN_LOCATION
+	      && (location = gimple_location (last)) != UNKNOWN_LOCATION
+	      && !optimize)
 	    break;
+	  /* When optimizing, replace return stmts in noreturn functions
+	     with __builtin_unreachable () call.  */
+	  if (optimize && gimple_code (last) == GIMPLE_RETURN)
+	    {
+	      tree fndecl = builtin_decl_implicit (BUILT_IN_UNREACHABLE);
+	      gimple *new_stmt = gimple_build_call (fndecl, 0);
+	      gimple_set_location (new_stmt, gimple_location (last));
+	      gimple_stmt_iterator gsi = gsi_for_stmt (last);
+	      gsi_replace (&gsi, new_stmt, true);
+	      remove_edge (e);
+	    }
+	  else
+	    ei_next (&ei);
 	}
       if (location == UNKNOWN_LOCATION)
 	location = cfun->function_end_locus;
@@ -9286,23 +9258,18 @@ execute_fixup_cfg (void)
   basic_block bb;
   gimple_stmt_iterator gsi;
   int todo = 0;
-  edge e;
-  edge_iterator ei;
   cgraph_node *node = cgraph_node::get (current_function_decl);
   profile_count num = node->count;
   profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
-  bool scale = num.initialized_p ()
-	       && (den > 0 || num == profile_count::zero ())
-	       && !(num == den);
+  bool scale = num.initialized_p () && den.ipa_p ()
+	       && (den.nonzero_p () || num == profile_count::zero ())
+	       && !(num == den.ipa ());
 
   if (scale)
     {
       ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
       EXIT_BLOCK_PTR_FOR_FN (cfun)->count
         = EXIT_BLOCK_PTR_FOR_FN (cfun)->count.apply_scale (num, den);
-
-      FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs)
-	e->count = e->count.apply_scale (num, den);
     }
 
   FOR_EACH_BB_FN (bb, cfun)
@@ -9377,10 +9344,6 @@ execute_fixup_cfg (void)
 	  gsi_next (&gsi);
 	}
 
-      if (scale)
-	FOR_EACH_EDGE (e, ei, bb->succs)
-	  e->count = e->count.apply_scale (num, den);
-
       /* If we have a basic block with no successors that does not
 	 end with a control statement or a noreturn call end it with
 	 a call to __builtin_unreachable.  This situation can occur
diff --git a/gcc/tree-cfgcleanup.c b/gcc/tree-cfgcleanup.c
index 1a71c93aeed..9b7f08c586c 100644
--- a/gcc/tree-cfgcleanup.c
+++ b/gcc/tree-cfgcleanup.c
@@ -195,7 +195,6 @@ cleanup_control_expr_graph (basic_block bb, gimple_stmt_iterator gsi,
 		}
 
 	      taken_edge->probability += e->probability;
-	      taken_edge->count += e->count;
 	      remove_edge_and_dominated_blocks (e);
 	      retval = true;
 	    }
diff --git a/gcc/tree-chkp.c b/gcc/tree-chkp.c
index de18984f29c..f73db4000ce 100644
--- a/gcc/tree-chkp.c
+++ b/gcc/tree-chkp.c
@@ -2276,8 +2276,7 @@ chkp_build_returned_bound (gcall *call)
      it separately.  */
   if (fndecl
       && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
-      && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_ALLOCA
-	  || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_ALLOCA_WITH_ALIGN))
+      && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (fndecl)))
     {
       tree size = gimple_call_arg (call, 0);
       gimple_stmt_iterator iter = gsi_for_stmt (call);
diff --git a/gcc/tree-complex.c b/gcc/tree-complex.c
index d61047bbf5f..146b52bbd52 100644
--- a/gcc/tree-complex.c
+++ b/gcc/tree-complex.c
@@ -60,6 +60,11 @@ typedef int complex_lattice_t;
 
 #define PAIR(a, b)  ((a) << 2 | (b))
 
+class complex_propagate : public ssa_propagation_engine
+{
+  enum ssa_prop_result visit_stmt (gimple *, edge *, tree *) FINAL OVERRIDE;
+  enum ssa_prop_result visit_phi (gphi *) FINAL OVERRIDE;
+};
 
 static vec<complex_lattice_t> complex_lattice_values;
 
@@ -300,9 +305,9 @@ init_dont_simulate_again (void)
 
 /* Evaluate statement STMT against the complex lattice defined above.  */
 
-static enum ssa_prop_result
-complex_visit_stmt (gimple *stmt, edge *taken_edge_p ATTRIBUTE_UNUSED,
-		    tree *result_p)
+enum ssa_prop_result
+complex_propagate::visit_stmt (gimple *stmt, edge *taken_edge_p ATTRIBUTE_UNUSED,
+			       tree *result_p)
 {
   complex_lattice_t new_l, old_l, op1_l, op2_l;
   unsigned int ver;
@@ -395,8 +400,8 @@ complex_visit_stmt (gimple *stmt, edge *taken_edge_p ATTRIBUTE_UNUSED,
 
 /* Evaluate a PHI node against the complex lattice defined above.  */
 
-static enum ssa_prop_result
-complex_visit_phi (gphi *phi)
+enum ssa_prop_result
+complex_propagate::visit_phi (gphi *phi)
 {
   complex_lattice_t new_l, old_l;
   unsigned int ver;
@@ -1186,19 +1191,16 @@ expand_complex_div_wide (gimple_stmt_iterator *gsi, tree inner_type,
       bb_join = e->dest;
       bb_true = create_empty_bb (bb_cond);
       bb_false = create_empty_bb (bb_true);
-      bb_true->frequency = bb_false->frequency = bb_cond->frequency / 2;
       bb_true->count = bb_false->count
 	 = bb_cond->count.apply_probability (profile_probability::even ());
 
       /* Wire the blocks together.  */
       e->flags = EDGE_TRUE_VALUE;
-      e->count = bb_true->count;
       /* TODO: With value profile we could add an historgram to determine real
 	 branch outcome.  */
       e->probability = profile_probability::even ();
       redirect_edge_succ (e, bb_true);
       edge e2 = make_edge (bb_cond, bb_false, EDGE_FALSE_VALUE);
-      e2->count = bb_false->count;
       e2->probability = profile_probability::even ();
       make_single_succ_edge (bb_true, bb_join, EDGE_FALLTHRU);
       make_single_succ_edge (bb_false, bb_join, EDGE_FALLTHRU);
@@ -1675,7 +1677,8 @@ tree_lower_complex (void)
   complex_lattice_values.safe_grow_cleared (num_ssa_names);
 
   init_parameter_lattice_values ();
-  ssa_propagate (complex_visit_stmt, complex_visit_phi);
+  class complex_propagate complex_propagate;
+  complex_propagate.ssa_propagate ();
 
   complex_variable_components = new int_tree_htab_type (10);
 
diff --git a/gcc/tree-core.h b/gcc/tree-core.h
index 4e18dc650c5..bd85898a6f0 100644
--- a/gcc/tree-core.h
+++ b/gcc/tree-core.h
@@ -1544,7 +1544,6 @@ struct GTY(()) tree_type_common {
   tree reference_to;
   union tree_type_symtab {
     int GTY ((tag ("TYPE_SYMTAB_IS_ADDRESS"))) address;
-    const char * GTY ((tag ("TYPE_SYMTAB_IS_POINTER"))) pointer;
     struct die_struct * GTY ((tag ("TYPE_SYMTAB_IS_DIE"))) die;
   } GTY ((desc ("debug_hooks->tree_type_symtab_field"))) symtab;
   tree canonical;
@@ -2065,7 +2064,7 @@ struct floatn_type_info {
                                 Global variables
 ---------------------------------------------------------------------------*/
 /* Matrix describing the structures contained in a given tree code.  */
-extern unsigned char tree_contains_struct[MAX_TREE_CODES][64];
+extern bool tree_contains_struct[MAX_TREE_CODES][64];
 
 /* Class of tree given its code.  */
 extern const enum tree_code_class tree_code_type[];
diff --git a/gcc/tree-dump.c b/gcc/tree-dump.c
index ac0c7b868a1..d691278bbb2 100644
--- a/gcc/tree-dump.c
+++ b/gcc/tree-dump.c
@@ -337,7 +337,8 @@ dequeue_and_dump (dump_info_p di)
       /* All declarations have names.  */
       if (DECL_NAME (t))
 	dump_child ("name", DECL_NAME (t));
-      if (DECL_ASSEMBLER_NAME_SET_P (t)
+      if (HAS_DECL_ASSEMBLER_NAME_P (t)
+	  && DECL_ASSEMBLER_NAME_SET_P (t)
 	  && DECL_ASSEMBLER_NAME (t) != DECL_NAME (t))
 	dump_child ("mngl", DECL_ASSEMBLER_NAME (t));
       if (DECL_ABSTRACT_ORIGIN (t))
diff --git a/gcc/tree-eh.c b/gcc/tree-eh.c
index a1d35bace3a..21b2fa9c959 100644
--- a/gcc/tree-eh.c
+++ b/gcc/tree-eh.c
@@ -3223,6 +3223,7 @@ lower_resx (basic_block bb, gresx *stmt,
 	      gimple_stmt_iterator gsi2;
 
 	      new_bb = create_empty_bb (bb);
+	      new_bb->count = bb->count;
 	      add_bb_to_loop (new_bb, bb->loop_father);
 	      lab = gimple_block_label (new_bb);
 	      gsi2 = gsi_start_bb (new_bb);
@@ -3258,7 +3259,6 @@ lower_resx (basic_block bb, gresx *stmt,
 	  gcc_assert (e->flags & EDGE_EH);
 	  e->flags = (e->flags & ~EDGE_EH) | EDGE_FALLTHRU;
 	  e->probability = profile_probability::always ();
-	  e->count = bb->count;
 
 	  /* If there are no more EH users of the landing pad, delete it.  */
 	  FOR_EACH_EDGE (e, ei, e->dest->preds)
@@ -3779,7 +3779,10 @@ pass_lower_eh_dispatch::execute (function *fun)
     }
 
   if (redirected)
-    delete_unreachable_blocks ();
+    {
+      free_dominance_info (CDI_DOMINATORS);
+      delete_unreachable_blocks ();
+    }
   return flags;
 }
 
@@ -4098,7 +4101,6 @@ unsplit_eh (eh_landing_pad lp)
   redirect_edge_pred (e_out, e_in->src);
   e_out->flags = e_in->flags;
   e_out->probability = e_in->probability;
-  e_out->count = e_in->count;
   remove_edge (e_in);
 
   return true;
@@ -4291,7 +4293,6 @@ cleanup_empty_eh_move_lp (basic_block bb, edge e_out,
   /* Clean up E_OUT for the fallthru.  */
   e_out->flags = (e_out->flags & ~EDGE_EH) | EDGE_FALLTHRU;
   e_out->probability = profile_probability::always ();
-  e_out->count = e_out->src->count;
 }
 
 /* A subroutine of cleanup_empty_eh.  Handle more complex cases of
diff --git a/gcc/tree-if-conv.c b/gcc/tree-if-conv.c
index 2e32267bbc4..e5965b00168 100644
--- a/gcc/tree-if-conv.c
+++ b/gcc/tree-if-conv.c
@@ -2357,7 +2357,8 @@ predicate_statements (loop_p loop)
 	  gassign *stmt = dyn_cast <gassign *> (gsi_stmt (gsi));
 	  if (!stmt)
 	    ;
-	  else if (is_false_predicate (cond))
+	  else if (is_false_predicate (cond)
+		   && gimple_vdef (stmt))
 	    {
 	      unlink_stmt_vdef (stmt);
 	      gsi_remove (&gsi, true);
@@ -2386,10 +2387,7 @@ predicate_statements (loop_p loop)
 					 TREE_OPERAND (cond, 0),
 					 TREE_OPERAND (cond, 1));
 		  else
-		    {
-		      gcc_assert (TREE_CODE (cond) == SSA_NAME);
-		      mask = cond;
-		    }
+		    mask = cond;
 
 		  if (swap)
 		    {
diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 3d8a7f10bdb..9eac215e4dc 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -1763,16 +1763,15 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id)
    later  */
 
 static basic_block
-copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
+copy_bb (copy_body_data *id, basic_block bb,
          profile_count num, profile_count den)
 {
   gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
   basic_block copy_basic_block;
   tree decl;
-  gcov_type freq;
   basic_block prev;
-  bool scale = num.initialized_p ()
-	       && (den > 0 || num == profile_count::zero ());
+  bool scale = !num.initialized_p ()
+	       || (den.nonzero_p () || num == profile_count::zero ());
 
   /* Search for previous copied basic block.  */
   prev = bb->prev_bb;
@@ -1784,15 +1783,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
   copy_basic_block = create_basic_block (NULL, (basic_block) prev->aux);
   if (scale)
     copy_basic_block->count = bb->count.apply_scale (num, den);
-
-  /* We are going to rebuild frequencies from scratch.  These values
-     have just small importance to drive canonicalize_loop_headers.  */
-  freq = apply_scale ((gcov_type)bb->frequency, frequency_scale);
-
-  /* We recompute frequencies after inlining, so this is quite safe.  */
-  if (freq > BB_FREQ_MAX)
-    freq = BB_FREQ_MAX;
-  copy_basic_block->frequency = freq;
+  else if (num.initialized_p ())
+    copy_basic_block->count = bb->count;
 
   copy_gsi = gsi_start_bb (copy_basic_block);
 
@@ -2068,8 +2060,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 			      fprintf (dump_file,
 				       "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
 				       bb->index,
-				       bb->frequency,
-				       copy_basic_block->frequency);
+				       bb->count.to_frequency (cfun),
+				       copy_basic_block->count.to_frequency (cfun));
 			    }
 			}
 		    }
@@ -2215,7 +2207,7 @@ update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb,
    debug stmts are left after a statement that must end the basic block.  */
 
 static bool
-copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
+copy_edges_for_bb (basic_block bb,
 		   basic_block ret_bb, basic_block abnormal_goto_dest)
 {
   basic_block new_bb = (basic_block) bb->aux;
@@ -2224,8 +2216,6 @@ copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
   gimple_stmt_iterator si;
   int flags;
   bool need_debug_cleanup = false;
-  bool scale = num.initialized_p ()
-	       && (den > 0 || num == profile_count::zero ());
 
   /* Use the indices from the original blocks to create edges for the
      new ones.  */
@@ -2242,8 +2232,6 @@ copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
 	    && old_edge->dest->aux != EXIT_BLOCK_PTR_FOR_FN (cfun))
 	  flags |= EDGE_FALLTHRU;
 	new_edge = make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
-	if (scale)
-	  new_edge->count = old_edge->count.apply_scale (num, den);
 	new_edge->probability = old_edge->probability;
       }
 
@@ -2324,17 +2312,11 @@ copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
 		&& (e = find_edge (copy_stmt_bb,
 				   (basic_block) old_edge->dest->aux))
 		&& (e->flags & EDGE_EH))
-	      {
-		e->probability = old_edge->probability;
-		e->count = old_edge->count;
-	      }
+	      e->probability = old_edge->probability;
 	    
           FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs)
 	    if ((e->flags & EDGE_EH) && !e->probability.initialized_p ())
-	      {
-	        e->probability = profile_probability::never ();
-	        e->count = profile_count::zero ();
-	      }
+	      e->probability = profile_probability::never ();
         }
 
 
@@ -2517,11 +2499,8 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
 
   profile_status_for_fn (cfun) = profile_status_for_fn (src_cfun);
 
-  /* FIXME: When all counts are known to be zero, scaling is also meaningful.
-   */
   if (ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.initialized_p ()
-      && count.initialized_p ()
-      && ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.initialized_p ())
+      && count.ipa ().initialized_p ())
     {
       ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
 	ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (count,
@@ -2530,10 +2509,13 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
 	EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (count,
 				    ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
     }
-  ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency
-    = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->frequency;
-  EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency =
-    EXIT_BLOCK_PTR_FOR_FN (src_cfun)->frequency;
+  else
+    {
+      ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
+	   = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
+      EXIT_BLOCK_PTR_FOR_FN (cfun)->count
+	   = EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count;
+    }
   if (src_cfun->eh)
     init_eh_for_function ();
 
@@ -2690,33 +2672,11 @@ redirect_all_calls (copy_body_data * id, basic_block bb)
     }
 }
 
-/* Convert estimated frequencies into counts for NODE, scaling COUNT
-   with each bb's frequency. Used when NODE has a 0-weight entry
-   but we are about to inline it into a non-zero count call bb.
-   See the comments for handle_missing_profiles() in predict.c for
-   when this can happen for COMDATs.  */
-
-void
-freqs_to_counts (struct cgraph_node *node, profile_count count)
-{
-  basic_block bb;
-  edge_iterator ei;
-  edge e;
-  struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
-
-  FOR_ALL_BB_FN(bb, fn)
-    {
-      bb->count = count.apply_scale (bb->frequency, BB_FREQ_MAX);
-      FOR_EACH_EDGE (e, ei, bb->succs)
-        e->count = e->src->count.apply_probability (e->probability);
-    }
-}
-
 /* Make a copy of the body of FN so that it can be inserted inline in
    another function.  Walks FN via CFG, returns new fndecl.  */
 
 static tree
-copy_cfg_body (copy_body_data * id, profile_count count, int frequency_scale,
+copy_cfg_body (copy_body_data * id, profile_count,
 	       basic_block entry_block_map, basic_block exit_block_map,
 	       basic_block new_entry)
 {
@@ -2728,31 +2688,10 @@ copy_cfg_body (copy_body_data * id, profile_count count, int frequency_scale,
   tree new_fndecl = NULL;
   bool need_debug_cleanup = false;
   int last;
-  int incoming_frequency = 0;
-  profile_count incoming_count = profile_count::zero ();
-  profile_count num = count;
   profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
-  bool scale = num.initialized_p ()
-	       && (den > 0 || num == profile_count::zero ());
-
-  /* This can happen for COMDAT routines that end up with 0 counts
-     despite being called (see the comments for handle_missing_profiles()
-     in predict.c as to why). Apply counts to the blocks in the callee
-     before inlining, using the guessed edge frequencies, so that we don't
-     end up with a 0-count inline body which can confuse downstream
-     optimizations such as function splitting.  */
-  if (!(ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count > 0) && count > 0)
-    {
-      /* Apply the larger of the call bb count and the total incoming
-         call edge count to the callee.  */
-      profile_count in_count = profile_count::zero ();
-      struct cgraph_edge *in_edge;
-      for (in_edge = id->src_node->callers; in_edge;
-           in_edge = in_edge->next_caller)
-	if (in_edge->count.initialized_p ())
-          in_count += in_edge->count;
-      freqs_to_counts (id->src_node, count > in_count ? count : in_count);
-    }
+  profile_count num = entry_block_map->count;
+
+  cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();
@@ -2766,28 +2705,18 @@ copy_cfg_body (copy_body_data * id, profile_count count, int frequency_scale,
     {
       edge e;
       edge_iterator ei;
+      den = profile_count::zero ();
 
       FOR_EACH_EDGE (e, ei, new_entry->preds)
 	if (!e->src->aux)
-	  {
-	    incoming_frequency += EDGE_FREQUENCY (e);
-	    incoming_count += e->count;
-	  }
-      if (scale)
-        incoming_count = incoming_count.apply_scale (num, den);
-      else
-	incoming_count = profile_count::uninitialized ();
-      incoming_frequency
-	= apply_scale ((gcov_type)incoming_frequency, frequency_scale);
-      ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = incoming_count;
-      ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency = incoming_frequency;
+	  den += e->count ();
+      ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = den;
     }
 
   /* Must have a CFG here at this point.  */
   gcc_assert (ENTRY_BLOCK_PTR_FOR_FN
 	      (DECL_STRUCT_FUNCTION (callee_fndecl)));
 
-  cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
 
   ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = entry_block_map;
   EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = exit_block_map;
@@ -2803,7 +2732,7 @@ copy_cfg_body (copy_body_data * id, profile_count count, int frequency_scale,
   FOR_EACH_BB_FN (bb, cfun_to_copy)
     if (!id->blocks_to_copy || bitmap_bit_p (id->blocks_to_copy, bb->index))
       {
-	basic_block new_bb = copy_bb (id, bb, frequency_scale, num, den);
+	basic_block new_bb = copy_bb (id, bb, num, den);
 	bb->aux = new_bb;
 	new_bb->aux = bb;
 	new_bb->loop_father = entry_block_map->loop_father;
@@ -2826,14 +2755,13 @@ copy_cfg_body (copy_body_data * id, profile_count count, int frequency_scale,
   FOR_ALL_BB_FN (bb, cfun_to_copy)
     if (!id->blocks_to_copy
 	|| (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
-      need_debug_cleanup |= copy_edges_for_bb (bb, num, den, exit_block_map,
+      need_debug_cleanup |= copy_edges_for_bb (bb, exit_block_map,
 					       abnormal_goto_dest);
 
   if (new_entry)
     {
       edge e = make_edge (entry_block_map, (basic_block)new_entry->aux, EDGE_FALLTHRU);
       e->probability = profile_probability::always ();
-      e->count = incoming_count;
     }
 
   /* Duplicate the loop tree, if available and wanted.  */
@@ -3031,7 +2959,7 @@ copy_tree_body (copy_body_data *id)
    another function.  */
 
 static tree
-copy_body (copy_body_data *id, profile_count count, int frequency_scale,
+copy_body (copy_body_data *id, profile_count count,
 	   basic_block entry_block_map, basic_block exit_block_map,
 	   basic_block new_entry)
 {
@@ -3040,7 +2968,7 @@ copy_body (copy_body_data *id, profile_count count, int frequency_scale,
 
   /* If this body has a CFG, walk CFG and copy.  */
   gcc_assert (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (fndecl)));
-  body = copy_cfg_body (id, count, frequency_scale, entry_block_map, exit_block_map,
+  body = copy_cfg_body (id, count, entry_block_map, exit_block_map,
 			new_entry);
   copy_debug_stmts (id);
 
@@ -4797,7 +4725,6 @@ expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id)
      a self-referential call; if we're calling ourselves, we need to
      duplicate our body before altering anything.  */
   copy_body (id, cg_edge->callee->count,
-  	     GCOV_COMPUTE_SCALE (cg_edge->frequency, CGRAPH_FREQ_BASE),
 	     bb, return_block, NULL);
 
   reset_debug_bindings (id, stmt_gsi);
@@ -5172,6 +5099,7 @@ optimize_inline_calls (tree fn)
     }
 
   /* Fold queued statements.  */
+  counts_to_freqs ();
   fold_marked_statements (last, id.statements_to_fold);
   delete id.statements_to_fold;
 
@@ -6116,7 +6044,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
     }
 
   /* Copy the Function's body.  */
-  copy_body (&id, old_entry_block->count, REG_BR_PROB_BASE,
+  copy_body (&id, old_entry_block->count,
 	     ENTRY_BLOCK_PTR_FOR_FN (cfun), EXIT_BLOCK_PTR_FOR_FN (cfun),
 	     new_entry);
 
@@ -6148,6 +6076,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
   free_dominance_info (CDI_DOMINATORS);
   free_dominance_info (CDI_POST_DOMINATORS);
 
+  counts_to_freqs ();
   fold_marked_statements (0, id.statements_to_fold);
   delete id.statements_to_fold;
   delete_unreachable_blocks_update_callgraph (&id);
@@ -6167,20 +6096,20 @@ tree_function_versioning (tree old_decl, tree new_decl,
       struct cgraph_edge *e;
       rebuild_frequencies ();
 
-      new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
+      new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ();
       for (e = new_version_node->callees; e; e = e->next_callee)
 	{
 	  basic_block bb = gimple_bb (e->call_stmt);
 	  e->frequency = compute_call_stmt_bb_frequency (current_function_decl,
 							 bb);
-	  e->count = bb->count;
+	  e->count = bb->count.ipa ();
 	}
       for (e = new_version_node->indirect_calls; e; e = e->next_callee)
 	{
 	  basic_block bb = gimple_bb (e->call_stmt);
 	  e->frequency = compute_call_stmt_bb_frequency (current_function_decl,
 							 bb);
-	  e->count = bb->count;
+	  e->count = bb->count.ipa ();
 	}
     }
 
diff --git a/gcc/tree-loop-distribution.c b/gcc/tree-loop-distribution.c
index 30091453e39..fbb891fdedf 100644
--- a/gcc/tree-loop-distribution.c
+++ b/gcc/tree-loop-distribution.c
@@ -90,6 +90,7 @@ along with GCC; see the file COPYING3.  If not see
 	data reuse.  */
 
 #include "config.h"
+#define INCLUDE_ALGORITHM /* stable_sort */
 #include "system.h"
 #include "coretypes.h"
 #include "backend.h"
@@ -106,6 +107,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "stor-layout.h"
 #include "tree-cfg.h"
 #include "tree-ssa-loop-manip.h"
+#include "tree-ssa-loop-ivopts.h"
 #include "tree-ssa-loop.h"
 #include "tree-into-ssa.h"
 #include "tree-ssa.h"
@@ -604,6 +606,10 @@ struct builtin_info
   tree dst_base;
   tree src_base;
   tree size;
+  /* Base and offset part of dst_base after stripping constant offset.  This
+     is only used in memset builtin distribution for now.  */
+  tree dst_base_base;
+  unsigned HOST_WIDE_INT dst_base_offset;
 };
 
 /* Partition for loop distribution.  */
@@ -1286,12 +1292,12 @@ build_rdg_partition_for_vertex (struct graph *rdg, int v)
   return partition;
 }
 
-/* Given PARTITION of RDG, record single load/store data references for
-   builtin partition in SRC_DR/DST_DR, return false if there is no such
+/* Given PARTITION of LOOP and RDG, record single load/store data references
+   for builtin partition in SRC_DR/DST_DR, return false if there is no such
    data references.  */
 
 static bool
-find_single_drs (struct graph *rdg, partition *partition,
+find_single_drs (struct loop *loop, struct graph *rdg, partition *partition,
 		 data_reference_p *dst_dr, data_reference_p *src_dr)
 {
   unsigned i;
@@ -1347,10 +1353,12 @@ find_single_drs (struct graph *rdg, partition *partition,
       && DECL_BIT_FIELD (TREE_OPERAND (DR_REF (single_st), 1)))
     return false;
 
-  /* Data reference must be executed exactly once per iteration.  */
+  /* Data reference must be executed exactly once per iteration of each
+     loop in the loop nest.  We only need to check dominance information
+     against the outermost one in a perfect loop nest because a bb can't
+     dominate outermost loop's latch without dominating inner loop's.  */
   basic_block bb_st = gimple_bb (DR_STMT (single_st));
-  struct loop *inner = bb_st->loop_father;
-  if (!dominated_by_p (CDI_DOMINATORS, inner->latch, bb_st))
+  if (!dominated_by_p (CDI_DOMINATORS, loop->latch, bb_st))
     return false;
 
   if (single_ld)
@@ -1368,14 +1376,16 @@ find_single_drs (struct graph *rdg, partition *partition,
 
       /* Load and store must be in the same loop nest.  */
       basic_block bb_ld = gimple_bb (DR_STMT (single_ld));
-      if (inner != bb_ld->loop_father)
+      if (bb_st->loop_father != bb_ld->loop_father)
 	return false;
 
-      /* Data reference must be executed exactly once per iteration.  */
-      if (!dominated_by_p (CDI_DOMINATORS, inner->latch, bb_ld))
+      /* Data reference must be executed exactly once per iteration.
+	 Same as single_st, we only need to check against the outermost
+	 loop.  */
+      if (!dominated_by_p (CDI_DOMINATORS, loop->latch, bb_ld))
 	return false;
 
-      edge e = single_exit (inner);
+      edge e = single_exit (bb_st->loop_father);
       bool dom_ld = dominated_by_p (CDI_DOMINATORS, e->src, bb_ld);
       bool dom_st = dominated_by_p (CDI_DOMINATORS, e->src, bb_st);
       if (dom_ld != dom_st)
@@ -1503,7 +1513,17 @@ classify_builtin_st (loop_p loop, partition *partition, data_reference_p dr)
   if (!compute_access_range (loop, dr, &base, &size))
     return;
 
-  partition->builtin = alloc_builtin (dr, NULL, base, NULL_TREE, size);
+  poly_uint64 base_offset;
+  unsigned HOST_WIDE_INT const_base_offset;
+  tree base_base = strip_offset (base, &base_offset);
+  if (!base_offset.is_constant (&const_base_offset))
+    return;
+
+  struct builtin_info *builtin;
+  builtin = alloc_builtin (dr, NULL, base, NULL_TREE, size);
+  builtin->dst_base_base = base_base;
+  builtin->dst_base_offset = const_base_offset;
+  partition->builtin = builtin;
   partition->kind = PKIND_MEMSET;
 }
 
@@ -1614,7 +1634,7 @@ classify_partition (loop_p loop, struct graph *rdg, partition *partition,
     return;
 
   /* Find single load/store data references for builtin partition.  */
-  if (!find_single_drs (rdg, partition, &single_st, &single_ld))
+  if (!find_single_drs (loop, rdg, partition, &single_st, &single_ld))
     return;
 
   /* Classify the builtin kind.  */
@@ -2482,6 +2502,113 @@ version_for_distribution_p (vec<struct partition *> *partitions,
   return (alias_ddrs->length () > 0);
 }
 
+/* Compare base offset of builtin mem* partitions P1 and P2.  */
+
+static bool
+offset_cmp (struct partition *p1, struct partition *p2)
+{
+  gcc_assert (p1 != NULL && p1->builtin != NULL);
+  gcc_assert (p2 != NULL && p2->builtin != NULL);
+  return p1->builtin->dst_base_offset < p2->builtin->dst_base_offset;
+}
+
+/* Fuse adjacent memset builtin PARTITIONS if possible.  This is a special
+   case optimization transforming below code:
+
+     __builtin_memset (&obj, 0, 100);
+     _1 = &obj + 100;
+     __builtin_memset (_1, 0, 200);
+     _2 = &obj + 300;
+     __builtin_memset (_2, 0, 100);
+
+   into:
+
+     __builtin_memset (&obj, 0, 400);
+
+   Note we don't have dependence information between different partitions
+   at this point, as a result, we can't handle nonadjacent memset builtin
+   partitions since dependence might be broken.  */
+
+static void
+fuse_memset_builtins (vec<struct partition *> *partitions)
+{
+  unsigned i, j;
+  struct partition *part1, *part2;
+
+  for (i = 0; partitions->iterate (i, &part1);)
+    {
+      if (part1->kind != PKIND_MEMSET)
+	{
+	  i++;
+	  continue;
+	}
+
+      /* Find sub-array of memset builtins of the same base.  Index range
+	 of the sub-array is [i, j) with "j > i".  */
+      for (j = i + 1; partitions->iterate (j, &part2); ++j)
+	{
+	  if (part2->kind != PKIND_MEMSET
+	      || !operand_equal_p (part1->builtin->dst_base_base,
+				   part2->builtin->dst_base_base, 0))
+	    break;
+	}
+
+      /* Stable sort is required in order to avoid breaking dependence.  */
+      std::stable_sort (&(*partitions)[i],
+			&(*partitions)[i] + j - i, offset_cmp);
+      /* Continue with next partition.  */
+      i = j;
+    }
+
+  /* Merge all consecutive memset builtin partitions.  */
+  for (i = 0; i < partitions->length () - 1;)
+    {
+      part1 = (*partitions)[i];
+      if (part1->kind != PKIND_MEMSET)
+	{
+	  i++;
+	  continue;
+	}
+
+      part2 = (*partitions)[i + 1];
+      /* Only merge memset partitions of the same base and with constant
+	 access sizes.  */
+      if (part2->kind != PKIND_MEMSET
+	  || TREE_CODE (part1->builtin->size) != INTEGER_CST
+	  || TREE_CODE (part2->builtin->size) != INTEGER_CST
+	  || !operand_equal_p (part1->builtin->dst_base_base,
+			       part2->builtin->dst_base_base, 0))
+	{
+	  i++;
+	  continue;
+	}
+      tree rhs1 = gimple_assign_rhs1 (DR_STMT (part1->builtin->dst_dr));
+      tree rhs2 = gimple_assign_rhs1 (DR_STMT (part2->builtin->dst_dr));
+      int bytev1 = const_with_all_bytes_same (rhs1);
+      int bytev2 = const_with_all_bytes_same (rhs2);
+      /* Only merge memset partitions of the same value.  */
+      if (bytev1 != bytev2 || bytev1 == -1)
+	{
+	  i++;
+	  continue;
+	}
+      wide_int end1 = wi::add (part1->builtin->dst_base_offset,
+			       wi::to_wide (part1->builtin->size));
+      /* Only merge adjacent memset partitions.  */
+      if (wi::ne_p (end1, part2->builtin->dst_base_offset))
+	{
+	  i++;
+	  continue;
+	}
+      /* Merge partitions[i] and partitions[i+1].  */
+      part1->builtin->size = fold_build2 (PLUS_EXPR, sizetype,
+					  part1->builtin->size,
+					  part2->builtin->size);
+      partition_free (part2);
+      partitions->ordered_remove (i + 1);
+    }
+}
+
 /* Fuse PARTITIONS of LOOP if necessary before finalizing distribution.
    ALIAS_DDRS contains ddrs which need runtime alias check.  */
 
@@ -2525,6 +2652,10 @@ finalize_partitions (struct loop *loop, vec<struct partition *> *partitions,
 	}
       partitions->truncate (1);
     }
+
+  /* Fuse memset builtins if possible.  */
+  if (partitions->length () > 1)
+    fuse_memset_builtins (partitions);
 }
 
 /* Distributes the code from LOOP in such a way that producer statements
diff --git a/gcc/tree-object-size.c b/gcc/tree-object-size.c
index c5a153f8411..3d288ec951c 100644
--- a/gcc/tree-object-size.c
+++ b/gcc/tree-object-size.c
@@ -436,8 +436,7 @@ alloc_object_size (const gcall *call, int object_size_type)
 	arg2 = 1;
 	/* fall through */
       case BUILT_IN_MALLOC:
-      case BUILT_IN_ALLOCA:
-      case BUILT_IN_ALLOCA_WITH_ALIGN:
+      CASE_BUILT_IN_ALLOCA:
 	arg1 = 0;
       default:
 	break;
diff --git a/gcc/tree-outof-ssa.h b/gcc/tree-outof-ssa.h
index 1220b6256ca..ebbaea1a03e 100644
--- a/gcc/tree-outof-ssa.h
+++ b/gcc/tree-outof-ssa.h
@@ -74,18 +74,6 @@ get_gimple_for_ssa_name (tree exp)
   return NULL;
 }
 
-/* Return whether the RTX expression representing the storage of the outof-SSA
-   partition that the SSA name EXP is a member of is always initialized.  */
-static inline bool
-always_initialized_rtx_for_ssa_name_p (tree exp)
-{
-  int p = partition_find (SA.map->var_partition, SSA_NAME_VERSION (exp));
-  if (SA.map->partition_to_view)
-    p = SA.map->partition_to_view[p];
-  gcc_assert (p != NO_PARTITION);
-  return !bitmap_bit_p (SA.partitions_for_undefined_values, p);
-}
-
 extern bool ssa_is_replaceable_p (gimple *stmt);
 extern void finish_out_of_ssa (struct ssaexpand *sa);
 extern unsigned int rewrite_out_of_ssa (struct ssaexpand *sa);
diff --git a/gcc/tree-pass.h b/gcc/tree-pass.h
index a6599eb8bf0..fbd0dbdf924 100644
--- a/gcc/tree-pass.h
+++ b/gcc/tree-pass.h
@@ -223,6 +223,7 @@ protected:
 						   current choices have
 						   been optimized.  */
 #define PROP_gimple_lomp_dev	(1 << 16)	/* done omp_device_lower */
+#define PROP_rtl_split_insns	(1 << 17)	/* RTL has insns split.  */
 
 #define PROP_trees \
   (PROP_gimple_any | PROP_gimple_lcf | PROP_gimple_leh | PROP_gimple_lomp)
diff --git a/gcc/tree-scalar-evolution.c b/gcc/tree-scalar-evolution.c
index a863de3d1d0..059f820e65b 100644
--- a/gcc/tree-scalar-evolution.c
+++ b/gcc/tree-scalar-evolution.c
@@ -1770,7 +1770,7 @@ interpret_rhs_expr (struct loop *loop, gimple *at_stmt,
 	      res = chrec_fold_plus (type, res, chrec2);
 	    }
 
-	  if (maybe_nonzero (bitpos))
+	  if (may_ne (bitpos, 0))
 	    {
 	      unitpos = size_int (exact_div (bitpos, BITS_PER_UNIT));
 	      chrec3 = analyze_scalar_evolution (loop, unitpos);
@@ -2356,11 +2356,9 @@ instantiate_scev_name (edge instantiate_below,
   struct loop *def_loop;
   basic_block def_bb = gimple_bb (SSA_NAME_DEF_STMT (chrec));
 
-  /* A parameter (or loop invariant and we do not want to include
-     evolutions in outer loops), nothing to do.  */
+  /* A parameter, nothing to do.  */
   if (!def_bb
-      || loop_depth (def_bb->loop_father) == 0
-      || ! dominated_by_p (CDI_DOMINATORS, def_bb, instantiate_below->dest))
+      || !dominated_by_p (CDI_DOMINATORS, def_bb, instantiate_below->dest))
     return chrec;
 
   /* We cache the value of instantiated variable to avoid exponential
diff --git a/gcc/tree-ssa-address.c b/gcc/tree-ssa-address.c
index d732c5f6b0c..417ca8f3c54 100644
--- a/gcc/tree-ssa-address.c
+++ b/gcc/tree-ssa-address.c
@@ -693,7 +693,7 @@ addr_to_parts (tree type, aff_tree *addr, tree iv_cand, tree base_hint,
   parts->index = NULL_TREE;
   parts->step = NULL_TREE;
 
-  if (maybe_nonzero (addr->offset))
+  if (may_ne (addr->offset, 0))
     parts->offset = wide_int_to_tree (sizetype, addr->offset);
   else
     parts->offset = NULL_TREE;
@@ -747,18 +747,15 @@ gimplify_mem_ref_parts (gimple_stmt_iterator *gsi, struct mem_address *parts)
 }
 
 /* Return true if the STEP in PARTS gives a valid BASE + INDEX * STEP
-   address for type TYPE and if some other component (the symbol or
-   offset) is making it appear invalid.  */
+   address for type TYPE and if the offset is making it appear invalid.  */
 
 static bool
 keep_index_p (tree type, mem_address parts)
 {
   if (!parts.base)
-    parts.base = parts.symbol;
-  if (!parts.base)
     return false;
 
-  parts.symbol = NULL_TREE;
+  gcc_assert (!parts.symbol);
   parts.offset = NULL_TREE;
   return valid_mem_ref_p (TYPE_MODE (type), TYPE_ADDR_SPACE (type), &parts);
 }
@@ -826,9 +823,8 @@ create_mem_ref (gimple_stmt_iterator *gsi, tree type, aff_tree *addr,
      into:
        index' = index << step;
        [... + index' + ,,,].  */
-  if (parts.step
-      && !integer_onep (parts.step)
-      && !keep_index_p (type, parts))
+  bool scaled_p = (parts.step && !integer_onep (parts.step));
+  if (scaled_p && !keep_index_p (type, parts))
     {
       gcc_assert (parts.index);
       parts.index = force_gimple_operand_gsi (gsi,
@@ -840,6 +836,7 @@ create_mem_ref (gimple_stmt_iterator *gsi, tree type, aff_tree *addr,
       mem_ref = create_mem_ref_raw (type, alias_ptr_type, &parts, true);
       if (mem_ref)
 	return mem_ref;
+      scaled_p = false;
     }
 
   /* Add offset to invariant part by transforming address expression:
@@ -853,7 +850,7 @@ create_mem_ref (gimple_stmt_iterator *gsi, tree type, aff_tree *addr,
      depending on which one is invariant.  */
   if (parts.offset
       && !integer_zerop (parts.offset)
-      && (!var_in_base || !parts.step || integer_onep (parts.step)))
+      && (!var_in_base || !scaled_p))
     {
       tree old_base = unshare_expr (parts.base);
       tree old_index = unshare_expr (parts.index);
@@ -903,7 +900,7 @@ create_mem_ref (gimple_stmt_iterator *gsi, tree type, aff_tree *addr,
   /* Transform [base + index + ...] into:
        base' = base + index;
        [base' + ...].  */
-  if (parts.index && (!parts.step || integer_onep (parts.step)))
+  if (parts.index && !scaled_p)
     {
       tmp = parts.index;
       parts.index = NULL_TREE;
diff --git a/gcc/tree-ssa-alias.c b/gcc/tree-ssa-alias.c
index f378993f453..b9f25a3ad78 100644
--- a/gcc/tree-ssa-alias.c
+++ b/gcc/tree-ssa-alias.c
@@ -1753,8 +1753,7 @@ ref_maybe_used_by_call_p_1 (gcall *call, ao_ref *ref)
 	case BUILT_IN_POSIX_MEMALIGN:
 	case BUILT_IN_ALIGNED_ALLOC:
 	case BUILT_IN_CALLOC:
-	case BUILT_IN_ALLOCA:
-	case BUILT_IN_ALLOCA_WITH_ALIGN:
+	CASE_BUILT_IN_ALLOCA:
 	case BUILT_IN_STACK_SAVE:
 	case BUILT_IN_STACK_RESTORE:
 	case BUILT_IN_MEMSET:
@@ -2092,8 +2091,7 @@ call_may_clobber_ref_p_1 (gcall *call, ao_ref *ref)
 	    return true;
 	  return false;
 	case BUILT_IN_STACK_SAVE:
-	case BUILT_IN_ALLOCA:
-	case BUILT_IN_ALLOCA_WITH_ALIGN:
+	CASE_BUILT_IN_ALLOCA:
 	case BUILT_IN_ASSUME_ALIGNED:
 	  return false;
 	/* But posix_memalign stores a pointer into the memory pointed to
@@ -2302,8 +2300,8 @@ same_addr_size_stores_p (tree base1, poly_int64 offset1, poly_int64 size1,
 			 poly_int64 max_size2)
 {
   /* Offsets need to be 0.  */
-  if (maybe_nonzero (offset1)
-      || maybe_nonzero (offset2))
+  if (may_ne (offset1, 0)
+      || may_ne (offset2, 0))
     return false;
 
   bool base1_obj_p = SSA_VAR_P (base1);
diff --git a/gcc/tree-ssa-ccp.c b/gcc/tree-ssa-ccp.c
index 7b075102df2..cc98d18e46e 100644
--- a/gcc/tree-ssa-ccp.c
+++ b/gcc/tree-ssa-ccp.c
@@ -171,6 +171,13 @@ struct ccp_prop_value_t {
     widest_int mask;
 };
 
+class ccp_propagate : public ssa_propagation_engine
+{
+ public:
+  enum ssa_prop_result visit_stmt (gimple *, edge *, tree *) FINAL OVERRIDE;
+  enum ssa_prop_result visit_phi (gphi *) FINAL OVERRIDE;
+};
+
 /* Array of propagated constant values.  After propagation,
    CONST_VAL[I].VALUE holds the constant value for SSA_NAME(I).  If
    the constant is held in an SSA name representing a memory store
@@ -181,7 +188,6 @@ static ccp_prop_value_t *const_val;
 static unsigned n_const_val;
 
 static void canonicalize_value (ccp_prop_value_t *);
-static bool ccp_fold_stmt (gimple_stmt_iterator *);
 static void ccp_lattice_meet (ccp_prop_value_t *, ccp_prop_value_t *);
 
 /* Dump constant propagation value VAL to file OUTF prefixed by PREFIX.  */
@@ -902,6 +908,24 @@ do_dbg_cnt (void)
 }
 
 
+/* We want to provide our own GET_VALUE and FOLD_STMT virtual methods.  */
+class ccp_folder : public substitute_and_fold_engine
+{
+ public:
+  tree get_value (tree) FINAL OVERRIDE;
+  bool fold_stmt (gimple_stmt_iterator *) FINAL OVERRIDE;
+};
+
+/* This method just wraps GET_CONSTANT_VALUE for now.  Over time
+   naked calls to GET_CONSTANT_VALUE should be eliminated in favor
+   of calling member functions.  */
+
+tree
+ccp_folder::get_value (tree op)
+{
+  return get_constant_value (op);
+}
+
 /* Do final substitution of propagated values, cleanup the flowgraph and
    free allocated storage.  If NONZERO_P, record nonzero bits.
 
@@ -960,7 +984,8 @@ ccp_finalize (bool nonzero_p)
     }
 
   /* Perform substitutions based on the known constant values.  */
-  something_changed = substitute_and_fold (get_constant_value, ccp_fold_stmt);
+  class ccp_folder ccp_folder;
+  something_changed = ccp_folder.substitute_and_fold ();
 
   free (const_val);
   const_val = NULL;
@@ -1064,8 +1089,8 @@ ccp_lattice_meet (ccp_prop_value_t *val1, ccp_prop_value_t *val2)
    PHI node is determined calling ccp_lattice_meet with all the arguments
    of the PHI node that are incoming via executable edges.  */
 
-static enum ssa_prop_result
-ccp_visit_phi_node (gphi *phi)
+enum ssa_prop_result
+ccp_propagate::visit_phi (gphi *phi)
 {
   unsigned i;
   ccp_prop_value_t new_val;
@@ -1886,11 +1911,10 @@ evaluate_stmt (gimple *stmt)
 			   / BITS_PER_UNIT - 1);
 	      break;
 
-	    case BUILT_IN_ALLOCA:
-	    case BUILT_IN_ALLOCA_WITH_ALIGN:
-	      align = (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_ALLOCA_WITH_ALIGN
-		       ? TREE_INT_CST_LOW (gimple_call_arg (stmt, 1))
-		       : BIGGEST_ALIGNMENT);
+	    CASE_BUILT_IN_ALLOCA:
+	      align = (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_ALLOCA
+		       ? BIGGEST_ALIGNMENT
+		       : TREE_INT_CST_LOW (gimple_call_arg (stmt, 1)));
 	      val.lattice_val = CONSTANT;
 	      val.value = build_int_cst (TREE_TYPE (gimple_get_lhs (stmt)), 0);
 	      val.mask = ~((HOST_WIDE_INT) align / BITS_PER_UNIT - 1);
@@ -2170,8 +2194,8 @@ fold_builtin_alloca_with_align (gimple *stmt)
 /* Fold the stmt at *GSI with CCP specific information that propagating
    and regular folding does not catch.  */
 
-static bool
-ccp_fold_stmt (gimple_stmt_iterator *gsi)
+bool
+ccp_folder::fold_stmt (gimple_stmt_iterator *gsi)
 {
   gimple *stmt = gsi_stmt (*gsi);
 
@@ -2243,7 +2267,8 @@ ccp_fold_stmt (gimple_stmt_iterator *gsi)
         /* The heuristic of fold_builtin_alloca_with_align differs before and
 	   after inlining, so we don't require the arg to be changed into a
 	   constant for folding, but just to be constant.  */
-        if (gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA_WITH_ALIGN))
+        if (gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA_WITH_ALIGN)
+	    || gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX))
           {
             tree new_rhs = fold_builtin_alloca_with_align (stmt);
             if (new_rhs)
@@ -2378,8 +2403,8 @@ visit_cond_stmt (gimple *stmt, edge *taken_edge_p)
    value, set *TAKEN_EDGE_P accordingly.  If STMT produces a varying
    value, return SSA_PROP_VARYING.  */
 
-static enum ssa_prop_result
-ccp_visit_stmt (gimple *stmt, edge *taken_edge_p, tree *output_p)
+enum ssa_prop_result
+ccp_propagate::visit_stmt (gimple *stmt, edge *taken_edge_p, tree *output_p)
 {
   tree def;
   ssa_op_iter iter;
@@ -2441,7 +2466,8 @@ do_ssa_ccp (bool nonzero_p)
   calculate_dominance_info (CDI_DOMINATORS);
 
   ccp_initialize ();
-  ssa_propagate (ccp_visit_stmt, ccp_visit_phi_node);
+  class ccp_propagate ccp_propagate;
+  ccp_propagate.ssa_propagate ();
   if (ccp_finalize (nonzero_p || flag_ipa_bit_cp))
     {
       todo = (TODO_cleanup_cfg | TODO_update_ssa);
@@ -2535,8 +2561,7 @@ optimize_stack_restore (gimple_stmt_iterator i)
       if (!callee
 	  || DECL_BUILT_IN_CLASS (callee) != BUILT_IN_NORMAL
 	  /* All regular builtins are ok, just obviously not alloca.  */
-	  || DECL_FUNCTION_CODE (callee) == BUILT_IN_ALLOCA
-	  || DECL_FUNCTION_CODE (callee) == BUILT_IN_ALLOCA_WITH_ALIGN)
+	  || ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (callee)))
 	return NULL_TREE;
 
       if (DECL_FUNCTION_CODE (callee) == BUILT_IN_STACK_RESTORE)
diff --git a/gcc/tree-ssa-coalesce.c b/gcc/tree-ssa-coalesce.c
index 3938f064f67..057d51dcf37 100644
--- a/gcc/tree-ssa-coalesce.c
+++ b/gcc/tree-ssa-coalesce.c
@@ -164,7 +164,7 @@ coalesce_cost (int frequency, bool optimize_for_size)
 static inline int
 coalesce_cost_bb (basic_block bb)
 {
-  return coalesce_cost (bb->frequency, optimize_bb_for_size_p (bb));
+  return coalesce_cost (bb->count.to_frequency (cfun), optimize_bb_for_size_p (bb));
 }
 
 
diff --git a/gcc/tree-ssa-copy.c b/gcc/tree-ssa-copy.c
index 9f0fe541ded..1f9dbf52346 100644
--- a/gcc/tree-ssa-copy.c
+++ b/gcc/tree-ssa-copy.c
@@ -68,6 +68,13 @@ struct prop_value_t {
     tree value;
 };
 
+class copy_prop : public ssa_propagation_engine
+{
+ public:
+  enum ssa_prop_result visit_stmt (gimple *, edge *, tree *) FINAL OVERRIDE;
+  enum ssa_prop_result visit_phi (gphi *) FINAL OVERRIDE;
+};
+
 static prop_value_t *copy_of;
 static unsigned n_copy_of;
 
@@ -263,8 +270,8 @@ copy_prop_visit_cond_stmt (gimple *stmt, edge *taken_edge_p)
    If the new value produced by STMT is varying, return
    SSA_PROP_VARYING.  */
 
-static enum ssa_prop_result
-copy_prop_visit_stmt (gimple *stmt, edge *taken_edge_p, tree *result_p)
+enum ssa_prop_result
+copy_prop::visit_stmt (gimple *stmt, edge *taken_edge_p, tree *result_p)
 {
   enum ssa_prop_result retval;
 
@@ -317,8 +324,8 @@ copy_prop_visit_stmt (gimple *stmt, edge *taken_edge_p, tree *result_p)
 /* Visit PHI node PHI.  If all the arguments produce the same value,
    set it to be the value of the LHS of PHI.  */
 
-static enum ssa_prop_result
-copy_prop_visit_phi_node (gphi *phi)
+enum ssa_prop_result
+copy_prop::visit_phi (gphi *phi)
 {
   enum ssa_prop_result retval;
   unsigned i;
@@ -482,10 +489,16 @@ init_copy_prop (void)
     }
 }
 
+class copy_folder : public substitute_and_fold_engine
+{
+ public:
+  tree get_value (tree) FINAL OVERRIDE;
+};
+
 /* Callback for substitute_and_fold to get at the final copy-of values.  */
 
-static tree
-get_value (tree name)
+tree
+copy_folder::get_value (tree name)
 {
   tree val;
   if (SSA_NAME_VERSION (name) >= n_copy_of)
@@ -550,7 +563,8 @@ fini_copy_prop (void)
 	}
     }
 
-  bool changed = substitute_and_fold (get_value, NULL);
+  class copy_folder copy_folder;
+  bool changed = copy_folder.substitute_and_fold ();
   if (changed)
     {
       free_numbers_of_iterations_estimates (cfun);
@@ -601,7 +615,8 @@ static unsigned int
 execute_copy_prop (void)
 {
   init_copy_prop ();
-  ssa_propagate (copy_prop_visit_stmt, copy_prop_visit_phi_node);
+  class copy_prop copy_prop;
+  copy_prop.ssa_propagate ();
   if (fini_copy_prop ())
     return TODO_cleanup_cfg;
   return 0;
diff --git a/gcc/tree-ssa-dce.c b/gcc/tree-ssa-dce.c
index 270253bbfb2..794a1b3a4d7 100644
--- a/gcc/tree-ssa-dce.c
+++ b/gcc/tree-ssa-dce.c
@@ -231,8 +231,7 @@ mark_stmt_if_obviously_necessary (gimple *stmt, bool aggressive)
 	    case BUILT_IN_MALLOC:
 	    case BUILT_IN_ALIGNED_ALLOC:
 	    case BUILT_IN_CALLOC:
-	    case BUILT_IN_ALLOCA:
-	    case BUILT_IN_ALLOCA_WITH_ALIGN:
+	    CASE_BUILT_IN_ALLOCA:
 	    case BUILT_IN_STRDUP:
 	    case BUILT_IN_STRNDUP:
 	      return;
@@ -572,8 +571,7 @@ mark_all_reaching_defs_necessary_1 (ao_ref *ref ATTRIBUTE_UNUSED,
 	  case BUILT_IN_MALLOC:
 	  case BUILT_IN_ALIGNED_ALLOC:
 	  case BUILT_IN_CALLOC:
-	  case BUILT_IN_ALLOCA:
-	  case BUILT_IN_ALLOCA_WITH_ALIGN:
+	  CASE_BUILT_IN_ALLOCA:
 	  case BUILT_IN_FREE:
 	    return false;
 
@@ -841,9 +839,7 @@ propagate_necessity (bool aggressive)
 		      || DECL_FUNCTION_CODE (callee) == BUILT_IN_CALLOC
 		      || DECL_FUNCTION_CODE (callee) == BUILT_IN_FREE
 		      || DECL_FUNCTION_CODE (callee) == BUILT_IN_VA_END
-		      || DECL_FUNCTION_CODE (callee) == BUILT_IN_ALLOCA
-		      || (DECL_FUNCTION_CODE (callee)
-			  == BUILT_IN_ALLOCA_WITH_ALIGN)
+		      || ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (callee))
 		      || DECL_FUNCTION_CODE (callee) == BUILT_IN_STACK_SAVE
 		      || DECL_FUNCTION_CODE (callee) == BUILT_IN_STACK_RESTORE
 		      || DECL_FUNCTION_CODE (callee) == BUILT_IN_ASSUME_ALIGNED))
@@ -1051,7 +1047,6 @@ remove_dead_stmt (gimple_stmt_iterator *i, basic_block bb)
 	}
       gcc_assert (e);
       e->probability = profile_probability::always ();
-      e->count = bb->count;
 
       /* The edge is no longer associated with a conditional, so it does
 	 not have TRUE/FALSE flags.
@@ -1344,9 +1339,8 @@ eliminate_unnecessary_stmts (void)
 		      || (DECL_FUNCTION_CODE (call) != BUILT_IN_ALIGNED_ALLOC
 			  && DECL_FUNCTION_CODE (call) != BUILT_IN_MALLOC
 			  && DECL_FUNCTION_CODE (call) != BUILT_IN_CALLOC
-			  && DECL_FUNCTION_CODE (call) != BUILT_IN_ALLOCA
-			  && (DECL_FUNCTION_CODE (call)
-			      != BUILT_IN_ALLOCA_WITH_ALIGN)))
+			  && !ALLOCA_FUNCTION_CODE_P
+			      (DECL_FUNCTION_CODE (call))))
 		  /* Avoid doing so for bndret calls for the same reason.  */
 		  && !chkp_gimple_call_builtin_p (stmt, BUILT_IN_CHKP_BNDRET))
 		{
diff --git a/gcc/tree-ssa-dom.c b/gcc/tree-ssa-dom.c
index 06be69a530c..eb85b4a09ad 100644
--- a/gcc/tree-ssa-dom.c
+++ b/gcc/tree-ssa-dom.c
@@ -113,7 +113,6 @@ static void eliminate_redundant_computations (gimple_stmt_iterator *,
 					      class avail_exprs_stack *);
 static void record_equivalences_from_stmt (gimple *, int,
 					   class avail_exprs_stack *);
-static edge single_incoming_edge_ignoring_loop_edges (basic_block);
 static void dump_dominator_optimization_stats (FILE *file,
 					       hash_table<expr_elt_hasher> *);
 
@@ -1057,39 +1056,6 @@ record_equivalences_from_phis (basic_block bb)
     }
 }
 
-/* Ignoring loop backedges, if BB has precisely one incoming edge then
-   return that edge.  Otherwise return NULL.  */
-static edge
-single_incoming_edge_ignoring_loop_edges (basic_block bb)
-{
-  edge retval = NULL;
-  edge e;
-  edge_iterator ei;
-
-  FOR_EACH_EDGE (e, ei, bb->preds)
-    {
-      /* A loop back edge can be identified by the destination of
-	 the edge dominating the source of the edge.  */
-      if (dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
-	continue;
-
-      /* We can safely ignore edges that are not executable.  */
-      if ((e->flags & EDGE_EXECUTABLE) == 0)
-	continue;
-
-      /* If we have already seen a non-loop edge, then we must have
-	 multiple incoming non-loop edges and thus we return NULL.  */
-      if (retval)
-	return NULL;
-
-      /* This is the first non-loop incoming edge we have found.  Record
-	 it.  */
-      retval = e;
-    }
-
-  return retval;
-}
-
 /* Record any equivalences created by the incoming edge to BB into
    CONST_AND_COPIES and AVAIL_EXPRS_STACK.  If BB has more than one
    incoming edge, then no equivalence is created.  */
@@ -1107,7 +1073,7 @@ record_equivalences_from_incoming_edge (basic_block bb,
      the parent was followed.  */
   parent = get_immediate_dominator (CDI_DOMINATORS, bb);
 
-  e = single_incoming_edge_ignoring_loop_edges (bb);
+  e = single_pred_edge_ignoring_loop_edges (bb, true);
 
   /* If we had a single incoming edge from our parent block, then enter
      any data associated with the edge into our tables.  */
diff --git a/gcc/tree-ssa-dse.c b/gcc/tree-ssa-dse.c
index 2aeb1f53410..741180e51bf 100644
--- a/gcc/tree-ssa-dse.c
+++ b/gcc/tree-ssa-dse.c
@@ -129,7 +129,7 @@ valid_ao_ref_for_dse (ao_ref *ref)
 {
   return (ao_ref_base (ref)
 	  && known_size_p (ref->max_size)
-	  && maybe_nonzero (ref->size)
+	  && may_ne (ref->size, 0)
 	  && must_eq (ref->max_size, ref->size)
 	  && must_ge (ref->offset, 0)
 	  && multiple_p (ref->offset, BITS_PER_UNIT)
@@ -606,7 +606,7 @@ dse_classify_store (ao_ref *ref, gimple *stmt, gimple **use_stmt,
 		      ao_ref use_ref;
 		      ao_ref_init (&use_ref, gimple_assign_rhs1 (use_stmt));
 		      if (valid_ao_ref_for_dse (&use_ref)
-			  && must_eq (use_ref.base, ref->base)
+			  && use_ref.base == ref->base
 			  && must_eq (use_ref.size, use_ref.max_size)
 			  && !live_bytes_read (use_ref, ref, live_bytes))
 			{
diff --git a/gcc/tree-ssa-forwprop.c b/gcc/tree-ssa-forwprop.c
index ec54a82da24..eb47a08004d 100644
--- a/gcc/tree-ssa-forwprop.c
+++ b/gcc/tree-ssa-forwprop.c
@@ -1174,7 +1174,7 @@ constant_pointer_difference (tree p1, tree p2)
 	      if (base)
 		{
 		  q = base;
-		  if (maybe_nonzero (offset))
+		  if (may_ne (offset, 0))
 		    off = size_binop (PLUS_EXPR, off, size_int (offset));
 		}
 	      if (TREE_CODE (q) == MEM_REF
@@ -1491,9 +1491,14 @@ defcodefor_name (tree name, enum tree_code *code, tree *arg1, tree *arg2)
    applied, otherwise return false.
 
    We are looking for X with unsigned type T with bitsize B, OP being
-   +, | or ^, some type T2 wider than T and
+   +, | or ^, some type T2 wider than T.  For:
    (X << CNT1) OP (X >> CNT2)				iff CNT1 + CNT2 == B
    ((T) ((T2) X << CNT1)) OP ((T) ((T2) X >> CNT2))	iff CNT1 + CNT2 == B
+
+   transform these into:
+   X r<< CNT1
+
+   Or for:
    (X << Y) OP (X >> (B - Y))
    (X << (int) Y) OP (X >> (int) (B - Y))
    ((T) ((T2) X << Y)) OP ((T) ((T2) X >> (B - Y)))
@@ -1503,12 +1508,23 @@ defcodefor_name (tree name, enum tree_code *code, tree *arg1, tree *arg2)
    ((T) ((T2) X << Y)) | ((T) ((T2) X >> ((-Y) & (B - 1))))
    ((T) ((T2) X << (int) Y)) | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
 
-   and transform these into:
-   X r<< CNT1
+   transform these into:
    X r<< Y
 
+   Or for:
+   (X << (Y & (B - 1))) | (X >> ((-Y) & (B - 1)))
+   (X << (int) (Y & (B - 1))) | (X >> (int) ((-Y) & (B - 1)))
+   ((T) ((T2) X << (Y & (B - 1)))) | ((T) ((T2) X >> ((-Y) & (B - 1))))
+   ((T) ((T2) X << (int) (Y & (B - 1)))) \
+     | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
+
+   transform these into:
+   X r<< (Y & (B - 1))
+
    Note, in the patterns with T2 type, the type of OP operands
-   might be even a signed type, but should have precision B.  */
+   might be even a signed type, but should have precision B.
+   Expressions with & (B - 1) should be recognized only if B is
+   a power of 2.  */
 
 static bool
 simplify_rotate (gimple_stmt_iterator *gsi)
@@ -1578,7 +1594,9 @@ simplify_rotate (gimple_stmt_iterator *gsi)
 	def_arg1[i] = tem;
       }
   /* Both shifts have to use the same first operand.  */
-  if (TREE_CODE (def_arg1[0]) != SSA_NAME || def_arg1[0] != def_arg1[1])
+  if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
+      || !types_compatible_p (TREE_TYPE (def_arg1[0]),
+			      TREE_TYPE (def_arg1[1])))
     return false;
   if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[0])))
     return false;
@@ -1649,8 +1667,10 @@ simplify_rotate (gimple_stmt_iterator *gsi)
 	/* The above sequence isn't safe for Y being 0,
 	   because then one of the shifts triggers undefined behavior.
 	   This alternative is safe even for rotation count of 0.
-	   One shift count is Y and the other (-Y) & (B - 1).  */
+	   One shift count is Y and the other (-Y) & (B - 1).
+	   Or one shift count is Y & (B - 1) and the other (-Y) & (B - 1).  */
 	else if (cdef_code[i] == BIT_AND_EXPR
+		 && pow2p_hwi (TYPE_PRECISION (rtype))
 		 && tree_fits_shwi_p (cdef_arg2[i])
 		 && tree_to_shwi (cdef_arg2[i])
 		    == TYPE_PRECISION (rtype) - 1
@@ -1675,17 +1695,50 @@ simplify_rotate (gimple_stmt_iterator *gsi)
 		    rotcnt = tem;
 		    break;
 		  }
-		defcodefor_name (tem, &code, &tem, NULL);
+		tree tem2;
+		defcodefor_name (tem, &code, &tem2, NULL);
 		if (CONVERT_EXPR_CODE_P (code)
-		    && INTEGRAL_TYPE_P (TREE_TYPE (tem))
-		    && TYPE_PRECISION (TREE_TYPE (tem))
+		    && INTEGRAL_TYPE_P (TREE_TYPE (tem2))
+		    && TYPE_PRECISION (TREE_TYPE (tem2))
 		       > floor_log2 (TYPE_PRECISION (rtype))
-		    && type_has_mode_precision_p (TREE_TYPE (tem))
-		    && (tem == def_arg2[1 - i]
-			|| tem == def_arg2_alt[1 - i]))
+		    && type_has_mode_precision_p (TREE_TYPE (tem2)))
 		  {
-		    rotcnt = tem;
-		    break;
+		    if (tem2 == def_arg2[1 - i]
+			|| tem2 == def_arg2_alt[1 - i])
+		      {
+			rotcnt = tem2;
+			break;
+		      }
+		  }
+		else
+		  tem2 = NULL_TREE;
+
+		if (cdef_code[1 - i] == BIT_AND_EXPR
+		    && tree_fits_shwi_p (cdef_arg2[1 - i])
+		    && tree_to_shwi (cdef_arg2[1 - i])
+		       == TYPE_PRECISION (rtype) - 1
+		    && TREE_CODE (cdef_arg1[1 - i]) == SSA_NAME)
+		  {
+		    if (tem == cdef_arg1[1 - i]
+			|| tem2 == cdef_arg1[1 - i])
+		      {
+			rotcnt = def_arg2[1 - i];
+			break;
+		      }
+		    tree tem3;
+		    defcodefor_name (cdef_arg1[1 - i], &code, &tem3, NULL);
+		    if (CONVERT_EXPR_CODE_P (code)
+			&& INTEGRAL_TYPE_P (TREE_TYPE (tem3))
+			&& TYPE_PRECISION (TREE_TYPE (tem3))
+			   > floor_log2 (TYPE_PRECISION (rtype))
+			&& type_has_mode_precision_p (TREE_TYPE (tem3)))
+		      {
+			if (tem == tem3 || tem2 == tem3)
+			  {
+			    rotcnt = def_arg2[1 - i];
+			    break;
+			  }
+		      }
 		  }
 	      }
 	  }
diff --git a/gcc/tree-ssa-ifcombine.c b/gcc/tree-ssa-ifcombine.c
index a211335889b..ff26dd1f731 100644
--- a/gcc/tree-ssa-ifcombine.c
+++ b/gcc/tree-ssa-ifcombine.c
@@ -358,10 +358,7 @@ update_profile_after_ifcombine (basic_block inner_cond_bb,
      outer_cond_bb->(outer_to_inner)->inner_cond_bb->(inner_taken)
      and probability of inner_not_taken updated.  */
 
-  outer_to_inner->count = outer_cond_bb->count;
   inner_cond_bb->count = outer_cond_bb->count;
-  inner_taken->count += outer2->count;
-  outer2->count = profile_count::zero ();
 
   inner_taken->probability = outer2->probability + outer_to_inner->probability
 			     * inner_taken->probability;
@@ -369,7 +366,6 @@ update_profile_after_ifcombine (basic_block inner_cond_bb,
 				 - inner_taken->probability;
 
   outer_to_inner->probability = profile_probability::always ();
-  inner_cond_bb->frequency = outer_cond_bb->frequency;
   outer2->probability = profile_probability::never ();
 }
 
diff --git a/gcc/tree-ssa-loop-im.c b/gcc/tree-ssa-loop-im.c
index 3866709fa6c..5103d12cf87 100644
--- a/gcc/tree-ssa-loop-im.c
+++ b/gcc/tree-ssa-loop-im.c
@@ -1802,7 +1802,7 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag,
   for (hash_set<basic_block>::iterator it = flag_bbs->begin ();
        it != flag_bbs->end (); ++it)
     {
-       freq_sum += (*it)->frequency;
+       freq_sum += (*it)->count.to_frequency (cfun);
        if ((*it)->count.initialized_p ())
          count_sum += (*it)->count, ncount ++;
        if (dominated_by_p (CDI_DOMINATORS, ex->src, *it))
@@ -1814,20 +1814,15 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag,
 
   if (flag_probability.initialized_p ())
     ;
-  else if (ncount == nbbs && count_sum > 0 && preheader->count >= count_sum)
+  else if (ncount == nbbs
+	   && preheader->count () >= count_sum && preheader->count ().nonzero_p ())
     {
-      flag_probability = count_sum.probability_in (preheader->count);
+      flag_probability = count_sum.probability_in (preheader->count ());
       if (flag_probability > cap)
 	flag_probability = cap;
     }
-  else if (freq_sum > 0 && EDGE_FREQUENCY (preheader) >= freq_sum)
-    {
-      flag_probability = profile_probability::from_reg_br_prob_base
-		(GCOV_COMPUTE_SCALE (freq_sum, EDGE_FREQUENCY (preheader)));
-      if (flag_probability > cap)
-	flag_probability = cap;
-    }
-  else
+
+  if (!flag_probability.initialized_p ())
     flag_probability = cap;
 
   /* ?? Insert store after previous store if applicable.  See note
@@ -1860,7 +1855,6 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag,
   old_dest = ex->dest;
   new_bb = split_edge (ex);
   then_bb = create_empty_bb (new_bb);
-  then_bb->frequency = flag_probability.apply (new_bb->frequency);
   then_bb->count = new_bb->count.apply_probability (flag_probability);
   if (irr)
     then_bb->flags = BB_IRREDUCIBLE_LOOP;
@@ -1880,13 +1874,11 @@ execute_sm_if_changed (edge ex, tree mem, tree tmp_var, tree flag,
   edge e2 = make_edge (new_bb, then_bb,
 	               EDGE_TRUE_VALUE | (irr ? EDGE_IRREDUCIBLE_LOOP : 0));
   e2->probability = flag_probability;
-  e2->count = then_bb->count;
 
   e1->flags |= EDGE_FALSE_VALUE | (irr ? EDGE_IRREDUCIBLE_LOOP : 0);
   e1->flags &= ~EDGE_FALLTHRU;
 
   e1->probability = flag_probability.invert ();
-  e1->count = new_bb->count - then_bb->count;
 
   then_old_edge = make_single_succ_edge (then_bb, old_dest,
 			     EDGE_FALLTHRU | (irr ? EDGE_IRREDUCIBLE_LOOP : 0));
diff --git a/gcc/tree-ssa-loop-ivcanon.c b/gcc/tree-ssa-loop-ivcanon.c
index efb199aaaa2..8b1daa66b3e 100644
--- a/gcc/tree-ssa-loop-ivcanon.c
+++ b/gcc/tree-ssa-loop-ivcanon.c
@@ -530,7 +530,6 @@ remove_exits_and_undefined_stmts (struct loop *loop, unsigned int npeeled)
 	  if (!loop_exit_edge_p (loop, exit_edge))
 	    exit_edge = EDGE_SUCC (bb, 1);
 	  exit_edge->probability = profile_probability::always ();
-	  exit_edge->count = exit_edge->src->count;
 	  gcc_checking_assert (loop_exit_edge_p (loop, exit_edge));
 	  gcond *cond_stmt = as_a <gcond *> (elt->stmt);
 	  if (exit_edge->flags & EDGE_TRUE_VALUE)
@@ -643,13 +642,11 @@ unloop_loops (bitmap loop_closed_ssa_invalidated,
       stmt = gimple_build_call (builtin_decl_implicit (BUILT_IN_UNREACHABLE), 0);
       latch_edge = make_edge (latch, create_basic_block (NULL, NULL, latch), flags);
       latch_edge->probability = profile_probability::never ();
-      latch_edge->count = profile_count::zero ();
       latch_edge->flags |= flags;
       latch_edge->goto_locus = locus;
 
       add_bb_to_loop (latch_edge->dest, current_loops->tree_root);
       latch_edge->dest->count = profile_count::zero ();
-      latch_edge->dest->frequency = 0;
       set_immediate_dominator (CDI_DOMINATORS, latch_edge->dest, latch_edge->src);
 
       gsi = gsi_start_bb (latch_edge->dest);
@@ -1092,7 +1089,6 @@ try_peel_loop (struct loop *loop,
 	}
     }
   profile_count entry_count = profile_count::zero ();
-  int entry_freq = 0;
 
   edge e;
   edge_iterator ei;
@@ -1101,15 +1097,10 @@ try_peel_loop (struct loop *loop,
       {
 	if (e->src->count.initialized_p ())
 	  entry_count = e->src->count + e->src->count;
-	entry_freq += e->src->frequency;
 	gcc_assert (!flow_bb_inside_loop_p (loop, e->src));
       }
   profile_probability p = profile_probability::very_unlikely ();
-  if (loop->header->count > 0)
-    p = entry_count.probability_in (loop->header->count);
-  else if (loop->header->frequency)
-    p = profile_probability::probability_in_gcov_type
-		 (entry_freq, loop->header->frequency);
+  p = entry_count.probability_in (loop->header->count);
   scale_loop_profile (loop, p, 0);
   bitmap_set_bit (peeled_loops, loop->num);
   return true;
diff --git a/gcc/tree-ssa-loop-ivopts.c b/gcc/tree-ssa-loop-ivopts.c
index fd23eba8158..e550c850e25 100644
--- a/gcc/tree-ssa-loop-ivopts.c
+++ b/gcc/tree-ssa-loop-ivopts.c
@@ -1571,9 +1571,6 @@ record_invariant (struct ivopts_data *data, tree op, bool nonlinear_use)
   bitmap_set_bit (data->relevant, SSA_NAME_VERSION (op));
 }
 
-static tree
-strip_offset (tree expr, poly_uint64 *offset);
-
 /* Record a group of TYPE.  */
 
 static struct iv_group *
@@ -2724,7 +2721,7 @@ split_address_groups (struct ivopts_data *data)
 	  /* Split group if aksed to, or the offset against the first
 	     use can't fit in offset part of addressing mode.  IV uses
 	     having the same offset are still kept in one group.  */
-	  if (maybe_nonzero (offset)
+	  if (may_ne (offset, 0)
 	      && (split_p || !addr_offset_valid_p (use, offset)))
 	    {
 	      if (!new_group)
@@ -2920,7 +2917,7 @@ strip_offset_1 (tree expr, bool inside_addr, bool top_compref,
       break;
 
     default:
-      if (ptrdiff_tree_p (expr, offset) && maybe_nonzero (*offset))
+      if (ptrdiff_tree_p (expr, offset) && may_ne (*offset, 0))
 	return build_int_cst (orig_type, 0);
       return orig_expr;
     }
@@ -2950,8 +2947,8 @@ strip_offset_1 (tree expr, bool inside_addr, bool top_compref,
 
 /* Strips constant offsets from EXPR and stores them to OFFSET.  */
 
-static tree
-strip_offset (tree expr, poly_uint64 *offset)
+tree
+strip_offset (tree expr, poly_uint64_pod *offset)
 {
   poly_int64 off;
   tree core = strip_offset_1 (expr, false, false, &off);
@@ -3228,7 +3225,7 @@ add_autoinc_candidates (struct ivopts_data *data, tree base, tree step,
      statement.  */
   if (use_bb->loop_father != data->current_loop
       || !dominated_by_p (CDI_DOMINATORS, data->current_loop->latch, use_bb)
-      || stmt_could_throw_p (use->stmt)
+      || stmt_can_throw_internal (use->stmt)
       || !cst_and_fits_in_hwi (step))
     return;
 
@@ -3499,7 +3496,7 @@ add_iv_candidate_for_use (struct ivopts_data *data, struct iv_use *use)
   /* Record common candidate with constant offset stripped in base.
      Like the use itself, we also add candidate directly for it.  */
   base = strip_offset (iv->base, &offset);
-  if (maybe_nonzero (offset) || base != iv->base)
+  if (may_ne (offset, 0U) || base != iv->base)
     {
       record_common_cand (data, base, iv->step, use);
       add_candidate (data, base, iv->step, false, use);
@@ -3518,7 +3515,7 @@ add_iv_candidate_for_use (struct ivopts_data *data, struct iv_use *use)
       record_common_cand (data, base, step, use);
       /* Also record common candidate with offset stripped.  */
       base = strip_offset (base, &offset);
-      if (maybe_nonzero (offset))
+      if (may_ne (offset, 0U))
 	record_common_cand (data, base, step, use);
     }
 
@@ -4385,9 +4382,9 @@ get_address_cost_ainc (poly_int64 ainc_step, poly_int64 ainc_offset,
     }
 
   poly_int64 msize = GET_MODE_SIZE (mem_mode);
-  if (known_zero (ainc_offset) && must_eq (msize, ainc_step))
+  if (must_eq (ainc_offset, 0) && must_eq (msize, ainc_step))
     return comp_cost (data->costs[AINC_POST_INC], 0);
-  if (known_zero (ainc_offset) && must_eq (msize, -ainc_step))
+  if (must_eq (ainc_offset, 0) && must_eq (msize, -ainc_step))
     return comp_cost (data->costs[AINC_POST_DEC], 0);
   if (must_eq (ainc_offset, msize) && must_eq (msize, ainc_step))
     return comp_cost (data->costs[AINC_PRE_INC], 0);
@@ -4429,19 +4426,19 @@ get_address_cost (struct ivopts_data *data, struct iv_use *use,
   if (!aff_combination_const_p (aff_inv))
     {
       parts.index = integer_one_node;
-      /* Addressing mode "base + index [<< scale]".  */
-      parts.step = NULL_TREE;
+      /* Addressing mode "base + index".  */
       ok_without_ratio_p = valid_mem_ref_p (mem_mode, as, &parts);
       if (ratio != 1)
 	{
 	  parts.step = wide_int_to_tree (type, ratio);
+	  /* Addressing mode "base + index << scale".  */
 	  ok_with_ratio_p = valid_mem_ref_p (mem_mode, as, &parts);
 	  if (!ok_with_ratio_p)
 	    parts.step = NULL_TREE;
 	}
       if (ok_with_ratio_p || ok_without_ratio_p)
 	{
-	  if (maybe_nonzero (aff_inv->offset))
+	  if (may_ne (aff_inv->offset, 0))
 	    {
 	      parts.offset = wide_int_to_tree (sizetype, aff_inv->offset);
 	      /* Addressing mode "base + index [<< scale] + offset".  */
@@ -4542,7 +4539,7 @@ get_address_cost (struct ivopts_data *data, struct iv_use *use,
     cost.complexity += 1;
   /* Don't increase the complexity of adding a scaled index if it's
      the only kind of index that the target allows.  */
-  if (ok_with_ratio_p && ok_without_ratio_p)
+  if (parts.step != NULL_TREE && ok_without_ratio_p)
     cost.complexity += 1;
   if (parts.base != NULL_TREE && parts.index != NULL_TREE)
     cost.complexity += 1;
@@ -4559,8 +4556,8 @@ get_address_cost (struct ivopts_data *data, struct iv_use *use,
 static comp_cost
 get_scaled_computation_cost_at (ivopts_data *data, gimple *at, comp_cost cost)
 {
-   int loop_freq = data->current_loop->header->frequency;
-   int bb_freq = gimple_bb (at)->frequency;
+   int loop_freq = data->current_loop->header->count.to_frequency (cfun);
+   int bb_freq = gimple_bb (at)->count.to_frequency (cfun);
    if (loop_freq != 0)
      {
        gcc_assert (cost.scratch <= cost.cost);
diff --git a/gcc/tree-ssa-loop-ivopts.h b/gcc/tree-ssa-loop-ivopts.h
index f8f31e93856..6326bb01eb9 100644
--- a/gcc/tree-ssa-loop-ivopts.h
+++ b/gcc/tree-ssa-loop-ivopts.h
@@ -28,6 +28,7 @@ extern void dump_cand (FILE *, struct iv_cand *);
 extern bool contains_abnormal_ssa_name_p (tree);
 extern struct loop *outermost_invariant_loop_for_expr (struct loop *, tree);
 extern bool expr_invariant_in_loop_p (struct loop *, tree);
+extern tree strip_offset (tree, poly_uint64_pod *);
 bool may_be_nonaddressable_p (tree expr);
 void tree_ssa_iv_optimize (void);
 
diff --git a/gcc/tree-ssa-loop-manip.c b/gcc/tree-ssa-loop-manip.c
index b08b8b9b92c..1efcd272241 100644
--- a/gcc/tree-ssa-loop-manip.c
+++ b/gcc/tree-ssa-loop-manip.c
@@ -1122,6 +1122,9 @@ niter_for_unrolled_loop (struct loop *loop, unsigned factor)
      converts back.  */
   gcov_type new_est_niter = est_niter / factor;
 
+  if (est_niter == -1)
+    return -1;
+
   /* Without profile feedback, loops for which we do not know a better estimate
      are assumed to roll 10 times.  When we unroll such loop, it appears to
      roll too little, and it may even seem to be cold.  To avoid this, we
@@ -1294,12 +1297,10 @@ tree_transform_and_unroll_loop (struct loop *loop, unsigned factor,
   /* Set the probability of new exit to the same of the old one.  Fix
      the frequency of the latch block, by scaling it back by
      1 - exit->probability.  */
-  new_exit->count = exit->count;
   new_exit->probability = exit->probability;
   new_nonexit = single_pred_edge (loop->latch);
   new_nonexit->probability = exit->probability.invert ();
   new_nonexit->flags = EDGE_TRUE_VALUE;
-  new_nonexit->count -= exit->count;
   if (new_nonexit->probability.initialized_p ())
     scale_bbs_frequencies (&loop->latch, 1, new_nonexit->probability);
 
@@ -1371,15 +1372,8 @@ tree_transform_and_unroll_loop (struct loop *loop, unsigned factor,
      exit edge.  */
 
   freq_h = loop->header->count;
-  freq_e = (loop_preheader_edge (loop))->count;
-  /* Use frequency only if counts are zero.  */
-  if (!(freq_h > 0) && !(freq_e > 0))
-    {
-      freq_h = profile_count::from_gcov_type (loop->header->frequency);
-      freq_e = profile_count::from_gcov_type
-		 (EDGE_FREQUENCY (loop_preheader_edge (loop)));
-    }
-  if (freq_h > 0)
+  freq_e = (loop_preheader_edge (loop))->count ();
+  if (freq_h.nonzero_p ())
     {
       /* Avoid dropping loop body profile counter to 0 because of zero count
 	 in loop's preheader.  */
@@ -1390,17 +1384,14 @@ tree_transform_and_unroll_loop (struct loop *loop, unsigned factor,
 
   exit_bb = single_pred (loop->latch);
   new_exit = find_edge (exit_bb, rest);
-  new_exit->count = loop_preheader_edge (loop)->count;
   new_exit->probability = profile_probability::always ()
 				.apply_scale (1, new_est_niter + 1);
 
-  rest->count += new_exit->count;
-  rest->frequency += EDGE_FREQUENCY (new_exit);
+  rest->count += new_exit->count ();
 
   new_nonexit = single_pred_edge (loop->latch);
   prob = new_nonexit->probability;
   new_nonexit->probability = new_exit->probability.invert ();
-  new_nonexit->count = exit_bb->count - new_exit->count;
   prob = new_nonexit->probability / prob;
   if (prob.initialized_p ())
     scale_bbs_frequencies (&loop->latch, 1, prob);
diff --git a/gcc/tree-ssa-loop-niter.c b/gcc/tree-ssa-loop-niter.c
index 89e57931745..9b8b11048db 100644
--- a/gcc/tree-ssa-loop-niter.c
+++ b/gcc/tree-ssa-loop-niter.c
@@ -3901,7 +3901,7 @@ estimate_numbers_of_iterations (struct loop *loop)
      recomputing iteration bounds later in the compilation process will just
      introduce random roundoff errors.  */
   if (!loop->any_estimate
-      && loop->header->count > 0)
+      && loop->header->count.reliable_p ())
     {
       gcov_type nit = expected_loop_iterations_unbounded (loop);
       bound = gcov_type_to_wide_int (nit);
diff --git a/gcc/tree-ssa-loop-split.c b/gcc/tree-ssa-loop-split.c
index e454cc5dc93..dcb7c1ee4c8 100644
--- a/gcc/tree-ssa-loop-split.c
+++ b/gcc/tree-ssa-loop-split.c
@@ -353,11 +353,8 @@ connect_loops (struct loop *loop1, struct loop *loop2)
       new_e->flags |= EDGE_TRUE_VALUE;
     }
 
-  new_e->count = skip_bb->count;
   new_e->probability = profile_probability::likely ();
-  new_e->count = skip_e->count.apply_probability (PROB_LIKELY);
-  skip_e->count -= new_e->count;
-  skip_e->probability = profile_probability::unlikely ();
+  skip_e->probability = new_e->probability.invert ();
 
   return new_e;
 }
@@ -560,7 +557,6 @@ split_loop (struct loop *loop1, struct tree_niter_desc *niter)
 	initialize_original_copy_tables ();
 	basic_block cond_bb;
 
-	/* FIXME: probabilities seems wrong here.  */
 	struct loop *loop2 = loop_version (loop1, cond, &cond_bb,
 					   profile_probability::always (),
 					   profile_probability::always (),
diff --git a/gcc/tree-ssa-loop-unswitch.c b/gcc/tree-ssa-loop-unswitch.c
index 57aba4f1dd0..ecc72cbaf5c 100644
--- a/gcc/tree-ssa-loop-unswitch.c
+++ b/gcc/tree-ssa-loop-unswitch.c
@@ -852,17 +852,16 @@ hoist_guard (struct loop *loop, edge guard)
   /* Determine the probability that we skip the loop.  Assume that loop has
      same average number of iterations regardless outcome of guard.  */
   new_edge->probability = guard->probability;
-  profile_count skip_count = guard->src->count > 0
-		   ? guard->count.apply_scale (pre_header->count,
+  profile_count skip_count = guard->src->count.nonzero_p ()
+		   ? guard->count ().apply_scale (pre_header->count,
 					       guard->src->count)
-		   : guard->count.apply_probability (new_edge->probability);
+		   : guard->count ().apply_probability (new_edge->probability);
 
-  if (skip_count > e->count)
+  if (skip_count > e->count ())
     {
       fprintf (dump_file, "  Capping count; expect profile inconsistency\n");
-      skip_count = e->count;
+      skip_count = e->count ();
     }
-  new_edge->count = skip_count;
   if (dump_file && (dump_flags & TDF_DETAILS))
     {
       fprintf (dump_file, "  Estimated probability of skipping loop is ");
@@ -874,19 +873,13 @@ hoist_guard (struct loop *loop, edge guard)
 
      First decrease count of path from newly hoisted loop guard
      to loop header...  */
-  e->count -= skip_count;
   e->probability = new_edge->probability.invert ();
-  e->dest->count = e->count;
-  e->dest->frequency = EDGE_FREQUENCY (e);
+  e->dest->count = e->count ();
 
   /* ... now update profile to represent that original guard will be optimized
      away ...  */
   guard->probability = profile_probability::never ();
-  guard->count = profile_count::zero ();
   not_guard->probability = profile_probability::always ();
-  /* This count is wrong (frequency of not_guard does not change),
-     but will be scaled later.  */
-  not_guard->count = guard->src->count;
 
   /* ... finally scale everything in the loop except for guarded basic blocks
      where profile does not change.  */
diff --git a/gcc/tree-ssa-math-opts.c b/gcc/tree-ssa-math-opts.c
index 1722ecb3d01..a600516ded8 100644
--- a/gcc/tree-ssa-math-opts.c
+++ b/gcc/tree-ssa-math-opts.c
@@ -515,6 +515,7 @@ internal_fn_reciprocal (gcall *call)
   switch (gimple_call_combined_fn (call))
     {
     CASE_CFN_SQRT:
+    CASE_CFN_SQRT_FN:
       ifn = IFN_RSQRT;
       break;
 
@@ -3258,6 +3259,9 @@ convert_mult_to_widen (gimple *stmt, gimple_stmt_iterator *gsi)
 
   to_mode = SCALAR_INT_TYPE_MODE (type);
   from_mode = SCALAR_INT_TYPE_MODE (type1);
+  if (to_mode == from_mode)
+    return false;
+
   from_unsigned1 = TYPE_UNSIGNED (type1);
   from_unsigned2 = TYPE_UNSIGNED (type2);
 
@@ -3448,6 +3452,9 @@ convert_plusminus_to_widen (gimple_stmt_iterator *gsi, gimple *stmt,
 
   to_mode = SCALAR_TYPE_MODE (type);
   from_mode = SCALAR_TYPE_MODE (type1);
+  if (to_mode == from_mode)
+    return false;
+
   from_unsigned1 = TYPE_UNSIGNED (type1);
   from_unsigned2 = TYPE_UNSIGNED (type2);
   optype = type1;
diff --git a/gcc/tree-ssa-phionlycprop.c b/gcc/tree-ssa-phionlycprop.c
index 65af44834df..fe39aa71f98 100644
--- a/gcc/tree-ssa-phionlycprop.c
+++ b/gcc/tree-ssa-phionlycprop.c
@@ -298,7 +298,6 @@ propagate_rhs_into_lhs (gimple *stmt, tree lhs, tree rhs,
 
 			  te->probability += e->probability;
 
-			  te->count += e->count;
 			  remove_edge (e);
 			  cfg_altered = true;
 			}
diff --git a/gcc/tree-ssa-phiopt.c b/gcc/tree-ssa-phiopt.c
index 1682f18eabb..ed342dac46d 100644
--- a/gcc/tree-ssa-phiopt.c
+++ b/gcc/tree-ssa-phiopt.c
@@ -375,7 +375,6 @@ replace_phi_edge_with_variable (basic_block cond_block,
       EDGE_SUCC (cond_block, 0)->flags |= EDGE_FALLTHRU;
       EDGE_SUCC (cond_block, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
       EDGE_SUCC (cond_block, 0)->probability = profile_probability::always ();
-      EDGE_SUCC (cond_block, 0)->count += EDGE_SUCC (cond_block, 1)->count;
 
       block_to_remove = EDGE_SUCC (cond_block, 1)->dest;
     }
@@ -385,7 +384,6 @@ replace_phi_edge_with_variable (basic_block cond_block,
       EDGE_SUCC (cond_block, 1)->flags
 	&= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
       EDGE_SUCC (cond_block, 1)->probability = profile_probability::always ();
-      EDGE_SUCC (cond_block, 1)->count += EDGE_SUCC (cond_block, 0)->count;
 
       block_to_remove = EDGE_SUCC (cond_block, 0)->dest;
     }
@@ -697,7 +695,7 @@ jump_function_from_stmt (tree *arg, gimple *stmt)
 						&offset);
       if (tem
 	  && TREE_CODE (tem) == MEM_REF
-	  && known_zero (mem_ref_offset (tem) + offset))
+	  && must_eq (mem_ref_offset (tem) + offset, 0))
 	{
 	  *arg = TREE_OPERAND (tem, 0);
 	  return true;
@@ -995,11 +993,13 @@ value_replacement (basic_block cond_bb, basic_block middle_bb,
 
     }
 
-  /* Now optimize (x != 0) ? x + y : y to just y.
-     The following condition is too restrictive, there can easily be another
-     stmt in middle_bb, for instance a CONVERT_EXPR for the second argument.  */
-  gimple *assign = last_and_only_stmt (middle_bb);
-  if (!assign || gimple_code (assign) != GIMPLE_ASSIGN
+  /* Now optimize (x != 0) ? x + y : y to just x + y.  */
+  gsi = gsi_last_nondebug_bb (middle_bb);
+  if (gsi_end_p (gsi))
+    return 0;
+
+  gimple *assign = gsi_stmt (gsi);
+  if (!is_gimple_assign (assign)
       || gimple_assign_rhs_class (assign) != GIMPLE_BINARY_RHS
       || (!INTEGRAL_TYPE_P (TREE_TYPE (arg0))
 	  && !POINTER_TYPE_P (TREE_TYPE (arg0))))
@@ -1009,6 +1009,71 @@ value_replacement (basic_block cond_bb, basic_block middle_bb,
   if (!gimple_seq_empty_p (phi_nodes (middle_bb)))
     return 0;
 
+  /* Allow up to 2 cheap preparation statements that prepare argument
+     for assign, e.g.:
+      if (y_4 != 0)
+	goto <bb 3>;
+      else
+	goto <bb 4>;
+     <bb 3>:
+      _1 = (int) y_4;
+      iftmp.0_6 = x_5(D) r<< _1;
+     <bb 4>:
+      # iftmp.0_2 = PHI <iftmp.0_6(3), x_5(D)(2)>
+     or:
+      if (y_3(D) == 0)
+	goto <bb 4>;
+      else
+	goto <bb 3>;
+     <bb 3>:
+      y_4 = y_3(D) & 31;
+      _1 = (int) y_4;
+      _6 = x_5(D) r<< _1;
+     <bb 4>:
+      # _2 = PHI <x_5(D)(2), _6(3)>  */
+  gimple *prep_stmt[2] = { NULL, NULL };
+  int prep_cnt;
+  for (prep_cnt = 0; ; prep_cnt++)
+    {
+      gsi_prev_nondebug (&gsi);
+      if (gsi_end_p (gsi))
+	break;
+
+      gimple *g = gsi_stmt (gsi);
+      if (gimple_code (g) == GIMPLE_LABEL)
+	break;
+
+      if (prep_cnt == 2 || !is_gimple_assign (g))
+	return 0;
+
+      tree lhs = gimple_assign_lhs (g);
+      tree rhs1 = gimple_assign_rhs1 (g);
+      use_operand_p use_p;
+      gimple *use_stmt;
+      if (TREE_CODE (lhs) != SSA_NAME
+	  || TREE_CODE (rhs1) != SSA_NAME
+	  || !INTEGRAL_TYPE_P (TREE_TYPE (lhs))
+	  || !INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
+	  || !single_imm_use (lhs, &use_p, &use_stmt)
+	  || use_stmt != (prep_cnt ? prep_stmt[prep_cnt - 1] : assign))
+	return 0;
+      switch (gimple_assign_rhs_code (g))
+	{
+	CASE_CONVERT:
+	  break;
+	case PLUS_EXPR:
+	case BIT_AND_EXPR:
+	case BIT_IOR_EXPR:
+	case BIT_XOR_EXPR:
+	  if (TREE_CODE (gimple_assign_rhs2 (g)) != INTEGER_CST)
+	    return 0;
+	  break;
+	default:
+	  return 0;
+	}
+      prep_stmt[prep_cnt] = g;
+    }
+
   /* Only transform if it removes the condition.  */
   if (!single_non_singleton_phi_for_edges (phi_nodes (gimple_bb (phi)), e0, e1))
     return 0;
@@ -1019,7 +1084,7 @@ value_replacement (basic_block cond_bb, basic_block middle_bb,
       && profile_status_for_fn (cfun) != PROFILE_ABSENT
       && EDGE_PRED (middle_bb, 0)->probability < profile_probability::even ()
       /* If assign is cheap, there is no point avoiding it.  */
-      && estimate_num_insns (assign, &eni_time_weights)
+      && estimate_num_insns (bb_seq (middle_bb), &eni_time_weights)
 	 >= 3 * estimate_num_insns (cond, &eni_time_weights))
     return 0;
 
@@ -1030,6 +1095,32 @@ value_replacement (basic_block cond_bb, basic_block middle_bb,
   tree cond_lhs = gimple_cond_lhs (cond);
   tree cond_rhs = gimple_cond_rhs (cond);
 
+  /* Propagate the cond_rhs constant through preparation stmts,
+     make sure UB isn't invoked while doing that.  */
+  for (int i = prep_cnt - 1; i >= 0; --i)
+    {
+      gimple *g = prep_stmt[i];
+      tree grhs1 = gimple_assign_rhs1 (g);
+      if (!operand_equal_for_phi_arg_p (cond_lhs, grhs1))
+	return 0;
+      cond_lhs = gimple_assign_lhs (g);
+      cond_rhs = fold_convert (TREE_TYPE (grhs1), cond_rhs);
+      if (TREE_CODE (cond_rhs) != INTEGER_CST
+	  || TREE_OVERFLOW (cond_rhs))
+	return 0;
+      if (gimple_assign_rhs_class (g) == GIMPLE_BINARY_RHS)
+	{
+	  cond_rhs = int_const_binop (gimple_assign_rhs_code (g), cond_rhs,
+				      gimple_assign_rhs2 (g));
+	  if (TREE_OVERFLOW (cond_rhs))
+	    return 0;
+	}
+      cond_rhs = fold_convert (TREE_TYPE (cond_lhs), cond_rhs);
+      if (TREE_CODE (cond_rhs) != INTEGER_CST
+	  || TREE_OVERFLOW (cond_rhs))
+	return 0;
+    }
+
   if (((code == NE_EXPR && e1 == false_edge)
 	|| (code == EQ_EXPR && e1 == true_edge))
       && arg0 == lhs
@@ -1071,7 +1162,15 @@ value_replacement (basic_block cond_bb, basic_block middle_bb,
 	    duplicate_ssa_name_range_info (lhs, SSA_NAME_RANGE_TYPE (phires),
 					   phires_range_info);
 	}
-      gimple_stmt_iterator gsi_from = gsi_for_stmt (assign);
+      gimple_stmt_iterator gsi_from;
+      for (int i = prep_cnt - 1; i >= 0; --i)
+	{
+	  tree plhs = gimple_assign_lhs (prep_stmt[i]);
+	  SSA_NAME_RANGE_INFO (plhs) = NULL;
+	  gsi_from = gsi_for_stmt (prep_stmt[i]);
+	  gsi_move_before (&gsi_from, &gsi);
+	}
+      gsi_from = gsi_for_stmt (assign);
       gsi_move_before (&gsi_from, &gsi);
       replace_phi_edge_with_variable (cond_bb, e1, phi, lhs);
       return 2;
@@ -1813,9 +1912,24 @@ cond_store_replacement (basic_block middle_bb, basic_block join_bb,
   gsi_remove (&gsi, true);
   release_defs (assign);
 
+  /* Make both store and load use alias-set zero as we have to
+     deal with the case of the store being a conditional change
+     of the dynamic type.  */
+  lhs = unshare_expr (lhs);
+  tree *basep = &lhs;
+  while (handled_component_p (*basep))
+    basep = &TREE_OPERAND (*basep, 0);
+  if (TREE_CODE (*basep) == MEM_REF
+      || TREE_CODE (*basep) == TARGET_MEM_REF)
+    TREE_OPERAND (*basep, 1)
+      = fold_convert (ptr_type_node, TREE_OPERAND (*basep, 1));
+  else
+    *basep = build2 (MEM_REF, TREE_TYPE (*basep),
+		     build_fold_addr_expr (*basep),
+		     build_zero_cst (ptr_type_node));
+
   /* 2) Insert a load from the memory of the store to the temporary
         on the edge which did not contain the store.  */
-  lhs = unshare_expr (lhs);
   name = make_temp_ssa_name (TREE_TYPE (lhs), NULL, "cstore");
   new_stmt = gimple_build_assign (name, lhs);
   gimple_set_location (new_stmt, locus);
diff --git a/gcc/tree-ssa-pre.c b/gcc/tree-ssa-pre.c
index c9079cd6751..45a82f95eb2 100644
--- a/gcc/tree-ssa-pre.c
+++ b/gcc/tree-ssa-pre.c
@@ -39,7 +39,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimplify.h"
 #include "gimple-iterator.h"
 #include "tree-cfg.h"
-#include "tree-ssa-loop.h"
 #include "tree-into-ssa.h"
 #include "tree-dfa.h"
 #include "tree-ssa.h"
@@ -50,9 +49,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "dbgcnt.h"
 #include "domwalk.h"
 #include "tree-ssa-propagate.h"
-#include "ipa-utils.h"
 #include "tree-cfgcleanup.h"
-#include "langhooks.h"
 #include "alias.h"
 
 /* Even though this file is called tree-ssa-pre.c, we actually
@@ -516,9 +513,6 @@ typedef struct bb_bitmap_sets
    optimization PRE was able to perform.  */
 static struct
 {
-  /* The number of RHS computations eliminated by PRE.  */
-  int eliminations;
-
   /* The number of new expressions/temporaries generated by PRE.  */
   int insertions;
 
@@ -537,7 +531,6 @@ static pre_expr bitmap_find_leader (bitmap_set_t, unsigned int);
 static void bitmap_value_insert_into_set (bitmap_set_t, pre_expr);
 static void bitmap_value_replace_in_set (bitmap_set_t, pre_expr);
 static void bitmap_set_copy (bitmap_set_t, bitmap_set_t);
-static void bitmap_set_and (bitmap_set_t, bitmap_set_t);
 static bool bitmap_set_contains_value (bitmap_set_t, unsigned int);
 static void bitmap_insert_into_set (bitmap_set_t, pre_expr);
 static bitmap_set_t bitmap_set_new (void);
@@ -552,12 +545,6 @@ static unsigned int get_expr_value_id (pre_expr);
 static object_allocator<bitmap_set> bitmap_set_pool ("Bitmap sets");
 static bitmap_obstack grand_bitmap_obstack;
 
-/* Set of blocks with statements that have had their EH properties changed.  */
-static bitmap need_eh_cleanup;
-
-/* Set of blocks with statements that have had their AB properties changed.  */
-static bitmap need_ab_cleanup;
-
 /* A three tuple {e, pred, v} used to cache phi translations in the
    phi_translate_table.  */
 
@@ -720,14 +707,11 @@ sccvn_valnum_from_value_id (unsigned int val)
 /* Remove an expression EXPR from a bitmapped set.  */
 
 static void
-bitmap_remove_from_set (bitmap_set_t set, pre_expr expr)
+bitmap_remove_expr_from_set (bitmap_set_t set, pre_expr expr)
 {
   unsigned int val  = get_expr_value_id (expr);
-  if (!value_id_constant_p (val))
-    {
-      bitmap_clear_bit (&set->values, val);
-      bitmap_clear_bit (&set->expressions, get_expression_id (expr));
-    }
+  bitmap_clear_bit (&set->values, val);
+  bitmap_clear_bit (&set->expressions, get_expression_id (expr));
 }
 
 /* Insert an expression EXPR into a bitmapped set.  */
@@ -800,40 +784,10 @@ sorted_array_from_bitmap_set (bitmap_set_t set)
   return result;
 }
 
-/* Perform bitmapped set operation DEST &= ORIG.  */
-
-static void
-bitmap_set_and (bitmap_set_t dest, bitmap_set_t orig)
-{
-  bitmap_iterator bi;
-  unsigned int i;
-
-  if (dest != orig)
-    {
-      bitmap_and_into (&dest->values, &orig->values);
-
-      unsigned int to_clear = -1U;
-      FOR_EACH_EXPR_ID_IN_SET (dest, i, bi)
-	{
-	  if (to_clear != -1U)
-	    {
-	      bitmap_clear_bit (&dest->expressions, to_clear);
-	      to_clear = -1U;
-	    }
-	  pre_expr expr = expression_for_id (i);
-	  unsigned int value_id = get_expr_value_id (expr);
-	  if (!bitmap_bit_p (&dest->values, value_id))
-	    to_clear = i;
-	}
-      if (to_clear != -1U)
-	bitmap_clear_bit (&dest->expressions, to_clear);
-    }
-}
-
 /* Subtract all expressions contained in ORIG from DEST.  */
 
 static bitmap_set_t
-bitmap_set_subtract (bitmap_set_t dest, bitmap_set_t orig)
+bitmap_set_subtract_expressions (bitmap_set_t dest, bitmap_set_t orig)
 {
   bitmap_set_t result = bitmap_set_new ();
   bitmap_iterator bi;
@@ -864,15 +818,15 @@ bitmap_set_subtract_values (bitmap_set_t a, bitmap_set_t b)
     {
       if (to_remove)
 	{
-	  bitmap_remove_from_set (a, to_remove);
+	  bitmap_remove_expr_from_set (a, to_remove);
 	  to_remove = NULL;
 	}
       pre_expr expr = expression_for_id (i);
-      if (bitmap_set_contains_value (b, get_expr_value_id (expr)))
+      if (bitmap_bit_p (&b->values, get_expr_value_id (expr)))
 	to_remove = expr;
     }
   if (to_remove)
-    bitmap_remove_from_set (a, to_remove);
+    bitmap_remove_expr_from_set (a, to_remove);
 }
 
 
@@ -884,9 +838,6 @@ bitmap_set_contains_value (bitmap_set_t set, unsigned int value_id)
   if (value_id_constant_p (value_id))
     return true;
 
-  if (!set || bitmap_empty_p (&set->expressions))
-    return false;
-
   return bitmap_bit_p (&set->values, value_id);
 }
 
@@ -896,44 +847,6 @@ bitmap_set_contains_expr (bitmap_set_t set, const pre_expr expr)
   return bitmap_bit_p (&set->expressions, get_expression_id (expr));
 }
 
-/* Replace an instance of value LOOKFOR with expression EXPR in SET.  */
-
-static void
-bitmap_set_replace_value (bitmap_set_t set, unsigned int lookfor,
-			  const pre_expr expr)
-{
-  bitmap exprset;
-  unsigned int i;
-  bitmap_iterator bi;
-
-  if (value_id_constant_p (lookfor))
-    return;
-
-  if (!bitmap_set_contains_value (set, lookfor))
-    return;
-
-  /* The number of expressions having a given value is usually
-     significantly less than the total number of expressions in SET.
-     Thus, rather than check, for each expression in SET, whether it
-     has the value LOOKFOR, we walk the reverse mapping that tells us
-     what expressions have a given value, and see if any of those
-     expressions are in our set.  For large testcases, this is about
-     5-10x faster than walking the bitmap.  If this is somehow a
-     significant lose for some cases, we can choose which set to walk
-     based on the set size.  */
-  exprset = value_expressions[lookfor];
-  EXECUTE_IF_SET_IN_BITMAP (exprset, 0, i, bi)
-    {
-      if (bitmap_clear_bit (&set->expressions, i))
-	{
-	  bitmap_set_bit (&set->expressions, get_expression_id (expr));
-	  return;
-	}
-    }
-
-  gcc_unreachable ();
-}
-
 /* Return true if two bitmap sets are equal.  */
 
 static bool
@@ -949,9 +862,33 @@ static void
 bitmap_value_replace_in_set (bitmap_set_t set, pre_expr expr)
 {
   unsigned int val = get_expr_value_id (expr);
+  if (value_id_constant_p (val))
+    return;
 
   if (bitmap_set_contains_value (set, val))
-    bitmap_set_replace_value (set, val, expr);
+    {
+      /* The number of expressions having a given value is usually
+	 significantly less than the total number of expressions in SET.
+	 Thus, rather than check, for each expression in SET, whether it
+	 has the value LOOKFOR, we walk the reverse mapping that tells us
+	 what expressions have a given value, and see if any of those
+	 expressions are in our set.  For large testcases, this is about
+	 5-10x faster than walking the bitmap.  If this is somehow a
+	 significant lose for some cases, we can choose which set to walk
+	 based on the set size.  */
+      unsigned int i;
+      bitmap_iterator bi;
+      bitmap exprset = value_expressions[val];
+      EXECUTE_IF_SET_IN_BITMAP (exprset, 0, i, bi)
+	{
+	  if (bitmap_clear_bit (&set->expressions, i))
+	    {
+	      bitmap_set_bit (&set->expressions, get_expression_id (expr));
+	      return;
+	    }
+	}
+      gcc_unreachable ();
+    }
   else
     bitmap_insert_into_set (set, expr);
 }
@@ -2010,14 +1947,12 @@ valid_in_sets (bitmap_set_t set1, bitmap_set_t set2, pre_expr expr)
     }
 }
 
-/* Clean the set of expressions that are no longer valid in SET1 or
-   SET2.  This means expressions that are made up of values we have no
-   leaders for in SET1 or SET2.  This version is used for partial
-   anticipation, which means it is not valid in either ANTIC_IN or
-   PA_IN.  */
+/* Clean the set of expressions SET1 that are no longer valid in SET1 or SET2.
+   This means expressions that are made up of values we have no leaders for
+   in SET1 or SET2.  */
 
 static void
-dependent_clean (bitmap_set_t set1, bitmap_set_t set2)
+clean (bitmap_set_t set1, bitmap_set_t set2 = NULL)
 {
   vec<pre_expr> exprs = sorted_array_from_bitmap_set (set1);
   pre_expr expr;
@@ -2026,26 +1961,7 @@ dependent_clean (bitmap_set_t set1, bitmap_set_t set2)
   FOR_EACH_VEC_ELT (exprs, i, expr)
     {
       if (!valid_in_sets (set1, set2, expr))
-	bitmap_remove_from_set (set1, expr);
-    }
-  exprs.release ();
-}
-
-/* Clean the set of expressions that are no longer valid in SET.  This
-   means expressions that are made up of values we have no leaders for
-   in SET.  */
-
-static void
-clean (bitmap_set_t set)
-{
-  vec<pre_expr> exprs = sorted_array_from_bitmap_set (set);
-  pre_expr expr;
-  int i;
-
-  FOR_EACH_VEC_ELT (exprs, i, expr)
-    {
-      if (!valid_in_sets (set, NULL, expr))
-	bitmap_remove_from_set (set, expr);
+	bitmap_remove_expr_from_set (set1, expr);
     }
   exprs.release ();
 }
@@ -2065,7 +1981,7 @@ prune_clobbered_mems (bitmap_set_t set, basic_block block)
       /* Remove queued expr.  */
       if (to_remove)
 	{
-	  bitmap_remove_from_set (set, to_remove);
+	  bitmap_remove_expr_from_set (set, to_remove);
 	  to_remove = NULL;
 	}
 
@@ -2100,7 +2016,7 @@ prune_clobbered_mems (bitmap_set_t set, basic_block block)
 
   /* Remove queued expr.  */
   if (to_remove)
-    bitmap_remove_from_set (set, to_remove);
+    bitmap_remove_expr_from_set (set, to_remove);
 }
 
 static sbitmap has_abnormal_preds;
@@ -2182,17 +2098,54 @@ compute_antic_aux (basic_block block, bool block_has_abnormal_pred_edge)
 
       phi_translate_set (ANTIC_OUT, ANTIC_IN (first), block, first);
 
+      /* If we have multiple successors we need to intersect the ANTIC_OUT
+         sets.  For values that's a simple intersection but for
+	 expressions it is a union.  Given we want to have a single
+	 expression per value in our sets we have to canonicalize.
+	 Avoid randomness and running into cycles like for PR82129 and
+	 canonicalize the expression we choose to the one with the
+	 lowest id.  This requires we actually compute the union first.  */
       FOR_EACH_VEC_ELT (worklist, i, bprime)
 	{
 	  if (!gimple_seq_empty_p (phi_nodes (bprime)))
 	    {
 	      bitmap_set_t tmp = bitmap_set_new ();
 	      phi_translate_set (tmp, ANTIC_IN (bprime), block, bprime);
-	      bitmap_set_and (ANTIC_OUT, tmp);
+	      bitmap_and_into (&ANTIC_OUT->values, &tmp->values);
+	      bitmap_ior_into (&ANTIC_OUT->expressions, &tmp->expressions);
 	      bitmap_set_free (tmp);
 	    }
 	  else
-	    bitmap_set_and (ANTIC_OUT, ANTIC_IN (bprime));
+	    {
+	      bitmap_and_into (&ANTIC_OUT->values, &ANTIC_IN (bprime)->values);
+	      bitmap_ior_into (&ANTIC_OUT->expressions,
+			       &ANTIC_IN (bprime)->expressions);
+	    }
+	}
+      if (! worklist.is_empty ())
+	{
+	  /* Prune expressions not in the value set, canonicalizing to
+	     expression with lowest ID.  */
+	  bitmap_iterator bi;
+	  unsigned int i;
+	  unsigned int to_clear = -1U;
+	  bitmap seen_value = BITMAP_ALLOC (NULL);
+	  FOR_EACH_EXPR_ID_IN_SET (ANTIC_OUT, i, bi)
+	    {
+	      if (to_clear != -1U)
+		{
+		  bitmap_clear_bit (&ANTIC_OUT->expressions, to_clear);
+		  to_clear = -1U;
+		}
+	      pre_expr expr = expression_for_id (i);
+	      unsigned int value_id = get_expr_value_id (expr);
+	      if (!bitmap_bit_p (&ANTIC_OUT->values, value_id)
+		  || !bitmap_set_bit (seen_value, value_id))
+		to_clear = i;
+	    }
+	  if (to_clear != -1U)
+	    bitmap_clear_bit (&ANTIC_OUT->expressions, to_clear);
+	  BITMAP_FREE (seen_value);
 	}
     }
 
@@ -2201,11 +2154,11 @@ compute_antic_aux (basic_block block, bool block_has_abnormal_pred_edge)
   prune_clobbered_mems (ANTIC_OUT, block);
 
   /* Generate ANTIC_OUT - TMP_GEN.  */
-  S = bitmap_set_subtract (ANTIC_OUT, TMP_GEN (block));
+  S = bitmap_set_subtract_expressions (ANTIC_OUT, TMP_GEN (block));
 
   /* Start ANTIC_IN with EXP_GEN - TMP_GEN.  */
-  ANTIC_IN (block) = bitmap_set_subtract (EXP_GEN (block),
-					  TMP_GEN (block));
+  ANTIC_IN (block) = bitmap_set_subtract_expressions (EXP_GEN (block),
+						      TMP_GEN (block));
 
   /* Then union in the ANTIC_OUT - TMP_GEN values,
      to get ANTIC_OUT U EXP_GEN - TMP_GEN */
@@ -2250,8 +2203,7 @@ compute_antic_aux (basic_block block, bool block_has_abnormal_pred_edge)
    else if succs(BLOCK) == 1 then
      PA_OUT[BLOCK] = phi_translate (PA_IN[succ(BLOCK)])
 
-   PA_IN[BLOCK] = dependent_clean(PA_OUT[BLOCK] - TMP_GEN[BLOCK]
-				  - ANTIC_IN[BLOCK])
+   PA_IN[BLOCK] = clean(PA_OUT[BLOCK] - TMP_GEN[BLOCK] - ANTIC_IN[BLOCK])
 
 */
 static void
@@ -2344,7 +2296,7 @@ compute_partial_antic_aux (basic_block block,
 
   /* PA_IN starts with PA_OUT - TMP_GEN.
      Then we subtract things from ANTIC_IN.  */
-  PA_IN (block) = bitmap_set_subtract (PA_OUT, TMP_GEN (block));
+  PA_IN (block) = bitmap_set_subtract_expressions (PA_OUT, TMP_GEN (block));
 
   /* For partial antic, we want to put back in the phi results, since
      we will properly avoid making them partially antic over backedges.  */
@@ -2354,7 +2306,7 @@ compute_partial_antic_aux (basic_block block,
   /* PA_IN[block] = PA_IN[block] - ANTIC_IN[block] */
   bitmap_set_subtract_values (PA_IN (block), ANTIC_IN (block));
 
-  dependent_clean (PA_IN (block), ANTIC_IN (block));
+  clean (PA_IN (block), ANTIC_IN (block));
 
  maybe_dump_sets:
   if (dump_file && (dump_flags & TDF_DETAILS))
@@ -4088,810 +4040,6 @@ compute_avail (void)
   free (worklist);
 }
 
-
-/* Local state for the eliminate domwalk.  */
-static vec<gimple *> el_to_remove;
-static vec<gimple *> el_to_fixup;
-static unsigned int el_todo;
-static vec<tree> el_avail;
-static vec<tree> el_avail_stack;
-
-/* Return a leader for OP that is available at the current point of the
-   eliminate domwalk.  */
-
-static tree
-eliminate_avail (tree op)
-{
-  tree valnum = VN_INFO (op)->valnum;
-  if (TREE_CODE (valnum) == SSA_NAME)
-    {
-      if (SSA_NAME_IS_DEFAULT_DEF (valnum))
-	return valnum;
-      if (el_avail.length () > SSA_NAME_VERSION (valnum))
-	return el_avail[SSA_NAME_VERSION (valnum)];
-    }
-  else if (is_gimple_min_invariant (valnum))
-    return valnum;
-  return NULL_TREE;
-}
-
-/* At the current point of the eliminate domwalk make OP available.  */
-
-static void
-eliminate_push_avail (tree op)
-{
-  tree valnum = VN_INFO (op)->valnum;
-  if (TREE_CODE (valnum) == SSA_NAME)
-    {
-      if (el_avail.length () <= SSA_NAME_VERSION (valnum))
-	el_avail.safe_grow_cleared (SSA_NAME_VERSION (valnum) + 1);
-      tree pushop = op;
-      if (el_avail[SSA_NAME_VERSION (valnum)])
-	pushop = el_avail[SSA_NAME_VERSION (valnum)];
-      el_avail_stack.safe_push (pushop);
-      el_avail[SSA_NAME_VERSION (valnum)] = op;
-    }
-}
-
-/* Insert the expression recorded by SCCVN for VAL at *GSI.  Returns
-   the leader for the expression if insertion was successful.  */
-
-static tree
-eliminate_insert (gimple_stmt_iterator *gsi, tree val)
-{
-  /* We can insert a sequence with a single assignment only.  */
-  gimple_seq stmts = VN_INFO (val)->expr;
-  if (!gimple_seq_singleton_p (stmts))
-    return NULL_TREE;
-  gassign *stmt = dyn_cast <gassign *> (gimple_seq_first_stmt (stmts));
-  if (!stmt
-      || (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
-	  && gimple_assign_rhs_code (stmt) != VIEW_CONVERT_EXPR
-	  && gimple_assign_rhs_code (stmt) != BIT_FIELD_REF
-	  && (gimple_assign_rhs_code (stmt) != BIT_AND_EXPR
-	      || TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)))
-    return NULL_TREE;
-
-  tree op = gimple_assign_rhs1 (stmt);
-  if (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
-      || gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
-    op = TREE_OPERAND (op, 0);
-  tree leader = TREE_CODE (op) == SSA_NAME ? eliminate_avail (op) : op;
-  if (!leader)
-    return NULL_TREE;
-
-  tree res;
-  stmts = NULL;
-  if (gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
-    res = gimple_build (&stmts, BIT_FIELD_REF,
-			TREE_TYPE (val), leader,
-			TREE_OPERAND (gimple_assign_rhs1 (stmt), 1),
-			TREE_OPERAND (gimple_assign_rhs1 (stmt), 2));
-  else if (gimple_assign_rhs_code (stmt) == BIT_AND_EXPR)
-    res = gimple_build (&stmts, BIT_AND_EXPR,
-			TREE_TYPE (val), leader, gimple_assign_rhs2 (stmt));
-  else
-    res = gimple_build (&stmts, gimple_assign_rhs_code (stmt),
-			TREE_TYPE (val), leader);
-  if (TREE_CODE (res) != SSA_NAME
-      || SSA_NAME_IS_DEFAULT_DEF (res)
-      || gimple_bb (SSA_NAME_DEF_STMT (res)))
-    {
-      gimple_seq_discard (stmts);
-
-      /* During propagation we have to treat SSA info conservatively
-         and thus we can end up simplifying the inserted expression
-	 at elimination time to sth not defined in stmts.  */
-      /* But then this is a redundancy we failed to detect.  Which means
-         res now has two values.  That doesn't play well with how
-	 we track availability here, so give up.  */
-      if (dump_file && (dump_flags & TDF_DETAILS))
-	{
-	  if (TREE_CODE (res) == SSA_NAME)
-	    res = eliminate_avail (res);
-	  if (res)
-	    {
-	      fprintf (dump_file, "Failed to insert expression for value ");
-	      print_generic_expr (dump_file, val);
-	      fprintf (dump_file, " which is really fully redundant to ");
-	      print_generic_expr (dump_file, res);
-	      fprintf (dump_file, "\n");
-	    }
-	}
-
-      return NULL_TREE;
-    }
-  else
-    {
-      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
-      VN_INFO_GET (res)->valnum = val;
-    }
-
-  pre_stats.insertions++;
-  if (dump_file && (dump_flags & TDF_DETAILS))
-    {
-      fprintf (dump_file, "Inserted ");
-      print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (res), 0);
-    }
-
-  return res;
-}
-
-class eliminate_dom_walker : public dom_walker
-{
-public:
-  eliminate_dom_walker (cdi_direction direction, bool do_pre_)
-      : dom_walker (direction), do_pre (do_pre_) {}
-
-  virtual edge before_dom_children (basic_block);
-  virtual void after_dom_children (basic_block);
-
-  bool do_pre;
-};
-
-/* Perform elimination for the basic-block B during the domwalk.  */
-
-edge
-eliminate_dom_walker::before_dom_children (basic_block b)
-{
-  /* Mark new bb.  */
-  el_avail_stack.safe_push (NULL_TREE);
-
-  /* Skip unreachable blocks marked unreachable during the SCCVN domwalk.  */
-  edge_iterator ei;
-  edge e;
-  FOR_EACH_EDGE (e, ei, b->preds)
-    if (e->flags & EDGE_EXECUTABLE)
-      break;
-  if (! e)
-    return NULL;
-
-  for (gphi_iterator gsi = gsi_start_phis (b); !gsi_end_p (gsi);)
-    {
-      gphi *phi = gsi.phi ();
-      tree res = PHI_RESULT (phi);
-
-      if (virtual_operand_p (res))
-	{
-	  gsi_next (&gsi);
-	  continue;
-	}
-
-      tree sprime = eliminate_avail (res);
-      if (sprime
-	  && sprime != res)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    {
-	      fprintf (dump_file, "Replaced redundant PHI node defining ");
-	      print_generic_expr (dump_file, res);
-	      fprintf (dump_file, " with ");
-	      print_generic_expr (dump_file, sprime);
-	      fprintf (dump_file, "\n");
-	    }
-
-	  /* If we inserted this PHI node ourself, it's not an elimination.  */
-	  if (inserted_exprs
-	      && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (res)))
-	    pre_stats.phis--;
-	  else
-	    pre_stats.eliminations++;
-
-	  /* If we will propagate into all uses don't bother to do
-	     anything.  */
-	  if (may_propagate_copy (res, sprime))
-	    {
-	      /* Mark the PHI for removal.  */
-	      el_to_remove.safe_push (phi);
-	      gsi_next (&gsi);
-	      continue;
-	    }
-
-	  remove_phi_node (&gsi, false);
-
-	  if (!useless_type_conversion_p (TREE_TYPE (res), TREE_TYPE (sprime)))
-	    sprime = fold_convert (TREE_TYPE (res), sprime);
-	  gimple *stmt = gimple_build_assign (res, sprime);
-	  gimple_stmt_iterator gsi2 = gsi_after_labels (b);
-	  gsi_insert_before (&gsi2, stmt, GSI_NEW_STMT);
-	  continue;
-	}
-
-      eliminate_push_avail (res);
-      gsi_next (&gsi);
-    }
-
-  for (gimple_stmt_iterator gsi = gsi_start_bb (b);
-       !gsi_end_p (gsi);
-       gsi_next (&gsi))
-    {
-      tree sprime = NULL_TREE;
-      gimple *stmt = gsi_stmt (gsi);
-      tree lhs = gimple_get_lhs (stmt);
-      if (lhs && TREE_CODE (lhs) == SSA_NAME
-	  && !gimple_has_volatile_ops (stmt)
-	  /* See PR43491.  Do not replace a global register variable when
-	     it is a the RHS of an assignment.  Do replace local register
-	     variables since gcc does not guarantee a local variable will
-	     be allocated in register.
-	     ???  The fix isn't effective here.  This should instead
-	     be ensured by not value-numbering them the same but treating
-	     them like volatiles?  */
-	  && !(gimple_assign_single_p (stmt)
-	       && (TREE_CODE (gimple_assign_rhs1 (stmt)) == VAR_DECL
-		   && DECL_HARD_REGISTER (gimple_assign_rhs1 (stmt))
-		   && is_global_var (gimple_assign_rhs1 (stmt)))))
-	{
-	  sprime = eliminate_avail (lhs);
-	  if (!sprime)
-	    {
-	      /* If there is no existing usable leader but SCCVN thinks
-		 it has an expression it wants to use as replacement,
-		 insert that.  */
-	      tree val = VN_INFO (lhs)->valnum;
-	      if (val != VN_TOP
-		  && TREE_CODE (val) == SSA_NAME
-		  && VN_INFO (val)->needs_insertion
-		  && VN_INFO (val)->expr != NULL
-		  && (sprime = eliminate_insert (&gsi, val)) != NULL_TREE)
-		eliminate_push_avail (sprime);
-	    }
-
-	  /* If this now constitutes a copy duplicate points-to
-	     and range info appropriately.  This is especially
-	     important for inserted code.  See tree-ssa-copy.c
-	     for similar code.  */
-	  if (sprime
-	      && TREE_CODE (sprime) == SSA_NAME)
-	    {
-	      basic_block sprime_b = gimple_bb (SSA_NAME_DEF_STMT (sprime));
-	      if (POINTER_TYPE_P (TREE_TYPE (lhs))
-		  && VN_INFO_PTR_INFO (lhs)
-		  && ! VN_INFO_PTR_INFO (sprime))
-		{
-		  duplicate_ssa_name_ptr_info (sprime,
-					       VN_INFO_PTR_INFO (lhs));
-		  if (b != sprime_b)
-		    mark_ptr_info_alignment_unknown
-			(SSA_NAME_PTR_INFO (sprime));
-		}
-	      else if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
-		       && VN_INFO_RANGE_INFO (lhs)
-		       && ! VN_INFO_RANGE_INFO (sprime)
-		       && b == sprime_b)
-		duplicate_ssa_name_range_info (sprime,
-					       VN_INFO_RANGE_TYPE (lhs),
-					       VN_INFO_RANGE_INFO (lhs));
-	    }
-
-	  /* Inhibit the use of an inserted PHI on a loop header when
-	     the address of the memory reference is a simple induction
-	     variable.  In other cases the vectorizer won't do anything
-	     anyway (either it's loop invariant or a complicated
-	     expression).  */
-	  if (sprime
-	      && TREE_CODE (sprime) == SSA_NAME
-	      && do_pre
-	      && (flag_tree_loop_vectorize || flag_tree_parallelize_loops > 1)
-	      && loop_outer (b->loop_father)
-	      && has_zero_uses (sprime)
-	      && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))
-	      && gimple_assign_load_p (stmt))
-	    {
-	      gimple *def_stmt = SSA_NAME_DEF_STMT (sprime);
-	      basic_block def_bb = gimple_bb (def_stmt);
-	      if (gimple_code (def_stmt) == GIMPLE_PHI
-		  && def_bb->loop_father->header == def_bb)
-		{
-		  loop_p loop = def_bb->loop_father;
-		  ssa_op_iter iter;
-		  tree op;
-		  bool found = false;
-		  FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
-		    {
-		      affine_iv iv;
-		      def_bb = gimple_bb (SSA_NAME_DEF_STMT (op));
-		      if (def_bb
-			  && flow_bb_inside_loop_p (loop, def_bb)
-			  && simple_iv (loop, loop, op, &iv, true))
-			{
-			  found = true;
-			  break;
-			}
-		    }
-		  if (found)
-		    {
-		      if (dump_file && (dump_flags & TDF_DETAILS))
-			{
-			  fprintf (dump_file, "Not replacing ");
-			  print_gimple_expr (dump_file, stmt, 0);
-			  fprintf (dump_file, " with ");
-			  print_generic_expr (dump_file, sprime);
-			  fprintf (dump_file, " which would add a loop"
-				   " carried dependence to loop %d\n",
-				   loop->num);
-			}
-		      /* Don't keep sprime available.  */
-		      sprime = NULL_TREE;
-		    }
-		}
-	    }
-
-	  if (sprime)
-	    {
-	      /* If we can propagate the value computed for LHS into
-		 all uses don't bother doing anything with this stmt.  */
-	      if (may_propagate_copy (lhs, sprime))
-		{
-		  /* Mark it for removal.  */
-		  el_to_remove.safe_push (stmt);
-
-		  /* ???  Don't count copy/constant propagations.  */
-		  if (gimple_assign_single_p (stmt)
-		      && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
-			  || gimple_assign_rhs1 (stmt) == sprime))
-		    continue;
-
-		  if (dump_file && (dump_flags & TDF_DETAILS))
-		    {
-		      fprintf (dump_file, "Replaced ");
-		      print_gimple_expr (dump_file, stmt, 0);
-		      fprintf (dump_file, " with ");
-		      print_generic_expr (dump_file, sprime);
-		      fprintf (dump_file, " in all uses of ");
-		      print_gimple_stmt (dump_file, stmt, 0);
-		    }
-
-		  pre_stats.eliminations++;
-		  continue;
-		}
-
-	      /* If this is an assignment from our leader (which
-	         happens in the case the value-number is a constant)
-		 then there is nothing to do.  */
-	      if (gimple_assign_single_p (stmt)
-		  && sprime == gimple_assign_rhs1 (stmt))
-		continue;
-
-	      /* Else replace its RHS.  */
-	      bool can_make_abnormal_goto
-		  = is_gimple_call (stmt)
-		  && stmt_can_make_abnormal_goto (stmt);
-
-	      if (dump_file && (dump_flags & TDF_DETAILS))
-		{
-		  fprintf (dump_file, "Replaced ");
-		  print_gimple_expr (dump_file, stmt, 0);
-		  fprintf (dump_file, " with ");
-		  print_generic_expr (dump_file, sprime);
-		  fprintf (dump_file, " in ");
-		  print_gimple_stmt (dump_file, stmt, 0);
-		}
-
-	      pre_stats.eliminations++;
-	      gimple *orig_stmt = stmt;
-	      if (!useless_type_conversion_p (TREE_TYPE (lhs),
-					      TREE_TYPE (sprime)))
-		sprime = fold_convert (TREE_TYPE (lhs), sprime);
-	      tree vdef = gimple_vdef (stmt);
-	      tree vuse = gimple_vuse (stmt);
-	      propagate_tree_value_into_stmt (&gsi, sprime);
-	      stmt = gsi_stmt (gsi);
-	      update_stmt (stmt);
-	      if (vdef != gimple_vdef (stmt))
-		VN_INFO (vdef)->valnum = vuse;
-
-	      /* If we removed EH side-effects from the statement, clean
-		 its EH information.  */
-	      if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
-		{
-		  bitmap_set_bit (need_eh_cleanup,
-				  gimple_bb (stmt)->index);
-		  if (dump_file && (dump_flags & TDF_DETAILS))
-		    fprintf (dump_file, "  Removed EH side-effects.\n");
-		}
-
-	      /* Likewise for AB side-effects.  */
-	      if (can_make_abnormal_goto
-		  && !stmt_can_make_abnormal_goto (stmt))
-		{
-		  bitmap_set_bit (need_ab_cleanup,
-				  gimple_bb (stmt)->index);
-		  if (dump_file && (dump_flags & TDF_DETAILS))
-		    fprintf (dump_file, "  Removed AB side-effects.\n");
-		}
-
-	      continue;
-	    }
-	}
-
-      /* If the statement is a scalar store, see if the expression
-         has the same value number as its rhs.  If so, the store is
-         dead.  */
-      if (gimple_assign_single_p (stmt)
-	  && !gimple_has_volatile_ops (stmt)
-	  && !is_gimple_reg (gimple_assign_lhs (stmt))
-	  && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
-	      || is_gimple_min_invariant (gimple_assign_rhs1 (stmt))))
-	{
-	  tree val;
-	  tree rhs = gimple_assign_rhs1 (stmt);
-	  vn_reference_t vnresult;
-	  val = vn_reference_lookup (lhs, gimple_vuse (stmt), VN_WALKREWRITE,
-				     &vnresult, false);
-	  if (TREE_CODE (rhs) == SSA_NAME)
-	    rhs = VN_INFO (rhs)->valnum;
-	  if (val
-	      && operand_equal_p (val, rhs, 0))
-	    {
-	      /* We can only remove the later store if the former aliases
-		 at least all accesses the later one does or if the store
-		 was to readonly memory storing the same value.  */
-	      alias_set_type set = get_alias_set (lhs);
-	      if (! vnresult
-		  || vnresult->set == set
-		  || alias_set_subset_of (set, vnresult->set))
-		{
-		  if (dump_file && (dump_flags & TDF_DETAILS))
-		    {
-		      fprintf (dump_file, "Deleted redundant store ");
-		      print_gimple_stmt (dump_file, stmt, 0);
-		    }
-
-		  /* Queue stmt for removal.  */
-		  el_to_remove.safe_push (stmt);
-		  continue;
-		}
-	    }
-	}
-
-      /* If this is a control statement value numbering left edges
-	 unexecuted on force the condition in a way consistent with
-	 that.  */
-      if (gcond *cond = dyn_cast <gcond *> (stmt))
-	{
-	  if ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE)
-	      ^ (EDGE_SUCC (b, 1)->flags & EDGE_EXECUTABLE))
-	    {
-              if (dump_file && (dump_flags & TDF_DETAILS))
-                {
-                  fprintf (dump_file, "Removing unexecutable edge from ");
-		  print_gimple_stmt (dump_file, stmt, 0);
-                }
-	      if (((EDGE_SUCC (b, 0)->flags & EDGE_TRUE_VALUE) != 0)
-		  == ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE) != 0))
-		gimple_cond_make_true (cond);
-	      else
-		gimple_cond_make_false (cond);
-	      update_stmt (cond);
-	      el_todo |= TODO_cleanup_cfg;
-	      continue;
-	    }
-	}
-
-      bool can_make_abnormal_goto = stmt_can_make_abnormal_goto (stmt);
-      bool was_noreturn = (is_gimple_call (stmt)
-			   && gimple_call_noreturn_p (stmt));
-      tree vdef = gimple_vdef (stmt);
-      tree vuse = gimple_vuse (stmt);
-
-      /* If we didn't replace the whole stmt (or propagate the result
-         into all uses), replace all uses on this stmt with their
-	 leaders.  */
-      bool modified = false;
-      use_operand_p use_p;
-      ssa_op_iter iter;
-      FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
-	{
-	  tree use = USE_FROM_PTR (use_p);
-	  /* ???  The call code above leaves stmt operands un-updated.  */
-	  if (TREE_CODE (use) != SSA_NAME)
-	    continue;
-	  tree sprime = eliminate_avail (use);
-	  if (sprime && sprime != use
-	      && may_propagate_copy (use, sprime)
-	      /* We substitute into debug stmts to avoid excessive
-	         debug temporaries created by removed stmts, but we need
-		 to avoid doing so for inserted sprimes as we never want
-		 to create debug temporaries for them.  */
-	      && (!inserted_exprs
-		  || TREE_CODE (sprime) != SSA_NAME
-		  || !is_gimple_debug (stmt)
-		  || !bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))))
-	    {
-	      propagate_value (use_p, sprime);
-	      modified = true;
-	    }
-	}
-
-      /* Fold the stmt if modified, this canonicalizes MEM_REFs we propagated
-         into which is a requirement for the IPA devirt machinery.  */
-      gimple *old_stmt = stmt;
-      if (modified)
-	{
-	  /* If a formerly non-invariant ADDR_EXPR is turned into an
-	     invariant one it was on a separate stmt.  */
-	  if (gimple_assign_single_p (stmt)
-	      && TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
-	    recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
-	  gimple_stmt_iterator prev = gsi;
-	  gsi_prev (&prev);
-	  if (fold_stmt (&gsi))
-	    {
-	      /* fold_stmt may have created new stmts inbetween
-		 the previous stmt and the folded stmt.  Mark
-		 all defs created there as varying to not confuse
-		 the SCCVN machinery as we're using that even during
-		 elimination.  */
-	      if (gsi_end_p (prev))
-		prev = gsi_start_bb (b);
-	      else
-		gsi_next (&prev);
-	      if (gsi_stmt (prev) != gsi_stmt (gsi))
-		do
-		  {
-		    tree def;
-		    ssa_op_iter dit;
-		    FOR_EACH_SSA_TREE_OPERAND (def, gsi_stmt (prev),
-					       dit, SSA_OP_ALL_DEFS)
-		      /* As existing DEFs may move between stmts
-			 we have to guard VN_INFO_GET.  */
-		      if (! has_VN_INFO (def))
-			VN_INFO_GET (def)->valnum = def;
-		    if (gsi_stmt (prev) == gsi_stmt (gsi))
-		      break;
-		    gsi_next (&prev);
-		  }
-		while (1);
-	    }
-	  stmt = gsi_stmt (gsi);
-	  /* In case we folded the stmt away schedule the NOP for removal.  */
-	  if (gimple_nop_p (stmt))
-	    el_to_remove.safe_push (stmt);
-	}
-
-      /* Visit indirect calls and turn them into direct calls if
-	 possible using the devirtualization machinery.  Do this before
-	 checking for required EH/abnormal/noreturn cleanup as devird
-	 may expose more of those.  */
-      if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
-	{
-	  tree fn = gimple_call_fn (call_stmt);
-	  if (fn
-	      && flag_devirtualize
-	      && virtual_method_call_p (fn))
-	    {
-	      tree otr_type = obj_type_ref_class (fn);
-	      unsigned HOST_WIDE_INT otr_tok
-		= tree_to_uhwi (OBJ_TYPE_REF_TOKEN (fn));
-	      tree instance;
-	      ipa_polymorphic_call_context context (current_function_decl,
-						    fn, stmt, &instance);
-	      context.get_dynamic_type (instance, OBJ_TYPE_REF_OBJECT (fn),
-					otr_type, stmt);
-	      bool final;
-	      vec <cgraph_node *> targets
-		= possible_polymorphic_call_targets (obj_type_ref_class (fn),
-						     otr_tok, context, &final);
-	      if (dump_file)
-		dump_possible_polymorphic_call_targets (dump_file, 
-							obj_type_ref_class (fn),
-							otr_tok, context);
-	      if (final && targets.length () <= 1 && dbg_cnt (devirt))
-		{
-		  tree fn;
-		  if (targets.length () == 1)
-		    fn = targets[0]->decl;
-		  else
-		    fn = builtin_decl_implicit (BUILT_IN_UNREACHABLE);
-		  if (dump_enabled_p ())
-		    {
-		      location_t loc = gimple_location (stmt);
-		      dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
-				       "converting indirect call to "
-				       "function %s\n",
-				       lang_hooks.decl_printable_name (fn, 2));
-		    }
-		  gimple_call_set_fndecl (call_stmt, fn);
-		  /* If changing the call to __builtin_unreachable
-		     or similar noreturn function, adjust gimple_call_fntype
-		     too.  */
-		  if (gimple_call_noreturn_p (call_stmt)
-		      && VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fn)))
-		      && TYPE_ARG_TYPES (TREE_TYPE (fn))
-		      && (TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fn)))
-			  == void_type_node))
-		    gimple_call_set_fntype (call_stmt, TREE_TYPE (fn));
-		  maybe_remove_unused_call_args (cfun, call_stmt);
-		  modified = true;
-		}
-	    }
-	}
-
-      if (modified)
-	{
-	  /* When changing a call into a noreturn call, cfg cleanup
-	     is needed to fix up the noreturn call.  */
-	  if (!was_noreturn
-	      && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
-	    el_to_fixup.safe_push  (stmt);
-	  /* When changing a condition or switch into one we know what
-	     edge will be executed, schedule a cfg cleanup.  */
-	  if ((gimple_code (stmt) == GIMPLE_COND
-	       && (gimple_cond_true_p (as_a <gcond *> (stmt))
-		   || gimple_cond_false_p (as_a <gcond *> (stmt))))
-	      || (gimple_code (stmt) == GIMPLE_SWITCH
-		  && TREE_CODE (gimple_switch_index
-				  (as_a <gswitch *> (stmt))) == INTEGER_CST))
-	    el_todo |= TODO_cleanup_cfg;
-	  /* If we removed EH side-effects from the statement, clean
-	     its EH information.  */
-	  if (maybe_clean_or_replace_eh_stmt (old_stmt, stmt))
-	    {
-	      bitmap_set_bit (need_eh_cleanup,
-			      gimple_bb (stmt)->index);
-	      if (dump_file && (dump_flags & TDF_DETAILS))
-		fprintf (dump_file, "  Removed EH side-effects.\n");
-	    }
-	  /* Likewise for AB side-effects.  */
-	  if (can_make_abnormal_goto
-	      && !stmt_can_make_abnormal_goto (stmt))
-	    {
-	      bitmap_set_bit (need_ab_cleanup,
-			      gimple_bb (stmt)->index);
-	      if (dump_file && (dump_flags & TDF_DETAILS))
-		fprintf (dump_file, "  Removed AB side-effects.\n");
-	    }
-	  update_stmt (stmt);
-	  if (vdef != gimple_vdef (stmt))
-	    VN_INFO (vdef)->valnum = vuse;
-	}
-
-      /* Make new values available - for fully redundant LHS we
-         continue with the next stmt above and skip this.  */
-      def_operand_p defp;
-      FOR_EACH_SSA_DEF_OPERAND (defp, stmt, iter, SSA_OP_DEF)
-	eliminate_push_avail (DEF_FROM_PTR (defp));
-    }
-
-  /* Replace destination PHI arguments.  */
-  FOR_EACH_EDGE (e, ei, b->succs)
-    if (e->flags & EDGE_EXECUTABLE)
-      for (gphi_iterator gsi = gsi_start_phis (e->dest);
-	   !gsi_end_p (gsi);
-	   gsi_next (&gsi))
-	{
-	  gphi *phi = gsi.phi ();
-	  use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
-	  tree arg = USE_FROM_PTR (use_p);
-	  if (TREE_CODE (arg) != SSA_NAME
-	      || virtual_operand_p (arg))
-	    continue;
-	  tree sprime = eliminate_avail (arg);
-	  if (sprime && may_propagate_copy (arg, sprime))
-	    propagate_value (use_p, sprime);
-	}
-  return NULL;
-}
-
-/* Make no longer available leaders no longer available.  */
-
-void
-eliminate_dom_walker::after_dom_children (basic_block)
-{
-  tree entry;
-  while ((entry = el_avail_stack.pop ()) != NULL_TREE)
-    {
-      tree valnum = VN_INFO (entry)->valnum;
-      tree old = el_avail[SSA_NAME_VERSION (valnum)];
-      if (old == entry)
-	el_avail[SSA_NAME_VERSION (valnum)] = NULL_TREE;
-      else
-	el_avail[SSA_NAME_VERSION (valnum)] = entry;
-    }
-}
-
-/* Eliminate fully redundant computations.  */
-
-static unsigned int
-eliminate (bool do_pre)
-{
-  need_eh_cleanup = BITMAP_ALLOC (NULL);
-  need_ab_cleanup = BITMAP_ALLOC (NULL);
-
-  el_to_remove.create (0);
-  el_to_fixup.create (0);
-  el_todo = 0;
-  el_avail.create (num_ssa_names);
-  el_avail_stack.create (0);
-
-  eliminate_dom_walker (CDI_DOMINATORS,
-			do_pre).walk (cfun->cfg->x_entry_block_ptr);
-
-  el_avail.release ();
-  el_avail_stack.release ();
-
-  return el_todo;
-}
-
-/* Perform CFG cleanups made necessary by elimination.  */
-
-static unsigned 
-fini_eliminate (void)
-{
-  gimple_stmt_iterator gsi;
-  gimple *stmt;
-  unsigned todo = 0;
-
-  /* We cannot remove stmts during BB walk, especially not release SSA
-     names there as this confuses the VN machinery.  The stmts ending
-     up in el_to_remove are either stores or simple copies.
-     Remove stmts in reverse order to make debug stmt creation possible.  */
-  while (!el_to_remove.is_empty ())
-    {
-      stmt = el_to_remove.pop ();
-
-      if (dump_file && (dump_flags & TDF_DETAILS))
-	{
-	  fprintf (dump_file, "Removing dead stmt ");
-	  print_gimple_stmt (dump_file, stmt, 0, 0);
-	}
-
-      gsi = gsi_for_stmt (stmt);
-      if (gimple_code (stmt) == GIMPLE_PHI)
-	remove_phi_node (&gsi, true);
-      else
-	{
-	  basic_block bb = gimple_bb (stmt);
-	  unlink_stmt_vdef (stmt);
-	  if (gsi_remove (&gsi, true))
-	    bitmap_set_bit (need_eh_cleanup, bb->index);
-	  if (is_gimple_call (stmt) && stmt_can_make_abnormal_goto (stmt))
-	    bitmap_set_bit (need_ab_cleanup, bb->index);
-	  release_defs (stmt);
-	}
-
-      /* Removing a stmt may expose a forwarder block.  */
-      todo |= TODO_cleanup_cfg;
-    }
-  el_to_remove.release ();
-
-  /* Fixup stmts that became noreturn calls.  This may require splitting
-     blocks and thus isn't possible during the dominator walk.  Do this
-     in reverse order so we don't inadvertedly remove a stmt we want to
-     fixup by visiting a dominating now noreturn call first.  */
-  while (!el_to_fixup.is_empty ())
-    {
-      stmt = el_to_fixup.pop ();
-
-      if (dump_file && (dump_flags & TDF_DETAILS))
-	{
-	  fprintf (dump_file, "Fixing up noreturn call ");
-	  print_gimple_stmt (dump_file, stmt, 0);
-	}
-
-      if (fixup_noreturn_call (stmt))
-	todo |= TODO_cleanup_cfg;
-    }
-  el_to_fixup.release ();
-
-  bool do_eh_cleanup = !bitmap_empty_p (need_eh_cleanup);
-  bool do_ab_cleanup = !bitmap_empty_p (need_ab_cleanup);
-
-  if (do_eh_cleanup)
-    gimple_purge_all_dead_eh_edges (need_eh_cleanup);
-
-  if (do_ab_cleanup)
-    gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
-
-  BITMAP_FREE (need_eh_cleanup);
-  BITMAP_FREE (need_ab_cleanup);
-
-  if (do_eh_cleanup || do_ab_cleanup)
-    todo |= TODO_cleanup_cfg;
-  return todo;
-}
-
 /* Cheap DCE of a known set of possibly dead stmts.
 
    Because we don't follow exactly the standard PRE algorithm, and decide not
@@ -5078,18 +4226,16 @@ pass_pre::execute (function *fun)
   gcc_assert (!need_ssa_update_p (fun));
 
   /* Remove all the redundant expressions.  */
-  todo |= eliminate (true);
+  todo |= vn_eliminate (inserted_exprs);
 
   statistics_counter_event (fun, "Insertions", pre_stats.insertions);
   statistics_counter_event (fun, "PA inserted", pre_stats.pa_insert);
   statistics_counter_event (fun, "HOIST inserted", pre_stats.hoist_insert);
   statistics_counter_event (fun, "New PHIs", pre_stats.phis);
-  statistics_counter_event (fun, "Eliminated", pre_stats.eliminations);
 
   clear_expression_ids ();
 
   scev_finalize ();
-  todo |= fini_eliminate ();
   remove_dead_inserted_code ();
   fini_pre ();
   loop_optimizer_finalize ();
@@ -5125,63 +4271,3 @@ make_pass_pre (gcc::context *ctxt)
 {
   return new pass_pre (ctxt);
 }
-
-namespace {
-
-const pass_data pass_data_fre =
-{
-  GIMPLE_PASS, /* type */
-  "fre", /* name */
-  OPTGROUP_NONE, /* optinfo_flags */
-  TV_TREE_FRE, /* tv_id */
-  ( PROP_cfg | PROP_ssa ), /* properties_required */
-  0, /* properties_provided */
-  0, /* properties_destroyed */
-  0, /* todo_flags_start */
-  0, /* todo_flags_finish */
-};
-
-class pass_fre : public gimple_opt_pass
-{
-public:
-  pass_fre (gcc::context *ctxt)
-    : gimple_opt_pass (pass_data_fre, ctxt)
-  {}
-
-  /* opt_pass methods: */
-  opt_pass * clone () { return new pass_fre (m_ctxt); }
-  virtual bool gate (function *) { return flag_tree_fre != 0; }
-  virtual unsigned int execute (function *);
-
-}; // class pass_fre
-
-unsigned int
-pass_fre::execute (function *fun)
-{
-  unsigned int todo = 0;
-
-  run_scc_vn (VN_WALKREWRITE);
-
-  memset (&pre_stats, 0, sizeof (pre_stats));
-
-  /* Remove all the redundant expressions.  */
-  todo |= eliminate (false);
-
-  todo |= fini_eliminate ();
-
-  scc_vn_restore_ssa_info ();
-  free_scc_vn ();
-
-  statistics_counter_event (fun, "Insertions", pre_stats.insertions);
-  statistics_counter_event (fun, "Eliminated", pre_stats.eliminations);
-
-  return todo;
-}
-
-} // anon namespace
-
-gimple_opt_pass *
-make_pass_fre (gcc::context *ctxt)
-{
-  return new pass_fre (ctxt);
-}
diff --git a/gcc/tree-ssa-propagate.c b/gcc/tree-ssa-propagate.c
index 00ab3d72564..62955bef9c5 100644
--- a/gcc/tree-ssa-propagate.c
+++ b/gcc/tree-ssa-propagate.c
@@ -108,10 +108,6 @@
      [3] Advanced Compiler Design and Implementation,
 	 Steven Muchnick, Morgan Kaufmann, 1997, Section 12.6  */
 
-/* Function pointers used to parameterize the propagation engine.  */
-static ssa_prop_visit_stmt_fn ssa_prop_visit_stmt;
-static ssa_prop_visit_phi_fn ssa_prop_visit_phi;
-
 /* Worklist of control flow edge destinations.  This contains
    the CFG order number of the blocks so we can iterate in CFG
    order by visiting in bit-order.  */
@@ -217,8 +213,8 @@ add_control_edge (edge e)
 
 /* Simulate the execution of STMT and update the work lists accordingly.  */
 
-static void
-simulate_stmt (gimple *stmt)
+void
+ssa_propagation_engine::simulate_stmt (gimple *stmt)
 {
   enum ssa_prop_result val = SSA_PROP_NOT_INTERESTING;
   edge taken_edge = NULL;
@@ -234,11 +230,11 @@ simulate_stmt (gimple *stmt)
 
   if (gimple_code (stmt) == GIMPLE_PHI)
     {
-      val = ssa_prop_visit_phi (as_a <gphi *> (stmt));
+      val = visit_phi (as_a <gphi *> (stmt));
       output_name = gimple_phi_result (stmt);
     }
   else
-    val = ssa_prop_visit_stmt (stmt, &taken_edge, &output_name);
+    val = visit_stmt (stmt, &taken_edge, &output_name);
 
   if (val == SSA_PROP_VARYING)
     {
@@ -321,8 +317,8 @@ simulate_stmt (gimple *stmt)
    when an SSA edge is added to it in simulate_stmt.  Return true if a stmt
    was simulated.  */
 
-static void
-process_ssa_edge_worklist ()
+void
+ssa_propagation_engine::process_ssa_edge_worklist (void)
 {
   /* Process the next entry from the worklist.  */
   unsigned stmt_uid = bitmap_first_set_bit (ssa_edge_worklist);
@@ -345,8 +341,8 @@ process_ssa_edge_worklist ()
 /* Simulate the execution of BLOCK.  Evaluate the statement associated
    with each variable reference inside the block.  */
 
-static void
-simulate_block (basic_block block)
+void
+ssa_propagation_engine::simulate_block (basic_block block)
 {
   gimple_stmt_iterator gsi;
 
@@ -781,19 +777,15 @@ update_call_from_tree (gimple_stmt_iterator *si_p, tree expr)
     return false;
 }
 
-
 /* Entry point to the propagation engine.
 
-   VISIT_STMT is called for every statement visited.
-   VISIT_PHI is called for every PHI node visited.  */
+   The VISIT_STMT virtual function is called for every statement
+   visited and the VISIT_PHI virtual function is called for every PHI
+   node visited.  */
 
 void
-ssa_propagate (ssa_prop_visit_stmt_fn visit_stmt,
-	       ssa_prop_visit_phi_fn visit_phi)
+ssa_propagation_engine::ssa_propagate (void)
 {
-  ssa_prop_visit_stmt = visit_stmt;
-  ssa_prop_visit_phi = visit_phi;
-
   ssa_prop_init ();
 
   /* Iterate until the worklists are empty.  */
@@ -861,7 +853,7 @@ static struct prop_stats_d prop_stats;
    PROP_VALUE. Return true if at least one reference was replaced.  */
 
 bool
-replace_uses_in (gimple *stmt, ssa_prop_get_value_fn get_value)
+substitute_and_fold_engine::replace_uses_in (gimple *stmt)
 {
   bool replaced = false;
   use_operand_p use;
@@ -870,7 +862,7 @@ replace_uses_in (gimple *stmt, ssa_prop_get_value_fn get_value)
   FOR_EACH_SSA_USE_OPERAND (use, stmt, iter, SSA_OP_USE)
     {
       tree tuse = USE_FROM_PTR (use);
-      tree val = (*get_value) (tuse);
+      tree val = get_value (tuse);
 
       if (val == tuse || val == NULL_TREE)
 	continue;
@@ -899,8 +891,8 @@ replace_uses_in (gimple *stmt, ssa_prop_get_value_fn get_value)
 /* Replace propagated values into all the arguments for PHI using the
    values from PROP_VALUE.  */
 
-static bool
-replace_phi_args_in (gphi *phi, ssa_prop_get_value_fn get_value)
+bool
+substitute_and_fold_engine::replace_phi_args_in (gphi *phi)
 {
   size_t i;
   bool replaced = false;
@@ -917,7 +909,7 @@ replace_phi_args_in (gphi *phi, ssa_prop_get_value_fn get_value)
 
       if (TREE_CODE (arg) == SSA_NAME)
 	{
-	  tree val = (*get_value) (arg);
+	  tree val = get_value (arg);
 
 	  if (val && val != arg && may_propagate_copy (arg, val))
 	    {
@@ -968,10 +960,10 @@ class substitute_and_fold_dom_walker : public dom_walker
 {
 public:
     substitute_and_fold_dom_walker (cdi_direction direction,
-				    ssa_prop_get_value_fn get_value_fn_,
-				    ssa_prop_fold_stmt_fn fold_fn_)
-	: dom_walker (direction), get_value_fn (get_value_fn_),
-      fold_fn (fold_fn_), something_changed (false)
+				    class substitute_and_fold_engine *engine)
+	: dom_walker (direction),
+          something_changed (false),
+	  substitute_and_fold_engine (engine)
     {
       stmts_to_remove.create (0);
       stmts_to_fixup.create (0);
@@ -987,12 +979,12 @@ public:
     virtual edge before_dom_children (basic_block);
     virtual void after_dom_children (basic_block) {}
 
-    ssa_prop_get_value_fn get_value_fn;
-    ssa_prop_fold_stmt_fn fold_fn;
     bool something_changed;
     vec<gimple *> stmts_to_remove;
     vec<gimple *> stmts_to_fixup;
     bitmap need_eh_cleanup;
+
+    class substitute_and_fold_engine *substitute_and_fold_engine;
 };
 
 edge
@@ -1009,7 +1001,7 @@ substitute_and_fold_dom_walker::before_dom_children (basic_block bb)
 	continue;
       if (res && TREE_CODE (res) == SSA_NAME)
 	{
-	  tree sprime = get_value_fn (res);
+	  tree sprime = substitute_and_fold_engine->get_value (res);
 	  if (sprime
 	      && sprime != res
 	      && may_propagate_copy (res, sprime))
@@ -1018,7 +1010,7 @@ substitute_and_fold_dom_walker::before_dom_children (basic_block bb)
 	      continue;
 	    }
 	}
-      something_changed |= replace_phi_args_in (phi, get_value_fn);
+      something_changed |= substitute_and_fold_engine->replace_phi_args_in (phi);
     }
 
   /* Propagate known values into stmts.  In some case it exposes
@@ -1035,7 +1027,7 @@ substitute_and_fold_dom_walker::before_dom_children (basic_block bb)
       tree lhs = gimple_get_lhs (stmt);
       if (lhs && TREE_CODE (lhs) == SSA_NAME)
 	{
-	  tree sprime = get_value_fn (lhs);
+	  tree sprime = substitute_and_fold_engine->get_value (lhs);
 	  if (sprime
 	      && sprime != lhs
 	      && may_propagate_copy (lhs, sprime)
@@ -1064,7 +1056,7 @@ substitute_and_fold_dom_walker::before_dom_children (basic_block bb)
 			   && gimple_call_noreturn_p (stmt));
 
       /* Replace real uses in the statement.  */
-      did_replace |= replace_uses_in (stmt, get_value_fn);
+      did_replace |= substitute_and_fold_engine->replace_uses_in (stmt);
 
       /* If we made a replacement, fold the statement.  */
       if (did_replace)
@@ -1077,16 +1069,13 @@ substitute_and_fold_dom_walker::before_dom_children (basic_block bb)
       /* Some statements may be simplified using propagator
 	 specific information.  Do this before propagating
 	 into the stmt to not disturb pass specific information.  */
-      if (fold_fn)
+      update_stmt_if_modified (stmt);
+      if (substitute_and_fold_engine->fold_stmt(&i))
 	{
-	  update_stmt_if_modified (stmt);
-	  if ((*fold_fn)(&i))
-	    {
-	      did_replace = true;
-	      prop_stats.num_stmts_folded++;
-	      stmt = gsi_stmt (i);
-	      gimple_set_modified (stmt, true);
-	    }
+	  did_replace = true;
+	  prop_stats.num_stmts_folded++;
+	  stmt = gsi_stmt (i);
+	  gimple_set_modified (stmt, true);
 	}
 
       /* If this is a control statement the propagator left edges
@@ -1172,19 +1161,15 @@ substitute_and_fold_dom_walker::before_dom_children (basic_block bb)
    Return TRUE when something changed.  */
 
 bool
-substitute_and_fold (ssa_prop_get_value_fn get_value_fn,
-		     ssa_prop_fold_stmt_fn fold_fn)
+substitute_and_fold_engine::substitute_and_fold (void)
 {
-  gcc_assert (get_value_fn);
-
   if (dump_file && (dump_flags & TDF_DETAILS))
     fprintf (dump_file, "\nSubstituting values and folding statements\n\n");
 
   memset (&prop_stats, 0, sizeof (prop_stats));
 
   calculate_dominance_info (CDI_DOMINATORS);
-  substitute_and_fold_dom_walker walker(CDI_DOMINATORS,
-					get_value_fn, fold_fn);
+  substitute_and_fold_dom_walker walker (CDI_DOMINATORS, this);
   walker.walk (ENTRY_BLOCK_PTR_FOR_FN (cfun));
 
   /* We cannot remove stmts during the BB walk, especially not release
diff --git a/gcc/tree-ssa-propagate.h b/gcc/tree-ssa-propagate.h
index 9a8ecc845d1..be4500bc83a 100644
--- a/gcc/tree-ssa-propagate.h
+++ b/gcc/tree-ssa-propagate.h
@@ -61,21 +61,11 @@ enum ssa_prop_result {
 };
 
 
-/* Call-back functions used by the value propagation engine.  */
-typedef enum ssa_prop_result (*ssa_prop_visit_stmt_fn) (gimple *, edge *,
-							tree *);
-typedef enum ssa_prop_result (*ssa_prop_visit_phi_fn) (gphi *);
-typedef bool (*ssa_prop_fold_stmt_fn) (gimple_stmt_iterator *gsi);
-typedef tree (*ssa_prop_get_value_fn) (tree);
-
-
 extern bool valid_gimple_rhs_p (tree);
 extern void move_ssa_defining_stmt_for_defs (gimple *, gimple *);
 extern bool update_gimple_call (gimple_stmt_iterator *, tree, int, ...);
 extern bool update_call_from_tree (gimple_stmt_iterator *, tree);
-extern void ssa_propagate (ssa_prop_visit_stmt_fn, ssa_prop_visit_phi_fn);
 extern bool stmt_makes_single_store (gimple *);
-extern bool substitute_and_fold (ssa_prop_get_value_fn, ssa_prop_fold_stmt_fn);
 extern bool may_propagate_copy (tree, tree);
 extern bool may_propagate_copy_into_stmt (gimple *, tree);
 extern bool may_propagate_copy_into_asm (tree);
@@ -83,6 +73,42 @@ extern void propagate_value (use_operand_p, tree);
 extern void replace_exp (use_operand_p, tree);
 extern void propagate_tree_value (tree *, tree);
 extern void propagate_tree_value_into_stmt (gimple_stmt_iterator *, tree);
-extern bool replace_uses_in (gimple *stmt, ssa_prop_get_value_fn get_value);
+
+/* Public interface into the SSA propagation engine.  Clients should inherit
+   from this class and provide their own visitors.  */
+
+class ssa_propagation_engine
+{
+ public:
+
+  virtual ~ssa_propagation_engine (void) { }
+
+  /* Virtual functions the clients must provide to visit statements
+     and phi nodes respectively.  */
+  virtual enum ssa_prop_result visit_stmt (gimple *, edge *, tree *) = 0;
+  virtual enum ssa_prop_result visit_phi (gphi *) = 0;
+
+  /* Main interface into the propagation engine.  */
+  void ssa_propagate (void);
+
+ private:
+  /* Internal implementation details.  */
+  void simulate_stmt (gimple *stmt);
+  void process_ssa_edge_worklist (void);
+  void simulate_block (basic_block);
+
+};
+
+class substitute_and_fold_engine
+{
+ public:
+  virtual ~substitute_and_fold_engine (void) { }
+  virtual bool fold_stmt (gimple_stmt_iterator *) { return false; }
+  virtual tree get_value (tree) { return NULL_TREE; }
+
+  bool substitute_and_fold (void);
+  bool replace_uses_in (gimple *);
+  bool replace_phi_args_in (gphi *);
+};
 
 #endif /* _TREE_SSA_PROPAGATE_H  */
diff --git a/gcc/tree-ssa-reassoc.c b/gcc/tree-ssa-reassoc.c
index cc57ae320a3..5e8cac69d5d 100644
--- a/gcc/tree-ssa-reassoc.c
+++ b/gcc/tree-ssa-reassoc.c
@@ -5625,6 +5625,7 @@ attempt_builtin_copysign (vec<operand_entry *> *ops)
 	      switch (gimple_call_combined_fn (old_call))
 		{
 		CASE_CFN_COPYSIGN:
+		CASE_CFN_COPYSIGN_FN:
 		  arg0 = gimple_call_arg (old_call, 0);
 		  arg1 = gimple_call_arg (old_call, 1);
 		  /* The first argument of copysign must be a constant,
@@ -5910,7 +5911,7 @@ reassociate_bb (basic_block bb)
 		     move it to the front.  This helps ensure that we generate
 		     (X & Y) & C rather than (X & C) & Y.  The former will
 		     often match a canonical bit test when we get to RTL.  */
-		  if (ops.length () != 2
+		  if (ops.length () > 2
 		      && (rhs_code == BIT_AND_EXPR
 		          || rhs_code == BIT_IOR_EXPR
 		          || rhs_code == BIT_XOR_EXPR)
@@ -6033,12 +6034,10 @@ branch_fixup (void)
 
       edge etrue = make_edge (cond_bb, merge_bb, EDGE_TRUE_VALUE);
       etrue->probability = profile_probability::even ();
-      etrue->count = cond_bb->count.apply_scale (1, 2);
       edge efalse = find_edge (cond_bb, then_bb);
       efalse->flags = EDGE_FALSE_VALUE;
       efalse->probability -= etrue->probability;
-      efalse->count -= etrue->count;
-      then_bb->count -= etrue->count;
+      then_bb->count -= etrue->count ();
 
       tree othervar = NULL_TREE;
       if (gimple_assign_rhs1 (use_stmt) == var)
diff --git a/gcc/tree-ssa-sccvn.c b/gcc/tree-ssa-sccvn.c
index 1a130d0d133..a5beb2a1c9f 100644
--- a/gcc/tree-ssa-sccvn.c
+++ b/gcc/tree-ssa-sccvn.c
@@ -55,13 +55,21 @@ along with GCC; see the file COPYING3.  If not see
 #include "cfgloop.h"
 #include "params.h"
 #include "tree-ssa-propagate.h"
-#include "tree-ssa-sccvn.h"
 #include "tree-cfg.h"
 #include "domwalk.h"
 #include "gimple-iterator.h"
 #include "gimple-match.h"
 #include "stringpool.h"
 #include "attribs.h"
+#include "tree-pass.h"
+#include "statistics.h"
+#include "langhooks.h"
+#include "ipa-utils.h"
+#include "dbgcnt.h"
+#include "tree-cfgcleanup.h"
+#include "tree-ssa-loop.h"
+#include "tree-scalar-evolution.h"
+#include "tree-ssa-sccvn.h"
 
 /* This algorithm is based on the SCC algorithm presented by Keith
    Cooper and L. Taylor Simpson in "SCC-Based Value numbering"
@@ -791,7 +799,7 @@ copy_reference_ops_from_ref (tree ref, vec<vn_reference_op_s> *result)
 		       (checking cfun->after_inlining does the
 		       trick here).  */
 		    if (TREE_CODE (orig) != ADDR_EXPR
-			|| maybe_nonzero (off)
+			|| may_ne (off, 0)
 			|| cfun->after_inlining)
 		      off.to_shwi (&temp.off);
 		  }
@@ -1212,7 +1220,7 @@ vn_reference_maybe_forwprop_address (vec<vn_reference_op_s> *ops,
 	 dereference isn't offsetted.  */
       if (!addr_base
 	  && *i_p == ops->length () - 1
-	  && known_zero (off)
+	  && must_eq (off, 0)
 	  /* This makes us disable this transform for PRE where the
 	     reference ops might be also used for code insertion which
 	     is invalid.  */
@@ -2148,7 +2156,7 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *vr_,
       copy_reference_ops_from_ref (gimple_assign_rhs1 (def_stmt), &rhs);
 
       /* Apply an extra offset to the inner MEM_REF of the RHS.  */
-      if (maybe_nonzero (extra_off))
+      if (may_ne (extra_off, 0))
 	{
 	  if (rhs.length () < 2
 	      || rhs[0].opcode != MEM_REF
@@ -4824,34 +4832,18 @@ sccvn_dom_walker::after_dom_children (basic_block bb)
 edge
 sccvn_dom_walker::before_dom_children (basic_block bb)
 {
-  edge e;
-  edge_iterator ei;
-
   if (dump_file && (dump_flags & TDF_DETAILS))
     fprintf (dump_file, "Visiting BB %d\n", bb->index);
 
   /* If we have a single predecessor record the equivalence from a
      possible condition on the predecessor edge.  */
-  edge pred_e = NULL;
-  FOR_EACH_EDGE (e, ei, bb->preds)
-    {
-      /* Ignore simple backedges from this to allow recording conditions
-         in loop headers.  */
-      if (dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
-	continue;
-      if (! pred_e)
-	pred_e = e;
-      else
-	{
-	  pred_e = NULL;
-	  break;
-	}
-    }
+  edge pred_e = single_pred_edge_ignoring_loop_edges (bb, false);
   if (pred_e)
     {
       /* Check if there are multiple executable successor edges in
 	 the source block.  Otherwise there is no additional info
 	 to be recorded.  */
+      edge_iterator ei;
       edge e2;
       FOR_EACH_EDGE (e2, ei, pred_e->src->succs)
 	if (e2 != pred_e
@@ -5134,3 +5126,868 @@ vn_nary_may_trap (vn_nary_op_t nary)
 
   return false;
 }
+
+
+class eliminate_dom_walker : public dom_walker
+{
+public:
+  eliminate_dom_walker (cdi_direction, bitmap);
+  ~eliminate_dom_walker ();
+
+  virtual edge before_dom_children (basic_block);
+  virtual void after_dom_children (basic_block);
+
+  tree eliminate_avail (tree op);
+  void eliminate_push_avail (tree op);
+  tree eliminate_insert (gimple_stmt_iterator *gsi, tree val);
+
+  bool do_pre;
+  unsigned int el_todo;
+  unsigned int eliminations;
+  unsigned int insertions;
+
+  /* SSA names that had their defs inserted by PRE if do_pre.  */
+  bitmap inserted_exprs;
+
+  /* Blocks with statements that have had their EH properties changed.  */
+  bitmap need_eh_cleanup;
+
+  /* Blocks with statements that have had their AB properties changed.  */
+  bitmap need_ab_cleanup;
+
+  auto_vec<gimple *> to_remove;
+  auto_vec<gimple *> to_fixup;
+  auto_vec<tree> avail;
+  auto_vec<tree> avail_stack;
+};
+
+eliminate_dom_walker::eliminate_dom_walker (cdi_direction direction,
+					    bitmap inserted_exprs_)
+  : dom_walker (direction), do_pre (inserted_exprs_ != NULL),
+    el_todo (0), eliminations (0), insertions (0),
+    inserted_exprs (inserted_exprs_)
+{
+  need_eh_cleanup = BITMAP_ALLOC (NULL);
+  need_ab_cleanup = BITMAP_ALLOC (NULL);
+}
+
+eliminate_dom_walker::~eliminate_dom_walker ()
+{
+  BITMAP_FREE (need_eh_cleanup);
+  BITMAP_FREE (need_ab_cleanup);
+}
+
+/* Return a leader for OP that is available at the current point of the
+   eliminate domwalk.  */
+
+tree
+eliminate_dom_walker::eliminate_avail (tree op)
+{
+  tree valnum = VN_INFO (op)->valnum;
+  if (TREE_CODE (valnum) == SSA_NAME)
+    {
+      if (SSA_NAME_IS_DEFAULT_DEF (valnum))
+	return valnum;
+      if (avail.length () > SSA_NAME_VERSION (valnum))
+	return avail[SSA_NAME_VERSION (valnum)];
+    }
+  else if (is_gimple_min_invariant (valnum))
+    return valnum;
+  return NULL_TREE;
+}
+
+/* At the current point of the eliminate domwalk make OP available.  */
+
+void
+eliminate_dom_walker::eliminate_push_avail (tree op)
+{
+  tree valnum = VN_INFO (op)->valnum;
+  if (TREE_CODE (valnum) == SSA_NAME)
+    {
+      if (avail.length () <= SSA_NAME_VERSION (valnum))
+	avail.safe_grow_cleared (SSA_NAME_VERSION (valnum) + 1);
+      tree pushop = op;
+      if (avail[SSA_NAME_VERSION (valnum)])
+	pushop = avail[SSA_NAME_VERSION (valnum)];
+      avail_stack.safe_push (pushop);
+      avail[SSA_NAME_VERSION (valnum)] = op;
+    }
+}
+
+/* Insert the expression recorded by SCCVN for VAL at *GSI.  Returns
+   the leader for the expression if insertion was successful.  */
+
+tree
+eliminate_dom_walker::eliminate_insert (gimple_stmt_iterator *gsi, tree val)
+{
+  /* We can insert a sequence with a single assignment only.  */
+  gimple_seq stmts = VN_INFO (val)->expr;
+  if (!gimple_seq_singleton_p (stmts))
+    return NULL_TREE;
+  gassign *stmt = dyn_cast <gassign *> (gimple_seq_first_stmt (stmts));
+  if (!stmt
+      || (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
+	  && gimple_assign_rhs_code (stmt) != VIEW_CONVERT_EXPR
+	  && gimple_assign_rhs_code (stmt) != BIT_FIELD_REF
+	  && (gimple_assign_rhs_code (stmt) != BIT_AND_EXPR
+	      || TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)))
+    return NULL_TREE;
+
+  tree op = gimple_assign_rhs1 (stmt);
+  if (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
+      || gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
+    op = TREE_OPERAND (op, 0);
+  tree leader = TREE_CODE (op) == SSA_NAME ? eliminate_avail (op) : op;
+  if (!leader)
+    return NULL_TREE;
+
+  tree res;
+  stmts = NULL;
+  if (gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
+    res = gimple_build (&stmts, BIT_FIELD_REF,
+			TREE_TYPE (val), leader,
+			TREE_OPERAND (gimple_assign_rhs1 (stmt), 1),
+			TREE_OPERAND (gimple_assign_rhs1 (stmt), 2));
+  else if (gimple_assign_rhs_code (stmt) == BIT_AND_EXPR)
+    res = gimple_build (&stmts, BIT_AND_EXPR,
+			TREE_TYPE (val), leader, gimple_assign_rhs2 (stmt));
+  else
+    res = gimple_build (&stmts, gimple_assign_rhs_code (stmt),
+			TREE_TYPE (val), leader);
+  if (TREE_CODE (res) != SSA_NAME
+      || SSA_NAME_IS_DEFAULT_DEF (res)
+      || gimple_bb (SSA_NAME_DEF_STMT (res)))
+    {
+      gimple_seq_discard (stmts);
+
+      /* During propagation we have to treat SSA info conservatively
+         and thus we can end up simplifying the inserted expression
+	 at elimination time to sth not defined in stmts.  */
+      /* But then this is a redundancy we failed to detect.  Which means
+         res now has two values.  That doesn't play well with how
+	 we track availability here, so give up.  */
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	{
+	  if (TREE_CODE (res) == SSA_NAME)
+	    res = eliminate_avail (res);
+	  if (res)
+	    {
+	      fprintf (dump_file, "Failed to insert expression for value ");
+	      print_generic_expr (dump_file, val);
+	      fprintf (dump_file, " which is really fully redundant to ");
+	      print_generic_expr (dump_file, res);
+	      fprintf (dump_file, "\n");
+	    }
+	}
+
+      return NULL_TREE;
+    }
+  else
+    {
+      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+      VN_INFO_GET (res)->valnum = val;
+    }
+
+  insertions++;
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Inserted ");
+      print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (res), 0);
+    }
+
+  return res;
+}
+
+
+
+/* Perform elimination for the basic-block B during the domwalk.  */
+
+edge
+eliminate_dom_walker::before_dom_children (basic_block b)
+{
+  /* Mark new bb.  */
+  avail_stack.safe_push (NULL_TREE);
+
+  /* Skip unreachable blocks marked unreachable during the SCCVN domwalk.  */
+  edge_iterator ei;
+  edge e;
+  FOR_EACH_EDGE (e, ei, b->preds)
+    if (e->flags & EDGE_EXECUTABLE)
+      break;
+  if (! e)
+    return NULL;
+
+  for (gphi_iterator gsi = gsi_start_phis (b); !gsi_end_p (gsi);)
+    {
+      gphi *phi = gsi.phi ();
+      tree res = PHI_RESULT (phi);
+
+      if (virtual_operand_p (res))
+	{
+	  gsi_next (&gsi);
+	  continue;
+	}
+
+      tree sprime = eliminate_avail (res);
+      if (sprime
+	  && sprime != res)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    {
+	      fprintf (dump_file, "Replaced redundant PHI node defining ");
+	      print_generic_expr (dump_file, res);
+	      fprintf (dump_file, " with ");
+	      print_generic_expr (dump_file, sprime);
+	      fprintf (dump_file, "\n");
+	    }
+
+	  /* If we inserted this PHI node ourself, it's not an elimination.  */
+	  if (! inserted_exprs
+	      || ! bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (res)))
+	    eliminations++;
+
+	  /* If we will propagate into all uses don't bother to do
+	     anything.  */
+	  if (may_propagate_copy (res, sprime))
+	    {
+	      /* Mark the PHI for removal.  */
+	      to_remove.safe_push (phi);
+	      gsi_next (&gsi);
+	      continue;
+	    }
+
+	  remove_phi_node (&gsi, false);
+
+	  if (!useless_type_conversion_p (TREE_TYPE (res), TREE_TYPE (sprime)))
+	    sprime = fold_convert (TREE_TYPE (res), sprime);
+	  gimple *stmt = gimple_build_assign (res, sprime);
+	  gimple_stmt_iterator gsi2 = gsi_after_labels (b);
+	  gsi_insert_before (&gsi2, stmt, GSI_NEW_STMT);
+	  continue;
+	}
+
+      eliminate_push_avail (res);
+      gsi_next (&gsi);
+    }
+
+  for (gimple_stmt_iterator gsi = gsi_start_bb (b);
+       !gsi_end_p (gsi);
+       gsi_next (&gsi))
+    {
+      tree sprime = NULL_TREE;
+      gimple *stmt = gsi_stmt (gsi);
+      tree lhs = gimple_get_lhs (stmt);
+      if (lhs && TREE_CODE (lhs) == SSA_NAME
+	  && !gimple_has_volatile_ops (stmt)
+	  /* See PR43491.  Do not replace a global register variable when
+	     it is a the RHS of an assignment.  Do replace local register
+	     variables since gcc does not guarantee a local variable will
+	     be allocated in register.
+	     ???  The fix isn't effective here.  This should instead
+	     be ensured by not value-numbering them the same but treating
+	     them like volatiles?  */
+	  && !(gimple_assign_single_p (stmt)
+	       && (TREE_CODE (gimple_assign_rhs1 (stmt)) == VAR_DECL
+		   && DECL_HARD_REGISTER (gimple_assign_rhs1 (stmt))
+		   && is_global_var (gimple_assign_rhs1 (stmt)))))
+	{
+	  sprime = eliminate_avail (lhs);
+	  if (!sprime)
+	    {
+	      /* If there is no existing usable leader but SCCVN thinks
+		 it has an expression it wants to use as replacement,
+		 insert that.  */
+	      tree val = VN_INFO (lhs)->valnum;
+	      if (val != VN_TOP
+		  && TREE_CODE (val) == SSA_NAME
+		  && VN_INFO (val)->needs_insertion
+		  && VN_INFO (val)->expr != NULL
+		  && (sprime = eliminate_insert (&gsi, val)) != NULL_TREE)
+		eliminate_push_avail (sprime);
+	    }
+
+	  /* If this now constitutes a copy duplicate points-to
+	     and range info appropriately.  This is especially
+	     important for inserted code.  See tree-ssa-copy.c
+	     for similar code.  */
+	  if (sprime
+	      && TREE_CODE (sprime) == SSA_NAME)
+	    {
+	      basic_block sprime_b = gimple_bb (SSA_NAME_DEF_STMT (sprime));
+	      if (POINTER_TYPE_P (TREE_TYPE (lhs))
+		  && VN_INFO_PTR_INFO (lhs)
+		  && ! VN_INFO_PTR_INFO (sprime))
+		{
+		  duplicate_ssa_name_ptr_info (sprime,
+					       VN_INFO_PTR_INFO (lhs));
+		  if (b != sprime_b)
+		    mark_ptr_info_alignment_unknown
+			(SSA_NAME_PTR_INFO (sprime));
+		}
+	      else if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
+		       && VN_INFO_RANGE_INFO (lhs)
+		       && ! VN_INFO_RANGE_INFO (sprime)
+		       && b == sprime_b)
+		duplicate_ssa_name_range_info (sprime,
+					       VN_INFO_RANGE_TYPE (lhs),
+					       VN_INFO_RANGE_INFO (lhs));
+	    }
+
+	  /* Inhibit the use of an inserted PHI on a loop header when
+	     the address of the memory reference is a simple induction
+	     variable.  In other cases the vectorizer won't do anything
+	     anyway (either it's loop invariant or a complicated
+	     expression).  */
+	  if (sprime
+	      && TREE_CODE (sprime) == SSA_NAME
+	      && do_pre
+	      && (flag_tree_loop_vectorize || flag_tree_parallelize_loops > 1)
+	      && loop_outer (b->loop_father)
+	      && has_zero_uses (sprime)
+	      && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))
+	      && gimple_assign_load_p (stmt))
+	    {
+	      gimple *def_stmt = SSA_NAME_DEF_STMT (sprime);
+	      basic_block def_bb = gimple_bb (def_stmt);
+	      if (gimple_code (def_stmt) == GIMPLE_PHI
+		  && def_bb->loop_father->header == def_bb)
+		{
+		  loop_p loop = def_bb->loop_father;
+		  ssa_op_iter iter;
+		  tree op;
+		  bool found = false;
+		  FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
+		    {
+		      affine_iv iv;
+		      def_bb = gimple_bb (SSA_NAME_DEF_STMT (op));
+		      if (def_bb
+			  && flow_bb_inside_loop_p (loop, def_bb)
+			  && simple_iv (loop, loop, op, &iv, true))
+			{
+			  found = true;
+			  break;
+			}
+		    }
+		  if (found)
+		    {
+		      if (dump_file && (dump_flags & TDF_DETAILS))
+			{
+			  fprintf (dump_file, "Not replacing ");
+			  print_gimple_expr (dump_file, stmt, 0);
+			  fprintf (dump_file, " with ");
+			  print_generic_expr (dump_file, sprime);
+			  fprintf (dump_file, " which would add a loop"
+				   " carried dependence to loop %d\n",
+				   loop->num);
+			}
+		      /* Don't keep sprime available.  */
+		      sprime = NULL_TREE;
+		    }
+		}
+	    }
+
+	  if (sprime)
+	    {
+	      /* If we can propagate the value computed for LHS into
+		 all uses don't bother doing anything with this stmt.  */
+	      if (may_propagate_copy (lhs, sprime))
+		{
+		  /* Mark it for removal.  */
+		  to_remove.safe_push (stmt);
+
+		  /* ???  Don't count copy/constant propagations.  */
+		  if (gimple_assign_single_p (stmt)
+		      && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
+			  || gimple_assign_rhs1 (stmt) == sprime))
+		    continue;
+
+		  if (dump_file && (dump_flags & TDF_DETAILS))
+		    {
+		      fprintf (dump_file, "Replaced ");
+		      print_gimple_expr (dump_file, stmt, 0);
+		      fprintf (dump_file, " with ");
+		      print_generic_expr (dump_file, sprime);
+		      fprintf (dump_file, " in all uses of ");
+		      print_gimple_stmt (dump_file, stmt, 0);
+		    }
+
+		  eliminations++;
+		  continue;
+		}
+
+	      /* If this is an assignment from our leader (which
+	         happens in the case the value-number is a constant)
+		 then there is nothing to do.  */
+	      if (gimple_assign_single_p (stmt)
+		  && sprime == gimple_assign_rhs1 (stmt))
+		continue;
+
+	      /* Else replace its RHS.  */
+	      bool can_make_abnormal_goto
+		  = is_gimple_call (stmt)
+		  && stmt_can_make_abnormal_goto (stmt);
+
+	      if (dump_file && (dump_flags & TDF_DETAILS))
+		{
+		  fprintf (dump_file, "Replaced ");
+		  print_gimple_expr (dump_file, stmt, 0);
+		  fprintf (dump_file, " with ");
+		  print_generic_expr (dump_file, sprime);
+		  fprintf (dump_file, " in ");
+		  print_gimple_stmt (dump_file, stmt, 0);
+		}
+
+	      eliminations++;
+	      gimple *orig_stmt = stmt;
+	      if (!useless_type_conversion_p (TREE_TYPE (lhs),
+					      TREE_TYPE (sprime)))
+		sprime = fold_convert (TREE_TYPE (lhs), sprime);
+	      tree vdef = gimple_vdef (stmt);
+	      tree vuse = gimple_vuse (stmt);
+	      propagate_tree_value_into_stmt (&gsi, sprime);
+	      stmt = gsi_stmt (gsi);
+	      update_stmt (stmt);
+	      if (vdef != gimple_vdef (stmt))
+		VN_INFO (vdef)->valnum = vuse;
+
+	      /* If we removed EH side-effects from the statement, clean
+		 its EH information.  */
+	      if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
+		{
+		  bitmap_set_bit (need_eh_cleanup,
+				  gimple_bb (stmt)->index);
+		  if (dump_file && (dump_flags & TDF_DETAILS))
+		    fprintf (dump_file, "  Removed EH side-effects.\n");
+		}
+
+	      /* Likewise for AB side-effects.  */
+	      if (can_make_abnormal_goto
+		  && !stmt_can_make_abnormal_goto (stmt))
+		{
+		  bitmap_set_bit (need_ab_cleanup,
+				  gimple_bb (stmt)->index);
+		  if (dump_file && (dump_flags & TDF_DETAILS))
+		    fprintf (dump_file, "  Removed AB side-effects.\n");
+		}
+
+	      continue;
+	    }
+	}
+
+      /* If the statement is a scalar store, see if the expression
+         has the same value number as its rhs.  If so, the store is
+         dead.  */
+      if (gimple_assign_single_p (stmt)
+	  && !gimple_has_volatile_ops (stmt)
+	  && !is_gimple_reg (gimple_assign_lhs (stmt))
+	  && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
+	      || is_gimple_min_invariant (gimple_assign_rhs1 (stmt))))
+	{
+	  tree val;
+	  tree rhs = gimple_assign_rhs1 (stmt);
+	  vn_reference_t vnresult;
+	  val = vn_reference_lookup (lhs, gimple_vuse (stmt), VN_WALKREWRITE,
+				     &vnresult, false);
+	  if (TREE_CODE (rhs) == SSA_NAME)
+	    rhs = VN_INFO (rhs)->valnum;
+	  if (val
+	      && operand_equal_p (val, rhs, 0))
+	    {
+	      /* We can only remove the later store if the former aliases
+		 at least all accesses the later one does or if the store
+		 was to readonly memory storing the same value.  */
+	      alias_set_type set = get_alias_set (lhs);
+	      if (! vnresult
+		  || vnresult->set == set
+		  || alias_set_subset_of (set, vnresult->set))
+		{
+		  if (dump_file && (dump_flags & TDF_DETAILS))
+		    {
+		      fprintf (dump_file, "Deleted redundant store ");
+		      print_gimple_stmt (dump_file, stmt, 0);
+		    }
+
+		  /* Queue stmt for removal.  */
+		  to_remove.safe_push (stmt);
+		  continue;
+		}
+	    }
+	}
+
+      /* If this is a control statement value numbering left edges
+	 unexecuted on force the condition in a way consistent with
+	 that.  */
+      if (gcond *cond = dyn_cast <gcond *> (stmt))
+	{
+	  if ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE)
+	      ^ (EDGE_SUCC (b, 1)->flags & EDGE_EXECUTABLE))
+	    {
+              if (dump_file && (dump_flags & TDF_DETAILS))
+                {
+                  fprintf (dump_file, "Removing unexecutable edge from ");
+		  print_gimple_stmt (dump_file, stmt, 0);
+                }
+	      if (((EDGE_SUCC (b, 0)->flags & EDGE_TRUE_VALUE) != 0)
+		  == ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE) != 0))
+		gimple_cond_make_true (cond);
+	      else
+		gimple_cond_make_false (cond);
+	      update_stmt (cond);
+	      el_todo |= TODO_cleanup_cfg;
+	      continue;
+	    }
+	}
+
+      bool can_make_abnormal_goto = stmt_can_make_abnormal_goto (stmt);
+      bool was_noreturn = (is_gimple_call (stmt)
+			   && gimple_call_noreturn_p (stmt));
+      tree vdef = gimple_vdef (stmt);
+      tree vuse = gimple_vuse (stmt);
+
+      /* If we didn't replace the whole stmt (or propagate the result
+         into all uses), replace all uses on this stmt with their
+	 leaders.  */
+      bool modified = false;
+      use_operand_p use_p;
+      ssa_op_iter iter;
+      FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
+	{
+	  tree use = USE_FROM_PTR (use_p);
+	  /* ???  The call code above leaves stmt operands un-updated.  */
+	  if (TREE_CODE (use) != SSA_NAME)
+	    continue;
+	  tree sprime = eliminate_avail (use);
+	  if (sprime && sprime != use
+	      && may_propagate_copy (use, sprime)
+	      /* We substitute into debug stmts to avoid excessive
+	         debug temporaries created by removed stmts, but we need
+		 to avoid doing so for inserted sprimes as we never want
+		 to create debug temporaries for them.  */
+	      && (!inserted_exprs
+		  || TREE_CODE (sprime) != SSA_NAME
+		  || !is_gimple_debug (stmt)
+		  || !bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))))
+	    {
+	      propagate_value (use_p, sprime);
+	      modified = true;
+	    }
+	}
+
+      /* Fold the stmt if modified, this canonicalizes MEM_REFs we propagated
+         into which is a requirement for the IPA devirt machinery.  */
+      gimple *old_stmt = stmt;
+      if (modified)
+	{
+	  /* If a formerly non-invariant ADDR_EXPR is turned into an
+	     invariant one it was on a separate stmt.  */
+	  if (gimple_assign_single_p (stmt)
+	      && TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
+	    recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
+	  gimple_stmt_iterator prev = gsi;
+	  gsi_prev (&prev);
+	  if (fold_stmt (&gsi))
+	    {
+	      /* fold_stmt may have created new stmts inbetween
+		 the previous stmt and the folded stmt.  Mark
+		 all defs created there as varying to not confuse
+		 the SCCVN machinery as we're using that even during
+		 elimination.  */
+	      if (gsi_end_p (prev))
+		prev = gsi_start_bb (b);
+	      else
+		gsi_next (&prev);
+	      if (gsi_stmt (prev) != gsi_stmt (gsi))
+		do
+		  {
+		    tree def;
+		    ssa_op_iter dit;
+		    FOR_EACH_SSA_TREE_OPERAND (def, gsi_stmt (prev),
+					       dit, SSA_OP_ALL_DEFS)
+		      /* As existing DEFs may move between stmts
+			 we have to guard VN_INFO_GET.  */
+		      if (! has_VN_INFO (def))
+			VN_INFO_GET (def)->valnum = def;
+		    if (gsi_stmt (prev) == gsi_stmt (gsi))
+		      break;
+		    gsi_next (&prev);
+		  }
+		while (1);
+	    }
+	  stmt = gsi_stmt (gsi);
+	  /* In case we folded the stmt away schedule the NOP for removal.  */
+	  if (gimple_nop_p (stmt))
+	    to_remove.safe_push (stmt);
+	}
+
+      /* Visit indirect calls and turn them into direct calls if
+	 possible using the devirtualization machinery.  Do this before
+	 checking for required EH/abnormal/noreturn cleanup as devird
+	 may expose more of those.  */
+      if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
+	{
+	  tree fn = gimple_call_fn (call_stmt);
+	  if (fn
+	      && flag_devirtualize
+	      && virtual_method_call_p (fn))
+	    {
+	      tree otr_type = obj_type_ref_class (fn);
+	      unsigned HOST_WIDE_INT otr_tok
+		= tree_to_uhwi (OBJ_TYPE_REF_TOKEN (fn));
+	      tree instance;
+	      ipa_polymorphic_call_context context (current_function_decl,
+						    fn, stmt, &instance);
+	      context.get_dynamic_type (instance, OBJ_TYPE_REF_OBJECT (fn),
+					otr_type, stmt);
+	      bool final;
+	      vec <cgraph_node *> targets
+		= possible_polymorphic_call_targets (obj_type_ref_class (fn),
+						     otr_tok, context, &final);
+	      if (dump_file)
+		dump_possible_polymorphic_call_targets (dump_file, 
+							obj_type_ref_class (fn),
+							otr_tok, context);
+	      if (final && targets.length () <= 1 && dbg_cnt (devirt))
+		{
+		  tree fn;
+		  if (targets.length () == 1)
+		    fn = targets[0]->decl;
+		  else
+		    fn = builtin_decl_implicit (BUILT_IN_UNREACHABLE);
+		  if (dump_enabled_p ())
+		    {
+		      location_t loc = gimple_location (stmt);
+		      dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
+				       "converting indirect call to "
+				       "function %s\n",
+				       lang_hooks.decl_printable_name (fn, 2));
+		    }
+		  gimple_call_set_fndecl (call_stmt, fn);
+		  /* If changing the call to __builtin_unreachable
+		     or similar noreturn function, adjust gimple_call_fntype
+		     too.  */
+		  if (gimple_call_noreturn_p (call_stmt)
+		      && VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fn)))
+		      && TYPE_ARG_TYPES (TREE_TYPE (fn))
+		      && (TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fn)))
+			  == void_type_node))
+		    gimple_call_set_fntype (call_stmt, TREE_TYPE (fn));
+		  maybe_remove_unused_call_args (cfun, call_stmt);
+		  modified = true;
+		}
+	    }
+	}
+
+      if (modified)
+	{
+	  /* When changing a call into a noreturn call, cfg cleanup
+	     is needed to fix up the noreturn call.  */
+	  if (!was_noreturn
+	      && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
+	    to_fixup.safe_push  (stmt);
+	  /* When changing a condition or switch into one we know what
+	     edge will be executed, schedule a cfg cleanup.  */
+	  if ((gimple_code (stmt) == GIMPLE_COND
+	       && (gimple_cond_true_p (as_a <gcond *> (stmt))
+		   || gimple_cond_false_p (as_a <gcond *> (stmt))))
+	      || (gimple_code (stmt) == GIMPLE_SWITCH
+		  && TREE_CODE (gimple_switch_index
+				  (as_a <gswitch *> (stmt))) == INTEGER_CST))
+	    el_todo |= TODO_cleanup_cfg;
+	  /* If we removed EH side-effects from the statement, clean
+	     its EH information.  */
+	  if (maybe_clean_or_replace_eh_stmt (old_stmt, stmt))
+	    {
+	      bitmap_set_bit (need_eh_cleanup,
+			      gimple_bb (stmt)->index);
+	      if (dump_file && (dump_flags & TDF_DETAILS))
+		fprintf (dump_file, "  Removed EH side-effects.\n");
+	    }
+	  /* Likewise for AB side-effects.  */
+	  if (can_make_abnormal_goto
+	      && !stmt_can_make_abnormal_goto (stmt))
+	    {
+	      bitmap_set_bit (need_ab_cleanup,
+			      gimple_bb (stmt)->index);
+	      if (dump_file && (dump_flags & TDF_DETAILS))
+		fprintf (dump_file, "  Removed AB side-effects.\n");
+	    }
+	  update_stmt (stmt);
+	  if (vdef != gimple_vdef (stmt))
+	    VN_INFO (vdef)->valnum = vuse;
+	}
+
+      /* Make new values available - for fully redundant LHS we
+         continue with the next stmt above and skip this.  */
+      def_operand_p defp;
+      FOR_EACH_SSA_DEF_OPERAND (defp, stmt, iter, SSA_OP_DEF)
+	eliminate_push_avail (DEF_FROM_PTR (defp));
+    }
+
+  /* Replace destination PHI arguments.  */
+  FOR_EACH_EDGE (e, ei, b->succs)
+    if (e->flags & EDGE_EXECUTABLE)
+      for (gphi_iterator gsi = gsi_start_phis (e->dest);
+	   !gsi_end_p (gsi);
+	   gsi_next (&gsi))
+	{
+	  gphi *phi = gsi.phi ();
+	  use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
+	  tree arg = USE_FROM_PTR (use_p);
+	  if (TREE_CODE (arg) != SSA_NAME
+	      || virtual_operand_p (arg))
+	    continue;
+	  tree sprime = eliminate_avail (arg);
+	  if (sprime && may_propagate_copy (arg, sprime))
+	    propagate_value (use_p, sprime);
+	}
+  return NULL;
+}
+
+/* Make no longer available leaders no longer available.  */
+
+void
+eliminate_dom_walker::after_dom_children (basic_block)
+{
+  tree entry;
+  while ((entry = avail_stack.pop ()) != NULL_TREE)
+    {
+      tree valnum = VN_INFO (entry)->valnum;
+      tree old = avail[SSA_NAME_VERSION (valnum)];
+      if (old == entry)
+	avail[SSA_NAME_VERSION (valnum)] = NULL_TREE;
+      else
+	avail[SSA_NAME_VERSION (valnum)] = entry;
+    }
+}
+
+/* Eliminate fully redundant computations.  */
+
+unsigned int
+vn_eliminate (bitmap inserted_exprs)
+{
+  eliminate_dom_walker el (CDI_DOMINATORS, inserted_exprs);
+  el.avail.reserve (num_ssa_names);
+
+  el.walk (cfun->cfg->x_entry_block_ptr);
+
+  /* We cannot remove stmts during BB walk, especially not release SSA
+     names there as this confuses the VN machinery.  The stmts ending
+     up in to_remove are either stores or simple copies.
+     Remove stmts in reverse order to make debug stmt creation possible.  */
+  while (!el.to_remove.is_empty ())
+    {
+      gimple *stmt = el.to_remove.pop ();
+
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	{
+	  fprintf (dump_file, "Removing dead stmt ");
+	  print_gimple_stmt (dump_file, stmt, 0, 0);
+	}
+
+      gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
+      if (gimple_code (stmt) == GIMPLE_PHI)
+	remove_phi_node (&gsi, true);
+      else
+	{
+	  basic_block bb = gimple_bb (stmt);
+	  unlink_stmt_vdef (stmt);
+	  if (gsi_remove (&gsi, true))
+	    bitmap_set_bit (el.need_eh_cleanup, bb->index);
+	  if (is_gimple_call (stmt) && stmt_can_make_abnormal_goto (stmt))
+	    bitmap_set_bit (el.need_ab_cleanup, bb->index);
+	  release_defs (stmt);
+	}
+
+      /* Removing a stmt may expose a forwarder block.  */
+      el.el_todo |= TODO_cleanup_cfg;
+    }
+
+  /* Fixup stmts that became noreturn calls.  This may require splitting
+     blocks and thus isn't possible during the dominator walk.  Do this
+     in reverse order so we don't inadvertedly remove a stmt we want to
+     fixup by visiting a dominating now noreturn call first.  */
+  while (!el.to_fixup.is_empty ())
+    {
+      gimple *stmt = el.to_fixup.pop ();
+
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	{
+	  fprintf (dump_file, "Fixing up noreturn call ");
+	  print_gimple_stmt (dump_file, stmt, 0);
+	}
+
+      if (fixup_noreturn_call (stmt))
+	el.el_todo |= TODO_cleanup_cfg;
+    }
+
+  bool do_eh_cleanup = !bitmap_empty_p (el.need_eh_cleanup);
+  bool do_ab_cleanup = !bitmap_empty_p (el.need_ab_cleanup);
+
+  if (do_eh_cleanup)
+    gimple_purge_all_dead_eh_edges (el.need_eh_cleanup);
+
+  if (do_ab_cleanup)
+    gimple_purge_all_dead_abnormal_call_edges (el.need_ab_cleanup);
+
+  if (do_eh_cleanup || do_ab_cleanup)
+    el.el_todo |= TODO_cleanup_cfg;
+
+  statistics_counter_event (cfun, "Eliminated", el.eliminations);
+  statistics_counter_event (cfun, "Insertions", el.insertions);
+
+  return el.el_todo;
+}
+
+
+namespace {
+
+const pass_data pass_data_fre =
+{
+  GIMPLE_PASS, /* type */
+  "fre", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_TREE_FRE, /* tv_id */
+  ( PROP_cfg | PROP_ssa ), /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
+
+class pass_fre : public gimple_opt_pass
+{
+public:
+  pass_fre (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_fre, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  opt_pass * clone () { return new pass_fre (m_ctxt); }
+  virtual bool gate (function *) { return flag_tree_fre != 0; }
+  virtual unsigned int execute (function *);
+
+}; // class pass_fre
+
+unsigned int
+pass_fre::execute (function *)
+{
+  unsigned int todo = 0;
+
+  run_scc_vn (VN_WALKREWRITE);
+
+  /* Remove all the redundant expressions.  */
+  todo |= vn_eliminate (NULL);
+
+  scc_vn_restore_ssa_info ();
+  free_scc_vn ();
+
+  return todo;
+}
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_fre (gcc::context *ctxt)
+{
+  return new pass_fre (ctxt);
+}
diff --git a/gcc/tree-ssa-sccvn.h b/gcc/tree-ssa-sccvn.h
index 68191562b85..830876849bf 100644
--- a/gcc/tree-ssa-sccvn.h
+++ b/gcc/tree-ssa-sccvn.h
@@ -214,6 +214,7 @@ extern vn_ssa_aux_t VN_INFO (tree);
 extern vn_ssa_aux_t VN_INFO_GET (tree);
 tree vn_get_expr_for (tree);
 void run_scc_vn (vn_lookup_kind);
+unsigned int vn_eliminate (bitmap);
 void free_scc_vn (void);
 void scc_vn_restore_ssa_info (void);
 tree vn_nary_op_lookup (tree, vn_nary_op_t *);
diff --git a/gcc/tree-ssa-sink.c b/gcc/tree-ssa-sink.c
index acf832d66f6..1c5d7dd7556 100644
--- a/gcc/tree-ssa-sink.c
+++ b/gcc/tree-ssa-sink.c
@@ -226,7 +226,8 @@ select_best_block (basic_block early_bb,
   /* If BEST_BB is at the same nesting level, then require it to have
      significantly lower execution frequency to avoid gratutious movement.  */
   if (bb_loop_depth (best_bb) == bb_loop_depth (early_bb)
-      && best_bb->frequency < (early_bb->frequency * threshold / 100.0))
+      && best_bb->count.to_frequency (cfun)
+	 < (early_bb->count.to_frequency (cfun) * threshold / 100.0))
     return best_bb;
 
   /* No better block found, so return EARLY_BB, which happens to be the
diff --git a/gcc/tree-ssa-structalias.c b/gcc/tree-ssa-structalias.c
index 3296058dd4b..e6d708f77b4 100644
--- a/gcc/tree-ssa-structalias.c
+++ b/gcc/tree-ssa-structalias.c
@@ -3256,7 +3256,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
 	 we may have to do something cute here.  */
 
       if (may_lt (poly_uint64 (bitpos), get_varinfo (result.var)->fullsize)
-	  && maybe_nonzero (bitmaxsize))
+	  && may_ne (bitmaxsize, 0))
 	{
 	  /* It's also not true that the constraint will actually start at the
 	     right offset, it may start in some padding.  We only care about
@@ -3302,7 +3302,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
 	      results->safe_push (cexpr);
 	    }
 	}
-      else if (known_zero (bitmaxsize))
+      else if (must_eq (bitmaxsize, 0))
 	{
 	  if (dump_file && (dump_flags & TDF_DETAILS))
 	    fprintf (dump_file, "Access to zero-sized part of variable, "
diff --git a/gcc/tree-ssa-tail-merge.c b/gcc/tree-ssa-tail-merge.c
index a65ff31d900..97e90233d58 100644
--- a/gcc/tree-ssa-tail-merge.c
+++ b/gcc/tree-ssa-tail-merge.c
@@ -1530,8 +1530,6 @@ static void
 replace_block_by (basic_block bb1, basic_block bb2)
 {
   edge pred_edge;
-  edge e1, e2;
-  edge_iterator ei;
   unsigned int i;
   gphi *bb2_phi;
 
@@ -1560,9 +1558,13 @@ replace_block_by (basic_block bb1, basic_block bb2)
 
   bb2->count += bb1->count;
 
+  /* FIXME: Fix merging of probabilities.  They need to be redistributed
+     according to the relative counts of merged BBs.  */
+#if 0
   /* Merge the outgoing edge counts from bb1 onto bb2.  */
   profile_count out_sum = profile_count::zero ();
   int out_freq_sum = 0;
+  edge e1, e2;
 
   /* Recompute the edge probabilities from the new merged edge count.
      Use the sum of the new merged edge counts computed above instead
@@ -1570,40 +1572,36 @@ replace_block_by (basic_block bb1, basic_block bb2)
      making the bb count inconsistent with the edge weights.  */
   FOR_EACH_EDGE (e1, ei, bb1->succs)
     {
-      if (e1->count.initialized_p ())
-	out_sum += e1->count;
+      if (e1->count ().initialized_p ())
+	out_sum += e1->count ();
       out_freq_sum += EDGE_FREQUENCY (e1);
     }
   FOR_EACH_EDGE (e1, ei, bb2->succs)
     {
-      if (e1->count.initialized_p ())
-	out_sum += e1->count;
+      if (e1->count ().initialized_p ())
+	out_sum += e1->count ();
       out_freq_sum += EDGE_FREQUENCY (e1);
     }
-
   FOR_EACH_EDGE (e1, ei, bb1->succs)
     {
       e2 = find_edge (bb2, e1->dest);
       gcc_assert (e2);
-      e2->count += e1->count;
-      if (out_sum > 0 && e2->count.initialized_p ())
+      if (out_sum > 0 && e2->count ().initialized_p ())
 	{
-	  e2->probability = e2->count.probability_in (bb2->count);
+	  e2->probability = e2->count ().probability_in (bb2->count);
 	}
-      else if (bb1->frequency && bb2->frequency)
+      else if (bb1->count.to_frequency (cfun) && bb2->count.to_frequency (cfun))
 	e2->probability = e1->probability;
-      else if (bb2->frequency && !bb1->frequency)
+      else if (bb2->count.to_frequency (cfun) && !bb1->count.to_frequency (cfun))
 	;
       else if (out_freq_sum)
 	e2->probability = profile_probability::from_reg_br_prob_base
 		(GCOV_COMPUTE_SCALE (EDGE_FREQUENCY (e1)
 				     + EDGE_FREQUENCY (e2),
 				     out_freq_sum));
-      out_sum += e2->count;
+      out_sum += e2->count ();
     }
-  bb2->frequency += bb1->frequency;
-  if (bb2->frequency > BB_FREQ_MAX)
-    bb2->frequency = BB_FREQ_MAX;
+#endif
 
   /* Move over any user labels from bb1 after the bb2 labels.  */
   gimple_stmt_iterator gsi1 = gsi_start_bb (bb1);
diff --git a/gcc/tree-ssa-threadupdate.c b/gcc/tree-ssa-threadupdate.c
index 28c81a6ec57..1dab0f1fab4 100644
--- a/gcc/tree-ssa-threadupdate.c
+++ b/gcc/tree-ssa-threadupdate.c
@@ -303,7 +303,6 @@ remove_ctrl_stmt_and_useless_edges (basic_block bb, basic_block dest_bb)
       else
 	{
 	  e->probability = profile_probability::always ();
-	  e->count = bb->count;
 	  ei_next (&ei);
 	}
     }
@@ -340,7 +339,6 @@ create_block_for_threading (basic_block bb,
     e->aux = NULL;
 
   /* Zero out the profile, since the block is unreachable for now.  */
-  rd->dup_blocks[count]->frequency = 0;
   rd->dup_blocks[count]->count = profile_count::uninitialized ();
   if (duplicate_blocks)
     bitmap_set_bit (*duplicate_blocks, rd->dup_blocks[count]->index);
@@ -591,7 +589,7 @@ any_remaining_duplicated_blocks (vec<jump_thread_edge *> *path,
 }
 
 
-/* Compute the amount of profile count/frequency coming into the jump threading
+/* Compute the amount of profile count coming into the jump threading
    path stored in RD that we are duplicating, returned in PATH_IN_COUNT_PTR and
    PATH_IN_FREQ_PTR, as well as the amount of counts flowing out of the
    duplicated path, returned in PATH_OUT_COUNT_PTR.  LOCAL_INFO is used to
@@ -599,7 +597,7 @@ any_remaining_duplicated_blocks (vec<jump_thread_edge *> *path,
    edges that need to be ignored in the analysis.  Return true if path contains
    a joiner, false otherwise.
 
-   In the non-joiner case, this is straightforward - all the counts/frequency
+   In the non-joiner case, this is straightforward - all the counts
    flowing into the jump threading path should flow through the duplicated
    block and out of the duplicated path.
 
@@ -741,7 +739,7 @@ compute_path_counts (struct redirection_data *rd,
 	     same last path edge in the case where the last edge has a nocopy
 	     source block.  */
 	  gcc_assert (ein_path->last ()->e == elast);
-	  path_in_count += ein->count;
+	  path_in_count += ein->count ();
 	  path_in_freq += EDGE_FREQUENCY (ein);
 	}
       else if (!ein_path)
@@ -749,7 +747,7 @@ compute_path_counts (struct redirection_data *rd,
 	  /* Keep track of the incoming edges that are not on any jump-threading
 	     path.  These counts will still flow out of original path after all
 	     jump threading is complete.  */
-	    nonpath_count += ein->count;
+	    nonpath_count += ein->count ();
 	}
     }
 
@@ -789,7 +787,7 @@ compute_path_counts (struct redirection_data *rd,
   for (unsigned int i = 1; i < path->length (); i++)
     {
       edge epath = (*path)[i]->e;
-      profile_count cur_count = epath->count;
+      profile_count cur_count = epath->count ();
       if ((*path)[i]->type == EDGE_COPY_SRC_JOINER_BLOCK)
 	{
 	  has_joiner = true;
@@ -809,13 +807,13 @@ compute_path_counts (struct redirection_data *rd,
 		     they are redirected by an invocation of this routine.  */
 		  && !bitmap_bit_p (local_info->duplicate_blocks,
 				    ein->src->index))
-		nonpath_count += ein->count;
+		nonpath_count += ein->count ();
 	    }
 	}
       if (cur_count < path_out_count)
 	path_out_count = cur_count;
-      if (epath->count < min_path_count)
-	min_path_count = epath->count;
+      if (epath->count () < min_path_count)
+	min_path_count = epath->count ();
     }
 
   /* We computed path_out_count above assuming that this path targeted
@@ -830,12 +828,12 @@ compute_path_counts (struct redirection_data *rd,
      (since any path through the joiner with a different elast will not
      include a copy of this elast in its duplicated path).
      So ensure that this path's path_out_count is at least the
-     difference between elast->count and nonpath_count.  Otherwise the edge
+     difference between elast->count () and nonpath_count.  Otherwise the edge
      counts after threading will not be sane.  */
   if (local_info->need_profile_correction
-      && has_joiner && path_out_count < elast->count - nonpath_count)
+      && has_joiner && path_out_count < elast->count () - nonpath_count)
     {
-      path_out_count = elast->count - nonpath_count;
+      path_out_count = elast->count () - nonpath_count;
       /* But neither can we go above the minimum count along the path
 	 we are duplicating.  This can be an issue due to profile
 	 insanities coming in to this pass.  */
@@ -852,267 +850,95 @@ compute_path_counts (struct redirection_data *rd,
 
 /* Update the counts and frequencies for both an original path
    edge EPATH and its duplicate EDUP.  The duplicate source block
-   will get a count/frequency of PATH_IN_COUNT and PATH_IN_FREQ,
+   will get a count of PATH_IN_COUNT and PATH_IN_FREQ,
    and the duplicate edge EDUP will have a count of PATH_OUT_COUNT.  */
 static void
 update_profile (edge epath, edge edup, profile_count path_in_count,
-		profile_count path_out_count, int path_in_freq)
+		profile_count path_out_count)
 {
 
-  /* First update the duplicated block's count / frequency.  */
+  /* First update the duplicated block's count.  */
   if (edup)
     {
       basic_block dup_block = edup->src;
+
+      /* Edup's count is reduced by path_out_count.  We need to redistribute
+         probabilities to the remaining edges.  */
+
+      edge esucc;
+      edge_iterator ei;
+      profile_probability edup_prob
+	 = path_out_count.probability_in (path_in_count);
+
+      /* Either scale up or down the remaining edges.
+	 probabilities are always in range <0,1> and thus we can't do
+	 both by same loop.  */
+      if (edup->probability > edup_prob)
+	{
+	   profile_probability rev_scale
+	     = (profile_probability::always () - edup->probability)
+	       / (profile_probability::always () - edup_prob);
+	   FOR_EACH_EDGE (esucc, ei, dup_block->succs)
+	     if (esucc != edup)
+	       esucc->probability /= rev_scale;
+	}
+      else if (edup->probability < edup_prob)
+	{
+	   profile_probability scale
+	     = (profile_probability::always () - edup_prob)
+	       / (profile_probability::always () - edup->probability);
+	  FOR_EACH_EDGE (esucc, ei, dup_block->succs)
+	    if (esucc != edup)
+	      esucc->probability *= scale;
+	}
+      if (edup_prob.initialized_p ())
+        edup->probability = edup_prob;
+
       gcc_assert (!dup_block->count.initialized_p ());
-      gcc_assert (dup_block->frequency == 0);
       dup_block->count = path_in_count;
-      dup_block->frequency = path_in_freq;
     }
 
-  /* Now update the original block's count and frequency in the
+  if (path_in_count == profile_count::zero ())
+    return;
+
+  profile_count final_count = epath->count () - path_out_count;
+
+  /* Now update the original block's count in the
      opposite manner - remove the counts/freq that will flow
      into the duplicated block.  Handle underflow due to precision/
      rounding issues.  */
   epath->src->count -= path_in_count;
-  epath->src->frequency -= path_in_freq;
-  if (epath->src->frequency < 0)
-    epath->src->frequency = 0;
 
   /* Next update this path edge's original and duplicated counts.  We know
      that the duplicated path will have path_out_count flowing
      out of it (in the joiner case this is the count along the duplicated path
      out of the duplicated joiner).  This count can then be removed from the
      original path edge.  */
-  if (edup)
-    edup->count = path_out_count;
-  epath->count -= path_out_count;
-  /* FIXME: can epath->count be legally uninitialized here?  */
-}
-
 
-/* The duplicate and original joiner blocks may end up with different
-   probabilities (different from both the original and from each other).
-   Recompute the probabilities here once we have updated the edge
-   counts and frequencies.  */
-
-static void
-recompute_probabilities (basic_block bb)
-{
   edge esucc;
   edge_iterator ei;
-  FOR_EACH_EDGE (esucc, ei, bb->succs)
-    {
-      if (!(bb->count > 0))
-	continue;
-
-      /* Prevent overflow computation due to insane profiles.  */
-      if (esucc->count < bb->count)
-	esucc->probability = esucc->count.probability_in (bb->count).guessed ();
-      else
-	/* Can happen with missing/guessed probabilities, since we
-	   may determine that more is flowing along duplicated
-	   path than joiner succ probabilities allowed.
-	   Counts and freqs will be insane after jump threading,
-	   at least make sure probability is sane or we will
-	   get a flow verification error.
-	   Not much we can do to make counts/freqs sane without
-	   redoing the profile estimation.  */
-	esucc->probability = profile_probability::guessed_always ();
-    }
-}
-
-
-/* Update the counts of the original and duplicated edges from a joiner
-   that go off path, given that we have already determined that the
-   duplicate joiner DUP_BB has incoming count PATH_IN_COUNT and
-   outgoing count along the path PATH_OUT_COUNT.  The original (on-)path
-   edge from joiner is EPATH.  */
-
-static void
-update_joiner_offpath_counts (edge epath, basic_block dup_bb,
-			      profile_count path_in_count,
-			      profile_count path_out_count)
-{
-  /* Compute the count that currently flows off path from the joiner.
-     In other words, the total count of joiner's out edges other than
-     epath.  Compute this by walking the successors instead of
-     subtracting epath's count from the joiner bb count, since there
-     are sometimes slight insanities where the total out edge count is
-     larger than the bb count (possibly due to rounding/truncation
-     errors).  */
-  profile_count total_orig_off_path_count = profile_count::zero ();
-  edge enonpath;
-  edge_iterator ei;
-  FOR_EACH_EDGE (enonpath, ei, epath->src->succs)
-    {
-      if (enonpath == epath)
-	continue;
-      total_orig_off_path_count += enonpath->count;
-    }
-
-  /* For the path that we are duplicating, the amount that will flow
-     off path from the duplicated joiner is the delta between the
-     path's cumulative in count and the portion of that count we
-     estimated above as flowing from the joiner along the duplicated
-     path.  */
-  profile_count total_dup_off_path_count = path_in_count - path_out_count;
-
-  /* Now do the actual updates of the off-path edges.  */
-  FOR_EACH_EDGE (enonpath, ei, epath->src->succs)
-    {
-      /* Look for edges going off of the threading path.  */
-      if (enonpath == epath)
-	continue;
-
-      /* Find the corresponding edge out of the duplicated joiner.  */
-      edge enonpathdup = find_edge (dup_bb, enonpath->dest);
-      gcc_assert (enonpathdup);
-
-      /* We can't use the original probability of the joiner's out
-	 edges, since the probabilities of the original branch
-	 and the duplicated branches may vary after all threading is
-	 complete.  But apportion the duplicated joiner's off-path
-	 total edge count computed earlier (total_dup_off_path_count)
-	 among the duplicated off-path edges based on their original
-	 ratio to the full off-path count (total_orig_off_path_count).
-	 */
-      profile_probability scale
-		 = enonpath->count.probability_in (total_orig_off_path_count);
-      /* Give the duplicated offpath edge a portion of the duplicated
-	 total.  */
-      enonpathdup->count = total_dup_off_path_count.apply_probability (scale);
-      /* Now update the original offpath edge count, handling underflow
-	 due to rounding errors.  */
-      enonpath->count -= enonpathdup->count;
-    }
-}
-
-
-/* Check if the paths through RD all have estimated frequencies but zero
-   profile counts.  This is more accurate than checking the entry block
-   for a zero profile count, since profile insanities sometimes creep in.  */
-
-static bool
-estimated_freqs_path (struct redirection_data *rd)
-{
-  edge e = rd->incoming_edges->e;
-  vec<jump_thread_edge *> *path = THREAD_PATH (e);
-  edge ein;
-  edge_iterator ei;
-  bool non_zero_freq = false;
-  FOR_EACH_EDGE (ein, ei, e->dest->preds)
-    {
-      if (ein->count > 0)
-	return false;
-      non_zero_freq |= ein->src->frequency != 0;
-    }
-
-  for (unsigned int i = 1; i < path->length (); i++)
-    {
-      edge epath = (*path)[i]->e;
-      if (epath->src->count > 0)
-	return false;
-      non_zero_freq |= epath->src->frequency != 0;
-      edge esucc;
-      FOR_EACH_EDGE (esucc, ei, epath->src->succs)
-	{
-	  if (esucc->count > 0)
-	    return false;
-	  non_zero_freq |= esucc->src->frequency != 0;
-	}
-    }
-  return non_zero_freq;
-}
-
-
-/* Invoked for routines that have guessed frequencies and no profile
-   counts to record the block and edge frequencies for paths through RD
-   in the profile count fields of those blocks and edges.  This is because
-   ssa_fix_duplicate_block_edges incrementally updates the block and
-   edge counts as edges are redirected, and it is difficult to do that
-   for edge frequencies which are computed on the fly from the source
-   block frequency and probability.  When a block frequency is updated
-   its outgoing edge frequencies are affected and become difficult to
-   adjust.  */
-
-static void
-freqs_to_counts_path (struct redirection_data *rd)
-{
-  edge e = rd->incoming_edges->e;
-  vec<jump_thread_edge *> *path = THREAD_PATH (e);
-  edge ein;
-  edge_iterator ei;
-  FOR_EACH_EDGE (ein, ei, e->dest->preds)
-    {
-      /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
-	 errors applying the probability when the frequencies are very
-	 small.  */
-      if (ein->probability.initialized_p ())
-        ein->count = profile_count::from_gcov_type
-		  (apply_probability (ein->src->frequency * REG_BR_PROB_BASE,
-				        ein->probability
-					  .to_reg_br_prob_base ())).guessed ();
-      else
-	/* FIXME: this is hack; we should track uninitialized values.  */
-	ein->count = profile_count::zero ();
-    }
+  profile_probability epath_prob = final_count.probability_in (epath->src->count);
 
-  for (unsigned int i = 1; i < path->length (); i++)
+  if (epath->probability > epath_prob)
     {
-      edge epath = (*path)[i]->e;
-      edge esucc;
-      /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
-	 errors applying the edge probability when the frequencies are very
-	 small.  */
-      epath->src->count = 
-	profile_count::from_gcov_type
-	  (epath->src->frequency * REG_BR_PROB_BASE);
-      FOR_EACH_EDGE (esucc, ei, epath->src->succs)
-	esucc->count = 
-	   esucc->src->count.apply_probability (esucc->probability);
+       profile_probability rev_scale
+	 = (profile_probability::always () - epath->probability)
+	   / (profile_probability::always () - epath_prob);
+       FOR_EACH_EDGE (esucc, ei, epath->src->succs)
+	 if (esucc != epath)
+	   esucc->probability /= rev_scale;
     }
-}
-
-
-/* For routines that have guessed frequencies and no profile counts, where we
-   used freqs_to_counts_path to record block and edge frequencies for paths
-   through RD, we clear the counts after completing all updates for RD.
-   The updates in ssa_fix_duplicate_block_edges are based off the count fields,
-   but the block frequencies and edge probabilities were updated as well,
-   so we can simply clear the count fields.  */
-
-static void
-clear_counts_path (struct redirection_data *rd)
-{
-  edge e = rd->incoming_edges->e;
-  vec<jump_thread_edge *> *path = THREAD_PATH (e);
-  edge ein, esucc;
-  edge_iterator ei;
-  profile_count val = profile_count::uninitialized ();
-  if (profile_status_for_fn (cfun) == PROFILE_READ)
-    val = profile_count::zero ();
-
-  FOR_EACH_EDGE (ein, ei, e->dest->preds)
-    ein->count = val;
-
-  /* First clear counts along original path.  */
-  for (unsigned int i = 1; i < path->length (); i++)
+  else if (epath->probability < epath_prob)
     {
-      edge epath = (*path)[i]->e;
+       profile_probability scale
+	 = (profile_probability::always () - epath_prob)
+	   / (profile_probability::always () - epath->probability);
       FOR_EACH_EDGE (esucc, ei, epath->src->succs)
-	esucc->count = val;
-      epath->src->count = val;
-    }
-  /* Also need to clear the counts along duplicated path.  */
-  for (unsigned int i = 0; i < 2; i++)
-    {
-      basic_block dup = rd->dup_blocks[i];
-      if (!dup)
-	continue;
-      FOR_EACH_EDGE (esucc, ei, dup->succs)
-	esucc->count = val;
-      dup->count = val;
+	if (esucc != epath)
+	  esucc->probability *= scale;
     }
+  if (epath_prob.initialized_p ())
+    epath->probability = epath_prob;
 }
 
 /* Wire up the outgoing edges from the duplicate blocks and
@@ -1130,20 +956,6 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
   profile_count path_out_count = profile_count::zero ();
   int path_in_freq = 0;
 
-  /* This routine updates profile counts, frequencies, and probabilities
-     incrementally. Since it is difficult to do the incremental updates
-     using frequencies/probabilities alone, for routines without profile
-     data we first take a snapshot of the existing block and edge frequencies
-     by copying them into the empty profile count fields.  These counts are
-     then used to do the incremental updates, and cleared at the end of this
-     routine.  If the function is marked as having a profile, we still check
-     to see if the paths through RD are using estimated frequencies because
-     the routine had zero profile counts.  */
-  bool do_freqs_to_counts = (profile_status_for_fn (cfun) != PROFILE_READ
-			     || estimated_freqs_path (rd));
-  if (do_freqs_to_counts)
-    freqs_to_counts_path (rd);
-
   /* First determine how much profile count to move from original
      path to the duplicate path.  This is tricky in the presence of
      a joiner (see comments for compute_path_counts), where some portion
@@ -1154,7 +966,6 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 					 &path_in_count, &path_out_count,
 					 &path_in_freq);
 
-  int cur_path_freq = path_in_freq;
   for (unsigned int count = 0, i = 1; i < path->length (); i++)
     {
       edge epath = (*path)[i]->e;
@@ -1220,30 +1031,14 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 		}
 	    }
 
-	  /* Update the counts and frequency of both the original block
+	  /* Update the counts of both the original block
 	     and path edge, and the duplicates.  The path duplicate's
-	     incoming count and frequency are the totals for all edges
+	     incoming count are the totals for all edges
 	     incoming to this jump threading path computed earlier.
 	     And we know that the duplicated path will have path_out_count
 	     flowing out of it (i.e. along the duplicated path out of the
 	     duplicated joiner).  */
-	  update_profile (epath, e2, path_in_count, path_out_count,
-			  path_in_freq);
-
-	  /* Next we need to update the counts of the original and duplicated
-	     edges from the joiner that go off path.  */
-	  update_joiner_offpath_counts (epath, e2->src, path_in_count,
-					path_out_count);
-
-	  /* Finally, we need to set the probabilities on the duplicated
-	     edges out of the duplicated joiner (e2->src).  The probabilities
-	     along the original path will all be updated below after we finish
-	     processing the whole path.  */
-	  recompute_probabilities (e2->src);
-
-	  /* Record the frequency flowing to the downstream duplicated
-	     path blocks.  */
-	  cur_path_freq = EDGE_FREQUENCY (e2);
+	  update_profile (epath, e2, path_in_count, path_out_count);
 	}
       else if ((*path)[i]->type == EDGE_COPY_SRC_BLOCK)
 	{
@@ -1253,7 +1048,7 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 	  if (count == 1)
 	    single_succ_edge (rd->dup_blocks[1])->aux = NULL;
 
-	  /* Update the counts and frequency of both the original block
+	  /* Update the counts of both the original block
 	     and path edge, and the duplicates.  Since we are now after
 	     any joiner that may have existed on the path, the count
 	     flowing along the duplicated threaded path is path_out_count.
@@ -1263,8 +1058,7 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 	     been updated at the end of that handling to the edge frequency
 	     along the duplicated joiner path edge.  */
 	  update_profile (epath, EDGE_SUCC (rd->dup_blocks[count], 0),
-			  path_out_count, path_out_count,
-			  cur_path_freq);
+			  path_out_count, path_out_count);
 	}
       else
 	{
@@ -1281,8 +1075,7 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 	     thread path (path_in_freq).  If we had a joiner, it would have
 	     been updated at the end of that handling to the edge frequency
 	     along the duplicated joiner path edge.  */
-	   update_profile (epath, NULL, path_out_count, path_out_count,
-			   cur_path_freq);
+	   update_profile (epath, NULL, path_out_count, path_out_count);
 	}
 
       /* Increment the index into the duplicated path when we processed
@@ -1293,19 +1086,6 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 	  count++;
 	}
     }
-
-  /* Now walk orig blocks and update their probabilities, since the
-     counts and freqs should be updated properly by above loop.  */
-  for (unsigned int i = 1; i < path->length (); i++)
-    {
-      edge epath = (*path)[i]->e;
-      recompute_probabilities (epath->src);
-    }
-
-  /* Done with all profile and frequency updates, clear counts if they
-     were copied.  */
-  if (do_freqs_to_counts)
-    clear_counts_path (rd);
 }
 
 /* Hash table traversal callback routine to create duplicate blocks.  */
@@ -2215,7 +1995,6 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
   struct loop *loop = entry->dest->loop_father;
   edge exit_copy;
   edge redirected;
-  int curr_freq;
   profile_count curr_count;
 
   if (!can_copy_bbs_p (region, n_region))
@@ -2247,8 +2026,7 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
      invalidating the property that is propagated by executing all the blocks of
      the jump-thread path in order.  */
 
-  curr_count = entry->count;
-  curr_freq = EDGE_FREQUENCY (entry);
+  curr_count = entry->count ();
 
   for (i = 0; i < n_region; i++)
     {
@@ -2259,10 +2037,8 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
       /* Watch inconsistent profile.  */
       if (curr_count > region[i]->count)
 	curr_count = region[i]->count;
-      if (curr_freq > region[i]->frequency)
-	curr_freq = region[i]->frequency;
       /* Scale current BB.  */
-      if (region[i]->count > 0 && curr_count.initialized_p ())
+      if (region[i]->count.nonzero_p () && curr_count.initialized_p ())
 	{
 	  /* In the middle of the path we only scale the frequencies.
 	     In last BB we need to update probabilities of outgoing edges
@@ -2273,24 +2049,11 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
 					         region[i]->count);
 	  else
 	    update_bb_profile_for_threading (region[i],
-					     curr_freq, curr_count,
+					     curr_count,
 					     exit);
 	  scale_bbs_frequencies_profile_count (region_copy + i, 1, curr_count,
 					       region_copy[i]->count);
 	}
-      else if (region[i]->frequency)
-	{
-	  if (i + 1 != n_region)
-	    scale_bbs_frequencies_int (region + i, 1,
-				       region[i]->frequency - curr_freq,
-				       region[i]->frequency);
-	  else
-	    update_bb_profile_for_threading (region[i],
-					     curr_freq, curr_count,
-					     exit);
-	  scale_bbs_frequencies_int (region_copy + i, 1, curr_freq,
-				     region_copy[i]->frequency);
-	}
 
       if (single_succ_p (bb))
 	{
@@ -2299,8 +2062,7 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
 		      || region_copy[i + 1] == single_succ_edge (bb)->dest);
 	  if (i + 1 != n_region)
 	    {
-	      curr_freq = EDGE_FREQUENCY (single_succ_edge (bb));
-	      curr_count = single_succ_edge (bb)->count;
+	      curr_count = single_succ_edge (bb)->count ();
 	    }
 	  continue;
 	}
@@ -2330,8 +2092,7 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
 	  }
 	else
 	  {
-	    curr_freq = EDGE_FREQUENCY (e);
-	    curr_count = e->count;
+	    curr_count = e->count ();
 	  }
     }
 
@@ -2353,7 +2114,6 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
     {
       rescan_loop_exit (e, true, false);
       e->probability = profile_probability::always ();
-      e->count = region_copy[n_region - 1]->count;
     }
 
   /* Redirect the entry and add the phi node arguments.  */
diff --git a/gcc/tree-ssa-uncprop.c b/gcc/tree-ssa-uncprop.c
index 200ec70c9b7..35a49d28420 100644
--- a/gcc/tree-ssa-uncprop.c
+++ b/gcc/tree-ssa-uncprop.c
@@ -408,40 +408,10 @@ uncprop_into_successor_phis (basic_block bb)
     }
 }
 
-/* Ignoring loop backedges, if BB has precisely one incoming edge then
-   return that edge.  Otherwise return NULL.  */
-static edge
-single_incoming_edge_ignoring_loop_edges (basic_block bb)
-{
-  edge retval = NULL;
-  edge e;
-  edge_iterator ei;
-
-  FOR_EACH_EDGE (e, ei, bb->preds)
-    {
-      /* A loop back edge can be identified by the destination of
-	 the edge dominating the source of the edge.  */
-      if (dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
-	continue;
-
-      /* If we have already seen a non-loop edge, then we must have
-	 multiple incoming non-loop edges and thus we return NULL.  */
-      if (retval)
-	return NULL;
-
-      /* This is the first non-loop incoming edge we have found.  Record
-	 it.  */
-      retval = e;
-    }
-
-  return retval;
-}
-
 edge
 uncprop_dom_walker::before_dom_children (basic_block bb)
 {
   basic_block parent;
-  edge e;
   bool recorded = false;
 
   /* If this block is dominated by a single incoming edge and that edge
@@ -450,7 +420,7 @@ uncprop_dom_walker::before_dom_children (basic_block bb)
   parent = get_immediate_dominator (CDI_DOMINATORS, bb);
   if (parent)
     {
-      e = single_incoming_edge_ignoring_loop_edges (bb);
+      edge e = single_pred_edge_ignoring_loop_edges (bb, false);
 
       if (e && e->src == parent && e->aux)
 	{
diff --git a/gcc/tree-switch-conversion.c b/gcc/tree-switch-conversion.c
index dc9fc84c6a0..f0d158391df 100644
--- a/gcc/tree-switch-conversion.c
+++ b/gcc/tree-switch-conversion.c
@@ -107,8 +107,7 @@ hoist_edge_and_branch_if_true (gimple_stmt_iterator *gsip,
   e_false->flags &= ~EDGE_FALLTHRU;
   e_false->flags |= EDGE_FALSE_VALUE;
   e_false->probability = e_true->probability.invert ();
-  e_false->count = split_bb->count - e_true->count;
-  new_bb->count = e_false->count;
+  new_bb->count = e_false->count ();
 
   if (update_dominators)
     {
@@ -239,9 +238,9 @@ case_bit_test_cmp (const void *p1, const void *p2)
   const struct case_bit_test *const d1 = (const struct case_bit_test *) p1;
   const struct case_bit_test *const d2 = (const struct case_bit_test *) p2;
 
-  if (d2->target_edge->count < d1->target_edge->count)
+  if (d2->target_edge->count () < d1->target_edge->count ())
     return -1;
-  if (d2->target_edge->count > d1->target_edge->count)
+  if (d2->target_edge->count () > d1->target_edge->count ())
     return 1;
   if (d2->bits != d1->bits)
     return d2->bits - d1->bits;
@@ -635,10 +634,10 @@ collect_switch_conv_info (gswitch *swtch, struct switch_conv_info *info)
     = label_to_block (CASE_LABEL (gimple_switch_default_label (swtch)));
   e_default = find_edge (info->switch_bb, info->default_bb);
   info->default_prob = e_default->probability;
-  info->default_count = e_default->count;
+  info->default_count = e_default->count ();
   FOR_EACH_EDGE (e, ei, info->switch_bb->succs)
     if (e != e_default)
-      info->other_count += e->count;
+      info->other_count += e->count ();
 
   /* Get upper and lower bounds of case values, and the covered range.  */
   min_case = gimple_switch_label (swtch, 1);
@@ -1424,19 +1423,16 @@ gen_inbound_check (gswitch *swtch, struct switch_conv_info *info)
   if (!info->default_case_nonstandard)
     e01 = make_edge (bb0, bb1, EDGE_TRUE_VALUE);
   e01->probability = info->default_prob.invert ();
-  e01->count = info->other_count;
 
   /* flags and profiles of the edge taking care of out-of-range values */
   e02->flags &= ~EDGE_FALLTHRU;
   e02->flags |= EDGE_FALSE_VALUE;
   e02->probability = info->default_prob;
-  e02->count = info->default_count;
 
   bbf = info->final_bb;
 
   e1f = make_edge (bb1, bbf, EDGE_FALLTHRU);
   e1f->probability = profile_probability::always ();
-  e1f->count = info->other_count;
 
   if (info->default_case_nonstandard)
     e2f = NULL;
@@ -1444,14 +1440,13 @@ gen_inbound_check (gswitch *swtch, struct switch_conv_info *info)
     {
       e2f = make_edge (bb2, bbf, EDGE_FALLTHRU);
       e2f->probability = profile_probability::always ();
-      e2f->count = info->default_count;
     }
 
   /* frequencies of the new BBs */
-  bb1->frequency = EDGE_FREQUENCY (e01);
-  bb2->frequency = EDGE_FREQUENCY (e02);
+  bb1->count = e01->count ();
+  bb2->count = e02->count ();
   if (!info->default_case_nonstandard)
-    bbf->frequency = EDGE_FREQUENCY (e1f) + EDGE_FREQUENCY (e2f);
+    bbf->count = e1f->count () + e2f->count ();
 
   /* Tidy blocks that have become unreachable.  */
   prune_bbs (bbd, info->final_bb,
@@ -2248,12 +2243,10 @@ do_jump_if_equal (basic_block bb, tree op0, tree op1, basic_block label_bb,
   edge false_edge = split_block (bb, cond);
   false_edge->flags = EDGE_FALSE_VALUE;
   false_edge->probability = prob.invert ();
-  false_edge->count = bb->count.apply_probability (false_edge->probability);
 
   edge true_edge = make_edge (bb, label_bb, EDGE_TRUE_VALUE);
   fix_phi_operands_for_edge (true_edge, phi_mapping);
   true_edge->probability = prob;
-  true_edge->count = bb->count.apply_probability (true_edge->probability);
 
   return false_edge->dest;
 }
@@ -2293,12 +2286,10 @@ emit_cmp_and_jump_insns (basic_block bb, tree op0, tree op1,
   edge false_edge = split_block (bb, cond);
   false_edge->flags = EDGE_FALSE_VALUE;
   false_edge->probability = prob.invert ();
-  false_edge->count = bb->count.apply_probability (false_edge->probability);
 
   edge true_edge = make_edge (bb, label_bb, EDGE_TRUE_VALUE);
   fix_phi_operands_for_edge (true_edge, phi_mapping);
   true_edge->probability = prob;
-  true_edge->count = bb->count.apply_probability (true_edge->probability);
 
   return false_edge->dest;
 }
diff --git a/gcc/tree-tailcall.c b/gcc/tree-tailcall.c
index c4b8cee4d27..0e637147e8c 100644
--- a/gcc/tree-tailcall.c
+++ b/gcc/tree-tailcall.c
@@ -805,20 +805,14 @@ adjust_return_value (basic_block bb, tree m, tree a)
 /* Subtract COUNT and FREQUENCY from the basic block and it's
    outgoing edge.  */
 static void
-decrease_profile (basic_block bb, profile_count count, int frequency)
+decrease_profile (basic_block bb, profile_count count)
 {
-  edge e;
   bb->count = bb->count - count;
-  bb->frequency -= frequency;
-  if (bb->frequency < 0)
-    bb->frequency = 0;
   if (!single_succ_p (bb))
     {
       gcc_assert (!EDGE_COUNT (bb->succs));
       return;
     }
-  e = single_succ_edge (bb);
-  e->count -= count;
 }
 
 /* Returns true if argument PARAM of the tail recursive call needs to be copied
@@ -895,11 +889,10 @@ eliminate_tail_call (struct tailcall *t)
 
   /* Number of executions of function has reduced by the tailcall.  */
   e = single_succ_edge (gsi_bb (t->call_gsi));
-  decrease_profile (EXIT_BLOCK_PTR_FOR_FN (cfun), e->count, EDGE_FREQUENCY (e));
-  decrease_profile (ENTRY_BLOCK_PTR_FOR_FN (cfun), e->count,
-		    EDGE_FREQUENCY (e));
+  decrease_profile (EXIT_BLOCK_PTR_FOR_FN (cfun), e->count ());
+  decrease_profile (ENTRY_BLOCK_PTR_FOR_FN (cfun), e->count ());
   if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
-    decrease_profile (e->dest, e->count, EDGE_FREQUENCY (e));
+    decrease_profile (e->dest, e->count ());
 
   /* Replace the call by a jump to the start of function.  */
   e = redirect_edge_and_branch (single_succ_edge (gsi_bb (t->call_gsi)),
diff --git a/gcc/tree-vect-generic.c b/gcc/tree-vect-generic.c
index 1152222be08..b19b6fbc0f0 100644
--- a/gcc/tree-vect-generic.c
+++ b/gcc/tree-vect-generic.c
@@ -1443,6 +1443,7 @@ ssa_uniform_vector_p (tree op)
 {
   if (TREE_CODE (op) == VECTOR_CST
       || TREE_CODE (op) == VEC_DUPLICATE_CST
+      || TREE_CODE (op) == VEC_DUPLICATE_EXPR
       || TREE_CODE (op) == CONSTRUCTOR)
     return uniform_vector_p (op);
   if (TREE_CODE (op) == SSA_NAME)
diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c
index 8cdc3c2521e..8811679e2bf 100644
--- a/gcc/tree-vect-loop-manip.c
+++ b/gcc/tree-vect-loop-manip.c
@@ -1481,13 +1481,10 @@ slpeel_add_loop_guard (basic_block guard_bb, tree cond,
   /* Add new edge to connect guard block to the merge/loop-exit block.  */
   new_e = make_edge (guard_bb, guard_to, EDGE_TRUE_VALUE);
 
-  new_e->count = guard_bb->count;
   new_e->probability = probability;
-  new_e->count = enter_e->count.apply_probability (probability);
   if (irreducible_p)
     new_e->flags |= EDGE_IRREDUCIBLE_LOOP;
 
-  enter_e->count -= new_e->count;
   enter_e->probability = probability.invert ();
   set_immediate_dominator (CDI_DOMINATORS, guard_to, dom_bb);
 
@@ -2914,9 +2911,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
 
 	  /* Simply propagate profile info from guard_bb to guard_to which is
 	     a merge point of control flow.  */
-	  guard_to->frequency = guard_bb->frequency;
 	  guard_to->count = guard_bb->count;
-	  single_succ_edge (guard_to)->count = guard_to->count;
 	  /* Scale probability of epilog loop back.
 	     FIXME: We should avoid scaling down and back up.  Profile may
 	     get lost if we scale down to 0.  */
@@ -3288,7 +3283,7 @@ vect_loop_versioning (loop_vec_info loop_vinfo,
     cond_expr = fold_build2 (GE_EXPR, boolean_type_node, scalar_loop_iters,
 			     build_int_cst (TREE_TYPE (scalar_loop_iters),
 					    th - 1));
-  if (maybe_nonzero (versioning_threshold))
+  if (may_ne (versioning_threshold, 0U))
     {
       tree expr = fold_build2 (GE_EXPR, boolean_type_node, scalar_loop_iters,
 			       build_int_cst (TREE_TYPE (scalar_loop_iters),
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 61da0e7998d..91a3610a1a0 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -1841,7 +1841,7 @@ vect_update_vf_for_slp (loop_vec_info loop_vinfo)
 		     "=== vect_update_vf_for_slp ===\n");
 
   vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-  gcc_assert (known_nonzero (vectorization_factor));
+  gcc_assert (must_ne (vectorization_factor, 0U));
 
   /* If all the stmts in the loop can be SLPed, we perform only SLP, and
      vectorization factor of the loop is the unrolling factor required by
@@ -2366,7 +2366,7 @@ start_over:
 
   /* Now the vectorization factor is final.  */
   poly_uint64 vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-  gcc_assert (known_nonzero (vectorization_factor));
+  gcc_assert (must_ne (vectorization_factor, 0U));
 
   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && dump_enabled_p ())
     {
@@ -2814,7 +2814,7 @@ vect_analyze_loop (struct loop *loop, loop_vec_info orig_loop_vinfo)
 
       if (fatal
 	  || next_size == vector_sizes.length ()
-	  || known_zero (current_vector_size))
+	  || must_eq (current_vector_size, 0U))
 	return NULL;
 
       /* Try the next biggest vector size.  */
@@ -6666,9 +6666,13 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi,
 	  reduc_index = i;
 	  continue;
 	}
-      else
+      else if (tem)
 	{
-	  if (!vectype_in)
+	  /* To properly compute ncopies we are interested in the widest
+	     input type in case we're looking at a widening accumulation.  */
+	  if (!vectype_in
+	      || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in)))
+		  < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (tem)))))
 	    vectype_in = tem;
 	}
 
@@ -8642,36 +8646,27 @@ scale_profile_for_vect_loop (struct loop *loop, unsigned vf)
   edge preheader = loop_preheader_edge (loop);
   /* Reduce loop iterations by the vectorization factor.  */
   gcov_type new_est_niter = niter_for_unrolled_loop (loop, vf);
-  profile_count freq_h = loop->header->count, freq_e = preheader->count;
+  profile_count freq_h = loop->header->count, freq_e = preheader->count ();
 
-  /* Use frequency only if counts are zero.  */
-  if (!(freq_h > 0) && !(freq_e > 0))
-    {
-      freq_h = profile_count::from_gcov_type (loop->header->frequency);
-      freq_e = profile_count::from_gcov_type (EDGE_FREQUENCY (preheader));
-    }
-  if (freq_h > 0)
+  if (freq_h.nonzero_p ())
     {
       profile_probability p;
 
       /* Avoid dropping loop body profile counter to 0 because of zero count
 	 in loop's preheader.  */
-      if (!(freq_e > profile_count::from_gcov_type (1)))
-       freq_e = profile_count::from_gcov_type (1);
+      if (!(freq_e == profile_count::zero ()))
+        freq_e = freq_e.force_nonzero ();
       p = freq_e.apply_scale (new_est_niter + 1, 1).probability_in (freq_h);
       scale_loop_frequencies (loop, p);
     }
 
-  basic_block exit_bb = single_pred (loop->latch);
   edge exit_e = single_exit (loop);
-  exit_e->count = loop_preheader_edge (loop)->count;
   exit_e->probability = profile_probability::always ()
 				 .apply_scale (1, new_est_niter + 1);
 
   edge exit_l = single_pred_edge (loop->latch);
   profile_probability prob = exit_l->probability;
   exit_l->probability = exit_e->probability.invert ();
-  exit_l->count = exit_bb->count - exit_e->count;
   if (prob.initialized_p () && exit_l->probability.initialized_p ())
     scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob);
 }
@@ -9327,7 +9322,7 @@ optimize_mask_stores (struct loop *loop)
       efalse = make_edge (bb, store_bb, EDGE_FALSE_VALUE);
       /* Put STORE_BB to likely part.  */
       efalse->probability = profile_probability::unlikely ();
-      store_bb->frequency = PROB_ALWAYS - EDGE_FREQUENCY (efalse);
+      store_bb->count = efalse->count ();
       make_single_succ_edge (store_bb, join_bb, EDGE_FALLTHRU);
       if (dom_info_available_p (CDI_DOMINATORS))
 	set_immediate_dominator (CDI_DOMINATORS, store_bb, bb);
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index 57c76364b9c..066ec48c056 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -3979,6 +3979,20 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
 	}
       else if (COMPARISON_CLASS_P (rhs1))
 	{
+	  /* Check whether we're comparing scalar booleans and (if so)
+	     whether a better mask type exists than the mask associated
+	     with boolean-sized elements.  This avoids unnecessary packs
+	     and unpacks if the booleans are set from comparisons of
+	     wider types.  E.g. in:
+
+	       int x1, x2, x3, x4, y1, y1;
+	       ...
+	       bool b1 = (x1 == x2);
+	       bool b2 = (x3 == x4);
+	       ... = b1 == b2 ? y1 : y2;
+
+	     it is better for b1 and b2 to use the mask type associated
+	     with int elements rather bool (byte) elements.  */
 	  rhs1_type = search_type_for_mask (TREE_OPERAND (rhs1, 0), vinfo);
 	  if (!rhs1_type)
 	    rhs1_type = TREE_TYPE (TREE_OPERAND (rhs1, 0));
@@ -3992,9 +4006,11 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
 	return NULL;
 
       /* Continue if a conversion is needed.  Also continue if we have
-	 a comparison whose natural vector type is different from VECTYPE2;
-	 in that case we'll replace the comparison with an SSA name and
-	 behave as though the comparison was an SSA name from the outset.  */
+	 a comparison whose vector type would normally be different from
+	 VECTYPE2 when considered in isolation.  In that case we'll
+	 replace the comparison with an SSA name (so that we can record
+	 its vector type) and behave as though the comparison was an SSA
+	 name from the outset.  */
       if (must_eq (TYPE_VECTOR_SUBPARTS (vectype1),
 		   TYPE_VECTOR_SUBPARTS (vectype2))
 	  && (TREE_CODE (rhs1) == SSA_NAME
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index c6ad2c557fa..01dfecaafb6 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -3242,7 +3242,7 @@ vect_slp_bb (basic_block bb)
 
       if (vectorized
 	  || next_size == vector_sizes.length ()
-	  || known_zero (current_vector_size)
+	  || must_eq (current_vector_size, 0U)
 	  /* If vect_slp_analyze_bb_1 signaled that analysis for all
 	     vector sizes will fail do not bother iterating.  */
 	  || fatal)
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 1bc2d43773c..f810212ca1b 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -98,6 +98,12 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
 		  enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
 		  int misalign, enum vect_cost_model_location where)
 {
+  if ((kind == vector_load || kind == unaligned_load)
+      && STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+    kind = vector_gather_load;
+  if ((kind == vector_store || kind == unaligned_store)
+      && STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+    kind = vector_scatter_store;
   if (body_cost_vec)
     {
       tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
@@ -4756,7 +4762,7 @@ vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi,
   vec<tree> vargs = vNULL;
   size_t i, nargs;
   tree lhs, rtype, ratype;
-  vec<constructor_elt, va_gc> *ret_ctor_elts;
+  vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
 
   /* Is STMT a vectorizable call?   */
   if (!is_gimple_call (stmt))
@@ -8975,7 +8981,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
 	         we need to skip the gaps after we manage to fully load
 		 all elements.  group_gap_adj is GROUP_SIZE here.  */
 	      group_elt += nunits;
-	      if (maybe_nonzero (group_gap_adj)
+	      if (may_ne (group_gap_adj, 0U)
 		  && !slp_perm
 		  && must_eq (group_elt, group_size - group_gap_adj))
 		{
@@ -8990,7 +8996,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
 	    }
 	  /* Bump the vector pointer to account for a gap or for excess
 	     elements loaded for a permuted SLP load.  */
-	  if (maybe_nonzero (group_gap_adj) && slp_perm)
+	  if (may_ne (group_gap_adj, 0U) && slp_perm)
 	    {
 	      poly_wide_int bump_val
 		= (wi::to_wide (TYPE_SIZE_UNIT (elem_type))
@@ -10541,7 +10547,7 @@ get_vectype_for_scalar_type_and_size (tree scalar_type, poly_uint64 size)
 
   /* If no size was supplied use the mode the target prefers.   Otherwise
      lookup a vector mode of the specified size.  */
-  if (known_zero (size))
+  if (must_eq (size, 0U))
     simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
   else if (!multiple_p (size, nbytes, &nunits)
 	   || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
@@ -10579,7 +10585,7 @@ get_vectype_for_scalar_type (tree scalar_type)
   vectype = get_vectype_for_scalar_type_and_size (scalar_type,
 						  current_vector_size);
   if (vectype
-      && known_zero (current_vector_size))
+      && must_eq (current_vector_size, 0U))
     current_vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
   return vectype;
 }
diff --git a/gcc/tree-vrp.c b/gcc/tree-vrp.c
index a34dea905d3..f229c8983bf 100644
--- a/gcc/tree-vrp.c
+++ b/gcc/tree-vrp.c
@@ -1619,21 +1619,20 @@ extract_range_from_ssa_name (value_range *vr, tree var)
 }
 
 
-/* Wrapper around int_const_binop.  If the operation overflows and
-   overflow is undefined, then adjust the result to be
-   -INF or +INF depending on CODE, VAL1 and VAL2.  Sets *OVERFLOW_P
-   to whether the operation overflowed.  For division by zero
-   the result is indeterminate but *OVERFLOW_P is set.  */
+/* Wrapper around int_const_binop.  Return true if we can compute the
+   result; i.e. if the operation doesn't overflow or if the overflow is
+   undefined.  In the latter case (if the operation overflows and
+   overflow is undefined), then adjust the result to be -INF or +INF
+   depending on CODE, VAL1 and VAL2.  Return the value in *RES.
 
-static wide_int
-vrp_int_const_binop (enum tree_code code, tree val1, tree val2,
-		     bool *overflow_p)
+   Return false for division by zero, for which the result is
+   indeterminate.  */
+
+static bool
+vrp_int_const_binop (enum tree_code code, tree val1, tree val2, wide_int *res)
 {
   bool overflow = false;
   signop sign = TYPE_SIGN (TREE_TYPE (val1));
-  wide_int res;
-
-  *overflow_p = false;
 
   switch (code)
     {
@@ -1654,57 +1653,45 @@ vrp_int_const_binop (enum tree_code code, tree val1, tree val2,
 	  /* It's unclear from the C standard whether shifts can overflow.
 	     The following code ignores overflow; perhaps a C standard
 	     interpretation ruling is needed.  */
-	  res = wi::rshift (wi::to_wide (val1), wval2, sign);
+	  *res = wi::rshift (wi::to_wide (val1), wval2, sign);
 	else
-	  res = wi::lshift (wi::to_wide (val1), wval2);
+	  *res = wi::lshift (wi::to_wide (val1), wval2);
 	break;
       }
 
     case MULT_EXPR:
-      res = wi::mul (wi::to_wide (val1),
-		     wi::to_wide (val2), sign, &overflow);
+      *res = wi::mul (wi::to_wide (val1),
+		      wi::to_wide (val2), sign, &overflow);
       break;
 
     case TRUNC_DIV_EXPR:
     case EXACT_DIV_EXPR:
       if (val2 == 0)
-	{
-	  *overflow_p = true;
-	  return res;
-	}
+	return false;
       else
-	res = wi::div_trunc (wi::to_wide (val1),
-			     wi::to_wide (val2), sign, &overflow);
+	*res = wi::div_trunc (wi::to_wide (val1),
+			      wi::to_wide (val2), sign, &overflow);
       break;
 
     case FLOOR_DIV_EXPR:
       if (val2 == 0)
-	{
-	  *overflow_p = true;
-	  return res;
-	}
-      res = wi::div_floor (wi::to_wide (val1),
-			   wi::to_wide (val2), sign, &overflow);
+	return false;
+      *res = wi::div_floor (wi::to_wide (val1),
+			    wi::to_wide (val2), sign, &overflow);
       break;
 
     case CEIL_DIV_EXPR:
       if (val2 == 0)
-	{
-	  *overflow_p = true;
-	  return res;
-	}
-      res = wi::div_ceil (wi::to_wide (val1),
-			  wi::to_wide (val2), sign, &overflow);
+	return false;
+      *res = wi::div_ceil (wi::to_wide (val1),
+			   wi::to_wide (val2), sign, &overflow);
       break;
 
     case ROUND_DIV_EXPR:
       if (val2 == 0)
-	{
-	  *overflow_p = 0;
-	  return res;
-	}
-      res = wi::div_round (wi::to_wide (val1),
-			   wi::to_wide (val2), sign, &overflow);
+	return false;
+      *res = wi::div_round (wi::to_wide (val1),
+			    wi::to_wide (val2), sign, &overflow);
       break;
 
     default:
@@ -1747,16 +1734,15 @@ vrp_int_const_binop (enum tree_code code, tree val1, tree val2,
 	  || code == CEIL_DIV_EXPR
 	  || code == EXACT_DIV_EXPR
 	  || code == ROUND_DIV_EXPR)
-	return wi::max_value (TYPE_PRECISION (TREE_TYPE (val1)),
+	*res = wi::max_value (TYPE_PRECISION (TREE_TYPE (val1)),
 			      TYPE_SIGN (TREE_TYPE (val1)));
       else
-	return wi::min_value (TYPE_PRECISION (TREE_TYPE (val1)),
+	*res = wi::min_value (TYPE_PRECISION (TREE_TYPE (val1)),
 			      TYPE_SIGN (TREE_TYPE (val1)));
+      return true;
     }
 
-  *overflow_p = overflow;
-
-  return res;
+  return !overflow;
 }
 
 
@@ -1852,7 +1838,6 @@ extract_range_from_multiplicative_op_1 (value_range *vr,
 {
   enum value_range_type rtype;
   wide_int val, min, max;
-  bool sop;
   tree type;
 
   /* Multiplications, divisions and shifts are a bit tricky to handle,
@@ -1883,58 +1868,50 @@ extract_range_from_multiplicative_op_1 (value_range *vr,
   signop sgn = TYPE_SIGN (type);
 
   /* Compute the 4 cross operations and their minimum and maximum value.  */
-  sop = false;
-  val = vrp_int_const_binop (code, vr0->min, vr1->min, &sop);
-  if (! sop)
-    min = max = val;
-
-  if (vr1->max == vr1->min)
-    ;
-  else if (! sop)
+  if (!vrp_int_const_binop (code, vr0->min, vr1->min, &val))
     {
-      val = vrp_int_const_binop (code, vr0->min, vr1->max, &sop);
-      if (! sop)
-	{
-	  if (wi::lt_p (val, min, sgn))
-	    min = val;
-	  else if (wi::gt_p (val, max, sgn))
-	    max = val;
-	}
+      set_value_range_to_varying (vr);
+      return;
     }
+  min = max = val;
 
-  if (vr0->max == vr0->min)
-    ;
-  else if (! sop)
+  if (vr1->max != vr1->min)
     {
-      val = vrp_int_const_binop (code, vr0->max, vr1->min, &sop);
-      if (! sop)
+      if (!vrp_int_const_binop (code, vr0->min, vr1->max, &val))
 	{
-	  if (wi::lt_p (val, min, sgn))
-	    min = val;
-	  else if (wi::gt_p (val, max, sgn))
-	    max = val;
+	  set_value_range_to_varying (vr);
+	  return;
 	}
+      if (wi::lt_p (val, min, sgn))
+	min = val;
+      else if (wi::gt_p (val, max, sgn))
+	max = val;
     }
 
-  if (vr0->min == vr0->max || vr1->min == vr1->max)
-    ;
-  else if (! sop)
+  if (vr0->max != vr0->min)
     {
-      val = vrp_int_const_binop (code, vr0->max, vr1->max, &sop);
-      if (! sop)
+      if (!vrp_int_const_binop (code, vr0->max, vr1->min, &val))
 	{
-	  if (wi::lt_p (val, min, sgn))
-	    min = val;
-	  else if (wi::gt_p (val, max, sgn))
-	    max = val;
+	  set_value_range_to_varying (vr);
+	  return;
 	}
+      if (wi::lt_p (val, min, sgn))
+	min = val;
+      else if (wi::gt_p (val, max, sgn))
+	max = val;
     }
 
-  /* If either operation overflowed, drop to VARYING.  */
-  if (sop)
+  if (vr0->min != vr0->max && vr1->min != vr1->max)
     {
-      set_value_range_to_varying (vr);
-      return;
+      if (!vrp_int_const_binop (code, vr0->max, vr1->max, &val))
+	{
+	  set_value_range_to_varying (vr);
+	  return;
+	}
+      if (wi::lt_p (val, min, sgn))
+	min = val;
+      else if (wi::gt_p (val, max, sgn))
+	max = val;
     }
 
   /* If the new range has its limits swapped around (MIN > MAX),
@@ -2804,7 +2781,7 @@ extract_range_from_binary_expr_1 (value_range *vr,
 	      return;
 	    }
 	}
-      else if (!symbolic_range_p (&vr0) && !symbolic_range_p (&vr1))
+      else if (range_int_cst_p (&vr0) && range_int_cst_p (&vr1))
 	{
 	  extract_range_from_multiplicative_op_1 (vr, code, &vr0, &vr1);
 	  return;
@@ -6856,10 +6833,7 @@ check_array_bounds (tree *tp, int *walk_subtree, void *data)
   if (EXPR_HAS_LOCATION (t))
     location = EXPR_LOCATION (t);
   else
-    {
-      location_t *locp = (location_t *) wi->info;
-      location = *locp;
-    }
+    location = gimple_location (wi->stmt);
 
   *walk_subtree = TRUE;
 
@@ -6906,9 +6880,6 @@ check_all_array_refs (void)
 
 	  memset (&wi, 0, sizeof (wi));
 
-	  location_t loc = gimple_location (stmt);
-	  wi.info = &loc;
-
 	  walk_gimple_op (gsi_stmt (si),
 			  check_array_bounds,
 			  &wi);
@@ -8111,6 +8082,13 @@ extract_range_from_stmt (gimple *stmt, edge *taken_edge_p,
     vrp_visit_switch_stmt (as_a <gswitch *> (stmt), taken_edge_p);
 }
 
+class vrp_prop : public ssa_propagation_engine
+{
+ public:
+  enum ssa_prop_result visit_stmt (gimple *, edge *, tree *) FINAL OVERRIDE;
+  enum ssa_prop_result visit_phi (gphi *) FINAL OVERRIDE;
+};
+
 /* Evaluate statement STMT.  If the statement produces a useful range,
    return SSA_PROP_INTERESTING and record the SSA name with the
    interesting range into *OUTPUT_P.
@@ -8120,8 +8098,8 @@ extract_range_from_stmt (gimple *stmt, edge *taken_edge_p,
 
    If STMT produces a varying value, return SSA_PROP_VARYING.  */
 
-static enum ssa_prop_result
-vrp_visit_stmt (gimple *stmt, edge *taken_edge_p, tree *output_p)
+enum ssa_prop_result
+vrp_prop::visit_stmt (gimple *stmt, edge *taken_edge_p, tree *output_p)
 {
   value_range vr = VR_INITIALIZER;
   tree lhs = gimple_get_lhs (stmt);
@@ -9212,8 +9190,8 @@ update_range:
    edges.  If a valid value range can be derived from all the incoming
    value ranges, set a new range for the LHS of PHI.  */
 
-static enum ssa_prop_result
-vrp_visit_phi_node (gphi *phi)
+enum ssa_prop_result
+vrp_prop::visit_phi (gphi *phi)
 {
   tree lhs = PHI_RESULT (phi);
   value_range vr_result = VR_INITIALIZER;
@@ -10548,10 +10526,17 @@ fold_predicate_in (gimple_stmt_iterator *si)
   return false;
 }
 
+class vrp_folder : public substitute_and_fold_engine
+{
+ public:
+  tree get_value (tree) FINAL OVERRIDE;
+  bool fold_stmt (gimple_stmt_iterator *) FINAL OVERRIDE;
+};
+
 /* Callback for substitute_and_fold folding the stmt at *SI.  */
 
-static bool
-vrp_fold_stmt (gimple_stmt_iterator *si)
+bool
+vrp_folder::fold_stmt (gimple_stmt_iterator *si)
 {
   if (fold_predicate_in (si))
     return true;
@@ -10559,6 +10544,18 @@ vrp_fold_stmt (gimple_stmt_iterator *si)
   return simplify_stmt_using_ranges (si);
 }
 
+/* If OP has a value range with a single constant value return that,
+   otherwise return NULL_TREE.  This returns OP itself if OP is a
+   constant.
+
+   Implemented as a pure wrapper right now, but this will change.  */
+
+tree
+vrp_folder::get_value (tree op)
+{
+  return op_with_constant_singleton_value_range (op);
+}
+
 /* Return the LHS of any ASSERT_EXPR where OP appears as the first
    argument to the ASSERT_EXPR and in which the ASSERT_EXPR dominates
    BB.  If no such ASSERT_EXPR is found, return OP.  */
@@ -10900,7 +10897,8 @@ vrp_finalize (bool warn_array_bounds_p)
 			  wi::to_wide (vr_value[i]->max));
       }
 
-  substitute_and_fold (op_with_constant_singleton_value_range, vrp_fold_stmt);
+  class vrp_folder vrp_folder;
+  vrp_folder.substitute_and_fold ();
 
   if (warn_array_bounds && warn_array_bounds_p)
     check_all_array_refs ();
@@ -10968,33 +10966,17 @@ evrp_dom_walker::try_find_new_range (tree name,
 edge
 evrp_dom_walker::before_dom_children (basic_block bb)
 {
-  tree op0 = NULL_TREE;
-  edge_iterator ei;
-  edge e;
-
   if (dump_file && (dump_flags & TDF_DETAILS))
     fprintf (dump_file, "Visiting BB%d\n", bb->index);
 
   stack.safe_push (std::make_pair (NULL_TREE, (value_range *)NULL));
 
-  edge pred_e = NULL;
-  FOR_EACH_EDGE (e, ei, bb->preds)
-    {
-      /* Ignore simple backedges from this to allow recording conditions
-	 in loop headers.  */
-      if (dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
-	continue;
-      if (! pred_e)
-	pred_e = e;
-      else
-	{
-	  pred_e = NULL;
-	  break;
-	}
-    }
+  edge pred_e = single_pred_edge_ignoring_loop_edges (bb, false);
   if (pred_e)
     {
       gimple *stmt = last_stmt (pred_e->src);
+      tree op0 = NULL_TREE;
+
       if (stmt
 	  && gimple_code (stmt) == GIMPLE_COND
 	  && (op0 = gimple_cond_lhs (stmt))
@@ -11038,6 +11020,8 @@ evrp_dom_walker::before_dom_children (basic_block bb)
 
   /* Visit PHI stmts and discover any new VRs possible.  */
   bool has_unvisited_preds = false;
+  edge_iterator ei;
+  edge e;
   FOR_EACH_EDGE (e, ei, bb->preds)
     if (e->flags & EDGE_EXECUTABLE
 	&& !(e->src->flags & BB_VISITED))
@@ -11237,8 +11221,8 @@ evrp_dom_walker::before_dom_children (basic_block bb)
 	}
 
       /* Try folding stmts with the VR discovered.  */
-      bool did_replace
-	= replace_uses_in (stmt, op_with_constant_singleton_value_range);
+      class vrp_folder vrp_folder;
+      bool did_replace = vrp_folder.replace_uses_in (stmt);
       if (fold_stmt (&gsi, follow_single_use_edges)
 	  || did_replace)
 	{
@@ -11488,7 +11472,8 @@ execute_vrp (bool warn_array_bounds_p)
 
   vrp_initialize_lattice ();
   vrp_initialize ();
-  ssa_propagate (vrp_visit_stmt, vrp_visit_phi_node);
+  class vrp_prop vrp_prop;
+  vrp_prop.ssa_propagate ();
   vrp_finalize (warn_array_bounds_p);
 
   /* We must identify jump threading opportunities before we release
diff --git a/gcc/tree.c b/gcc/tree.c
index ee12d3f3c4f..e5ee29e49ce 100644
--- a/gcc/tree.c
+++ b/gcc/tree.c
@@ -270,7 +270,7 @@ tree integer_types[itk_none];
 bool int_n_enabled_p[NUM_INT_N_ENTS];
 struct int_n_trees_t int_n_trees [NUM_INT_N_ENTS];
 
-unsigned char tree_contains_struct[MAX_TREE_CODES][64];
+bool tree_contains_struct[MAX_TREE_CODES][64];
 
 /* Number of operands for each OpenMP clause.  */
 unsigned const char omp_clause_num_ops[] =
@@ -780,40 +780,53 @@ tree_code_size (enum tree_code code)
   switch (TREE_CODE_CLASS (code))
     {
     case tcc_declaration:  /* A decl node */
-      {
-	switch (code)
-	  {
-	  case FIELD_DECL:
-	    return sizeof (struct tree_field_decl);
-	  case PARM_DECL:
-	    return sizeof (struct tree_parm_decl);
-	  case VAR_DECL:
-	    return sizeof (struct tree_var_decl);
-	  case LABEL_DECL:
-	    return sizeof (struct tree_label_decl);
-	  case RESULT_DECL:
-	    return sizeof (struct tree_result_decl);
-	  case CONST_DECL:
-	    return sizeof (struct tree_const_decl);
-	  case TYPE_DECL:
-	    return sizeof (struct tree_type_decl);
-	  case FUNCTION_DECL:
-	    return sizeof (struct tree_function_decl);
-	  case DEBUG_EXPR_DECL:
-	    return sizeof (struct tree_decl_with_rtl);
-	  case TRANSLATION_UNIT_DECL:
-	    return sizeof (struct tree_translation_unit_decl);
-	  case NAMESPACE_DECL:
-	  case IMPORTED_DECL:
-	  case NAMELIST_DECL:
-	    return sizeof (struct tree_decl_non_common);
-	  default:
-	    return lang_hooks.tree_size (code);
-	  }
-      }
+      switch (code)
+	{
+	case FIELD_DECL:	return sizeof (tree_field_decl);
+	case PARM_DECL:		return sizeof (tree_parm_decl);
+	case VAR_DECL:		return sizeof (tree_var_decl);
+	case LABEL_DECL:	return sizeof (tree_label_decl);
+	case RESULT_DECL:	return sizeof (tree_result_decl);
+	case CONST_DECL:	return sizeof (tree_const_decl);
+	case TYPE_DECL:		return sizeof (tree_type_decl);
+	case FUNCTION_DECL:	return sizeof (tree_function_decl);
+	case DEBUG_EXPR_DECL:	return sizeof (tree_decl_with_rtl);
+	case TRANSLATION_UNIT_DECL: return sizeof (tree_translation_unit_decl);
+	case NAMESPACE_DECL:
+	case IMPORTED_DECL:
+	case NAMELIST_DECL:	return sizeof (tree_decl_non_common);
+	default:
+	  gcc_checking_assert (code >= NUM_TREE_CODES);
+	  return lang_hooks.tree_size (code);
+	}
 
     case tcc_type:  /* a type node */
-      return sizeof (struct tree_type_non_common);
+      switch (code)
+	{
+	case OFFSET_TYPE:
+	case ENUMERAL_TYPE:
+	case BOOLEAN_TYPE:
+	case INTEGER_TYPE:
+	case REAL_TYPE:
+	case POINTER_TYPE:
+	case REFERENCE_TYPE:
+	case NULLPTR_TYPE:
+	case FIXED_POINT_TYPE:
+	case COMPLEX_TYPE:
+	case VECTOR_TYPE:
+	case ARRAY_TYPE:
+	case RECORD_TYPE:
+	case UNION_TYPE:
+	case QUAL_UNION_TYPE:
+	case VOID_TYPE:
+	case POINTER_BOUNDS_TYPE:
+	case FUNCTION_TYPE:
+	case METHOD_TYPE:
+	case LANG_TYPE:		return sizeof (tree_type_non_common);
+	default:
+	  gcc_checking_assert (code >= NUM_TREE_CODES);
+	  return lang_hooks.tree_size (code);
+	}
 
     case tcc_reference:   /* a reference */
     case tcc_expression:  /* an expression */
@@ -827,18 +840,18 @@ tree_code_size (enum tree_code code)
     case tcc_constant:  /* a constant */
       switch (code)
 	{
-	case VOID_CST:		return sizeof (struct tree_typed);
+	case VOID_CST:		return sizeof (tree_typed);
 	case INTEGER_CST:	gcc_unreachable ();
-	case POLY_INT_CST:	return sizeof (struct tree_poly_int_cst);
-	case REAL_CST:		return sizeof (struct tree_real_cst);
-	case FIXED_CST:		return sizeof (struct tree_fixed_cst);
-	case COMPLEX_CST:	return sizeof (struct tree_complex);
-	case VECTOR_CST:	return sizeof (struct tree_vector);
-	case VEC_DUPLICATE_CST:	return sizeof (struct tree_vector);
-	case VEC_SERIES_CST:
-	  return sizeof (struct tree_vector) + sizeof (tree);
+	case POLY_INT_CST:	return sizeof (tree_poly_int_cst);
+	case REAL_CST:		return sizeof (tree_real_cst);
+	case FIXED_CST:		return sizeof (tree_fixed_cst);
+	case COMPLEX_CST:	return sizeof (tree_complex);
+	case VECTOR_CST:	return sizeof (tree_vector);
+	case VEC_DUPLICATE_CST:	return sizeof (tree_vector);
+	case VEC_SERIES_CST:	return sizeof (tree_vector) + sizeof (tree);
 	case STRING_CST:	gcc_unreachable ();
 	default:
+	  gcc_checking_assert (code >= NUM_TREE_CODES);
 	  return lang_hooks.tree_size (code);
 	}
 
@@ -846,23 +859,24 @@ tree_code_size (enum tree_code code)
       switch (code)
 	{
 	case IDENTIFIER_NODE:	return lang_hooks.identifier_size;
-	case TREE_LIST:		return sizeof (struct tree_list);
+	case TREE_LIST:		return sizeof (tree_list);
 
 	case ERROR_MARK:
-	case PLACEHOLDER_EXPR:	return sizeof (struct tree_common);
+	case PLACEHOLDER_EXPR:	return sizeof (tree_common);
 
-	case TREE_VEC:
+	case TREE_VEC:		gcc_unreachable ();
 	case OMP_CLAUSE:	gcc_unreachable ();
 
-	case SSA_NAME:		return sizeof (struct tree_ssa_name);
+	case SSA_NAME:		return sizeof (tree_ssa_name);
 
-	case STATEMENT_LIST:	return sizeof (struct tree_statement_list);
+	case STATEMENT_LIST:	return sizeof (tree_statement_list);
 	case BLOCK:		return sizeof (struct tree_block);
-	case CONSTRUCTOR:	return sizeof (struct tree_constructor);
-	case OPTIMIZATION_NODE: return sizeof (struct tree_optimization_option);
-	case TARGET_OPTION_NODE: return sizeof (struct tree_target_option);
+	case CONSTRUCTOR:	return sizeof (tree_constructor);
+	case OPTIMIZATION_NODE: return sizeof (tree_optimization_option);
+	case TARGET_OPTION_NODE: return sizeof (tree_target_option);
 
 	default:
+	  gcc_checking_assert (code >= NUM_TREE_CODES);
 	  return lang_hooks.tree_size (code);
 	}
 
@@ -1218,8 +1232,8 @@ copy_node (tree node MEM_STAT_DECL)
 	 The two statements usually duplicate each other
 	 (because they clear fields of the same union),
 	 but the optimizer should catch that.  */
-      TYPE_SYMTAB_POINTER (t) = 0;
       TYPE_SYMTAB_ADDRESS (t) = 0;
+      TYPE_SYMTAB_DIE (t) = 0;
 
       /* Do not copy the values cache.  */
       if (TYPE_CACHED_VALUES_P (t))
@@ -1804,13 +1818,16 @@ cst_and_fits_in_hwi (const_tree x)
 
 /* Build a new VEC_DUPLICATE_CST with type TYPE and operand EXP.
 
-   Note that this function is only suitable for callers that specifically
-   need a VEC_DUPLICATE_CST node.  Use build_vector_from_val to duplicate
-   a general scalar into a general vector type.  */
+   This function is only suitable for callers that know TYPE is a
+   variable-length vector and specifically need a VEC_DUPLICATE_CST node.
+   Use build_vector_from_val to duplicate a general scalar into a general
+   vector type.  */
 
-tree
+static tree
 build_vec_duplicate_cst (tree type, tree exp MEM_STAT_DECL)
 {
+  gcc_assert (!TYPE_VECTOR_SUBPARTS (type).is_constant ());
+
   int length = sizeof (struct tree_vector);
 
   record_node_allocation_statistics (VEC_DUPLICATE_CST, length);
@@ -1832,9 +1849,11 @@ build_vec_duplicate_cst (tree type, tree exp MEM_STAT_DECL)
    need a VEC_SERIES_CST node.  Use build_vec_series to build a general
    series vector from a general base and step.  */
 
-tree
+static tree
 build_vec_series_cst (tree type, tree base, tree step MEM_STAT_DECL)
 {
+  gcc_assert (!TYPE_VECTOR_SUBPARTS (type).is_constant ());
+
   int length = sizeof (struct tree_vector) + sizeof (tree);
 
   record_node_allocation_statistics (VEC_SERIES_CST, length);
@@ -1970,7 +1989,8 @@ build_vector_from_val (tree vectype, tree sc)
 }
 
 /* Build a vector series of type TYPE in which element I has the value
-   BASE + I * STEP.  */
+   BASE + I * STEP.  The result is a constant if BASE and STEP are constant
+   and a VEC_SERIES_EXPR otherwise.  */
 
 tree
 build_vec_series (tree type, tree base, tree step)
@@ -1978,7 +1998,20 @@ build_vec_series (tree type, tree base, tree step)
   if (integer_zerop (step))
     return build_vector_from_val (type, base);
   if (CONSTANT_CLASS_P (base) && CONSTANT_CLASS_P (step))
-    return build_vec_series_cst (type, base, step);
+    {
+      unsigned HOST_WIDE_INT nunits;
+      if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nunits))
+	return build_vec_series_cst (type, base, step);
+
+      auto_vec<tree, 32> v (nunits);
+      v.quick_push (base);
+      for (unsigned int i = 1; i < nunits; ++i)
+	{
+	  base = const_binop (PLUS_EXPR, TREE_TYPE (base), base, step);
+	  v.quick_push (base);
+	}
+      return build_vector (type, v);
+    }
   return build2 (VEC_SERIES_EXPR, type, base, step);
 }
 
@@ -10309,6 +10342,13 @@ build_common_builtin_nodes (void)
 			"__builtin_alloca_with_align",
 			alloca_flags);
 
+  ftype = build_function_type_list (ptr_type_node, size_type_node,
+				    size_type_node, size_type_node, NULL_TREE);
+  local_define_builtin ("__builtin_alloca_with_align_and_max", ftype,
+			BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX,
+			"__builtin_alloca_with_align_and_max",
+			alloca_flags);
+
   ftype = build_function_type_list (void_type_node,
 				    ptr_type_node, ptr_type_node,
 				    ptr_type_node, NULL_TREE);
@@ -10623,7 +10663,7 @@ build_same_sized_truth_vector_type (tree vectype)
 
   poly_uint64 size = GET_MODE_SIZE (TYPE_MODE (vectype));
 
-  if (known_zero (size))
+  if (must_eq (size, 0U))
     size = tree_to_uhwi (TYPE_SIZE_UNIT (vectype));
 
   return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype), size);
@@ -11060,6 +11100,33 @@ maybe_build_call_expr_loc (location_t loc, combined_fn fn, tree type,
     }
 }
 
+/* Return a function call to the appropriate builtin alloca variant.
+
+   SIZE is the size to be allocated.  ALIGN, if non-zero, is the requested
+   alignment of the allocated area.  MAX_SIZE, if non-negative, is an upper
+   bound for SIZE in case it is not a fixed value.  */
+
+tree
+build_alloca_call_expr (tree size, unsigned int align, HOST_WIDE_INT max_size)
+{
+  if (max_size >= 0)
+    {
+      tree t = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX);
+      return
+	build_call_expr (t, 3, size, size_int (align), size_int (max_size));
+    }
+  else if (align > 0)
+    {
+      tree t = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
+      return build_call_expr (t, 2, size, size_int (align));
+    }
+  else
+    {
+      tree t = builtin_decl_explicit (BUILT_IN_ALLOCA);
+      return build_call_expr (t, 1, size);
+    }
+}
+
 /* Create a new constant string literal and return a char* pointer to it.
    The STRING_CST value is the LEN characters at STR.  */
 tree
@@ -12394,7 +12461,7 @@ get_binfo_at_offset (tree binfo, poly_int64 offset, tree expected_type)
 
       /* Offset 0 indicates the primary base, whose vtable contents are
 	 represented in the binfo for the derived class.  */
-      else if (maybe_nonzero (offset))
+      else if (may_ne (offset, 0))
 	{
 	  tree found_binfo = NULL, base_binfo;
 	  /* Offsets in BINFO are in bytes relative to the whole structure
@@ -12876,6 +12943,9 @@ array_at_struct_end_p (tree ref)
   else
     return false;
 
+  if (TREE_CODE (ref) == STRING_CST)
+    return false;
+
   while (handled_component_p (ref))
     {
       /* If the reference chain contains a component reference to a
@@ -14180,10 +14250,15 @@ test_integer_constants ()
 static void
 test_vec_duplicate_predicates_int (tree type)
 {
-  tree vec_type = build_vector_type (type, 4);
+  scalar_int_mode int_mode = SCALAR_INT_TYPE_MODE (type);
+  machine_mode vec_mode = targetm.vectorize.preferred_simd_mode (int_mode);
+  /* This will be 1 if VEC_MODE isn't a vector mode.  */
+  poly_uint64 nunits = GET_MODE_NUNITS (vec_mode);
+
+  tree vec_type = build_vector_type (type, nunits);
 
   tree zero = build_zero_cst (type);
-  tree vec_zero = build_vec_duplicate_cst (vec_type, zero);
+  tree vec_zero = build_vector_from_val (vec_type, zero);
   ASSERT_TRUE (integer_zerop (vec_zero));
   ASSERT_FALSE (integer_onep (vec_zero));
   ASSERT_FALSE (integer_minus_onep (vec_zero));
@@ -14192,7 +14267,7 @@ test_vec_duplicate_predicates_int (tree type)
   ASSERT_TRUE (initializer_zerop (vec_zero));
 
   tree one = build_one_cst (type);
-  tree vec_one = build_vec_duplicate_cst (vec_type, one);
+  tree vec_one = build_vector_from_val (vec_type, one);
   ASSERT_FALSE (integer_zerop (vec_one));
   ASSERT_TRUE (integer_onep (vec_one));
   ASSERT_FALSE (integer_minus_onep (vec_one));
@@ -14201,7 +14276,7 @@ test_vec_duplicate_predicates_int (tree type)
   ASSERT_FALSE (initializer_zerop (vec_one));
 
   tree minus_one = build_minus_one_cst (type);
-  tree vec_minus_one = build_vec_duplicate_cst (vec_type, minus_one);
+  tree vec_minus_one = build_vector_from_val (vec_type, minus_one);
   ASSERT_FALSE (integer_zerop (vec_minus_one));
   ASSERT_FALSE (integer_onep (vec_minus_one));
   ASSERT_TRUE (integer_minus_onep (vec_minus_one));
@@ -14223,24 +14298,29 @@ test_vec_duplicate_predicates_int (tree type)
 static void
 test_vec_duplicate_predicates_float (tree type)
 {
-  tree vec_type = build_vector_type (type, 4);
+  scalar_float_mode float_mode = SCALAR_FLOAT_TYPE_MODE (type);
+  machine_mode vec_mode = targetm.vectorize.preferred_simd_mode (float_mode);
+  /* This will be 1 if VEC_MODE isn't a vector mode.  */
+  poly_uint64 nunits = GET_MODE_NUNITS (vec_mode);
+
+  tree vec_type = build_vector_type (type, nunits);
 
   tree zero = build_zero_cst (type);
-  tree vec_zero = build_vec_duplicate_cst (vec_type, zero);
+  tree vec_zero = build_vector_from_val (vec_type, zero);
   ASSERT_TRUE (real_zerop (vec_zero));
   ASSERT_FALSE (real_onep (vec_zero));
   ASSERT_FALSE (real_minus_onep (vec_zero));
   ASSERT_TRUE (initializer_zerop (vec_zero));
 
   tree one = build_one_cst (type);
-  tree vec_one = build_vec_duplicate_cst (vec_type, one);
+  tree vec_one = build_vector_from_val (vec_type, one);
   ASSERT_FALSE (real_zerop (vec_one));
   ASSERT_TRUE (real_onep (vec_one));
   ASSERT_FALSE (real_minus_onep (vec_one));
   ASSERT_FALSE (initializer_zerop (vec_one));
 
   tree minus_one = build_minus_one_cst (type);
-  tree vec_minus_one = build_vec_duplicate_cst (vec_type, minus_one);
+  tree vec_minus_one = build_vector_from_val (vec_type, minus_one);
   ASSERT_FALSE (real_zerop (vec_minus_one));
   ASSERT_FALSE (real_onep (vec_minus_one));
   ASSERT_TRUE (real_minus_onep (vec_minus_one));
diff --git a/gcc/tree.def b/gcc/tree.def
index 608d950b20e..051ecd4897e 100644
--- a/gcc/tree.def
+++ b/gcc/tree.def
@@ -308,11 +308,14 @@ DEFTREECODE (COMPLEX_CST, "complex_cst", tcc_constant, 0)
 DEFTREECODE (VECTOR_CST, "vector_cst", tcc_constant, 0)
 
 /* Represents a vector constant in which every element is equal to
-   VEC_DUPLICATE_CST_ELT.  */
+   VEC_DUPLICATE_CST_ELT.  This is only ever used for variable-length
+   vectors; fixed-length vectors must use VECTOR_CST instead.  */
 DEFTREECODE (VEC_DUPLICATE_CST, "vec_duplicate_cst", tcc_constant, 0)
 
 /* Represents a vector constant in which element i is equal to
-   VEC_SERIES_CST_BASE + i * VEC_SERIES_CST_STEP.  */
+   VEC_SERIES_CST_BASE + i * VEC_SERIES_CST_STEP.  This is only ever
+   used for variable-length vectors; fixed-length vectors must use
+   VECTOR_CST instead.  */
 DEFTREECODE (VEC_SERIES_CST, "vec_series_cst", tcc_constant, 0)
 
 /* Contents are TREE_STRING_LENGTH and the actual contents of the string.  */
diff --git a/gcc/tree.h b/gcc/tree.h
index 1b05d969541..a73928fa3ee 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -76,64 +76,43 @@ as_internal_fn (combined_fn code)
 
 /* Macros for initializing `tree_contains_struct'.  */
 #define MARK_TS_BASE(C)					\
-  do {							\
-    tree_contains_struct[C][TS_BASE] = 1;		\
-  } while (0)
+  (tree_contains_struct[C][TS_BASE] = true)
 
 #define MARK_TS_TYPED(C)				\
-  do {							\
-    MARK_TS_BASE (C);					\
-    tree_contains_struct[C][TS_TYPED] = 1;		\
-  } while (0)
+  (MARK_TS_BASE (C),					\
+   tree_contains_struct[C][TS_TYPED] = true)
 
 #define MARK_TS_COMMON(C)				\
-  do {							\
-    MARK_TS_TYPED (C);					\
-    tree_contains_struct[C][TS_COMMON] = 1;		\
-  } while (0)
+  (MARK_TS_TYPED (C),					\
+   tree_contains_struct[C][TS_COMMON] = true)
 
 #define MARK_TS_TYPE_COMMON(C)				\
-  do {							\
-    MARK_TS_COMMON (C);					\
-    tree_contains_struct[C][TS_TYPE_COMMON] = 1;	\
-  } while (0)
+  (MARK_TS_COMMON (C),					\
+   tree_contains_struct[C][TS_TYPE_COMMON] = true)
 
 #define MARK_TS_TYPE_WITH_LANG_SPECIFIC(C)		\
-  do {							\
-    MARK_TS_TYPE_COMMON (C);				\
-    tree_contains_struct[C][TS_TYPE_WITH_LANG_SPECIFIC] = 1;	\
-  } while (0)
+  (MARK_TS_TYPE_COMMON (C),				\
+   tree_contains_struct[C][TS_TYPE_WITH_LANG_SPECIFIC] = true)
 
 #define MARK_TS_DECL_MINIMAL(C)				\
-  do {							\
-    MARK_TS_COMMON (C);					\
-    tree_contains_struct[C][TS_DECL_MINIMAL] = 1;	\
-  } while (0)
+  (MARK_TS_COMMON (C),					\
+   tree_contains_struct[C][TS_DECL_MINIMAL] = true)
 
 #define MARK_TS_DECL_COMMON(C)				\
-  do {							\
-    MARK_TS_DECL_MINIMAL (C);				\
-    tree_contains_struct[C][TS_DECL_COMMON] = 1;	\
-  } while (0)
+  (MARK_TS_DECL_MINIMAL (C),				\
+   tree_contains_struct[C][TS_DECL_COMMON] = true)
 
 #define MARK_TS_DECL_WRTL(C)				\
-  do {							\
-    MARK_TS_DECL_COMMON (C);				\
-    tree_contains_struct[C][TS_DECL_WRTL] = 1;		\
-  } while (0)
+  (MARK_TS_DECL_COMMON (C),				\
+   tree_contains_struct[C][TS_DECL_WRTL] = true)
 
 #define MARK_TS_DECL_WITH_VIS(C)			\
-  do {							\
-    MARK_TS_DECL_WRTL (C);				\
-    tree_contains_struct[C][TS_DECL_WITH_VIS] = 1;	\
-  } while (0)
+  (MARK_TS_DECL_WRTL (C),				\
+   tree_contains_struct[C][TS_DECL_WITH_VIS] = true)
 
 #define MARK_TS_DECL_NON_COMMON(C)			\
-  do {							\
-    MARK_TS_DECL_WITH_VIS (C);				\
-    tree_contains_struct[C][TS_DECL_NON_COMMON] = 1;	\
-  } while (0)
-
+  (MARK_TS_DECL_WITH_VIS (C),				\
+   tree_contains_struct[C][TS_DECL_NON_COMMON] = true)
 
 /* Returns the string representing CLASS.  */
 
@@ -2103,11 +2082,6 @@ extern machine_mode vector_type_mode (const_tree);
 #define TYPE_SYMTAB_ADDRESS(NODE) \
   (TYPE_CHECK (NODE)->type_common.symtab.address)
 
-/* Symtab field as a string.  Used by COFF generator in sdbout.c to
-   hold struct/union type tag names.  */
-#define TYPE_SYMTAB_POINTER(NODE) \
-  (TYPE_CHECK (NODE)->type_common.symtab.pointer)
-
 /* Symtab field as a pointer to a DWARF DIE.  Used by DWARF generator
    in dwarf2out.c to point to the DIE generated for the type.  */
 #define TYPE_SYMTAB_DIE(NODE) \
@@ -2118,8 +2092,7 @@ extern machine_mode vector_type_mode (const_tree);
    union.  */
 
 #define TYPE_SYMTAB_IS_ADDRESS (0)
-#define TYPE_SYMTAB_IS_POINTER (1)
-#define TYPE_SYMTAB_IS_DIE (2)
+#define TYPE_SYMTAB_IS_DIE (1)
 
 #define TYPE_LANG_SPECIFIC(NODE) \
   (TYPE_CHECK (NODE)->type_with_lang_specific.lang_specific)
@@ -2427,6 +2400,18 @@ extern machine_mode vector_type_mode (const_tree);
 #define DECL_FUNCTION_CODE(NODE) \
   (FUNCTION_DECL_CHECK (NODE)->function_decl.function_code)
 
+/* Test if FCODE is a function code for an alloca operation.  */
+#define ALLOCA_FUNCTION_CODE_P(FCODE)				\
+  ((FCODE) == BUILT_IN_ALLOCA					\
+   || (FCODE) == BUILT_IN_ALLOCA_WITH_ALIGN			\
+   || (FCODE) == BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX)
+
+/* Generate case for an alloca operation.  */
+#define CASE_BUILT_IN_ALLOCA			\
+  case BUILT_IN_ALLOCA:				\
+  case BUILT_IN_ALLOCA_WITH_ALIGN:		\
+  case BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX
+
 #define DECL_FUNCTION_PERSONALITY(NODE) \
   (FUNCTION_DECL_CHECK (NODE)->function_decl.personality)
 
@@ -4093,8 +4078,6 @@ extern tree build_int_cst (tree, poly_int64);
 extern tree build_int_cstu (tree type, poly_uint64);
 extern tree build_int_cst_type (tree, poly_int64);
 extern tree make_vector (unsigned CXX_MEM_STAT_INFO);
-extern tree build_vec_duplicate_cst (tree, tree CXX_MEM_STAT_INFO);
-extern tree build_vec_series_cst (tree, tree, tree CXX_MEM_STAT_INFO);
 extern tree build_vector (tree, vec<tree> CXX_MEM_STAT_INFO);
 extern tree build_vector_from_ctor (tree, vec<constructor_elt, va_gc> *);
 extern tree build_vector_from_val (tree, tree);
@@ -4144,6 +4127,7 @@ extern tree build_call_expr_internal_loc_array (location_t, enum internal_fn,
 						tree, int, const tree *);
 extern tree maybe_build_call_expr_loc (location_t, combined_fn, tree,
 				       int, ...);
+extern tree build_alloca_call_expr (tree, unsigned int, HOST_WIDE_INT);
 extern tree build_string_literal (int, const char *);
 
 /* Construct various nodes representing data types.  */
@@ -5601,8 +5585,9 @@ template <typename T>
 bool
 wi::fits_to_boolean_p (const T &x, const_tree type)
 {
-  return (known_zero (x)
-	  || (TYPE_UNSIGNED (type) ? known_one (x) : known_all_ones (x)));
+  typedef typename poly_int_traits<T>::int_type int_type;
+  return (must_eq (x, int_type (0))
+	  || must_eq (x, int_type (TYPE_UNSIGNED (type) ? 1 : -1)));
 }
 
 template <typename T>
diff --git a/gcc/ubsan.c b/gcc/ubsan.c
index cfa08c0e6b6..3a0584271a3 100644
--- a/gcc/ubsan.c
+++ b/gcc/ubsan.c
@@ -804,6 +804,7 @@ ubsan_expand_null_ifn (gimple_stmt_iterator *gsip)
      this edge is unlikely taken, so set up the probability accordingly.  */
   e = make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
   e->probability = profile_probability::very_unlikely ();
+  then_bb->count = e->count ();
 
   /* Connect 'then block' with the 'else block'.  This is needed
      as the ubsan routines we call in the 'then block' are not noreturn.
@@ -813,7 +814,6 @@ ubsan_expand_null_ifn (gimple_stmt_iterator *gsip)
   /* Set up the fallthrough basic block.  */
   e = find_edge (cond_bb, fallthru_bb);
   e->flags = EDGE_FALSE_VALUE;
-  e->count = cond_bb->count;
   e->probability = profile_probability::very_likely ();
 
   /* Update dominance info for the newly created then_bb; note that
@@ -830,15 +830,17 @@ ubsan_expand_null_ifn (gimple_stmt_iterator *gsip)
       enum built_in_function bcode
 	= (flag_sanitize_recover & ((check_align ? SANITIZE_ALIGNMENT : 0)
 				    | (check_null ? SANITIZE_NULL : 0)))
-	  ? BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH
-	  : BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_ABORT;
+	  ? BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1
+	  : BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1_ABORT;
       tree fn = builtin_decl_implicit (bcode);
+      int align_log = tree_log2 (align);
       tree data
 	= ubsan_create_data ("__ubsan_null_data", 1, &loc,
 			     ubsan_type_descriptor (TREE_TYPE (ckind),
 						    UBSAN_PRINT_POINTER),
 			     NULL_TREE,
-			     align,
+			     build_int_cst (unsigned_char_type_node,
+					    MAX (align_log, 0)),
 			     fold_convert (unsigned_char_type_node, ckind),
 			     NULL_TREE);
       data = build_fold_addr_expr_loc (loc, data);
@@ -882,7 +884,6 @@ ubsan_expand_null_ifn (gimple_stmt_iterator *gsip)
 	  /* Set up the fallthrough basic block.  */
 	  e = find_edge (cond1_bb, cond2_bb);
 	  e->flags = EDGE_FALSE_VALUE;
-	  e->count = cond1_bb->count;
 	  e->probability = profile_probability::very_likely ();
 
 	  /* Update dominance info.  */
@@ -1001,14 +1002,14 @@ ubsan_expand_objsize_ifn (gimple_stmt_iterator *gsi)
 				 ubsan_type_descriptor (TREE_TYPE (ptr),
 							UBSAN_PRINT_POINTER),
 				 NULL_TREE,
-				 build_zero_cst (pointer_sized_int_node),
+				 build_zero_cst (unsigned_char_type_node),
 				 ckind,
 				 NULL_TREE);
 	  data = build_fold_addr_expr_loc (loc, data);
 	  enum built_in_function bcode
 	    = (flag_sanitize_recover & SANITIZE_OBJECT_SIZE)
-	      ? BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH
-	      : BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_ABORT;
+	      ? BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1
+	      : BUILT_IN_UBSAN_HANDLE_TYPE_MISMATCH_V1_ABORT;
 	  tree p = make_ssa_name (pointer_sized_int_node);
 	  g = gimple_build_assign (p, NOP_EXPR, ptr);
 	  gimple_set_location (g, loc);
@@ -1073,7 +1074,6 @@ ubsan_expand_ptr_ifn (gimple_stmt_iterator *gsip)
   e->flags = EDGE_FALSE_VALUE;
   if (pos_neg != 3)
     {
-      e->count = cond_bb->count;
       e->probability = profile_probability::very_likely ();
 
       /* Connect 'then block' with the 'else block'.  This is needed
@@ -1086,35 +1086,33 @@ ubsan_expand_ptr_ifn (gimple_stmt_iterator *gsip)
 	 accordingly.  */
       e = make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
       e->probability = profile_probability::very_unlikely ();
+      then_bb->count = e->count ();
     }
   else
     {
-      profile_count count = cond_bb->count.apply_probability (PROB_EVEN);
-      e->count = count;
       e->probability = profile_probability::even ();
 
       e = split_block (fallthru_bb, (gimple *) NULL);
       cond_neg_bb = e->src;
       fallthru_bb = e->dest;
-      e->count = count;
       e->probability = profile_probability::very_likely ();
       e->flags = EDGE_FALSE_VALUE;
 
       e = make_edge (cond_neg_bb, then_bb, EDGE_TRUE_VALUE);
       e->probability = profile_probability::very_unlikely ();
+      then_bb->count = e->count ();
 
       cond_pos_bb = create_empty_bb (cond_bb);
       add_bb_to_loop (cond_pos_bb, cond_bb->loop_father);
 
       e = make_edge (cond_bb, cond_pos_bb, EDGE_TRUE_VALUE);
-      e->count = count;
       e->probability = profile_probability::even ();
+      cond_pos_bb->count = e->count ();
 
       e = make_edge (cond_pos_bb, then_bb, EDGE_TRUE_VALUE);
       e->probability = profile_probability::very_unlikely ();
 
       e = make_edge (cond_pos_bb, fallthru_bb, EDGE_FALSE_VALUE);
-      e->count = count;
       e->probability = profile_probability::very_likely ();
 
       make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU);
@@ -1449,7 +1447,7 @@ maybe_instrument_pointer_overflow (gimple_stmt_iterator *gsi, tree t)
 	 fits, don't instrument anything.  */
       poly_int64 base_size;
       if (offset == NULL_TREE
-	  && maybe_nonzero (bitpos)
+	  && may_ne (bitpos, 0)
 	  && (VAR_P (base)
 	      || TREE_CODE (base) == PARM_DECL
 	      || TREE_CODE (base) == RESULT_DECL)
@@ -1476,7 +1474,7 @@ maybe_instrument_pointer_overflow (gimple_stmt_iterator *gsi, tree t)
   if (!POINTER_TYPE_P (TREE_TYPE (base)) && !DECL_P (base))
     return;
   bytepos = bits_to_bytes_round_down (bitpos);
-  if (offset == NULL_TREE && known_zero (bytepos) && moff == NULL_TREE)
+  if (offset == NULL_TREE && must_eq (bytepos, 0) && moff == NULL_TREE)
     return;
 
   tree base_addr = base;
@@ -1484,7 +1482,7 @@ maybe_instrument_pointer_overflow (gimple_stmt_iterator *gsi, tree t)
     base_addr = build1 (ADDR_EXPR,
 			build_pointer_type (TREE_TYPE (base)), base);
   t = offset;
-  if (maybe_nonzero (bytepos))
+  if (may_ne (bytepos, 0))
     {
       if (t)
 	t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t,
@@ -2025,15 +2023,18 @@ instrument_nonnull_return (gimple_stmt_iterator *gsi)
       else
 	{
 	  tree data = ubsan_create_data ("__ubsan_nonnull_return_data",
-					 2, loc, NULL_TREE, NULL_TREE);
+					 1, &loc[1], NULL_TREE, NULL_TREE);
 	  data = build_fold_addr_expr_loc (loc[0], data);
+	  tree data2 = ubsan_create_data ("__ubsan_nonnull_return_data",
+					  1, &loc[0], NULL_TREE, NULL_TREE);
+	  data2 = build_fold_addr_expr_loc (loc[0], data2);
 	  enum built_in_function bcode
 	    = (flag_sanitize_recover & SANITIZE_RETURNS_NONNULL_ATTRIBUTE)
-	      ? BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN
-	      : BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_ABORT;
+	      ? BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_V1
+	      : BUILT_IN_UBSAN_HANDLE_NONNULL_RETURN_V1_ABORT;
 	  tree fn = builtin_decl_explicit (bcode);
 
-	  g = gimple_build_call (fn, 1, data);
+	  g = gimple_build_call (fn, 2, data, data2);
 	}
       gimple_set_location (g, loc[0]);
       gsi_insert_before (gsi, g, GSI_SAME_STMT);
@@ -2217,6 +2218,72 @@ instrument_object_size (gimple_stmt_iterator *gsi, tree t, bool is_lhs)
   gsi_insert_before (gsi, g, GSI_SAME_STMT);
 }
 
+/* Instrument values passed to builtin functions.  */
+
+static void
+instrument_builtin (gimple_stmt_iterator *gsi)
+{
+  gimple *stmt = gsi_stmt (*gsi);
+  location_t loc = gimple_location (stmt);
+  tree arg;
+  enum built_in_function fcode
+    = DECL_FUNCTION_CODE (gimple_call_fndecl (stmt));
+  int kind = 0;
+  switch (fcode)
+    {
+    CASE_INT_FN (BUILT_IN_CLZ):
+      kind = 1;
+      gcc_fallthrough ();
+    CASE_INT_FN (BUILT_IN_CTZ):
+      arg = gimple_call_arg (stmt, 0);
+      if (!integer_nonzerop (arg))
+	{
+	  gimple *g;
+	  if (!is_gimple_val (arg))
+	    {
+	      g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
+	      gimple_set_location (g, loc);
+	      gsi_insert_before (gsi, g, GSI_SAME_STMT);
+	      arg = gimple_assign_lhs (g);
+	    }
+
+	  basic_block then_bb, fallthru_bb;
+	  *gsi = create_cond_insert_point (gsi, true, false, true,
+					   &then_bb, &fallthru_bb);
+	  g = gimple_build_cond (EQ_EXPR, arg,
+				 build_zero_cst (TREE_TYPE (arg)),
+				 NULL_TREE, NULL_TREE);
+	  gimple_set_location (g, loc);
+	  gsi_insert_after (gsi, g, GSI_NEW_STMT);
+
+	  *gsi = gsi_after_labels (then_bb);
+	  if (flag_sanitize_undefined_trap_on_error)
+	    g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
+	  else
+	    {
+	      tree t = build_int_cst (unsigned_char_type_node, kind);
+	      tree data = ubsan_create_data ("__ubsan_builtin_data",
+					     1, &loc, NULL_TREE, t, NULL_TREE);
+	      data = build_fold_addr_expr_loc (loc, data);
+	      enum built_in_function bcode
+		= (flag_sanitize_recover & SANITIZE_BUILTIN)
+		  ? BUILT_IN_UBSAN_HANDLE_INVALID_BUILTIN
+		  : BUILT_IN_UBSAN_HANDLE_INVALID_BUILTIN_ABORT;
+	      tree fn = builtin_decl_explicit (bcode);
+
+	      g = gimple_build_call (fn, 1, data);
+	    }
+	  gimple_set_location (g, loc);
+	  gsi_insert_before (gsi, g, GSI_SAME_STMT);
+	  ubsan_create_edge (g);
+	}
+      *gsi = gsi_for_stmt (stmt);
+      break;
+    default:
+      break;
+    }
+}
+
 namespace {
 
 const pass_data pass_data_ubsan =
@@ -2248,7 +2315,8 @@ public:
 				| SANITIZE_NONNULL_ATTRIBUTE
 				| SANITIZE_RETURNS_NONNULL_ATTRIBUTE
 				| SANITIZE_OBJECT_SIZE
-				| SANITIZE_POINTER_OVERFLOW));
+				| SANITIZE_POINTER_OVERFLOW
+				| SANITIZE_BUILTIN));
     }
 
   virtual unsigned int execute (function *);
@@ -2313,6 +2381,13 @@ pass_ubsan::execute (function *fun)
 	      bb = gimple_bb (stmt);
 	    }
 
+	  if (sanitize_flags_p (SANITIZE_BUILTIN, fun->decl)
+	      && gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
+	    {
+	      instrument_builtin (&gsi);
+	      bb = gimple_bb (stmt);
+	    }
+
 	  if (sanitize_flags_p (SANITIZE_RETURNS_NONNULL_ATTRIBUTE, fun->decl)
 	      && gimple_code (stmt) == GIMPLE_RETURN)
 	    {
diff --git a/gcc/unique-ptr-tests.cc b/gcc/unique-ptr-tests.cc
new file mode 100644
index 00000000000..d2756947189
--- /dev/null
+++ b/gcc/unique-ptr-tests.cc
@@ -0,0 +1,234 @@
+/* Unit tests for unique-ptr.h.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#define INCLUDE_UNIQUE_PTR
+#include "system.h"
+#include "coretypes.h"
+#include "selftest.h"
+
+#if CHECKING_P
+
+namespace selftest {
+
+namespace {
+
+/* A class for counting ctor and dtor invocations.  */
+
+struct stats
+{
+  stats () : ctor_count (0), dtor_count (0) {}
+
+  int ctor_count;
+  int dtor_count;
+};
+
+/* A class that uses "stats" to track its ctor and dtor invocations.  */
+
+class foo
+{
+public:
+  foo (stats &s) : m_s (s) { ++m_s.ctor_count; }
+  ~foo () { ++m_s.dtor_count; }
+
+  int example_method () const { return 42; }
+
+private:
+  foo (const foo&);
+  foo & operator= (const foo &);
+
+private:
+  stats &m_s;
+};
+
+/* A struct for testing unique_ptr<T[]>.  */
+
+struct has_default_ctor
+{
+  has_default_ctor () : m_field (42) {}
+  int m_field;
+};
+
+/* A dummy struct for testing unique_xmalloc_ptr.  */
+
+struct dummy
+{
+  int field;
+};
+
+} // anonymous namespace
+
+/* Verify that the default ctor inits ptrs to NULL.  */
+
+static void
+test_null_ptr ()
+{
+  gnu::unique_ptr<void *> p;
+  ASSERT_EQ (NULL, p);
+
+  gnu::unique_xmalloc_ptr<void *> q;
+  ASSERT_EQ (NULL, q);
+}
+
+/* Verify that deletion happens when a unique_ptr goes out of scope.  */
+
+static void
+test_implicit_deletion ()
+{
+  stats s;
+  ASSERT_EQ (0, s.ctor_count);
+  ASSERT_EQ (0, s.dtor_count);
+
+  {
+    gnu::unique_ptr<foo> f (new foo (s));
+    ASSERT_NE (NULL, f);
+    ASSERT_EQ (1, s.ctor_count);
+    ASSERT_EQ (0, s.dtor_count);
+  }
+
+  /* Verify that the foo was implicitly deleted.  */
+  ASSERT_EQ (1, s.ctor_count);
+  ASSERT_EQ (1, s.dtor_count);
+}
+
+/* Verify that we can assign to a NULL unique_ptr.  */
+
+static void
+test_overwrite_of_null ()
+{
+  stats s;
+  ASSERT_EQ (0, s.ctor_count);
+  ASSERT_EQ (0, s.dtor_count);
+
+  {
+    gnu::unique_ptr<foo> f;
+    ASSERT_EQ (NULL, f);
+    ASSERT_EQ (0, s.ctor_count);
+    ASSERT_EQ (0, s.dtor_count);
+
+    /* Overwrite with a non-NULL value.  */
+    f = gnu::unique_ptr<foo> (new foo (s));
+    ASSERT_EQ (1, s.ctor_count);
+    ASSERT_EQ (0, s.dtor_count);
+  }
+
+  /* Verify that the foo is implicitly deleted.  */
+  ASSERT_EQ (1, s.ctor_count);
+  ASSERT_EQ (1, s.dtor_count);
+}
+
+/* Verify that we can assign to a non-NULL unique_ptr.  */
+
+static void
+test_overwrite_of_non_null ()
+{
+  stats s;
+  ASSERT_EQ (0, s.ctor_count);
+  ASSERT_EQ (0, s.dtor_count);
+
+  {
+    gnu::unique_ptr<foo> f (new foo (s));
+    ASSERT_NE (NULL, f);
+    ASSERT_EQ (1, s.ctor_count);
+    ASSERT_EQ (0, s.dtor_count);
+
+    /* Overwrite with a different value.  */
+    f = gnu::unique_ptr<foo> (new foo (s));
+    ASSERT_EQ (2, s.ctor_count);
+    ASSERT_EQ (1, s.dtor_count);
+  }
+
+  /* Verify that the 2nd foo was implicitly deleted.  */
+  ASSERT_EQ (2, s.ctor_count);
+  ASSERT_EQ (2, s.dtor_count);
+}
+
+/* Verify that unique_ptr's overloaded ops work.  */
+
+static void
+test_overloaded_ops ()
+{
+  stats s;
+  gnu::unique_ptr<foo> f (new foo (s));
+  ASSERT_EQ (42, f->example_method ());
+  ASSERT_EQ (42, (*f).example_method ());
+  ASSERT_EQ (f, f);
+  ASSERT_NE (NULL, f.get ());
+
+  gnu::unique_ptr<foo> g (new foo (s));
+  ASSERT_NE (f, g);
+}
+
+/* Verify that the gnu::unique_ptr specialization for T[] works.  */
+
+static void
+test_array_new ()
+{
+  const int num = 10;
+  gnu::unique_ptr<has_default_ctor[]> p (new has_default_ctor[num]);
+  ASSERT_NE (NULL, p.get ());
+  /* Verify that operator[] works, and that the default ctor was called
+     on each element.  */
+  for (int i = 0; i < num; i++)
+    ASSERT_EQ (42, p[i].m_field);
+}
+
+/* Verify that gnu::unique_xmalloc_ptr works.  */
+
+static void
+test_xmalloc ()
+{
+  gnu::unique_xmalloc_ptr<dummy> p (XNEW (dummy));
+  ASSERT_NE (NULL, p.get ());
+}
+
+/* Verify the gnu::unique_xmalloc_ptr specialization for T[].  */
+
+static void
+test_xmalloc_array ()
+{
+  const int num = 10;
+  gnu::unique_xmalloc_ptr<dummy[]> p (XNEWVEC (dummy, num));
+  ASSERT_NE (NULL, p.get ());
+
+  /* Verify that operator[] works.  */
+  for (int i = 0; i < num; i++)
+    p[i].field = 42;
+  for (int i = 0; i < num; i++)
+    ASSERT_EQ (42, p[i].field);
+}
+
+/* Run all of the selftests within this file.  */
+
+void
+unique_ptr_tests_cc_tests ()
+{
+  test_null_ptr ();
+  test_implicit_deletion ();
+  test_overwrite_of_null ();
+  test_overwrite_of_non_null ();
+  test_overloaded_ops ();
+  test_array_new ();
+  test_xmalloc ();
+  test_xmalloc_array ();
+}
+
+} // namespace selftest
+
+#endif /* #if CHECKING_P */
diff --git a/gcc/value-prof.c b/gcc/value-prof.c
index 23b8dc26471..85de3189f83 100644
--- a/gcc/value-prof.c
+++ b/gcc/value-prof.c
@@ -583,7 +583,7 @@ static bool
 check_counter (gimple *stmt, const char * name,
 	       gcov_type *count, gcov_type *all, profile_count bb_count_d)
 {
-  gcov_type bb_count = bb_count_d.to_gcov_type ();
+  gcov_type bb_count = bb_count_d.ipa ().to_gcov_type ();
   if (*all != bb_count || *count > *all)
     {
       location_t locus;
@@ -745,20 +745,16 @@ gimple_divmod_fixed_value (gassign *stmt, tree value, profile_probability prob,
   e12->flags &= ~EDGE_FALLTHRU;
   e12->flags |= EDGE_FALSE_VALUE;
   e12->probability = prob;
-  e12->count = profile_count::from_gcov_type (count);
 
   e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
   e13->probability = prob.invert ();
-  e13->count = profile_count::from_gcov_type (all - count);
 
   remove_edge (e23);
 
   e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
   e24->probability = profile_probability::always ();
-  e24->count = profile_count::from_gcov_type (count);
 
   e34->probability = profile_probability::always ();
-  e34->count = profile_count::from_gcov_type (all - count);
 
   return tmp2;
 }
@@ -910,20 +906,16 @@ gimple_mod_pow2 (gassign *stmt, profile_probability prob, gcov_type count, gcov_
   e12->flags &= ~EDGE_FALLTHRU;
   e12->flags |= EDGE_FALSE_VALUE;
   e12->probability = prob;
-  e12->count = profile_count::from_gcov_type (count);
 
   e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
   e13->probability = prob.invert ();
-  e13->count = profile_count::from_gcov_type (all - count);
 
   remove_edge (e23);
 
   e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
   e24->probability = profile_probability::always ();
-  e24->count = profile_count::from_gcov_type (count);
 
   e34->probability = profile_probability::always ();
-  e34->count = profile_count::from_gcov_type (all - count);
 
   return result;
 }
@@ -1076,26 +1068,21 @@ gimple_mod_subtract (gassign *stmt, profile_probability prob1,
   e12->flags &= ~EDGE_FALLTHRU;
   e12->flags |= EDGE_FALSE_VALUE;
   e12->probability = prob1.invert ();
-  e12->count = profile_count::from_gcov_type (all - count1);
 
   e14 = make_edge (bb, bb4, EDGE_TRUE_VALUE);
   e14->probability = prob1;
-  e14->count = profile_count::from_gcov_type (count1);
 
   if (ncounts)  /* Assumed to be 0 or 1.  */
     {
       e23->flags &= ~EDGE_FALLTHRU;
       e23->flags |= EDGE_FALSE_VALUE;
-      e23->count = profile_count::from_gcov_type (all - count1 - count2);
       e23->probability = prob2.invert ();
 
       e24 = make_edge (bb2, bb4, EDGE_TRUE_VALUE);
       e24->probability = prob2;
-      e24->count = profile_count::from_gcov_type (count2);
     }
 
   e34->probability = profile_probability::always ();
-  e34->count = profile_count::from_gcov_type (all - count1 - count2);
 
   return result;
 }
@@ -1312,7 +1299,7 @@ check_ic_target (gcall *call_stmt, struct cgraph_node *target)
 
 gcall *
 gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call,
-	   profile_probability prob, profile_count count, profile_count all)
+	   profile_probability prob)
 {
   gcall *dcall_stmt;
   gassign *load_stmt;
@@ -1367,11 +1354,11 @@ gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call,
   /* Edge e_cd connects cond_bb to dcall_bb, etc; note the first letters. */
   e_cd = split_block (cond_bb, cond_stmt);
   dcall_bb = e_cd->dest;
-  dcall_bb->count = count;
+  dcall_bb->count = cond_bb->count.apply_probability (prob);
 
   e_di = split_block (dcall_bb, dcall_stmt);
   icall_bb = e_di->dest;
-  icall_bb->count = all - count;
+  icall_bb->count = cond_bb->count - dcall_bb->count;
 
   /* Do not disturb existing EH edges from the indirect call.  */
   if (!stmt_ends_bb_p (icall_stmt))
@@ -1383,37 +1370,29 @@ gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call,
       if (e_ij != NULL)
 	{
 	  e_ij->probability = profile_probability::always ();
-	  e_ij->count = all - count;
 	  e_ij = single_pred_edge (split_edge (e_ij));
 	}
     }
   if (e_ij != NULL)
     {
       join_bb = e_ij->dest;
-      join_bb->count = all;
+      join_bb->count = cond_bb->count;
     }
 
   e_cd->flags = (e_cd->flags & ~EDGE_FALLTHRU) | EDGE_TRUE_VALUE;
   e_cd->probability = prob;
-  e_cd->count = count;
 
   e_ci = make_edge (cond_bb, icall_bb, EDGE_FALSE_VALUE);
   e_ci->probability = prob.invert ();
-  e_ci->count = all - count;
 
   remove_edge (e_di);
 
   if (e_ij != NULL)
     {
-      if ((dflags & ECF_NORETURN) != 0)
-	e_ij->count = all;
-      else
+      if ((dflags & ECF_NORETURN) == 0)
 	{
 	  e_dj = make_edge (dcall_bb, join_bb, EDGE_FALLTHRU);
 	  e_dj->probability = profile_probability::always ();
-	  e_dj->count = count;
-
-	  e_ij->count = all - count;
 	}
       e_ij->probability = profile_probability::always ();
     }
@@ -1494,7 +1473,6 @@ gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call,
       {
 	e = make_edge (dcall_bb, e_eh->dest, e_eh->flags);
 	e->probability = e_eh->probability;
-	e->count = e_eh->count;
 	for (gphi_iterator psi = gsi_start_phis (e_eh->dest);
 	     !gsi_end_p (psi); gsi_next (&psi))
 	  {
@@ -1540,7 +1518,7 @@ gimple_ic_transform (gimple_stmt_iterator *gsi)
   count = histogram->hvalue.counters [1];
   all = histogram->hvalue.counters [2];
 
-  bb_all = gimple_bb (stmt)->count.to_gcov_type ();
+  bb_all = gimple_bb (stmt)->count.ipa ().to_gcov_type ();
   /* The order of CHECK_COUNTER calls is important -
      since check_counter can correct the third parameter
      and we want to make count <= all <= bb_all. */
@@ -1704,20 +1682,16 @@ gimple_stringop_fixed_value (gcall *vcall_stmt, tree icall_size, profile_probabi
 
   e_ci->flags = (e_ci->flags & ~EDGE_FALLTHRU) | EDGE_TRUE_VALUE;
   e_ci->probability = prob;
-  e_ci->count = profile_count::from_gcov_type (count);
 
   e_cv = make_edge (cond_bb, vcall_bb, EDGE_FALSE_VALUE);
   e_cv->probability = prob.invert ();
-  e_cv->count = profile_count::from_gcov_type (all - count);
 
   remove_edge (e_iv);
 
   e_ij = make_edge (icall_bb, join_bb, EDGE_FALLTHRU);
   e_ij->probability = profile_probability::always ();
-  e_ij->count = profile_count::from_gcov_type (count);
 
   e_vj->probability = profile_probability::always ();
-  e_vj->count = profile_count::from_gcov_type (all - count);
 
   /* Insert PHI node for the call result if necessary.  */
   if (gimple_call_lhs (vcall_stmt)
diff --git a/gcc/value-prof.h b/gcc/value-prof.h
index f72bb2d2241..8190bfd074f 100644
--- a/gcc/value-prof.h
+++ b/gcc/value-prof.h
@@ -90,8 +90,7 @@ void gimple_move_stmt_histograms (struct function *, gimple *, gimple *);
 void verify_histograms (void);
 void free_histograms (function *);
 void stringop_block_profile (gimple *, unsigned int *, HOST_WIDE_INT *);
-gcall *gimple_ic (gcall *, struct cgraph_node *, profile_probability,
-		  profile_count, profile_count);
+gcall *gimple_ic (gcall *, struct cgraph_node *, profile_probability);
 bool check_ic_target (gcall *, struct cgraph_node *);
 
 
diff --git a/gcc/var-tracking.c b/gcc/var-tracking.c
index 4682aabc18f..92a0f77551f 100644
--- a/gcc/var-tracking.c
+++ b/gcc/var-tracking.c
@@ -5350,7 +5350,7 @@ track_loc_p (rtx loc, tree expr, poly_int64 offset, bool store_reg_p,
        || (store_reg_p
 	   && !COMPLEX_MODE_P (DECL_MODE (expr))
 	   && hard_regno_nregs (REGNO (loc), DECL_MODE (expr)) == 1))
-      && known_zero (offset + byte_lowpart_offset (DECL_MODE (expr), mode)))
+      && must_eq (offset + byte_lowpart_offset (DECL_MODE (expr), mode), 0))
     {
       mode = DECL_MODE (expr);
       offset = 0;
diff --git a/gcc/varasm.c b/gcc/varasm.c
index bf19ab7f413..046fd8bc2f0 100644
--- a/gcc/varasm.c
+++ b/gcc/varasm.c
@@ -2397,8 +2397,7 @@ incorporeal_function_p (tree decl)
       const char *name;
 
       if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL
-	  && (DECL_FUNCTION_CODE (decl) == BUILT_IN_ALLOCA
-	      || DECL_FUNCTION_CODE (decl) == BUILT_IN_ALLOCA_WITH_ALIGN))
+	  && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
 	return true;
 
       name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
@@ -2894,8 +2893,10 @@ decode_addr_const (tree exp, struct addr_const *value)
       else if (TREE_CODE (target) == ARRAY_REF
 	       || TREE_CODE (target) == ARRAY_RANGE_REF)
 	{
-	  offset += (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (target)))
-		     * tree_to_poly_int64 (TREE_OPERAND (target, 1)));
+	  /* Truncate big offset.  */
+	  offset
+	    += (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (target)))
+		* wi::to_poly_widest (TREE_OPERAND (target, 1)).force_shwi ());
 	  target = TREE_OPERAND (target, 0);
 	}
       else if (TREE_CODE (target) == MEM_REF
@@ -3931,8 +3932,8 @@ output_constant_pool_2 (fixed_size_mode mode, rtx x, unsigned int align)
 	gcc_assert (GET_CODE (x) == CONST_VECTOR);
 
 	/* Pick the smallest integer mode that contains at least one
-	   whole element.  Often this will be byte_mode and will contain
-	   more than one element.  */
+	   whole element.  Often this is byte_mode and contains more
+	   than one element.  */
 	unsigned int nelts = CONST_VECTOR_NUNITS (x);
 	unsigned int elt_bits = GET_MODE_BITSIZE (mode) / nelts;
 	unsigned int int_bits = MAX (elt_bits, BITS_PER_UNIT);
@@ -3947,9 +3948,8 @@ output_constant_pool_2 (fixed_size_mode mode, rtx x, unsigned int align)
 	    for (unsigned int j = 0; j < limit; ++j)
 	      if (INTVAL (CONST_VECTOR_ELT (x, i + j)) != 0)
 		value |= 1 << (j * elt_bits);
-	    output_constant_pool_2 (byte_mode,
-				    gen_int_mode (value, int_mode),
-				    i ? MIN (align, int_bits) : align);
+	    output_constant_pool_2 (int_mode, gen_int_mode (value, int_mode),
+				    i != 0 ? MIN (align, int_bits) : align);
 	  }
 	break;
       }
diff --git a/gcc/wide-int.cc b/gcc/wide-int.cc
index a2c8fa72302..ba0fd25b093 100644
--- a/gcc/wide-int.cc
+++ b/gcc/wide-int.cc
@@ -2146,6 +2146,39 @@ template void generic_wide_int <wide_int_ref_storage <true> >::dump () const;
 template void offset_int::dump () const;
 template void widest_int::dump () const;
 
+/* We could add all the above ::dump variants here, but wide_int and
+   widest_int should handle the common cases.  Besides, you can always
+   call the dump method directly.  */
+
+DEBUG_FUNCTION void
+debug (const wide_int &ref)
+{
+  ref.dump ();
+}
+
+DEBUG_FUNCTION void
+debug (const wide_int *ptr)
+{
+  if (ptr)
+    debug (*ptr);
+  else
+    fprintf (stderr, "<nil>\n");
+}
+
+DEBUG_FUNCTION void
+debug (const widest_int &ref)
+{
+  ref.dump ();
+}
+
+DEBUG_FUNCTION void
+debug (const widest_int *ptr)
+{
+  if (ptr)
+    debug (*ptr);
+  else
+    fprintf (stderr, "<nil>\n");
+}
 
 #if CHECKING_P
 
diff --git a/gcc/xcoffout.c b/gcc/xcoffout.c
index 17b201aced6..cf2064d5ba5 100644
--- a/gcc/xcoffout.c
+++ b/gcc/xcoffout.c
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 /* Output xcoff-format symbol table data.  The main functionality is contained
    in dbxout.c.  This file implements the sdbout-like parts of the xcoff
    interface.  Many functions are very similar to their counterparts in
-   sdbout.c.  */
+   the former sdbout.c file.  */
 
 #include "config.h"
 #include "system.h"
@@ -452,7 +452,7 @@ xcoffout_begin_prologue (unsigned int line,
   ASM_OUTPUT_LFB (asm_out_file, line);
   dbxout_parms (DECL_ARGUMENTS (current_function_decl));
 
-  /* Emit the symbols for the outermost BLOCK's variables.  sdbout.c does this
+  /* Emit the symbols for the outermost BLOCK's variables.  sdbout.c did this
      in sdbout_begin_block, but there is no guarantee that there will be any
      inner block 1, so we must do it here.  This gives a result similar to
      dbxout, so it does make some sense.  */
diff --git a/gotools/ChangeLog b/gotools/ChangeLog
index a78e762dd77..bd6e13f9ed5 100644
--- a/gotools/ChangeLog
+++ b/gotools/ChangeLog
@@ -1,3 +1,9 @@
+2017-10-25  Ian Lance Taylor  <iant@golang.org>
+
+	* Makefile.am (check-go-tool): Output colon after ${fl}.
+	(check-runtime, check-cgo-test, check-carchive-test): Likewise.
+	* Makefile.in: Rebuild.
+
 2017-09-19  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
 
 	* Makefile.am (GOTESTFLAGS): New variable.
diff --git a/gotools/Makefile.am b/gotools/Makefile.am
index 7ed5bc315c4..cbb64ae85d6 100644
--- a/gotools/Makefile.am
+++ b/gotools/Makefile.am
@@ -193,7 +193,7 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 	$(CHECK_ENV) \
 	GOPATH=`cd check-go-dir && $(PWD_COMMAND)`; \
 	export GOPATH; \
-	(cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_go-testlog 2>&1 || echo "--- $${fl} go test cmd/go (0.00s)" >> cmd_go-testlog
+	(cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_go-testlog 2>&1 || echo "--- $${fl}: go test cmd/go (0.00s)" >> cmd_go-testlog
 	grep '^--- ' cmd_go-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-runtime runs `go test runtime` in our environment.
@@ -215,7 +215,7 @@ check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 	GOARCH=`$(abs_builddir)/go$(EXEEXT) env GOARCH`; \
 	GOOS=`$(abs_builddir)/go$(EXEEXT) env GOOS`; \
 	files=`$(SHELL) $(libgosrcdir)/../match.sh --goarch=$${GOARCH} --goos=$${GOOS} --srcdir=$(libgosrcdir)/runtime --extrafiles="$(libgodir)/runtime_sysinfo.go $(libgodir)/sigtab.go" --tag=libffi`; \
-	$(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" $(GOTESTFLAGS) -test.v >> runtime-testlog 2>&1 || echo "--- $${fl} go test runtime (0.00s)" >> runtime-testlog
+	$(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" $(GOTESTFLAGS) -test.v >> runtime-testlog 2>&1 || echo "--- $${fl}: go test runtime (0.00s)" >> runtime-testlog
 	grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-cgo-test runs `go test misc/cgo/test` in our environment.
@@ -228,7 +228,7 @@ check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 	$(CHECK_ENV) \
 	GOTRACEBACK=2; \
 	export GOTRACEBACK; \
-	(cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/test (0.00s)" >> cgo-testlog
+	(cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/test (0.00s)" >> cgo-testlog
 	grep '^--- ' cgo-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
@@ -242,7 +242,7 @@ check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 	$(CHECK_ENV) \
 	LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \
 	export LIBRARY_PATH; \
-	(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
+	(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
 	grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # The check targets runs the tests and assembles the output files.
diff --git a/gotools/Makefile.in b/gotools/Makefile.in
index ad0fc2e1705..4f2b1073986 100644
--- a/gotools/Makefile.in
+++ b/gotools/Makefile.in
@@ -751,7 +751,7 @@ mostlyclean-local:
 @NATIVE_TRUE@	$(CHECK_ENV) \
 @NATIVE_TRUE@	GOPATH=`cd check-go-dir && $(PWD_COMMAND)`; \
 @NATIVE_TRUE@	export GOPATH; \
-@NATIVE_TRUE@	(cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_go-testlog 2>&1 || echo "--- $${fl} go test cmd/go (0.00s)" >> cmd_go-testlog
+@NATIVE_TRUE@	(cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_go-testlog 2>&1 || echo "--- $${fl}: go test cmd/go (0.00s)" >> cmd_go-testlog
 @NATIVE_TRUE@	grep '^--- ' cmd_go-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-runtime runs `go test runtime` in our environment.
@@ -773,7 +773,7 @@ mostlyclean-local:
 @NATIVE_TRUE@	GOARCH=`$(abs_builddir)/go$(EXEEXT) env GOARCH`; \
 @NATIVE_TRUE@	GOOS=`$(abs_builddir)/go$(EXEEXT) env GOOS`; \
 @NATIVE_TRUE@	files=`$(SHELL) $(libgosrcdir)/../match.sh --goarch=$${GOARCH} --goos=$${GOOS} --srcdir=$(libgosrcdir)/runtime --extrafiles="$(libgodir)/runtime_sysinfo.go $(libgodir)/sigtab.go" --tag=libffi`; \
-@NATIVE_TRUE@	$(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" $(GOTESTFLAGS) -test.v >> runtime-testlog 2>&1 || echo "--- $${fl} go test runtime (0.00s)" >> runtime-testlog
+@NATIVE_TRUE@	$(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" $(GOTESTFLAGS) -test.v >> runtime-testlog 2>&1 || echo "--- $${fl}: go test runtime (0.00s)" >> runtime-testlog
 @NATIVE_TRUE@	grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-cgo-test runs `go test misc/cgo/test` in our environment.
@@ -786,7 +786,7 @@ mostlyclean-local:
 @NATIVE_TRUE@	$(CHECK_ENV) \
 @NATIVE_TRUE@	GOTRACEBACK=2; \
 @NATIVE_TRUE@	export GOTRACEBACK; \
-@NATIVE_TRUE@	(cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/test (0.00s)" >> cgo-testlog
+@NATIVE_TRUE@	(cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/test (0.00s)" >> cgo-testlog
 @NATIVE_TRUE@	grep '^--- ' cgo-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
@@ -800,7 +800,7 @@ mostlyclean-local:
 @NATIVE_TRUE@	$(CHECK_ENV) \
 @NATIVE_TRUE@	LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \
 @NATIVE_TRUE@	export LIBRARY_PATH; \
-@NATIVE_TRUE@	(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
+@NATIVE_TRUE@	(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
 @NATIVE_TRUE@	grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # The check targets runs the tests and assembles the output files.
diff --git a/include/ChangeLog b/include/ChangeLog
index 0221586d778..8ccc57ae783 100644
--- a/include/ChangeLog
+++ b/include/ChangeLog
@@ -1,3 +1,13 @@
+2017-10-23  David Malcolm  <dmalcolm@redhat.com>
+
+	* unique-ptr.h: Make include of <memory> conditional on C++11 or
+	later.
+
+2017-10-16  Trevor Saunders  <tbsaunde+gcc@tbsaunde.org>
+	    David Malcolm  <dmalcolm@redhat.com>
+
+	* unique-ptr.h: New file.
+
 2017-09-15  Yao Qi  <yao.qi@linaro.org>
 	    Pedro Alves  <palves@redhat.com>
 
diff --git a/include/unique-ptr.h b/include/unique-ptr.h
new file mode 100644
index 00000000000..0b6f11a8146
--- /dev/null
+++ b/include/unique-ptr.h
@@ -0,0 +1,405 @@
+/* gnu::unique_ptr, a simple std::unique_ptr replacement for C++03.
+
+   Copyright (C) 2007-2016 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* gnu::unique_ptr defines a C++ owning smart pointer that exposes a
+   subset of the std::unique_ptr API.
+
+   In fact, when compiled with a C++11 compiler, gnu::unique_ptr
+   actually _is_ std::unique_ptr.  When compiled with a C++03 compiler
+   OTOH, it's an hand coded std::unique_ptr emulation that assumes
+   code is correct and doesn't try to be too smart.
+
+   This supports custom deleters, but not _stateful_ deleters, so you
+   can't use those in C++11 mode either.  Only the managed pointer is
+   stored in the smart pointer.  That could be changed; it simply
+   wasn't found necessary.
+
+   At the end of the file you'll find a gnu::unique_ptr partial
+   specialization that uses a custom (stateless) deleter:
+   gnu::unique_xmalloc_ptr.  That is used to manage pointers to
+   objects allocated with xmalloc.
+
+   The C++03 version was originally based on GCC 7.0's std::auto_ptr
+   and then heavily customized to behave more like C++11's
+   std::unique_ptr, but at this point, it no longer shares much at all
+   with the original file.  But, that's the history and the reason for
+   the copyright's starting year.
+
+   The C++03 version lets you shoot yourself in the foot, since
+   similarly to std::auto_ptr, the copy constructor and assignment
+   operators actually move.  Also, in the name of simplicity, no
+   effort is spent on using SFINAE to prevent invalid conversions,
+   etc.  This is not really a problem, because the goal here is to
+   allow code that would be correct using std::unique_ptr to be
+   equally correct in C++03 mode, and, just as efficient.  If client
+   code compiles correctly with a C++11 (or newer) compiler, we know
+   we're not doing anything invalid by mistake.
+
+   Usage notes:
+
+   - Putting gnu::unique_ptr in standard containers is not supported,
+     since C++03 containers are not move-aware (and our emulation
+     relies on copy actually moving).
+
+   - Since there's no nullptr in C++03, gnu::unique_ptr allows
+     implicit initialization and assignment from NULL instead.
+
+   - To check whether there's an associated managed object, all these
+     work as expected:
+
+      if (ptr)
+      if (!ptr)
+      if (ptr != NULL)
+      if (ptr == NULL)
+      if (NULL != ptr)
+      if (NULL == ptr)
+*/
+
+#ifndef GNU_UNIQUE_PTR_H
+#define GNU_UNIQUE_PTR_H 1
+
+#if __cplusplus >= 201103
+# include <memory>
+#endif
+
+namespace gnu
+{
+
+#if __cplusplus >= 201103
+
+/* In C++11 mode, all we need is import the standard
+   std::unique_ptr.  */
+template<typename T> using unique_ptr = std::unique_ptr<T>;
+
+/* Pull in move as well.  */
+using std::move;
+
+#else /* C++11 */
+
+/* Default destruction policy used by gnu::unique_ptr when no deleter
+   is specified.  Uses delete.  */
+
+template<typename T>
+struct default_delete
+{
+  void operator () (T *ptr) const { delete ptr; }
+};
+
+/* Specialization for arrays.  Uses delete[].  */
+
+template<typename T>
+struct default_delete<T[]>
+{
+  void operator () (T *ptr) const { delete [] ptr; }
+};
+
+namespace detail
+{
+/* Type used to support implicit construction from NULL:
+
+     gnu::unique_ptr<foo> func (....)
+     {
+     return NULL;
+     }
+
+   and assignment from NULL:
+
+     gnu::unique_ptr<foo> ptr (....);
+     ...
+     ptr = NULL;
+
+  It is intentionally not defined anywhere.  */
+struct nullptr_t;
+
+/* Base class of our unique_ptr emulation.  Contains code common to
+   both unique_ptr<T, D> and unique_ptr<T[], D>.  */
+
+template<typename T, typename D>
+class unique_ptr_base
+{
+public:
+  typedef T *pointer;
+  typedef T element_type;
+  typedef D deleter_type;
+
+  /* Takes ownership of a pointer.  P is a pointer to an object of
+     element_type type.  Defaults to NULL.  */
+  explicit unique_ptr_base (element_type *p = NULL) throw () : m_ptr (p) {}
+
+  /* The "move" constructor.  Really a copy constructor that actually
+     moves.  Even though std::unique_ptr is not copyable, our little
+     simpler emulation allows it, because:
+
+       - There are no rvalue references in C++03.  Our move emulation
+       instead relies on copy/assignment moving, like std::auto_ptr.
+       - RVO/NRVO requires an accessible copy constructor
+  */
+  unique_ptr_base (const unique_ptr_base &other) throw ()
+    : m_ptr (const_cast<unique_ptr_base &> (other).release ()) {}
+
+  /* Converting "move" constructor.  Really an lvalue ref converting
+     constructor that actually moves.  This allows constructs such as:
+
+      unique_ptr<Derived> func_returning_unique_ptr (.....);
+      ...
+      unique_ptr<Base> ptr = func_returning_unique_ptr (.....);
+  */
+  template<typename T1, typename D1>
+  unique_ptr_base (const unique_ptr_base<T1, D1> &other) throw ()
+    : m_ptr (const_cast<unique_ptr_base<T1, D1> &> (other).release ()) {}
+
+  /* The "move" assignment operator.  Really an lvalue ref copy
+     assignment operator that actually moves.  See comments above.  */
+  unique_ptr_base &operator= (const unique_ptr_base &other) throw ()
+  {
+    reset (const_cast<unique_ptr_base &> (other).release ());
+    return *this;
+  }
+
+  /* Converting "move" assignment.  Really an lvalue ref converting
+     copy assignment operator that moves.  See comments above.  */
+  template<typename T1, typename D1>
+  unique_ptr_base &operator= (const unique_ptr_base<T1, D1> &other) throw ()
+  {
+    reset (const_cast<unique_ptr_base<T1, D1> &> (other).release ());
+    return *this;
+  }
+
+  /* std::unique_ptr does not allow assignment, except from nullptr.
+     nullptr doesn't exist in C++03, so we allow assignment from NULL
+     instead [ptr = NULL;].
+  */
+  unique_ptr_base &operator= (detail::nullptr_t *) throw ()
+  {
+    reset ();
+    return *this;
+  }
+
+  ~unique_ptr_base () { call_deleter (); }
+
+  /* "explicit operator bool ()" emulation using the safe bool
+     idiom.  */
+private:
+  typedef void (unique_ptr_base::*explicit_operator_bool) () const;
+  void this_type_does_not_support_comparisons () const {}
+
+public:
+  operator explicit_operator_bool () const
+  {
+    return (m_ptr != NULL
+	    ? &unique_ptr_base::this_type_does_not_support_comparisons
+	    : 0);
+  }
+
+  element_type *get () const throw () { return m_ptr; }
+
+  element_type *release () throw ()
+  {
+    pointer tmp = m_ptr;
+    m_ptr = NULL;
+    return tmp;
+  }
+
+  void reset (element_type *p = NULL) throw ()
+  {
+    if (p != m_ptr)
+      {
+	call_deleter ();
+	m_ptr = p;
+      }
+  }
+
+private:
+
+  /* Call the deleter.  Note we assume the deleter is "stateless".  */
+  void call_deleter ()
+  {
+    D d;
+
+    d (m_ptr);
+  }
+
+  element_type *m_ptr;
+};
+
+} /* namespace detail */
+
+/* Macro used to create a unique_ptr_base "partial specialization" --
+   a subclass that uses a specific deleter.  Basically this re-defines
+   the necessary constructors.  This is necessary because C++03
+   doesn't support inheriting constructors with "using".  While at it,
+   we inherit the assignment operator.  TYPE is the name of the type
+   being defined.  Assumes that 'base_type' is a typedef of the
+   baseclass TYPE is inheriting from.  */
+#define DEFINE_GNU_UNIQUE_PTR(TYPE)						\
+public:									\
+  explicit TYPE (T *p = NULL) throw ()					\
+    : base_type (p) {}							\
+									\
+  TYPE (const TYPE &other) throw () : base_type (other) {}		\
+									\
+  TYPE (detail::nullptr_t *) throw () : base_type (NULL) {}		\
+									\
+  template<typename T1, typename D1>					\
+  TYPE (const detail::unique_ptr_base<T1, D1> &other) throw ()		\
+    : base_type (other) {}						\
+									\
+  using base_type::operator=;
+
+/* Define single-object gnu::unique_ptr.  */
+
+template <typename T, typename D = default_delete<T> >
+class unique_ptr : public detail::unique_ptr_base<T, D>
+{
+  typedef detail::unique_ptr_base<T, D> base_type;
+
+  DEFINE_GNU_UNIQUE_PTR (unique_ptr)
+
+public:
+  /* Dereferencing.  */
+  T &operator* () const throw () { return *this->get (); }
+  T *operator-> () const throw () { return this->get (); }
+};
+
+/* Define gnu::unique_ptr specialization for T[].  */
+
+template <typename T, typename D>
+class unique_ptr<T[], D> : public detail::unique_ptr_base<T, D>
+{
+  typedef detail::unique_ptr_base<T, D> base_type;
+
+  DEFINE_GNU_UNIQUE_PTR (unique_ptr)
+
+public:
+  /* Indexing operator.  */
+  T &operator[] (size_t i) const { return this->get ()[i]; }
+};
+
+/* Comparison operators.  */
+
+template <typename T, typename D,
+	  typename U, typename E>
+inline bool
+operator== (const detail::unique_ptr_base<T, D> &x,
+	    const detail::unique_ptr_base<U, E> &y)
+{ return x.get() == y.get(); }
+
+template <typename T, typename D,
+	  typename U, typename E>
+inline bool
+operator!= (const detail::unique_ptr_base<T, D> &x,
+	    const detail::unique_ptr_base<U, E> &y)
+{ return x.get() != y.get(); }
+
+template<typename T, typename D,
+	 typename U, typename E>
+inline bool
+operator< (const detail::unique_ptr_base<T, D> &x,
+	   const detail::unique_ptr_base<U, E> &y)
+{ return x.get() < y.get (); }
+
+template<typename T, typename D,
+	 typename U, typename E>
+inline bool
+operator<= (const detail::unique_ptr_base<T, D> &x,
+	    const detail::unique_ptr_base<U, E> &y)
+{ return !(y < x); }
+
+template<typename T, typename D,
+	 typename U, typename E>
+inline bool
+operator> (const detail::unique_ptr_base<T, D> &x,
+	   const detail::unique_ptr_base<U, E> &y)
+{ return y < x; }
+
+template<typename T, typename D,
+	 typename U, typename E>
+inline bool
+operator>= (const detail::unique_ptr_base<T, D> &x,
+	    const detail::unique_ptr_base<U, E> &y)
+{ return !(x < y); }
+
+/* std::move "emulation".  This is as simple as it can be -- no
+   attempt is made to emulate rvalue references.  Instead relies on
+   the fact that gnu::unique_ptr has move semantics like
+   std::auto_ptr.  I.e., copy/assignment actually moves.  */
+
+template<typename T, typename D>
+unique_ptr<T, D>
+move (unique_ptr<T, D> v)
+{
+  return v;
+}
+
+#endif /* C++11 */
+
+/* Define gnu::unique_xmalloc_ptr, a gnu::unique_ptr that manages
+   xmalloc'ed memory.  */
+
+/* The deleter for gnu::unique_xmalloc_ptr.  Uses free.  */
+template <typename T>
+struct xmalloc_deleter
+{
+  void operator() (T *ptr) const { free (ptr); }
+};
+
+/* Same, for arrays.  */
+template <typename T>
+struct xmalloc_deleter<T[]>
+{
+  void operator() (T *ptr) const { free (ptr); }
+};
+
+#if __cplusplus >= 201103
+
+/* In C++11, we just import the standard unique_ptr to our namespace
+   with a custom deleter.  */
+
+template<typename T> using unique_xmalloc_ptr
+  = std::unique_ptr<T, xmalloc_deleter<T>>;
+
+#else /* C++11 */
+
+/* In C++03, we don't have template aliases, so we need to define a
+   subclass instead, and re-define the constructors, because C++03
+   doesn't support inheriting constructors either.  */
+
+template <typename T>
+class unique_xmalloc_ptr : public unique_ptr<T, xmalloc_deleter<T> >
+{
+  typedef unique_ptr<T, xmalloc_deleter<T> > base_type;
+
+  DEFINE_GNU_UNIQUE_PTR (unique_xmalloc_ptr)
+};
+
+/* Define gnu::unique_xmalloc_ptr specialization for T[].  */
+
+template <typename T>
+class unique_xmalloc_ptr<T[]> : public unique_ptr<T[], xmalloc_deleter<T[]> >
+{
+  typedef unique_ptr<T[], xmalloc_deleter<T[]> > base_type;
+
+  DEFINE_GNU_UNIQUE_PTR (unique_xmalloc_ptr)
+};
+
+#endif /* C++11 */
+
+} /* namespace gnu */
+
+#endif /* GNU_UNIQUE_PTR_H */
diff --git a/libatomic/ChangeLog b/libatomic/ChangeLog
index 835ea6c5093..eed12e63a68 100644
--- a/libatomic/ChangeLog
+++ b/libatomic/ChangeLog
@@ -1,3 +1,9 @@
+2017-10-20  Richard Earnshaw  <rearnsha@arm.com>
+
+	* Makefile.am: (IFUNC_OPTIONS): Set the architecture to
+	-march=armv7-a+fp on Linux/Arm.
+	* Makefile.in: Regenerated.
+
 2017-10-02  Martin Sebor  <msebor@redhat.com>
 
 	PR c/81854
diff --git a/libatomic/Makefile.am b/libatomic/Makefile.am
index d731406fdbd..9c457008232 100644
--- a/libatomic/Makefile.am
+++ b/libatomic/Makefile.am
@@ -123,7 +123,7 @@ libatomic_la_LIBADD = $(foreach s,$(SIZES),$(addsuffix _$(s)_.lo,$(SIZEOBJS)))
 ## On a target-specific basis, include alternates to be selected by IFUNC.
 if HAVE_IFUNC
 if ARCH_ARM_LINUX
-IFUNC_OPTIONS	     = -march=armv7-a -DHAVE_KERNEL64
+IFUNC_OPTIONS	     = -march=armv7-a+fp -DHAVE_KERNEL64
 libatomic_la_LIBADD += $(foreach s,$(SIZES),$(addsuffix _$(s)_1_.lo,$(SIZEOBJS)))
 libatomic_la_LIBADD += $(addsuffix _8_2_.lo,$(SIZEOBJS))
 endif
diff --git a/libatomic/Makefile.in b/libatomic/Makefile.in
index f6eeab312ea..0f0382e0713 100644
--- a/libatomic/Makefile.in
+++ b/libatomic/Makefile.in
@@ -346,7 +346,7 @@ M_SRC = $(firstword $(filter %/$(M_FILE), $(all_c_files)))
 libatomic_la_LIBADD = $(foreach s,$(SIZES),$(addsuffix \
 	_$(s)_.lo,$(SIZEOBJS))) $(am__append_1) $(am__append_2) \
 	$(am__append_3)
-@ARCH_ARM_LINUX_TRUE@@HAVE_IFUNC_TRUE@IFUNC_OPTIONS = -march=armv7-a -DHAVE_KERNEL64
+@ARCH_ARM_LINUX_TRUE@@HAVE_IFUNC_TRUE@IFUNC_OPTIONS = -march=armv7-a+fp -DHAVE_KERNEL64
 @ARCH_I386_TRUE@@HAVE_IFUNC_TRUE@IFUNC_OPTIONS = -march=i586
 @ARCH_X86_64_TRUE@@HAVE_IFUNC_TRUE@IFUNC_OPTIONS = -mcx16
 libatomic_convenience_la_SOURCES = $(libatomic_la_SOURCES)
diff --git a/libcc1/ChangeLog b/libcc1/ChangeLog
index ea7a1aeb25b..2194aa8a5a4 100644
--- a/libcc1/ChangeLog
+++ b/libcc1/ChangeLog
@@ -1,3 +1,14 @@
+2017-11-01  Nathan Sidwell  <nathan@acm.org>
+
+	* libcp1plugin.cc (plugin_build_decl): Use
+	DECL_OVERLOADED_OPERATOR_CODE_RAW.
+
+2017-10-31  Nathan Sidwell  <nathan@acm.org>
+
+	* libcp1plugin.cc (plugin_build_decl): Use ovl_op_identifier.
+	Directly set operator code.
+	(plugin_build_dependent_expr): Use ovl_op_identifier.
+
 2017-10-04  Nathan Sidwell  <nathan@acm.org>
 
 	* libcp1plugin.cc (supplement_binding): Don't use
diff --git a/libcc1/libcp1plugin.cc b/libcc1/libcp1plugin.cc
index 12ea4ed84e0..030721340c1 100644
--- a/libcc1/libcp1plugin.cc
+++ b/libcc1/libcp1plugin.cc
@@ -1346,12 +1346,7 @@ plugin_build_decl (cc1_plugin::connection *self,
 	    }
 
 	  if (opcode != ERROR_MARK)
-	    {
-	      if (assop)
-		identifier = cp_assignment_operator_id (opcode);
-	      else
-		identifier = cp_operator_id (opcode);
-	    }
+	    identifier = ovl_op_identifier (assop, opcode);
 	}
       decl = build_lang_decl_loc (loc, code, identifier, sym_type);
       /* FIXME: current_lang_name is lang_name_c while compiling an
@@ -1410,19 +1405,14 @@ plugin_build_decl (cc1_plugin::connection *self,
 	  DECL_DECLARED_INLINE_P (decl) = 1;
 	  DECL_INITIAL (decl) = error_mark_node;
 	}
-      if (ctor || dtor)
-	{
-	  if (ctor)
-	    DECL_CXX_CONSTRUCTOR_P (decl) = 1;
-	  if (dtor)
-	    DECL_CXX_DESTRUCTOR_P (decl) = 1;
-	}
-      else
-	{
-	  if ((sym_flags & GCC_CP_FLAG_SPECIAL_FUNCTION)
-	      && opcode != ERROR_MARK)
-	    SET_OVERLOADED_OPERATOR_CODE (decl, opcode);
-	}
+
+      if (ctor)
+	DECL_CXX_CONSTRUCTOR_P (decl) = 1;
+      else if (dtor)
+	DECL_CXX_DESTRUCTOR_P (decl) = 1;
+      else if ((sym_flags & GCC_CP_FLAG_SPECIAL_FUNCTION)
+	       && opcode != ERROR_MARK)
+	DECL_OVERLOADED_OPERATOR_CODE_RAW (decl) = ovl_op_mapping[opcode];
     }
   else if (RECORD_OR_UNION_CODE_P (code))
     {
@@ -2649,12 +2639,7 @@ plugin_build_dependent_expr (cc1_plugin::connection *self,
       gcc_assert (convop || !conv_type);
 
       if (opcode != ERROR_MARK)
-	{
-	  if (assop)
-	    identifier = cp_assignment_operator_id (opcode);
-	  else
-	    identifier = cp_operator_id (opcode);
-	}
+	identifier = ovl_op_identifier (assop, opcode);
 
       gcc_assert (identifier);
     }
diff --git a/libcpp/ChangeLog b/libcpp/ChangeLog
index f2c0d4d63fd..39e12bda927 100644
--- a/libcpp/ChangeLog
+++ b/libcpp/ChangeLog
@@ -1,3 +1,25 @@
+2017-10-31  David Malcolm  <dmalcolm@redhat.com>
+
+	* directives.c (_cpp_handle_directive): Update for renaming of
+	cpp_error_at_richloc to cpp_error_at.
+	* errors.c (cpp_diagnostic_at_richloc): Rename to...
+	(cpp_diagnostic_at): ...this, dropping the location_t-based
+	implementation.
+	(cpp_diagnostic): Update for removal of location_t-based
+	cpp_diagnostic_at.
+	(cpp_error_at): Likewise.
+	(cpp_error_at_richloc): Rename to...
+	(cpp_error_at): ...this, and update for renaming of
+	cpp_diagnostic_at_richloc.
+	* include/cpplib.h (cpp_error_at_richloc): Rename to...
+	(cpp_error_at): ...this.
+
+2017-10-30  Joseph Myers  <joseph@codesourcery.com>
+
+	* include/cpplib.h (enum c_lang): Add CLK_GNUC17 and CLK_STDC17.
+	* init.c (lang_defaults): Add GNUC17 and STDC17 data.
+	(cpp_init_builtins): Handle C17 value of __STDC_VERSION__.
+
 2017-10-10  Nathan Sidwell  <nathan@acm.org>
 
 	PR preprocessor/82506
diff --git a/libcpp/directives.c b/libcpp/directives.c
index 7cac653dbbc..e2952216c65 100644
--- a/libcpp/directives.c
+++ b/libcpp/directives.c
@@ -527,10 +527,10 @@ _cpp_handle_directive (cpp_reader *pfile, int indented)
 	      source_range misspelled_token_range
 		= get_range_from_loc (pfile->line_table, dname->src_loc);
 	      richloc.add_fixit_replace (misspelled_token_range, hint);
-	      cpp_error_at_richloc (pfile, CPP_DL_ERROR, &richloc,
-				    "invalid preprocessing directive #%s;"
-				    " did you mean #%s?",
-				    unrecognized, hint);
+	      cpp_error_at (pfile, CPP_DL_ERROR, &richloc,
+			    "invalid preprocessing directive #%s;"
+			    " did you mean #%s?",
+			    unrecognized, hint);
 	    }
 	  else
 	    cpp_error (pfile, CPP_DL_ERROR,
diff --git a/libcpp/errors.c b/libcpp/errors.c
index 7fdee3c6254..d49c85324d8 100644
--- a/libcpp/errors.c
+++ b/libcpp/errors.c
@@ -31,33 +31,15 @@ along with this program; see the file COPYING3.  If not see
 
 ATTRIBUTE_FPTR_PRINTF(5,0)
 static bool
-cpp_diagnostic_at_richloc (cpp_reader * pfile, int level, int reason,
-			   rich_location *richloc,
-			   const char *msgid, va_list *ap)
-{
-  bool ret;
-
-  if (!pfile->cb.error)
-    abort ();
-  ret = pfile->cb.error (pfile, level, reason, richloc, _(msgid), ap);
-
-  return ret;
-}
-
-/* Print a diagnostic at the given location.  */
-
-ATTRIBUTE_FPTR_PRINTF(5,0)
-static bool
 cpp_diagnostic_at (cpp_reader * pfile, int level, int reason,
-		   source_location src_loc,
+		   rich_location *richloc,
 		   const char *msgid, va_list *ap)
 {
   bool ret;
 
   if (!pfile->cb.error)
     abort ();
-  rich_location richloc (pfile->line_table, src_loc);
-  ret = pfile->cb.error (pfile, level, reason, &richloc, _(msgid), ap);
+  ret = pfile->cb.error (pfile, level, reason, richloc, _(msgid), ap);
 
   return ret;
 }
@@ -88,7 +70,8 @@ cpp_diagnostic (cpp_reader * pfile, int level, int reason,
     {
       src_loc = pfile->cur_token[-1].src_loc;
     }
-  return cpp_diagnostic_at (pfile, level, reason, src_loc, msgid, ap);
+  rich_location richloc (pfile->line_table, src_loc);
+  return cpp_diagnostic_at (pfile, level, reason, &richloc, msgid, ap);
 }
 
 /* Print a warning or error, depending on the value of LEVEL.  */
@@ -265,7 +248,8 @@ cpp_error_at (cpp_reader * pfile, int level, source_location src_loc,
 
   va_start (ap, msgid);
 
-  ret = cpp_diagnostic_at (pfile, level, CPP_W_NONE, src_loc,
+  rich_location richloc (pfile->line_table, src_loc);
+  ret = cpp_diagnostic_at (pfile, level, CPP_W_NONE, &richloc,
 			   msgid, &ap);
 
   va_end (ap);
@@ -276,16 +260,16 @@ cpp_error_at (cpp_reader * pfile, int level, source_location src_loc,
    a column override.  */
 
 bool
-cpp_error_at_richloc (cpp_reader * pfile, int level, rich_location *richloc,
-		      const char *msgid, ...)
+cpp_error_at (cpp_reader * pfile, int level, rich_location *richloc,
+	      const char *msgid, ...)
 {
   va_list ap;
   bool ret;
 
   va_start (ap, msgid);
 
-  ret = cpp_diagnostic_at_richloc (pfile, level, CPP_W_NONE, richloc,
-				   msgid, &ap);
+  ret = cpp_diagnostic_at (pfile, level, CPP_W_NONE, richloc,
+			   msgid, &ap);
 
   va_end (ap);
   return ret;
diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index 804132a44da..5a14858c44f 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -168,8 +168,8 @@ enum cpp_ttype
 #undef TK
 
 /* C language kind, used when calling cpp_create_reader.  */
-enum c_lang {CLK_GNUC89 = 0, CLK_GNUC99, CLK_GNUC11,
-	     CLK_STDC89, CLK_STDC94, CLK_STDC99, CLK_STDC11,
+enum c_lang {CLK_GNUC89 = 0, CLK_GNUC99, CLK_GNUC11, CLK_GNUC17,
+	     CLK_STDC89, CLK_STDC94, CLK_STDC99, CLK_STDC11, CLK_STDC17,
 	     CLK_GNUCXX, CLK_CXX98, CLK_GNUCXX11, CLK_CXX11,
 	     CLK_GNUCXX14, CLK_CXX14, CLK_GNUCXX17, CLK_CXX17,
 	     CLK_GNUCXX2A, CLK_CXX2A, CLK_ASM};
@@ -1096,9 +1096,9 @@ extern bool cpp_error_at (cpp_reader * pfile, int level,
 			  source_location src_loc, const char *msgid, ...)
   ATTRIBUTE_PRINTF_4;
 
-extern bool cpp_error_at_richloc (cpp_reader * pfile, int level,
-				  rich_location *richloc, const char *msgid,
-				  ...)
+extern bool cpp_error_at (cpp_reader * pfile, int level,
+			  rich_location *richloc, const char *msgid,
+			  ...)
   ATTRIBUTE_PRINTF_4;
 
 /* In lex.c */
diff --git a/libcpp/init.c b/libcpp/init.c
index 16ff202c8cf..ecc81e3138a 100644
--- a/libcpp/init.c
+++ b/libcpp/init.c
@@ -98,10 +98,12 @@ static const struct lang_flags lang_defaults[] =
   /* GNUC89   */  { 0,  0,  1,  0,  0,  0,  1,   0,   0,   0,    0,     0,     0,   0 },
   /* GNUC99   */  { 1,  0,  1,  1,  0,  0,  1,   1,   1,   0,    0,     0,     0,   0 },
   /* GNUC11   */  { 1,  0,  1,  1,  1,  0,  1,   1,   1,   0,    0,     0,     0,   0 },
+  /* GNUC17   */  { 1,  0,  1,  1,  1,  0,  1,   1,   1,   0,    0,     0,     0,   0 },
   /* STDC89   */  { 0,  0,  0,  0,  0,  1,  0,   0,   0,   0,    0,     0,     1,   0 },
   /* STDC94   */  { 0,  0,  0,  0,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0 },
   /* STDC99   */  { 1,  0,  1,  1,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0 },
   /* STDC11   */  { 1,  0,  1,  1,  1,  1,  1,   1,   0,   0,    0,     0,     1,   0 },
+  /* STDC17   */  { 1,  0,  1,  1,  1,  1,  1,   1,   0,   0,    0,     0,     1,   0 },
   /* GNUCXX   */  { 0,  1,  1,  1,  0,  0,  1,   0,   0,   0,    0,     0,     0,   0 },
   /* CXX98    */  { 0,  1,  0,  1,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0 },
   /* GNUCXX11 */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    0,     0,     0,   0 },
@@ -518,6 +520,9 @@ cpp_init_builtins (cpp_reader *pfile, int hosted)
     _cpp_define_builtin (pfile, "__ASSEMBLER__ 1");
   else if (CPP_OPTION (pfile, lang) == CLK_STDC94)
     _cpp_define_builtin (pfile, "__STDC_VERSION__ 199409L");
+  else if (CPP_OPTION (pfile, lang) == CLK_STDC17
+	   || CPP_OPTION (pfile, lang) == CLK_GNUC17)
+    _cpp_define_builtin (pfile, "__STDC_VERSION__ 201710L");
   else if (CPP_OPTION (pfile, lang) == CLK_STDC11
 	   || CPP_OPTION (pfile, lang) == CLK_GNUC11)
     _cpp_define_builtin (pfile, "__STDC_VERSION__ 201112L");
diff --git a/libgcc/ChangeLog b/libgcc/ChangeLog
index 7b1ffb1a0cc..a99d6186ca5 100644
--- a/libgcc/ChangeLog
+++ b/libgcc/ChangeLog
@@ -1,3 +1,27 @@
+2017-11-03  Cupertino Miranda  <cmiranda@synopsys.com>
+	    Vineet Gupta <vgupta@synopsys.com>
+
+	* config.host (arc*-*-linux*): Set md_unwind_header variable.
+	* config/arc/linux-unwind-reg.def: New file.
+	* config/arc/linux-unwind-reg.h: Likewise.
+
+2017-10-23  Sebastian Perta  <sebastian.perta@renesas.com>
+
+	* config/rl78/subdi3.S: New assembly file.
+	* config/rl78/t-rl78: Added subdi3.S to LIB2ADD.
+
+2017-10-13  Sebastian Perta  <sebastian.perta@renesas.com>
+
+	* config/rl78/adddi3.S: New assembly file.
+	* config/rl78/t-rl78: Added adddi3.S to LIB2ADD.
+
+2017-10-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/82274
+	* libgcc2.c (__mulvDI3): If both operands have
+	the same highpart of -1 and the topmost bit of lowpart is 0,
+	multiplication overflows even if both lowparts are 0.
+
 2017-09-28  James Bowman  <james.bowman@ftdichip.com>
 
 	* config/ft32/crti-hw.S: Add watchdog vector, FT930 IRQ support.
diff --git a/libgcc/config.host b/libgcc/config.host
index 68d8fe616e6..ad5a9ff621f 100644
--- a/libgcc/config.host
+++ b/libgcc/config.host
@@ -390,6 +390,7 @@ arc*-*-linux*)
 	tmake_file="${tmake_file} t-slibgcc-libgcc t-slibgcc-nolc-override arc/t-arc-uClibc arc/t-arc"
 	extra_parts="$extra_parts crti.o crtn.o"
 	extra_parts="$extra_parts crttls.o"
+	md_unwind_header=arc/linux-unwind.h
 	;;
 arm-wrs-vxworks|arm-wrs-vxworks7)
 	tmake_file="$tmake_file arm/t-arm arm/t-elf t-softfp-sfdf t-softfp-excl arm/t-softfp t-softfp"
diff --git a/libgcc/config/aarch64/value-unwind.h b/libgcc/config/aarch64/value-unwind.h
index c79e832144f..4019d847324 100644
--- a/libgcc/config/aarch64/value-unwind.h
+++ b/libgcc/config/aarch64/value-unwind.h
@@ -23,3 +23,19 @@
 #if defined __aarch64__ && !defined __LP64__
 # define REG_VALUE_IN_UNWIND_CONTEXT
 #endif
+
+/* Return the value of the pseudo VG register.  This should only be
+   called if we know this is an SVE host.  */
+static inline int
+aarch64_vg (void)
+{
+  register int vg asm ("x0");
+  /* CNTD X0.  */
+  asm (".inst 0x04e0e3e0" : "=r" (vg));
+  return vg;
+}
+
+/* Lazily provide a value for VG, so that we don't try to execute SVE
+   instructions unless we know they're needed.  */
+#define DWARF_LAZY_REGISTER_VALUE(REGNO, VALUE) \
+  ((REGNO) == AARCH64_DWARF_VG && ((*VALUE) = aarch64_vg (), 1))
diff --git a/libgcc/config/arc/linux-unwind-reg.def b/libgcc/config/arc/linux-unwind-reg.def
new file mode 100644
index 00000000000..8c680376c89
--- /dev/null
+++ b/libgcc/config/arc/linux-unwind-reg.def
@@ -0,0 +1,42 @@
+/* Copyright (C) 2017 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
+   License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.
+*/
+
+REGISTER_IN_STACK(bta, -1)
+REGISTER_IN_STACK(lp_start, -1)
+REGISTER_IN_STACK(lp_end, -1)
+REGISTER_IN_STACK(lp_count, -1)
+REGISTER_IN_STACK(status32, -1)
+REGISTER_IN_STACK(ret, -1)
+REGISTER_IN_STACK(blink, 31)
+REGISTER_IN_STACK(fp, 27)
+REGISTER_IN_STACK(gp, 26)
+REGISTER_IN_STACK(r12, 12)
+REGISTER_IN_STACK(r11, 11)
+REGISTER_IN_STACK(r10, 10)
+REGISTER_IN_STACK(r9, 9)
+REGISTER_IN_STACK(r8, 8)
+REGISTER_IN_STACK(r7, 7)
+REGISTER_IN_STACK(r6, 6)
+REGISTER_IN_STACK(r5, 5)
+REGISTER_IN_STACK(r4, 4)
+REGISTER_IN_STACK(r3, 3)
+REGISTER_IN_STACK(r2, 2)
+REGISTER_IN_STACK(r1, 1)
+REGISTER_IN_STACK(r0, 0)
+REGISTER_IN_STACK(sp, 28)
diff --git a/libgcc/config/arc/linux-unwind.h b/libgcc/config/arc/linux-unwind.h
new file mode 100644
index 00000000000..136c9b7ed03
--- /dev/null
+++ b/libgcc/config/arc/linux-unwind.h
@@ -0,0 +1,153 @@
+/* DWARF2 EH unwinding support for ARC Linux.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
+   License for more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License
+   and a copy of the GCC Runtime Library Exception along with this
+   program; see the files COPYING3 and COPYING.RUNTIME respectively.
+   If not, see <http://www.gnu.org/licenses/>.  */
+
+/* This order is defined by a structure in the kernel, in file
+   arch/arc/kernel/signal.c.  */
+
+#define REGISTER_IN_STACK(REG_NAME, ID) \
+  REG_NAME,
+enum registers_stack_order {
+  REGISTER_STACK_ORDER_START = -1,
+  #include "config/arc/linux-unwind-reg.def"
+  REGISTER_STACK_ORDER_SIZE,
+};
+
+struct register_position {
+  int reg_id;
+  int offset_in_stack;
+};
+#undef REGISTER_IN_STACK
+
+#define REGISTER_SIZE_IN_WORDS 4
+#define REGISTER_IN_STACK(REG_NAME, ID) (int) ID,
+int
+register_id_for_index[REGISTER_STACK_ORDER_SIZE] = {
+  #include "config/arc/linux-unwind-reg.def"
+};
+#undef REGISTER_IN_STACK
+
+#ifndef inhibit_libc
+/* Do code reading to identify a signal frame, and set the frame
+   state data appropriately.  See unwind-dw2.c for the structs.  */
+
+#include <signal.h>
+#include <asm/unistd.h>
+
+/*
+00010edc <__default_rt_sa_restorer>:
+   10edc:	208a 12c2		mov     r8,139
+   10ee0:	781e			trap_s  0
+   10ee2:	7ee0			j_s     [blink]
+*/
+
+#if __BIG_ENDIAN__
+#define MOV_R8_139	  0x8a20c212
+#define TRAP_S_J_S_BLINK  0x1e78e07e
+#define SWI		  0x6f223f00
+#elif __LITTLE_ENDIAN__
+#define MOV_R8_139	  0x12c2208a
+#define TRAP_S_J_S_BLINK  0x7ee0781e
+#define SWI		  0x003f226f
+#endif
+
+#define MD_FALLBACK_FRAME_STATE_FOR arc_fallback_frame_state
+
+static __attribute__((noinline)) _Unwind_Reason_Code
+arc_fallback_frame_state (struct _Unwind_Context *context,
+			   _Unwind_FrameState *fs)
+{
+  struct rt_sigframe {
+    siginfo_t info;
+    ucontext_t uc;
+    unsigned int sigret_magic;
+  };
+
+  struct rt_sigframe *rt_;
+  u_int32_t *pc = (u_int32_t *) context->ra;
+  struct sigcontext *sc;
+  _Unwind_Ptr new_cfa;
+  int i;
+
+#ifdef __ARC700__
+  if (pc[1] != SWI)
+    return _URC_END_OF_STACK;
+#else
+  if (pc[1] != TRAP_S_J_S_BLINK)
+    return _URC_END_OF_STACK;
+#endif
+
+  if (pc[0] == MOV_R8_139)
+    {
+      rt_ = context->cfa;
+      sc = &rt_->uc.uc_mcontext;
+    }
+  else
+    return _URC_END_OF_STACK;
+
+  new_cfa = (_Unwind_Ptr) sc;
+  fs->regs.cfa_how = CFA_REG_OFFSET;
+  fs->regs.cfa_reg = __LIBGCC_STACK_POINTER_REGNUM__;
+  fs->regs.cfa_offset = new_cfa - (_Unwind_Ptr) context->cfa;
+
+  unsigned long *regs = &sc->regs.scratch.bta;
+  for (i = 0; i < REGISTER_STACK_ORDER_SIZE; i++)
+    {
+      if (register_id_for_index[i] == -1)
+	continue;
+      fs->regs.reg[register_id_for_index[i]].how = REG_SAVED_OFFSET;
+      fs->regs.reg[register_id_for_index[i]].loc.offset
+	= ((_Unwind_Ptr) &(regs[i])) - new_cfa;
+    }
+
+  fs->regs.reg[31].how = REG_SAVED_VAL_OFFSET;
+  fs->regs.reg[31].loc.offset = ((_Unwind_Ptr) (regs[ret])) - new_cfa;
+
+  fs->retaddr_column = 31;
+
+  return _URC_NO_REASON;
+}
+#endif
+
+#define MD_FROB_UPDATE_CONTEXT arc_frob_update_context
+/* Save fp register for unwinding to work.  */
+
+static void
+arc_frob_update_context (struct _Unwind_Context *context,
+			 _Unwind_FrameState *fs)
+{
+  _Unwind_Word fp_val;
+  asm ("mov %0,fp" : "=r" (fp_val));
+
+  switch (fs->regs.reg[27].how)
+    {
+    case REG_UNSAVED:
+    case REG_UNDEFINED:
+      if (context->reg[27] == NULL)
+	_Unwind_SetGRValue (context, 27, fp_val);
+      break;
+
+    default:
+      break;
+    }
+}
diff --git a/libgcc/config/rl78/adddi3.S b/libgcc/config/rl78/adddi3.S
new file mode 100644
index 00000000000..c805055c4f8
--- /dev/null
+++ b/libgcc/config/rl78/adddi3.S
@@ -0,0 +1,58 @@
+;   Copyright (C) 2017 Free Software Foundation, Inc.
+;   Contributed by Sebastian Perta.
+;
+; This file is free software; you can redistribute it and/or modify it
+; under the terms of the GNU General Public License as published by the
+; Free Software Foundation; either version 3, or (at your option) any
+; later version.
+;
+; This file is distributed in the hope that it will be useful, but
+; WITHOUT ANY WARRANTY; without even the implied warranty of
+; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+; General Public License for more details.
+;
+; Under Section 7 of GPL version 3, you are granted additional
+; permissions described in the GCC Runtime Library Exception, version
+; 3.1, as published by the Free Software Foundation.
+;
+; You should have received a copy of the GNU General Public License and
+; a copy of the GCC Runtime Library Exception along with this program;
+; see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+; <http://www.gnu.org/licenses/>.
+
+
+#include "vregs.h"
+
+    .text
+
+START_FUNC ___adddi3
+
+    movw  hl, sp       ; use HL-based addressing (allows for direct addw)
+
+    movw  ax, [hl+4]
+    addw  ax, [hl+12]
+    movw  r8, ax
+
+    mov   a, [hl+6]    ; middle bytes of the result are determined using 8-bit
+    addc  a, [hl+14]   ; ADDC insns which both account for and update the carry bit
+    mov   r10, a       ; (no ADDWC instruction is available)
+    mov   a, [hl+7]
+    addc  a, [hl+15]
+    mov   r11, a
+
+    mov   a, [hl+8]
+    addc  a, [hl+16]
+    mov   r12, a
+    mov   a, [hl+9]
+    addc  a, [hl+17]
+    mov   r13, a
+
+    movw  ax, [hl+10]
+    sknc               ; account for the possible carry from the
+    incw  ax           ; latest 8-bit operation
+    addw  ax, [hl+18]
+    movw  r14, ax
+
+    ret
+
+END_FUNC ___adddi3
diff --git a/libgcc/config/rl78/subdi3.S b/libgcc/config/rl78/subdi3.S
new file mode 100644
index 00000000000..f6fcaba89ac
--- /dev/null
+++ b/libgcc/config/rl78/subdi3.S
@@ -0,0 +1,58 @@
+;   Copyright (C) 2017 Free Software Foundation, Inc.
+;   Contributed by Sebastian Perta.
+;
+; This file is free software; you can redistribute it and/or modify it
+; under the terms of the GNU General Public License as published by the
+; Free Software Foundation; either version 3, or (at your option) any
+; later version.
+;
+; This file is distributed in the hope that it will be useful, but
+; WITHOUT ANY WARRANTY; without even the implied warranty of
+; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+; General Public License for more details.
+;
+; Under Section 7 of GPL version 3, you are granted additional
+; permissions described in the GCC Runtime Library Exception, version
+; 3.1, as published by the Free Software Foundation.
+;
+; You should have received a copy of the GNU General Public License and
+; a copy of the GCC Runtime Library Exception along with this program;
+; see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+; <http://www.gnu.org/licenses/>.
+
+
+#include "vregs.h"
+
+    .text
+
+START_FUNC ___subdi3
+
+    movw  hl, sp       ; use HL-based addressing (allows for direct subw)
+
+    movw  ax, [hl+4]
+    subw  ax, [hl+12]
+    movw  r8, ax
+
+    mov   a, [hl+6]    ; middle bytes of the result are determined using 8-bit
+    subc  a, [hl+14]   ; SUBC insns which both account for and update the carry bit
+    mov   r10, a       ; (no SUBWC instruction is available)
+    mov   a, [hl+7]
+    subc  a, [hl+15]
+    mov   r11, a
+
+    mov   a, [hl+8]
+    subc  a, [hl+16]
+    mov   r12, a
+    mov   a, [hl+9]
+    subc  a, [hl+17]
+    mov   r13, a
+
+    movw  ax, [hl+10]
+    sknc               ; account for the possible carry from the
+    decw  ax           ; latest 8-bit operation
+    subw  ax, [hl+18]
+    movw  r14, ax
+
+    ret
+
+END_FUNC ___subdi3
diff --git a/libgcc/config/rl78/t-rl78 b/libgcc/config/rl78/t-rl78
index 6e48a856a7f..e035d58c09c 100644
--- a/libgcc/config/rl78/t-rl78
+++ b/libgcc/config/rl78/t-rl78
@@ -30,7 +30,9 @@ LIB2ADD = \
 	$(srcdir)/config/rl78/bit-count.S \
 	$(srcdir)/config/rl78/fpbit-sf.S \
 	$(srcdir)/config/rl78/fpmath-sf.S \
-	$(srcdir)/config/rl78/cmpsi2.S
+	$(srcdir)/config/rl78/cmpsi2.S \
+	$(srcdir)/config/rl78/adddi3.S \
+	$(srcdir)/config/rl78/subdi3.S
 
 LIB2FUNCS_EXCLUDE = _clzhi2 _clzsi2 _ctzhi2 _ctzsi2 \
   _popcounthi2 _popcountsi2 \
diff --git a/libgcc/libgcc2.c b/libgcc/libgcc2.c
index 5d3c69f16aa..83f865a04db 100644
--- a/libgcc/libgcc2.c
+++ b/libgcc/libgcc2.c
@@ -375,7 +375,8 @@ __mulvDI3 (DWtype u, DWtype v)
 		}
 	      else
 		{
-		  if (uu.s.high == (Wtype) -1 && vv.s.high == (Wtype) - 1)
+		  if ((uu.s.high & vv.s.high) == (Wtype) -1
+		      && (uu.s.low | vv.s.low) != 0)
 		    {
 		      DWunion ww = {.ll = (UDWtype) (UWtype) uu.s.low
 				    * (UDWtype) (UWtype) vv.s.low};
diff --git a/libgcc/unwind-dw2.c b/libgcc/unwind-dw2.c
index 3f26eaf73de..d514af9f607 100644
--- a/libgcc/unwind-dw2.c
+++ b/libgcc/unwind-dw2.c
@@ -216,12 +216,12 @@ _Unwind_IsExtendedContext (struct _Unwind_Context *context)
 	  || (context->flags & EXTENDED_CONTEXT_BIT));
 }
 
-/* Get the value of register INDEX as saved in CONTEXT.  */
+/* Get the value of register REGNO as saved in CONTEXT.  */
 
 inline _Unwind_Word
-_Unwind_GetGR (struct _Unwind_Context *context, int index)
+_Unwind_GetGR (struct _Unwind_Context *context, int regno)
 {
-  int size;
+  int size, index;
   _Unwind_Context_Reg_Val val;
 
 #ifdef DWARF_ZERO_REG
@@ -229,7 +229,7 @@ _Unwind_GetGR (struct _Unwind_Context *context, int index)
     return 0;
 #endif
 
-  index = DWARF_REG_TO_UNWIND_COLUMN (index);
+  index = DWARF_REG_TO_UNWIND_COLUMN (regno);
   gcc_assert (index < (int) sizeof(dwarf_reg_size_table));
   size = dwarf_reg_size_table[index];
   val = context->reg[index];
@@ -237,6 +237,14 @@ _Unwind_GetGR (struct _Unwind_Context *context, int index)
   if (_Unwind_IsExtendedContext (context) && context->by_value[index])
     return _Unwind_Get_Unwind_Word (val);
 
+#ifdef DWARF_LAZY_REGISTER_VALUE
+  {
+    _Unwind_Word value;
+    if (DWARF_LAZY_REGISTER_VALUE (regno, &value))
+      return value;
+  }
+#endif
+
   /* This will segfault if the register hasn't been saved.  */
   if (size == sizeof(_Unwind_Ptr))
     return * (_Unwind_Ptr *) (_Unwind_Internal_Ptr) val;
diff --git a/libgfortran/ChangeLog b/libgfortran/ChangeLog
index ef9ef19b68b..9e52ece6e15 100644
--- a/libgfortran/ChangeLog
+++ b/libgfortran/ChangeLog
@@ -1,3 +1,10 @@
+2017-10-27  Jerry DeLisle  <jvdelisle@gcc.gnu.org>
+	    Rimvydas (RJ)
+
+	PR libgfortran/81938
+	io/format.c (free_format_data): Don't try to free vlist
+	descriptors past the end of the fnode array.
+	
 2017-10-10  Thomas Koenig  <tkoenig@gcc.gnu.org>
 
 	PR libfortran/82233
diff --git a/libgfortran/io/format.c b/libgfortran/io/format.c
index bd14ac37bb3..c2abdd7b3af 100644
--- a/libgfortran/io/format.c
+++ b/libgfortran/io/format.c
@@ -266,7 +266,8 @@ free_format_data (format_data *fmt)
     return;
 
   /* Free vlist descriptors in the fnode_array if one was allocated.  */
-  for (fnp = fmt->array.array; fnp->format != FMT_NONE; fnp++)
+  for (fnp = fmt->array.array; fnp < &fmt->array.array[FARRAY_SIZE] &&
+       fnp->format != FMT_NONE; fnp++)
     if (fnp->format == FMT_DT)
 	{
 	  if (GFC_DESCRIPTOR_DATA(fnp->u.udf.vlist))
diff --git a/libgo/go/debug/dwarf/entry.go b/libgo/go/debug/dwarf/entry.go
index 80bf14cb22f..ffa61c28d15 100644
--- a/libgo/go/debug/dwarf/entry.go
+++ b/libgo/go/debug/dwarf/entry.go
@@ -33,13 +33,13 @@ type abbrevTable map[uint32]abbrev
 
 // ParseAbbrev returns the abbreviation table that starts at byte off
 // in the .debug_abbrev section.
-func (d *Data) parseAbbrev(off uint32, vers int) (abbrevTable, error) {
+func (d *Data) parseAbbrev(off uint64, vers int) (abbrevTable, error) {
 	if m, ok := d.abbrevCache[off]; ok {
 		return m, nil
 	}
 
 	data := d.abbrev
-	if off > uint32(len(data)) {
+	if off > uint64(len(data)) {
 		data = nil
 	} else {
 		data = data[off:]
diff --git a/libgo/go/debug/dwarf/entry_test.go b/libgo/go/debug/dwarf/entry_test.go
index 58a5d570be5..58f3023d296 100644
--- a/libgo/go/debug/dwarf/entry_test.go
+++ b/libgo/go/debug/dwarf/entry_test.go
@@ -135,3 +135,63 @@ func TestReaderRanges(t *testing.T) {
 		t.Errorf("saw only %d subprograms, expected %d", i, len(subprograms))
 	}
 }
+
+func Test64Bit(t *testing.T) {
+	// I don't know how to generate a 64-bit DWARF debug
+	// compilation unit except by using XCOFF, so this is
+	// hand-written.
+	tests := []struct {
+		name string
+		info []byte
+	}{
+		{
+			"32-bit little",
+			[]byte{0x30, 0, 0, 0, // comp unit length
+				4, 0, // DWARF version 4
+				0, 0, 0, 0, // abbrev offset
+				8, // address size
+				0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+			},
+		},
+		{
+			"64-bit little",
+			[]byte{0xff, 0xff, 0xff, 0xff, // 64-bit DWARF
+				0x30, 0, 0, 0, 0, 0, 0, 0, // comp unit length
+				4, 0, // DWARF version 4
+				0, 0, 0, 0, 0, 0, 0, 0, // abbrev offset
+				8, // address size
+				0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+			},
+		},
+		{
+			"64-bit big",
+			[]byte{0xff, 0xff, 0xff, 0xff, // 64-bit DWARF
+				0, 0, 0, 0, 0, 0, 0, 0x30, // comp unit length
+				0, 4, // DWARF version 4
+				0, 0, 0, 0, 0, 0, 0, 0, // abbrev offset
+				8, // address size
+				0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		_, err := New(nil, nil, nil, test.info, nil, nil, nil, nil)
+		if err != nil {
+			t.Errorf("%s: %v", test.name, err)
+		}
+	}
+}
diff --git a/libgo/go/debug/dwarf/open.go b/libgo/go/debug/dwarf/open.go
index 253a43cf579..938186998ff 100644
--- a/libgo/go/debug/dwarf/open.go
+++ b/libgo/go/debug/dwarf/open.go
@@ -23,7 +23,7 @@ type Data struct {
 	str      []byte
 
 	// parsed data
-	abbrevCache map[uint32]abbrevTable
+	abbrevCache map[uint64]abbrevTable
 	order       binary.ByteOrder
 	typeCache   map[Offset]Type
 	typeSigs    map[uint64]*typeUnit
@@ -48,17 +48,26 @@ func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Dat
 		pubnames:    pubnames,
 		ranges:      ranges,
 		str:         str,
-		abbrevCache: make(map[uint32]abbrevTable),
+		abbrevCache: make(map[uint64]abbrevTable),
 		typeCache:   make(map[Offset]Type),
 		typeSigs:    make(map[uint64]*typeUnit),
 	}
 
 	// Sniff .debug_info to figure out byte order.
-	// bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3).
+	// 32-bit DWARF: 4 byte length, 2 byte version.
+	// 64-bit DWARf: 4 bytes of 0xff, 8 byte length, 2 byte version.
 	if len(d.info) < 6 {
 		return nil, DecodeError{"info", Offset(len(d.info)), "too short"}
 	}
-	x, y := d.info[4], d.info[5]
+	offset := 4
+	if d.info[0] == 0xff && d.info[1] == 0xff && d.info[2] == 0xff && d.info[3] == 0xff {
+		if len(d.info) < 14 {
+			return nil, DecodeError{"info", Offset(len(d.info)), "too short"}
+		}
+		offset = 12
+	}
+	// Fetch the version, a tiny 16-bit number (1, 2, 3, 4, 5).
+	x, y := d.info[offset], d.info[offset+1]
 	switch {
 	case x == 0 && y == 0:
 		return nil, DecodeError{"info", 4, "unsupported version 0"}
diff --git a/libgo/go/debug/dwarf/typeunit.go b/libgo/go/debug/dwarf/typeunit.go
index 652e02d9172..76b357ce28b 100644
--- a/libgo/go/debug/dwarf/typeunit.go
+++ b/libgo/go/debug/dwarf/typeunit.go
@@ -38,16 +38,11 @@ func (d *Data) parseTypes(name string, types []byte) error {
 			b.error("unsupported DWARF version " + strconv.Itoa(vers))
 			return b.err
 		}
-		var ao uint32
+		var ao uint64
 		if !dwarf64 {
-			ao = b.uint32()
+			ao = uint64(b.uint32())
 		} else {
-			ao64 := b.uint64()
-			if ao64 != uint64(uint32(ao64)) {
-				b.error("type unit abbrev offset overflow")
-				return b.err
-			}
-			ao = uint32(ao64)
+			ao = b.uint64()
 		}
 		atable, err := d.parseAbbrev(ao, vers)
 		if err != nil {
diff --git a/libgo/go/debug/dwarf/unit.go b/libgo/go/debug/dwarf/unit.go
index e45aed7ad1d..98024ca1f84 100644
--- a/libgo/go/debug/dwarf/unit.go
+++ b/libgo/go/debug/dwarf/unit.go
@@ -61,13 +61,20 @@ func (d *Data) parseUnits() ([]unit, error) {
 		u.base = b.off
 		var n Offset
 		n, u.is64 = b.unitLength()
+		dataOff := b.off
 		vers := b.uint16()
 		if vers != 2 && vers != 3 && vers != 4 {
 			b.error("unsupported DWARF version " + strconv.Itoa(int(vers)))
 			break
 		}
 		u.vers = int(vers)
-		atable, err := d.parseAbbrev(b.uint32(), u.vers)
+		var abbrevOff uint64
+		if u.is64 {
+			abbrevOff = b.uint64()
+		} else {
+			abbrevOff = uint64(b.uint32())
+		}
+		atable, err := d.parseAbbrev(abbrevOff, u.vers)
 		if err != nil {
 			if b.err == nil {
 				b.err = err
@@ -77,7 +84,7 @@ func (d *Data) parseUnits() ([]unit, error) {
 		u.atable = atable
 		u.asize = int(b.uint8())
 		u.off = b.off
-		u.data = b.bytes(int(n - (2 + 4 + 1)))
+		u.data = b.bytes(int(n - (b.off - dataOff)))
 	}
 	if b.err != nil {
 		return nil, b.err
diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog
index 1c5c5a9a912..f68604ca424 100644
--- a/libgomp/ChangeLog
+++ b/libgomp/ChangeLog
@@ -1,3 +1,48 @@
+2017-10-31  Tom de Vries  <tom@codesourcery.com>
+
+	* plugin/plugin-hsa.c (HSA_LOG): Remove semicolon after
+	"do {} while (false)".
+	(init_single_kernel, GOMP_OFFLOAD_async_run): Add missing semicolon
+	after HSA_DEBUG call.
+
+2017-10-28  Jakub Jelinek  <jakub@redhat.com>
+
+	* target.c (struct gomp_coalesce_buf): New type.
+	(MAX_COALESCE_BUF_SIZE, MAX_COALESCE_BUF_GAP): Define.
+	(gomp_coalesce_buf_add, gomp_to_device_kind_p): New functions.
+	(gomp_copy_host2dev): Add CBUF argument, if copying into
+	the cached ranges, memcpy into buffer instead of copying
+	into device.
+	(gomp_map_vars_existing, gomp_map_pointer, gomp_map_fields_existing):
+	Add CBUF argument, pass it through to other calls.
+	(gomp_map_vars): Aggregate copies from host to device if small enough
+	and with small enough gaps in between into memcpy into a buffer and
+	fewer host to device copies from the buffer.
+	(gomp_update): Adjust gomp_copy_host2dev caller.
+
+2017-10-17  Thomas Schwinge  <thomas@codesourcery.com>
+
+	* testsuite/libgomp.oacc-fortran/declare-1.f90: Restore "dg-do
+	run" directive.
+	* testsuite/libgomp.oacc-fortran/declare-2.f90: Likewise.
+	* testsuite/libgomp.oacc-fortran/declare-3.f90: Likewise.
+	* testsuite/libgomp.oacc-fortran/declare-4.f90: Likewise.
+	* testsuite/libgomp.oacc-fortran/declare-5.f90: Likewise.
+
+2017-10-16  Tom de Vries  <tom@codesourcery.com>
+
+	* testsuite/libgomp.oacc-c-c++-common/declare-1.c: Don't require
+	openacc_nvidia_accel_selected.
+	* testsuite/libgomp.oacc-c-c++-common/declare-2.c: Same.
+	* testsuite/libgomp.oacc-c-c++-common/declare-4.c: Same.
+	* testsuite/libgomp.oacc-fortran/declare-2.f90: Same.
+	* testsuite/libgomp.oacc-fortran/declare-4.f90: Same
+	* testsuite/libgomp.oacc-fortran/declare-5.f90: Same.
+	* testsuite/libgomp.oacc-c-c++-common/declare-5.c: Don't require
+	openacc_nvidia_accel_selected. Skip for shared memory device.
+	* testsuite/libgomp.oacc-fortran/declare-1.f90: Same.
+	* testsuite/libgomp.oacc-fortran/declare-3.f90: Same.
+
 2017-10-09  Martin Jambor  <mjambor@suse.cz>
 
 	PR hsa/82416
diff --git a/libgomp/plugin/plugin-hsa.c b/libgomp/plugin/plugin-hsa.c
index fc08f5df058..cf3a4d5142d 100644
--- a/libgomp/plugin/plugin-hsa.c
+++ b/libgomp/plugin/plugin-hsa.c
@@ -261,7 +261,7 @@ init_enviroment_variables (void)
 	fprintf (stderr, __VA_ARGS__); \
       } \
   } \
-  while (false);
+  while (false)
 
 /* Print a debugging message to stderr.  */
 
@@ -1241,7 +1241,7 @@ init_single_kernel (struct kernel_info *kernel, unsigned *max_omp_data_size)
       if (dependency->dependencies_count > 0)
 	{
 	  HSA_DEBUG ("HSA does not allow kernel dispatching code with "
-		     "a depth bigger than one\n")
+		     "a depth bigger than one\n");
 	  goto failure;
 	}
 
@@ -1664,7 +1664,7 @@ GOMP_OFFLOAD_async_run (int device, void *tgt_fn, void *tgt_vars,
 {
   pthread_t pt;
   struct async_run_info *info;
-  HSA_DEBUG ("GOMP_OFFLOAD_async_run invoked\n")
+  HSA_DEBUG ("GOMP_OFFLOAD_async_run invoked\n");
   info = GOMP_PLUGIN_malloc (sizeof (struct async_run_info));
 
   info->device = device;
diff --git a/libgomp/target.c b/libgomp/target.c
index 3dd119f52e5..8ac05e8c641 100644
--- a/libgomp/target.c
+++ b/libgomp/target.c
@@ -177,10 +177,122 @@ gomp_device_copy (struct gomp_device_descr *devicep,
     }
 }
 
+/* Infrastructure for coalescing adjacent or nearly adjacent (in device addresses)
+   host to device memory transfers.  */
+
+struct gomp_coalesce_buf
+{
+  /* Buffer into which gomp_copy_host2dev will memcpy data and from which
+     it will be copied to the device.  */
+  void *buf;
+  struct target_mem_desc *tgt;
+  /* Array with offsets, chunks[2 * i] is the starting offset and
+     chunks[2 * i + 1] ending offset relative to tgt->tgt_start device address
+     of chunks which are to be copied to buf and later copied to device.  */
+  size_t *chunks;
+  /* Number of chunks in chunks array, or -1 if coalesce buffering should not
+     be performed.  */
+  long chunk_cnt;
+  /* During construction of chunks array, how many memory regions are within
+     the last chunk.  If there is just one memory region for a chunk, we copy
+     it directly to device rather than going through buf.  */
+  long use_cnt;
+};
+
+/* Maximum size of memory region considered for coalescing.  Larger copies
+   are performed directly.  */
+#define MAX_COALESCE_BUF_SIZE	(32 * 1024)
+
+/* Maximum size of a gap in between regions to consider them being copied
+   within the same chunk.  All the device offsets considered are within
+   newly allocated device memory, so it isn't fatal if we copy some padding
+   in between from host to device.  The gaps come either from alignment
+   padding or from memory regions which are not supposed to be copied from
+   host to device (e.g. map(alloc:), map(from:) etc.).  */
+#define MAX_COALESCE_BUF_GAP	(4 * 1024)
+
+/* Add region with device tgt_start relative offset and length to CBUF.  */
+
+static inline void
+gomp_coalesce_buf_add (struct gomp_coalesce_buf *cbuf, size_t start, size_t len)
+{
+  if (len > MAX_COALESCE_BUF_SIZE || len == 0)
+    return;
+  if (cbuf->chunk_cnt)
+    {
+      if (cbuf->chunk_cnt < 0)
+	return;
+      if (start < cbuf->chunks[2 * cbuf->chunk_cnt - 1])
+	{
+	  cbuf->chunk_cnt = -1;
+	  return;
+	}
+      if (start < cbuf->chunks[2 * cbuf->chunk_cnt - 1] + MAX_COALESCE_BUF_GAP)
+	{
+	  cbuf->chunks[2 * cbuf->chunk_cnt - 1] = start + len;
+	  cbuf->use_cnt++;
+	  return;
+	}
+      /* If the last chunk is only used by one mapping, discard it,
+	 as it will be one host to device copy anyway and
+	 memcpying it around will only waste cycles.  */
+      if (cbuf->use_cnt == 1)
+	cbuf->chunk_cnt--;
+    }
+  cbuf->chunks[2 * cbuf->chunk_cnt] = start;
+  cbuf->chunks[2 * cbuf->chunk_cnt + 1] = start + len;
+  cbuf->chunk_cnt++;
+  cbuf->use_cnt = 1;
+}
+
+/* Return true for mapping kinds which need to copy data from the
+   host to device for regions that weren't previously mapped.  */
+
+static inline bool
+gomp_to_device_kind_p (int kind)
+{
+  switch (kind)
+    {
+    case GOMP_MAP_ALLOC:
+    case GOMP_MAP_FROM:
+    case GOMP_MAP_FORCE_ALLOC:
+    case GOMP_MAP_ALWAYS_FROM:
+      return false;
+    default:
+      return true;
+    }
+}
+
 static void
 gomp_copy_host2dev (struct gomp_device_descr *devicep,
-		    void *d, const void *h, size_t sz)
+		    void *d, const void *h, size_t sz,
+		    struct gomp_coalesce_buf *cbuf)
 {
+  if (cbuf)
+    {
+      uintptr_t doff = (uintptr_t) d - cbuf->tgt->tgt_start;
+      if (doff < cbuf->chunks[2 * cbuf->chunk_cnt - 1])
+	{
+	  long first = 0;
+	  long last = cbuf->chunk_cnt - 1;
+	  while (first <= last)
+	    {
+	      long middle = (first + last) >> 1;
+	      if (cbuf->chunks[2 * middle + 1] <= doff)
+		first = middle + 1;
+	      else if (cbuf->chunks[2 * middle] <= doff)
+		{
+		  if (doff + sz > cbuf->chunks[2 * middle + 1])
+		    gomp_fatal ("internal libgomp cbuf error");
+		  memcpy ((char *) cbuf->buf + (doff - cbuf->chunks[0]),
+			  h, sz);
+		  return;
+		}
+	      else
+		last = middle - 1;
+	    }
+	}
+    }
   gomp_device_copy (devicep, devicep->host2dev_func, "dev", d, "host", h, sz);
 }
 
@@ -208,7 +320,7 @@ gomp_free_device_memory (struct gomp_device_descr *devicep, void *devptr)
 static inline void
 gomp_map_vars_existing (struct gomp_device_descr *devicep, splay_tree_key oldn,
 			splay_tree_key newn, struct target_var_desc *tgt_var,
-			unsigned char kind)
+			unsigned char kind, struct gomp_coalesce_buf *cbuf)
 {
   tgt_var->key = oldn;
   tgt_var->copy_from = GOMP_MAP_COPY_FROM_P (kind);
@@ -232,7 +344,7 @@ gomp_map_vars_existing (struct gomp_device_descr *devicep, splay_tree_key oldn,
 			(void *) (oldn->tgt->tgt_start + oldn->tgt_offset
 				  + newn->host_start - oldn->host_start),
 			(void *) newn->host_start,
-			newn->host_end - newn->host_start);
+			newn->host_end - newn->host_start, cbuf);
 
   if (oldn->refcount != REFCOUNT_INFINITY)
     oldn->refcount++;
@@ -247,7 +359,8 @@ get_kind (bool short_mapkind, void *kinds, int idx)
 
 static void
 gomp_map_pointer (struct target_mem_desc *tgt, uintptr_t host_ptr,
-		  uintptr_t target_offset, uintptr_t bias)
+		  uintptr_t target_offset, uintptr_t bias,
+		  struct gomp_coalesce_buf *cbuf)
 {
   struct gomp_device_descr *devicep = tgt->device_descr;
   struct splay_tree_s *mem_map = &devicep->mem_map;
@@ -257,11 +370,10 @@ gomp_map_pointer (struct target_mem_desc *tgt, uintptr_t host_ptr,
   if (cur_node.host_start == (uintptr_t) NULL)
     {
       cur_node.tgt_offset = (uintptr_t) NULL;
-      /* FIXME: see comment about coalescing host/dev transfers below.  */
       gomp_copy_host2dev (devicep,
 			  (void *) (tgt->tgt_start + target_offset),
 			  (void *) &cur_node.tgt_offset,
-			  sizeof (void *));
+			  sizeof (void *), cbuf);
       return;
     }
   /* Add bias to the pointer value.  */
@@ -280,15 +392,15 @@ gomp_map_pointer (struct target_mem_desc *tgt, uintptr_t host_ptr,
      array section.  Now subtract bias to get what we want
      to initialize the pointer with.  */
   cur_node.tgt_offset -= bias;
-  /* FIXME: see comment about coalescing host/dev transfers below.  */
   gomp_copy_host2dev (devicep, (void *) (tgt->tgt_start + target_offset),
-		      (void *) &cur_node.tgt_offset, sizeof (void *));
+		      (void *) &cur_node.tgt_offset, sizeof (void *), cbuf);
 }
 
 static void
 gomp_map_fields_existing (struct target_mem_desc *tgt, splay_tree_key n,
 			  size_t first, size_t i, void **hostaddrs,
-			  size_t *sizes, void *kinds)
+			  size_t *sizes, void *kinds,
+			  struct gomp_coalesce_buf *cbuf)
 {
   struct gomp_device_descr *devicep = tgt->device_descr;
   struct splay_tree_s *mem_map = &devicep->mem_map;
@@ -306,7 +418,7 @@ gomp_map_fields_existing (struct target_mem_desc *tgt, splay_tree_key n,
       && n2->host_start - n->host_start == n2->tgt_offset - n->tgt_offset)
     {
       gomp_map_vars_existing (devicep, n2, &cur_node,
-			      &tgt->list[i], kind & typemask);
+			      &tgt->list[i], kind & typemask, cbuf);
       return;
     }
   if (sizes[i] == 0)
@@ -322,7 +434,7 @@ gomp_map_fields_existing (struct target_mem_desc *tgt, splay_tree_key n,
 		 == n2->tgt_offset - n->tgt_offset)
 	    {
 	      gomp_map_vars_existing (devicep, n2, &cur_node, &tgt->list[i],
-				      kind & typemask);
+				      kind & typemask, cbuf);
 	      return;
 	    }
 	}
@@ -334,7 +446,7 @@ gomp_map_fields_existing (struct target_mem_desc *tgt, splay_tree_key n,
 	  && n2->host_start - n->host_start == n2->tgt_offset - n->tgt_offset)
 	{
 	  gomp_map_vars_existing (devicep, n2, &cur_node, &tgt->list[i],
-				  kind & typemask);
+				  kind & typemask, cbuf);
 	  return;
 	}
     }
@@ -381,6 +493,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
   tgt->list_count = mapnum;
   tgt->refcount = pragma_kind == GOMP_MAP_VARS_ENTER_DATA ? 0 : 1;
   tgt->device_descr = devicep;
+  struct gomp_coalesce_buf cbuf, *cbufp = NULL;
 
   if (mapnum == 0)
     {
@@ -391,11 +504,25 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 
   tgt_align = sizeof (void *);
   tgt_size = 0;
+  cbuf.chunks = NULL;
+  cbuf.chunk_cnt = -1;
+  cbuf.use_cnt = 0;
+  cbuf.buf = NULL;
+  if (mapnum > 1 || pragma_kind == GOMP_MAP_VARS_TARGET)
+    {
+      cbuf.chunks
+	= (size_t *) gomp_alloca ((2 * mapnum + 2) * sizeof (size_t));
+      cbuf.chunk_cnt = 0;
+    }
   if (pragma_kind == GOMP_MAP_VARS_TARGET)
     {
       size_t align = 4 * sizeof (void *);
       tgt_align = align;
       tgt_size = mapnum * sizeof (void *);
+      cbuf.chunk_cnt = 1;
+      cbuf.use_cnt = 1 + (mapnum > 1);
+      cbuf.chunks[0] = 0;
+      cbuf.chunks[1] = tgt_size;
     }
 
   gomp_mutex_lock (&devicep->lock);
@@ -449,19 +576,26 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 	      size_t align = (size_t) 1 << (kind >> rshift);
 	      if (tgt_align < align)
 		tgt_align = align;
-	      tgt_size -= (uintptr_t) hostaddrs[first]
-			  - (uintptr_t) hostaddrs[i];
+	      tgt_size -= (uintptr_t) hostaddrs[first] - cur_node.host_start;
 	      tgt_size = (tgt_size + align - 1) & ~(align - 1);
-	      tgt_size += cur_node.host_end - (uintptr_t) hostaddrs[i];
+	      tgt_size += cur_node.host_end - cur_node.host_start;
 	      not_found_cnt += last - i;
 	      for (i = first; i <= last; i++)
-		tgt->list[i].key = NULL;
+		{
+		  tgt->list[i].key = NULL;
+		  if (gomp_to_device_kind_p (get_kind (short_mapkind, kinds, i)
+					     & typemask))
+		    gomp_coalesce_buf_add (&cbuf,
+					   tgt_size - cur_node.host_end
+					   + (uintptr_t) hostaddrs[i],
+					   sizes[i]);
+		}
 	      i--;
 	      continue;
 	    }
 	  for (i = first; i <= last; i++)
 	    gomp_map_fields_existing (tgt, n, first, i, hostaddrs,
-				      sizes, kinds);
+				      sizes, kinds, NULL);
 	  i--;
 	  continue;
 	}
@@ -485,6 +619,8 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 	  if (tgt_align < align)
 	    tgt_align = align;
 	  tgt_size = (tgt_size + align - 1) & ~(align - 1);
+	  gomp_coalesce_buf_add (&cbuf, tgt_size,
+				 cur_node.host_end - cur_node.host_start);
 	  tgt_size += cur_node.host_end - cur_node.host_start;
 	  has_firstprivate = true;
 	  continue;
@@ -504,7 +640,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 	n = splay_tree_lookup (mem_map, &cur_node);
       if (n && n->refcount != REFCOUNT_LINK)
 	gomp_map_vars_existing (devicep, n, &cur_node, &tgt->list[i],
-				kind & typemask);
+				kind & typemask, NULL);
       else
 	{
 	  tgt->list[i].key = NULL;
@@ -514,6 +650,9 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 	  if (tgt_align < align)
 	    tgt_align = align;
 	  tgt_size = (tgt_size + align - 1) & ~(align - 1);
+	  if (gomp_to_device_kind_p (kind & typemask))
+	    gomp_coalesce_buf_add (&cbuf, tgt_size,
+				   cur_node.host_end - cur_node.host_start);
 	  tgt_size += cur_node.host_end - cur_node.host_start;
 	  if ((kind & typemask) == GOMP_MAP_TO_PSET)
 	    {
@@ -562,6 +701,19 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
       tgt->tgt_start = (uintptr_t) tgt->to_free;
       tgt->tgt_start = (tgt->tgt_start + tgt_align - 1) & ~(tgt_align - 1);
       tgt->tgt_end = tgt->tgt_start + tgt_size;
+
+      if (cbuf.use_cnt == 1)
+	cbuf.chunk_cnt--;
+      if (cbuf.chunk_cnt > 0)
+	{
+	  cbuf.buf
+	    = malloc (cbuf.chunks[2 * cbuf.chunk_cnt - 1] - cbuf.chunks[0]);
+	  if (cbuf.buf)
+	    {
+	      cbuf.tgt = tgt;
+	      cbufp = &cbuf;
+	    }
+	}
     }
   else
     {
@@ -600,7 +752,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 		len = sizes[i];
 		gomp_copy_host2dev (devicep,
 				    (void *) (tgt->tgt_start + tgt_size),
-				    (void *) hostaddrs[i], len);
+				    (void *) hostaddrs[i], len, cbufp);
 		tgt_size += len;
 		continue;
 	      case GOMP_MAP_FIRSTPRIVATE_INT:
@@ -633,7 +785,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 		  }
 		for (i = first; i <= last; i++)
 		  gomp_map_fields_existing (tgt, n, first, i, hostaddrs,
-					    sizes, kinds);
+					    sizes, kinds, cbufp);
 		i--;
 		continue;
 	      case GOMP_MAP_ALWAYS_POINTER:
@@ -658,7 +810,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 					      + cur_node.host_start
 					      - n->host_start),
 				    (void *) &cur_node.tgt_offset,
-				    sizeof (void *));
+				    sizeof (void *), cbufp);
 		cur_node.tgt_offset = n->tgt->tgt_start + n->tgt_offset
 				      + cur_node.host_start - n->host_start;
 		continue;
@@ -674,7 +826,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 	    splay_tree_key n = splay_tree_lookup (mem_map, k);
 	    if (n && n->refcount != REFCOUNT_LINK)
 	      gomp_map_vars_existing (devicep, n, k, &tgt->list[i],
-				      kind & typemask);
+				      kind & typemask, cbufp);
 	    else
 	      {
 		k->link_key = NULL;
@@ -725,26 +877,22 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 		  case GOMP_MAP_FORCE_TOFROM:
 		  case GOMP_MAP_ALWAYS_TO:
 		  case GOMP_MAP_ALWAYS_TOFROM:
-		    /* FIXME: Perhaps add some smarts, like if copying
-		       several adjacent fields from host to target, use some
-		       host buffer to avoid sending each var individually.  */
 		    gomp_copy_host2dev (devicep,
 					(void *) (tgt->tgt_start
 						  + k->tgt_offset),
 					(void *) k->host_start,
-					k->host_end - k->host_start);
+					k->host_end - k->host_start, cbufp);
 		    break;
 		  case GOMP_MAP_POINTER:
 		    gomp_map_pointer (tgt, (uintptr_t) *(void **) k->host_start,
-				      k->tgt_offset, sizes[i]);
+				      k->tgt_offset, sizes[i], cbufp);
 		    break;
 		  case GOMP_MAP_TO_PSET:
-		    /* FIXME: see above FIXME comment.  */
 		    gomp_copy_host2dev (devicep,
 					(void *) (tgt->tgt_start
 						  + k->tgt_offset),
 					(void *) k->host_start,
-					k->host_end - k->host_start);
+					k->host_end - k->host_start, cbufp);
 
 		    for (j = i + 1; j < mapnum; j++)
 		      if (!GOMP_MAP_POINTER_P (get_kind (short_mapkind, kinds,
@@ -767,7 +915,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 					    k->tgt_offset
 					    + ((uintptr_t) hostaddrs[j]
 					       - k->host_start),
-					    sizes[j]);
+					    sizes[j], cbufp);
 			  i++;
 			}
 		    break;
@@ -795,7 +943,7 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
 					(void *) (tgt->tgt_start
 						  + k->tgt_offset),
 					(void *) k->host_start,
-					sizeof (void *));
+					sizeof (void *), cbufp);
 		    break;
 		  default:
 		    gomp_mutex_unlock (&devicep->lock);
@@ -822,13 +970,23 @@ gomp_map_vars (struct gomp_device_descr *devicep, size_t mapnum,
       for (i = 0; i < mapnum; i++)
 	{
 	  cur_node.tgt_offset = gomp_map_val (tgt, hostaddrs, i);
-	  /* FIXME: see above FIXME comment.  */
 	  gomp_copy_host2dev (devicep,
 			      (void *) (tgt->tgt_start + i * sizeof (void *)),
-			      (void *) &cur_node.tgt_offset, sizeof (void *));
+			      (void *) &cur_node.tgt_offset, sizeof (void *),
+			      cbufp);
 	}
     }
 
+  if (cbufp)
+    {
+      long c = 0;
+      for (c = 0; c < cbuf.chunk_cnt; ++c)
+	gomp_copy_host2dev (devicep, (void *) (tgt->tgt_start + cbuf.chunks[2 * c]),
+			    (char *) cbuf.buf + (cbuf.chunks[2 * c] - cbuf.chunks[0]),
+			    cbuf.chunks[2 * c + 1] - cbuf.chunks[2 * c], NULL);
+      free (cbuf.buf);
+    }
+
   /* If the variable from "omp target enter data" map-list was already mapped,
      tgt is not needed.  Otherwise tgt will be freed by gomp_unmap_vars or
      gomp_exit_data.  */
@@ -970,7 +1128,7 @@ gomp_update (struct gomp_device_descr *devicep, size_t mapnum, void **hostaddrs,
 	    size_t size = cur_node.host_end - cur_node.host_start;
 
 	    if (GOMP_MAP_COPY_TO_P (kind & typemask))
-	      gomp_copy_host2dev (devicep, devaddr, hostaddr, size);
+	      gomp_copy_host2dev (devicep, devaddr, hostaddr, size, NULL);
 	    if (GOMP_MAP_COPY_FROM_P (kind & typemask))
 	      gomp_copy_dev2host (devicep, hostaddr, devaddr, size);
 	  }
diff --git a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-1.c b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-1.c
index c63a68dbab7..bc726174252 100644
--- a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-1.c
+++ b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-1.c
@@ -1,5 +1,3 @@
-/* { dg-do run { target openacc_nvidia_accel_selected } } */
-
 #include <openacc.h>
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-2.c b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-2.c
index 2078a33afa9..d212458dada 100644
--- a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-2.c
+++ b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-2.c
@@ -1,5 +1,3 @@
-/* { dg-do run { target openacc_nvidia_accel_selected } } */
-
 #include <stdlib.h>
 
 #define N 16
diff --git a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-4.c b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-4.c
index 36bf0ebdd0b..ca48e801314 100644
--- a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-4.c
+++ b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-4.c
@@ -1,5 +1,3 @@
-/* { dg-do run  { target openacc_nvidia_accel_selected } } */
-
 #include <stdlib.h>
 #include <openacc.h>
 
diff --git a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-5.c b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-5.c
index 38c5de063d9..229e96c08a0 100644
--- a/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-5.c
+++ b/libgomp/testsuite/libgomp.oacc-c-c++-common/declare-5.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target openacc_nvidia_accel_selected } } */
+/* { dg-skip-if "" { *-*-* } { "-DACC_MEM_SHARED=1" } } */
 
 #include <stdio.h>
 
diff --git a/libgomp/testsuite/libgomp.oacc-fortran/declare-1.f90 b/libgomp/testsuite/libgomp.oacc-fortran/declare-1.f90
index 2d4b70720e3..b502df42a28 100644
--- a/libgomp/testsuite/libgomp.oacc-fortran/declare-1.f90
+++ b/libgomp/testsuite/libgomp.oacc-fortran/declare-1.f90
@@ -1,4 +1,5 @@
-! { dg-do run  { target openacc_nvidia_accel_selected } }
+! { dg-do run }
+! { dg-skip-if "" { *-*-* } { "-DACC_MEM_SHARED=1" } }
 
 ! Tests to exercise the declare directive along with
 ! the clauses: copy
diff --git a/libgomp/testsuite/libgomp.oacc-fortran/declare-2.f90 b/libgomp/testsuite/libgomp.oacc-fortran/declare-2.f90
index 2aa79079d91..0e759dda98e 100644
--- a/libgomp/testsuite/libgomp.oacc-fortran/declare-2.f90
+++ b/libgomp/testsuite/libgomp.oacc-fortran/declare-2.f90
@@ -1,4 +1,4 @@
-! { dg-do run  { target openacc_nvidia_accel_selected } }
+! { dg-do run }
 
 module globalvars
   implicit none
diff --git a/libgomp/testsuite/libgomp.oacc-fortran/declare-3.f90 b/libgomp/testsuite/libgomp.oacc-fortran/declare-3.f90
index 3a6b420f1c7..16164cd0ac6 100644
--- a/libgomp/testsuite/libgomp.oacc-fortran/declare-3.f90
+++ b/libgomp/testsuite/libgomp.oacc-fortran/declare-3.f90
@@ -1,4 +1,5 @@
-! { dg-do run  { target openacc_nvidia_accel_selected } }
+! { dg-do run }
+! { dg-skip-if "" { *-*-* } { "-DACC_MEM_SHARED=1" } }
 
 module globalvars
   implicit none
diff --git a/libgomp/testsuite/libgomp.oacc-fortran/declare-4.f90 b/libgomp/testsuite/libgomp.oacc-fortran/declare-4.f90
index 226264e38c1..6c4e7c515ce 100644
--- a/libgomp/testsuite/libgomp.oacc-fortran/declare-4.f90
+++ b/libgomp/testsuite/libgomp.oacc-fortran/declare-4.f90
@@ -1,4 +1,4 @@
-! { dg-do run  { target openacc_nvidia_accel_selected } }
+! { dg-do run }
 
 module vars
   implicit none
diff --git a/libgomp/testsuite/libgomp.oacc-fortran/declare-5.f90 b/libgomp/testsuite/libgomp.oacc-fortran/declare-5.f90
index bcd9c9c72b5..4f5c8f00120 100644
--- a/libgomp/testsuite/libgomp.oacc-fortran/declare-5.f90
+++ b/libgomp/testsuite/libgomp.oacc-fortran/declare-5.f90
@@ -1,4 +1,4 @@
-! { dg-do run  { target openacc_nvidia_accel_selected } }
+! { dg-do run }
 
 module vars
   implicit none
diff --git a/libiberty/ChangeLog b/libiberty/ChangeLog
index ecbf3e7d41d..c8e1e137ab7 100644
--- a/libiberty/ChangeLog
+++ b/libiberty/ChangeLog
@@ -1,3 +1,31 @@
+2017-10-30  Richard Biener  <rguenther@suse.de>
+
+	PR lto/82757
+	* simple-object-elf.c (simple_object_elf_copy_lto_debug_sections):
+	Strip two leading _s from the __gnu_lto_* symbols.
+
+2017-10-24  Alan Modra  <amodra@gmail.com>
+
+	PR lto/82687
+	PR lto/82575
+	* simple-object-elf.c (simple_object_elf_copy_lto_debug_sections):
+	Only make __gnu_lto symbols hidden.
+
+2017-10-20  Alan Modra  <amodra@gmail.com>
+
+	PR lto/82575
+	* simple-object-elf.c (simple_object_elf_copy_lto_debug_sections):
+	Make discarded non-local symbols weak and hidden.
+
+2017-10-18  Jakub Jelinek  <jakub@redhat.com>
+
+	PR lto/82598
+	* simple-object.c (handle_lto_debug_sections): Copy over also
+	.note.GNU-stack section with unchanged name.
+	* simple-object-elf.c (SHF_EXECINSTR): Define.
+	(simple_object_elf_copy_lto_debug_section): Drop SHF_EXECINSTR bit
+	on .note.GNU-stack section.
+
 2017-09-25  Nathan Sidwell  <nathan@acm.org>
 
 	PR demangler/82195
diff --git a/libiberty/simple-object-elf.c b/libiberty/simple-object-elf.c
index 7eb3df85668..27431db98c7 100644
--- a/libiberty/simple-object-elf.c
+++ b/libiberty/simple-object-elf.c
@@ -196,6 +196,7 @@ typedef struct {
 
 /* Values for sh_flags field.  */
 
+#define SHF_EXECINSTR	0x00000004	/* Executable section.  */
 #define SHF_EXCLUDE	0x80000000	/* Link editor is to exclude this
 					   section from executable and
 					   shared library that it builds
@@ -235,8 +236,10 @@ typedef struct
 
 #define STB_LOCAL	0	/* Local symbol */
 #define STB_GLOBAL	1	/* Global symbol */
+#define STB_WEAK	2	/* Weak global */
 
 #define STV_DEFAULT	0	/* Visibility is specified by binding type */
+#define STV_HIDDEN	2	/* Can only be seen inside currect component */
 
 /* Functions to fetch and store different ELF types, depending on the
    endianness and size.  */
@@ -1085,6 +1088,7 @@ simple_object_elf_copy_lto_debug_sections (simple_object_read *sobj,
   off_t shstroff;
   unsigned char *names;
   unsigned int i;
+  int changed;
   int *pfnret;
   const char **pfnname;
 
@@ -1158,7 +1162,6 @@ simple_object_elf_copy_lto_debug_sections (simple_object_read *sobj,
 
   /* Mark sections as preserved that are required by to be preserved
      sections.  */
-  int changed;
   do
     {
       changed = 0;
@@ -1346,9 +1349,6 @@ simple_object_elf_copy_lto_debug_sections (simple_object_read *sobj,
 		     and __gnu_lto_slim which otherwise cause endless
 		     LTO plugin invocation.  */
 		  if (st_shndx == SHN_COMMON)
-		    /* Setting st_name to "" seems to work to purge
-		       COMMON symbols (in addition to setting their
-		       size to zero).  */
 		    discard = 1;
 		  /* We also need to remove symbols refering to sections
 		     we'll eventually remove as with fat LTO objects
@@ -1364,18 +1364,42 @@ simple_object_elf_copy_lto_debug_sections (simple_object_read *sobj,
 		    {
 		      /* Make discarded symbols undefined and unnamed
 		         in case it is local.  */
-		      if (ELF_ST_BIND (*st_info) == STB_LOCAL)
+		      int bind = ELF_ST_BIND (*st_info);
+		      int other = STV_DEFAULT;
+		      size_t st_name;
+
+		      if (bind == STB_LOCAL)
 			ELF_SET_FIELD (type_functions, ei_class, Sym,
 				       ent, st_name, Elf_Word, 0);
+		      else
+			{
+			  bind = STB_WEAK;
+			  st_name = ELF_FETCH_FIELD (type_functions, ei_class,
+						     Sym, ent, st_name,
+						     Elf_Word);
+			  if (st_name < strsz)
+			    {
+			      char *p = strings + st_name;
+			      if (p[0] == '_'
+				  && p[1] == '_'
+				  && strncmp (p + (p[2] == '_'),
+					      "__gnu_lto_", 10) == 0)
+				{
+				  other = STV_HIDDEN;
+				  ELF_SET_FIELD (type_functions, ei_class, Sym,
+						 ent, st_name, Elf_Word,
+						 st_name + 2);
+				}
+			    }
+			}
+		      *st_other = other;
+		      *st_info = ELF_ST_INFO (bind, STT_NOTYPE);
 		      ELF_SET_FIELD (type_functions, ei_class, Sym,
 				     ent, st_value, Elf_Addr, 0);
 		      ELF_SET_FIELD (type_functions, ei_class, Sym,
 				     ent, st_size, Elf_Word, 0);
 		      ELF_SET_FIELD (type_functions, ei_class, Sym,
 				     ent, st_shndx, Elf_Half, SHN_UNDEF);
-		      *st_info = ELF_ST_INFO (ELF_ST_BIND (*st_info),
-					      STT_NOTYPE);
-		      *st_other = STV_DEFAULT;
 		    }
 		}
 	      XDELETEVEC (strings);
@@ -1403,7 +1427,14 @@ simple_object_elf_copy_lto_debug_sections (simple_object_read *sobj,
       flags = ELF_FETCH_FIELD (type_functions, ei_class, Shdr,
 			       shdr, sh_flags, Elf_Addr);
       if (ret == 0)
-	flags &= ~SHF_EXCLUDE;
+	{
+	  /* The debugobj doesn't contain any code, thus no trampolines.
+	     Even when the original object needs trampolines, debugobj
+	     doesn't.  */
+	  if (strcmp (name, ".note.GNU-stack") == 0)
+	    flags &= ~SHF_EXECINSTR;
+	  flags &= ~SHF_EXCLUDE;
+	}
       else if (ret == -1)
 	flags = SHF_EXCLUDE;
       ELF_SET_FIELD (type_functions, ei_class, Shdr,
diff --git a/libiberty/simple-object.c b/libiberty/simple-object.c
index 553e90f5048..19d222fb1cd 100644
--- a/libiberty/simple-object.c
+++ b/libiberty/simple-object.c
@@ -273,6 +273,9 @@ handle_lto_debug_sections (const char **name)
       *name = *name + sizeof (".gnu.lto_") - 1;
       return 1;
     }
+  /* Copy over .note.GNU-stack section under the same name if present.  */
+  else if (strcmp (*name, ".note.GNU-stack") == 0)
+    return 1;
   return 0;
 }
 
diff --git a/libsanitizer/ChangeLog b/libsanitizer/ChangeLog
index b893e852e90..63e71317cbf 100644
--- a/libsanitizer/ChangeLog
+++ b/libsanitizer/ChangeLog
@@ -1,3 +1,38 @@
+2017-10-20  Jakub Jelinek  <jakub@redhat.com>
+
+	PR sanitizer/82595
+	* lsan/lsan.h (__lsan_init): Add SANITIZER_INTERFACE_ATTRIBUTE.
+	* lsan/Makefile.am (nodist_toolexeclib_HEADERS): Add
+	liblsan_preinit.o.
+	(lsan_files): Remove lsan_preinit.cc.
+	(liblsan_preinit.o): New rule.
+	* lsan/Makefile.in: Regenerated.
+
+2017-10-19  Jakub Jelinek  <jakub@redhat.com>
+
+	* All source files: Merge from upstream 315899.
+        * asan/Makefile.am (nodist_saninclude_HEADERS): Add
+	include/sanitizer/tsan_interface.h.
+        * asan/libtool-version: Bump the libasan SONAME.
+	* lsan/Makefile.am (sanitizer_lsan_files): Add lsan_common_mac.cc.
+	(lsan_files): Add lsan_linux.cc, lsan_mac.cc and lsan_malloc_mac.cc.
+        * sanitizer_common/Makefile.am (sanitizer_common_files): Add
+	sancov_flags.cc, sanitizer_allocator_checks.cc,
+	sanitizer_coverage_libcdep_new.cc, sanitizer_errno.cc,
+	sanitizer_file.cc, sanitizer_mac_libcdep.cc and
+	sanitizer_stoptheworld_mac.cc.  Remove sanitizer_coverage_libcdep.cc
+	and sanitizer_coverage_mapping_libcdep.cc.
+        * tsan/Makefile.am (tsan_files): Add tsan_external.cc.
+	* ubsan/Makefile.am (DEFS): Add -DUBSAN_CAN_USE_CXXABI=1.
+	(ubsan_files): Add ubsan_init_standalone.cc and
+	ubsan_signals_standalone.cc.
+	* ubsan/libtool-version: Bump the libubsan SONAME.
+        * asan/Makefile.in: Regenerate.
+        * lsan/Makefile.in: Regenerate.
+        * sanitizer_common/Makefile.in: Regenerate.
+        * tsan/Makefile.in: Regenerate.
+	* ubsan/Makefile.in: Regenerate.
+
 2017-10-05  H.J. Lu  <hongjiu.lu@intel.com>
 
 	PR sanitizer/82379
diff --git a/libsanitizer/MERGE b/libsanitizer/MERGE
index 21c2f390e97..c92f826445e 100644
--- a/libsanitizer/MERGE
+++ b/libsanitizer/MERGE
@@ -1,4 +1,4 @@
-285547
+315899
 
 The first line of this file holds the svn revision number of the
 last merge done from the master library sources.
diff --git a/libsanitizer/Makefile.am b/libsanitizer/Makefile.am
index 6afb2b0e048..018f0b06823 100644
--- a/libsanitizer/Makefile.am
+++ b/libsanitizer/Makefile.am
@@ -17,7 +17,8 @@ endif
 SUBDIRS += lsan asan ubsan
 nodist_saninclude_HEADERS += \
   include/sanitizer/lsan_interface.h \
-  include/sanitizer/asan_interface.h
+  include/sanitizer/asan_interface.h \
+  include/sanitizer/tsan_interface.h
 if TSAN_SUPPORTED
 SUBDIRS += tsan
 endif
diff --git a/libsanitizer/Makefile.in b/libsanitizer/Makefile.in
index 1f4bb8cd6bb..fd0f7fd78f9 100644
--- a/libsanitizer/Makefile.in
+++ b/libsanitizer/Makefile.in
@@ -54,7 +54,8 @@ host_triplet = @host@
 target_triplet = @target@
 @SANITIZER_SUPPORTED_TRUE@am__append_1 = include/sanitizer/common_interface_defs.h \
 @SANITIZER_SUPPORTED_TRUE@	include/sanitizer/lsan_interface.h \
-@SANITIZER_SUPPORTED_TRUE@	include/sanitizer/asan_interface.h
+@SANITIZER_SUPPORTED_TRUE@	include/sanitizer/asan_interface.h \
+@SANITIZER_SUPPORTED_TRUE@	include/sanitizer/tsan_interface.h
 @SANITIZER_SUPPORTED_TRUE@@USING_MAC_INTERPOSE_FALSE@am__append_2 = interception
 @LIBBACKTRACE_SUPPORTED_TRUE@@SANITIZER_SUPPORTED_TRUE@am__append_3 = libbacktrace
 @SANITIZER_SUPPORTED_TRUE@@TSAN_SUPPORTED_TRUE@am__append_4 = tsan
diff --git a/libsanitizer/asan/Makefile.am b/libsanitizer/asan/Makefile.am
index 43377a95622..709b6c8d75b 100644
--- a/libsanitizer/asan/Makefile.am
+++ b/libsanitizer/asan/Makefile.am
@@ -25,6 +25,7 @@ asan_files = \
 	asan_flags.cc \
 	asan_globals.cc \
 	asan_interceptors.cc \
+	asan_interceptors_memintrinsics.cc \
 	asan_linux.cc \
 	asan_mac.cc \
 	asan_malloc_linux.cc \
@@ -36,6 +37,7 @@ asan_files = \
 	asan_posix.cc \
 	asan_report.cc \
 	asan_rtl.cc \
+	asan_shadow_setup.cc \
 	asan_stack.cc \
 	asan_stats.cc \
 	asan_suppressions.cc \
diff --git a/libsanitizer/asan/Makefile.in b/libsanitizer/asan/Makefile.in
index 4dad60ba1ae..db3c3ced8b9 100644
--- a/libsanitizer/asan/Makefile.in
+++ b/libsanitizer/asan/Makefile.in
@@ -114,12 +114,13 @@ libasan_la_DEPENDENCIES =  \
 am__objects_1 = asan_activation.lo asan_allocator.lo asan_debugging.lo \
 	asan_descriptions.lo asan_errors.lo asan_fake_stack.lo \
 	asan_flags.lo asan_globals.lo asan_interceptors.lo \
-	asan_linux.lo asan_mac.lo asan_malloc_linux.lo \
-	asan_malloc_mac.lo asan_malloc_win.lo asan_memory_profile.lo \
-	asan_new_delete.lo asan_poisoning.lo asan_posix.lo \
-	asan_report.lo asan_rtl.lo asan_stack.lo asan_stats.lo \
-	asan_suppressions.lo asan_thread.lo asan_win.lo \
-	asan_win_dll_thunk.lo asan_win_dynamic_runtime_thunk.lo
+	asan_interceptors_memintrinsics.lo asan_linux.lo asan_mac.lo \
+	asan_malloc_linux.lo asan_malloc_mac.lo asan_malloc_win.lo \
+	asan_memory_profile.lo asan_new_delete.lo asan_poisoning.lo \
+	asan_posix.lo asan_report.lo asan_rtl.lo asan_shadow_setup.lo \
+	asan_stack.lo asan_stats.lo asan_suppressions.lo \
+	asan_thread.lo asan_win.lo asan_win_dll_thunk.lo \
+	asan_win_dynamic_runtime_thunk.lo
 am_libasan_la_OBJECTS = $(am__objects_1)
 libasan_la_OBJECTS = $(am_libasan_la_OBJECTS)
 libasan_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \
@@ -317,6 +318,7 @@ asan_files = \
 	asan_flags.cc \
 	asan_globals.cc \
 	asan_interceptors.cc \
+	asan_interceptors_memintrinsics.cc \
 	asan_linux.cc \
 	asan_mac.cc \
 	asan_malloc_linux.cc \
@@ -328,6 +330,7 @@ asan_files = \
 	asan_posix.cc \
 	asan_report.cc \
 	asan_rtl.cc \
+	asan_shadow_setup.cc \
 	asan_stack.cc \
 	asan_stats.cc \
 	asan_suppressions.cc \
@@ -466,6 +469,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_flags.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_globals.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_interceptors.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_interceptors_memintrinsics.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_linux.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_mac.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_malloc_linux.Plo@am__quote@
@@ -477,6 +481,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_posix.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_report.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_rtl.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_shadow_setup.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_stack.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_stats.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_suppressions.Plo@am__quote@
diff --git a/libsanitizer/asan/asan_activation.cc b/libsanitizer/asan/asan_activation.cc
index ecd767c5985..599e56b9e3f 100644
--- a/libsanitizer/asan/asan_activation.cc
+++ b/libsanitizer/asan/asan_activation.cc
@@ -75,13 +75,16 @@ static struct AsanDeactivatedFlags {
 
   void Print() {
     Report(
-        "quarantine_size_mb %d, max_redzone %d, poison_heap %d, "
-        "malloc_context_size %d, alloc_dealloc_mismatch %d, "
-        "allocator_may_return_null %d, coverage %d, coverage_dir %s\n",
-        allocator_options.quarantine_size_mb, allocator_options.max_redzone,
-        poison_heap, malloc_context_size,
+        "quarantine_size_mb %d, thread_local_quarantine_size_kb %d, "
+        "max_redzone %d, poison_heap %d, malloc_context_size %d, "
+        "alloc_dealloc_mismatch %d, allocator_may_return_null %d, coverage %d, "
+        "coverage_dir %s, allocator_release_to_os_interval_ms %d\n",
+        allocator_options.quarantine_size_mb,
+        allocator_options.thread_local_quarantine_size_kb,
+        allocator_options.max_redzone, poison_heap, malloc_context_size,
         allocator_options.alloc_dealloc_mismatch,
-        allocator_options.may_return_null, coverage, coverage_dir);
+        allocator_options.may_return_null, coverage, coverage_dir,
+        allocator_options.release_to_os_interval_ms);
   }
 } asan_deactivated_flags;
 
@@ -101,10 +104,10 @@ void AsanDeactivate() {
   // Deactivate the runtime.
   SetCanPoisonMemory(false);
   SetMallocContextSize(1);
-  ReInitializeCoverage(false, nullptr);
 
   AllocatorOptions disabled = asan_deactivated_flags.allocator_options;
   disabled.quarantine_size_mb = 0;
+  disabled.thread_local_quarantine_size_kb = 0;
   disabled.min_redzone = 16;  // Redzone must be at least 16 bytes long.
   disabled.max_redzone = 16;
   disabled.alloc_dealloc_mismatch = false;
@@ -124,8 +127,6 @@ void AsanActivate() {
 
   SetCanPoisonMemory(asan_deactivated_flags.poison_heap);
   SetMallocContextSize(asan_deactivated_flags.malloc_context_size);
-  ReInitializeCoverage(asan_deactivated_flags.coverage,
-                       asan_deactivated_flags.coverage_dir);
   ReInitializeAllocator(asan_deactivated_flags.allocator_options);
 
   asan_is_deactivated = false;
diff --git a/libsanitizer/asan/asan_activation_flags.inc b/libsanitizer/asan/asan_activation_flags.inc
index 4bab38213c1..e71abb96e5b 100644
--- a/libsanitizer/asan/asan_activation_flags.inc
+++ b/libsanitizer/asan/asan_activation_flags.inc
@@ -22,6 +22,7 @@
 ASAN_ACTIVATION_FLAG(int, redzone)
 ASAN_ACTIVATION_FLAG(int, max_redzone)
 ASAN_ACTIVATION_FLAG(int, quarantine_size_mb)
+ASAN_ACTIVATION_FLAG(int, thread_local_quarantine_size_kb)
 ASAN_ACTIVATION_FLAG(bool, alloc_dealloc_mismatch)
 ASAN_ACTIVATION_FLAG(bool, poison_heap)
 
@@ -31,3 +32,4 @@ COMMON_ACTIVATION_FLAG(bool, coverage)
 COMMON_ACTIVATION_FLAG(const char *, coverage_dir)
 COMMON_ACTIVATION_FLAG(int, verbosity)
 COMMON_ACTIVATION_FLAG(bool, help)
+COMMON_ACTIVATION_FLAG(s32, allocator_release_to_os_interval_ms)
diff --git a/libsanitizer/asan/asan_allocator.cc b/libsanitizer/asan/asan_allocator.cc
index d3ddb904d58..1b465469c75 100644
--- a/libsanitizer/asan/asan_allocator.cc
+++ b/libsanitizer/asan/asan_allocator.cc
@@ -19,7 +19,9 @@
 #include "asan_report.h"
 #include "asan_stack.h"
 #include "asan_thread.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_list.h"
@@ -158,7 +160,11 @@ struct QuarantineCallback {
   }
 
   void *Allocate(uptr size) {
-    return get_allocator().Allocate(cache_, size, 1, false);
+    void *res = get_allocator().Allocate(cache_, size, 1);
+    // TODO(alekseys): Consider making quarantine OOM-friendly.
+    if (UNLIKELY(!res))
+      return DieOnFailure::OnOOM();
+    return res;
   }
 
   void Deallocate(void *p) {
@@ -205,25 +211,27 @@ QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {
 
 void AllocatorOptions::SetFrom(const Flags *f, const CommonFlags *cf) {
   quarantine_size_mb = f->quarantine_size_mb;
+  thread_local_quarantine_size_kb = f->thread_local_quarantine_size_kb;
   min_redzone = f->redzone;
   max_redzone = f->max_redzone;
   may_return_null = cf->allocator_may_return_null;
   alloc_dealloc_mismatch = f->alloc_dealloc_mismatch;
+  release_to_os_interval_ms = cf->allocator_release_to_os_interval_ms;
 }
 
 void AllocatorOptions::CopyTo(Flags *f, CommonFlags *cf) {
   f->quarantine_size_mb = quarantine_size_mb;
+  f->thread_local_quarantine_size_kb = thread_local_quarantine_size_kb;
   f->redzone = min_redzone;
   f->max_redzone = max_redzone;
   cf->allocator_may_return_null = may_return_null;
   f->alloc_dealloc_mismatch = alloc_dealloc_mismatch;
+  cf->allocator_release_to_os_interval_ms = release_to_os_interval_ms;
 }
 
 struct Allocator {
   static const uptr kMaxAllowedMallocSize =
       FIRST_32_SECOND_64(3UL << 30, 1ULL << 40);
-  static const uptr kMaxThreadLocalQuarantine =
-      FIRST_32_SECOND_64(1 << 18, 1 << 20);
 
   AsanAllocator allocator;
   AsanQuarantine quarantine;
@@ -231,6 +239,8 @@ struct Allocator {
   AllocatorCache fallback_allocator_cache;
   QuarantineCache fallback_quarantine_cache;
 
+  atomic_uint8_t rss_limit_exceeded;
+
   // ------------------- Options --------------------------
   atomic_uint16_t min_redzone;
   atomic_uint16_t max_redzone;
@@ -252,7 +262,7 @@ struct Allocator {
   void SharedInitCode(const AllocatorOptions &options) {
     CheckOptions(options);
     quarantine.Init((uptr)options.quarantine_size_mb << 20,
-                    kMaxThreadLocalQuarantine);
+                    (uptr)options.thread_local_quarantine_size_kb << 10);
     atomic_store(&alloc_dealloc_mismatch, options.alloc_dealloc_mismatch,
                  memory_order_release);
     atomic_store(&min_redzone, options.min_redzone, memory_order_release);
@@ -260,35 +270,45 @@ struct Allocator {
   }
 
   void Initialize(const AllocatorOptions &options) {
-    allocator.Init(options.may_return_null);
+    SetAllocatorMayReturnNull(options.may_return_null);
+    allocator.Init(options.release_to_os_interval_ms);
     SharedInitCode(options);
   }
 
+  bool RssLimitExceeded() {
+    return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
+  }
+
+  void SetRssLimitExceeded(bool limit_exceeded) {
+    atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
+  }
+
   void RePoisonChunk(uptr chunk) {
-    // This could a user-facing chunk (with redzones), or some internal
+    // This could be a user-facing chunk (with redzones), or some internal
     // housekeeping chunk, like TransferBatch. Start by assuming the former.
     AsanChunk *ac = GetAsanChunk((void *)chunk);
     uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)ac);
     uptr beg = ac->Beg();
     uptr end = ac->Beg() + ac->UsedSize(true);
     uptr chunk_end = chunk + allocated_size;
-    if (chunk < beg && beg < end && end <= chunk_end) {
-      // Looks like a valid AsanChunk. Or maybe not. Be conservative and only
-      // poison the redzones.
+    if (chunk < beg && beg < end && end <= chunk_end &&
+        ac->chunk_state == CHUNK_ALLOCATED) {
+      // Looks like a valid AsanChunk in use, poison redzones only.
       PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic);
       uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY);
       FastPoisonShadowPartialRightRedzone(
           end_aligned_down, end - end_aligned_down,
           chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic);
     } else {
-      // This can not be an AsanChunk. Poison everything. It may be reused as
-      // AsanChunk later.
+      // This is either not an AsanChunk or freed or quarantined AsanChunk.
+      // In either case, poison everything.
       PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic);
     }
   }
 
   void ReInitialize(const AllocatorOptions &options) {
-    allocator.SetMayReturnNull(options.may_return_null);
+    SetAllocatorMayReturnNull(options.may_return_null);
+    allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms);
     SharedInitCode(options);
 
     // Poison all existing allocation's redzones.
@@ -305,11 +325,13 @@ struct Allocator {
 
   void GetOptions(AllocatorOptions *options) const {
     options->quarantine_size_mb = quarantine.GetSize() >> 20;
+    options->thread_local_quarantine_size_kb = quarantine.GetCacheSize() >> 10;
     options->min_redzone = atomic_load(&min_redzone, memory_order_acquire);
     options->max_redzone = atomic_load(&max_redzone, memory_order_acquire);
-    options->may_return_null = allocator.MayReturnNull();
+    options->may_return_null = AllocatorMayReturnNull();
     options->alloc_dealloc_mismatch =
         atomic_load(&alloc_dealloc_mismatch, memory_order_acquire);
+    options->release_to_os_interval_ms = allocator.ReleaseToOSIntervalMs();
   }
 
   // -------------------- Helper methods. -------------------------
@@ -356,6 +378,8 @@ struct Allocator {
                  AllocType alloc_type, bool can_fill) {
     if (UNLIKELY(!asan_inited))
       AsanInitFromRtl();
+    if (RssLimitExceeded())
+      return AsanAllocator::FailureHandler::OnOOM();
     Flags &fl = *flags();
     CHECK(stack);
     const uptr min_alignment = SHADOW_GRANULARITY;
@@ -388,24 +412,21 @@ struct Allocator {
     if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) {
       Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
              (void*)size);
-      return allocator.ReturnNullOrDieOnBadRequest();
+      return AsanAllocator::FailureHandler::OnBadRequest();
     }
 
     AsanThread *t = GetCurrentThread();
     void *allocated;
-    bool check_rss_limit = true;
     if (t) {
       AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
-      allocated =
-          allocator.Allocate(cache, needed_size, 8, false, check_rss_limit);
+      allocated = allocator.Allocate(cache, needed_size, 8);
     } else {
       SpinMutexLock l(&fallback_mutex);
       AllocatorCache *cache = &fallback_allocator_cache;
-      allocated =
-          allocator.Allocate(cache, needed_size, 8, false, check_rss_limit);
+      allocated = allocator.Allocate(cache, needed_size, 8);
     }
-
-    if (!allocated) return allocator.ReturnNullOrDieOnOOM();
+    if (!allocated)
+      return nullptr;
 
     if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && CanPoisonMemory()) {
       // Heap poisoning is enabled, but the allocator provides an unpoisoned
@@ -507,8 +528,7 @@ struct Allocator {
 
   // Expects the chunk to already be marked as quarantined by using
   // AtomicallySetQuarantineFlagIfAllocated.
-  void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack,
-                       AllocType alloc_type) {
+  void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack) {
     CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE);
     CHECK_GE(m->alloc_tid, 0);
     if (SANITIZER_WORDSIZE == 64)  // On 32-bits this resides in user area.
@@ -516,6 +536,18 @@ struct Allocator {
     AsanThread *t = GetCurrentThread();
     m->free_tid = t ? t->tid() : 0;
     m->free_context_id = StackDepotPut(*stack);
+
+    Flags &fl = *flags();
+    if (fl.max_free_fill_size > 0) {
+      // We have to skip the chunk header, it contains free_context_id.
+      uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size;
+      if (m->UsedSize() >= kChunkHeader2Size) {  // Skip Header2 in user area.
+        uptr size_to_fill = m->UsedSize() - kChunkHeader2Size;
+        size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size);
+        REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill);
+      }
+    }
+
     // Poison the region.
     PoisonShadow(m->Beg(),
                  RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
@@ -547,7 +579,17 @@ struct Allocator {
     uptr chunk_beg = p - kChunkHeaderSize;
     AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
 
+    // On Windows, uninstrumented DLLs may allocate memory before ASan hooks
+    // malloc. Don't report an invalid free in this case.
+    if (SANITIZER_WINDOWS &&
+        !get_allocator().PointerIsMine(ptr)) {
+      if (!IsSystemHeapAddress(p))
+        ReportFreeNotMalloced(p, stack);
+      return;
+    }
+
     ASAN_FREE_HOOK(ptr);
+
     // Must mark the chunk as quarantined before any changes to its metadata.
     // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag.
     if (!AtomicallySetQuarantineFlagIfAllocated(m, ptr, stack)) return;
@@ -564,7 +606,7 @@ struct Allocator {
       ReportNewDeleteSizeMismatch(p, delete_size, stack);
     }
 
-    QuarantineChunk(m, ptr, stack, alloc_type);
+    QuarantineChunk(m, ptr, stack);
   }
 
   void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) {
@@ -593,8 +635,8 @@ struct Allocator {
   }
 
   void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
-    if (CallocShouldReturnNullDueToOverflow(size, nmemb))
-      return allocator.ReturnNullOrDieOnBadRequest();
+    if (CheckForCallocOverflow(size, nmemb))
+      return AsanAllocator::FailureHandler::OnBadRequest();
     void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false);
     // If the memory comes from the secondary allocator no need to clear it
     // as it comes directly from mmap.
@@ -674,6 +716,7 @@ struct Allocator {
 
   void PrintStats() {
     allocator.PrintStats();
+    quarantine.PrintStats();
   }
 
   void ForceLock() {
@@ -685,8 +728,6 @@ struct Allocator {
     fallback_mutex.Unlock();
     allocator.ForceUnlock();
   }
-
-  void ReleaseToOS() { allocator.ReleaseToOS(); }
 };
 
 static Allocator instance(LINKER_INITIALIZED);
@@ -695,18 +736,21 @@ static AsanAllocator &get_allocator() {
   return instance.allocator;
 }
 
-bool AsanChunkView::IsValid() {
+bool AsanChunkView::IsValid() const {
   return chunk_ && chunk_->chunk_state != CHUNK_AVAILABLE;
 }
-bool AsanChunkView::IsAllocated() {
+bool AsanChunkView::IsAllocated() const {
   return chunk_ && chunk_->chunk_state == CHUNK_ALLOCATED;
 }
-uptr AsanChunkView::Beg() { return chunk_->Beg(); }
-uptr AsanChunkView::End() { return Beg() + UsedSize(); }
-uptr AsanChunkView::UsedSize() { return chunk_->UsedSize(); }
-uptr AsanChunkView::AllocTid() { return chunk_->alloc_tid; }
-uptr AsanChunkView::FreeTid() { return chunk_->free_tid; }
-AllocType AsanChunkView::GetAllocType() {
+bool AsanChunkView::IsQuarantined() const {
+  return chunk_ && chunk_->chunk_state == CHUNK_QUARANTINE;
+}
+uptr AsanChunkView::Beg() const { return chunk_->Beg(); }
+uptr AsanChunkView::End() const { return Beg() + UsedSize(); }
+uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); }
+uptr AsanChunkView::AllocTid() const { return chunk_->alloc_tid; }
+uptr AsanChunkView::FreeTid() const { return chunk_->free_tid; }
+AllocType AsanChunkView::GetAllocType() const {
   return (AllocType)chunk_->alloc_type;
 }
 
@@ -717,22 +761,19 @@ static StackTrace GetStackTraceFromId(u32 id) {
   return res;
 }
 
-u32 AsanChunkView::GetAllocStackId() { return chunk_->alloc_context_id; }
-u32 AsanChunkView::GetFreeStackId() { return chunk_->free_context_id; }
+u32 AsanChunkView::GetAllocStackId() const { return chunk_->alloc_context_id; }
+u32 AsanChunkView::GetFreeStackId() const { return chunk_->free_context_id; }
 
-StackTrace AsanChunkView::GetAllocStack() {
+StackTrace AsanChunkView::GetAllocStack() const {
   return GetStackTraceFromId(GetAllocStackId());
 }
 
-StackTrace AsanChunkView::GetFreeStack() {
+StackTrace AsanChunkView::GetFreeStack() const {
   return GetStackTraceFromId(GetFreeStackId());
 }
 
-void ReleaseToOS() { instance.ReleaseToOS(); }
-
 void InitializeAllocator(const AllocatorOptions &options) {
   instance.Initialize(options);
-  SetAllocatorReleaseToOSCallback(ReleaseToOS);
 }
 
 void ReInitializeAllocator(const AllocatorOptions &options) {
@@ -758,11 +799,6 @@ void PrintInternalAllocatorStats() {
   instance.PrintStats();
 }
 
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
-                    AllocType alloc_type) {
-  return instance.Allocate(size, alignment, stack, alloc_type, true);
-}
-
 void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) {
   instance.Deallocate(ptr, 0, stack, alloc_type);
 }
@@ -773,40 +809,63 @@ void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack,
 }
 
 void *asan_malloc(uptr size, BufferedStackTrace *stack) {
-  return instance.Allocate(size, 8, stack, FROM_MALLOC, true);
+  return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
 }
 
 void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
-  return instance.Calloc(nmemb, size, stack);
+  return SetErrnoOnNull(instance.Calloc(nmemb, size, stack));
 }
 
 void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
   if (!p)
-    return instance.Allocate(size, 8, stack, FROM_MALLOC, true);
+    return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
   if (size == 0) {
-    instance.Deallocate(p, 0, stack, FROM_MALLOC);
-    return nullptr;
+    if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
+      instance.Deallocate(p, 0, stack, FROM_MALLOC);
+      return nullptr;
+    }
+    // Allocate a size of 1 if we shouldn't free() on Realloc to 0
+    size = 1;
   }
-  return instance.Reallocate(p, size, stack);
+  return SetErrnoOnNull(instance.Reallocate(p, size, stack));
 }
 
 void *asan_valloc(uptr size, BufferedStackTrace *stack) {
-  return instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true);
+  return SetErrnoOnNull(
+      instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true));
 }
 
 void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
   uptr PageSize = GetPageSizeCached();
-  size = RoundUpTo(size, PageSize);
-  if (size == 0) {
-    // pvalloc(0) should allocate one page.
-    size = PageSize;
+  if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+    errno = errno_ENOMEM;
+    return AsanAllocator::FailureHandler::OnBadRequest();
   }
-  return instance.Allocate(size, PageSize, stack, FROM_MALLOC, true);
+  // pvalloc(0) should allocate one page.
+  size = size ? RoundUpTo(size, PageSize) : PageSize;
+  return SetErrnoOnNull(
+      instance.Allocate(size, PageSize, stack, FROM_MALLOC, true));
+}
+
+void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
+                    AllocType alloc_type) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    return AsanAllocator::FailureHandler::OnBadRequest();
+  }
+  return SetErrnoOnNull(
+      instance.Allocate(size, alignment, stack, alloc_type, true));
 }
 
 int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
                         BufferedStackTrace *stack) {
+  if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+    AsanAllocator::FailureHandler::OnBadRequest();
+    return errno_EINVAL;
+  }
   void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true);
+  if (UNLIKELY(!ptr))
+    return errno_ENOMEM;
   CHECK(IsAligned((uptr)ptr, alignment));
   *memptr = ptr;
   return 0;
@@ -834,8 +893,8 @@ void asan_mz_force_unlock() {
   instance.ForceUnlock();
 }
 
-void AsanSoftRssLimitExceededCallback(bool exceeded) {
-  instance.allocator.SetRssLimitIsExceeded(exceeded);
+void AsanSoftRssLimitExceededCallback(bool limit_exceeded) {
+  instance.SetRssLimitExceeded(limit_exceeded);
 }
 
 } // namespace __asan
@@ -952,15 +1011,13 @@ uptr __sanitizer_get_allocated_size(const void *p) {
 
 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
 // Provide default (no-op) implementation of malloc hooks.
-extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_malloc_hook(void *ptr, uptr size) {
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook,
+                             void *ptr, uptr size) {
   (void)ptr;
   (void)size;
 }
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_free_hook(void *ptr) {
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
   (void)ptr;
 }
-} // extern "C"
 #endif
diff --git a/libsanitizer/asan/asan_allocator.h b/libsanitizer/asan/asan_allocator.h
index 7eeddadd547..63260ff9895 100644
--- a/libsanitizer/asan/asan_allocator.h
+++ b/libsanitizer/asan/asan_allocator.h
@@ -31,10 +31,12 @@ struct AsanChunk;
 
 struct AllocatorOptions {
   u32 quarantine_size_mb;
+  u32 thread_local_quarantine_size_kb;
   u16 min_redzone;
   u16 max_redzone;
   u8 may_return_null;
   u8 alloc_dealloc_mismatch;
+  s32 release_to_os_interval_ms;
 
   void SetFrom(const Flags *f, const CommonFlags *cf);
   void CopyTo(Flags *f, CommonFlags *cf);
@@ -47,28 +49,29 @@ void GetAllocatorOptions(AllocatorOptions *options);
 class AsanChunkView {
  public:
   explicit AsanChunkView(AsanChunk *chunk) : chunk_(chunk) {}
-  bool IsValid();        // Checks if AsanChunkView points to a valid allocated
-                         // or quarantined chunk.
-  bool IsAllocated();    // Checks if the memory is currently allocated.
-  uptr Beg();            // First byte of user memory.
-  uptr End();            // Last byte of user memory.
-  uptr UsedSize();       // Size requested by the user.
-  uptr AllocTid();
-  uptr FreeTid();
+  bool IsValid() const;        // Checks if AsanChunkView points to a valid
+                               // allocated or quarantined chunk.
+  bool IsAllocated() const;    // Checks if the memory is currently allocated.
+  bool IsQuarantined() const;  // Checks if the memory is currently quarantined.
+  uptr Beg() const;            // First byte of user memory.
+  uptr End() const;            // Last byte of user memory.
+  uptr UsedSize() const;       // Size requested by the user.
+  uptr AllocTid() const;
+  uptr FreeTid() const;
   bool Eq(const AsanChunkView &c) const { return chunk_ == c.chunk_; }
-  u32 GetAllocStackId();
-  u32 GetFreeStackId();
-  StackTrace GetAllocStack();
-  StackTrace GetFreeStack();
-  AllocType GetAllocType();
-  bool AddrIsInside(uptr addr, uptr access_size, sptr *offset) {
+  u32 GetAllocStackId() const;
+  u32 GetFreeStackId() const;
+  StackTrace GetAllocStack() const;
+  StackTrace GetFreeStack() const;
+  AllocType GetAllocType() const;
+  bool AddrIsInside(uptr addr, uptr access_size, sptr *offset) const {
     if (addr >= Beg() && (addr + access_size) <= End()) {
       *offset = addr - Beg();
       return true;
     }
     return false;
   }
-  bool AddrIsAtLeft(uptr addr, uptr access_size, sptr *offset) {
+  bool AddrIsAtLeft(uptr addr, uptr access_size, sptr *offset) const {
     (void)access_size;
     if (addr < Beg()) {
       *offset = Beg() - addr;
@@ -76,7 +79,7 @@ class AsanChunkView {
     }
     return false;
   }
-  bool AddrIsAtRight(uptr addr, uptr access_size, sptr *offset) {
+  bool AddrIsAtRight(uptr addr, uptr access_size, sptr *offset) const {
     if (addr + access_size > End()) {
       *offset = addr - End();
       return true;
@@ -114,7 +117,11 @@ struct AsanMapUnmapCallback {
 };
 
 #if SANITIZER_CAN_USE_ALLOCATOR64
-# if defined(__powerpc64__)
+# if SANITIZER_FUCHSIA
+const uptr kAllocatorSpace = ~(uptr)0;
+const uptr kAllocatorSize  =  0x40000000000ULL;  // 4T.
+typedef DefaultSizeClassMap SizeClassMap;
+# elif defined(__powerpc64__)
 const uptr kAllocatorSpace =  0xa0000000000ULL;
 const uptr kAllocatorSize  =  0x20000000000ULL;  // 2T.
 typedef DefaultSizeClassMap SizeClassMap;
@@ -156,10 +163,17 @@ typedef FlatByteMap<kNumRegions> ByteMap;
 typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
 # endif
 typedef CompactSizeClassMap SizeClassMap;
-typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, 16,
-  SizeClassMap, kRegionSizeLog,
-  ByteMap,
-  AsanMapUnmapCallback> PrimaryAllocator;
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = 16;
+  typedef __asan::SizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = __asan::kRegionSizeLog;
+  typedef __asan::ByteMap ByteMap;
+  typedef AsanMapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
 #endif  // SANITIZER_CAN_USE_ALLOCATOR64
 
 static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses;
diff --git a/libsanitizer/asan/asan_descriptions.cc b/libsanitizer/asan/asan_descriptions.cc
index 35d1619f2d9..d46962adf27 100644
--- a/libsanitizer/asan/asan_descriptions.cc
+++ b/libsanitizer/asan/asan_descriptions.cc
@@ -148,7 +148,7 @@ static void PrintHeapChunkAccess(uptr addr, const ChunkAccess &descr) {
   str.append(" %zu-byte region [%p,%p)\n", descr.chunk_size,
              (void *)descr.chunk_begin,
              (void *)(descr.chunk_begin + descr.chunk_size));
-  str.append("%s", d.EndLocation());
+  str.append("%s", d.Default());
   Printf("%s", str.data());
 }
 
@@ -250,12 +250,15 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr,
     str.append("%c", var.name_pos[i]);
   }
   str.append("'");
+  if (var.line > 0) {
+    str.append(" (line %d)", var.line);
+  }
   if (pos_descr) {
     Decorator d;
     // FIXME: we may want to also print the size of the access here,
     // but in case of accesses generated by memset it may be confusing.
     str.append("%s <== Memory access at offset %zd %s this variable%s\n",
-               d.Location(), addr, pos_descr, d.EndLocation());
+               d.Location(), addr, pos_descr, d.Default());
   } else {
     str.append("\n");
   }
@@ -290,7 +293,7 @@ static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size,
              MaybeDemangleGlobalName(g.name));
   PrintGlobalLocation(&str, g);
   str.append("' (0x%zx) of size %zu\n", g.beg, g.size);
-  str.append("%s", d.EndLocation());
+  str.append("%s", d.Default());
   PrintGlobalNameIfASCII(&str, g);
   Printf("%s", str.data());
 }
@@ -338,10 +341,10 @@ void StackAddressDescription::Print() const {
          ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
 
   if (!frame_descr) {
-    Printf("%s\n", d.EndLocation());
+    Printf("%s\n", d.Default());
     return;
   }
-  Printf(" at offset %zu in frame%s\n", offset, d.EndLocation());
+  Printf(" at offset %zu in frame%s\n", offset, d.Default());
 
   // Now we print the frame where the alloca has happened.
   // We print this frame as a stack trace with one element.
@@ -350,7 +353,7 @@ void StackAddressDescription::Print() const {
   // previously. That's unfortunate, but I have no better solution,
   // especially given that the alloca may be from entirely different place
   // (e.g. use-after-scope, or different thread's stack).
-  Printf("%s", d.EndLocation());
+  Printf("%s", d.Default());
   StackTrace alloca_stack(&frame_pc, 1);
   alloca_stack.Print();
 
@@ -400,18 +403,18 @@ void HeapAddressDescription::Print() const {
     Printf("%sfreed by thread T%d%s here:%s\n", d.Allocation(),
            free_thread->tid,
            ThreadNameWithParenthesis(free_thread, tname, sizeof(tname)),
-           d.EndAllocation());
+           d.Default());
     StackTrace free_stack = GetStackTraceFromId(free_stack_id);
     free_stack.Print();
     Printf("%spreviously allocated by thread T%d%s here:%s\n", d.Allocation(),
            alloc_thread->tid,
            ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)),
-           d.EndAllocation());
+           d.Default());
   } else {
     Printf("%sallocated by thread T%d%s here:%s\n", d.Allocation(),
            alloc_thread->tid,
            ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)),
-           d.EndAllocation());
+           d.Default());
   }
   alloc_stack.Print();
   DescribeThread(GetCurrentThread());
diff --git a/libsanitizer/asan/asan_descriptions.h b/libsanitizer/asan/asan_descriptions.h
index 584b9ba6491..0fbb531492a 100644
--- a/libsanitizer/asan/asan_descriptions.h
+++ b/libsanitizer/asan/asan_descriptions.h
@@ -32,11 +32,8 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator {
  public:
   Decorator() : SanitizerCommonDecorator() {}
   const char *Access() { return Blue(); }
-  const char *EndAccess() { return Default(); }
   const char *Location() { return Green(); }
-  const char *EndLocation() { return Default(); }
   const char *Allocation() { return Magenta(); }
-  const char *EndAllocation() { return Default(); }
 
   const char *ShadowByte(u8 byte) {
     switch (byte) {
@@ -70,9 +67,6 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator {
         return Default();
     }
   }
-  const char *EndShadowByte() { return Default(); }
-  const char *MemoryByte() { return Magenta(); }
-  const char *EndMemoryByte() { return Default(); }
 };
 
 enum ShadowKind : u8 {
diff --git a/libsanitizer/asan/asan_errors.cc b/libsanitizer/asan/asan_errors.cc
index 73c4cca843b..b469b16f004 100644
--- a/libsanitizer/asan/asan_errors.cc
+++ b/libsanitizer/asan/asan_errors.cc
@@ -20,64 +20,27 @@
 
 namespace __asan {
 
-void ErrorStackOverflow::Print() {
-  Decorator d;
-  Printf("%s", d.Warning());
-  Report(
-      "ERROR: AddressSanitizer: stack-overflow on address %p"
-      " (pc %p bp %p sp %p T%d)\n",
-      (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid);
-  Printf("%s", d.EndWarning());
-  scariness.Print();
-  BufferedStackTrace stack;
-  GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context,
-                                  common_flags()->fast_unwind_on_fatal);
-  stack.Print();
-  ReportErrorSummary("stack-overflow", &stack);
-}
-
-static void MaybeDumpInstructionBytes(uptr pc) {
-  if (!flags()->dump_instruction_bytes || (pc < GetPageSizeCached())) return;
-  InternalScopedString str(1024);
-  str.append("First 16 instruction bytes at pc: ");
-  if (IsAccessibleMemoryRange(pc, 16)) {
-    for (int i = 0; i < 16; ++i) {
-      PrintMemoryByte(&str, "", ((u8 *)pc)[i], /*in_shadow*/ false, " ");
-    }
-    str.append("\n");
-  } else {
-    str.append("unaccessible\n");
-  }
-  Report("%s", str.data());
+static void OnStackUnwind(const SignalContext &sig,
+                          const void *callback_context,
+                          BufferedStackTrace *stack) {
+  bool fast = common_flags()->fast_unwind_on_fatal;
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+  // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace()
+  // yields the call stack of the signal's handler and not of the code
+  // that raised the signal (as it does on Linux).
+  fast = true;
+#endif
+  // Tests and maybe some users expect that scariness is going to be printed
+  // just before the stack. As only asan has scariness score we have no
+  // corresponding code in the sanitizer_common and we use this callback to
+  // print it.
+  static_cast<const ScarinessScoreBase *>(callback_context)->Print();
+  GetStackTraceWithPcBpAndContext(stack, kStackTraceMax, sig.pc, sig.bp,
+                                  sig.context, fast);
 }
 
 void ErrorDeadlySignal::Print() {
-  Decorator d;
-  Printf("%s", d.Warning());
-  const char *description = DescribeSignalOrException(signo);
-  Report(
-      "ERROR: AddressSanitizer: %s on unknown address %p (pc %p bp %p sp %p "
-      "T%d)\n",
-      description, (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid);
-  Printf("%s", d.EndWarning());
-  if (pc < GetPageSizeCached()) Report("Hint: pc points to the zero page.\n");
-  if (is_memory_access) {
-    const char *access_type =
-        write_flag == SignalContext::WRITE
-            ? "WRITE"
-            : (write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
-    Report("The signal is caused by a %s memory access.\n", access_type);
-    if (addr < GetPageSizeCached())
-      Report("Hint: address points to the zero page.\n");
-  }
-  scariness.Print();
-  BufferedStackTrace stack;
-  GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context,
-                                  common_flags()->fast_unwind_on_fatal);
-  stack.Print();
-  MaybeDumpInstructionBytes(pc);
-  Printf("AddressSanitizer can not provide additional info.\n");
-  ReportErrorSummary(description, &stack);
+  ReportDeadlySignal(signal, tid, &OnStackUnwind, &scariness);
 }
 
 void ErrorDoubleFree::Print() {
@@ -85,17 +48,17 @@ void ErrorDoubleFree::Print() {
   Printf("%s", d.Warning());
   char tname[128];
   Report(
-      "ERROR: AddressSanitizer: attempting double-free on %p in "
+      "ERROR: AddressSanitizer: attempting %s on %p in "
       "thread T%d%s:\n",
-      addr_description.addr, tid,
+      scariness.GetDescription(), addr_description.addr, tid,
       ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
   scariness.Print();
   GET_STACK_TRACE_FATAL(second_free_stack->trace[0],
                         second_free_stack->top_frame_bp);
   stack.Print();
   addr_description.Print();
-  ReportErrorSummary("double-free", &stack);
+  ReportErrorSummary(scariness.GetDescription(), &stack);
 }
 
 void ErrorNewDeleteSizeMismatch::Print() {
@@ -103,11 +66,11 @@ void ErrorNewDeleteSizeMismatch::Print() {
   Printf("%s", d.Warning());
   char tname[128];
   Report(
-      "ERROR: AddressSanitizer: new-delete-type-mismatch on %p in thread "
+      "ERROR: AddressSanitizer: %s on %p in thread "
       "T%d%s:\n",
-      addr_description.addr, tid,
+      scariness.GetDescription(), addr_description.addr, tid,
       ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
-  Printf("%s  object passed to delete has wrong type:\n", d.EndWarning());
+  Printf("%s  object passed to delete has wrong type:\n", d.Default());
   Printf(
       "  size of the allocated type:   %zd bytes;\n"
       "  size of the deallocated type: %zd bytes.\n",
@@ -117,7 +80,7 @@ void ErrorNewDeleteSizeMismatch::Print() {
   GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp);
   stack.Print();
   addr_description.Print();
-  ReportErrorSummary("new-delete-type-mismatch", &stack);
+  ReportErrorSummary(scariness.GetDescription(), &stack);
   Report(
       "HINT: if you don't care about these errors you may set "
       "ASAN_OPTIONS=new_delete_type_mismatch=0\n");
@@ -132,13 +95,13 @@ void ErrorFreeNotMalloced::Print() {
       "which was not malloc()-ed: %p in thread T%d%s\n",
       addr_description.Address(), tid,
       ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
   CHECK_GT(free_stack->size, 0);
   scariness.Print();
   GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp);
   stack.Print();
   addr_description.Print();
-  ReportErrorSummary("bad-free", &stack);
+  ReportErrorSummary(scariness.GetDescription(), &stack);
 }
 
 void ErrorAllocTypeMismatch::Print() {
@@ -149,16 +112,17 @@ void ErrorAllocTypeMismatch::Print() {
   CHECK_NE(alloc_type, dealloc_type);
   Decorator d;
   Printf("%s", d.Warning());
-  Report("ERROR: AddressSanitizer: alloc-dealloc-mismatch (%s vs %s) on %p\n",
+  Report("ERROR: AddressSanitizer: %s (%s vs %s) on %p\n",
+         scariness.GetDescription(),
          alloc_names[alloc_type], dealloc_names[dealloc_type],
          addr_description.addr);
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
   CHECK_GT(dealloc_stack->size, 0);
   scariness.Print();
   GET_STACK_TRACE_FATAL(dealloc_stack->trace[0], dealloc_stack->top_frame_bp);
   stack.Print();
   addr_description.Print();
-  ReportErrorSummary("alloc-dealloc-mismatch", &stack);
+  ReportErrorSummary(scariness.GetDescription(), &stack);
   Report(
       "HINT: if you don't care about these errors you may set "
       "ASAN_OPTIONS=alloc_dealloc_mismatch=0\n");
@@ -171,10 +135,10 @@ void ErrorMallocUsableSizeNotOwned::Print() {
       "ERROR: AddressSanitizer: attempting to call malloc_usable_size() for "
       "pointer which is not owned: %p\n",
       addr_description.Address());
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
   stack->Print();
   addr_description.Print();
-  ReportErrorSummary("bad-malloc_usable_size", stack);
+  ReportErrorSummary(scariness.GetDescription(), stack);
 }
 
 void ErrorSanitizerGetAllocatedSizeNotOwned::Print() {
@@ -184,10 +148,10 @@ void ErrorSanitizerGetAllocatedSizeNotOwned::Print() {
       "ERROR: AddressSanitizer: attempting to call "
       "__sanitizer_get_allocated_size() for pointer which is not owned: %p\n",
       addr_description.Address());
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
   stack->Print();
   addr_description.Print();
-  ReportErrorSummary("bad-__sanitizer_get_allocated_size", stack);
+  ReportErrorSummary(scariness.GetDescription(), stack);
 }
 
 void ErrorStringFunctionMemoryRangesOverlap::Print() {
@@ -201,7 +165,7 @@ void ErrorStringFunctionMemoryRangesOverlap::Print() {
       bug_type, addr1_description.Address(),
       addr1_description.Address() + length1, addr2_description.Address(),
       addr2_description.Address() + length2);
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
   scariness.Print();
   stack->Print();
   addr1_description.Print();
@@ -212,13 +176,13 @@ void ErrorStringFunctionMemoryRangesOverlap::Print() {
 void ErrorStringFunctionSizeOverflow::Print() {
   Decorator d;
   Printf("%s", d.Warning());
-  const char *bug_type = "negative-size-param";
-  Report("ERROR: AddressSanitizer: %s: (size=%zd)\n", bug_type, size);
-  Printf("%s", d.EndWarning());
+  Report("ERROR: AddressSanitizer: %s: (size=%zd)\n",
+         scariness.GetDescription(), size);
+  Printf("%s", d.Default());
   scariness.Print();
   stack->Print();
   addr_description.Print();
-  ReportErrorSummary(bug_type, stack);
+  ReportErrorSummary(scariness.GetDescription(), stack);
 }
 
 void ErrorBadParamsToAnnotateContiguousContainer::Print() {
@@ -234,14 +198,15 @@ void ErrorBadParamsToAnnotateContiguousContainer::Print() {
   if (!IsAligned(beg, granularity))
     Report("ERROR: beg is not aligned by %d\n", granularity);
   stack->Print();
-  ReportErrorSummary("bad-__sanitizer_annotate_contiguous_container", stack);
+  ReportErrorSummary(scariness.GetDescription(), stack);
 }
 
 void ErrorODRViolation::Print() {
   Decorator d;
   Printf("%s", d.Warning());
-  Report("ERROR: AddressSanitizer: odr-violation (%p):\n", global1.beg);
-  Printf("%s", d.EndWarning());
+  Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(),
+         global1.beg);
+  Printf("%s", d.Default());
   InternalScopedString g1_loc(256), g2_loc(256);
   PrintGlobalLocation(&g1_loc, global1);
   PrintGlobalLocation(&g2_loc, global2);
@@ -260,23 +225,22 @@ void ErrorODRViolation::Print() {
       "HINT: if you don't care about these errors you may set "
       "ASAN_OPTIONS=detect_odr_violation=0\n");
   InternalScopedString error_msg(256);
-  error_msg.append("odr-violation: global '%s' at %s",
+  error_msg.append("%s: global '%s' at %s", scariness.GetDescription(),
                    MaybeDemangleGlobalName(global1.name), g1_loc.data());
   ReportErrorSummary(error_msg.data());
 }
 
 void ErrorInvalidPointerPair::Print() {
-  const char *bug_type = "invalid-pointer-pair";
   Decorator d;
   Printf("%s", d.Warning());
-  Report("ERROR: AddressSanitizer: invalid-pointer-pair: %p %p\n",
+  Report("ERROR: AddressSanitizer: %s: %p %p\n", scariness.GetDescription(),
          addr1_description.Address(), addr2_description.Address());
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
   GET_STACK_TRACE_FATAL(pc, bp);
   stack.Print();
   addr1_description.Print();
   addr2_description.Print();
-  ReportErrorSummary(bug_type, &stack);
+  ReportErrorSummary(scariness.GetDescription(), &stack);
 }
 
 static bool AdjacentShadowValuesAreFullyPoisoned(u8 *s) {
@@ -470,13 +434,13 @@ void ErrorGeneric::Print() {
   uptr addr = addr_description.Address();
   Report("ERROR: AddressSanitizer: %s on address %p at pc %p bp %p sp %p\n",
          bug_descr, (void *)addr, pc, bp, sp);
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
 
   char tname[128];
   Printf("%s%s of size %zu at %p thread T%d%s%s\n", d.Access(),
          access_size ? (is_write ? "WRITE" : "READ") : "ACCESS", access_size,
          (void *)addr, tid,
-         ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.EndAccess());
+         ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.Default());
 
   scariness.Print();
   GET_STACK_TRACE_FATAL(pc, bp);
diff --git a/libsanitizer/asan/asan_errors.h b/libsanitizer/asan/asan_errors.h
index 6262dcf3506..ea8fd01e287 100644
--- a/libsanitizer/asan/asan_errors.h
+++ b/libsanitizer/asan/asan_errors.h
@@ -25,61 +25,28 @@ struct ErrorBase {
   u32 tid;
 };
 
-struct ErrorStackOverflow : ErrorBase {
-  uptr addr, pc, bp, sp;
-  // ErrorStackOverflow never owns the context.
-  void *context;
-  // VS2013 doesn't implement unrestricted unions, so we need a trivial default
-  // constructor
-  ErrorStackOverflow() = default;
-  ErrorStackOverflow(u32 tid, const SignalContext &sig)
-      : ErrorBase(tid),
-        addr(sig.addr),
-        pc(sig.pc),
-        bp(sig.bp),
-        sp(sig.sp),
-        context(sig.context) {
-    scariness.Clear();
-    scariness.Scare(10, "stack-overflow");
-  }
-  void Print();
-};
-
 struct ErrorDeadlySignal : ErrorBase {
-  uptr addr, pc, bp, sp;
-  // ErrorDeadlySignal never owns the context.
-  void *context;
-  int signo;
-  SignalContext::WriteFlag write_flag;
-  bool is_memory_access;
+  SignalContext signal;
   // VS2013 doesn't implement unrestricted unions, so we need a trivial default
   // constructor
   ErrorDeadlySignal() = default;
-  ErrorDeadlySignal(u32 tid, const SignalContext &sig, int signo_)
-      : ErrorBase(tid),
-        addr(sig.addr),
-        pc(sig.pc),
-        bp(sig.bp),
-        sp(sig.sp),
-        context(sig.context),
-        signo(signo_),
-        write_flag(sig.write_flag),
-        is_memory_access(sig.is_memory_access) {
+  ErrorDeadlySignal(u32 tid, const SignalContext &sig)
+      : ErrorBase(tid), signal(sig) {
     scariness.Clear();
-    if (is_memory_access) {
-      if (addr < GetPageSizeCached()) {
-        scariness.Scare(10, "null-deref");
-      } else if (addr == pc) {
-        scariness.Scare(60, "wild-jump");
-      } else if (write_flag == SignalContext::WRITE) {
-        scariness.Scare(30, "wild-addr-write");
-      } else if (write_flag == SignalContext::READ) {
-        scariness.Scare(20, "wild-addr-read");
-      } else {
-        scariness.Scare(25, "wild-addr");
-      }
-    } else {
+    if (signal.IsStackOverflow()) {
+      scariness.Scare(10, "stack-overflow");
+    } else if (!signal.is_memory_access) {
       scariness.Scare(10, "signal");
+    } else if (signal.addr < GetPageSizeCached()) {
+      scariness.Scare(10, "null-deref");
+    } else if (signal.addr == signal.pc) {
+      scariness.Scare(60, "wild-jump");
+    } else if (signal.write_flag == SignalContext::WRITE) {
+      scariness.Scare(30, "wild-addr-write");
+    } else if (signal.write_flag == SignalContext::READ) {
+      scariness.Scare(20, "wild-addr-read");
+    } else {
+      scariness.Scare(25, "wild-addr");
     }
   }
   void Print();
@@ -170,6 +137,7 @@ struct ErrorMallocUsableSizeNotOwned : ErrorBase {
         stack(stack_),
         addr_description(addr, /*shouldLockThreadRegistry=*/false) {
     scariness.Clear();
+    scariness.Scare(10, "bad-malloc_usable_size");
   }
   void Print();
 };
@@ -187,6 +155,7 @@ struct ErrorSanitizerGetAllocatedSizeNotOwned : ErrorBase {
         stack(stack_),
         addr_description(addr, /*shouldLockThreadRegistry=*/false) {
     scariness.Clear();
+    scariness.Scare(10, "bad-__sanitizer_get_allocated_size");
   }
   void Print();
 };
@@ -256,7 +225,10 @@ struct ErrorBadParamsToAnnotateContiguousContainer : ErrorBase {
         beg(beg_),
         end(end_),
         old_mid(old_mid_),
-        new_mid(new_mid_) {}
+        new_mid(new_mid_) {
+    scariness.Clear();
+    scariness.Scare(10, "bad-__sanitizer_annotate_contiguous_container");
+  }
   void Print();
 };
 
@@ -272,7 +244,10 @@ struct ErrorODRViolation : ErrorBase {
         global1(*g1),
         global2(*g2),
         stack_id1(stack_id1_),
-        stack_id2(stack_id2_) {}
+        stack_id2(stack_id2_) {
+    scariness.Clear();
+    scariness.Scare(10, "odr-violation");
+  }
   void Print();
 };
 
@@ -290,7 +265,10 @@ struct ErrorInvalidPointerPair : ErrorBase {
         bp(bp_),
         sp(sp_),
         addr1_description(p1, 1, /*shouldLockThreadRegistry=*/false),
-        addr2_description(p2, 1, /*shouldLockThreadRegistry=*/false) {}
+        addr2_description(p2, 1, /*shouldLockThreadRegistry=*/false)  {
+    scariness.Clear();
+    scariness.Scare(10, "invalid-pointer-pair");
+  }
   void Print();
 };
 
@@ -311,7 +289,6 @@ struct ErrorGeneric : ErrorBase {
 
 // clang-format off
 #define ASAN_FOR_EACH_ERROR_KIND(macro)         \
-  macro(StackOverflow)                          \
   macro(DeadlySignal)                           \
   macro(DoubleFree)                             \
   macro(NewDeleteSizeMismatch)                  \
@@ -348,6 +325,7 @@ struct ErrorDescription {
   // We can add a wrapper around it to make it "more c++-like", but that would
   // add a lot of code and the benefit wouldn't be that big.
   union {
+    ErrorBase Base;
     ASAN_FOR_EACH_ERROR_KIND(ASAN_ERROR_DESCRIPTION_MEMBER)
   };
 
diff --git a/libsanitizer/asan/asan_fake_stack.cc b/libsanitizer/asan/asan_fake_stack.cc
index bf7566a334e..3140f9a2aeb 100644
--- a/libsanitizer/asan/asan_fake_stack.cc
+++ b/libsanitizer/asan/asan_fake_stack.cc
@@ -169,7 +169,7 @@ void FakeStack::ForEachFakeFrame(RangeIteratorCallback callback, void *arg) {
   }
 }
 
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA
 static THREADLOCAL FakeStack *fake_stack_tls;
 
 FakeStack *GetTLSFakeStack() {
@@ -181,7 +181,7 @@ void SetTLSFakeStack(FakeStack *fs) {
 #else
 FakeStack *GetTLSFakeStack() { return 0; }
 void SetTLSFakeStack(FakeStack *fs) { }
-#endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
+#endif  // (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA
 
 static FakeStack *GetFakeStack() {
   AsanThread *t = GetCurrentThread();
diff --git a/libsanitizer/asan/asan_flags.cc b/libsanitizer/asan/asan_flags.cc
index c18174beed0..0c83dac6747 100644
--- a/libsanitizer/asan/asan_flags.cc
+++ b/libsanitizer/asan/asan_flags.cc
@@ -59,7 +59,7 @@ void InitializeFlags() {
   {
     CommonFlags cf;
     cf.CopyFrom(*common_flags());
-    cf.detect_leaks = CAN_SANITIZE_LEAKS;
+    cf.detect_leaks = cf.detect_leaks && CAN_SANITIZE_LEAKS;
     cf.external_symbolizer_path = GetEnv("ASAN_SYMBOLIZER_PATH");
     cf.malloc_context_size = kDefaultMallocContextSize;
     cf.intercept_tls_get_addr = true;
@@ -93,6 +93,18 @@ void InitializeFlags() {
   RegisterCommonFlags(&ubsan_parser);
 #endif
 
+  if (SANITIZER_MAC) {
+    // Support macOS MallocScribble and MallocPreScribble:
+    // <https://developer.apple.com/library/content/documentation/Performance/
+    // Conceptual/ManagingMemory/Articles/MallocDebug.html>
+    if (GetEnv("MallocScribble")) {
+      f->max_free_fill_size = 0x1000;
+    }
+    if (GetEnv("MallocPreScribble")) {
+      f->malloc_fill_byte = 0xaa;
+    }
+  }
+
   // Override from ASan compile definition.
   const char *asan_compile_def = MaybeUseAsanDefaultOptionsCompileDefinition();
   asan_parser.ParseString(asan_compile_def);
@@ -104,6 +116,10 @@ void InitializeFlags() {
   const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
   ubsan_parser.ParseString(ubsan_default_options);
 #endif
+#if CAN_SANITIZE_LEAKS
+  const char *lsan_default_options = __lsan::MaybeCallLsanDefaultOptions();
+  lsan_parser.ParseString(lsan_default_options);
+#endif
 
   // Override from command line.
   asan_parser.ParseString(GetEnv("ASAN_OPTIONS"));
@@ -154,9 +170,24 @@ void InitializeFlags() {
     f->quarantine_size_mb = f->quarantine_size >> 20;
   if (f->quarantine_size_mb < 0) {
     const int kDefaultQuarantineSizeMb =
-        (ASAN_LOW_MEMORY) ? 1UL << 6 : 1UL << 8;
+        (ASAN_LOW_MEMORY) ? 1UL << 4 : 1UL << 8;
     f->quarantine_size_mb = kDefaultQuarantineSizeMb;
   }
+  if (f->thread_local_quarantine_size_kb < 0) {
+    const u32 kDefaultThreadLocalQuarantineSizeKb =
+        // It is not advised to go lower than 64Kb, otherwise quarantine batches
+        // pushed from thread local quarantine to global one will create too
+        // much overhead. One quarantine batch size is 8Kb and it  holds up to
+        // 1021 chunk, which amounts to 1/8 memory overhead per batch when
+        // thread local quarantine is set to 64Kb.
+        (ASAN_LOW_MEMORY) ? 1 << 6 : FIRST_32_SECOND_64(1 << 8, 1 << 10);
+    f->thread_local_quarantine_size_kb = kDefaultThreadLocalQuarantineSizeKb;
+  }
+  if (f->thread_local_quarantine_size_kb == 0 && f->quarantine_size_mb > 0) {
+    Report("%s: thread_local_quarantine_size_kb can be set to 0 only when "
+           "quarantine_size_mb is set to 0\n", SanitizerToolName);
+    Die();
+  }
   if (!f->replace_str && common_flags()->intercept_strlen) {
     Report("WARNING: strlen interceptor is enabled even though replace_str=0. "
            "Use intercept_strlen=0 to disable it.");
@@ -165,13 +196,14 @@ void InitializeFlags() {
     Report("WARNING: strchr* interceptors are enabled even though "
            "replace_str=0. Use intercept_strchr=0 to disable them.");
   }
+  if (!f->replace_str && common_flags()->intercept_strndup) {
+    Report("WARNING: strndup* interceptors are enabled even though "
+           "replace_str=0. Use intercept_strndup=0 to disable them.");
+  }
 }
 
 }  // namespace __asan
 
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char* __asan_default_options() { return ""; }
-}  // extern "C"
-#endif
+SANITIZER_INTERFACE_WEAK_DEF(const char*, __asan_default_options, void) {
+  return "";
+}
diff --git a/libsanitizer/asan/asan_flags.inc b/libsanitizer/asan/asan_flags.inc
index 34493eededf..3784f065be6 100644
--- a/libsanitizer/asan/asan_flags.inc
+++ b/libsanitizer/asan/asan_flags.inc
@@ -21,6 +21,12 @@ ASAN_FLAG(int, quarantine_size_mb, -1,
           "Size (in Mb) of quarantine used to detect use-after-free "
           "errors. Lower value may reduce memory usage but increase the "
           "chance of false negatives.")
+ASAN_FLAG(int, thread_local_quarantine_size_kb, -1,
+          "Size (in Kb) of thread local quarantine used to detect "
+          "use-after-free errors. Lower value may reduce memory usage but "
+          "increase the chance of false negatives. It is not advised to go "
+          "lower than 64Kb, otherwise frequent transfers to global quarantine "
+          "might affect performance.")
 ASAN_FLAG(int, redzone, 16,
           "Minimal size (in bytes) of redzones around heap objects. "
           "Requirement: redzone >= 16, is a power of two.")
@@ -55,8 +61,14 @@ ASAN_FLAG(
     int, max_malloc_fill_size, 0x1000,  // By default, fill only the first 4K.
     "ASan allocator flag. max_malloc_fill_size is the maximal amount of "
     "bytes that will be filled with malloc_fill_byte on malloc.")
+ASAN_FLAG(
+    int, max_free_fill_size, 0,
+    "ASan allocator flag. max_free_fill_size is the maximal amount of "
+    "bytes that will be filled with free_fill_byte during free.")
 ASAN_FLAG(int, malloc_fill_byte, 0xbe,
           "Value used to fill the newly allocated memory.")
+ASAN_FLAG(int, free_fill_byte, 0x55,
+          "Value used to fill deallocated memory.")
 ASAN_FLAG(bool, allow_user_poisoning, true,
           "If set, user may manually mark memory regions as poisoned or "
           "unpoisoned.")
@@ -65,6 +77,10 @@ ASAN_FLAG(
     "Number of seconds to sleep between printing an error report and "
     "terminating the program. Useful for debugging purposes (e.g. when one "
     "needs to attach gdb).")
+ASAN_FLAG(
+    int, sleep_after_init, 0,
+    "Number of seconds to sleep after AddressSanitizer is initialized. "
+    "Useful for debugging purposes (e.g. when one needs to attach gdb).")
 ASAN_FLAG(bool, check_malloc_usable_size, true,
           "Allows the users to work around the bug in Nvidia drivers prior to "
           "295.*.")
@@ -129,11 +145,16 @@ ASAN_FLAG(int, detect_odr_violation, 2,
           "If >=2, detect violation of One-Definition-Rule (ODR); "
           "If ==1, detect ODR-violation only if the two variables "
           "have different sizes")
-ASAN_FLAG(bool, dump_instruction_bytes, false,
-          "If true, dump 16 bytes starting at the instruction that caused SEGV")
 ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
 ASAN_FLAG(bool, halt_on_error, true,
           "Crash the program after printing the first error report "
           "(WARNING: USE AT YOUR OWN RISK!)")
 ASAN_FLAG(bool, use_odr_indicator, false,
           "Use special ODR indicator symbol for ODR violation detection")
+ASAN_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true,
+          "realloc(p, 0) is equivalent to free(p) by default (Same as the "
+          "POSIX standard). If set to false, realloc(p, 0) will return a "
+          "pointer to an allocated space which can not be used.")
+ASAN_FLAG(bool, verify_asan_link_order, true,
+          "Check position of ASan runtime in library list (needs to be disabled"
+          " when other library has to be preloaded system-wide)")
diff --git a/libsanitizer/asan/asan_fuchsia.cc b/libsanitizer/asan/asan_fuchsia.cc
new file mode 100644
index 00000000000..6b1b48921ef
--- /dev/null
+++ b/libsanitizer/asan/asan_fuchsia.cc
@@ -0,0 +1,216 @@
+//===-- asan_fuchsia.cc --------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Fuchsia-specific details.
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "asan_interceptors.h"
+#include "asan_internal.h"
+#include "asan_stack.h"
+#include "asan_thread.h"
+
+#include <limits.h>
+#include <zircon/sanitizer.h>
+#include <zircon/syscalls.h>
+#include <zircon/threads.h>
+
+namespace __asan {
+
+// The system already set up the shadow memory for us.
+// __sanitizer::GetMaxVirtualAddress has already been called by
+// AsanInitInternal->InitializeHighMemEnd (asan_rtl.cc).
+// Just do some additional sanity checks here.
+void InitializeShadowMemory() {
+  if (Verbosity()) PrintAddressSpaceLayout();
+
+  // Make sure SHADOW_OFFSET doesn't use __asan_shadow_memory_dynamic_address.
+  __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
+  DCHECK(kLowShadowBeg != kDefaultShadowSentinel);
+  __asan_shadow_memory_dynamic_address = kLowShadowBeg;
+
+  CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1);
+  CHECK_EQ(kHighMemEnd, __sanitizer::ShadowBounds.memory_limit - 1);
+  CHECK_EQ(kHighMemBeg, __sanitizer::ShadowBounds.shadow_limit);
+  CHECK_EQ(kHighShadowBeg, __sanitizer::ShadowBounds.shadow_base);
+  CHECK_EQ(kShadowGapEnd, __sanitizer::ShadowBounds.shadow_base - 1);
+  CHECK_EQ(kLowShadowEnd, 0);
+  CHECK_EQ(kLowShadowBeg, 0);
+}
+
+void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
+  UNIMPLEMENTED();
+}
+
+void AsanCheckDynamicRTPrereqs() {}
+void AsanCheckIncompatibleRT() {}
+void InitializeAsanInterceptors() {}
+
+void *AsanDoesNotSupportStaticLinkage() { return nullptr; }
+
+void InitializePlatformExceptionHandlers() {}
+void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+  UNIMPLEMENTED();
+}
+
+// We can use a plain thread_local variable for TSD.
+static thread_local void *per_thread;
+
+void *AsanTSDGet() { return per_thread; }
+
+void AsanTSDSet(void *tsd) { per_thread = tsd; }
+
+// There's no initialization needed, and the passed-in destructor
+// will never be called.  Instead, our own thread destruction hook
+// (below) will call AsanThread::TSDDtor directly.
+void AsanTSDInit(void (*destructor)(void *tsd)) {
+  DCHECK(destructor == &PlatformTSDDtor);
+}
+
+void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); }
+
+static inline size_t AsanThreadMmapSize() {
+  return RoundUpTo(sizeof(AsanThread), PAGE_SIZE);
+}
+
+struct AsanThread::InitOptions {
+  uptr stack_bottom, stack_size;
+};
+
+// Shared setup between thread creation and startup for the initial thread.
+static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
+                                    uptr user_id, bool detached,
+                                    const char *name, uptr stack_bottom,
+                                    uptr stack_size) {
+  // In lieu of AsanThread::Create.
+  AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__);
+
+  AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
+  u32 tid =
+      asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args);
+  asanThreadRegistry().SetThreadName(tid, name);
+
+  // On other systems, AsanThread::Init() is called from the new
+  // thread itself.  But on Fuchsia we already know the stack address
+  // range beforehand, so we can do most of the setup right now.
+  const AsanThread::InitOptions options = {stack_bottom, stack_size};
+  thread->Init(&options);
+
+  return thread;
+}
+
+// This gets the same arguments passed to Init by CreateAsanThread, above.
+// We're in the creator thread before the new thread is actually started,
+// but its stack address range is already known.  We don't bother tracking
+// the static TLS address range because the system itself already uses an
+// ASan-aware allocator for that.
+void AsanThread::SetThreadStackAndTls(const AsanThread::InitOptions *options) {
+  DCHECK_NE(GetCurrentThread(), this);
+  DCHECK_NE(GetCurrentThread(), nullptr);
+  CHECK_NE(options->stack_bottom, 0);
+  CHECK_NE(options->stack_size, 0);
+  stack_bottom_ = options->stack_bottom;
+  stack_top_ = options->stack_bottom + options->stack_size;
+}
+
+// Called by __asan::AsanInitInternal (asan_rtl.c).
+AsanThread *CreateMainThread() {
+  thrd_t self = thrd_current();
+  char name[ZX_MAX_NAME_LEN];
+  CHECK_NE(__sanitizer::MainThreadStackBase, 0);
+  CHECK_GT(__sanitizer::MainThreadStackSize, 0);
+  AsanThread *t = CreateAsanThread(
+      nullptr, 0, reinterpret_cast<uptr>(self), true,
+      _zx_object_get_property(thrd_get_zx_handle(self), ZX_PROP_NAME, name,
+                              sizeof(name)) == ZX_OK
+          ? name
+          : nullptr,
+      __sanitizer::MainThreadStackBase, __sanitizer::MainThreadStackSize);
+  SetCurrentThread(t);
+  return t;
+}
+
+// This is called before each thread creation is attempted.  So, in
+// its first call, the calling thread is the initial and sole thread.
+static void *BeforeThreadCreateHook(uptr user_id, bool detached,
+                                    const char *name, uptr stack_bottom,
+                                    uptr stack_size) {
+  EnsureMainThreadIDIsCorrect();
+  // Strict init-order checking is thread-hostile.
+  if (flags()->strict_init_order) StopInitOrderChecking();
+
+  GET_STACK_TRACE_THREAD;
+  u32 parent_tid = GetCurrentTidOrInvalid();
+
+  return CreateAsanThread(&stack, parent_tid, user_id, detached, name,
+                          stack_bottom, stack_size);
+}
+
+// This is called after creating a new thread (in the creating thread),
+// with the pointer returned by BeforeThreadCreateHook (above).
+static void ThreadCreateHook(void *hook, bool aborted) {
+  AsanThread *thread = static_cast<AsanThread *>(hook);
+  if (!aborted) {
+    // The thread was created successfully.
+    // ThreadStartHook is already running in the new thread.
+  } else {
+    // The thread wasn't created after all.
+    // Clean up everything we set up in BeforeThreadCreateHook.
+    asanThreadRegistry().FinishThread(thread->tid());
+    UnmapOrDie(thread, AsanThreadMmapSize());
+  }
+}
+
+// This is called in the newly-created thread before it runs anything else,
+// with the pointer returned by BeforeThreadCreateHook (above).
+// cf. asan_interceptors.cc:asan_thread_start
+static void ThreadStartHook(void *hook, uptr os_id) {
+  AsanThread *thread = static_cast<AsanThread *>(hook);
+  SetCurrentThread(thread);
+
+  // In lieu of AsanThread::ThreadStart.
+  asanThreadRegistry().StartThread(thread->tid(), os_id, /*workerthread*/ false,
+                                   nullptr);
+}
+
+// Each thread runs this just before it exits,
+// with the pointer returned by BeforeThreadCreateHook (above).
+// All per-thread destructors have already been called.
+static void ThreadExitHook(void *hook, uptr os_id) {
+  AsanThread::TSDDtor(per_thread);
+}
+
+}  // namespace __asan
+
+// These are declared (in extern "C") by <zircon/sanitizer.h>.
+// The system runtime will call our definitions directly.
+
+void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
+                                            const char *name, void *stack_base,
+                                            size_t stack_size) {
+  return __asan::BeforeThreadCreateHook(
+      reinterpret_cast<uptr>(thread), detached, name,
+      reinterpret_cast<uptr>(stack_base), stack_size);
+}
+
+void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) {
+  __asan::ThreadCreateHook(hook, error != thrd_success);
+}
+
+void __sanitizer_thread_start_hook(void *hook, thrd_t self) {
+  __asan::ThreadStartHook(hook, reinterpret_cast<uptr>(self));
+}
+
+void __sanitizer_thread_exit_hook(void *hook, thrd_t self) {
+  __asan::ThreadExitHook(hook, reinterpret_cast<uptr>(self));
+}
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/libsanitizer/asan/asan_globals.cc b/libsanitizer/asan/asan_globals.cc
index f2292926e6a..c33b0ac7561 100644
--- a/libsanitizer/asan/asan_globals.cc
+++ b/libsanitizer/asan/asan_globals.cc
@@ -311,6 +311,26 @@ void __asan_unregister_image_globals(uptr *flag) {
   *flag = 0;
 }
 
+void __asan_register_elf_globals(uptr *flag, void *start, void *stop) {
+  if (*flag) return;
+  if (!start) return;
+  CHECK_EQ(0, ((uptr)stop - (uptr)start) % sizeof(__asan_global));
+  __asan_global *globals_start = (__asan_global*)start;
+  __asan_global *globals_stop = (__asan_global*)stop;
+  __asan_register_globals(globals_start, globals_stop - globals_start);
+  *flag = 1;
+}
+
+void __asan_unregister_elf_globals(uptr *flag, void *start, void *stop) {
+  if (!*flag) return;
+  if (!start) return;
+  CHECK_EQ(0, ((uptr)stop - (uptr)start) % sizeof(__asan_global));
+  __asan_global *globals_start = (__asan_global*)start;
+  __asan_global *globals_stop = (__asan_global*)stop;
+  __asan_unregister_globals(globals_start, globals_stop - globals_start);
+  *flag = 0;
+}
+
 // Register an array of globals.
 void __asan_register_globals(__asan_global *globals, uptr n) {
   if (!flags()->report_globals) return;
@@ -327,8 +347,26 @@ void __asan_register_globals(__asan_global *globals, uptr n) {
     Printf("=== ID %d; %p %p\n", stack_id, &globals[0], &globals[n - 1]);
   }
   for (uptr i = 0; i < n; i++) {
+    if (SANITIZER_WINDOWS && globals[i].beg == 0) {
+      // The MSVC incremental linker may pad globals out to 256 bytes. As long
+      // as __asan_global is less than 256 bytes large and its size is a power
+      // of two, we can skip over the padding.
+      static_assert(
+          sizeof(__asan_global) < 256 &&
+              (sizeof(__asan_global) & (sizeof(__asan_global) - 1)) == 0,
+          "sizeof(__asan_global) incompatible with incremental linker padding");
+      // If these are padding bytes, the rest of the global should be zero.
+      CHECK(globals[i].size == 0 && globals[i].size_with_redzone == 0 &&
+            globals[i].name == nullptr && globals[i].module_name == nullptr &&
+            globals[i].odr_indicator == 0);
+      continue;
+    }
     RegisterGlobal(&globals[i]);
   }
+
+  // Poison the metadata. It should not be accessible to user code.
+  PoisonShadow(reinterpret_cast<uptr>(globals), n * sizeof(__asan_global),
+               kAsanGlobalRedzoneMagic);
 }
 
 // Unregister an array of globals.
@@ -337,8 +375,16 @@ void __asan_unregister_globals(__asan_global *globals, uptr n) {
   if (!flags()->report_globals) return;
   BlockingMutexLock lock(&mu_for_globals);
   for (uptr i = 0; i < n; i++) {
+    if (SANITIZER_WINDOWS && globals[i].beg == 0) {
+      // Skip globals that look like padding from the MSVC incremental linker.
+      // See comment in __asan_register_globals.
+      continue;
+    }
     UnregisterGlobal(&globals[i]);
   }
+
+  // Unpoison the metadata.
+  PoisonShadow(reinterpret_cast<uptr>(globals), n * sizeof(__asan_global), 0);
 }
 
 // This method runs immediately prior to dynamic initialization in each TU,
diff --git a/libsanitizer/asan/asan_globals_win.cc b/libsanitizer/asan/asan_globals_win.cc
new file mode 100644
index 00000000000..118c0ac991c
--- /dev/null
+++ b/libsanitizer/asan/asan_globals_win.cc
@@ -0,0 +1,60 @@
+//===-- asan_globals_win.cc -----------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Global registration code that is linked into every Windows DLL and EXE.
+//
+//===----------------------------------------------------------------------===//
+
+#include "asan_interface_internal.h"
+#if SANITIZER_WINDOWS
+
+namespace __asan {
+
+#pragma section(".ASAN$GA", read, write)  // NOLINT
+#pragma section(".ASAN$GZ", read, write)  // NOLINT
+extern "C" __declspec(allocate(".ASAN$GA"))
+__asan_global __asan_globals_start = {};
+extern "C" __declspec(allocate(".ASAN$GZ"))
+__asan_global __asan_globals_end = {};
+#pragma comment(linker, "/merge:.ASAN=.data")
+
+static void call_on_globals(void (*hook)(__asan_global *, uptr)) {
+  __asan_global *start = &__asan_globals_start + 1;
+  __asan_global *end = &__asan_globals_end;
+  uptr bytediff = (uptr)end - (uptr)start;
+  if (bytediff % sizeof(__asan_global) != 0) {
+#ifdef SANITIZER_DLL_THUNK
+    __debugbreak();
+#else
+    CHECK("corrupt asan global array");
+#endif
+  }
+  // We know end >= start because the linker sorts the portion after the dollar
+  // sign alphabetically.
+  uptr n = end - start;
+  hook(start, n);
+}
+
+static void register_dso_globals() {
+  call_on_globals(&__asan_register_globals);
+}
+
+static void unregister_dso_globals() {
+  call_on_globals(&__asan_unregister_globals);
+}
+
+// Register globals
+#pragma section(".CRT$XCU", long, read)  // NOLINT
+#pragma section(".CRT$XTX", long, read)  // NOLINT
+extern "C" __declspec(allocate(".CRT$XCU"))
+void (*const __asan_dso_reg_hook)() = &register_dso_globals;
+extern "C" __declspec(allocate(".CRT$XTX"))
+void (*const __asan_dso_unreg_hook)() = &unregister_dso_globals;
+
+} // namespace __asan
+
+#endif  // SANITIZER_WINDOWS
diff --git a/libsanitizer/asan/asan_interceptors.cc b/libsanitizer/asan/asan_interceptors.cc
index 743abe51481..a8f4b72723f 100644
--- a/libsanitizer/asan/asan_interceptors.cc
+++ b/libsanitizer/asan/asan_interceptors.cc
@@ -22,6 +22,11 @@
 #include "lsan/lsan_common.h"
 #include "sanitizer_common/sanitizer_libc.h"
 
+// There is no general interception at all on Fuchsia.
+// Only the functions in asan_interceptors_memintrinsics.cc are
+// really defined to replace libc functions.
+#if !SANITIZER_FUCHSIA
+
 #if SANITIZER_POSIX
 #include "sanitizer_common/sanitizer_posix.h"
 #endif
@@ -34,56 +39,6 @@
 
 namespace __asan {
 
-// Return true if we can quickly decide that the region is unpoisoned.
-static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) {
-  if (size == 0) return true;
-  if (size <= 32)
-    return !AddressIsPoisoned(beg) &&
-           !AddressIsPoisoned(beg + size - 1) &&
-           !AddressIsPoisoned(beg + size / 2);
-  return false;
-}
-
-struct AsanInterceptorContext {
-  const char *interceptor_name;
-};
-
-// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE,
-// and ASAN_WRITE_RANGE as macro instead of function so
-// that no extra frames are created, and stack trace contains
-// relevant information only.
-// We check all shadow bytes.
-#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do {            \
-    uptr __offset = (uptr)(offset);                                     \
-    uptr __size = (uptr)(size);                                         \
-    uptr __bad = 0;                                                     \
-    if (__offset > __offset + __size) {                                 \
-      GET_STACK_TRACE_FATAL_HERE;                                       \
-      ReportStringFunctionSizeOverflow(__offset, __size, &stack);       \
-    }                                                                   \
-    if (!QuickCheckForUnpoisonedRegion(__offset, __size) &&             \
-        (__bad = __asan_region_is_poisoned(__offset, __size))) {        \
-      AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx;     \
-      bool suppressed = false;                                          \
-      if (_ctx) {                                                       \
-        suppressed = IsInterceptorSuppressed(_ctx->interceptor_name);   \
-        if (!suppressed && HaveStackTraceBasedSuppressions()) {         \
-          GET_STACK_TRACE_FATAL_HERE;                                   \
-          suppressed = IsStackTraceSuppressed(&stack);                  \
-        }                                                               \
-      }                                                                 \
-      if (!suppressed) {                                                \
-        GET_CURRENT_PC_BP_SP;                                           \
-        ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\
-      }                                                                 \
-    }                                                                   \
-  } while (0)
-
-#define ASAN_READ_RANGE(ctx, offset, size) \
-  ACCESS_MEMORY_RANGE(ctx, offset, size, false)
-#define ASAN_WRITE_RANGE(ctx, offset, size) \
-  ACCESS_MEMORY_RANGE(ctx, offset, size, true)
-
 #define ASAN_READ_STRING_OF_LEN(ctx, s, len, n)                 \
   ASAN_READ_RANGE((ctx), (s),                                   \
     common_flags()->strict_string_checks ? (len) + 1 : (n))
@@ -91,23 +46,6 @@ struct AsanInterceptorContext {
 #define ASAN_READ_STRING(ctx, s, n)                             \
   ASAN_READ_STRING_OF_LEN((ctx), (s), REAL(strlen)(s), (n))
 
-// Behavior of functions like "memcpy" or "strcpy" is undefined
-// if memory intervals overlap. We report error in this case.
-// Macro is used to avoid creation of new frames.
-static inline bool RangesOverlap(const char *offset1, uptr length1,
-                                 const char *offset2, uptr length2) {
-  return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1));
-}
-#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \
-  const char *offset1 = (const char*)_offset1; \
-  const char *offset2 = (const char*)_offset2; \
-  if (RangesOverlap(offset1, length1, offset2, length2)) { \
-    GET_STACK_TRACE_FATAL_HERE; \
-    ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \
-                                            offset2, length2, &stack); \
-  } \
-} while (0)
-
 static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) {
 #if SANITIZER_INTERCEPT_STRNLEN
   if (REAL(strnlen)) {
@@ -124,6 +62,10 @@ void SetThreadName(const char *name) {
 }
 
 int OnExit() {
+  if (CAN_SANITIZE_LEAKS && common_flags()->detect_leaks &&
+      __lsan::HasReportedLeaks()) {
+    return common_flags()->exitcode;
+  }
   // FIXME: ask frontend whether we need to return failure.
   return 0;
 }
@@ -181,13 +123,14 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
 // Strict init-order checking is dlopen-hostile:
 // https://github.com/google/sanitizers/issues/178
 #define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag)                           \
-  if (flags()->strict_init_order) {                                            \
-    StopInitOrderChecking();                                                   \
-  }
+  do {                                                                         \
+    if (flags()->strict_init_order)                                            \
+      StopInitOrderChecking();                                                 \
+    CheckNoDeepBind(filename, flag);                                           \
+  } while (false)
 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
-#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
-  CoverageUpdateMapping()
-#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() CoverageUpdateMapping()
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle)
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED()
 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!asan_inited)
 #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end)                           \
   if (AsanThread *t = GetCurrentThread()) {                                    \
@@ -196,11 +139,27 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
   } else {                                                                     \
     *begin = *end = 0;                                                         \
   }
-// Asan needs custom handling of these:
-#undef SANITIZER_INTERCEPT_MEMSET
-#undef SANITIZER_INTERCEPT_MEMMOVE
-#undef SANITIZER_INTERCEPT_MEMCPY
+
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \
+  do {                                                       \
+    ASAN_INTERCEPTOR_ENTER(ctx, memmove);                    \
+    ASAN_MEMMOVE_IMPL(ctx, to, from, size);                  \
+  } while (false)
+
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \
+  do {                                                      \
+    ASAN_INTERCEPTOR_ENTER(ctx, memcpy);                    \
+    ASAN_MEMCPY_IMPL(ctx, to, from, size);                  \
+  } while (false)
+
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \
+  do {                                                      \
+    ASAN_INTERCEPTOR_ENTER(ctx, memset);                    \
+    ASAN_MEMSET_IMPL(ctx, block, c, size);                  \
+  } while (false)
+
 #include "sanitizer_common/sanitizer_common_interceptors.inc"
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
 
 // Syscall interceptors don't have contexts, we don't support suppressions
 // for them.
@@ -282,48 +241,6 @@ INTERCEPTOR(int, pthread_join, void *t, void **arg) {
 DEFINE_REAL_PTHREAD_FUNCTIONS
 #endif  // ASAN_INTERCEPT_PTHREAD_CREATE
 
-#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION
-
-#if SANITIZER_ANDROID
-INTERCEPTOR(void*, bsd_signal, int signum, void *handler) {
-  if (!IsHandledDeadlySignal(signum) ||
-      common_flags()->allow_user_segv_handler) {
-    return REAL(bsd_signal)(signum, handler);
-  }
-  return 0;
-}
-#endif
-
-INTERCEPTOR(void*, signal, int signum, void *handler) {
-  if (!IsHandledDeadlySignal(signum) ||
-      common_flags()->allow_user_segv_handler) {
-    return REAL(signal)(signum, handler);
-  }
-  return nullptr;
-}
-
-INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act,
-                            struct sigaction *oldact) {
-  if (!IsHandledDeadlySignal(signum) ||
-      common_flags()->allow_user_segv_handler) {
-    return REAL(sigaction)(signum, act, oldact);
-  }
-  return 0;
-}
-
-namespace __sanitizer {
-int real_sigaction(int signum, const void *act, void *oldact) {
-  return REAL(sigaction)(signum, (const struct sigaction *)act,
-                         (struct sigaction *)oldact);
-}
-} // namespace __sanitizer
-
-#elif SANITIZER_POSIX
-// We need to have defined REAL(sigaction) on posix systems.
-DEFINE_REAL(int, sigaction, int signum, const struct sigaction *act,
-    struct sigaction *oldact)
-#endif  // ASAN_INTERCEPT_SIGNAL_AND_SIGACTION
-
 #if ASAN_INTERCEPT_SWAPCONTEXT
 static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) {
   // Align to page size.
@@ -360,6 +277,11 @@ INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp,
 }
 #endif  // ASAN_INTERCEPT_SWAPCONTEXT
 
+#if SANITIZER_NETBSD
+#define longjmp __longjmp14
+#define siglongjmp __siglongjmp14
+#endif
+
 INTERCEPTOR(void, longjmp, void *env, int val) {
   __asan_handle_no_return();
   REAL(longjmp)(env, val);
@@ -372,6 +294,13 @@ INTERCEPTOR(void, _longjmp, void *env, int val) {
 }
 #endif
 
+#if ASAN_INTERCEPT___LONGJMP_CHK
+INTERCEPTOR(void, __longjmp_chk, void *env, int val) {
+  __asan_handle_no_return();
+  REAL(__longjmp_chk)(env, val);
+}
+#endif
+
 #if ASAN_INTERCEPT_SIGLONGJMP
 INTERCEPTOR(void, siglongjmp, void *env, int val) {
   __asan_handle_no_return();
@@ -387,90 +316,6 @@ INTERCEPTOR(void, __cxa_throw, void *a, void *b, void *c) {
 }
 #endif
 
-// memcpy is called during __asan_init() from the internals of printf(...).
-// We do not treat memcpy with to==from as a bug.
-// See http://llvm.org/bugs/show_bug.cgi?id=11763.
-#define ASAN_MEMCPY_IMPL(ctx, to, from, size) do {                             \
-    if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size);        \
-    if (asan_init_is_running) {                                                \
-      return REAL(memcpy)(to, from, size);                                     \
-    }                                                                          \
-    ENSURE_ASAN_INITED();                                                      \
-    if (flags()->replace_intrin) {                                             \
-      if (to != from) {                                                        \
-        CHECK_RANGES_OVERLAP("memcpy", to, size, from, size);                  \
-      }                                                                        \
-      ASAN_READ_RANGE(ctx, from, size);                                        \
-      ASAN_WRITE_RANGE(ctx, to, size);                                         \
-    }                                                                          \
-    return REAL(memcpy)(to, from, size);                                       \
-  } while (0)
-
-
-void *__asan_memcpy(void *to, const void *from, uptr size) {
-  ASAN_MEMCPY_IMPL(nullptr, to, from, size);
-}
-
-// memset is called inside Printf.
-#define ASAN_MEMSET_IMPL(ctx, block, c, size) do {                             \
-    if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size);        \
-    if (asan_init_is_running) {                                                \
-      return REAL(memset)(block, c, size);                                     \
-    }                                                                          \
-    ENSURE_ASAN_INITED();                                                      \
-    if (flags()->replace_intrin) {                                             \
-      ASAN_WRITE_RANGE(ctx, block, size);                                      \
-    }                                                                          \
-    return REAL(memset)(block, c, size);                                       \
-  } while (0)
-
-void *__asan_memset(void *block, int c, uptr size) {
-  ASAN_MEMSET_IMPL(nullptr, block, c, size);
-}
-
-#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) do {                            \
-    if (UNLIKELY(!asan_inited))                                                \
-      return internal_memmove(to, from, size);                                 \
-    ENSURE_ASAN_INITED();                                                      \
-    if (flags()->replace_intrin) {                                             \
-      ASAN_READ_RANGE(ctx, from, size);                                        \
-      ASAN_WRITE_RANGE(ctx, to, size);                                         \
-    }                                                                          \
-    return internal_memmove(to, from, size);                                   \
-  } while (0)
-
-void *__asan_memmove(void *to, const void *from, uptr size) {
-  ASAN_MEMMOVE_IMPL(nullptr, to, from, size);
-}
-
-INTERCEPTOR(void*, memmove, void *to, const void *from, uptr size) {
-  void *ctx;
-  ASAN_INTERCEPTOR_ENTER(ctx, memmove);
-  ASAN_MEMMOVE_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void*, memcpy, void *to, const void *from, uptr size) {
-  void *ctx;
-  ASAN_INTERCEPTOR_ENTER(ctx, memcpy);
-#if !SANITIZER_MAC
-  ASAN_MEMCPY_IMPL(ctx, to, from, size);
-#else
-  // At least on 10.7 and 10.8 both memcpy() and memmove() are being replaced
-  // with WRAP(memcpy). As a result, false positives are reported for memmove()
-  // calls. If we just disable error reporting with
-  // ASAN_OPTIONS=replace_intrin=0, memmove() is still replaced with
-  // internal_memcpy(), which may lead to crashes, see
-  // http://llvm.org/bugs/show_bug.cgi?id=16362.
-  ASAN_MEMMOVE_IMPL(ctx, to, from, size);
-#endif  // !SANITIZER_MAC
-}
-
-INTERCEPTOR(void*, memset, void *block, int c, uptr size) {
-  void *ctx;
-  ASAN_INTERCEPTOR_ENTER(ctx, memset);
-  ASAN_MEMSET_IMPL(ctx, block, c, size);
-}
-
 #if ASAN_INTERCEPT_INDEX
 # if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX
 INTERCEPTOR(char*, index, const char *string, int c)
@@ -580,17 +425,6 @@ INTERCEPTOR(char*, __strdup, const char *s) {
 }
 #endif // ASAN_INTERCEPT___STRDUP
 
-INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {
-  void *ctx;
-  ASAN_INTERCEPTOR_ENTER(ctx, wcslen);
-  SIZE_T length = internal_wcslen(s);
-  if (!asan_init_is_running) {
-    ENSURE_ASAN_INITED();
-    ASAN_READ_RANGE(ctx, s, (length + 1) * sizeof(wchar_t));
-  }
-  return length;
-}
-
 INTERCEPTOR(char*, strncpy, char *to, const char *from, uptr size) {
   void *ctx;
   ASAN_INTERCEPTOR_ENTER(ctx, strncpy);
@@ -707,9 +541,7 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
 #if ASAN_INTERCEPT_FORK
 INTERCEPTOR(int, fork, void) {
   ENSURE_ASAN_INITED();
-  if (common_flags()->coverage) CovBeforeFork();
   int pid = REAL(fork)();
-  if (common_flags()->coverage) CovAfterFork(pid);
   return pid;
 }
 #endif  // ASAN_INTERCEPT_FORK
@@ -721,22 +553,11 @@ void InitializeAsanInterceptors() {
   CHECK(!was_called_once);
   was_called_once = true;
   InitializeCommonInterceptors();
-
-  // Intercept mem* functions.
-  ASAN_INTERCEPT_FUNC(memmove);
-  ASAN_INTERCEPT_FUNC(memset);
-  if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) {
-    // In asan, REAL(memmove) is not used, but it is used in msan.
-    ASAN_INTERCEPT_FUNC(memcpy);
-  } else {
-    ASSIGN_REAL(memcpy, memmove);
-  }
-  CHECK(REAL(memcpy));
+  InitializeSignalInterceptors();
 
   // Intercept str* functions.
   ASAN_INTERCEPT_FUNC(strcat);  // NOLINT
   ASAN_INTERCEPT_FUNC(strcpy);  // NOLINT
-  ASAN_INTERCEPT_FUNC(wcslen);
   ASAN_INTERCEPT_FUNC(strncat);
   ASAN_INTERCEPT_FUNC(strncpy);
   ASAN_INTERCEPT_FUNC(strdup);
@@ -755,21 +576,18 @@ void InitializeAsanInterceptors() {
   ASAN_INTERCEPT_FUNC(strtoll);
 #endif
 
-  // Intecept signal- and jump-related functions.
+  // Intecept jump-related functions.
   ASAN_INTERCEPT_FUNC(longjmp);
-#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION
-  ASAN_INTERCEPT_FUNC(sigaction);
-#if SANITIZER_ANDROID
-  ASAN_INTERCEPT_FUNC(bsd_signal);
-#endif
-  ASAN_INTERCEPT_FUNC(signal);
-#endif
+
 #if ASAN_INTERCEPT_SWAPCONTEXT
   ASAN_INTERCEPT_FUNC(swapcontext);
 #endif
 #if ASAN_INTERCEPT__LONGJMP
   ASAN_INTERCEPT_FUNC(_longjmp);
 #endif
+#if ASAN_INTERCEPT___LONGJMP_CHK
+  ASAN_INTERCEPT_FUNC(__longjmp_chk);
+#endif
 #if ASAN_INTERCEPT_SIGLONGJMP
   ASAN_INTERCEPT_FUNC(siglongjmp);
 #endif
@@ -804,3 +622,5 @@ void InitializeAsanInterceptors() {
 }
 
 } // namespace __asan
+
+#endif  // !SANITIZER_FUCHSIA
diff --git a/libsanitizer/asan/asan_interceptors.h b/libsanitizer/asan/asan_interceptors.h
index 7053bb7faf5..e20d1af15a6 100644
--- a/libsanitizer/asan/asan_interceptors.h
+++ b/libsanitizer/asan/asan_interceptors.h
@@ -13,9 +13,30 @@
 #define ASAN_INTERCEPTORS_H
 
 #include "asan_internal.h"
+#include "asan_interceptors_memintrinsics.h"
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 
+namespace __asan {
+
+void InitializeAsanInterceptors();
+void InitializePlatformInterceptors();
+
+#define ENSURE_ASAN_INITED()      \
+  do {                            \
+    CHECK(!asan_init_is_running); \
+    if (UNLIKELY(!asan_inited)) { \
+      AsanInitFromRtl();          \
+    }                             \
+  } while (0)
+
+}  // namespace __asan
+
+// There is no general interception at all on Fuchsia.
+// Only the functions in asan_interceptors_memintrinsics.h are
+// really defined to replace libc functions.
+#if !SANITIZER_FUCHSIA
+
 // Use macro to describe if specific function should be
 // intercepted on a given platform.
 #if !SANITIZER_WINDOWS
@@ -32,7 +53,7 @@
 # define ASAN_INTERCEPT_FORK 0
 #endif
 
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 # define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 1
 #else
 # define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 0
@@ -45,15 +66,15 @@
 #endif
 
 #if !SANITIZER_WINDOWS
-# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 1
+# define ASAN_INTERCEPT_SIGLONGJMP 1
 #else
-# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 0
+# define ASAN_INTERCEPT_SIGLONGJMP 0
 #endif
 
-#if !SANITIZER_WINDOWS
-# define ASAN_INTERCEPT_SIGLONGJMP 1
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+# define ASAN_INTERCEPT___LONGJMP_CHK 1
 #else
-# define ASAN_INTERCEPT_SIGLONGJMP 0
+# define ASAN_INTERCEPT___LONGJMP_CHK 0
 #endif
 
 // Android bug: https://code.google.com/p/android/issues/detail?id=61799
@@ -77,8 +98,6 @@
 #endif
 
 DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size)
-DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size)
-DECLARE_REAL(void*, memset, void *block, int c, uptr size)
 DECLARE_REAL(char*, strchr, const char *str, int c)
 DECLARE_REAL(SIZE_T, strlen, const char *s)
 DECLARE_REAL(char*, strncpy, char *to, const char *from, uptr size)
@@ -105,18 +124,6 @@ DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act,
 #define ASAN_INTERCEPT_FUNC(name)
 #endif  // SANITIZER_MAC
 
-namespace __asan {
-
-void InitializeAsanInterceptors();
-void InitializePlatformInterceptors();
-
-#define ENSURE_ASAN_INITED() do { \
-  CHECK(!asan_init_is_running); \
-  if (UNLIKELY(!asan_inited)) { \
-    AsanInitFromRtl(); \
-  } \
-} while (0)
-
-}  // namespace __asan
+#endif  // !SANITIZER_FUCHSIA
 
 #endif  // ASAN_INTERCEPTORS_H
diff --git a/libsanitizer/asan/asan_interceptors_memintrinsics.cc b/libsanitizer/asan/asan_interceptors_memintrinsics.cc
new file mode 100644
index 00000000000..16de54cfeef
--- /dev/null
+++ b/libsanitizer/asan/asan_interceptors_memintrinsics.cc
@@ -0,0 +1,42 @@
+//===-- asan_interceptors_memintrinsics.cc --------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// ASan versions of memcpy, memmove, and memset.
+//===---------------------------------------------------------------------===//
+
+#include "asan_interceptors_memintrinsics.h"
+#include "asan_report.h"
+#include "asan_stack.h"
+#include "asan_suppressions.h"
+
+using namespace __asan;  // NOLINT
+
+void *__asan_memcpy(void *to, const void *from, uptr size) {
+  ASAN_MEMCPY_IMPL(nullptr, to, from, size);
+}
+
+void *__asan_memset(void *block, int c, uptr size) {
+  ASAN_MEMSET_IMPL(nullptr, block, c, size);
+}
+
+void *__asan_memmove(void *to, const void *from, uptr size) {
+  ASAN_MEMMOVE_IMPL(nullptr, to, from, size);
+}
+
+#if SANITIZER_FUCHSIA
+
+// Fuchsia doesn't use sanitizer_common_interceptors.inc, but the only
+// things there it wants are these three.  Just define them as aliases
+// here rather than repeating the contents.
+
+decltype(memcpy) memcpy[[gnu::alias("__asan_memcpy")]];
+decltype(memmove) memmove[[gnu::alias("__asan_memmove")]];
+decltype(memset) memset[[gnu::alias("__asan_memset")]];
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/libsanitizer/asan/asan_interceptors_memintrinsics.h b/libsanitizer/asan/asan_interceptors_memintrinsics.h
new file mode 100644
index 00000000000..1dc4d64c7fa
--- /dev/null
+++ b/libsanitizer/asan/asan_interceptors_memintrinsics.h
@@ -0,0 +1,146 @@
+//===-- asan_interceptors_memintrinsics.h -----------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// ASan-private header for asan_memintrin.cc
+//===---------------------------------------------------------------------===//
+#ifndef ASAN_MEMINTRIN_H
+#define ASAN_MEMINTRIN_H
+
+#include "asan_interface_internal.h"
+#include "asan_internal.h"
+#include "asan_mapping.h"
+#include "interception/interception.h"
+
+DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size)
+DECLARE_REAL(void*, memset, void *block, int c, uptr size)
+
+namespace __asan {
+
+// Return true if we can quickly decide that the region is unpoisoned.
+// We assume that a redzone is at least 16 bytes.
+static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) {
+  if (size == 0) return true;
+  if (size <= 32)
+    return !AddressIsPoisoned(beg) &&
+           !AddressIsPoisoned(beg + size - 1) &&
+           !AddressIsPoisoned(beg + size / 2);
+  if (size <= 64)
+    return !AddressIsPoisoned(beg) &&
+           !AddressIsPoisoned(beg + size / 4) &&
+           !AddressIsPoisoned(beg + size - 1) &&
+           !AddressIsPoisoned(beg + 3 * size / 4) &&
+           !AddressIsPoisoned(beg + size / 2);
+  return false;
+}
+
+struct AsanInterceptorContext {
+  const char *interceptor_name;
+};
+
+// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE,
+// and ASAN_WRITE_RANGE as macro instead of function so
+// that no extra frames are created, and stack trace contains
+// relevant information only.
+// We check all shadow bytes.
+#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do {            \
+    uptr __offset = (uptr)(offset);                                     \
+    uptr __size = (uptr)(size);                                         \
+    uptr __bad = 0;                                                     \
+    if (__offset > __offset + __size) {                                 \
+      GET_STACK_TRACE_FATAL_HERE;                                       \
+      ReportStringFunctionSizeOverflow(__offset, __size, &stack);       \
+    }                                                                   \
+    if (!QuickCheckForUnpoisonedRegion(__offset, __size) &&             \
+        (__bad = __asan_region_is_poisoned(__offset, __size))) {        \
+      AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx;     \
+      bool suppressed = false;                                          \
+      if (_ctx) {                                                       \
+        suppressed = IsInterceptorSuppressed(_ctx->interceptor_name);   \
+        if (!suppressed && HaveStackTraceBasedSuppressions()) {         \
+          GET_STACK_TRACE_FATAL_HERE;                                   \
+          suppressed = IsStackTraceSuppressed(&stack);                  \
+        }                                                               \
+      }                                                                 \
+      if (!suppressed) {                                                \
+        GET_CURRENT_PC_BP_SP;                                           \
+        ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\
+      }                                                                 \
+    }                                                                   \
+  } while (0)
+
+// memcpy is called during __asan_init() from the internals of printf(...).
+// We do not treat memcpy with to==from as a bug.
+// See http://llvm.org/bugs/show_bug.cgi?id=11763.
+#define ASAN_MEMCPY_IMPL(ctx, to, from, size)                           \
+  do {                                                                  \
+    if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \
+    if (asan_init_is_running) {                                         \
+      return REAL(memcpy)(to, from, size);                              \
+    }                                                                   \
+    ENSURE_ASAN_INITED();                                               \
+    if (flags()->replace_intrin) {                                      \
+      if (to != from) {                                                 \
+        CHECK_RANGES_OVERLAP("memcpy", to, size, from, size);           \
+      }                                                                 \
+      ASAN_READ_RANGE(ctx, from, size);                                 \
+      ASAN_WRITE_RANGE(ctx, to, size);                                  \
+    }                                                                   \
+    return REAL(memcpy)(to, from, size);                                \
+  } while (0)
+
+// memset is called inside Printf.
+#define ASAN_MEMSET_IMPL(ctx, block, c, size)                           \
+  do {                                                                  \
+    if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \
+    if (asan_init_is_running) {                                         \
+      return REAL(memset)(block, c, size);                              \
+    }                                                                   \
+    ENSURE_ASAN_INITED();                                               \
+    if (flags()->replace_intrin) {                                      \
+      ASAN_WRITE_RANGE(ctx, block, size);                               \
+    }                                                                   \
+    return REAL(memset)(block, c, size);                                \
+  } while (0)
+
+#define ASAN_MEMMOVE_IMPL(ctx, to, from, size)                           \
+  do {                                                                   \
+    if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \
+    ENSURE_ASAN_INITED();                                                \
+    if (flags()->replace_intrin) {                                       \
+      ASAN_READ_RANGE(ctx, from, size);                                  \
+      ASAN_WRITE_RANGE(ctx, to, size);                                   \
+    }                                                                    \
+    return internal_memmove(to, from, size);                             \
+  } while (0)
+
+#define ASAN_READ_RANGE(ctx, offset, size) \
+  ACCESS_MEMORY_RANGE(ctx, offset, size, false)
+#define ASAN_WRITE_RANGE(ctx, offset, size) \
+  ACCESS_MEMORY_RANGE(ctx, offset, size, true)
+
+// Behavior of functions like "memcpy" or "strcpy" is undefined
+// if memory intervals overlap. We report error in this case.
+// Macro is used to avoid creation of new frames.
+static inline bool RangesOverlap(const char *offset1, uptr length1,
+                                 const char *offset2, uptr length2) {
+  return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1));
+}
+#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \
+  const char *offset1 = (const char*)_offset1; \
+  const char *offset2 = (const char*)_offset2; \
+  if (RangesOverlap(offset1, length1, offset2, length2)) { \
+    GET_STACK_TRACE_FATAL_HERE; \
+    ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \
+                                            offset2, length2, &stack); \
+  } \
+} while (0)
+
+}  // namespace __asan
+
+#endif  // ASAN_MEMINTRIN_H
diff --git a/libsanitizer/asan/asan_interface.inc b/libsanitizer/asan/asan_interface.inc
new file mode 100644
index 00000000000..b2fcde1c770
--- /dev/null
+++ b/libsanitizer/asan/asan_interface.inc
@@ -0,0 +1,167 @@
+//===-- asan_interface.inc ------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Asan interface list.
+//===----------------------------------------------------------------------===//
+INTERFACE_FUNCTION(__asan_addr_is_in_fake_stack)
+INTERFACE_FUNCTION(__asan_address_is_poisoned)
+INTERFACE_FUNCTION(__asan_after_dynamic_init)
+INTERFACE_FUNCTION(__asan_alloca_poison)
+INTERFACE_FUNCTION(__asan_allocas_unpoison)
+INTERFACE_FUNCTION(__asan_before_dynamic_init)
+INTERFACE_FUNCTION(__asan_describe_address)
+INTERFACE_FUNCTION(__asan_exp_load1)
+INTERFACE_FUNCTION(__asan_exp_load2)
+INTERFACE_FUNCTION(__asan_exp_load4)
+INTERFACE_FUNCTION(__asan_exp_load8)
+INTERFACE_FUNCTION(__asan_exp_load16)
+INTERFACE_FUNCTION(__asan_exp_loadN)
+INTERFACE_FUNCTION(__asan_exp_store1)
+INTERFACE_FUNCTION(__asan_exp_store2)
+INTERFACE_FUNCTION(__asan_exp_store4)
+INTERFACE_FUNCTION(__asan_exp_store8)
+INTERFACE_FUNCTION(__asan_exp_store16)
+INTERFACE_FUNCTION(__asan_exp_storeN)
+INTERFACE_FUNCTION(__asan_get_alloc_stack)
+INTERFACE_FUNCTION(__asan_get_current_fake_stack)
+INTERFACE_FUNCTION(__asan_get_free_stack)
+INTERFACE_FUNCTION(__asan_get_report_access_size)
+INTERFACE_FUNCTION(__asan_get_report_access_type)
+INTERFACE_FUNCTION(__asan_get_report_address)
+INTERFACE_FUNCTION(__asan_get_report_bp)
+INTERFACE_FUNCTION(__asan_get_report_description)
+INTERFACE_FUNCTION(__asan_get_report_pc)
+INTERFACE_FUNCTION(__asan_get_report_sp)
+INTERFACE_FUNCTION(__asan_get_shadow_mapping)
+INTERFACE_FUNCTION(__asan_handle_no_return)
+INTERFACE_FUNCTION(__asan_init)
+INTERFACE_FUNCTION(__asan_load_cxx_array_cookie)
+INTERFACE_FUNCTION(__asan_load1)
+INTERFACE_FUNCTION(__asan_load2)
+INTERFACE_FUNCTION(__asan_load4)
+INTERFACE_FUNCTION(__asan_load8)
+INTERFACE_FUNCTION(__asan_load16)
+INTERFACE_FUNCTION(__asan_loadN)
+INTERFACE_FUNCTION(__asan_load1_noabort)
+INTERFACE_FUNCTION(__asan_load2_noabort)
+INTERFACE_FUNCTION(__asan_load4_noabort)
+INTERFACE_FUNCTION(__asan_load8_noabort)
+INTERFACE_FUNCTION(__asan_load16_noabort)
+INTERFACE_FUNCTION(__asan_loadN_noabort)
+INTERFACE_FUNCTION(__asan_locate_address)
+INTERFACE_FUNCTION(__asan_memcpy)
+INTERFACE_FUNCTION(__asan_memmove)
+INTERFACE_FUNCTION(__asan_memset)
+INTERFACE_FUNCTION(__asan_poison_cxx_array_cookie)
+INTERFACE_FUNCTION(__asan_poison_intra_object_redzone)
+INTERFACE_FUNCTION(__asan_poison_memory_region)
+INTERFACE_FUNCTION(__asan_poison_stack_memory)
+INTERFACE_FUNCTION(__asan_print_accumulated_stats)
+INTERFACE_FUNCTION(__asan_region_is_poisoned)
+INTERFACE_FUNCTION(__asan_register_globals)
+INTERFACE_FUNCTION(__asan_register_elf_globals)
+INTERFACE_FUNCTION(__asan_register_image_globals)
+INTERFACE_FUNCTION(__asan_report_error)
+INTERFACE_FUNCTION(__asan_report_exp_load1)
+INTERFACE_FUNCTION(__asan_report_exp_load2)
+INTERFACE_FUNCTION(__asan_report_exp_load4)
+INTERFACE_FUNCTION(__asan_report_exp_load8)
+INTERFACE_FUNCTION(__asan_report_exp_load16)
+INTERFACE_FUNCTION(__asan_report_exp_load_n)
+INTERFACE_FUNCTION(__asan_report_exp_store1)
+INTERFACE_FUNCTION(__asan_report_exp_store2)
+INTERFACE_FUNCTION(__asan_report_exp_store4)
+INTERFACE_FUNCTION(__asan_report_exp_store8)
+INTERFACE_FUNCTION(__asan_report_exp_store16)
+INTERFACE_FUNCTION(__asan_report_exp_store_n)
+INTERFACE_FUNCTION(__asan_report_load1)
+INTERFACE_FUNCTION(__asan_report_load2)
+INTERFACE_FUNCTION(__asan_report_load4)
+INTERFACE_FUNCTION(__asan_report_load8)
+INTERFACE_FUNCTION(__asan_report_load16)
+INTERFACE_FUNCTION(__asan_report_load_n)
+INTERFACE_FUNCTION(__asan_report_load1_noabort)
+INTERFACE_FUNCTION(__asan_report_load2_noabort)
+INTERFACE_FUNCTION(__asan_report_load4_noabort)
+INTERFACE_FUNCTION(__asan_report_load8_noabort)
+INTERFACE_FUNCTION(__asan_report_load16_noabort)
+INTERFACE_FUNCTION(__asan_report_load_n_noabort)
+INTERFACE_FUNCTION(__asan_report_present)
+INTERFACE_FUNCTION(__asan_report_store1)
+INTERFACE_FUNCTION(__asan_report_store2)
+INTERFACE_FUNCTION(__asan_report_store4)
+INTERFACE_FUNCTION(__asan_report_store8)
+INTERFACE_FUNCTION(__asan_report_store16)
+INTERFACE_FUNCTION(__asan_report_store_n)
+INTERFACE_FUNCTION(__asan_report_store1_noabort)
+INTERFACE_FUNCTION(__asan_report_store2_noabort)
+INTERFACE_FUNCTION(__asan_report_store4_noabort)
+INTERFACE_FUNCTION(__asan_report_store8_noabort)
+INTERFACE_FUNCTION(__asan_report_store16_noabort)
+INTERFACE_FUNCTION(__asan_report_store_n_noabort)
+INTERFACE_FUNCTION(__asan_set_death_callback)
+INTERFACE_FUNCTION(__asan_set_error_report_callback)
+INTERFACE_FUNCTION(__asan_set_shadow_00)
+INTERFACE_FUNCTION(__asan_set_shadow_f1)
+INTERFACE_FUNCTION(__asan_set_shadow_f2)
+INTERFACE_FUNCTION(__asan_set_shadow_f3)
+INTERFACE_FUNCTION(__asan_set_shadow_f5)
+INTERFACE_FUNCTION(__asan_set_shadow_f8)
+INTERFACE_FUNCTION(__asan_stack_free_0)
+INTERFACE_FUNCTION(__asan_stack_free_1)
+INTERFACE_FUNCTION(__asan_stack_free_2)
+INTERFACE_FUNCTION(__asan_stack_free_3)
+INTERFACE_FUNCTION(__asan_stack_free_4)
+INTERFACE_FUNCTION(__asan_stack_free_5)
+INTERFACE_FUNCTION(__asan_stack_free_6)
+INTERFACE_FUNCTION(__asan_stack_free_7)
+INTERFACE_FUNCTION(__asan_stack_free_8)
+INTERFACE_FUNCTION(__asan_stack_free_9)
+INTERFACE_FUNCTION(__asan_stack_free_10)
+INTERFACE_FUNCTION(__asan_stack_malloc_0)
+INTERFACE_FUNCTION(__asan_stack_malloc_1)
+INTERFACE_FUNCTION(__asan_stack_malloc_2)
+INTERFACE_FUNCTION(__asan_stack_malloc_3)
+INTERFACE_FUNCTION(__asan_stack_malloc_4)
+INTERFACE_FUNCTION(__asan_stack_malloc_5)
+INTERFACE_FUNCTION(__asan_stack_malloc_6)
+INTERFACE_FUNCTION(__asan_stack_malloc_7)
+INTERFACE_FUNCTION(__asan_stack_malloc_8)
+INTERFACE_FUNCTION(__asan_stack_malloc_9)
+INTERFACE_FUNCTION(__asan_stack_malloc_10)
+INTERFACE_FUNCTION(__asan_store1)
+INTERFACE_FUNCTION(__asan_store2)
+INTERFACE_FUNCTION(__asan_store4)
+INTERFACE_FUNCTION(__asan_store8)
+INTERFACE_FUNCTION(__asan_store16)
+INTERFACE_FUNCTION(__asan_storeN)
+INTERFACE_FUNCTION(__asan_store1_noabort)
+INTERFACE_FUNCTION(__asan_store2_noabort)
+INTERFACE_FUNCTION(__asan_store4_noabort)
+INTERFACE_FUNCTION(__asan_store8_noabort)
+INTERFACE_FUNCTION(__asan_store16_noabort)
+INTERFACE_FUNCTION(__asan_storeN_noabort)
+INTERFACE_FUNCTION(__asan_unpoison_intra_object_redzone)
+INTERFACE_FUNCTION(__asan_unpoison_memory_region)
+INTERFACE_FUNCTION(__asan_unpoison_stack_memory)
+INTERFACE_FUNCTION(__asan_unregister_globals)
+INTERFACE_FUNCTION(__asan_unregister_elf_globals)
+INTERFACE_FUNCTION(__asan_unregister_image_globals)
+INTERFACE_FUNCTION(__asan_version_mismatch_check_v8)
+INTERFACE_FUNCTION(__sanitizer_finish_switch_fiber)
+INTERFACE_FUNCTION(__sanitizer_print_stack_trace)
+INTERFACE_FUNCTION(__sanitizer_ptr_cmp)
+INTERFACE_FUNCTION(__sanitizer_ptr_sub)
+INTERFACE_FUNCTION(__sanitizer_start_switch_fiber)
+INTERFACE_FUNCTION(__sanitizer_unaligned_load16)
+INTERFACE_FUNCTION(__sanitizer_unaligned_load32)
+INTERFACE_FUNCTION(__sanitizer_unaligned_load64)
+INTERFACE_FUNCTION(__sanitizer_unaligned_store16)
+INTERFACE_FUNCTION(__sanitizer_unaligned_store32)
+INTERFACE_FUNCTION(__sanitizer_unaligned_store64)
+INTERFACE_WEAK_FUNCTION(__asan_default_options)
+INTERFACE_WEAK_FUNCTION(__asan_default_suppressions)
+INTERFACE_WEAK_FUNCTION(__asan_on_error)
diff --git a/libsanitizer/asan/asan_interface_internal.h b/libsanitizer/asan/asan_interface_internal.h
index 05605a8065f..be9605d9e6e 100644
--- a/libsanitizer/asan/asan_interface_internal.h
+++ b/libsanitizer/asan/asan_interface_internal.h
@@ -65,6 +65,11 @@ extern "C" {
   SANITIZER_INTERFACE_ATTRIBUTE
   void __asan_unregister_image_globals(uptr *flag);
 
+  SANITIZER_INTERFACE_ATTRIBUTE
+  void __asan_register_elf_globals(uptr *flag, void *start, void *stop);
+  SANITIZER_INTERFACE_ATTRIBUTE
+  void __asan_unregister_elf_globals(uptr *flag, void *start, void *stop);
+
   // These two functions should be called by the instrumented code.
   // 'globals' is an array of structures describing 'n' globals.
   SANITIZER_INTERFACE_ATTRIBUTE
@@ -163,12 +168,12 @@ extern "C" {
   void __asan_set_error_report_callback(void (*callback)(const char*));
 
   SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-  /* OPTIONAL */ void __asan_on_error();
+  void __asan_on_error();
 
   SANITIZER_INTERFACE_ATTRIBUTE void __asan_print_accumulated_stats();
 
   SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-  /* OPTIONAL */ const char* __asan_default_options();
+  const char* __asan_default_options();
 
   SANITIZER_INTERFACE_ATTRIBUTE
   extern uptr __asan_shadow_memory_dynamic_address;
@@ -240,6 +245,9 @@ extern "C" {
   void __asan_alloca_poison(uptr addr, uptr size);
   SANITIZER_INTERFACE_ATTRIBUTE
   void __asan_allocas_unpoison(uptr top, uptr bottom);
+
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  const char* __asan_default_suppressions();
 }  // extern "C"
 
 #endif  // ASAN_INTERFACE_INTERNAL_H
diff --git a/libsanitizer/asan/asan_internal.h b/libsanitizer/asan/asan_internal.h
index 15a28ff777b..a3fe755f523 100644
--- a/libsanitizer/asan/asan_internal.h
+++ b/libsanitizer/asan/asan_internal.h
@@ -34,7 +34,7 @@
 // If set, values like allocator chunk size, as well as defaults for some flags
 // will be changed towards less memory overhead.
 #ifndef ASAN_LOW_MEMORY
-# if SANITIZER_IOS || (SANITIZER_WORDSIZE == 32)
+# if SANITIZER_IOS || SANITIZER_ANDROID
 #  define ASAN_LOW_MEMORY 1
 # else
 #  define ASAN_LOW_MEMORY 0
@@ -62,21 +62,29 @@ void AsanInitFromRtl();
 
 // asan_win.cc
 void InitializePlatformExceptionHandlers();
-
-// asan_win.cc / asan_posix.cc
-const char *DescribeSignalOrException(int signo);
+// Returns whether an address is a valid allocated system heap block.
+// 'addr' must point to the beginning of the block.
+bool IsSystemHeapAddress(uptr addr);
 
 // asan_rtl.cc
+void PrintAddressSpaceLayout();
 void NORETURN ShowStatsAndAbort();
 
+// asan_shadow_setup.cc
+void InitializeShadowMemory();
+
 // asan_malloc_linux.cc / asan_malloc_mac.cc
 void ReplaceSystemMalloc();
 
 // asan_linux.cc / asan_mac.cc / asan_win.cc
+uptr FindDynamicShadowStart();
 void *AsanDoesNotSupportStaticLinkage();
 void AsanCheckDynamicRTPrereqs();
 void AsanCheckIncompatibleRT();
 
+// asan_thread.cc
+AsanThread *CreateMainThread();
+
 // Support function for __asan_(un)register_image_globals. Searches for the
 // loaded image containing `needle' and then enumerates all global metadata
 // structures declared in that image, applying `op' (e.g.,
@@ -101,17 +109,6 @@ void *AsanDlSymNext(const char *sym);
 
 void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name);
 
-// Platform-specific options.
-#if SANITIZER_MAC
-bool PlatformHasDifferentMemcpyAndMemmove();
-# define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE \
-    (PlatformHasDifferentMemcpyAndMemmove())
-#elif SANITIZER_WINDOWS64
-# define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
-#else
-# define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE true
-#endif  // SANITIZER_MAC
-
 // Add convenient macro for interface functions that may be represented as
 // weak hooks.
 #define ASAN_MALLOC_HOOK(ptr, size)                                   \
diff --git a/libsanitizer/asan/asan_linux.cc b/libsanitizer/asan/asan_linux.cc
index 9f058df71a5..a43243e3b43 100644
--- a/libsanitizer/asan/asan_linux.cc
+++ b/libsanitizer/asan/asan_linux.cc
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 
 #include "asan_interceptors.h"
 #include "asan_internal.h"
@@ -40,6 +40,10 @@
 #if SANITIZER_ANDROID || SANITIZER_FREEBSD
 #include <ucontext.h>
 extern "C" void* _DYNAMIC;
+#elif SANITIZER_NETBSD
+#include <link_elf.h>
+#include <ucontext.h>
+extern Elf_Dyn _DYNAMIC;
 #else
 #include <sys/ucontext.h>
 #include <link.h>
@@ -68,12 +72,18 @@ namespace __asan {
 
 void InitializePlatformInterceptors() {}
 void InitializePlatformExceptionHandlers() {}
+bool IsSystemHeapAddress (uptr addr) { return false; }
 
 void *AsanDoesNotSupportStaticLinkage() {
   // This will fail to link with -static.
   return &_DYNAMIC;  // defined in link.h
 }
 
+uptr FindDynamicShadowStart() {
+  UNREACHABLE("FindDynamicShadowStart is not available");
+  return 0;
+}
+
 void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
   UNIMPLEMENTED();
 }
@@ -93,6 +103,15 @@ static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size,
   if (internal_strncmp(info->dlpi_name, "linux-", sizeof("linux-") - 1) == 0)
     return 0;
 
+#if SANITIZER_NETBSD
+  // Ignore first entry (the main program)
+  char **p = (char **)data;
+  if (!(*p)) {
+    *p = (char *)-1;
+    return 0;
+  }
+#endif
+
   *(const char **)data = info->dlpi_name;
   return 1;
 }
@@ -108,7 +127,7 @@ static void ReportIncompatibleRT() {
 }
 
 void AsanCheckDynamicRTPrereqs() {
-  if (!ASAN_DYNAMIC)
+  if (!ASAN_DYNAMIC || !flags()->verify_asan_link_order)
     return;
 
   // Ensure that dynamic RT is the first DSO in the list
@@ -137,9 +156,9 @@ void AsanCheckIncompatibleRT() {
       // system libraries, causing crashes later in ASan initialization.
       MemoryMappingLayout proc_maps(/*cache_enabled*/true);
       char filename[128];
-      while (proc_maps.Next(nullptr, nullptr, nullptr, filename,
-                            sizeof(filename), nullptr)) {
-        if (IsDynamicRTName(filename)) {
+      MemoryMappedSegment segment(filename, sizeof(filename));
+      while (proc_maps.Next(&segment)) {
+        if (IsDynamicRTName(segment.filename)) {
           Report("Your application is linked against "
                  "incompatible ASan runtimes.\n");
           Die();
@@ -171,4 +190,4 @@ void *AsanDlSymNext(const char *sym) {
 
 } // namespace __asan
 
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/libsanitizer/asan/asan_mac.cc b/libsanitizer/asan/asan_mac.cc
index 4bf79be1728..45c66d8645c 100644
--- a/libsanitizer/asan/asan_mac.cc
+++ b/libsanitizer/asan/asan_mac.cc
@@ -46,21 +46,36 @@ namespace __asan {
 
 void InitializePlatformInterceptors() {}
 void InitializePlatformExceptionHandlers() {}
-
-bool PlatformHasDifferentMemcpyAndMemmove() {
-  // On OS X 10.7 memcpy() and memmove() are both resolved
-  // into memmove$VARIANT$sse42.
-  // See also https://github.com/google/sanitizers/issues/34.
-  // TODO(glider): need to check dynamically that memcpy() and memmove() are
-  // actually the same function.
-  return GetMacosVersion() == MACOS_VERSION_SNOW_LEOPARD;
-}
+bool IsSystemHeapAddress (uptr addr) { return false; }
 
 // No-op. Mac does not support static linkage anyway.
 void *AsanDoesNotSupportStaticLinkage() {
   return 0;
 }
 
+uptr FindDynamicShadowStart() {
+  uptr granularity = GetMmapGranularity();
+  uptr alignment = 8 * granularity;
+  uptr left_padding = granularity;
+  uptr space_size = kHighShadowEnd + left_padding;
+
+  uptr largest_gap_found = 0;
+  uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
+                                               granularity, &largest_gap_found);
+  // If the shadow doesn't fit, restrict the address space to make it fit.
+  if (shadow_start == 0) {
+    uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
+    RestrictMemoryToMaxAddress(new_max_vm);
+    kHighMemEnd = new_max_vm - 1;
+    space_size = kHighShadowEnd + left_padding;
+    shadow_start =
+        FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+  }
+  CHECK_NE((uptr)0, shadow_start);
+  CHECK(IsAligned(shadow_start, alignment));
+  return shadow_start;
+}
+
 // No-op. Mac does not support static linkage anyway.
 void AsanCheckDynamicRTPrereqs() {}
 
@@ -145,7 +160,8 @@ void asan_register_worker_thread(int parent_tid, StackTrace *stack) {
     t = AsanThread::Create(/* start_routine */ nullptr, /* arg */ nullptr,
                            parent_tid, stack, /* detached */ true);
     t->Init();
-    asanThreadRegistry().StartThread(t->tid(), 0, 0);
+    asanThreadRegistry().StartThread(t->tid(), GetTid(),
+                                     /* workerthread */ true, 0);
     SetCurrentThread(t);
   }
 }
diff --git a/libsanitizer/asan/asan_malloc_linux.cc b/libsanitizer/asan/asan_malloc_linux.cc
index cc50a388495..19f45fc3cee 100644
--- a/libsanitizer/asan/asan_malloc_linux.cc
+++ b/libsanitizer/asan/asan_malloc_linux.cc
@@ -13,7 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \
+    SANITIZER_NETBSD
 
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 #include "asan_allocator.h"
@@ -28,9 +29,9 @@ static uptr allocated_for_dlsym;
 static const uptr kDlsymAllocPoolSize = 1024;
 static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
 
-static bool IsInDlsymAllocPool(const void *ptr) {
+static INLINE bool IsInDlsymAllocPool(const void *ptr) {
   uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < sizeof(alloc_memory_for_dlsym);
+  return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
 }
 
 static void *AllocateFromLocalPool(uptr size_in_bytes) {
@@ -41,6 +42,26 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) {
   return mem;
 }
 
+static INLINE bool MaybeInDlsym() {
+  // Fuchsia doesn't use dlsym-based interceptors.
+  return !SANITIZER_FUCHSIA && asan_init_is_running;
+}
+
+static void *ReallocFromLocalPool(void *ptr, uptr size) {
+  const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+  const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+  void *new_ptr;
+  if (UNLIKELY(MaybeInDlsym())) {
+    new_ptr = AllocateFromLocalPool(size);
+  } else {
+    ENSURE_ASAN_INITED();
+    GET_STACK_TRACE_MALLOC;
+    new_ptr = asan_malloc(size, &stack);
+  }
+  internal_memcpy(new_ptr, ptr, copy_size);
+  return new_ptr;
+}
+
 INTERCEPTOR(void, free, void *ptr) {
   GET_STACK_TRACE_FREE;
   if (UNLIKELY(IsInDlsymAllocPool(ptr)))
@@ -48,63 +69,61 @@ INTERCEPTOR(void, free, void *ptr) {
   asan_free(ptr, &stack, FROM_MALLOC);
 }
 
+#if SANITIZER_INTERCEPT_CFREE
 INTERCEPTOR(void, cfree, void *ptr) {
   GET_STACK_TRACE_FREE;
   if (UNLIKELY(IsInDlsymAllocPool(ptr)))
     return;
   asan_free(ptr, &stack, FROM_MALLOC);
 }
+#endif // SANITIZER_INTERCEPT_CFREE
 
 INTERCEPTOR(void*, malloc, uptr size) {
-  if (UNLIKELY(!asan_inited))
+  if (UNLIKELY(MaybeInDlsym()))
     // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
     return AllocateFromLocalPool(size);
+  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_malloc(size, &stack);
 }
 
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
-  if (UNLIKELY(!asan_inited))
+  if (UNLIKELY(MaybeInDlsym()))
     // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
     return AllocateFromLocalPool(nmemb * size);
+  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_calloc(nmemb, size, &stack);
 }
 
 INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
+  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
+    return ReallocFromLocalPool(ptr, size);
+  if (UNLIKELY(MaybeInDlsym()))
+    return AllocateFromLocalPool(size);
+  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-    void *new_ptr;
-    if (UNLIKELY(!asan_inited)) {
-      new_ptr = AllocateFromLocalPool(size);
-    } else {
-      copy_size = size;
-      new_ptr = asan_malloc(copy_size, &stack);
-    }
-    internal_memcpy(new_ptr, ptr, copy_size);
-    return new_ptr;
-  }
   return asan_realloc(ptr, size, &stack);
 }
 
+#if SANITIZER_INTERCEPT_MEMALIGN
 INTERCEPTOR(void*, memalign, uptr boundary, uptr size) {
   GET_STACK_TRACE_MALLOC;
   return asan_memalign(boundary, size, &stack, FROM_MALLOC);
 }
 
-INTERCEPTOR(void*, aligned_alloc, uptr boundary, uptr size) {
-  GET_STACK_TRACE_MALLOC;
-  return asan_memalign(boundary, size, &stack, FROM_MALLOC);
-}
-
 INTERCEPTOR(void*, __libc_memalign, uptr boundary, uptr size) {
   GET_STACK_TRACE_MALLOC;
   void *res = asan_memalign(boundary, size, &stack, FROM_MALLOC);
   DTLS_on_libc_memalign(res, size);
   return res;
 }
+#endif // SANITIZER_INTERCEPT_MEMALIGN
+
+INTERCEPTOR(void*, aligned_alloc, uptr boundary, uptr size) {
+  GET_STACK_TRACE_MALLOC;
+  return asan_memalign(boundary, size, &stack, FROM_MALLOC);
+}
 
 INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
   GET_CURRENT_PC_BP_SP;
@@ -112,6 +131,7 @@ INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
   return asan_malloc_usable_size(ptr, pc, bp);
 }
 
+#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
 // We avoid including malloc.h for portability reasons.
 // man mallinfo says the fields are "long", but the implementation uses int.
 // It doesn't matter much -- we just need to make sure that the libc's mallinfo
@@ -129,6 +149,7 @@ INTERCEPTOR(struct fake_mallinfo, mallinfo, void) {
 INTERCEPTOR(int, mallopt, int cmd, int value) {
   return -1;
 }
+#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
 
 INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
   GET_STACK_TRACE_MALLOC;
@@ -141,10 +162,12 @@ INTERCEPTOR(void*, valloc, uptr size) {
   return asan_valloc(size, &stack);
 }
 
+#if SANITIZER_INTERCEPT_PVALLOC
 INTERCEPTOR(void*, pvalloc, uptr size) {
   GET_STACK_TRACE_MALLOC;
   return asan_pvalloc(size, &stack);
 }
+#endif // SANITIZER_INTERCEPT_PVALLOC
 
 INTERCEPTOR(void, malloc_stats, void) {
   __asan_print_accumulated_stats();
@@ -210,4 +233,5 @@ void ReplaceSystemMalloc() {
 }  // namespace __asan
 #endif  // SANITIZER_ANDROID
 
-#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif  // SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX ||
+        // SANITIZER_NETBSD
diff --git a/libsanitizer/asan/asan_malloc_win.cc b/libsanitizer/asan/asan_malloc_win.cc
index f38cd054bb0..017481d0b81 100644
--- a/libsanitizer/asan/asan_malloc_win.cc
+++ b/libsanitizer/asan/asan_malloc_win.cc
@@ -54,11 +54,6 @@ void _free_base(void *ptr) {
 }
 
 ALLOCATION_FUNCTION_ATTRIBUTE
-void cfree(void *ptr) {
-  CHECK(!"cfree() should not be used on Windows");
-}
-
-ALLOCATION_FUNCTION_ATTRIBUTE
 void *malloc(size_t size) {
   GET_STACK_TRACE_MALLOC;
   return asan_malloc(size, &stack);
@@ -103,7 +98,7 @@ void *realloc(void *ptr, size_t size) {
 
 ALLOCATION_FUNCTION_ATTRIBUTE
 void *_realloc_dbg(void *ptr, size_t size, int) {
-  CHECK(!"_realloc_dbg should not exist!");
+  UNREACHABLE("_realloc_dbg should not exist!");
   return 0;
 }
 
diff --git a/libsanitizer/asan/asan_mapping.h b/libsanitizer/asan/asan_mapping.h
index b9fa5f79481..5496df66deb 100644
--- a/libsanitizer/asan/asan_mapping.h
+++ b/libsanitizer/asan/asan_mapping.h
@@ -113,6 +113,13 @@
 // || `[0x40000000, 0x47ffffff]` || LowShadow  ||
 // || `[0x00000000, 0x3fffffff]` || LowMem     ||
 //
+// Shadow mapping on NetBSD/x86-64 with SHADOW_OFFSET == 0x400000000000:
+// || `[0x4feffffffe01, 0x7f7ffffff000]` || HighMem    ||
+// || `[0x49fdffffffc0, 0x4feffffffe00]` || HighShadow ||
+// || `[0x480000000000, 0x49fdffffffbf]` || ShadowGap  ||
+// || `[0x400000000000, 0x47ffffffffff]` || LowShadow  ||
+// || `[0x000000000000, 0x3fffffffffff]` || LowMem     ||
+//
 // Default Windows/i386 mapping:
 // (the exact location of HighShadow/HighMem may vary depending
 //  on WoW64, /LARGEADDRESSAWARE, etc).
@@ -138,12 +145,14 @@ static const u64 kPPC64_ShadowOffset64 = 1ULL << 41;
 static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52;
 static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30;  // 0x40000000
 static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46;  // 0x400000000000
+static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46;  // 0x400000000000
 static const u64 kWindowsShadowOffset32 = 3ULL << 28;  // 0x30000000
 
 #define SHADOW_SCALE kDefaultShadowScale
 
-
-#if SANITIZER_WORDSIZE == 32
+#if SANITIZER_FUCHSIA
+#  define SHADOW_OFFSET (0)
+#elif SANITIZER_WORDSIZE == 32
 #  if SANITIZER_ANDROID
 #    define SHADOW_OFFSET (0)
 #  elif defined(__mips__)
@@ -176,6 +185,8 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28;  // 0x30000000
 #    define SHADOW_OFFSET kSystemZ_ShadowOffset64
 #  elif SANITIZER_FREEBSD
 #    define SHADOW_OFFSET kFreeBSD_ShadowOffset64
+#  elif SANITIZER_NETBSD
+#    define SHADOW_OFFSET kNetBSD_ShadowOffset64
 #  elif SANITIZER_MAC
 #   define SHADOW_OFFSET kDefaultShadowOffset64
 #  elif defined(__mips64)
@@ -189,7 +200,6 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28;  // 0x30000000
 
 #define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE)
 #define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET))
-#define SHADOW_TO_MEM(shadow) (((shadow) - SHADOW_OFFSET) << SHADOW_SCALE)
 
 #define kLowMemBeg      0
 #define kLowMemEnd      (SHADOW_OFFSET ? SHADOW_OFFSET - 1 : 0)
diff --git a/libsanitizer/asan/asan_memory_profile.cc b/libsanitizer/asan/asan_memory_profile.cc
index 5a2578596ad..42d07c705f1 100644
--- a/libsanitizer/asan/asan_memory_profile.cc
+++ b/libsanitizer/asan/asan_memory_profile.cc
@@ -30,69 +30,99 @@ struct AllocationSite {
 class HeapProfile {
  public:
   HeapProfile() : allocations_(1024) {}
-  void Insert(u32 id, uptr size) {
-    total_allocated_ += size;
-    total_count_++;
-    // Linear lookup will be good enough for most cases (although not all).
-    for (uptr i = 0; i < allocations_.size(); i++) {
-      if (allocations_[i].id == id) {
-        allocations_[i].total_size += size;
-        allocations_[i].count++;
-        return;
-      }
+
+  void ProcessChunk(const AsanChunkView& cv) {
+    if (cv.IsAllocated()) {
+      total_allocated_user_size_ += cv.UsedSize();
+      total_allocated_count_++;
+      u32 id = cv.GetAllocStackId();
+      if (id)
+        Insert(id, cv.UsedSize());
+    } else if (cv.IsQuarantined()) {
+      total_quarantined_user_size_ += cv.UsedSize();
+      total_quarantined_count_++;
+    } else {
+      total_other_count_++;
     }
-    allocations_.push_back({id, size, 1});
   }
 
-  void Print(uptr top_percent) {
+  void Print(uptr top_percent, uptr max_number_of_contexts) {
     InternalSort(&allocations_, allocations_.size(),
                  [](const AllocationSite &a, const AllocationSite &b) {
                    return a.total_size > b.total_size;
                  });
-    CHECK(total_allocated_);
+    CHECK(total_allocated_user_size_);
     uptr total_shown = 0;
-    Printf("Live Heap Allocations: %zd bytes from %zd allocations; "
-           "showing top %zd%%\n", total_allocated_, total_count_, top_percent);
-    for (uptr i = 0; i < allocations_.size(); i++) {
+    Printf("Live Heap Allocations: %zd bytes in %zd chunks; quarantined: "
+           "%zd bytes in %zd chunks; %zd other chunks; total chunks: %zd; "
+           "showing top %zd%% (at most %zd unique contexts)\n",
+           total_allocated_user_size_, total_allocated_count_,
+           total_quarantined_user_size_, total_quarantined_count_,
+           total_other_count_, total_allocated_count_ +
+           total_quarantined_count_ + total_other_count_, top_percent,
+           max_number_of_contexts);
+    for (uptr i = 0; i < Min(allocations_.size(), max_number_of_contexts);
+         i++) {
       auto &a = allocations_[i];
       Printf("%zd byte(s) (%zd%%) in %zd allocation(s)\n", a.total_size,
-             a.total_size * 100 / total_allocated_, a.count);
+             a.total_size * 100 / total_allocated_user_size_, a.count);
       StackDepotGet(a.id).Print();
       total_shown += a.total_size;
-      if (total_shown * 100 / total_allocated_ > top_percent)
+      if (total_shown * 100 / total_allocated_user_size_ > top_percent)
         break;
     }
   }
 
  private:
-  uptr total_allocated_ = 0;
-  uptr total_count_ = 0;
+  uptr total_allocated_user_size_ = 0;
+  uptr total_allocated_count_ = 0;
+  uptr total_quarantined_user_size_ = 0;
+  uptr total_quarantined_count_ = 0;
+  uptr total_other_count_ = 0;
   InternalMmapVector<AllocationSite> allocations_;
+
+  void Insert(u32 id, uptr size) {
+    // Linear lookup will be good enough for most cases (although not all).
+    for (uptr i = 0; i < allocations_.size(); i++) {
+      if (allocations_[i].id == id) {
+        allocations_[i].total_size += size;
+        allocations_[i].count++;
+        return;
+      }
+    }
+    allocations_.push_back({id, size, 1});
+  }
 };
 
 static void ChunkCallback(uptr chunk, void *arg) {
-  HeapProfile *hp = reinterpret_cast<HeapProfile*>(arg);
-  AsanChunkView cv = FindHeapChunkByAllocBeg(chunk);
-  if (!cv.IsAllocated()) return;
-  u32 id = cv.GetAllocStackId();
-  if (!id) return;
-  hp->Insert(id, cv.UsedSize());
+  reinterpret_cast<HeapProfile*>(arg)->ProcessChunk(
+      FindHeapChunkByAllocBeg(chunk));
 }
 
 static void MemoryProfileCB(const SuspendedThreadsList &suspended_threads_list,
                             void *argument) {
   HeapProfile hp;
   __lsan::ForEachChunk(ChunkCallback, &hp);
-  hp.Print(reinterpret_cast<uptr>(argument));
+  uptr *Arg = reinterpret_cast<uptr*>(argument);
+  hp.Print(Arg[0], Arg[1]);
+
+  if (Verbosity())
+    __asan_print_accumulated_stats();
 }
 
 }  // namespace __asan
 
+#endif  // CAN_SANITIZE_LEAKS
+
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_print_memory_profile(uptr top_percent) {
-  __sanitizer::StopTheWorld(__asan::MemoryProfileCB, (void*)top_percent);
+void __sanitizer_print_memory_profile(uptr top_percent,
+                                      uptr max_number_of_contexts) {
+#if CAN_SANITIZE_LEAKS
+  uptr Arg[2];
+  Arg[0] = top_percent;
+  Arg[1] = max_number_of_contexts;
+  __sanitizer::StopTheWorld(__asan::MemoryProfileCB, Arg);
+#endif  // CAN_SANITIZE_LEAKS
 }
 }  // extern "C"
-
-#endif  // CAN_SANITIZE_LEAKS
diff --git a/libsanitizer/asan/asan_new_delete.cc b/libsanitizer/asan/asan_new_delete.cc
index 2c4642cb27e..e95b5287191 100644
--- a/libsanitizer/asan/asan_new_delete.cc
+++ b/libsanitizer/asan/asan_new_delete.cc
@@ -23,22 +23,26 @@
 // dllexport would normally do. We need to export them in order to make the
 // VS2015 dynamic CRT (MD) work.
 #if SANITIZER_WINDOWS
-# define CXX_OPERATOR_ATTRIBUTE
-# ifdef _WIN64
-#  pragma comment(linker, "/export:??2@YAPEAX_K@Z")   // operator new
-#  pragma comment(linker, "/export:??3@YAXPEAX@Z")    // operator delete
-#  pragma comment(linker, "/export:??3@YAXPEAX_K@Z")  // sized operator delete
-#  pragma comment(linker, "/export:??_U@YAPEAX_K@Z")  // operator new[]
-#  pragma comment(linker, "/export:??_V@YAXPEAX@Z")   // operator delete[]
-# else
-#  pragma comment(linker, "/export:??2@YAPAXI@Z")   // operator new
-#  pragma comment(linker, "/export:??3@YAXPAX@Z")   // operator delete
-#  pragma comment(linker, "/export:??3@YAXPAXI@Z")  // sized operator delete
-#  pragma comment(linker, "/export:??_U@YAPAXI@Z")  // operator new[]
-#  pragma comment(linker, "/export:??_V@YAXPAX@Z")  // operator delete[]
-# endif
+#define CXX_OPERATOR_ATTRIBUTE
+#define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:" sym))
+#ifdef _WIN64
+COMMENT_EXPORT("??2@YAPEAX_K@Z")                     // operator new
+COMMENT_EXPORT("??2@YAPEAX_KAEBUnothrow_t@std@@@Z")  // operator new nothrow
+COMMENT_EXPORT("??3@YAXPEAX@Z")                      // operator delete
+COMMENT_EXPORT("??3@YAXPEAX_K@Z")                    // sized operator delete
+COMMENT_EXPORT("??_U@YAPEAX_K@Z")                    // operator new[]
+COMMENT_EXPORT("??_V@YAXPEAX@Z")                     // operator delete[]
 #else
-# define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE
+COMMENT_EXPORT("??2@YAPAXI@Z")                    // operator new
+COMMENT_EXPORT("??2@YAPAXIABUnothrow_t@std@@@Z")  // operator new nothrow
+COMMENT_EXPORT("??3@YAXPAX@Z")                    // operator delete
+COMMENT_EXPORT("??3@YAXPAXI@Z")                   // sized operator delete
+COMMENT_EXPORT("??_U@YAPAXI@Z")                   // operator new[]
+COMMENT_EXPORT("??_V@YAXPAX@Z")                   // operator delete[]
+#endif
+#undef COMMENT_EXPORT
+#else
+#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE
 #endif
 
 using namespace __asan;  // NOLINT
@@ -61,12 +65,17 @@ struct nothrow_t {};
 enum class align_val_t: size_t {};
 }  // namespace std
 
-#define OPERATOR_NEW_BODY(type) \
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(type, nothrow) \
   GET_STACK_TRACE_MALLOC;\
-  return asan_memalign(0, size, &stack, type);
-#define OPERATOR_NEW_BODY_ALIGN(type) \
+  void *res = asan_memalign(0, size, &stack, type);\
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+  return res;
+#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \
   GET_STACK_TRACE_MALLOC;\
-  return asan_memalign((uptr)align, size, &stack, type);
+  void *res = asan_memalign((uptr)align, size, &stack, type);\
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+  return res;
 
 // On OS X it's not enough to just provide our own 'operator new' and
 // 'operator delete' implementations, because they're going to be in the
@@ -77,40 +86,42 @@ enum class align_val_t: size_t {};
 // OS X we need to intercept them using their mangled names.
 #if !SANITIZER_MAC
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size) { OPERATOR_NEW_BODY(FROM_NEW); }
+void *operator new(size_t size)
+{ OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size) { OPERATOR_NEW_BODY(FROM_NEW_BR); }
+void *operator new[](size_t size)
+{ OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW); }
+{ OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new[](size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW_BR); }
+{ OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new[](size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, true /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); }
 
 #else  // SANITIZER_MAC
 INTERCEPTOR(void *, _Znwm, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW);
+  OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/);
 }
 INTERCEPTOR(void *, _Znam, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR);
+  OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/);
 }
 INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW);
+  OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/);
 }
 INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR);
+  OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/);
 }
 #endif
 
diff --git a/libsanitizer/asan/asan_poisoning.cc b/libsanitizer/asan/asan_poisoning.cc
index 8fe2bd42bdb..15cd8eaac3e 100644
--- a/libsanitizer/asan/asan_poisoning.cc
+++ b/libsanitizer/asan/asan_poisoning.cc
@@ -62,12 +62,9 @@ struct ShadowSegmentEndpoint {
 };
 
 void FlushUnneededASanShadowMemory(uptr p, uptr size) {
-    // Since asan's mapping is compacting, the shadow chunk may be
-    // not page-aligned, so we only flush the page-aligned portion.
-    uptr page_size = GetPageSizeCached();
-    uptr shadow_beg = RoundUpTo(MemToShadow(p), page_size);
-    uptr shadow_end = RoundDownTo(MemToShadow(p + size), page_size);
-    ReleaseMemoryToOS(shadow_beg, shadow_end - shadow_beg);
+  // Since asan's mapping is compacting, the shadow chunk may be
+  // not page-aligned, so we only flush the page-aligned portion.
+  ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
 }
 
 void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) {
@@ -410,7 +407,7 @@ const void *__sanitizer_contiguous_container_find_bad_address(
   // ending with end.
   uptr kMaxRangeToCheck = 32;
   uptr r1_beg = beg;
-  uptr r1_end = Min(end + kMaxRangeToCheck, mid);
+  uptr r1_end = Min(beg + kMaxRangeToCheck, mid);
   uptr r2_beg = Max(beg, mid - kMaxRangeToCheck);
   uptr r2_end = Min(end, mid + kMaxRangeToCheck);
   uptr r3_beg = Max(end - kMaxRangeToCheck, mid);
diff --git a/libsanitizer/asan/asan_poisoning.h b/libsanitizer/asan/asan_poisoning.h
index 4ddcbc314d4..942e74174c2 100644
--- a/libsanitizer/asan/asan_poisoning.h
+++ b/libsanitizer/asan/asan_poisoning.h
@@ -44,8 +44,11 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size,
   // for mapping shadow and zeroing out pages doesn't "just work", so we should
   // probably provide higher-level interface for these operations.
   // For now, just memset on Windows.
-  if (value ||
-      SANITIZER_WINDOWS == 1 ||
+  if (value || SANITIZER_WINDOWS == 1 ||
+      // TODO(mcgrathr): Fuchsia doesn't allow the shadow mapping to be
+      // changed at all.  It doesn't currently have an efficient means
+      // to zero a bunch of pages, but maybe we should add one.
+      SANITIZER_FUCHSIA == 1 ||
       shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
     REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg);
   } else {
@@ -84,8 +87,8 @@ ALWAYS_INLINE void FastPoisonShadowPartialRightRedzone(
   }
 }
 
-// Calls __sanitizer::ReleaseMemoryToOS() on
-// [MemToShadow(p), MemToShadow(p+size)] with proper rounding.
+// Calls __sanitizer::ReleaseMemoryPagesToOS() on
+// [MemToShadow(p), MemToShadow(p+size)].
 void FlushUnneededASanShadowMemory(uptr p, uptr size);
 
 }  // namespace __asan
diff --git a/libsanitizer/asan/asan_posix.cc b/libsanitizer/asan/asan_posix.cc
index 532afb37f1a..e113c02a8cc 100644
--- a/libsanitizer/asan/asan_posix.cc
+++ b/libsanitizer/asan/asan_posix.cc
@@ -31,72 +31,10 @@
 
 namespace __asan {
 
-const char *DescribeSignalOrException(int signo) {
-  switch (signo) {
-    case SIGFPE:
-      return "FPE";
-    case SIGILL:
-      return "ILL";
-    case SIGABRT:
-      return "ABRT";
-    default:
-      return "SEGV";
-  }
-}
-
 void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
-  ScopedDeadlySignal signal_scope(GetCurrentThread());
-  int code = (int)((siginfo_t*)siginfo)->si_code;
-  // Write the first message using fd=2, just in case.
-  // It may actually fail to write in case stderr is closed.
-  internal_write(2, "ASAN:DEADLYSIGNAL\n", 18);
-  SignalContext sig = SignalContext::Create(siginfo, context);
-
-  // Access at a reasonable offset above SP, or slightly below it (to account
-  // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
-  // probably a stack overflow.
-#ifdef __s390__
-  // On s390, the fault address in siginfo points to start of the page, not
-  // to the precise word that was accessed.  Mask off the low bits of sp to
-  // take it into account.
-  bool IsStackAccess = sig.addr >= (sig.sp & ~0xFFF) &&
-                       sig.addr < sig.sp + 0xFFFF;
-#else
-  bool IsStackAccess = sig.addr + 512 > sig.sp && sig.addr < sig.sp + 0xFFFF;
-#endif
-
-#if __powerpc__
-  // Large stack frames can be allocated with e.g.
-  //   lis r0,-10000
-  //   stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
-  // If the store faults then sp will not have been updated, so test above
-  // will not work, becase the fault address will be more than just "slightly"
-  // below sp.
-  if (!IsStackAccess && IsAccessibleMemoryRange(sig.pc, 4)) {
-    u32 inst = *(unsigned *)sig.pc;
-    u32 ra = (inst >> 16) & 0x1F;
-    u32 opcd = inst >> 26;
-    u32 xo = (inst >> 1) & 0x3FF;
-    // Check for store-with-update to sp. The instructions we accept are:
-    //   stbu rs,d(ra)          stbux rs,ra,rb
-    //   sthu rs,d(ra)          sthux rs,ra,rb
-    //   stwu rs,d(ra)          stwux rs,ra,rb
-    //   stdu rs,ds(ra)         stdux rs,ra,rb
-    // where ra is r1 (the stack pointer).
-    if (ra == 1 &&
-        (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 ||
-         (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181))))
-      IsStackAccess = true;
-  }
-#endif // __powerpc__
-
-  // We also check si_code to filter out SEGV caused by something else other
-  // then hitting the guard page or unmapped memory, like, for example,
-  // unaligned memory access.
-  if (IsStackAccess && (code == si_SEGV_MAPERR || code == si_SEGV_ACCERR))
-    ReportStackOverflow(sig);
-  else
-    ReportDeadlySignal(signo, sig);
+  StartReportDeadlySignal();
+  SignalContext sig(siginfo, context);
+  ReportDeadlySignal(sig);
 }
 
 // ---------------------- TSD ---------------- {{{1
diff --git a/libsanitizer/asan/asan_report.cc b/libsanitizer/asan/asan_report.cc
index 84d67646b40..51bad6e45e3 100644
--- a/libsanitizer/asan/asan_report.cc
+++ b/libsanitizer/asan/asan_report.cc
@@ -58,9 +58,8 @@ void PrintMemoryByte(InternalScopedString *str, const char *before, u8 byte,
                      bool in_shadow, const char *after) {
   Decorator d;
   str->append("%s%s%x%x%s%s", before,
-              in_shadow ? d.ShadowByte(byte) : d.MemoryByte(),
-              byte >> 4, byte & 15,
-              in_shadow ? d.EndShadowByte() : d.EndMemoryByte(), after);
+              in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), byte >> 4,
+              byte & 15, d.Default(), after);
 }
 
 static void PrintZoneForPointer(uptr ptr, uptr zone_ptr,
@@ -86,7 +85,8 @@ bool ParseFrameDescription(const char *frame_descr,
   char *p;
   // This string is created by the compiler and has the following form:
   // "n alloc_1 alloc_2 ... alloc_n"
-  // where alloc_i looks like "offset size len ObjectName".
+  // where alloc_i looks like "offset size len ObjectName"
+  // or                       "offset size len ObjectName:line".
   uptr n_objects = (uptr)internal_simple_strtoll(frame_descr, &p, 10);
   if (n_objects == 0)
     return false;
@@ -99,7 +99,14 @@ bool ParseFrameDescription(const char *frame_descr,
       return false;
     }
     p++;
-    StackVarDescr var = {beg, size, p, len};
+    char *colon_pos = internal_strchr(p, ':');
+    uptr line = 0;
+    uptr name_len = len;
+    if (colon_pos != nullptr && colon_pos < p + len) {
+      name_len = colon_pos - p;
+      line = (uptr)internal_simple_strtoll(colon_pos + 1, nullptr, 10);
+    }
+    StackVarDescr var = {beg, size, p, name_len, line};
     vars->push_back(var);
     p += len;
   }
@@ -113,53 +120,15 @@ bool ParseFrameDescription(const char *frame_descr,
 // immediately after printing error report.
 class ScopedInErrorReport {
  public:
-  explicit ScopedInErrorReport(bool fatal = false) {
-    halt_on_error_ = fatal || flags()->halt_on_error;
-
-    if (lock_.TryLock()) {
-      StartReporting();
-      return;
-    }
-
-    // ASan found two bugs in different threads simultaneously.
-
-    u32 current_tid = GetCurrentTidOrInvalid();
-    if (reporting_thread_tid_ == current_tid ||
-        reporting_thread_tid_ == kInvalidTid) {
-      // This is either asynch signal or nested error during error reporting.
-      // Fail simple to avoid deadlocks in Report().
-
-      // Can't use Report() here because of potential deadlocks
-      // in nested signal handlers.
-      const char msg[] = "AddressSanitizer: nested bug in the same thread, "
-                         "aborting.\n";
-      WriteToFile(kStderrFd, msg, sizeof(msg));
-
-      internal__exit(common_flags()->exitcode);
-    }
-
-    if (halt_on_error_) {
-      // Do not print more than one report, otherwise they will mix up.
-      // Error reporting functions shouldn't return at this situation, as
-      // they are effectively no-returns.
-
-      Report("AddressSanitizer: while reporting a bug found another one. "
-             "Ignoring.\n");
-
-      // Sleep long enough to make sure that the thread which started
-      // to print an error report will finish doing it.
-      SleepForSeconds(Max(100, flags()->sleep_before_dying + 1));
-
-      // If we're still not dead for some reason, use raw _exit() instead of
-      // Die() to bypass any additional checks.
-      internal__exit(common_flags()->exitcode);
-    } else {
-      // The other thread will eventually finish reporting
-      // so it's safe to wait
-      lock_.Lock();
-    }
-
-    StartReporting();
+  explicit ScopedInErrorReport(bool fatal = false)
+      : halt_on_error_(fatal || flags()->halt_on_error) {
+    // Make sure the registry and sanitizer report mutexes are locked while
+    // we're printing an error report.
+    // We can lock them only here to avoid self-deadlock in case of
+    // recursive reports.
+    asanThreadRegistry().Lock();
+    Printf(
+        "=================================================================\n");
   }
 
   ~ScopedInErrorReport() {
@@ -177,6 +146,8 @@ class ScopedInErrorReport {
     if (common_flags()->print_cmdline)
       PrintCmdline();
 
+    if (common_flags()->print_module_map == 2) PrintModuleMap();
+
     // Copy the message buffer so that we could start logging without holding a
     // lock that gets aquired during printing.
     InternalScopedBuffer<char> buffer_copy(kErrorMessageBufferSize);
@@ -192,14 +163,19 @@ class ScopedInErrorReport {
       error_report_callback(buffer_copy.data());
     }
 
+    if (halt_on_error_ && common_flags()->abort_on_error) {
+      // On Android the message is truncated to 512 characters.
+      // FIXME: implement "compact" error format, possibly without, or with
+      // highly compressed stack traces?
+      // FIXME: or just use the summary line as abort message?
+      SetAbortMessage(buffer_copy.data());
+    }
+
     // In halt_on_error = false mode, reset the current error object (before
     // unlocking).
     if (!halt_on_error_)
       internal_memset(&current_error_, 0, sizeof(current_error_));
 
-    CommonSanitizerReportMutex.Unlock();
-    reporting_thread_tid_ = kInvalidTid;
-    lock_.Unlock();
     if (halt_on_error_) {
       Report("ABORTING\n");
       Die();
@@ -217,39 +193,18 @@ class ScopedInErrorReport {
   }
 
  private:
-  void StartReporting() {
-    // Make sure the registry and sanitizer report mutexes are locked while
-    // we're printing an error report.
-    // We can lock them only here to avoid self-deadlock in case of
-    // recursive reports.
-    asanThreadRegistry().Lock();
-    CommonSanitizerReportMutex.Lock();
-    reporting_thread_tid_ = GetCurrentTidOrInvalid();
-    Printf("===================================================="
-           "=============\n");
-  }
-
-  static StaticSpinMutex lock_;
-  static u32 reporting_thread_tid_;
+  ScopedErrorReportLock error_report_lock_;
   // Error currently being reported. This enables the destructor to interact
   // with the debugger and point it to an error description.
   static ErrorDescription current_error_;
   bool halt_on_error_;
 };
 
-StaticSpinMutex ScopedInErrorReport::lock_;
-u32 ScopedInErrorReport::reporting_thread_tid_ = kInvalidTid;
 ErrorDescription ScopedInErrorReport::current_error_;
 
-void ReportStackOverflow(const SignalContext &sig) {
+void ReportDeadlySignal(const SignalContext &sig) {
   ScopedInErrorReport in_report(/*fatal*/ true);
-  ErrorStackOverflow error(GetCurrentTidOrInvalid(), sig);
-  in_report.ReportError(error);
-}
-
-void ReportDeadlySignal(int signo, const SignalContext &sig) {
-  ScopedInErrorReport in_report(/*fatal*/ true);
-  ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig, signo);
+  ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig);
   in_report.ReportError(error);
 }
 
@@ -425,7 +380,7 @@ void __asan_describe_address(uptr addr) {
 }
 
 int __asan_report_present() {
-  return ScopedInErrorReport::CurrentError().kind == kErrorKindGeneric;
+  return ScopedInErrorReport::CurrentError().kind != kErrorKindInvalid;
 }
 
 uptr __asan_get_report_pc() {
@@ -447,9 +402,11 @@ uptr __asan_get_report_sp() {
 }
 
 uptr __asan_get_report_address() {
-  if (ScopedInErrorReport::CurrentError().kind == kErrorKindGeneric)
-    return ScopedInErrorReport::CurrentError()
-        .Generic.addr_description.Address();
+  ErrorDescription &err = ScopedInErrorReport::CurrentError();
+  if (err.kind == kErrorKindGeneric)
+    return err.Generic.addr_description.Address();
+  else if (err.kind == kErrorKindDoubleFree)
+    return err.DoubleFree.addr_description.addr;
   return 0;
 }
 
@@ -468,7 +425,7 @@ uptr __asan_get_report_access_size() {
 const char *__asan_get_report_description() {
   if (ScopedInErrorReport::CurrentError().kind == kErrorKindGeneric)
     return ScopedInErrorReport::CurrentError().Generic.bug_descr;
-  return nullptr;
+  return ScopedInErrorReport::CurrentError().Base.scariness.GetDescription();
 }
 
 extern "C" {
@@ -482,9 +439,6 @@ void __sanitizer_ptr_cmp(void *a, void *b) {
 }
 } // extern "C"
 
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
 // Provide default implementation of __asan_on_error that does nothing
 // and may be overriden by user.
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE
-void __asan_on_error() {}
-#endif
+SANITIZER_INTERFACE_WEAK_DEF(void, __asan_on_error, void) {}
diff --git a/libsanitizer/asan/asan_report.h b/libsanitizer/asan/asan_report.h
index 111b8400153..5d47712bd2d 100644
--- a/libsanitizer/asan/asan_report.h
+++ b/libsanitizer/asan/asan_report.h
@@ -21,6 +21,7 @@ struct StackVarDescr {
   uptr size;
   const char *name_pos;
   uptr name_len;
+  uptr line;
 };
 
 // Returns the number of globals close to the provided address and copies
@@ -43,8 +44,7 @@ bool ParseFrameDescription(const char *frame_descr,
 // Different kinds of error reports.
 void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write,
                         uptr access_size, u32 exp, bool fatal);
-void ReportStackOverflow(const SignalContext &sig);
-void ReportDeadlySignal(int signo, const SignalContext &sig);
+void ReportDeadlySignal(const SignalContext &sig);
 void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size,
                                  BufferedStackTrace *free_stack);
 void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack);
diff --git a/libsanitizer/asan/asan_rtl.cc b/libsanitizer/asan/asan_rtl.cc
index 38009d2905f..3905658a494 100644
--- a/libsanitizer/asan/asan_rtl.cc
+++ b/libsanitizer/asan/asan_rtl.cc
@@ -44,6 +44,7 @@ static void AsanDie() {
     // Don't die twice - run a busy loop.
     while (1) { }
   }
+  if (common_flags()->print_module_map >= 1) PrintModuleMap();
   if (flags()->sleep_before_dying) {
     Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying);
     SleepForSeconds(flags()->sleep_before_dying);
@@ -81,26 +82,6 @@ void ShowStatsAndAbort() {
   Die();
 }
 
-// ---------------------- mmap -------------------- {{{1
-// Reserve memory range [beg, end].
-// We need to use inclusive range because end+1 may not be representable.
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
-  CHECK_EQ((beg % GetMmapGranularity()), 0);
-  CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
-  uptr size = end - beg + 1;
-  DecreaseTotalMmap(size);  // Don't count the shadow against mmap_limit_mb.
-  void *res = MmapFixedNoReserve(beg, size, name);
-  if (res != (void*)beg) {
-    Report("ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
-           "Perhaps you're using ulimit -v\n", size);
-    Abort();
-  }
-  if (common_flags()->no_huge_pages_for_shadow)
-    NoHugePagesInRegion(beg, size);
-  if (common_flags()->use_madv_dontdump)
-    DontDumpShadowMemory(beg, size);
-}
-
 // --------------- LowLevelAllocateCallbac ---------- {{{1
 static void OnLowLevelAllocate(uptr ptr, uptr size) {
   PoisonShadow(ptr, size, kAsanInternalHeapMagic);
@@ -332,46 +313,7 @@ static void InitializeHighMemEnd() {
   CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
 }
 
-static void ProtectGap(uptr addr, uptr size) {
-  if (!flags()->protect_shadow_gap) {
-    // The shadow gap is unprotected, so there is a chance that someone
-    // is actually using this memory. Which means it needs a shadow...
-    uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached());
-    uptr GapShadowEnd =
-        RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1;
-    if (Verbosity())
-      Printf("protect_shadow_gap=0:"
-             " not protecting shadow gap, allocating gap's shadow\n"
-             "|| `[%p, %p]` || ShadowGap's shadow ||\n", GapShadowBeg,
-             GapShadowEnd);
-    ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
-                             "unprotected gap shadow");
-    return;
-  }
-  void *res = MmapFixedNoAccess(addr, size, "shadow gap");
-  if (addr == (uptr)res)
-    return;
-  // A few pages at the start of the address space can not be protected.
-  // But we really want to protect as much as possible, to prevent this memory
-  // being returned as a result of a non-FIXED mmap().
-  if (addr == kZeroBaseShadowStart) {
-    uptr step = GetMmapGranularity();
-    while (size > step && addr < kZeroBaseMaxShadowStart) {
-      addr += step;
-      size -= step;
-      void *res = MmapFixedNoAccess(addr, size, "shadow gap");
-      if (addr == (uptr)res)
-        return;
-    }
-  }
-
-  Report("ERROR: Failed to protect the shadow gap. "
-         "ASan cannot proceed correctly. ABORTING.\n");
-  DumpProcessMap();
-  Die();
-}
-
-static void PrintAddressSpaceLayout() {
+void PrintAddressSpaceLayout() {
   Printf("|| `[%p, %p]` || HighMem    ||\n",
          (void*)kHighMemBeg, (void*)kHighMemEnd);
   Printf("|| `[%p, %p]` || HighShadow ||\n",
@@ -408,6 +350,8 @@ static void PrintAddressSpaceLayout() {
   Printf("redzone=%zu\n", (uptr)flags()->redzone);
   Printf("max_redzone=%zu\n", (uptr)flags()->max_redzone);
   Printf("quarantine_size_mb=%zuM\n", (uptr)flags()->quarantine_size_mb);
+  Printf("thread_local_quarantine_size_kb=%zuK\n",
+         (uptr)flags()->thread_local_quarantine_size_kb);
   Printf("malloc_context_size=%zu\n",
          (uptr)common_flags()->malloc_context_size);
 
@@ -472,78 +416,9 @@ static void AsanInitInternal() {
 
   ReplaceSystemMalloc();
 
-  // Set the shadow memory address to uninitialized.
-  __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
-
-  uptr shadow_start = kLowShadowBeg;
-  // Detect if a dynamic shadow address must used and find a available location
-  // when necessary. When dynamic address is used, the macro |kLowShadowBeg|
-  // expands to |__asan_shadow_memory_dynamic_address| which is
-  // |kDefaultShadowSentinel|.
-  if (shadow_start == kDefaultShadowSentinel) {
-    __asan_shadow_memory_dynamic_address = 0;
-    CHECK_EQ(0, kLowShadowBeg);
-
-    uptr granularity = GetMmapGranularity();
-    uptr alignment = 8 * granularity;
-    uptr left_padding = granularity;
-    uptr space_size = kHighShadowEnd + left_padding;
-
-    shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity);
-    CHECK_NE((uptr)0, shadow_start);
-    CHECK(IsAligned(shadow_start, alignment));
-  }
-  // Update the shadow memory address (potentially) used by instrumentation.
-  __asan_shadow_memory_dynamic_address = shadow_start;
-
-  if (kLowShadowBeg)
-    shadow_start -= GetMmapGranularity();
-  bool full_shadow_is_available =
-      MemoryRangeIsAvailable(shadow_start, kHighShadowEnd);
-
-#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) &&                \
-    !ASAN_FIXED_MAPPING
-  if (!full_shadow_is_available) {
-    kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0;
-    kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0;
-  }
-#endif
-
-  if (Verbosity()) PrintAddressSpaceLayout();
-
   DisableCoreDumperIfNecessary();
 
-  if (full_shadow_is_available) {
-    // mmap the low shadow plus at least one page at the left.
-    if (kLowShadowBeg)
-      ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
-    // mmap the high shadow.
-    ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
-    // protect the gap.
-    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
-    CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1);
-  } else if (kMidMemBeg &&
-      MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) &&
-      MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) {
-    CHECK(kLowShadowBeg != kLowShadowEnd);
-    // mmap the low shadow plus at least one page at the left.
-    ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
-    // mmap the mid shadow.
-    ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow");
-    // mmap the high shadow.
-    ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
-    // protect the gaps.
-    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
-    ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
-    ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
-  } else {
-    Report("Shadow memory range interleaves with an existing memory mapping. "
-           "ASan cannot proceed correctly. ABORTING.\n");
-    Report("ASan shadow was supposed to be located in the [%p-%p] range.\n",
-           shadow_start, kHighShadowEnd);
-    DumpProcessMap();
-    Die();
-  }
+  InitializeShadowMemory();
 
   AsanTSDInit(PlatformTSDDtor);
   InstallDeadlySignalHandlers(AsanOnDeadlySignal);
@@ -574,20 +449,18 @@ static void AsanInitInternal() {
   InitTlsSize();
 
   // Create main thread.
-  AsanThread *main_thread = AsanThread::Create(
-      /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0,
-      /* stack */ nullptr, /* detached */ true);
+  AsanThread *main_thread = CreateMainThread();
   CHECK_EQ(0, main_thread->tid());
-  SetCurrentThread(main_thread);
-  main_thread->ThreadStart(internal_getpid(),
-                           /* signal_thread_is_registered */ nullptr);
   force_interface_symbols();  // no-op.
   SanitizerInitializeUnwinder();
 
   if (CAN_SANITIZE_LEAKS) {
     __lsan::InitCommonLsan();
     if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
-      Atexit(__lsan::DoLeakCheck);
+      if (flags()->halt_on_error)
+        Atexit(__lsan::DoLeakCheck);
+      else
+        Atexit(__lsan::DoRecoverableLeakCheckVoid);
     }
   }
 
@@ -607,6 +480,11 @@ static void AsanInitInternal() {
   }
 
   VReport(1, "AddressSanitizer Init done\n");
+
+  if (flags()->sleep_after_init) {
+    Report("Sleeping for %d second(s)\n", flags()->sleep_after_init);
+    SleepForSeconds(flags()->sleep_after_init);
+  }
 }
 
 // Initialize as requested from some part of ASan runtime library (interceptors,
@@ -646,6 +524,7 @@ void NOINLINE __asan_handle_no_return() {
     top = curr_thread->stack_top();
     bottom = ((uptr)&local_stack - PageSize) & ~(PageSize - 1);
   } else {
+    CHECK(!SANITIZER_FUCHSIA);
     // If we haven't seen this thread, try asking the OS for stack bounds.
     uptr tls_addr, tls_size, stack_size;
     GetThreadStackAndTls(/*main=*/false, &bottom, &stack_size, &tls_addr,
diff --git a/libsanitizer/asan/asan_scariness_score.h b/libsanitizer/asan/asan_scariness_score.h
index d72fce69d64..aa947ed2732 100644
--- a/libsanitizer/asan/asan_scariness_score.h
+++ b/libsanitizer/asan/asan_scariness_score.h
@@ -45,7 +45,7 @@ struct ScarinessScoreBase {
   };
   int GetScore() const { return score; }
   const char *GetDescription() const { return descr; }
-  void Print() {
+  void Print() const {
     if (score && flags()->print_scariness)
       Printf("SCARINESS: %d (%s)\n", score, descr);
   }
diff --git a/libsanitizer/asan/asan_shadow_setup.cc b/libsanitizer/asan/asan_shadow_setup.cc
new file mode 100644
index 00000000000..9629b36798f
--- /dev/null
+++ b/libsanitizer/asan/asan_shadow_setup.cc
@@ -0,0 +1,159 @@
+//===-- asan_shadow_setup.cc ----------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Set up the shadow memory.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+// asan_fuchsia.cc has its own InitializeShadowMemory implementation.
+#if !SANITIZER_FUCHSIA
+
+#include "asan_internal.h"
+#include "asan_mapping.h"
+
+namespace __asan {
+
+// ---------------------- mmap -------------------- {{{1
+// Reserve memory range [beg, end].
+// We need to use inclusive range because end+1 may not be representable.
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
+  CHECK_EQ((beg % GetMmapGranularity()), 0);
+  CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
+  uptr size = end - beg + 1;
+  DecreaseTotalMmap(size);  // Don't count the shadow against mmap_limit_mb.
+  void *res = MmapFixedNoReserve(beg, size, name);
+  if (res != (void *)beg) {
+    Report(
+        "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
+        "Perhaps you're using ulimit -v\n",
+        size);
+    Abort();
+  }
+  if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size);
+  if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size);
+}
+
+static void ProtectGap(uptr addr, uptr size) {
+  if (!flags()->protect_shadow_gap) {
+    // The shadow gap is unprotected, so there is a chance that someone
+    // is actually using this memory. Which means it needs a shadow...
+    uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached());
+    uptr GapShadowEnd =
+        RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1;
+    if (Verbosity())
+      Printf(
+          "protect_shadow_gap=0:"
+          " not protecting shadow gap, allocating gap's shadow\n"
+          "|| `[%p, %p]` || ShadowGap's shadow ||\n",
+          GapShadowBeg, GapShadowEnd);
+    ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
+                             "unprotected gap shadow");
+    return;
+  }
+  void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+  if (addr == (uptr)res) return;
+  // A few pages at the start of the address space can not be protected.
+  // But we really want to protect as much as possible, to prevent this memory
+  // being returned as a result of a non-FIXED mmap().
+  if (addr == kZeroBaseShadowStart) {
+    uptr step = GetMmapGranularity();
+    while (size > step && addr < kZeroBaseMaxShadowStart) {
+      addr += step;
+      size -= step;
+      void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+      if (addr == (uptr)res) return;
+    }
+  }
+
+  Report(
+      "ERROR: Failed to protect the shadow gap. "
+      "ASan cannot proceed correctly. ABORTING.\n");
+  DumpProcessMap();
+  Die();
+}
+
+static void MaybeReportLinuxPIEBug() {
+#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__aarch64__))
+  Report("This might be related to ELF_ET_DYN_BASE change in Linux 4.12.\n");
+  Report(
+      "See https://github.com/google/sanitizers/issues/856 for possible "
+      "workarounds.\n");
+#endif
+}
+
+void InitializeShadowMemory() {
+  // Set the shadow memory address to uninitialized.
+  __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
+
+  uptr shadow_start = kLowShadowBeg;
+  // Detect if a dynamic shadow address must used and find a available location
+  // when necessary. When dynamic address is used, the macro |kLowShadowBeg|
+  // expands to |__asan_shadow_memory_dynamic_address| which is
+  // |kDefaultShadowSentinel|.
+  if (shadow_start == kDefaultShadowSentinel) {
+    __asan_shadow_memory_dynamic_address = 0;
+    CHECK_EQ(0, kLowShadowBeg);
+    shadow_start = FindDynamicShadowStart();
+  }
+  // Update the shadow memory address (potentially) used by instrumentation.
+  __asan_shadow_memory_dynamic_address = shadow_start;
+
+  if (kLowShadowBeg) shadow_start -= GetMmapGranularity();
+  bool full_shadow_is_available =
+      MemoryRangeIsAvailable(shadow_start, kHighShadowEnd);
+
+#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) && \
+    !ASAN_FIXED_MAPPING
+  if (!full_shadow_is_available) {
+    kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0;
+    kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0;
+  }
+#endif
+
+  if (Verbosity()) PrintAddressSpaceLayout();
+
+  if (full_shadow_is_available) {
+    // mmap the low shadow plus at least one page at the left.
+    if (kLowShadowBeg)
+      ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
+    // mmap the high shadow.
+    ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
+    // protect the gap.
+    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+    CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1);
+  } else if (kMidMemBeg &&
+             MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) &&
+             MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) {
+    CHECK(kLowShadowBeg != kLowShadowEnd);
+    // mmap the low shadow plus at least one page at the left.
+    ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
+    // mmap the mid shadow.
+    ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow");
+    // mmap the high shadow.
+    ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
+    // protect the gaps.
+    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+    ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
+    ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
+  } else {
+    Report(
+        "Shadow memory range interleaves with an existing memory mapping. "
+        "ASan cannot proceed correctly. ABORTING.\n");
+    Report("ASan shadow was supposed to be located in the [%p-%p] range.\n",
+           shadow_start, kHighShadowEnd);
+    MaybeReportLinuxPIEBug();
+    DumpProcessMap();
+    Die();
+  }
+}
+
+}  // namespace __asan
+
+#endif  // !SANITIZER_FUCHSIA
diff --git a/libsanitizer/asan/asan_stack.h b/libsanitizer/asan/asan_stack.h
index 2b8738156ea..aa8c4cdc2f2 100644
--- a/libsanitizer/asan/asan_stack.h
+++ b/libsanitizer/asan/asan_stack.h
@@ -39,10 +39,6 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth,
   stack->size = 0;
   if (LIKELY(asan_inited)) {
     if ((t = GetCurrentThread()) && !t->isUnwinding()) {
-      // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace()
-      // yields the call stack of the signal's handler and not of the code
-      // that raised the signal (as it does on Linux).
-      if (SANITIZER_FREEBSD && t->isInDeadlySignal()) fast = true;
       uptr stack_top = t->stack_top();
       uptr stack_bottom = t->stack_bottom();
       ScopedUnwinding unwind_scope(t);
diff --git a/libsanitizer/asan/asan_suppressions.cc b/libsanitizer/asan/asan_suppressions.cc
index 1dc9d474240..00406020054 100644
--- a/libsanitizer/asan/asan_suppressions.cc
+++ b/libsanitizer/asan/asan_suppressions.cc
@@ -29,15 +29,9 @@ static const char *kSuppressionTypes[] = {
     kInterceptorName, kInterceptorViaFunction, kInterceptorViaLibrary,
     kODRViolation};
 
-extern "C" {
-#if SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char *__asan_default_suppressions();
-#else
-// No week hooks, provide empty implementation.
-const char *__asan_default_suppressions() { return ""; }
-#endif  // SANITIZER_SUPPORTS_WEAK_HOOKS
-}  // extern "C"
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __asan_default_suppressions, void) {
+  return "";
+}
 
 void InitializeSuppressions() {
   CHECK_EQ(nullptr, suppression_ctx);
diff --git a/libsanitizer/asan/asan_thread.cc b/libsanitizer/asan/asan_thread.cc
index 818e1261400..d0fdf6e9847 100644
--- a/libsanitizer/asan/asan_thread.cc
+++ b/libsanitizer/asan/asan_thread.cc
@@ -25,11 +25,6 @@ namespace __asan {
 
 // AsanThreadContext implementation.
 
-struct CreateThreadContextArgs {
-  AsanThread *thread;
-  StackTrace *stack;
-};
-
 void AsanThreadContext::OnCreated(void *arg) {
   CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs*>(arg);
   if (args->stack)
@@ -86,7 +81,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg,
   AsanThread *thread = (AsanThread*)MmapOrDie(size, __func__);
   thread->start_routine_ = start_routine;
   thread->arg_ = arg;
-  CreateThreadContextArgs args = { thread, stack };
+  AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
   asanThreadRegistry().CreateThread(*reinterpret_cast<uptr *>(thread), detached,
                                     parent_tid, &args);
 
@@ -164,16 +159,19 @@ void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save,
 }
 
 inline AsanThread::StackBounds AsanThread::GetStackBounds() const {
-  if (!atomic_load(&stack_switching_, memory_order_acquire))
-    return StackBounds{stack_bottom_, stack_top_};  // NOLINT
+  if (!atomic_load(&stack_switching_, memory_order_acquire)) {
+    // Make sure the stack bounds are fully initialized.
+    if (stack_bottom_ >= stack_top_) return {0, 0};
+    return {stack_bottom_, stack_top_};
+  }
   char local;
   const uptr cur_stack = (uptr)&local;
   // Note: need to check next stack first, because FinishSwitchFiber
   // may be in process of overwriting stack_top_/bottom_. But in such case
   // we are already on the next stack.
   if (cur_stack >= next_stack_bottom_ && cur_stack < next_stack_top_)
-    return StackBounds{next_stack_bottom_, next_stack_top_};  // NOLINT
-  return StackBounds{stack_bottom_, stack_top_};              // NOLINT
+    return {next_stack_bottom_, next_stack_top_};
+  return {stack_bottom_, stack_top_};
 }
 
 uptr AsanThread::stack_top() {
@@ -218,12 +216,12 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
   return nullptr;
 }
 
-void AsanThread::Init() {
+void AsanThread::Init(const InitOptions *options) {
   next_stack_top_ = next_stack_bottom_ = 0;
   atomic_store(&stack_switching_, false, memory_order_release);
   fake_stack_ = nullptr;  // Will be initialized lazily if needed.
   CHECK_EQ(this->stack_size(), 0U);
-  SetThreadStackAndTls();
+  SetThreadStackAndTls(options);
   CHECK_GT(this->stack_size(), 0U);
   CHECK(AddrIsInMem(stack_bottom_));
   CHECK(AddrIsInMem(stack_top_ - 1));
@@ -234,10 +232,15 @@ void AsanThread::Init() {
           &local);
 }
 
+// Fuchsia doesn't use ThreadStart.
+// asan_fuchsia.c defines CreateMainThread and SetThreadStackAndTls.
+#if !SANITIZER_FUCHSIA
+
 thread_return_t AsanThread::ThreadStart(
-    uptr os_id, atomic_uintptr_t *signal_thread_is_registered) {
+    tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) {
   Init();
-  asanThreadRegistry().StartThread(tid(), os_id, nullptr);
+  asanThreadRegistry().StartThread(tid(), os_id, /*workerthread*/ false,
+                                   nullptr);
   if (signal_thread_is_registered)
     atomic_store(signal_thread_is_registered, 1, memory_order_release);
 
@@ -264,7 +267,21 @@ thread_return_t AsanThread::ThreadStart(
   return res;
 }
 
-void AsanThread::SetThreadStackAndTls() {
+AsanThread *CreateMainThread() {
+  AsanThread *main_thread = AsanThread::Create(
+      /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0,
+      /* stack */ nullptr, /* detached */ true);
+  SetCurrentThread(main_thread);
+  main_thread->ThreadStart(internal_getpid(),
+                           /* signal_thread_is_registered */ nullptr);
+  return main_thread;
+}
+
+// This implementation doesn't use the argument, which is just passed down
+// from the caller of Init (which see, above).  It's only there to support
+// OS-specific implementations that need more information passed through.
+void AsanThread::SetThreadStackAndTls(const InitOptions *options) {
+  DCHECK_EQ(options, nullptr);
   uptr tls_size = 0;
   uptr stack_size = 0;
   GetThreadStackAndTls(tid() == 0, const_cast<uptr *>(&stack_bottom_),
@@ -277,6 +294,8 @@ void AsanThread::SetThreadStackAndTls() {
   CHECK(AddrIsInStack((uptr)&local));
 }
 
+#endif  // !SANITIZER_FUCHSIA
+
 void AsanThread::ClearShadowForThreadStackAndTLS() {
   PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0);
   if (tls_begin_ != tls_end_)
@@ -297,24 +316,27 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
     return true;
   }
   uptr aligned_addr = addr & ~(SANITIZER_WORDSIZE/8 - 1);  // align addr.
+  uptr mem_ptr = RoundDownTo(aligned_addr, SHADOW_GRANULARITY);
   u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr);
   u8 *shadow_bottom = (u8*)MemToShadow(bottom);
 
   while (shadow_ptr >= shadow_bottom &&
          *shadow_ptr != kAsanStackLeftRedzoneMagic) {
     shadow_ptr--;
+    mem_ptr -= SHADOW_GRANULARITY;
   }
 
   while (shadow_ptr >= shadow_bottom &&
          *shadow_ptr == kAsanStackLeftRedzoneMagic) {
     shadow_ptr--;
+    mem_ptr -= SHADOW_GRANULARITY;
   }
 
   if (shadow_ptr < shadow_bottom) {
     return false;
   }
 
-  uptr* ptr = (uptr*)SHADOW_TO_MEM((uptr)(shadow_ptr + 1));
+  uptr* ptr = (uptr*)(mem_ptr + SHADOW_GRANULARITY);
   CHECK(ptr[0] == kCurrentStackFrameMagic);
   access->offset = addr - (uptr)ptr;
   access->frame_pc = ptr[2];
@@ -389,7 +411,7 @@ void EnsureMainThreadIDIsCorrect() {
     context->os_id = GetTid();
 }
 
-__asan::AsanThread *GetAsanThreadByOsIDLocked(uptr os_id) {
+__asan::AsanThread *GetAsanThreadByOsIDLocked(tid_t os_id) {
   __asan::AsanThreadContext *context = static_cast<__asan::AsanThreadContext *>(
       __asan::asanThreadRegistry().FindThreadContextByOsIDLocked(os_id));
   if (!context) return nullptr;
@@ -399,7 +421,7 @@ __asan::AsanThread *GetAsanThreadByOsIDLocked(uptr os_id) {
 
 // --- Implementation of LSan-specific functions --- {{{1
 namespace __lsan {
-bool GetThreadRangesLocked(uptr os_id, uptr *stack_begin, uptr *stack_end,
+bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
                            uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
                            uptr *cache_end, DTLS **dtls) {
   __asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id);
@@ -415,7 +437,7 @@ bool GetThreadRangesLocked(uptr os_id, uptr *stack_begin, uptr *stack_end,
   return true;
 }
 
-void ForEachExtraStackRange(uptr os_id, RangeIteratorCallback callback,
+void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
                             void *arg) {
   __asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id);
   if (t && t->has_fake_stack())
diff --git a/libsanitizer/asan/asan_thread.h b/libsanitizer/asan/asan_thread.h
index c51a58ad0bb..f7a91f3e73b 100644
--- a/libsanitizer/asan/asan_thread.h
+++ b/libsanitizer/asan/asan_thread.h
@@ -47,6 +47,11 @@ class AsanThreadContext : public ThreadContextBase {
 
   void OnCreated(void *arg) override;
   void OnFinished() override;
+
+  struct CreateThreadContextArgs {
+    AsanThread *thread;
+    StackTrace *stack;
+  };
 };
 
 // AsanThreadContext objects are never freed, so we need many of them.
@@ -60,8 +65,10 @@ class AsanThread {
   static void TSDDtor(void *tsd);
   void Destroy();
 
-  void Init();  // Should be called from the thread itself.
-  thread_return_t ThreadStart(uptr os_id,
+  struct InitOptions;
+  void Init(const InitOptions *options = nullptr);
+
+  thread_return_t ThreadStart(tid_t os_id,
                               atomic_uintptr_t *signal_thread_is_registered);
 
   uptr stack_top();
@@ -116,17 +123,15 @@ class AsanThread {
   bool isUnwinding() const { return unwinding_; }
   void setUnwinding(bool b) { unwinding_ = b; }
 
-  // True if we are in a deadly signal handler.
-  bool isInDeadlySignal() const { return in_deadly_signal_; }
-  void setInDeadlySignal(bool b) { in_deadly_signal_ = b; }
-
   AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
   AsanStats &stats() { return stats_; }
 
  private:
   // NOTE: There is no AsanThread constructor. It is allocated
   // via mmap() and *must* be valid in zero-initialized state.
-  void SetThreadStackAndTls();
+
+  void SetThreadStackAndTls(const InitOptions *options);
+
   void ClearShadowForThreadStackAndTLS();
   FakeStack *AsyncSignalSafeLazyInitFakeStack();
 
@@ -156,7 +161,6 @@ class AsanThread {
   AsanThreadLocalMallocStorage malloc_storage_;
   AsanStats stats_;
   bool unwinding_;
-  bool in_deadly_signal_;
 };
 
 // ScopedUnwinding is a scope for stacktracing member of a context
@@ -171,20 +175,6 @@ class ScopedUnwinding {
   AsanThread *thread;
 };
 
-// ScopedDeadlySignal is a scope for handling deadly signals.
-class ScopedDeadlySignal {
- public:
-  explicit ScopedDeadlySignal(AsanThread *t) : thread(t) {
-    if (thread) thread->setInDeadlySignal(true);
-  }
-  ~ScopedDeadlySignal() {
-    if (thread) thread->setInDeadlySignal(false);
-  }
-
- private:
-  AsanThread *thread;
-};
-
 // Returns a single instance of registry.
 ThreadRegistry &asanThreadRegistry();
 
diff --git a/libsanitizer/asan/asan_win.cc b/libsanitizer/asan/asan_win.cc
index efd82bfc3b0..02c7ed11628 100644
--- a/libsanitizer/asan/asan_win.cc
+++ b/libsanitizer/asan/asan_win.cc
@@ -25,6 +25,8 @@
 #include "asan_mapping.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_win.h"
+#include "sanitizer_common/sanitizer_win_defs.h"
 
 using namespace __asan;  // NOLINT
 
@@ -40,35 +42,50 @@ uptr __asan_get_shadow_memory_dynamic_address() {
   __asan_init();
   return __asan_shadow_memory_dynamic_address;
 }
-
-// -------------------- A workaround for the absence of weak symbols ----- {{{
-// We don't have a direct equivalent of weak symbols when using MSVC, but we can
-// use the /alternatename directive to tell the linker to default a specific
-// symbol to a specific value, which works nicely for allocator hooks and
-// __asan_default_options().
-void __sanitizer_default_malloc_hook(void *ptr, uptr size) { }
-void __sanitizer_default_free_hook(void *ptr) { }
-const char* __asan_default_default_options() { return ""; }
-const char* __asan_default_default_suppressions() { return ""; }
-void __asan_default_on_error() {}
-// 64-bit msvc will not prepend an underscore for symbols.
-#ifdef _WIN64
-#pragma comment(linker, "/alternatename:__sanitizer_malloc_hook=__sanitizer_default_malloc_hook")  // NOLINT
-#pragma comment(linker, "/alternatename:__sanitizer_free_hook=__sanitizer_default_free_hook")      // NOLINT
-#pragma comment(linker, "/alternatename:__asan_default_options=__asan_default_default_options")    // NOLINT
-#pragma comment(linker, "/alternatename:__asan_default_suppressions=__asan_default_default_suppressions")    // NOLINT
-#pragma comment(linker, "/alternatename:__asan_on_error=__asan_default_on_error")                  // NOLINT
-#else
-#pragma comment(linker, "/alternatename:___sanitizer_malloc_hook=___sanitizer_default_malloc_hook")  // NOLINT
-#pragma comment(linker, "/alternatename:___sanitizer_free_hook=___sanitizer_default_free_hook")      // NOLINT
-#pragma comment(linker, "/alternatename:___asan_default_options=___asan_default_default_options")    // NOLINT
-#pragma comment(linker, "/alternatename:___asan_default_suppressions=___asan_default_default_suppressions")    // NOLINT
-#pragma comment(linker, "/alternatename:___asan_on_error=___asan_default_on_error")                  // NOLINT
-#endif
-// }}}
 }  // extern "C"
 
 // ---------------------- Windows-specific interceptors ---------------- {{{
+static LPTOP_LEVEL_EXCEPTION_FILTER default_seh_handler;
+static LPTOP_LEVEL_EXCEPTION_FILTER user_seh_handler;
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+long __asan_unhandled_exception_filter(EXCEPTION_POINTERS *info) {
+  EXCEPTION_RECORD *exception_record = info->ExceptionRecord;
+  CONTEXT *context = info->ContextRecord;
+
+  // FIXME: Handle EXCEPTION_STACK_OVERFLOW here.
+
+  SignalContext sig(exception_record, context);
+  ReportDeadlySignal(sig);
+  UNREACHABLE("returned from reporting deadly signal");
+}
+
+// Wrapper SEH Handler. If the exception should be handled by asan, we call
+// __asan_unhandled_exception_filter, otherwise, we execute the user provided
+// exception handler or the default.
+static long WINAPI SEHHandler(EXCEPTION_POINTERS *info) {
+  DWORD exception_code = info->ExceptionRecord->ExceptionCode;
+  if (__sanitizer::IsHandledDeadlyException(exception_code))
+    return __asan_unhandled_exception_filter(info);
+  if (user_seh_handler)
+    return user_seh_handler(info);
+  // Bubble out to the default exception filter.
+  if (default_seh_handler)
+    return default_seh_handler(info);
+  return EXCEPTION_CONTINUE_SEARCH;
+}
+
+INTERCEPTOR_WINAPI(LPTOP_LEVEL_EXCEPTION_FILTER, SetUnhandledExceptionFilter,
+    LPTOP_LEVEL_EXCEPTION_FILTER ExceptionFilter) {
+  CHECK(REAL(SetUnhandledExceptionFilter));
+  if (ExceptionFilter == &SEHHandler)
+    return REAL(SetUnhandledExceptionFilter)(ExceptionFilter);
+  // We record the user provided exception handler to be called for all the
+  // exceptions unhandled by asan.
+  Swap(ExceptionFilter, user_seh_handler);
+  return ExceptionFilter;
+}
+
 INTERCEPTOR_WINAPI(void, RtlRaiseException, EXCEPTION_RECORD *ExceptionRecord) {
   CHECK(REAL(RtlRaiseException));
   // This is a noreturn function, unless it's one of the exceptions raised to
@@ -141,6 +158,7 @@ namespace __asan {
 
 void InitializePlatformInterceptors() {
   ASAN_INTERCEPT_FUNC(CreateThread);
+  ASAN_INTERCEPT_FUNC(SetUnhandledExceptionFilter);
 
 #ifdef _WIN64
   ASAN_INTERCEPT_FUNC(__C_specific_handler);
@@ -197,6 +215,18 @@ void *AsanDoesNotSupportStaticLinkage() {
   return 0;
 }
 
+uptr FindDynamicShadowStart() {
+  uptr granularity = GetMmapGranularity();
+  uptr alignment = 8 * granularity;
+  uptr left_padding = granularity;
+  uptr space_size = kHighShadowEnd + left_padding;
+  uptr shadow_start =
+      FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+  CHECK_NE((uptr)0, shadow_start);
+  CHECK(IsAligned(shadow_start, alignment));
+  return shadow_start;
+}
+
 void AsanCheckDynamicRTPrereqs() {}
 
 void AsanCheckIncompatibleRT() {}
@@ -257,52 +287,8 @@ void InitializePlatformExceptionHandlers() {
 #endif
 }
 
-static LPTOP_LEVEL_EXCEPTION_FILTER default_seh_handler;
-
-// Check based on flags if we should report this exception.
-static bool ShouldReportDeadlyException(unsigned code) {
-  switch (code) {
-    case EXCEPTION_ACCESS_VIOLATION:
-    case EXCEPTION_IN_PAGE_ERROR:
-      return common_flags()->handle_segv;
-    case EXCEPTION_BREAKPOINT:
-    case EXCEPTION_ILLEGAL_INSTRUCTION: {
-      return common_flags()->handle_sigill;
-    }
-  }
-  return false;
-}
-
-// Return the textual name for this exception.
-const char *DescribeSignalOrException(int signo) {
-  unsigned code = signo;
-  // Get the string description of the exception if this is a known deadly
-  // exception.
-  switch (code) {
-    case EXCEPTION_ACCESS_VIOLATION:
-      return "access-violation";
-    case EXCEPTION_IN_PAGE_ERROR:
-      return "in-page-error";
-    case EXCEPTION_BREAKPOINT:
-      return "breakpoint";
-    case EXCEPTION_ILLEGAL_INSTRUCTION:
-      return "illegal-instruction";
-  }
-  return nullptr;
-}
-
-static long WINAPI SEHHandler(EXCEPTION_POINTERS *info) {
-  EXCEPTION_RECORD *exception_record = info->ExceptionRecord;
-  CONTEXT *context = info->ContextRecord;
-
-  if (ShouldReportDeadlyException(exception_record->ExceptionCode)) {
-    SignalContext sig = SignalContext::Create(exception_record, context);
-    ReportDeadlySignal(exception_record->ExceptionCode, sig);
-  }
-
-  // FIXME: Handle EXCEPTION_STACK_OVERFLOW here.
-
-  return default_seh_handler(info);
+bool IsSystemHeapAddress(uptr addr) {
+  return ::HeapValidate(GetProcessHeap(), 0, (void*)addr) != FALSE;
 }
 
 // We want to install our own exception handler (EH) to print helpful reports
@@ -341,10 +327,25 @@ int __asan_set_seh_filter() {
 // immediately after the CRT runs. This way, our exception filter is called
 // first and we can delegate to their filter if appropriate.
 #pragma section(".CRT$XCAB", long, read)  // NOLINT
-__declspec(allocate(".CRT$XCAB"))
-    int (*__intercept_seh)() = __asan_set_seh_filter;
+__declspec(allocate(".CRT$XCAB")) int (*__intercept_seh)() =
+    __asan_set_seh_filter;
+
+// Piggyback on the TLS initialization callback directory to initialize asan as
+// early as possible. Initializers in .CRT$XL* are called directly by ntdll,
+// which run before the CRT. Users also add code to .CRT$XLC, so it's important
+// to run our initializers first.
+static void NTAPI asan_thread_init(void *module, DWORD reason, void *reserved) {
+  if (reason == DLL_PROCESS_ATTACH) __asan_init();
+}
+
+#pragma section(".CRT$XLAB", long, read)  // NOLINT
+__declspec(allocate(".CRT$XLAB")) void (NTAPI *__asan_tls_init)(void *,
+    unsigned long, void *) = asan_thread_init;
 #endif
+
+WIN_FORCE_LINK(__asan_dso_reg_hook)
+
 // }}}
 }  // namespace __asan
 
-#endif  // _WIN32
+#endif  // SANITIZER_WINDOWS
diff --git a/libsanitizer/asan/asan_win_dll_thunk.cc b/libsanitizer/asan/asan_win_dll_thunk.cc
index f7c9a37bf79..31847efe77a 100644
--- a/libsanitizer/asan/asan_win_dll_thunk.cc
+++ b/libsanitizer/asan/asan_win_dll_thunk.cc
@@ -13,375 +13,41 @@
 // See https://github.com/google/sanitizers/issues/209 for the details.
 //===----------------------------------------------------------------------===//
 
-// Only compile this code when building asan_dll_thunk.lib
-// Using #ifdef rather than relying on Makefiles etc.
-// simplifies the build procedure.
-#ifdef ASAN_DLL_THUNK
+#ifdef SANITIZER_DLL_THUNK
 #include "asan_init_version.h"
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_win_defs.h"
+#include "sanitizer_common/sanitizer_win_dll_thunk.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 
-// ---------- Function interception helper functions and macros ----------- {{{1
-extern "C" {
-void *__stdcall GetModuleHandleA(const char *module_name);
-void *__stdcall GetProcAddress(void *module, const char *proc_name);
-void abort();
-}
-
-using namespace __sanitizer;
-
-static uptr getRealProcAddressOrDie(const char *name) {
-  uptr ret =
-      __interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name);
-  if (!ret)
-    abort();
-  return ret;
-}
-
-// We need to intercept some functions (e.g. ASan interface, memory allocator --
-// let's call them "hooks") exported by the DLL thunk and forward the hooks to
-// the runtime in the main module.
-// However, we don't want to keep two lists of these hooks.
-// To avoid that, the list of hooks should be defined using the
-// INTERCEPT_WHEN_POSSIBLE macro. Then, all these hooks can be intercepted
-// at once by calling INTERCEPT_HOOKS().
-
-// Use macro+template magic to automatically generate the list of hooks.
-// Each hook at line LINE defines a template class with a static
-// FunctionInterceptor<LINE>::Execute() method intercepting the hook.
-// The default implementation of FunctionInterceptor<LINE> is to call
-// the Execute() method corresponding to the previous line.
-template<int LINE>
-struct FunctionInterceptor {
-  static void Execute() { FunctionInterceptor<LINE-1>::Execute(); }
-};
-
-// There shouldn't be any hooks with negative definition line number.
-template<>
-struct FunctionInterceptor<0> {
-  static void Execute() {}
-};
-
-#define INTERCEPT_WHEN_POSSIBLE(main_function, dll_function)                   \
-  template <> struct FunctionInterceptor<__LINE__> {                           \
-    static void Execute() {                                                    \
-      uptr wrapper = getRealProcAddressOrDie(main_function);                   \
-      if (!__interception::OverrideFunction((uptr)dll_function, wrapper, 0))   \
-        abort();                                                               \
-      FunctionInterceptor<__LINE__ - 1>::Execute();                            \
-    }                                                                          \
-  };
-
-// Special case of hooks -- ASan own interface functions.  Those are only called
-// after __asan_init, thus an empty implementation is sufficient.
-#define INTERFACE_FUNCTION(name)                                               \
-  extern "C" __declspec(noinline) void name() {                                \
-    volatile int prevent_icf = (__LINE__ << 8); (void)prevent_icf;             \
-    __debugbreak();                                                            \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name)
-
-// INTERCEPT_HOOKS must be used after the last INTERCEPT_WHEN_POSSIBLE.
-#define INTERCEPT_HOOKS FunctionInterceptor<__LINE__>::Execute
-
-// We can't define our own version of strlen etc. because that would lead to
-// link-time or even type mismatch errors.  Instead, we can declare a function
-// just to be able to get its address.  Me may miss the first few calls to the
-// functions since it can be called before __asan_init, but that would lead to
-// false negatives in the startup code before user's global initializers, which
-// isn't a big deal.
-#define INTERCEPT_LIBRARY_FUNCTION(name)                                       \
-  extern "C" void name();                                                      \
-  INTERCEPT_WHEN_POSSIBLE(WRAPPER_NAME(name), name)
-
-// Disable compiler warnings that show up if we declare our own version
-// of a compiler intrinsic (e.g. strlen).
-#pragma warning(disable: 4391)
-#pragma warning(disable: 4392)
-
-static void InterceptHooks();
-// }}}
-
-// ---------- Function wrapping helpers ----------------------------------- {{{1
-#define WRAP_V_V(name)                                                         \
-  extern "C" void name() {                                                     \
-    typedef void (*fntype)();                                                  \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    fn();                                                                      \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_V_W(name)                                                         \
-  extern "C" void name(void *arg) {                                            \
-    typedef void (*fntype)(void *arg);                                         \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    fn(arg);                                                                   \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_V_WW(name)                                                        \
-  extern "C" void name(void *arg1, void *arg2) {                               \
-    typedef void (*fntype)(void *, void *);                                    \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    fn(arg1, arg2);                                                            \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_V_WWW(name)                                                       \
-  extern "C" void name(void *arg1, void *arg2, void *arg3) {                   \
-    typedef void *(*fntype)(void *, void *, void *);                           \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    fn(arg1, arg2, arg3);                                                      \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_W_V(name)                                                         \
-  extern "C" void *name() {                                                    \
-    typedef void *(*fntype)();                                                 \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    return fn();                                                               \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_W_W(name)                                                         \
-  extern "C" void *name(void *arg) {                                           \
-    typedef void *(*fntype)(void *arg);                                        \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    return fn(arg);                                                            \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_W_WW(name)                                                        \
-  extern "C" void *name(void *arg1, void *arg2) {                              \
-    typedef void *(*fntype)(void *, void *);                                   \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    return fn(arg1, arg2);                                                     \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_W_WWW(name)                                                       \
-  extern "C" void *name(void *arg1, void *arg2, void *arg3) {                  \
-    typedef void *(*fntype)(void *, void *, void *);                           \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    return fn(arg1, arg2, arg3);                                               \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_W_WWWW(name)                                                      \
-  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4) {      \
-    typedef void *(*fntype)(void *, void *, void *, void *);                   \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    return fn(arg1, arg2, arg3, arg4);                                         \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_W_WWWWW(name)                                                     \
-  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4,        \
-                        void *arg5) {                                          \
-    typedef void *(*fntype)(void *, void *, void *, void *, void *);           \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    return fn(arg1, arg2, arg3, arg4, arg5);                                   \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-
-#define WRAP_W_WWWWWW(name)                                                    \
-  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4,        \
-                        void *arg5, void *arg6) {                              \
-    typedef void *(*fntype)(void *, void *, void *, void *, void *, void *);   \
-    static fntype fn = (fntype)getRealProcAddressOrDie(#name);                 \
-    return fn(arg1, arg2, arg3, arg4, arg5, arg6);                             \
-  }                                                                            \
-  INTERCEPT_WHEN_POSSIBLE(#name, name);
-// }}}
-
-// ----------------- ASan own interface functions --------------------
-// Don't use the INTERFACE_FUNCTION machinery for this function as we actually
-// want to call it in the __asan_init interceptor.
-WRAP_W_V(__asan_should_detect_stack_use_after_return)
-WRAP_W_V(__asan_get_shadow_memory_dynamic_address)
-
-extern "C" {
-  int __asan_option_detect_stack_use_after_return;
-  uptr __asan_shadow_memory_dynamic_address;
-
-  // Manually wrap __asan_init as we need to initialize
-  // __asan_option_detect_stack_use_after_return afterwards.
-  void __asan_init() {
-    typedef void (*fntype)();
-    static fntype fn = 0;
-    // __asan_init is expected to be called by only one thread.
-    if (fn) return;
-
-    fn = (fntype)getRealProcAddressOrDie("__asan_init");
-    fn();
-    __asan_option_detect_stack_use_after_return =
-        (__asan_should_detect_stack_use_after_return() != 0);
-    __asan_shadow_memory_dynamic_address =
-        (uptr)__asan_get_shadow_memory_dynamic_address();
-    InterceptHooks();
-  }
-}
-
-extern "C" void __asan_version_mismatch_check() {
-  // Do nothing.
-}
-
-INTERFACE_FUNCTION(__asan_handle_no_return)
-
-INTERFACE_FUNCTION(__asan_report_store1)
-INTERFACE_FUNCTION(__asan_report_store2)
-INTERFACE_FUNCTION(__asan_report_store4)
-INTERFACE_FUNCTION(__asan_report_store8)
-INTERFACE_FUNCTION(__asan_report_store16)
-INTERFACE_FUNCTION(__asan_report_store_n)
-
-INTERFACE_FUNCTION(__asan_report_load1)
-INTERFACE_FUNCTION(__asan_report_load2)
-INTERFACE_FUNCTION(__asan_report_load4)
-INTERFACE_FUNCTION(__asan_report_load8)
-INTERFACE_FUNCTION(__asan_report_load16)
-INTERFACE_FUNCTION(__asan_report_load_n)
-
-INTERFACE_FUNCTION(__asan_store1)
-INTERFACE_FUNCTION(__asan_store2)
-INTERFACE_FUNCTION(__asan_store4)
-INTERFACE_FUNCTION(__asan_store8)
-INTERFACE_FUNCTION(__asan_store16)
-INTERFACE_FUNCTION(__asan_storeN)
-
-INTERFACE_FUNCTION(__asan_load1)
-INTERFACE_FUNCTION(__asan_load2)
-INTERFACE_FUNCTION(__asan_load4)
-INTERFACE_FUNCTION(__asan_load8)
-INTERFACE_FUNCTION(__asan_load16)
-INTERFACE_FUNCTION(__asan_loadN)
-
-INTERFACE_FUNCTION(__asan_memcpy);
-INTERFACE_FUNCTION(__asan_memset);
-INTERFACE_FUNCTION(__asan_memmove);
-
-INTERFACE_FUNCTION(__asan_set_shadow_00);
-INTERFACE_FUNCTION(__asan_set_shadow_f1);
-INTERFACE_FUNCTION(__asan_set_shadow_f2);
-INTERFACE_FUNCTION(__asan_set_shadow_f3);
-INTERFACE_FUNCTION(__asan_set_shadow_f5);
-INTERFACE_FUNCTION(__asan_set_shadow_f8);
-
-INTERFACE_FUNCTION(__asan_alloca_poison);
-INTERFACE_FUNCTION(__asan_allocas_unpoison);
-
-INTERFACE_FUNCTION(__asan_register_globals)
-INTERFACE_FUNCTION(__asan_unregister_globals)
-
-INTERFACE_FUNCTION(__asan_before_dynamic_init)
-INTERFACE_FUNCTION(__asan_after_dynamic_init)
-
-INTERFACE_FUNCTION(__asan_poison_stack_memory)
-INTERFACE_FUNCTION(__asan_unpoison_stack_memory)
+// ASan own interface functions.
+#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "asan_interface.inc"
 
-INTERFACE_FUNCTION(__asan_poison_memory_region)
-INTERFACE_FUNCTION(__asan_unpoison_memory_region)
+// Memory allocation functions.
+INTERCEPT_WRAP_V_W(free)
+INTERCEPT_WRAP_V_W(_free_base)
+INTERCEPT_WRAP_V_WW(_free_dbg)
 
-INTERFACE_FUNCTION(__asan_address_is_poisoned)
-INTERFACE_FUNCTION(__asan_region_is_poisoned)
+INTERCEPT_WRAP_W_W(malloc)
+INTERCEPT_WRAP_W_W(_malloc_base)
+INTERCEPT_WRAP_W_WWWW(_malloc_dbg)
 
-INTERFACE_FUNCTION(__asan_get_current_fake_stack)
-INTERFACE_FUNCTION(__asan_addr_is_in_fake_stack)
+INTERCEPT_WRAP_W_WW(calloc)
+INTERCEPT_WRAP_W_WW(_calloc_base)
+INTERCEPT_WRAP_W_WWWWW(_calloc_dbg)
+INTERCEPT_WRAP_W_WWW(_calloc_impl)
 
-INTERFACE_FUNCTION(__asan_stack_malloc_0)
-INTERFACE_FUNCTION(__asan_stack_malloc_1)
-INTERFACE_FUNCTION(__asan_stack_malloc_2)
-INTERFACE_FUNCTION(__asan_stack_malloc_3)
-INTERFACE_FUNCTION(__asan_stack_malloc_4)
-INTERFACE_FUNCTION(__asan_stack_malloc_5)
-INTERFACE_FUNCTION(__asan_stack_malloc_6)
-INTERFACE_FUNCTION(__asan_stack_malloc_7)
-INTERFACE_FUNCTION(__asan_stack_malloc_8)
-INTERFACE_FUNCTION(__asan_stack_malloc_9)
-INTERFACE_FUNCTION(__asan_stack_malloc_10)
+INTERCEPT_WRAP_W_WW(realloc)
+INTERCEPT_WRAP_W_WW(_realloc_base)
+INTERCEPT_WRAP_W_WWW(_realloc_dbg)
+INTERCEPT_WRAP_W_WWW(_recalloc)
+INTERCEPT_WRAP_W_WWW(_recalloc_base)
 
-INTERFACE_FUNCTION(__asan_stack_free_0)
-INTERFACE_FUNCTION(__asan_stack_free_1)
-INTERFACE_FUNCTION(__asan_stack_free_2)
-INTERFACE_FUNCTION(__asan_stack_free_4)
-INTERFACE_FUNCTION(__asan_stack_free_5)
-INTERFACE_FUNCTION(__asan_stack_free_6)
-INTERFACE_FUNCTION(__asan_stack_free_7)
-INTERFACE_FUNCTION(__asan_stack_free_8)
-INTERFACE_FUNCTION(__asan_stack_free_9)
-INTERFACE_FUNCTION(__asan_stack_free_10)
-
-// FIXME: we might want to have a sanitizer_win_dll_thunk?
-INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
-INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
-INTERFACE_FUNCTION(__sanitizer_cov)
-INTERFACE_FUNCTION(__sanitizer_cov_dump)
-INTERFACE_FUNCTION(__sanitizer_cov_indir_call16)
-INTERFACE_FUNCTION(__sanitizer_cov_init)
-INTERFACE_FUNCTION(__sanitizer_cov_module_init)
-INTERFACE_FUNCTION(__sanitizer_cov_trace_basic_block)
-INTERFACE_FUNCTION(__sanitizer_cov_trace_func_enter)
-INTERFACE_FUNCTION(__sanitizer_cov_with_check)
-INTERFACE_FUNCTION(__sanitizer_get_allocated_size)
-INTERFACE_FUNCTION(__sanitizer_get_coverage_guards)
-INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes)
-INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size)
-INTERFACE_FUNCTION(__sanitizer_get_free_bytes)
-INTERFACE_FUNCTION(__sanitizer_get_heap_size)
-INTERFACE_FUNCTION(__sanitizer_get_ownership)
-INTERFACE_FUNCTION(__sanitizer_get_total_unique_caller_callee_pairs)
-INTERFACE_FUNCTION(__sanitizer_get_total_unique_coverage)
-INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes)
-INTERFACE_FUNCTION(__sanitizer_maybe_open_cov_file)
-INTERFACE_FUNCTION(__sanitizer_print_stack_trace)
-INTERFACE_FUNCTION(__sanitizer_symbolize_pc)
-INTERFACE_FUNCTION(__sanitizer_symbolize_global)
-INTERFACE_FUNCTION(__sanitizer_ptr_cmp)
-INTERFACE_FUNCTION(__sanitizer_ptr_sub)
-INTERFACE_FUNCTION(__sanitizer_report_error_summary)
-INTERFACE_FUNCTION(__sanitizer_reset_coverage)
-INTERFACE_FUNCTION(__sanitizer_get_number_of_counters)
-INTERFACE_FUNCTION(__sanitizer_update_counter_bitset_and_clear_counters)
-INTERFACE_FUNCTION(__sanitizer_sandbox_on_notify)
-INTERFACE_FUNCTION(__sanitizer_set_death_callback)
-INTERFACE_FUNCTION(__sanitizer_set_report_path)
-INTERFACE_FUNCTION(__sanitizer_set_report_fd)
-INTERFACE_FUNCTION(__sanitizer_unaligned_load16)
-INTERFACE_FUNCTION(__sanitizer_unaligned_load32)
-INTERFACE_FUNCTION(__sanitizer_unaligned_load64)
-INTERFACE_FUNCTION(__sanitizer_unaligned_store16)
-INTERFACE_FUNCTION(__sanitizer_unaligned_store32)
-INTERFACE_FUNCTION(__sanitizer_unaligned_store64)
-INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
-INTERFACE_FUNCTION(__sanitizer_install_malloc_and_free_hooks)
-INTERFACE_FUNCTION(__sanitizer_start_switch_fiber)
-INTERFACE_FUNCTION(__sanitizer_finish_switch_fiber)
-
-// TODO(timurrrr): Add more interface functions on the as-needed basis.
-
-// ----------------- Memory allocation functions ---------------------
-WRAP_V_W(free)
-WRAP_V_W(_free_base)
-WRAP_V_WW(_free_dbg)
-
-WRAP_W_W(malloc)
-WRAP_W_W(_malloc_base)
-WRAP_W_WWWW(_malloc_dbg)
-
-WRAP_W_WW(calloc)
-WRAP_W_WW(_calloc_base)
-WRAP_W_WWWWW(_calloc_dbg)
-WRAP_W_WWW(_calloc_impl)
-
-WRAP_W_WW(realloc)
-WRAP_W_WW(_realloc_base)
-WRAP_W_WWW(_realloc_dbg)
-WRAP_W_WWW(_recalloc)
-WRAP_W_WWW(_recalloc_base)
-
-WRAP_W_W(_msize)
-WRAP_W_W(_expand)
-WRAP_W_W(_expand_dbg)
+INTERCEPT_WRAP_W_W(_msize)
+INTERCEPT_WRAP_W_W(_expand)
+INTERCEPT_WRAP_W_W(_expand_dbg)
 
 // TODO(timurrrr): Might want to add support for _aligned_* allocation
 // functions to detect a bit more bugs.  Those functions seem to wrap malloc().
@@ -390,20 +56,6 @@ WRAP_W_W(_expand_dbg)
 
 INTERCEPT_LIBRARY_FUNCTION(atoi);
 INTERCEPT_LIBRARY_FUNCTION(atol);
-
-#ifdef _WIN64
-INTERCEPT_LIBRARY_FUNCTION(__C_specific_handler);
-#else
-INTERCEPT_LIBRARY_FUNCTION(_except_handler3);
-
-// _except_handler4 checks -GS cookie which is different for each module, so we
-// can't use INTERCEPT_LIBRARY_FUNCTION(_except_handler4).
-INTERCEPTOR(int, _except_handler4, void *a, void *b, void *c, void *d) {
-  __asan_handle_no_return();
-  return REAL(_except_handler4)(a, b, c, d);
-}
-#endif
-
 INTERCEPT_LIBRARY_FUNCTION(frexp);
 INTERCEPT_LIBRARY_FUNCTION(longjmp);
 #if SANITIZER_INTERCEPT_MEMCHR
@@ -428,30 +80,71 @@ INTERCEPT_LIBRARY_FUNCTION(strpbrk);
 INTERCEPT_LIBRARY_FUNCTION(strrchr);
 INTERCEPT_LIBRARY_FUNCTION(strspn);
 INTERCEPT_LIBRARY_FUNCTION(strstr);
+INTERCEPT_LIBRARY_FUNCTION(strtok);
 INTERCEPT_LIBRARY_FUNCTION(strtol);
 INTERCEPT_LIBRARY_FUNCTION(wcslen);
+INTERCEPT_LIBRARY_FUNCTION(wcsnlen);
+
+#ifdef _WIN64
+INTERCEPT_LIBRARY_FUNCTION(__C_specific_handler);
+#else
+INTERCEPT_LIBRARY_FUNCTION(_except_handler3);
+// _except_handler4 checks -GS cookie which is different for each module, so we
+// can't use INTERCEPT_LIBRARY_FUNCTION(_except_handler4).
+INTERCEPTOR(int, _except_handler4, void *a, void *b, void *c, void *d) {
+  __asan_handle_no_return();
+  return REAL(_except_handler4)(a, b, c, d);
+}
+#endif
+
+// Window specific functions not included in asan_interface.inc.
+INTERCEPT_WRAP_W_V(__asan_should_detect_stack_use_after_return)
+INTERCEPT_WRAP_W_V(__asan_get_shadow_memory_dynamic_address)
+INTERCEPT_WRAP_W_W(__asan_unhandled_exception_filter)
+
+using namespace __sanitizer;
+
+extern "C" {
+int __asan_option_detect_stack_use_after_return;
+uptr __asan_shadow_memory_dynamic_address;
+} // extern "C"
+
+static int asan_dll_thunk_init() {
+  typedef void (*fntype)();
+  static fntype fn = 0;
+  // asan_dll_thunk_init is expected to be called by only one thread.
+  if (fn) return 0;
+
+  // Ensure all interception was executed.
+  __dll_thunk_init();
+
+  fn = (fntype) dllThunkGetRealAddrOrDie("__asan_init");
+  fn();
+  __asan_option_detect_stack_use_after_return =
+      (__asan_should_detect_stack_use_after_return() != 0);
+  __asan_shadow_memory_dynamic_address =
+      (uptr)__asan_get_shadow_memory_dynamic_address();
 
-// Must be after all the interceptor declarations due to the way INTERCEPT_HOOKS
-// is defined.
-void InterceptHooks() {
-  INTERCEPT_HOOKS();
 #ifndef _WIN64
   INTERCEPT_FUNCTION(_except_handler4);
 #endif
+  // In DLLs, the callbacks are expected to return 0,
+  // otherwise CRT initialization fails.
+  return 0;
 }
 
-// We want to call __asan_init before C/C++ initializers/constructors are
-// executed, otherwise functions like memset might be invoked.
-// For some strange reason, merely linking in asan_preinit.cc doesn't work
-// as the callback is never called...  Is link.exe doing something too smart?
+#pragma section(".CRT$XIB", long, read)  // NOLINT
+__declspec(allocate(".CRT$XIB")) int (*__asan_preinit)() = asan_dll_thunk_init;
 
-// In DLLs, the callbacks are expected to return 0,
-// otherwise CRT initialization fails.
-static int call_asan_init() {
-  __asan_init();
-  return 0;
+static void WINAPI asan_thread_init(void *mod, unsigned long reason,
+                                    void *reserved) {
+  if (reason == /*DLL_PROCESS_ATTACH=*/1) asan_dll_thunk_init();
 }
-#pragma section(".CRT$XIB", long, read)  // NOLINT
-__declspec(allocate(".CRT$XIB")) int (*__asan_preinit)() = call_asan_init;
 
-#endif // ASAN_DLL_THUNK
+#pragma section(".CRT$XLAB", long, read)  // NOLINT
+__declspec(allocate(".CRT$XLAB")) void (WINAPI *__asan_tls_init)(void *,
+    unsigned long, void *) = asan_thread_init;
+
+WIN_FORCE_LINK(__asan_dso_reg_hook)
+
+#endif // SANITIZER_DLL_THUNK
diff --git a/libsanitizer/asan/asan_win_dynamic_runtime_thunk.cc b/libsanitizer/asan/asan_win_dynamic_runtime_thunk.cc
index 8989159cc52..d431b78d605 100644
--- a/libsanitizer/asan/asan_win_dynamic_runtime_thunk.cc
+++ b/libsanitizer/asan/asan_win_dynamic_runtime_thunk.cc
@@ -12,24 +12,31 @@
 // using the default "import library" generated when linking the DLL RTL.
 //
 // This includes:
+//  - creating weak aliases to default implementation imported from asan dll.
 //  - forwarding the detect_stack_use_after_return runtime option
 //  - working around deficiencies of the MD runtime
 //  - installing a custom SEH handler
 //
 //===----------------------------------------------------------------------===//
 
-// Only compile this code when building asan_dynamic_runtime_thunk.lib
-// Using #ifdef rather than relying on Makefiles etc.
-// simplifies the build procedure.
-#ifdef ASAN_DYNAMIC_RUNTIME_THUNK
+#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
+#define SANITIZER_IMPORT_INTERFACE 1
+#include "sanitizer_common/sanitizer_win_defs.h"
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
+// Define weak alias for all weak functions imported from asan dll.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
+#include "asan_interface.inc"
+
 // First, declare CRT sections we'll be using in this file
+#pragma section(".CRT$XIB", long, read)  // NOLINT
 #pragma section(".CRT$XID", long, read)  // NOLINT
 #pragma section(".CRT$XCAB", long, read)  // NOLINT
 #pragma section(".CRT$XTW", long, read)  // NOLINT
 #pragma section(".CRT$XTY", long, read)  // NOLINT
+#pragma section(".CRT$XLAB", long, read)  // NOLINT
 
 ////////////////////////////////////////////////////////////////////////////////
 // Define a copy of __asan_option_detect_stack_use_after_return that should be
@@ -44,14 +51,33 @@
 // after initialization anyways.
 extern "C" {
 __declspec(dllimport) int __asan_should_detect_stack_use_after_return();
-int __asan_option_detect_stack_use_after_return =
-    __asan_should_detect_stack_use_after_return();
+int __asan_option_detect_stack_use_after_return;
 
 __declspec(dllimport) void* __asan_get_shadow_memory_dynamic_address();
-void* __asan_shadow_memory_dynamic_address =
+void* __asan_shadow_memory_dynamic_address;
+}
+
+static int InitializeClonedVariables() {
+  __asan_option_detect_stack_use_after_return =
+    __asan_should_detect_stack_use_after_return();
+  __asan_shadow_memory_dynamic_address =
     __asan_get_shadow_memory_dynamic_address();
+  return 0;
 }
 
+static void NTAPI asan_thread_init(void *mod, unsigned long reason,
+    void *reserved) {
+  if (reason == DLL_PROCESS_ATTACH) InitializeClonedVariables();
+}
+
+// Our cloned variables must be initialized before C/C++ constructors.  If TLS
+// is used, our .CRT$XLAB initializer will run first. If not, our .CRT$XIB
+// initializer is needed as a backup.
+__declspec(allocate(".CRT$XIB")) int (*__asan_initialize_cloned_variables)() =
+    InitializeClonedVariables;
+__declspec(allocate(".CRT$XLAB")) void (NTAPI *__asan_tls_init)(void *,
+    unsigned long, void *) = asan_thread_init;
+
 ////////////////////////////////////////////////////////////////////////////////
 // For some reason, the MD CRT doesn't call the C/C++ terminators during on DLL
 // unload or on exit.  ASan relies on LLVM global_dtors to call
@@ -98,4 +124,6 @@ __declspec(allocate(".CRT$XCAB")) int (*__asan_seh_interceptor)() =
     SetSEHFilter;
 }
 
-#endif // ASAN_DYNAMIC_RUNTIME_THUNK
+WIN_FORCE_LINK(__asan_dso_reg_hook)
+
+#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
diff --git a/libsanitizer/asan/asan_win_weak_interception.cc b/libsanitizer/asan/asan_win_weak_interception.cc
new file mode 100644
index 00000000000..74c1dcdb729
--- /dev/null
+++ b/libsanitizer/asan/asan_win_weak_interception.cc
@@ -0,0 +1,21 @@
+//===-- asan_win_weak_interception.cc -------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This module should be included in Address Sanitizer when it is implemented as
+// a shared library on Windows (dll), in order to delegate the calls of weak
+// functions to the implementation in the main executable when a strong
+// definition is provided.
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DYNAMIC
+#include "sanitizer_common/sanitizer_win_weak_interception.h"
+#include "asan_interface_internal.h"
+// Check if strong definitions for weak functions are present in the main
+// executable. If that is the case, override dll functions to point to strong
+// implementations.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "asan_interface.inc"
+#endif // SANITIZER_DYNAMIC
diff --git a/libsanitizer/asan/libtool-version b/libsanitizer/asan/libtool-version
index 0f14ee34185..e3138f3b9f8 100644
--- a/libsanitizer/asan/libtool-version
+++ b/libsanitizer/asan/libtool-version
@@ -3,4 +3,4 @@
 # a separate file so that version updates don't involve re-running
 # automake.
 # CURRENT:REVISION:AGE
-4:0:0
+5:0:0
diff --git a/libsanitizer/builtins/assembly.h b/libsanitizer/builtins/assembly.h
index 5e36b5a5edf..3f5e59b2544 100644
--- a/libsanitizer/builtins/assembly.h
+++ b/libsanitizer/builtins/assembly.h
@@ -44,7 +44,8 @@
 #endif
 #define CONST_SECTION .section .rodata
 
-#if defined(__GNU__) || defined(__ANDROID__) || defined(__FreeBSD__)
+#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
+    defined(__linux__)
 #define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
 #else
 #define NO_EXEC_STACK_DIRECTIVE
@@ -67,10 +68,42 @@
 #endif
 
 #if defined(__arm__)
+
+/*
+ * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
+ * - for '-mthumb -march=armv6' compiler defines '__thumb__'
+ * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__'
+ */
+#if defined(__thumb2__) || defined(__thumb__)
+#define DEFINE_CODE_STATE .thumb SEPARATOR
+#define DECLARE_FUNC_ENCODING    .thumb_func SEPARATOR
+#if defined(__thumb2__)
+#define USE_THUMB_2
+#define IT(cond)  it cond
+#define ITT(cond) itt cond
+#define ITE(cond) ite cond
+#else
+#define USE_THUMB_1
+#define IT(cond)
+#define ITT(cond)
+#define ITE(cond)
+#endif // defined(__thumb__2)
+#else // !defined(__thumb2__) && !defined(__thumb__)
+#define DEFINE_CODE_STATE .arm SEPARATOR
+#define DECLARE_FUNC_ENCODING
+#define IT(cond)
+#define ITT(cond)
+#define ITE(cond)
+#endif
+
+#if defined(USE_THUMB_1) && defined(USE_THUMB_2)
+#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together."
+#endif
+
 #if defined(__ARM_ARCH_4T__) || __ARM_ARCH >= 5
 #define ARM_HAS_BX
 #endif
-#if !defined(__ARM_FEATURE_CLZ) &&                                             \
+#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) &&  \
     (__ARM_ARCH >= 6 || (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__)))
 #define __ARM_FEATURE_CLZ
 #endif
@@ -92,19 +125,14 @@
   JMP(ip)
 #endif
 
-#if __ARM_ARCH_ISA_THUMB == 2
-#define IT(cond)  it cond
-#define ITT(cond) itt cond
-#else
-#define IT(cond)
-#define ITT(cond)
-#endif
-
-#if __ARM_ARCH_ISA_THUMB == 2
+#if defined(USE_THUMB_2)
 #define WIDE(op) op.w
 #else
 #define WIDE(op) op
 #endif
+#else // !defined(__arm)
+#define DECLARE_FUNC_ENCODING
+#define DEFINE_CODE_STATE
 #endif
 
 #define GLUE2(a, b) a##b
@@ -119,13 +147,16 @@
 #endif
 
 #define DEFINE_COMPILERRT_FUNCTION(name)                                       \
+  DEFINE_CODE_STATE                                                            \
   FILE_LEVEL_DIRECTIVE SEPARATOR                                               \
   .globl SYMBOL_NAME(name) SEPARATOR                                           \
   SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR                                  \
   DECLARE_SYMBOL_VISIBILITY(name)                                              \
+  DECLARE_FUNC_ENCODING                                                        \
   SYMBOL_NAME(name):
 
 #define DEFINE_COMPILERRT_THUMB_FUNCTION(name)                                 \
+  DEFINE_CODE_STATE                                                            \
   FILE_LEVEL_DIRECTIVE SEPARATOR                                               \
   .globl SYMBOL_NAME(name) SEPARATOR                                           \
   SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR                                  \
@@ -134,16 +165,20 @@
   SYMBOL_NAME(name):
 
 #define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name)                               \
+  DEFINE_CODE_STATE                                                            \
   FILE_LEVEL_DIRECTIVE SEPARATOR                                               \
   .globl SYMBOL_NAME(name) SEPARATOR                                           \
   SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR                                  \
   HIDDEN(SYMBOL_NAME(name)) SEPARATOR                                          \
+  DECLARE_FUNC_ENCODING                                                        \
   SYMBOL_NAME(name):
 
 #define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name)                     \
+  DEFINE_CODE_STATE                                                            \
   .globl name SEPARATOR                                                        \
   SYMBOL_IS_FUNC(name) SEPARATOR                                               \
   HIDDEN(name) SEPARATOR                                                       \
+  DECLARE_FUNC_ENCODING                                                        \
   name:
 
 #define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target)                         \
diff --git a/libsanitizer/include/sanitizer/asan_interface.h b/libsanitizer/include/sanitizer/asan_interface.h
index 448a0bcba01..ad69ab46c92 100644
--- a/libsanitizer/include/sanitizer/asan_interface.h
+++ b/libsanitizer/include/sanitizer/asan_interface.h
@@ -142,6 +142,10 @@ extern "C" {
   void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg,
                                      void **end);
 
+  // Performs cleanup before a [[noreturn]] function.  Must be called
+  // before things like _exit and execl to avoid false positives on stack.
+  void __asan_handle_no_return(void);
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/libsanitizer/include/sanitizer/common_interface_defs.h b/libsanitizer/include/sanitizer/common_interface_defs.h
index fd38c55a235..a66c932b0ae 100644
--- a/libsanitizer/include/sanitizer/common_interface_defs.h
+++ b/libsanitizer/include/sanitizer/common_interface_defs.h
@@ -156,8 +156,10 @@ extern "C" {
   // Prints stack traces for all live heap allocations ordered by total
   // allocation size until `top_percent` of total live heap is shown.
   // `top_percent` should be between 1 and 100.
+  // At most `max_number_of_contexts` contexts (stack traces) is printed.
   // Experimental feature currently available only with asan on Linux/x86_64.
-  void __sanitizer_print_memory_profile(size_t top_percent);
+  void __sanitizer_print_memory_profile(size_t top_percent,
+                                        size_t max_number_of_contexts);
 
   // Fiber annotation interface.
   // Before switching to a different stack, one must call
@@ -180,6 +182,13 @@ extern "C" {
   void __sanitizer_finish_switch_fiber(void *fake_stack_save,
                                        const void **bottom_old,
                                        size_t *size_old);
+
+  // Get full module name and calculate pc offset within it.
+  // Returns 1 if pc belongs to some module, 0 if module was not found.
+  int __sanitizer_get_module_and_offset_for_pc(void *pc, char *module_path,
+                                               size_t module_path_len,
+                                               void **pc_offset);
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/libsanitizer/include/sanitizer/coverage_interface.h b/libsanitizer/include/sanitizer/coverage_interface.h
index ffb956c39db..85447b68a46 100644
--- a/libsanitizer/include/sanitizer/coverage_interface.h
+++ b/libsanitizer/include/sanitizer/coverage_interface.h
@@ -17,45 +17,15 @@
 extern "C" {
 #endif
 
-  // Initialize coverage.
-  void __sanitizer_cov_init();
   // Record and dump coverage info.
   void __sanitizer_cov_dump();
-  // Open <name>.sancov.packed in the coverage directory and return the file
-  // descriptor. Returns -1 on failure, or if coverage dumping is disabled.
-  // This is intended for use by sandboxing code.
-  intptr_t __sanitizer_maybe_open_cov_file(const char *name);
-  // Get the number of unique covered blocks (or edges).
-  // This can be useful for coverage-directed in-process fuzzers.
-  uintptr_t __sanitizer_get_total_unique_coverage();
-  // Get the number of unique indirect caller-callee pairs.
-  uintptr_t __sanitizer_get_total_unique_caller_callee_pairs();
 
-  // Reset the basic-block (edge) coverage to the initial state.
-  // Useful for in-process fuzzing to start collecting coverage from scratch.
-  // Experimental, will likely not work for multi-threaded process.
-  void __sanitizer_reset_coverage();
-  // Set *data to the array of covered PCs and return the size of that array.
-  // Some of the entries in *data will be zero.
-  uintptr_t __sanitizer_get_coverage_guards(uintptr_t **data);
+  // Clear collected coverage info.
+  void __sanitizer_cov_reset();
 
-  // The coverage instrumentation may optionally provide imprecise counters.
-  // Rather than exposing the counter values to the user we instead map
-  // the counters to a bitset.
-  // Every counter is associated with 8 bits in the bitset.
-  // We define 8 value ranges: 1, 2, 3, 4-7, 8-15, 16-31, 32-127, 128+
-  // The i-th bit is set to 1 if the counter value is in the i-th range.
-  // This counter-based coverage implementation is *not* thread-safe.
-
-  // Returns the number of registered coverage counters.
-  uintptr_t __sanitizer_get_number_of_counters();
-  // Updates the counter 'bitset', clears the counters and returns the number of
-  // new bits in 'bitset'.
-  // If 'bitset' is nullptr, only clears the counters.
-  // Otherwise 'bitset' should be at least
-  // __sanitizer_get_number_of_counters bytes long and 8-aligned.
-  uintptr_t
-  __sanitizer_update_counter_bitset_and_clear_counters(uint8_t *bitset);
+  // Dump collected coverage info. Sorts pcs by module into individual .sancov
+  // files.
+  void __sanitizer_dump_coverage(const uintptr_t *pcs, uintptr_t len);
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/libsanitizer/include/sanitizer/lsan_interface.h b/libsanitizer/include/sanitizer/lsan_interface.h
index bdbe39084f7..32051e62a00 100644
--- a/libsanitizer/include/sanitizer/lsan_interface.h
+++ b/libsanitizer/include/sanitizer/lsan_interface.h
@@ -62,8 +62,14 @@ extern "C" {
   // for the program it is linked into (if the return value is non-zero). This
   // function must be defined as returning a constant value; any behavior beyond
   // that is unsupported.
+  // To avoid dead stripping, you may need to define this function with
+  // __attribute__((used))
   int __lsan_is_turned_off();
 
+  // This function may be optionally provided by user and should return
+  // a string containing LSan runtime options. See lsan_flags.inc for details.
+  const char *__lsan_default_options();
+
   // This function may be optionally provided by the user and should return
   // a string containing LSan suppressions.
   const char *__lsan_default_suppressions();
diff --git a/libsanitizer/include/sanitizer/tsan_interface.h b/libsanitizer/include/sanitizer/tsan_interface.h
new file mode 100644
index 00000000000..9d9119262f8
--- /dev/null
+++ b/libsanitizer/include/sanitizer/tsan_interface.h
@@ -0,0 +1,136 @@
+//===-- tsan_interface.h ----------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Public interface header for TSan.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_TSAN_INTERFACE_H
+#define SANITIZER_TSAN_INTERFACE_H
+
+#include <sanitizer/common_interface_defs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// __tsan_release establishes a happens-before relation with a preceding
+// __tsan_acquire on the same address.
+void __tsan_acquire(void *addr);
+void __tsan_release(void *addr);
+
+// Annotations for custom mutexes.
+// The annotations allow to get better reports (with sets of locked mutexes),
+// detect more types of bugs (e.g. mutex misuses, races between lock/unlock and
+// destruction and potential deadlocks) and improve precision and performance
+// (by ignoring individual atomic operations in mutex code). However, the
+// downside is that annotated mutex code itself is not checked for correctness.
+
+// Mutex creation flags are passed to __tsan_mutex_create annotation.
+// If mutex has no constructor and __tsan_mutex_create is not called,
+// the flags may be passed to __tsan_mutex_pre_lock/__tsan_mutex_post_lock
+// annotations.
+
+// Mutex has static storage duration and no-op constructor and destructor.
+// This effectively makes tsan ignore destroy annotation.
+const unsigned __tsan_mutex_linker_init      = 1 << 0;
+// Mutex is write reentrant.
+const unsigned __tsan_mutex_write_reentrant  = 1 << 1;
+// Mutex is read reentrant.
+const unsigned __tsan_mutex_read_reentrant   = 1 << 2;
+
+// Mutex operation flags:
+
+// Denotes read lock operation.
+const unsigned __tsan_mutex_read_lock        = 1 << 3;
+// Denotes try lock operation.
+const unsigned __tsan_mutex_try_lock         = 1 << 4;
+// Denotes that a try lock operation has failed to acquire the mutex.
+const unsigned __tsan_mutex_try_lock_failed  = 1 << 5;
+// Denotes that the lock operation acquires multiple recursion levels.
+// Number of levels is passed in recursion parameter.
+// This is useful for annotation of e.g. Java builtin monitors,
+// for which wait operation releases all recursive acquisitions of the mutex.
+const unsigned __tsan_mutex_recursive_lock   = 1 << 6;
+// Denotes that the unlock operation releases all recursion levels.
+// Number of released levels is returned and later must be passed to
+// the corresponding __tsan_mutex_post_lock annotation.
+const unsigned __tsan_mutex_recursive_unlock = 1 << 7;
+
+// Annotate creation of a mutex.
+// Supported flags: mutex creation flags.
+void __tsan_mutex_create(void *addr, unsigned flags);
+
+// Annotate destruction of a mutex.
+// Supported flags:
+//   - __tsan_mutex_linker_init
+void __tsan_mutex_destroy(void *addr, unsigned flags);
+
+// Annotate start of lock operation.
+// Supported flags:
+//   - __tsan_mutex_read_lock
+//   - __tsan_mutex_try_lock
+//   - all mutex creation flags
+void __tsan_mutex_pre_lock(void *addr, unsigned flags);
+
+// Annotate end of lock operation.
+// Supported flags:
+//   - __tsan_mutex_read_lock (must match __tsan_mutex_pre_lock)
+//   - __tsan_mutex_try_lock (must match __tsan_mutex_pre_lock)
+//   - __tsan_mutex_try_lock_failed
+//   - __tsan_mutex_recursive_lock
+//   - all mutex creation flags
+void __tsan_mutex_post_lock(void *addr, unsigned flags, int recursion);
+
+// Annotate start of unlock operation.
+// Supported flags:
+//   - __tsan_mutex_read_lock
+//   - __tsan_mutex_recursive_unlock
+int __tsan_mutex_pre_unlock(void *addr, unsigned flags);
+
+// Annotate end of unlock operation.
+// Supported flags:
+//   - __tsan_mutex_read_lock (must match __tsan_mutex_pre_unlock)
+void __tsan_mutex_post_unlock(void *addr, unsigned flags);
+
+// Annotate start/end of notify/signal/broadcast operation.
+// Supported flags: none.
+void __tsan_mutex_pre_signal(void *addr, unsigned flags);
+void __tsan_mutex_post_signal(void *addr, unsigned flags);
+
+// Annotate start/end of a region of code where lock/unlock/signal operation
+// diverts to do something else unrelated to the mutex. This can be used to
+// annotate, for example, calls into cooperative scheduler or contention
+// profiling code.
+// These annotations must be called only from within
+// __tsan_mutex_pre/post_lock, __tsan_mutex_pre/post_unlock,
+// __tsan_mutex_pre/post_signal regions.
+// Supported flags: none.
+void __tsan_mutex_pre_divert(void *addr, unsigned flags);
+void __tsan_mutex_post_divert(void *addr, unsigned flags);
+
+// External race detection API.
+// Can be used by non-instrumented libraries to detect when their objects are
+// being used in an unsafe manner.
+//   - __tsan_external_read/__tsan_external_write annotates the logical reads
+//       and writes of the object at the specified address. 'caller_pc' should
+//       be the PC of the library user, which the library can obtain with e.g.
+//       `__builtin_return_address(0)`.
+//   - __tsan_external_register_tag registers a 'tag' with the specified name,
+//       which is later used in read/write annotations to denote the object type
+//   - __tsan_external_assign_tag can optionally mark a heap object with a tag
+void *__tsan_external_register_tag(const char *object_type);
+void __tsan_external_register_header(void *tag, const char *header);
+void __tsan_external_assign_tag(void *addr, void *tag);
+void __tsan_external_read(void *addr, void *caller_pc, void *tag);
+void __tsan_external_write(void *addr, void *caller_pc, void *tag);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // SANITIZER_TSAN_INTERFACE_H
diff --git a/libsanitizer/interception/interception.h b/libsanitizer/interception/interception.h
index 0db36ddf28e..75631da55ee 100644
--- a/libsanitizer/interception/interception.h
+++ b/libsanitizer/interception/interception.h
@@ -13,8 +13,8 @@
 #ifndef INTERCEPTION_H
 #define INTERCEPTION_H
 
-#if !defined(__linux__) && !defined(__FreeBSD__) && \
-  !defined(__APPLE__) && !defined(_WIN32)
+#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__APPLE__) && \
+    !defined(__NetBSD__) && !defined(_WIN32) && !defined(__Fuchsia__)
 # error "Interception doesn't work on this operating system."
 #endif
 
@@ -127,7 +127,7 @@ const interpose_substitution substitution_##func_name[] \
     extern "C" ret_type func(__VA_ARGS__);
 # define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
     extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
-#elif defined(__FreeBSD__)
+#elif defined(__FreeBSD__) || defined(__NetBSD__)
 # define WRAP(x) __interceptor_ ## x
 # define WRAPPER_NAME(x) "__interceptor_" #x
 # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
@@ -137,7 +137,7 @@ const interpose_substitution substitution_##func_name[] \
 # define DECLARE_WRAPPER(ret_type, func, ...) \
      extern "C" ret_type func(__VA_ARGS__) \
      __attribute__((alias("__interceptor_" #func), visibility("default")));
-#else
+#elif !defined(__Fuchsia__)
 # define WRAP(x) __interceptor_ ## x
 # define WRAPPER_NAME(x) "__interceptor_" #x
 # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
@@ -146,7 +146,15 @@ const interpose_substitution substitution_##func_name[] \
     __attribute__((weak, alias("__interceptor_" #func), visibility("default")));
 #endif
 
-#if !defined(__APPLE__)
+#if defined(__Fuchsia__)
+// There is no general interception at all on Fuchsia.
+// Sanitizer runtimes just define functions directly to preempt them,
+// and have bespoke ways to access the underlying libc functions.
+# include <zircon/sanitizer.h>
+# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+# define REAL(x) __unsanitized_##x
+# define DECLARE_REAL(ret_type, func, ...)
+#elif !defined(__APPLE__)
 # define PTR_TO_REAL(x) real_##x
 # define REAL(x) __interception::PTR_TO_REAL(x)
 # define FUNC_TYPE(x) x##_f
@@ -164,15 +172,19 @@ const interpose_substitution substitution_##func_name[] \
 # define ASSIGN_REAL(x, y)
 #endif  // __APPLE__
 
+#if !defined(__Fuchsia__)
 #define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
   DECLARE_REAL(ret_type, func, __VA_ARGS__) \
   extern "C" ret_type WRAP(func)(__VA_ARGS__);
+#else
+#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
+#endif
 
 // Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR
 // macros does its job. In exceptional cases you may need to call REAL(foo)
 // without defining INTERCEPTOR(..., foo, ...). For example, if you override
 // foo with an interceptor for other function.
-#if !defined(__APPLE__)
+#if !defined(__APPLE__) && !defined(__Fuchsia__)
 # define DEFINE_REAL(ret_type, func, ...) \
     typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
     namespace __interception { \
@@ -182,7 +194,18 @@ const interpose_substitution substitution_##func_name[] \
 # define DEFINE_REAL(ret_type, func, ...)
 #endif
 
-#if !defined(__APPLE__)
+#if defined(__Fuchsia__)
+
+// We need to define the __interceptor_func name just to get
+// sanitizer_common/scripts/gen_dynamic_list.py to export func.
+// But we don't need to export __interceptor_func to get that.
+#define INTERCEPTOR(ret_type, func, ...)                                \
+  extern "C"[[ gnu::alias(#func), gnu::visibility("hidden") ]] ret_type \
+      __interceptor_##func(__VA_ARGS__);                                \
+  extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__)
+
+#elif !defined(__APPLE__)
+
 #define INTERCEPTOR(ret_type, func, ...) \
   DEFINE_REAL(ret_type, func, __VA_ARGS__) \
   DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
@@ -239,7 +262,7 @@ typedef unsigned long uptr;  // NOLINT
 
 #define INCLUDED_FROM_INTERCEPTION_LIB
 
-#if defined(__linux__) || defined(__FreeBSD__)
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__NetBSD__)
 # include "interception_linux.h"
 # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
 # define INTERCEPT_FUNCTION_VER(func, symver) \
@@ -249,7 +272,7 @@ typedef unsigned long uptr;  // NOLINT
 # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func)
 # define INTERCEPT_FUNCTION_VER(func, symver) \
     INTERCEPT_FUNCTION_VER_MAC(func, symver)
-#else  // defined(_WIN32)
+#elif defined(_WIN32)
 # include "interception_win.h"
 # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_WIN(func)
 # define INTERCEPT_FUNCTION_VER(func, symver) \
diff --git a/libsanitizer/interception/interception_linux.cc b/libsanitizer/interception/interception_linux.cc
index 0a8305b0bf8..888b2ceac13 100644
--- a/libsanitizer/interception/interception_linux.cc
+++ b/libsanitizer/interception/interception_linux.cc
@@ -10,14 +10,22 @@
 // Linux-specific interception methods.
 //===----------------------------------------------------------------------===//
 
-#if defined(__linux__) || defined(__FreeBSD__)
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__NetBSD__)
 #include "interception.h"
 
 #include <dlfcn.h>   // for dlsym() and dlvsym()
 
+#ifdef __NetBSD__
+#include "sanitizer_common/sanitizer_libc.h"
+#endif
+
 namespace __interception {
 bool GetRealFunctionAddress(const char *func_name, uptr *func_addr,
     uptr real, uptr wrapper) {
+#ifdef __NetBSD__
+  // XXX: Find a better way to handle renames
+  if (internal_strcmp(func_name, "sigaction") == 0) func_name = "__sigaction14";
+#endif
   *func_addr = (uptr)dlsym(RTLD_NEXT, func_name);
   return real == wrapper;
 }
@@ -30,5 +38,4 @@ void *GetFuncAddrVer(const char *func_name, const char *ver) {
 
 }  // namespace __interception
 
-
-#endif  // __linux__ || __FreeBSD__
+#endif  // __linux__ || __FreeBSD__ || __NetBSD__
diff --git a/libsanitizer/interception/interception_linux.h b/libsanitizer/interception/interception_linux.h
index 61bf48a7233..f5965180888 100644
--- a/libsanitizer/interception/interception_linux.h
+++ b/libsanitizer/interception/interception_linux.h
@@ -10,7 +10,7 @@
 // Linux-specific interception methods.
 //===----------------------------------------------------------------------===//
 
-#if defined(__linux__) || defined(__FreeBSD__)
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__NetBSD__)
 
 #if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
 # error "interception_linux.h should be included from interception library only"
@@ -42,4 +42,4 @@ void *GetFuncAddrVer(const char *func_name, const char *ver);
 #endif  // !defined(__ANDROID__)
 
 #endif  // INTERCEPTION_LINUX_H
-#endif  // __linux__ || __FreeBSD__
+#endif  // __linux__ || __FreeBSD__ || __NetBSD__
diff --git a/libsanitizer/interception/interception_win.cc b/libsanitizer/interception/interception_win.cc
index fa81162097e..1957397bdad 100644
--- a/libsanitizer/interception/interception_win.cc
+++ b/libsanitizer/interception/interception_win.cc
@@ -146,10 +146,16 @@ static void InterceptionFailed() {
 }
 
 static bool DistanceIsWithin2Gig(uptr from, uptr target) {
+#if SANITIZER_WINDOWS64
   if (from < target)
     return target - from <= (uptr)0x7FFFFFFFU;
   else
     return from - target <= (uptr)0x80000000U;
+#else
+  // In a 32-bit address space, the address calculation will wrap, so this check
+  // is unnecessary.
+  return true;
+#endif
 }
 
 static uptr GetMmapGranularity() {
@@ -215,9 +221,8 @@ static bool IsMemoryPadding(uptr address, uptr size) {
   return true;
 }
 
-static const u8 kHintNop10Bytes[] = {
-  0x66, 0x66, 0x0F, 0x1F, 0x84,
-  0x00, 0x00, 0x00, 0x00, 0x00
+static const u8 kHintNop9Bytes[] = {
+  0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
 };
 
 template<class T>
@@ -232,8 +237,8 @@ static bool FunctionHasPrefix(uptr address, const T &pattern) {
 static bool FunctionHasPadding(uptr address, uptr size) {
   if (IsMemoryPadding(address - size, size))
     return true;
-  if (size <= sizeof(kHintNop10Bytes) &&
-      FunctionHasPrefix(address, kHintNop10Bytes))
+  if (size <= sizeof(kHintNop9Bytes) &&
+      FunctionHasPrefix(address, kHintNop9Bytes))
     return true;
   return false;
 }
@@ -469,7 +474,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
   switch (*(u8*)address) {
     case 0xA1:  // A1 XX XX XX XX XX XX XX XX :
                 //   movabs eax, dword ptr ds:[XXXXXXXX]
-      return 8;
+      return 9;
   }
 
   switch (*(u16*)address) {
@@ -487,6 +492,11 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x5741:  // push r15
     case 0x9066:  // Two-byte NOP
       return 2;
+
+    case 0x058B:  // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
+      if (rel_offset)
+        *rel_offset = 2;
+      return 6;
   }
 
   switch (0x00FFFFFF & *(u32*)address) {
@@ -870,6 +880,8 @@ uptr InternalGetProcAddress(void *module, const char *func_name) {
 
   IMAGE_DATA_DIRECTORY *export_directory =
       &headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT];
+  if (export_directory->Size == 0)
+    return 0;
   RVAPtr<IMAGE_EXPORT_DIRECTORY> exports(module,
                                          export_directory->VirtualAddress);
   RVAPtr<DWORD> functions(module, exports->AddressOfFunctions);
diff --git a/libsanitizer/lsan/Makefile.am b/libsanitizer/lsan/Makefile.am
index 294342b1926..700d15fce57 100644
--- a/libsanitizer/lsan/Makefile.am
+++ b/libsanitizer/lsan/Makefile.am
@@ -12,18 +12,22 @@ ACLOCAL_AMFLAGS = -I m4
 noinst_LTLIBRARIES = libsanitizer_lsan.la
 if LSAN_SUPPORTED
 toolexeclib_LTLIBRARIES = liblsan.la
+nodist_toolexeclib_HEADERS = liblsan_preinit.o
 endif
 
 sanitizer_lsan_files = \
 	lsan_common.cc \
-	lsan_common_linux.cc
+	lsan_common_linux.cc \
+	lsan_common_mac.cc
 
 lsan_files = \
 	$(sanitizer_lsan_files) \
 	lsan.cc \
+	lsan_linux.cc \
+	lsan_mac.cc \
+	lsan_malloc_mac.cc \
 	lsan_allocator.cc \
 	lsan_interceptors.cc \
-	lsan_preinit.cc \
 	lsan_thread.cc
 
 libsanitizer_lsan_la_SOURCES = $(sanitizer_lsan_files)
@@ -36,6 +40,9 @@ endif
 liblsan_la_LIBADD += $(LIBSTDCXX_RAW_CXX_LDFLAGS)
 liblsan_la_LDFLAGS = -version-info `grep -v '^\#' $(srcdir)/libtool-version` $(link_liblsan)
 
+liblsan_preinit.o: lsan_preinit.o
+	cp $< $@
+
 # Work around what appears to be a GNU make bug handling MAKEFLAGS
 # values defined in terms of make variables, as is the case for CC and
 # friends when we are called from the top level Makefile.
diff --git a/libsanitizer/lsan/Makefile.in b/libsanitizer/lsan/Makefile.in
index d951741c300..85502500728 100644
--- a/libsanitizer/lsan/Makefile.in
+++ b/libsanitizer/lsan/Makefile.in
@@ -15,6 +15,7 @@
 
 @SET_MAKE@
 
+
 VPATH = @srcdir@
 am__make_dryrun = \
   { \
@@ -100,16 +101,18 @@ am__uninstall_files_from_dir = { \
     || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
          $(am__cd) "$$dir" && rm -f $$files; }; \
   }
-am__installdirs = "$(DESTDIR)$(toolexeclibdir)"
+am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \
+	"$(DESTDIR)$(toolexeclibdir)"
 LTLIBRARIES = $(noinst_LTLIBRARIES) $(toolexeclib_LTLIBRARIES)
 am__DEPENDENCIES_1 =
 liblsan_la_DEPENDENCIES =  \
 	$(top_builddir)/sanitizer_common/libsanitizer_common.la \
 	$(top_builddir)/interception/libinterception.la \
 	$(am__append_1) $(am__DEPENDENCIES_1)
-am__objects_1 = lsan_common.lo lsan_common_linux.lo
-am__objects_2 = $(am__objects_1) lsan.lo lsan_allocator.lo \
-	lsan_interceptors.lo lsan_preinit.lo lsan_thread.lo
+am__objects_1 = lsan_common.lo lsan_common_linux.lo lsan_common_mac.lo
+am__objects_2 = $(am__objects_1) lsan.lo lsan_linux.lo lsan_mac.lo \
+	lsan_malloc_mac.lo lsan_allocator.lo lsan_interceptors.lo \
+	lsan_thread.lo
 am_liblsan_la_OBJECTS = $(am__objects_2)
 liblsan_la_OBJECTS = $(am_liblsan_la_OBJECTS)
 liblsan_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \
@@ -138,6 +141,7 @@ am__can_run_installinfo = \
     n|no|NO) false;; \
     *) (install-info --version) >/dev/null 2>&1;; \
   esac
+HEADERS = $(nodist_toolexeclib_HEADERS)
 ETAGS = etags
 CTAGS = ctags
 ACLOCAL = @ACLOCAL@
@@ -297,16 +301,20 @@ AM_CXXFLAGS = -Wall -W -Wno-unused-parameter -Wwrite-strings -pedantic \
 ACLOCAL_AMFLAGS = -I m4
 noinst_LTLIBRARIES = libsanitizer_lsan.la
 @LSAN_SUPPORTED_TRUE@toolexeclib_LTLIBRARIES = liblsan.la
+@LSAN_SUPPORTED_TRUE@nodist_toolexeclib_HEADERS = liblsan_preinit.o
 sanitizer_lsan_files = \
 	lsan_common.cc \
-	lsan_common_linux.cc
+	lsan_common_linux.cc \
+	lsan_common_mac.cc
 
 lsan_files = \
 	$(sanitizer_lsan_files) \
 	lsan.cc \
+	lsan_linux.cc \
+	lsan_mac.cc \
+	lsan_malloc_mac.cc \
 	lsan_allocator.cc \
 	lsan_interceptors.cc \
-	lsan_preinit.cc \
 	lsan_thread.cc
 
 libsanitizer_lsan_la_SOURCES = $(sanitizer_lsan_files)
@@ -446,8 +454,11 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_allocator.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_common.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_common_linux.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_common_mac.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_interceptors.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_preinit.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_linux.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_mac.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_malloc_mac.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lsan_thread.Plo@am__quote@
 
 .cc.o:
@@ -476,6 +487,27 @@ mostlyclean-libtool:
 
 clean-libtool:
 	-rm -rf .libs _libs
+install-nodist_toolexeclibHEADERS: $(nodist_toolexeclib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(nodist_toolexeclib_HEADERS)'; test -n "$(toolexeclibdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(toolexeclibdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(toolexeclibdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(toolexeclibdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(toolexeclibdir)" || exit $$?; \
+	done
+
+uninstall-nodist_toolexeclibHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(nodist_toolexeclib_HEADERS)'; test -n "$(toolexeclibdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(toolexeclibdir)'; $(am__uninstall_files_from_dir)
 
 ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
 	list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
@@ -530,9 +562,9 @@ distclean-tags:
 	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
 check-am: all-am
 check: check-am
-all-am: Makefile $(LTLIBRARIES)
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
 installdirs:
-	for dir in "$(DESTDIR)$(toolexeclibdir)"; do \
+	for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)"; do \
 	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
 	done
 install: install-am
@@ -594,7 +626,8 @@ install-dvi: install-dvi-am
 
 install-dvi-am:
 
-install-exec-am: install-toolexeclibLTLIBRARIES
+install-exec-am: install-nodist_toolexeclibHEADERS \
+	install-toolexeclibLTLIBRARIES
 
 install-html: install-html-am
 
@@ -634,7 +667,8 @@ ps: ps-am
 
 ps-am:
 
-uninstall-am: uninstall-toolexeclibLTLIBRARIES
+uninstall-am: uninstall-nodist_toolexeclibHEADERS \
+	uninstall-toolexeclibLTLIBRARIES
 
 .MAKE: install-am install-strip
 
@@ -645,14 +679,19 @@ uninstall-am: uninstall-toolexeclibLTLIBRARIES
 	html html-am info info-am install install-am install-data \
 	install-data-am install-dvi install-dvi-am install-exec \
 	install-exec-am install-html install-html-am install-info \
-	install-info-am install-man install-pdf install-pdf-am \
-	install-ps install-ps-am install-strip \
-	install-toolexeclibLTLIBRARIES installcheck installcheck-am \
-	installdirs maintainer-clean maintainer-clean-generic \
-	mostlyclean mostlyclean-compile mostlyclean-generic \
-	mostlyclean-libtool pdf pdf-am ps ps-am tags uninstall \
-	uninstall-am uninstall-toolexeclibLTLIBRARIES
-
+	install-info-am install-man install-nodist_toolexeclibHEADERS \
+	install-pdf install-pdf-am install-ps install-ps-am \
+	install-strip install-toolexeclibLTLIBRARIES installcheck \
+	installcheck-am installdirs maintainer-clean \
+	maintainer-clean-generic mostlyclean mostlyclean-compile \
+	mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+	tags uninstall uninstall-am \
+	uninstall-nodist_toolexeclibHEADERS \
+	uninstall-toolexeclibLTLIBRARIES
+
+
+liblsan_preinit.o: lsan_preinit.o
+	cp $< $@
 
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.
diff --git a/libsanitizer/lsan/lsan.cc b/libsanitizer/lsan/lsan.cc
index 2ded5544c71..7540aeb327b 100644
--- a/libsanitizer/lsan/lsan.cc
+++ b/libsanitizer/lsan/lsan.cc
@@ -54,6 +54,9 @@ static void InitializeFlags() {
   RegisterLsanFlags(&parser, f);
   RegisterCommonFlags(&parser);
 
+  // Override from user-specified string.
+  const char *lsan_default_options = MaybeCallLsanDefaultOptions();
+  parser.ParseString(lsan_default_options);
   parser.ParseString(GetEnv("LSAN_OPTIONS"));
 
   SetVerbosity(common_flags()->verbosity);
@@ -63,6 +66,18 @@ static void InitializeFlags() {
   if (common_flags()->help) parser.PrintFlagDescriptions();
 }
 
+static void OnStackUnwind(const SignalContext &sig, const void *,
+                          BufferedStackTrace *stack) {
+  GetStackTraceWithPcBpAndContext(stack, kStackTraceMax, sig.pc, sig.bp,
+                                  sig.context,
+                                  common_flags()->fast_unwind_on_fatal);
+}
+
+void LsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+  HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind,
+                     nullptr);
+}
+
 extern "C" void __lsan_init() {
   CHECK(!lsan_init_is_running);
   if (lsan_inited)
@@ -74,9 +89,11 @@ extern "C" void __lsan_init() {
   InitializeFlags();
   InitCommonLsan();
   InitializeAllocator();
+  ReplaceSystemMalloc();
   InitTlsSize();
   InitializeInterceptors();
   InitializeThreadRegistry();
+  InstallDeadlySignalHandlers(LsanOnDeadlySignal);
   u32 tid = ThreadCreate(0, 0, true);
   CHECK_EQ(tid, 0);
   ThreadStart(tid, GetTid());
diff --git a/libsanitizer/lsan/lsan.h b/libsanitizer/lsan/lsan.h
index 6d2d427b27d..42d4214ed0f 100644
--- a/libsanitizer/lsan/lsan.h
+++ b/libsanitizer/lsan/lsan.h
@@ -10,24 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "lsan_thread.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 
-#define GET_STACK_TRACE(max_size, fast)                                        \
-  BufferedStackTrace stack;                                                    \
-  {                                                                            \
-    uptr stack_top = 0, stack_bottom = 0;                                      \
-    ThreadContext *t;                                                          \
-    if (fast && (t = CurrentThreadContext())) {                                \
-      stack_top = t->stack_end();                                              \
-      stack_bottom = t->stack_begin();                                         \
-    }                                                                          \
-    if (!SANITIZER_MIPS ||                                                     \
-        IsValidFrame(GET_CURRENT_FRAME(), stack_top, stack_bottom)) {          \
-      stack.Unwind(max_size, StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),  \
-                   /* context */ 0, stack_top, stack_bottom, fast);            \
-    }                                                                          \
-  }
+#define GET_STACK_TRACE(max_size, fast)                       \
+  __sanitizer::BufferedStackTrace stack;                      \
+  GetStackTraceWithPcBpAndContext(&stack, max_size,           \
+                                  StackTrace::GetCurrentPc(), \
+                                  GET_CURRENT_FRAME(), nullptr, fast);
 
 #define GET_STACK_TRACE_FATAL \
   GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
@@ -36,13 +27,41 @@
   GET_STACK_TRACE(__sanitizer::common_flags()->malloc_context_size, \
                   common_flags()->fast_unwind_on_malloc)
 
+#define GET_STACK_TRACE_THREAD GET_STACK_TRACE(kStackTraceMax, true)
+
 namespace __lsan {
 
 void InitializeInterceptors();
+void ReplaceSystemMalloc();
+
+#define ENSURE_LSAN_INITED do {   \
+  CHECK(!lsan_init_is_running);   \
+  if (!lsan_inited)               \
+    __lsan_init();                \
+} while (0)
+
+// Get the stack trace with the given pc and bp.
+// The pc will be in the position 0 of the resulting stack trace.
+// The bp may refer to the current frame or to the caller's frame.
+ALWAYS_INLINE
+void GetStackTraceWithPcBpAndContext(__sanitizer::BufferedStackTrace *stack,
+                                     __sanitizer::uptr max_depth,
+                                     __sanitizer::uptr pc, __sanitizer::uptr bp,
+                                     void *context, bool fast) {
+  uptr stack_top = 0, stack_bottom = 0;
+  ThreadContext *t;
+  if (fast && (t = CurrentThreadContext())) {
+    stack_top = t->stack_end();
+    stack_bottom = t->stack_begin();
+  }
+  if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) {
+    stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast);
+  }
+}
 
 }  // namespace __lsan
 
 extern bool lsan_inited;
 extern bool lsan_init_is_running;
 
-extern "C" void __lsan_init();
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __lsan_init();
diff --git a/libsanitizer/lsan/lsan_allocator.cc b/libsanitizer/lsan/lsan_allocator.cc
index 0d2fceac71b..9e166807791 100644
--- a/libsanitizer/lsan/lsan_allocator.cc
+++ b/libsanitizer/lsan/lsan_allocator.cc
@@ -13,7 +13,9 @@
 #include "lsan_allocator.h"
 
 #include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
@@ -22,51 +24,27 @@
 extern "C" void *memset(void *ptr, int value, uptr num);
 
 namespace __lsan {
-
-struct ChunkMetadata {
-  u8 allocated : 8;  // Must be first.
-  ChunkTag tag : 2;
-  uptr requested_size : 54;
-  u32 stack_trace_id;
-};
-
-#if defined(__mips64) || defined(__aarch64__)
+#if defined(__i386__) || defined(__arm__)
+static const uptr kMaxAllowedMallocSize = 1UL << 30;
+#elif defined(__mips64) || defined(__aarch64__)
 static const uptr kMaxAllowedMallocSize = 4UL << 30;
-static const uptr kRegionSizeLog = 20;
-static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog;
-typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
-typedef CompactSizeClassMap SizeClassMap;
-typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE,
-    sizeof(ChunkMetadata), SizeClassMap, kRegionSizeLog, ByteMap>
-    PrimaryAllocator;
 #else
 static const uptr kMaxAllowedMallocSize = 8UL << 30;
-
-struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
-  static const uptr kSpaceBeg = 0x600000000000ULL;
-  static const uptr kSpaceSize =  0x40000000000ULL; // 4T.
-  static const uptr kMetadataSize = sizeof(ChunkMetadata);
-  typedef DefaultSizeClassMap SizeClassMap;
-  typedef NoOpMapUnmapCallback MapUnmapCallback;
-  static const uptr kFlags = 0;
-};
-
-typedef SizeClassAllocator64<AP64> PrimaryAllocator;
 #endif
-typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
 typedef LargeMmapAllocator<> SecondaryAllocator;
 typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
           SecondaryAllocator> Allocator;
 
 static Allocator allocator;
-static THREADLOCAL AllocatorCache cache;
 
 void InitializeAllocator() {
-  allocator.InitLinkerInitialized(common_flags()->allocator_may_return_null);
+  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+  allocator.InitLinkerInitialized(
+      common_flags()->allocator_release_to_os_interval_ms);
 }
 
 void AllocatorThreadFinish() {
-  allocator.SwallowCache(&cache);
+  allocator.SwallowCache(GetAllocatorCache());
 }
 
 static ChunkMetadata *Metadata(const void *p) {
@@ -96,9 +74,9 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
     size = 1;
   if (size > kMaxAllowedMallocSize) {
     Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", size);
-    return nullptr;
+    return Allocator::FailureHandler::OnBadRequest();
   }
-  void *p = allocator.Allocate(&cache, size, alignment, false);
+  void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
   // Do not rely on the allocator to clear the memory (it's slow).
   if (cleared && allocator.FromPrimary(p))
     memset(p, 0, size);
@@ -108,11 +86,18 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
   return p;
 }
 
+static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) {
+  if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
+    return Allocator::FailureHandler::OnBadRequest();
+  size *= nmemb;
+  return Allocate(stack, size, 1, true);
+}
+
 void Deallocate(void *p) {
   if (&__sanitizer_free_hook) __sanitizer_free_hook(p);
   RunFreeHooks(p);
   RegisterDeallocation(p);
-  allocator.Deallocate(&cache, p);
+  allocator.Deallocate(GetAllocatorCache(), p);
 }
 
 void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
@@ -120,17 +105,17 @@ void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
   RegisterDeallocation(p);
   if (new_size > kMaxAllowedMallocSize) {
     Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", new_size);
-    allocator.Deallocate(&cache, p);
-    return nullptr;
+    allocator.Deallocate(GetAllocatorCache(), p);
+    return Allocator::FailureHandler::OnBadRequest();
   }
-  p = allocator.Reallocate(&cache, p, new_size, alignment);
+  p = allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
   RegisterAllocation(stack, p, new_size);
   return p;
 }
 
 void GetAllocatorCacheRange(uptr *begin, uptr *end) {
-  *begin = (uptr)&cache;
-  *end = *begin + sizeof(cache);
+  *begin = (uptr)GetAllocatorCache();
+  *end = *begin + sizeof(AllocatorCache);
 }
 
 uptr GetMallocUsableSize(const void *p) {
@@ -139,6 +124,39 @@ uptr GetMallocUsableSize(const void *p) {
   return m->requested_size;
 }
 
+void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory));
+}
+
+void *lsan_malloc(uptr size, const StackTrace &stack) {
+  return SetErrnoOnNull(Allocate(stack, size, 1, kAlwaysClearMemory));
+}
+
+void lsan_free(void *p) {
+  Deallocate(p);
+}
+
+void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
+  return SetErrnoOnNull(Reallocate(stack, p, size, 1));
+}
+
+void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) {
+  return SetErrnoOnNull(Calloc(nmemb, size, stack));
+}
+
+void *lsan_valloc(uptr size, const StackTrace &stack) {
+  return SetErrnoOnNull(
+      Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory));
+}
+
+uptr lsan_mz_size(const void *p) {
+  return GetMallocUsableSize(p);
+}
+
 ///// Interface to the common LSan module. /////
 
 void LockAllocator() {
@@ -254,4 +272,17 @@ SANITIZER_INTERFACE_ATTRIBUTE
 uptr __sanitizer_get_allocated_size(const void *p) {
   return GetMallocUsableSize(p);
 }
+
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+// Provide default (no-op) implementation of malloc hooks.
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void __sanitizer_malloc_hook(void *ptr, uptr size) {
+  (void)ptr;
+  (void)size;
+}
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void __sanitizer_free_hook(void *ptr) {
+  (void)ptr;
+}
+#endif
 } // extern "C"
diff --git a/libsanitizer/lsan/lsan_allocator.h b/libsanitizer/lsan/lsan_allocator.h
index aae0d28dc4a..b0c0ec241d9 100644
--- a/libsanitizer/lsan/lsan_allocator.h
+++ b/libsanitizer/lsan/lsan_allocator.h
@@ -13,8 +13,10 @@
 #ifndef LSAN_ALLOCATOR_H
 #define LSAN_ALLOCATOR_H
 
+#include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
+#include "lsan_common.h"
 
 namespace __lsan {
 
@@ -32,6 +34,61 @@ void GetAllocatorCacheRange(uptr *begin, uptr *end);
 void AllocatorThreadFinish();
 void InitializeAllocator();
 
+const bool kAlwaysClearMemory = true;
+
+struct ChunkMetadata {
+  u8 allocated : 8;  // Must be first.
+  ChunkTag tag : 2;
+#if SANITIZER_WORDSIZE == 64
+  uptr requested_size : 54;
+#else
+  uptr requested_size : 32;
+  uptr padding : 22;
+#endif
+  u32 stack_trace_id;
+};
+
+#if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \
+    defined(__arm__)
+static const uptr kRegionSizeLog = 20;
+static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog;
+typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
+
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = sizeof(ChunkMetadata);
+  typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = __lsan::kRegionSizeLog;
+  typedef __lsan::ByteMap ByteMap;
+  typedef NoOpMapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+#elif defined(__x86_64__) || defined(__powerpc64__)
+struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
+  static const uptr kSpaceBeg = 0x600000000000ULL;
+  static const uptr kSpaceSize =  0x40000000000ULL; // 4T.
+  static const uptr kMetadataSize = sizeof(ChunkMetadata);
+  typedef DefaultSizeClassMap SizeClassMap;
+  typedef NoOpMapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+
+typedef SizeClassAllocator64<AP64> PrimaryAllocator;
+#endif
+typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
+
+AllocatorCache *GetAllocatorCache();
+
+void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack);
+void *lsan_malloc(uptr size, const StackTrace &stack);
+void lsan_free(void *p);
+void *lsan_realloc(void *p, uptr size, const StackTrace &stack);
+void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack);
+void *lsan_valloc(uptr size, const StackTrace &stack);
+uptr lsan_mz_size(const void *p);
+
 }  // namespace __lsan
 
 #endif  // LSAN_ALLOCATOR_H
diff --git a/libsanitizer/lsan/lsan_common.cc b/libsanitizer/lsan/lsan_common.cc
index 41024e11873..a3274d5c1c3 100644
--- a/libsanitizer/lsan/lsan_common.cc
+++ b/libsanitizer/lsan/lsan_common.cc
@@ -30,20 +30,15 @@ namespace __lsan {
 // also to protect the global list of root regions.
 BlockingMutex global_mutex(LINKER_INITIALIZED);
 
-__attribute__((tls_model("initial-exec")))
-THREADLOCAL int disable_counter;
-bool DisabledInThisThread() { return disable_counter > 0; }
-void DisableInThisThread() { disable_counter++; }
-void EnableInThisThread() {
-  if (!disable_counter && common_flags()->detect_leaks) {
+Flags lsan_flags;
+
+void DisableCounterUnderflow() {
+  if (common_flags()->detect_leaks) {
     Report("Unmatched call to __lsan_enable().\n");
     Die();
   }
-  disable_counter--;
 }
 
-Flags lsan_flags;
-
 void Flags::SetDefaults() {
 #define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
 #include "lsan_flags.inc"
@@ -71,6 +66,19 @@ ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];
 static SuppressionContext *suppression_ctx = nullptr;
 static const char kSuppressionLeak[] = "leak";
 static const char *kSuppressionTypes[] = { kSuppressionLeak };
+static const char kStdSuppressions[] =
+#if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+  // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+  // definition.
+  "leak:*pthread_exit*\n"
+#endif  // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+#if SANITIZER_MAC
+  // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
+  "leak:*_os_trace*\n"
+#endif
+  // TLS leak in some glibc versions, described in
+  // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
+  "leak:*tls_get_addr*\n";
 
 void InitializeSuppressions() {
   CHECK_EQ(nullptr, suppression_ctx);
@@ -79,6 +87,7 @@ void InitializeSuppressions() {
   suppression_ctx->ParseFromFile(flags()->suppressions);
   if (&__lsan_default_suppressions)
     suppression_ctx->Parse(__lsan_default_suppressions());
+  suppression_ctx->Parse(kStdSuppressions);
 }
 
 static SuppressionContext *GetSuppressionContext() {
@@ -86,12 +95,9 @@ static SuppressionContext *GetSuppressionContext() {
   return suppression_ctx;
 }
 
-struct RootRegion {
-  const void *begin;
-  uptr size;
-};
+static InternalMmapVector<RootRegion> *root_regions;
 
-InternalMmapVector<RootRegion> *root_regions;
+InternalMmapVector<RootRegion> const *GetRootRegions() { return root_regions; }
 
 void InitializeRootRegions() {
   CHECK(!root_regions);
@@ -99,6 +105,10 @@ void InitializeRootRegions() {
   root_regions = new(placeholder) InternalMmapVector<RootRegion>(1);
 }
 
+const char *MaybeCallLsanDefaultOptions() {
+  return (&__lsan_default_options) ? __lsan_default_options() : "";
+}
+
 void InitCommonLsan() {
   InitializeRootRegions();
   if (common_flags()->detect_leaks) {
@@ -114,7 +124,6 @@ class Decorator: public __sanitizer::SanitizerCommonDecorator {
   Decorator() : SanitizerCommonDecorator() { }
   const char *Error() { return Red(); }
   const char *Leak() { return Blue(); }
-  const char *End() { return Default(); }
 };
 
 static inline bool CanBeAHeapPointer(uptr p) {
@@ -178,6 +187,23 @@ void ScanRangeForPointers(uptr begin, uptr end,
   }
 }
 
+// Scans a global range for pointers
+void ScanGlobalRange(uptr begin, uptr end, Frontier *frontier) {
+  uptr allocator_begin = 0, allocator_end = 0;
+  GetAllocatorGlobalRange(&allocator_begin, &allocator_end);
+  if (begin <= allocator_begin && allocator_begin < end) {
+    CHECK_LE(allocator_begin, allocator_end);
+    CHECK_LE(allocator_end, end);
+    if (begin < allocator_begin)
+      ScanRangeForPointers(begin, allocator_begin, frontier, "GLOBAL",
+                           kReachable);
+    if (allocator_end < end)
+      ScanRangeForPointers(allocator_end, end, frontier, "GLOBAL", kReachable);
+  } else {
+    ScanRangeForPointers(begin, end, frontier, "GLOBAL", kReachable);
+  }
+}
+
 void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg) {
   Frontier *frontier = reinterpret_cast<Frontier *>(arg);
   ScanRangeForPointers(begin, end, frontier, "FAKE STACK", kReachable);
@@ -186,11 +212,11 @@ void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg) {
 // Scans thread data (stacks and TLS) for heap pointers.
 static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
                            Frontier *frontier) {
-  InternalScopedBuffer<uptr> registers(SuspendedThreadsList::RegisterCount());
+  InternalScopedBuffer<uptr> registers(suspended_threads.RegisterCount());
   uptr registers_begin = reinterpret_cast<uptr>(registers.data());
   uptr registers_end = registers_begin + registers.size();
-  for (uptr i = 0; i < suspended_threads.thread_count(); i++) {
-    uptr os_id = static_cast<uptr>(suspended_threads.GetThreadID(i));
+  for (uptr i = 0; i < suspended_threads.ThreadCount(); i++) {
+    tid_t os_id = static_cast<tid_t>(suspended_threads.GetThreadID(i));
     LOG_THREADS("Processing thread %d.\n", os_id);
     uptr stack_begin, stack_end, tls_begin, tls_end, cache_begin, cache_end;
     DTLS *dtls;
@@ -204,11 +230,13 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
       continue;
     }
     uptr sp;
-    bool have_registers =
-        (suspended_threads.GetRegistersAndSP(i, registers.data(), &sp) == 0);
-    if (!have_registers) {
-      Report("Unable to get registers from thread %d.\n");
-      // If unable to get SP, consider the entire stack to be reachable.
+    PtraceRegistersStatus have_registers =
+        suspended_threads.GetRegistersAndSP(i, registers.data(), &sp);
+    if (have_registers != REGISTERS_AVAILABLE) {
+      Report("Unable to get registers from thread %d.\n", os_id);
+      // If unable to get SP, consider the entire stack to be reachable unless
+      // GetRegistersAndSP failed with ESRCH.
+      if (have_registers == REGISTERS_UNAVAILABLE_FATAL) continue;
       sp = stack_begin;
     }
 
@@ -242,21 +270,23 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
     }
 
     if (flags()->use_tls) {
-      LOG_THREADS("TLS at %p-%p.\n", tls_begin, tls_end);
-      if (cache_begin == cache_end) {
-        ScanRangeForPointers(tls_begin, tls_end, frontier, "TLS", kReachable);
-      } else {
-        // Because LSan should not be loaded with dlopen(), we can assume
-        // that allocator cache will be part of static TLS image.
-        CHECK_LE(tls_begin, cache_begin);
-        CHECK_GE(tls_end, cache_end);
-        if (tls_begin < cache_begin)
-          ScanRangeForPointers(tls_begin, cache_begin, frontier, "TLS",
-                               kReachable);
-        if (tls_end > cache_end)
-          ScanRangeForPointers(cache_end, tls_end, frontier, "TLS", kReachable);
+      if (tls_begin) {
+        LOG_THREADS("TLS at %p-%p.\n", tls_begin, tls_end);
+        // If the tls and cache ranges don't overlap, scan full tls range,
+        // otherwise, only scan the non-overlapping portions
+        if (cache_begin == cache_end || tls_end < cache_begin ||
+            tls_begin > cache_end) {
+          ScanRangeForPointers(tls_begin, tls_end, frontier, "TLS", kReachable);
+        } else {
+          if (tls_begin < cache_begin)
+            ScanRangeForPointers(tls_begin, cache_begin, frontier, "TLS",
+                                 kReachable);
+          if (tls_end > cache_end)
+            ScanRangeForPointers(cache_end, tls_end, frontier, "TLS",
+                                 kReachable);
+        }
       }
-      if (dtls) {
+      if (dtls && !DTLSInDestruction(dtls)) {
         for (uptr j = 0; j < dtls->dtv_size; ++j) {
           uptr dtls_beg = dtls->dtv[j].beg;
           uptr dtls_end = dtls_beg + dtls->dtv[j].size;
@@ -266,28 +296,36 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
                                  kReachable);
           }
         }
+      } else {
+        // We are handling a thread with DTLS under destruction. Log about
+        // this and continue.
+        LOG_THREADS("Thread %d has DTLS under destruction.\n", os_id);
       }
     }
   }
 }
 
-static void ProcessRootRegion(Frontier *frontier, uptr root_begin,
-                              uptr root_end) {
-  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr begin, end, prot;
-  while (proc_maps.Next(&begin, &end,
-                        /*offset*/ nullptr, /*filename*/ nullptr,
-                        /*filename_size*/ 0, &prot)) {
-    uptr intersection_begin = Max(root_begin, begin);
-    uptr intersection_end = Min(end, root_end);
-    if (intersection_begin >= intersection_end) continue;
-    bool is_readable = prot & MemoryMappingLayout::kProtectionRead;
-    LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
-                 root_begin, root_end, begin, end,
-                 is_readable ? "readable" : "unreadable");
-    if (is_readable)
-      ScanRangeForPointers(intersection_begin, intersection_end, frontier,
-                           "ROOT", kReachable);
+void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
+                    uptr region_begin, uptr region_end, bool is_readable) {
+  uptr intersection_begin = Max(root_region.begin, region_begin);
+  uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
+  if (intersection_begin >= intersection_end) return;
+  LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
+               root_region.begin, root_region.begin + root_region.size,
+               region_begin, region_end,
+               is_readable ? "readable" : "unreadable");
+  if (is_readable)
+    ScanRangeForPointers(intersection_begin, intersection_end, frontier, "ROOT",
+                         kReachable);
+}
+
+static void ProcessRootRegion(Frontier *frontier,
+                              const RootRegion &root_region) {
+  MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    ScanRootRegion(frontier, root_region, segment.start, segment.end,
+                   segment.IsReadable());
   }
 }
 
@@ -296,9 +334,7 @@ static void ProcessRootRegions(Frontier *frontier) {
   if (!flags()->use_root_regions) return;
   CHECK(root_regions);
   for (uptr i = 0; i < root_regions->size(); i++) {
-    RootRegion region = (*root_regions)[i];
-    uptr begin_addr = reinterpret_cast<uptr>(region.begin);
-    ProcessRootRegion(frontier, begin_addr, begin_addr + region.size);
+    ProcessRootRegion(frontier, (*root_regions)[i]);
   }
 }
 
@@ -336,6 +372,72 @@ static void CollectIgnoredCb(uptr chunk, void *arg) {
   }
 }
 
+static uptr GetCallerPC(u32 stack_id, StackDepotReverseMap *map) {
+  CHECK(stack_id);
+  StackTrace stack = map->Get(stack_id);
+  // The top frame is our malloc/calloc/etc. The next frame is the caller.
+  if (stack.size >= 2)
+    return stack.trace[1];
+  return 0;
+}
+
+struct InvalidPCParam {
+  Frontier *frontier;
+  StackDepotReverseMap *stack_depot_reverse_map;
+  bool skip_linker_allocations;
+};
+
+// ForEachChunk callback. If the caller pc is invalid or is within the linker,
+// mark as reachable. Called by ProcessPlatformSpecificAllocations.
+static void MarkInvalidPCCb(uptr chunk, void *arg) {
+  CHECK(arg);
+  InvalidPCParam *param = reinterpret_cast<InvalidPCParam *>(arg);
+  chunk = GetUserBegin(chunk);
+  LsanMetadata m(chunk);
+  if (m.allocated() && m.tag() != kReachable && m.tag() != kIgnored) {
+    u32 stack_id = m.stack_trace_id();
+    uptr caller_pc = 0;
+    if (stack_id > 0)
+      caller_pc = GetCallerPC(stack_id, param->stack_depot_reverse_map);
+    // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
+    // it as reachable, as we can't properly report its allocation stack anyway.
+    if (caller_pc == 0 || (param->skip_linker_allocations &&
+                           GetLinker()->containsAddress(caller_pc))) {
+      m.set_tag(kReachable);
+      param->frontier->push_back(chunk);
+    }
+  }
+}
+
+// On Linux, handles dynamically allocated TLS blocks by treating all chunks
+// allocated from ld-linux.so as reachable.
+// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
+// They are allocated with a __libc_memalign() call in allocate_and_init()
+// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
+// blocks, but we can make sure they come from our own allocator by intercepting
+// __libc_memalign(). On top of that, there is no easy way to reach them. Their
+// addresses are stored in a dynamically allocated array (the DTV) which is
+// referenced from the static TLS. Unfortunately, we can't just rely on the DTV
+// being reachable from the static TLS, and the dynamic TLS being reachable from
+// the DTV. This is because the initial DTV is allocated before our interception
+// mechanism kicks in, and thus we don't recognize it as allocated memory. We
+// can't special-case it either, since we don't know its size.
+// Our solution is to include in the root set all allocations made from
+// ld-linux.so (which is where allocate_and_init() is implemented). This is
+// guaranteed to include all dynamic TLS blocks (and possibly other allocations
+// which we don't care about).
+// On all other platforms, this simply checks to ensure that the caller pc is
+// valid before reporting chunks as leaked.
+void ProcessPC(Frontier *frontier) {
+  StackDepotReverseMap stack_depot_reverse_map;
+  InvalidPCParam arg;
+  arg.frontier = frontier;
+  arg.stack_depot_reverse_map = &stack_depot_reverse_map;
+  arg.skip_linker_allocations =
+      flags()->use_tls && flags()->use_ld_allocations && GetLinker() != nullptr;
+  ForEachChunk(MarkInvalidPCCb, &arg);
+}
+
 // Sets the appropriate tag on each chunk.
 static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads) {
   // Holds the flood fill frontier.
@@ -347,11 +449,13 @@ static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads) {
   ProcessRootRegions(&frontier);
   FloodFillTag(&frontier, kReachable);
 
+  CHECK_EQ(0, frontier.size());
+  ProcessPC(&frontier);
+
   // The check here is relatively expensive, so we do this in a separate flood
   // fill. That way we can skip the check for chunks that are reachable
   // otherwise.
   LOG_POINTERS("Processing platform-specific allocations.\n");
-  CHECK_EQ(0, frontier.size());
   ProcessPlatformSpecificAllocations(&frontier);
   FloodFillTag(&frontier, kReachable);
 
@@ -461,7 +565,7 @@ static bool CheckForLeaks() {
            "\n");
     Printf("%s", d.Error());
     Report("ERROR: LeakSanitizer: detected memory leaks\n");
-    Printf("%s", d.End());
+    Printf("%s", d.Default());
     param.leak_report.ReportTopLeaks(flags()->max_leaks);
   }
   if (common_flags()->print_suppressions)
@@ -473,18 +577,16 @@ static bool CheckForLeaks() {
   return false;
 }
 
+static bool has_reported_leaks = false;
+bool HasReportedLeaks() { return has_reported_leaks; }
+
 void DoLeakCheck() {
   BlockingMutexLock l(&global_mutex);
   static bool already_done;
   if (already_done) return;
   already_done = true;
-  bool have_leaks = CheckForLeaks();
-  if (!have_leaks) {
-    return;
-  }
-  if (common_flags()->exitcode) {
-    Die();
-  }
+  has_reported_leaks = CheckForLeaks();
+  if (has_reported_leaks) HandleLeaks();
 }
 
 static int DoRecoverableLeakCheck() {
@@ -493,6 +595,8 @@ static int DoRecoverableLeakCheck() {
   return have_leaks ? 1 : 0;
 }
 
+void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); }
+
 static Suppression *GetSuppressionForAddr(uptr addr) {
   Suppression *s = nullptr;
 
@@ -597,7 +701,7 @@ void LeakReport::PrintReportForLeak(uptr index) {
   Printf("%s leak of %zu byte(s) in %zu object(s) allocated from:\n",
          leaks_[index].is_directly_leaked ? "Direct" : "Indirect",
          leaks_[index].total_size, leaks_[index].hit_count);
-  Printf("%s", d.End());
+  Printf("%s", d.Default());
 
   PrintStackTraceById(leaks_[index].stack_trace_id);
 
@@ -655,6 +759,7 @@ uptr LeakReport::UnsuppressedLeakCount() {
 namespace __lsan {
 void InitCommonLsan() { }
 void DoLeakCheck() { }
+void DoRecoverableLeakCheckVoid() { }
 void DisableInThisThread() { }
 void EnableInThisThread() { }
 }
@@ -687,7 +792,7 @@ void __lsan_register_root_region(const void *begin, uptr size) {
 #if CAN_SANITIZE_LEAKS
   BlockingMutexLock l(&global_mutex);
   CHECK(root_regions);
-  RootRegion region = {begin, size};
+  RootRegion region = {reinterpret_cast<uptr>(begin), size};
   root_regions->push_back(region);
   VReport(1, "Registered root region at %p of size %llu\n", begin, size);
 #endif // CAN_SANITIZE_LEAKS
@@ -701,7 +806,7 @@ void __lsan_unregister_root_region(const void *begin, uptr size) {
   bool removed = false;
   for (uptr i = 0; i < root_regions->size(); i++) {
     RootRegion region = (*root_regions)[i];
-    if (region.begin == begin && region.size == size) {
+    if (region.begin == reinterpret_cast<uptr>(begin) && region.size == size) {
       removed = true;
       uptr last_index = root_regions->size() - 1;
       (*root_regions)[i] = (*root_regions)[last_index];
@@ -753,8 +858,18 @@ int __lsan_do_recoverable_leak_check() {
 
 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char * __lsan_default_options() {
+  return "";
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 int __lsan_is_turned_off() {
   return 0;
 }
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char *__lsan_default_suppressions() {
+  return "";
+}
 #endif
 } // extern "C"
diff --git a/libsanitizer/lsan/lsan_common.h b/libsanitizer/lsan/lsan_common.h
index 1091b84f108..e99cd9e1b52 100644
--- a/libsanitizer/lsan/lsan_common.h
+++ b/libsanitizer/lsan/lsan_common.h
@@ -20,8 +20,24 @@
 #include "sanitizer_common/sanitizer_stoptheworld.h"
 #include "sanitizer_common/sanitizer_symbolizer.h"
 
-#if (SANITIZER_LINUX && !SANITIZER_ANDROID) && (SANITIZER_WORDSIZE == 64) \
-     && (defined(__x86_64__) ||  defined(__mips64) ||  defined(__aarch64__))
+// LeakSanitizer relies on some Glibc's internals (e.g. TLS machinery) thus
+// supported for Linux only. Also, LSan doesn't like 32 bit architectures
+// because of "small" (4 bytes) pointer size that leads to high false negative
+// ratio on large leaks. But we still want to have it for some 32 bit arches
+// (e.g. x86), see https://github.com/google/sanitizers/issues/403.
+// To enable LeakSanitizer on new architecture, one need to implement
+// internal_clone function as well as (probably) adjust TLS machinery for
+// new architecture inside sanitizer library.
+#if (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC) && \
+    (SANITIZER_WORDSIZE == 64) &&                               \
+    (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \
+     defined(__powerpc64__))
+#define CAN_SANITIZE_LEAKS 1
+#elif defined(__i386__) && \
+    (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC)
+#define CAN_SANITIZE_LEAKS 1
+#elif defined(__arm__) && \
+    SANITIZER_LINUX && !SANITIZER_ANDROID
 #define CAN_SANITIZE_LEAKS 1
 #else
 #define CAN_SANITIZE_LEAKS 0
@@ -42,6 +58,8 @@ enum ChunkTag {
   kIgnored = 3
 };
 
+const u32 kInvalidTid = (u32) -1;
+
 struct Flags {
 #define LSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
 #include "lsan_flags.inc"
@@ -99,12 +117,22 @@ typedef InternalMmapVector<uptr> Frontier;
 void InitializePlatformSpecificModules();
 void ProcessGlobalRegions(Frontier *frontier);
 void ProcessPlatformSpecificAllocations(Frontier *frontier);
+
+struct RootRegion {
+  uptr begin;
+  uptr size;
+};
+
+InternalMmapVector<RootRegion> const *GetRootRegions();
+void ScanRootRegion(Frontier *frontier, RootRegion const &region,
+                    uptr region_begin, uptr region_end, bool is_readable);
 // Run stoptheworld while holding any platform-specific locks.
 void DoStopTheWorld(StopTheWorldCallback callback, void* argument);
 
 void ScanRangeForPointers(uptr begin, uptr end,
                           Frontier *frontier,
                           const char *region_type, ChunkTag tag);
+void ScanGlobalRange(uptr begin, uptr end, Frontier *frontier);
 
 enum IgnoreObjectResult {
   kIgnoreObjectSuccess,
@@ -113,8 +141,11 @@ enum IgnoreObjectResult {
 };
 
 // Functions called from the parent tool.
+const char *MaybeCallLsanDefaultOptions();
 void InitCommonLsan();
 void DoLeakCheck();
+void DoRecoverableLeakCheckVoid();
+void DisableCounterUnderflow();
 bool DisabledInThisThread();
 
 // Used to implement __lsan::ScopedDisabler.
@@ -127,13 +158,36 @@ struct ScopedInterceptorDisabler {
   ~ScopedInterceptorDisabler() { EnableInThisThread(); }
 };
 
+// According to Itanium C++ ABI array cookie is a one word containing
+// size of allocated array.
+static inline bool IsItaniumABIArrayCookie(uptr chunk_beg, uptr chunk_size,
+                                           uptr addr) {
+  return chunk_size == sizeof(uptr) && chunk_beg + chunk_size == addr &&
+         *reinterpret_cast<uptr *>(chunk_beg) == 0;
+}
+
+// According to ARM C++ ABI array cookie consists of two words:
+// struct array_cookie {
+//   std::size_t element_size; // element_size != 0
+//   std::size_t element_count;
+// };
+static inline bool IsARMABIArrayCookie(uptr chunk_beg, uptr chunk_size,
+                                       uptr addr) {
+  return chunk_size == 2 * sizeof(uptr) && chunk_beg + chunk_size == addr &&
+         *reinterpret_cast<uptr *>(chunk_beg + sizeof(uptr)) == 0;
+}
+
 // Special case for "new T[0]" where T is a type with DTOR.
-// new T[0] will allocate one word for the array size (0) and store a pointer
-// to the end of allocated chunk.
+// new T[0] will allocate a cookie (one or two words) for the array size (0)
+// and store a pointer to the end of allocated chunk. The actual cookie layout
+// varies between platforms according to their C++ ABI implementation.
 inline bool IsSpecialCaseOfOperatorNew0(uptr chunk_beg, uptr chunk_size,
                                         uptr addr) {
-  return chunk_size == sizeof(uptr) && chunk_beg + chunk_size == addr &&
-         *reinterpret_cast<uptr *>(chunk_beg) == 0;
+#if defined(__arm__)
+  return IsARMABIArrayCookie(chunk_beg, chunk_size, addr);
+#else
+  return IsItaniumABIArrayCookie(chunk_beg, chunk_size, addr);
+#endif
 }
 
 // The following must be implemented in the parent tool.
@@ -149,10 +203,10 @@ bool WordIsPoisoned(uptr addr);
 // Wrappers for ThreadRegistry access.
 void LockThreadRegistry();
 void UnlockThreadRegistry();
-bool GetThreadRangesLocked(uptr os_id, uptr *stack_begin, uptr *stack_end,
+bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
                            uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
                            uptr *cache_end, DTLS **dtls);
-void ForEachExtraStackRange(uptr os_id, RangeIteratorCallback callback,
+void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
                             void *arg);
 // If called from the main thread, updates the main thread's TID in the thread
 // registry. We need this to handle processes that fork() without a subsequent
@@ -168,6 +222,16 @@ uptr PointsIntoChunk(void *p);
 uptr GetUserBegin(uptr chunk);
 // Helper for __lsan_ignore_object().
 IgnoreObjectResult IgnoreObjectLocked(const void *p);
+
+// Return the linker module, if valid for the platform.
+LoadedModule *GetLinker();
+
+// Return true if LSan has finished leak checking and reported leaks.
+bool HasReportedLeaks();
+
+// Run platform-specific leak handlers.
+void HandleLeaks();
+
 // Wrapper for chunk metadata operations.
 class LsanMetadata {
  public:
@@ -186,6 +250,9 @@ class LsanMetadata {
 
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char *__lsan_default_options();
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 int __lsan_is_turned_off();
 
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
diff --git a/libsanitizer/lsan/lsan_common_linux.cc b/libsanitizer/lsan/lsan_common_linux.cc
index abbb61f07c9..677727229b1 100644
--- a/libsanitizer/lsan/lsan_common_linux.cc
+++ b/libsanitizer/lsan/lsan_common_linux.cc
@@ -32,6 +32,17 @@ static bool IsLinker(const char* full_name) {
   return LibraryNameIs(full_name, kLinkerName);
 }
 
+__attribute__((tls_model("initial-exec")))
+THREADLOCAL int disable_counter;
+bool DisabledInThisThread() { return disable_counter > 0; }
+void DisableInThisThread() { disable_counter++; }
+void EnableInThisThread() {
+  if (disable_counter == 0) {
+    DisableCounterUnderflow();
+  }
+  disable_counter--;
+}
+
 void InitializePlatformSpecificModules() {
   ListOfModules modules;
   modules.init();
@@ -49,8 +60,10 @@ void InitializePlatformSpecificModules() {
       return;
     }
   }
-  VReport(1, "LeakSanitizer: Dynamic linker not found. "
-             "TLS will not be handled correctly.\n");
+  if (linker == nullptr) {
+    VReport(1, "LeakSanitizer: Dynamic linker not found. "
+               "TLS will not be handled correctly.\n");
+  }
 }
 
 static int ProcessGlobalRegionsCallback(struct dl_phdr_info *info, size_t size,
@@ -65,20 +78,7 @@ static int ProcessGlobalRegionsCallback(struct dl_phdr_info *info, size_t size,
       continue;
     uptr begin = info->dlpi_addr + phdr->p_vaddr;
     uptr end = begin + phdr->p_memsz;
-    uptr allocator_begin = 0, allocator_end = 0;
-    GetAllocatorGlobalRange(&allocator_begin, &allocator_end);
-    if (begin <= allocator_begin && allocator_begin < end) {
-      CHECK_LE(allocator_begin, allocator_end);
-      CHECK_LE(allocator_end, end);
-      if (begin < allocator_begin)
-        ScanRangeForPointers(begin, allocator_begin, frontier, "GLOBAL",
-                             kReachable);
-      if (allocator_end < end)
-        ScanRangeForPointers(allocator_end, end, frontier, "GLOBAL",
-                             kReachable);
-    } else {
-      ScanRangeForPointers(begin, end, frontier, "GLOBAL", kReachable);
-    }
+    ScanGlobalRange(begin, end, frontier);
   }
   return 0;
 }
@@ -89,76 +89,22 @@ void ProcessGlobalRegions(Frontier *frontier) {
   dl_iterate_phdr(ProcessGlobalRegionsCallback, frontier);
 }
 
-static uptr GetCallerPC(u32 stack_id, StackDepotReverseMap *map) {
-  CHECK(stack_id);
-  StackTrace stack = map->Get(stack_id);
-  // The top frame is our malloc/calloc/etc. The next frame is the caller.
-  if (stack.size >= 2)
-    return stack.trace[1];
-  return 0;
-}
+LoadedModule *GetLinker() { return linker; }
 
-struct ProcessPlatformAllocParam {
-  Frontier *frontier;
-  StackDepotReverseMap *stack_depot_reverse_map;
-  bool skip_linker_allocations;
-};
-
-// ForEachChunk callback. Identifies unreachable chunks which must be treated as
-// reachable. Marks them as reachable and adds them to the frontier.
-static void ProcessPlatformSpecificAllocationsCb(uptr chunk, void *arg) {
-  CHECK(arg);
-  ProcessPlatformAllocParam *param =
-      reinterpret_cast<ProcessPlatformAllocParam *>(arg);
-  chunk = GetUserBegin(chunk);
-  LsanMetadata m(chunk);
-  if (m.allocated() && m.tag() != kReachable && m.tag() != kIgnored) {
-    u32 stack_id = m.stack_trace_id();
-    uptr caller_pc = 0;
-    if (stack_id > 0)
-      caller_pc = GetCallerPC(stack_id, param->stack_depot_reverse_map);
-    // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
-    // it as reachable, as we can't properly report its allocation stack anyway.
-    if (caller_pc == 0 || (param->skip_linker_allocations &&
-                           linker->containsAddress(caller_pc))) {
-      m.set_tag(kReachable);
-      param->frontier->push_back(chunk);
-    }
-  }
-}
-
-// Handles dynamically allocated TLS blocks by treating all chunks allocated
-// from ld-linux.so as reachable.
-// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
-// They are allocated with a __libc_memalign() call in allocate_and_init()
-// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
-// blocks, but we can make sure they come from our own allocator by intercepting
-// __libc_memalign(). On top of that, there is no easy way to reach them. Their
-// addresses are stored in a dynamically allocated array (the DTV) which is
-// referenced from the static TLS. Unfortunately, we can't just rely on the DTV
-// being reachable from the static TLS, and the dynamic TLS being reachable from
-// the DTV. This is because the initial DTV is allocated before our interception
-// mechanism kicks in, and thus we don't recognize it as allocated memory. We
-// can't special-case it either, since we don't know its size.
-// Our solution is to include in the root set all allocations made from
-// ld-linux.so (which is where allocate_and_init() is implemented). This is
-// guaranteed to include all dynamic TLS blocks (and possibly other allocations
-// which we don't care about).
-void ProcessPlatformSpecificAllocations(Frontier *frontier) {
-  StackDepotReverseMap stack_depot_reverse_map;
-  ProcessPlatformAllocParam arg;
-  arg.frontier = frontier;
-  arg.stack_depot_reverse_map = &stack_depot_reverse_map;
-  arg.skip_linker_allocations =
-      flags()->use_tls && flags()->use_ld_allocations && linker != nullptr;
-  ForEachChunk(ProcessPlatformSpecificAllocationsCb, &arg);
-}
+void ProcessPlatformSpecificAllocations(Frontier *frontier) {}
 
 struct DoStopTheWorldParam {
   StopTheWorldCallback callback;
   void *argument;
 };
 
+// While calling Die() here is undefined behavior and can potentially
+// cause race conditions, it isn't possible to intercept exit on linux,
+// so we have no choice but to call Die() from the atexit handler.
+void HandleLeaks() {
+  if (common_flags()->exitcode) Die();
+}
+
 static int DoStopTheWorldCallback(struct dl_phdr_info *info, size_t size,
                                   void *data) {
   DoStopTheWorldParam *param = reinterpret_cast<DoStopTheWorldParam *>(data);
diff --git a/libsanitizer/lsan/lsan_common_mac.cc b/libsanitizer/lsan/lsan_common_mac.cc
new file mode 100644
index 00000000000..e60b3d0f195
--- /dev/null
+++ b/libsanitizer/lsan/lsan_common_mac.cc
@@ -0,0 +1,197 @@
+//=-- lsan_common_mac.cc --------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer.
+// Implementation of common leak checking functionality. Darwin-specific code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#include "lsan_common.h"
+
+#if CAN_SANITIZE_LEAKS && SANITIZER_MAC
+
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "lsan_allocator.h"
+
+#include <pthread.h>
+
+#include <mach/mach.h>
+
+namespace __lsan {
+
+typedef struct {
+  int disable_counter;
+  u32 current_thread_id;
+  AllocatorCache cache;
+} thread_local_data_t;
+
+static pthread_key_t key;
+static pthread_once_t key_once = PTHREAD_ONCE_INIT;
+
+// The main thread destructor requires the current thread id,
+// so we can't destroy it until it's been used and reset to invalid tid
+void restore_tid_data(void *ptr) {
+  thread_local_data_t *data = (thread_local_data_t *)ptr;
+  if (data->current_thread_id != kInvalidTid)
+    pthread_setspecific(key, data);
+}
+
+static void make_tls_key() {
+  CHECK_EQ(pthread_key_create(&key, restore_tid_data), 0);
+}
+
+static thread_local_data_t *get_tls_val(bool alloc) {
+  pthread_once(&key_once, make_tls_key);
+
+  thread_local_data_t *ptr = (thread_local_data_t *)pthread_getspecific(key);
+  if (ptr == NULL && alloc) {
+    ptr = (thread_local_data_t *)InternalAlloc(sizeof(*ptr));
+    ptr->disable_counter = 0;
+    ptr->current_thread_id = kInvalidTid;
+    ptr->cache = AllocatorCache();
+    pthread_setspecific(key, ptr);
+  }
+
+  return ptr;
+}
+
+bool DisabledInThisThread() {
+  thread_local_data_t *data = get_tls_val(false);
+  return data ? data->disable_counter > 0 : false;
+}
+
+void DisableInThisThread() { ++get_tls_val(true)->disable_counter; }
+
+void EnableInThisThread() {
+  int *disable_counter = &get_tls_val(true)->disable_counter;
+  if (*disable_counter == 0) {
+    DisableCounterUnderflow();
+  }
+  --*disable_counter;
+}
+
+u32 GetCurrentThread() {
+  thread_local_data_t *data = get_tls_val(false);
+  return data ? data->current_thread_id : kInvalidTid;
+}
+
+void SetCurrentThread(u32 tid) { get_tls_val(true)->current_thread_id = tid; }
+
+AllocatorCache *GetAllocatorCache() { return &get_tls_val(true)->cache; }
+
+LoadedModule *GetLinker() { return nullptr; }
+
+// Required on Linux for initialization of TLS behavior, but should not be
+// required on Darwin.
+void InitializePlatformSpecificModules() {}
+
+// Sections which can't contain contain global pointers. This list errs on the
+// side of caution to avoid false positives, at the expense of performance.
+//
+// Other potentially safe sections include:
+// __all_image_info, __crash_info, __const, __got, __interpose, __objc_msg_break
+//
+// Sections which definitely cannot be included here are:
+// __objc_data, __objc_const, __data, __bss, __common, __thread_data,
+// __thread_bss, __thread_vars, __objc_opt_rw, __objc_opt_ptrs
+static const char *kSkippedSecNames[] = {
+    "__cfstring",       "__la_symbol_ptr",  "__mod_init_func",
+    "__mod_term_func",  "__nl_symbol_ptr",  "__objc_classlist",
+    "__objc_classrefs", "__objc_imageinfo", "__objc_nlclslist",
+    "__objc_protolist", "__objc_selrefs",   "__objc_superrefs"};
+
+// Scans global variables for heap pointers.
+void ProcessGlobalRegions(Frontier *frontier) {
+  for (auto name : kSkippedSecNames) CHECK(ARRAY_SIZE(name) < kMaxSegName);
+
+  MemoryMappingLayout memory_mapping(false);
+  InternalMmapVector<LoadedModule> modules(/*initial_capacity*/ 128);
+  memory_mapping.DumpListOfModules(&modules);
+  for (uptr i = 0; i < modules.size(); ++i) {
+    // Even when global scanning is disabled, we still need to scan
+    // system libraries for stashed pointers
+    if (!flags()->use_globals && modules[i].instrumented()) continue;
+
+    for (const __sanitizer::LoadedModule::AddressRange &range :
+         modules[i].ranges()) {
+      // Sections storing global variables are writable and non-executable
+      if (range.executable || !range.writable) continue;
+
+      for (auto name : kSkippedSecNames) {
+        if (!internal_strcmp(range.name, name)) continue;
+      }
+
+      ScanGlobalRange(range.beg, range.end, frontier);
+    }
+  }
+}
+
+void ProcessPlatformSpecificAllocations(Frontier *frontier) {
+  mach_port_name_t port;
+  if (task_for_pid(mach_task_self(), internal_getpid(), &port)
+      != KERN_SUCCESS) {
+    return;
+  }
+
+  unsigned depth = 1;
+  vm_size_t size = 0;
+  vm_address_t address = 0;
+  kern_return_t err = KERN_SUCCESS;
+  mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
+
+  InternalMmapVector<RootRegion> const *root_regions = GetRootRegions();
+
+  while (err == KERN_SUCCESS) {
+    struct vm_region_submap_info_64 info;
+    err = vm_region_recurse_64(port, &address, &size, &depth,
+                               (vm_region_info_t)&info, &count);
+
+    uptr end_address = address + size;
+
+    // libxpc stashes some pointers in the Kernel Alloc Once page,
+    // make sure not to report those as leaks.
+    if (info.user_tag == VM_MEMORY_OS_ALLOC_ONCE) {
+      ScanRangeForPointers(address, end_address, frontier, "GLOBAL",
+                           kReachable);
+
+      // Recursing over the full memory map is very slow, break out
+      // early if we don't need the full iteration.
+      if (!flags()->use_root_regions || !root_regions->size())
+        break;
+    }
+
+    // This additional root region scan is required on Darwin in order to
+    // detect root regions contained within mmap'd memory regions, because
+    // the Darwin implementation of sanitizer_procmaps traverses images
+    // as loaded by dyld, and not the complete set of all memory regions.
+    //
+    // TODO(fjricci) - remove this once sanitizer_procmaps_mac has the same
+    // behavior as sanitizer_procmaps_linux and traverses all memory regions
+    if (flags()->use_root_regions) {
+      for (uptr i = 0; i < root_regions->size(); i++) {
+        ScanRootRegion(frontier, (*root_regions)[i], address, end_address,
+                       info.protection & kProtectionRead);
+      }
+    }
+
+    address = end_address;
+  }
+}
+
+// On darwin, we can intercept _exit gracefully, and return a failing exit code
+// if required at that point. Calling Die() here is undefined behavior and
+// causes rare race conditions.
+void HandleLeaks() {}
+
+void DoStopTheWorld(StopTheWorldCallback callback, void *argument) {
+  StopTheWorld(callback, argument);
+}
+
+} // namespace __lsan
+
+#endif // CAN_SANITIZE_LEAKS && SANITIZER_MAC
diff --git a/libsanitizer/lsan/lsan_interceptors.cc b/libsanitizer/lsan/lsan_interceptors.cc
index 160ed5979c4..c9279aad676 100644
--- a/libsanitizer/lsan/lsan_interceptors.cc
+++ b/libsanitizer/lsan/lsan_interceptors.cc
@@ -17,13 +17,18 @@
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_posix.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 #include "lsan.h"
 #include "lsan_allocator.h"
 #include "lsan_common.h"
 #include "lsan_thread.h"
 
+#include <stddef.h>
+
 using namespace __lsan;
 
 extern "C" {
@@ -34,29 +39,23 @@ int pthread_key_create(unsigned *key, void (*destructor)(void* v));
 int pthread_setspecific(unsigned key, const void *v);
 }
 
-#define ENSURE_LSAN_INITED do {   \
-  CHECK(!lsan_init_is_running);   \
-  if (!lsan_inited)               \
-    __lsan_init();                \
-} while (0)
-
 ///// Malloc/free interceptors. /////
 
-const bool kAlwaysClearMemory = true;
-
 namespace std {
   struct nothrow_t;
+  enum class align_val_t: size_t;
 }
 
+#if !SANITIZER_MAC
 INTERCEPTOR(void*, malloc, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  return Allocate(stack, size, 1, kAlwaysClearMemory);
+  return lsan_malloc(size, stack);
 }
 
 INTERCEPTOR(void, free, void *p) {
   ENSURE_LSAN_INITED;
-  Deallocate(p);
+  lsan_free(p);
 }
 
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
@@ -71,60 +70,76 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
     CHECK(allocated < kCallocPoolSize);
     return mem;
   }
-  if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return nullptr;
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  size *= nmemb;
-  return Allocate(stack, size, 1, true);
+  return lsan_calloc(nmemb, size, stack);
 }
 
 INTERCEPTOR(void*, realloc, void *q, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  return Reallocate(stack, q, size, 1);
+  return lsan_realloc(q, size, stack);
 }
 
-INTERCEPTOR(void*, memalign, uptr alignment, uptr size) {
+INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  return Allocate(stack, size, alignment, kAlwaysClearMemory);
+  *memptr = lsan_memalign(alignment, size, stack);
+  // FIXME: Return ENOMEM if user requested more than max alloc size.
+  return 0;
 }
 
-INTERCEPTOR(void*, aligned_alloc, uptr alignment, uptr size) {
+INTERCEPTOR(void*, valloc, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  return Allocate(stack, size, alignment, kAlwaysClearMemory);
+  return lsan_valloc(size, stack);
 }
+#endif
 
-INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
+#if SANITIZER_INTERCEPT_MEMALIGN
+INTERCEPTOR(void*, memalign, uptr alignment, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  *memptr = Allocate(stack, size, alignment, kAlwaysClearMemory);
-  // FIXME: Return ENOMEM if user requested more than max alloc size.
-  return 0;
+  return lsan_memalign(alignment, size, stack);
 }
+#define LSAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign)
 
 INTERCEPTOR(void *, __libc_memalign, uptr alignment, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  void *res = Allocate(stack, size, alignment, kAlwaysClearMemory);
+  void *res = lsan_memalign(alignment, size, stack);
   DTLS_on_libc_memalign(res, size);
   return res;
 }
+#define LSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN INTERCEPT_FUNCTION(__libc_memalign)
+#else
+#define LSAN_MAYBE_INTERCEPT_MEMALIGN
+#define LSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN
+#endif // SANITIZER_INTERCEPT_MEMALIGN
 
-INTERCEPTOR(void*, valloc, uptr size) {
+#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
+INTERCEPTOR(void*, aligned_alloc, uptr alignment, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  if (size == 0)
-    size = GetPageSizeCached();
-  return Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory);
+  return lsan_memalign(alignment, size, stack);
 }
+#define LSAN_MAYBE_INTERCEPT_ALIGNED_ALLOC INTERCEPT_FUNCTION(aligned_alloc)
+#else
+#define LSAN_MAYBE_INTERCEPT_ALIGNED_ALLOC
+#endif
 
+#if SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE
 INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
   ENSURE_LSAN_INITED;
   return GetMallocUsableSize(ptr);
 }
+#define LSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE \
+        INTERCEPT_FUNCTION(malloc_usable_size)
+#else
+#define LSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE
+#endif
 
+#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
 struct fake_mallinfo {
   int x[10];
 };
@@ -134,11 +149,18 @@ INTERCEPTOR(struct fake_mallinfo, mallinfo, void) {
   internal_memset(&res, 0, sizeof(res));
   return res;
 }
+#define LSAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo)
 
 INTERCEPTOR(int, mallopt, int cmd, int value) {
   return -1;
 }
+#define LSAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt)
+#else
+#define LSAN_MAYBE_INTERCEPT_MALLINFO
+#define LSAN_MAYBE_INTERCEPT_MALLOPT
+#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
 
+#if SANITIZER_INTERCEPT_PVALLOC
 INTERCEPTOR(void*, pvalloc, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
@@ -150,26 +172,81 @@ INTERCEPTOR(void*, pvalloc, uptr size) {
   }
   return Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory);
 }
+#define LSAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc)
+#else
+#define LSAN_MAYBE_INTERCEPT_PVALLOC
+#endif // SANITIZER_INTERCEPT_PVALLOC
 
+#if SANITIZER_INTERCEPT_CFREE
 INTERCEPTOR(void, cfree, void *p) ALIAS(WRAPPER_NAME(free));
+#define LSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
+#else
+#define LSAN_MAYBE_INTERCEPT_CFREE
+#endif // SANITIZER_INTERCEPT_CFREE
+
+#if SANITIZER_INTERCEPT_MCHECK_MPROBE
+INTERCEPTOR(int, mcheck, void (*abortfunc)(int mstatus)) {
+  return 0;
+}
+
+INTERCEPTOR(int, mcheck_pedantic, void (*abortfunc)(int mstatus)) {
+  return 0;
+}
 
-#define OPERATOR_NEW_BODY                              \
-  ENSURE_LSAN_INITED;                                  \
-  GET_STACK_TRACE_MALLOC;                              \
-  return Allocate(stack, size, 1, kAlwaysClearMemory);
+INTERCEPTOR(int, mprobe, void *ptr) {
+  return 0;
+}
+#endif // SANITIZER_INTERCEPT_MCHECK_MPROBE
+
+
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow)                         \
+  ENSURE_LSAN_INITED;                                      \
+  GET_STACK_TRACE_MALLOC;                                  \
+  void *res = lsan_malloc(size, stack);                    \
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();   \
+  return res;
+#define OPERATOR_NEW_BODY_ALIGN(nothrow)                   \
+  ENSURE_LSAN_INITED;                                      \
+  GET_STACK_TRACE_MALLOC;                                  \
+  void *res = lsan_memalign((uptr)align, size, stack);     \
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();   \
+  return res;
+
+#define OPERATOR_DELETE_BODY \
+  ENSURE_LSAN_INITED;        \
+  lsan_free(ptr);
+
+// On OS X it's not enough to just provide our own 'operator new' and
+// 'operator delete' implementations, because they're going to be in the runtime
+// dylib, and the main executable will depend on both the runtime dylib and
+// libstdc++, each of has its implementation of new and delete.
+// To make sure that C++ allocation/deallocation operators are overridden on
+// OS X we need to intercept them using their mangled names.
+#if !SANITIZER_MAC
 
 INTERCEPTOR_ATTRIBUTE
-void *operator new(uptr size) { OPERATOR_NEW_BODY; }
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new[](uptr size) { OPERATOR_NEW_BODY; }
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new(uptr size, std::nothrow_t const&) { OPERATOR_NEW_BODY; }
+void *operator new(size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new[](uptr size, std::nothrow_t const&) { OPERATOR_NEW_BODY; }
-
-#define OPERATOR_DELETE_BODY \
-  ENSURE_LSAN_INITED;        \
-  Deallocate(ptr);
+void *operator new[](size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align)
+{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align)
+{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); }
 
 INTERCEPTOR_ATTRIBUTE
 void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
@@ -178,9 +255,55 @@ void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
 INTERCEPTOR_ATTRIBUTE
 void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
 INTERCEPTOR_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const &) {
-  OPERATOR_DELETE_BODY;
-}
+void operator delete[](void *ptr, std::nothrow_t const &)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+
+#else  // SANITIZER_MAC
+
+INTERCEPTOR(void *, _Znwm, size_t size)
+{ OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR(void *, _Znam, size_t size)
+{ OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
+INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
+
+INTERCEPTOR(void, _ZdlPv, void *ptr)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdaPv, void *ptr)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+
+#endif  // !SANITIZER_MAC
+
 
 ///// Thread initialization and finalization. /////
 
@@ -250,7 +373,8 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr,
     res = REAL(pthread_create)(th, attr, __lsan_thread_start_func, &p);
   }
   if (res == 0) {
-    int tid = ThreadCreate(GetCurrentThread(), *(uptr *)th, detached);
+    int tid = ThreadCreate(GetCurrentThread(), *(uptr *)th,
+                           IsStateDetached(detached));
     CHECK_NE(tid, 0);
     atomic_store(&p.tid, tid, memory_order_release);
     while (atomic_load(&p.tid, memory_order_acquire) != 0)
@@ -270,24 +394,36 @@ INTERCEPTOR(int, pthread_join, void *th, void **ret) {
   return res;
 }
 
+INTERCEPTOR(void, _exit, int status) {
+  if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode;
+  REAL(_exit)(status);
+}
+
+#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
 namespace __lsan {
 
 void InitializeInterceptors() {
+  InitializeSignalInterceptors();
+
   INTERCEPT_FUNCTION(malloc);
   INTERCEPT_FUNCTION(free);
-  INTERCEPT_FUNCTION(cfree);
+  LSAN_MAYBE_INTERCEPT_CFREE;
   INTERCEPT_FUNCTION(calloc);
   INTERCEPT_FUNCTION(realloc);
-  INTERCEPT_FUNCTION(memalign);
+  LSAN_MAYBE_INTERCEPT_MEMALIGN;
+  LSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN;
+  LSAN_MAYBE_INTERCEPT_ALIGNED_ALLOC;
   INTERCEPT_FUNCTION(posix_memalign);
-  INTERCEPT_FUNCTION(__libc_memalign);
   INTERCEPT_FUNCTION(valloc);
-  INTERCEPT_FUNCTION(pvalloc);
-  INTERCEPT_FUNCTION(malloc_usable_size);
-  INTERCEPT_FUNCTION(mallinfo);
-  INTERCEPT_FUNCTION(mallopt);
+  LSAN_MAYBE_INTERCEPT_PVALLOC;
+  LSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE;
+  LSAN_MAYBE_INTERCEPT_MALLINFO;
+  LSAN_MAYBE_INTERCEPT_MALLOPT;
   INTERCEPT_FUNCTION(pthread_create);
   INTERCEPT_FUNCTION(pthread_join);
+  INTERCEPT_FUNCTION(_exit);
 
   if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
     Report("LeakSanitizer: failed to create thread key.\n");
diff --git a/libsanitizer/lsan/lsan_linux.cc b/libsanitizer/lsan/lsan_linux.cc
new file mode 100644
index 00000000000..aa6445a9877
--- /dev/null
+++ b/libsanitizer/lsan/lsan_linux.cc
@@ -0,0 +1,31 @@
+//=-- lsan_linux.cc -------------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer. Linux-specific code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if SANITIZER_LINUX
+
+#include "lsan_allocator.h"
+
+namespace __lsan {
+
+static THREADLOCAL u32 current_thread_tid = kInvalidTid;
+u32 GetCurrentThread() { return current_thread_tid; }
+void SetCurrentThread(u32 tid) { current_thread_tid = tid; }
+
+static THREADLOCAL AllocatorCache allocator_cache;
+AllocatorCache *GetAllocatorCache() { return &allocator_cache; }
+
+void ReplaceSystemMalloc() {}
+
+} // namespace __lsan
+
+#endif // SANITIZER_LINUX
diff --git a/libsanitizer/lsan/lsan_mac.cc b/libsanitizer/lsan/lsan_mac.cc
new file mode 100644
index 00000000000..ca38c1c6f8a
--- /dev/null
+++ b/libsanitizer/lsan/lsan_mac.cc
@@ -0,0 +1,190 @@
+//===-- lsan_mac.cc -------------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer, a memory leak checker.
+//
+// Mac-specific details.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "interception/interception.h"
+#include "lsan.h"
+#include "lsan_allocator.h"
+#include "lsan_thread.h"
+
+#include <pthread.h>
+
+namespace __lsan {
+// Support for the following functions from libdispatch on Mac OS:
+//   dispatch_async_f()
+//   dispatch_async()
+//   dispatch_sync_f()
+//   dispatch_sync()
+//   dispatch_after_f()
+//   dispatch_after()
+//   dispatch_group_async_f()
+//   dispatch_group_async()
+// TODO(glider): libdispatch API contains other functions that we don't support
+// yet.
+//
+// dispatch_sync() and dispatch_sync_f() are synchronous, although chances are
+// they can cause jobs to run on a thread different from the current one.
+// TODO(glider): if so, we need a test for this (otherwise we should remove
+// them).
+//
+// The following functions use dispatch_barrier_async_f() (which isn't a library
+// function but is exported) and are thus supported:
+//   dispatch_source_set_cancel_handler_f()
+//   dispatch_source_set_cancel_handler()
+//   dispatch_source_set_event_handler_f()
+//   dispatch_source_set_event_handler()
+//
+// The reference manual for Grand Central Dispatch is available at
+//   http://developer.apple.com/library/mac/#documentation/Performance/Reference/GCD_libdispatch_Ref/Reference/reference.html
+// The implementation details are at
+//   http://libdispatch.macosforge.org/trac/browser/trunk/src/queue.c
+
+typedef void *dispatch_group_t;
+typedef void *dispatch_queue_t;
+typedef void *dispatch_source_t;
+typedef u64 dispatch_time_t;
+typedef void (*dispatch_function_t)(void *block);
+typedef void *(*worker_t)(void *block);
+
+// A wrapper for the ObjC blocks used to support libdispatch.
+typedef struct {
+  void *block;
+  dispatch_function_t func;
+  u32 parent_tid;
+} lsan_block_context_t;
+
+ALWAYS_INLINE
+void lsan_register_worker_thread(int parent_tid) {
+  if (GetCurrentThread() == kInvalidTid) {
+    u32 tid = ThreadCreate(parent_tid, 0, true);
+    ThreadStart(tid, GetTid());
+    SetCurrentThread(tid);
+  }
+}
+
+// For use by only those functions that allocated the context via
+// alloc_lsan_context().
+extern "C" void lsan_dispatch_call_block_and_release(void *block) {
+  lsan_block_context_t *context = (lsan_block_context_t *)block;
+  VReport(2,
+          "lsan_dispatch_call_block_and_release(): "
+          "context: %p, pthread_self: %p\n",
+          block, pthread_self());
+  lsan_register_worker_thread(context->parent_tid);
+  // Call the original dispatcher for the block.
+  context->func(context->block);
+  lsan_free(context);
+}
+
+}  // namespace __lsan
+
+using namespace __lsan;  // NOLINT
+
+// Wrap |ctxt| and |func| into an lsan_block_context_t.
+// The caller retains control of the allocated context.
+extern "C" lsan_block_context_t *alloc_lsan_context(void *ctxt,
+                                                    dispatch_function_t func) {
+  GET_STACK_TRACE_THREAD;
+  lsan_block_context_t *lsan_ctxt =
+      (lsan_block_context_t *)lsan_malloc(sizeof(lsan_block_context_t), stack);
+  lsan_ctxt->block = ctxt;
+  lsan_ctxt->func = func;
+  lsan_ctxt->parent_tid = GetCurrentThread();
+  return lsan_ctxt;
+}
+
+// Define interceptor for dispatch_*_f function with the three most common
+// parameters: dispatch_queue_t, context, dispatch_function_t.
+#define INTERCEPT_DISPATCH_X_F_3(dispatch_x_f)                        \
+  INTERCEPTOR(void, dispatch_x_f, dispatch_queue_t dq, void *ctxt,    \
+              dispatch_function_t func) {                             \
+    lsan_block_context_t *lsan_ctxt = alloc_lsan_context(ctxt, func); \
+    return REAL(dispatch_x_f)(dq, (void *)lsan_ctxt,                  \
+                              lsan_dispatch_call_block_and_release);  \
+  }
+
+INTERCEPT_DISPATCH_X_F_3(dispatch_async_f)
+INTERCEPT_DISPATCH_X_F_3(dispatch_sync_f)
+INTERCEPT_DISPATCH_X_F_3(dispatch_barrier_async_f)
+
+INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, dispatch_queue_t dq,
+            void *ctxt, dispatch_function_t func) {
+  lsan_block_context_t *lsan_ctxt = alloc_lsan_context(ctxt, func);
+  return REAL(dispatch_after_f)(when, dq, (void *)lsan_ctxt,
+                                lsan_dispatch_call_block_and_release);
+}
+
+INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
+            dispatch_queue_t dq, void *ctxt, dispatch_function_t func) {
+  lsan_block_context_t *lsan_ctxt = alloc_lsan_context(ctxt, func);
+  REAL(dispatch_group_async_f)
+  (group, dq, (void *)lsan_ctxt, lsan_dispatch_call_block_and_release);
+}
+
+#if !defined(MISSING_BLOCKS_SUPPORT)
+extern "C" {
+void dispatch_async(dispatch_queue_t dq, void (^work)(void));
+void dispatch_group_async(dispatch_group_t dg, dispatch_queue_t dq,
+                          void (^work)(void));
+void dispatch_after(dispatch_time_t when, dispatch_queue_t queue,
+                    void (^work)(void));
+void dispatch_source_set_cancel_handler(dispatch_source_t ds,
+                                        void (^work)(void));
+void dispatch_source_set_event_handler(dispatch_source_t ds,
+                                       void (^work)(void));
+}
+
+#define GET_LSAN_BLOCK(work)                 \
+  void (^lsan_block)(void);                  \
+  int parent_tid = GetCurrentThread();       \
+  lsan_block = ^(void) {                     \
+    lsan_register_worker_thread(parent_tid); \
+    work();                                  \
+  }
+
+INTERCEPTOR(void, dispatch_async, dispatch_queue_t dq, void (^work)(void)) {
+  GET_LSAN_BLOCK(work);
+  REAL(dispatch_async)(dq, lsan_block);
+}
+
+INTERCEPTOR(void, dispatch_group_async, dispatch_group_t dg,
+            dispatch_queue_t dq, void (^work)(void)) {
+  GET_LSAN_BLOCK(work);
+  REAL(dispatch_group_async)(dg, dq, lsan_block);
+}
+
+INTERCEPTOR(void, dispatch_after, dispatch_time_t when, dispatch_queue_t queue,
+            void (^work)(void)) {
+  GET_LSAN_BLOCK(work);
+  REAL(dispatch_after)(when, queue, lsan_block);
+}
+
+INTERCEPTOR(void, dispatch_source_set_cancel_handler, dispatch_source_t ds,
+            void (^work)(void)) {
+  if (!work) {
+    REAL(dispatch_source_set_cancel_handler)(ds, work);
+    return;
+  }
+  GET_LSAN_BLOCK(work);
+  REAL(dispatch_source_set_cancel_handler)(ds, lsan_block);
+}
+
+INTERCEPTOR(void, dispatch_source_set_event_handler, dispatch_source_t ds,
+            void (^work)(void)) {
+  GET_LSAN_BLOCK(work);
+  REAL(dispatch_source_set_event_handler)(ds, lsan_block);
+}
+#endif
+
+#endif  // SANITIZER_MAC
diff --git a/libsanitizer/lsan/lsan_malloc_mac.cc b/libsanitizer/lsan/lsan_malloc_mac.cc
new file mode 100644
index 00000000000..2d810af841f
--- /dev/null
+++ b/libsanitizer/lsan/lsan_malloc_mac.cc
@@ -0,0 +1,53 @@
+//===-- lsan_malloc_mac.cc ------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer (LSan), a memory leak detector.
+//
+// Mac-specific malloc interception.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "lsan.h"
+#include "lsan_allocator.h"
+#include "lsan_thread.h"
+
+using namespace __lsan;
+#define COMMON_MALLOC_ZONE_NAME "lsan"
+#define COMMON_MALLOC_ENTER() ENSURE_LSAN_INITED
+#define COMMON_MALLOC_SANITIZER_INITIALIZED lsan_inited
+#define COMMON_MALLOC_FORCE_LOCK()
+#define COMMON_MALLOC_FORCE_UNLOCK()
+#define COMMON_MALLOC_MEMALIGN(alignment, size) \
+  GET_STACK_TRACE_MALLOC; \
+  void *p = lsan_memalign(alignment, size, stack)
+#define COMMON_MALLOC_MALLOC(size) \
+  GET_STACK_TRACE_MALLOC; \
+  void *p = lsan_malloc(size, stack)
+#define COMMON_MALLOC_REALLOC(ptr, size) \
+  GET_STACK_TRACE_MALLOC; \
+  void *p = lsan_realloc(ptr, size, stack)
+#define COMMON_MALLOC_CALLOC(count, size) \
+  GET_STACK_TRACE_MALLOC; \
+  void *p = lsan_calloc(count, size, stack)
+#define COMMON_MALLOC_VALLOC(size) \
+  GET_STACK_TRACE_MALLOC; \
+  void *p = lsan_valloc(size, stack)
+#define COMMON_MALLOC_FREE(ptr) \
+  lsan_free(ptr)
+#define COMMON_MALLOC_SIZE(ptr) \
+  uptr size = lsan_mz_size(ptr)
+#define COMMON_MALLOC_FILL_STATS(zone, stats)
+#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \
+  (void)zone_name; \
+  Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr);
+#define COMMON_MALLOC_NAMESPACE __lsan
+
+#include "sanitizer_common/sanitizer_malloc_mac.inc"
+
+#endif // SANITIZER_MAC
diff --git a/libsanitizer/lsan/lsan_thread.cc b/libsanitizer/lsan/lsan_thread.cc
index af5ad47913f..e03e8766ae1 100644
--- a/libsanitizer/lsan/lsan_thread.cc
+++ b/libsanitizer/lsan/lsan_thread.cc
@@ -17,13 +17,11 @@
 #include "sanitizer_common/sanitizer_thread_registry.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 #include "lsan_allocator.h"
+#include "lsan_common.h"
 
 namespace __lsan {
 
-const u32 kInvalidTid = (u32) -1;
-
 static ThreadRegistry *thread_registry;
-static THREADLOCAL u32 current_thread_tid = kInvalidTid;
 
 static ThreadContextBase *CreateThreadContext(u32 tid) {
   void *mem = MmapOrDie(sizeof(ThreadContext), "ThreadContext");
@@ -39,14 +37,6 @@ void InitializeThreadRegistry() {
     ThreadRegistry(CreateThreadContext, kMaxThreads, kThreadQuarantineSize);
 }
 
-u32 GetCurrentThread() {
-  return current_thread_tid;
-}
-
-void SetCurrentThread(u32 tid) {
-  current_thread_tid = tid;
-}
-
 ThreadContext::ThreadContext(int tid)
     : ThreadContextBase(tid),
       stack_begin_(0),
@@ -85,7 +75,7 @@ u32 ThreadCreate(u32 parent_tid, uptr user_id, bool detached) {
                                        /* arg */ nullptr);
 }
 
-void ThreadStart(u32 tid, uptr os_id) {
+void ThreadStart(u32 tid, tid_t os_id, bool workerthread) {
   OnStartedArgs args;
   uptr stack_size = 0;
   uptr tls_size = 0;
@@ -95,11 +85,12 @@ void ThreadStart(u32 tid, uptr os_id) {
   args.tls_end = args.tls_begin + tls_size;
   GetAllocatorCacheRange(&args.cache_begin, &args.cache_end);
   args.dtls = DTLS_Get();
-  thread_registry->StartThread(tid, os_id, &args);
+  thread_registry->StartThread(tid, os_id, workerthread, &args);
 }
 
 void ThreadFinish() {
   thread_registry->FinishThread(GetCurrentThread());
+  SetCurrentThread(kInvalidTid);
 }
 
 ThreadContext *CurrentThreadContext() {
@@ -134,7 +125,7 @@ void EnsureMainThreadIDIsCorrect() {
 
 ///// Interface to the common LSan module. /////
 
-bool GetThreadRangesLocked(uptr os_id, uptr *stack_begin, uptr *stack_end,
+bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
                            uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
                            uptr *cache_end, DTLS **dtls) {
   ThreadContext *context = static_cast<ThreadContext *>(
@@ -150,7 +141,7 @@ bool GetThreadRangesLocked(uptr os_id, uptr *stack_begin, uptr *stack_end,
   return true;
 }
 
-void ForEachExtraStackRange(uptr os_id, RangeIteratorCallback callback,
+void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
                             void *arg) {
 }
 
diff --git a/libsanitizer/lsan/lsan_thread.h b/libsanitizer/lsan/lsan_thread.h
index dafd8af0a29..86758347432 100644
--- a/libsanitizer/lsan/lsan_thread.h
+++ b/libsanitizer/lsan/lsan_thread.h
@@ -43,7 +43,7 @@ class ThreadContext : public ThreadContextBase {
 
 void InitializeThreadRegistry();
 
-void ThreadStart(u32 tid, uptr os_id);
+void ThreadStart(u32 tid, tid_t os_id, bool workerthread = false);
 void ThreadFinish();
 u32 ThreadCreate(u32 tid, uptr uid, bool detached);
 void ThreadJoin(u32 tid);
diff --git a/libsanitizer/sanitizer_common/Makefile.am b/libsanitizer/sanitizer_common/Makefile.am
index 92c8419f33d..adaab4cee54 100644
--- a/libsanitizer/sanitizer_common/Makefile.am
+++ b/libsanitizer/sanitizer_common/Makefile.am
@@ -19,13 +19,16 @@ ACLOCAL_AMFLAGS = -I m4
 noinst_LTLIBRARIES = libsanitizer_common.la
 
 sanitizer_common_files = \
+	sancov_flags.cc \
 	sanitizer_allocator.cc \
+	sanitizer_allocator_checks.cc \
 	sanitizer_common.cc \
 	sanitizer_common_libcdep.cc \
-	sanitizer_coverage_libcdep.cc \
-	sanitizer_coverage_mapping_libcdep.cc \
+	sanitizer_coverage_libcdep_new.cc \
 	sanitizer_deadlock_detector1.cc \
 	sanitizer_deadlock_detector2.cc \
+	sanitizer_errno.cc \
+	sanitizer_file.cc \
 	sanitizer_flags.cc \
 	sanitizer_flag_parser.cc \
 	sanitizer_libc.cc \
@@ -34,6 +37,7 @@ sanitizer_common_files = \
 	sanitizer_linux_libcdep.cc \
 	sanitizer_linux_s390.cc \
 	sanitizer_mac.cc \
+	sanitizer_mac_libcdep.cc \
 	sanitizer_persistent_allocator.cc \
 	sanitizer_platform_limits_linux.cc \
 	sanitizer_platform_limits_posix.cc \
@@ -50,6 +54,7 @@ sanitizer_common_files = \
 	sanitizer_symbolizer_mac.cc \
 	sanitizer_stacktrace_printer.cc \
 	sanitizer_stoptheworld_linux_libcdep.cc \
+	sanitizer_stoptheworld_mac.cc \
 	sanitizer_suppressions.cc \
 	sanitizer_symbolizer.cc \
 	sanitizer_symbolizer_libbacktrace.cc \
diff --git a/libsanitizer/sanitizer_common/Makefile.in b/libsanitizer/sanitizer_common/Makefile.in
index c375f63a380..b2acc5caf56 100644
--- a/libsanitizer/sanitizer_common/Makefile.in
+++ b/libsanitizer/sanitizer_common/Makefile.in
@@ -80,15 +80,16 @@ CONFIG_CLEAN_FILES =
 CONFIG_CLEAN_VPATH_FILES =
 LTLIBRARIES = $(noinst_LTLIBRARIES)
 am__DEPENDENCIES_1 =
-am__objects_1 = sanitizer_allocator.lo sanitizer_common.lo \
-	sanitizer_common_libcdep.lo sanitizer_coverage_libcdep.lo \
-	sanitizer_coverage_mapping_libcdep.lo \
+am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \
+	sanitizer_allocator_checks.lo sanitizer_common.lo \
+	sanitizer_common_libcdep.lo sanitizer_coverage_libcdep_new.lo \
 	sanitizer_deadlock_detector1.lo \
-	sanitizer_deadlock_detector2.lo sanitizer_flags.lo \
-	sanitizer_flag_parser.lo sanitizer_libc.lo \
-	sanitizer_libignore.lo sanitizer_linux.lo \
+	sanitizer_deadlock_detector2.lo sanitizer_errno.lo \
+	sanitizer_file.lo sanitizer_flags.lo sanitizer_flag_parser.lo \
+	sanitizer_libc.lo sanitizer_libignore.lo sanitizer_linux.lo \
 	sanitizer_linux_libcdep.lo sanitizer_linux_s390.lo \
-	sanitizer_mac.lo sanitizer_persistent_allocator.lo \
+	sanitizer_mac.lo sanitizer_mac_libcdep.lo \
+	sanitizer_persistent_allocator.lo \
 	sanitizer_platform_limits_linux.lo \
 	sanitizer_platform_limits_posix.lo sanitizer_posix.lo \
 	sanitizer_posix_libcdep.lo sanitizer_printf.lo \
@@ -98,8 +99,8 @@ am__objects_1 = sanitizer_allocator.lo sanitizer_common.lo \
 	sanitizer_stacktrace_libcdep.lo sanitizer_symbolizer_mac.lo \
 	sanitizer_stacktrace_printer.lo \
 	sanitizer_stoptheworld_linux_libcdep.lo \
-	sanitizer_suppressions.lo sanitizer_symbolizer.lo \
-	sanitizer_symbolizer_libbacktrace.lo \
+	sanitizer_stoptheworld_mac.lo sanitizer_suppressions.lo \
+	sanitizer_symbolizer.lo sanitizer_symbolizer_libbacktrace.lo \
 	sanitizer_symbolizer_libcdep.lo \
 	sanitizer_symbolizer_posix_libcdep.lo \
 	sanitizer_symbolizer_win.lo sanitizer_termination.lo \
@@ -300,13 +301,16 @@ AM_CXXFLAGS = -Wall -W -Wno-unused-parameter -Wwrite-strings -pedantic \
 ACLOCAL_AMFLAGS = -I m4
 noinst_LTLIBRARIES = libsanitizer_common.la
 sanitizer_common_files = \
+	sancov_flags.cc \
 	sanitizer_allocator.cc \
+	sanitizer_allocator_checks.cc \
 	sanitizer_common.cc \
 	sanitizer_common_libcdep.cc \
-	sanitizer_coverage_libcdep.cc \
-	sanitizer_coverage_mapping_libcdep.cc \
+	sanitizer_coverage_libcdep_new.cc \
 	sanitizer_deadlock_detector1.cc \
 	sanitizer_deadlock_detector2.cc \
+	sanitizer_errno.cc \
+	sanitizer_file.cc \
 	sanitizer_flags.cc \
 	sanitizer_flag_parser.cc \
 	sanitizer_libc.cc \
@@ -315,6 +319,7 @@ sanitizer_common_files = \
 	sanitizer_linux_libcdep.cc \
 	sanitizer_linux_s390.cc \
 	sanitizer_mac.cc \
+	sanitizer_mac_libcdep.cc \
 	sanitizer_persistent_allocator.cc \
 	sanitizer_platform_limits_linux.cc \
 	sanitizer_platform_limits_posix.cc \
@@ -331,6 +336,7 @@ sanitizer_common_files = \
 	sanitizer_symbolizer_mac.cc \
 	sanitizer_stacktrace_printer.cc \
 	sanitizer_stoptheworld_linux_libcdep.cc \
+	sanitizer_stoptheworld_mac.cc \
 	sanitizer_suppressions.cc \
 	sanitizer_symbolizer.cc \
 	sanitizer_symbolizer_libbacktrace.cc \
@@ -439,13 +445,16 @@ mostlyclean-compile:
 distclean-compile:
 	-rm -f *.tab.c
 
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sancov_flags.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_allocator.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_allocator_checks.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_common.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_common_libcdep.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_coverage_libcdep.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_coverage_mapping_libcdep.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_coverage_libcdep_new.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_deadlock_detector1.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_deadlock_detector2.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_errno.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_file.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_flag_parser.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_flags.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_libc.Plo@am__quote@
@@ -456,6 +465,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_s390.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_x86_64.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mac.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mac_libcdep.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_persistent_allocator.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_platform_limits_linux.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_platform_limits_posix.Plo@am__quote@
@@ -471,6 +481,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stacktrace_libcdep.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stacktrace_printer.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stoptheworld_linux_libcdep.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stoptheworld_mac.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_suppressions.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_symbolizer.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_symbolizer_libbacktrace.Plo@am__quote@
diff --git a/libsanitizer/sanitizer_common/sancov_flags.cc b/libsanitizer/sanitizer_common/sancov_flags.cc
new file mode 100644
index 00000000000..e600cdac33c
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sancov_flags.cc
@@ -0,0 +1,57 @@
+//===-- sancov_flags.cc -----------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Sanitizer Coverage runtime flags.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sancov_flags.h"
+#include "sanitizer_flag_parser.h"
+#include "sanitizer_platform.h"
+
+SANITIZER_INTERFACE_WEAK_DEF(const char*, __sancov_default_options, void) {
+  return "";
+}
+
+using namespace __sanitizer;
+
+namespace __sancov {
+
+SancovFlags sancov_flags_dont_use_directly;  // use via flags();
+
+void SancovFlags::SetDefaults() {
+#define SANCOV_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "sancov_flags.inc"
+#undef SANCOV_FLAG
+}
+
+static void RegisterSancovFlags(FlagParser *parser, SancovFlags *f) {
+#define SANCOV_FLAG(Type, Name, DefaultValue, Description) \
+  RegisterFlag(parser, #Name, Description, &f->Name);
+#include "sancov_flags.inc"
+#undef SANCOV_FLAG
+}
+
+static const char *MaybeCallSancovDefaultOptions() {
+  return (&__sancov_default_options) ? __sancov_default_options() : "";
+}
+
+void InitializeSancovFlags() {
+  SancovFlags *f = sancov_flags();
+  f->SetDefaults();
+
+  FlagParser parser;
+  RegisterSancovFlags(&parser, f);
+
+  parser.ParseString(MaybeCallSancovDefaultOptions());
+  parser.ParseString(GetEnv("SANCOV_OPTIONS"));
+
+  ReportUnrecognizedFlags();
+  if (f->help) parser.PrintFlagDescriptions();
+}
+
+}  // namespace __sancov
diff --git a/libsanitizer/sanitizer_common/sancov_flags.h b/libsanitizer/sanitizer_common/sancov_flags.h
new file mode 100644
index 00000000000..f7e0186c511
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sancov_flags.h
@@ -0,0 +1,38 @@
+//===-- sancov_flags.h ------------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Sanitizer Coverage runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANCOV_FLAGS_H
+#define SANCOV_FLAGS_H
+
+#include "sanitizer_flag_parser.h"
+#include "sanitizer_internal_defs.h"
+
+namespace __sancov {
+
+struct SancovFlags {
+#define SANCOV_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "sancov_flags.inc"
+#undef SANCOV_FLAG
+
+  void SetDefaults();
+};
+
+extern SancovFlags sancov_flags_dont_use_directly;
+
+inline SancovFlags* sancov_flags() { return &sancov_flags_dont_use_directly; }
+
+void InitializeSancovFlags();
+
+}  // namespace __sancov
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char*
+__sancov_default_options();
+
+#endif
diff --git a/libsanitizer/sanitizer_common/sancov_flags.inc b/libsanitizer/sanitizer_common/sancov_flags.inc
new file mode 100644
index 00000000000..a6107cc8583
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sancov_flags.inc
@@ -0,0 +1,19 @@
+//===-- sancov_flags.inc ----------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Sanitizer Coverage runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANCOV_FLAG
+#error "Defnine SANCOV_FLAG prior to including this file!"
+#endif
+
+SANCOV_FLAG(bool, symbolize, true,
+            "If set, converage information will be symbolized by sancov tool "
+            "after dumping.")
+
+SANCOV_FLAG(bool, help, false, "Print flags help.")
diff --git a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
index 3bc40ef64f0..a67ec84373e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
+++ b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
@@ -71,6 +71,8 @@ class AddrHashMap {
 
     ~Handle();
     T *operator->();
+    T &operator*();
+    const T &operator*() const;
     bool created() const;
     bool exists() const;
 
@@ -134,6 +136,16 @@ T *AddrHashMap<T, kSize>::Handle::operator->() {
   return &cell_->val;
 }
 
+template <typename T, uptr kSize>
+const T &AddrHashMap<T, kSize>::Handle::operator*() const {
+  return cell_->val;
+}
+
+template <typename T, uptr kSize>
+T &AddrHashMap<T, kSize>::Handle::operator*() {
+  return cell_->val;
+}
+
 template<typename T, uptr kSize>
 bool AddrHashMap<T, kSize>::Handle::created() const {
   return created_;
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.cc b/libsanitizer/sanitizer_common/sanitizer_allocator.cc
index 2755853acbc..895efcf1be0 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator.cc
@@ -12,6 +12,7 @@
 
 #include "sanitizer_allocator.h"
 
+#include "sanitizer_allocator_checks.h"
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_atomic.h"
 #include "sanitizer_common.h"
@@ -92,7 +93,7 @@ InternalAllocator *internal_allocator() {
     SpinMutexLock l(&internal_alloc_init_mu);
     if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) ==
         0) {
-      internal_allocator_instance->Init(/* may_return_null*/ false);
+      internal_allocator_instance->Init(kReleaseToOSIntervalNever);
       atomic_store(&internal_allocator_initialized, 1, memory_order_release);
     }
   }
@@ -105,9 +106,9 @@ static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
   if (cache == 0) {
     SpinMutexLock l(&internal_allocator_cache_mu);
     return internal_allocator()->Allocate(&internal_allocator_cache, size,
-                                          alignment, false);
+                                          alignment);
   }
-  return internal_allocator()->Allocate(cache, size, alignment, false);
+  return internal_allocator()->Allocate(cache, size, alignment);
 }
 
 static void *RawInternalRealloc(void *ptr, uptr size,
@@ -158,8 +159,8 @@ void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
 }
 
 void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
-  if (CallocShouldReturnNullDueToOverflow(count, size))
-    return internal_allocator()->ReturnNullOrDieOnBadRequest();
+  if (UNLIKELY(CheckForCallocOverflow(count, size)))
+    return InternalAllocator::FailureHandler::OnBadRequest();
   void *p = InternalAlloc(count * size, cache);
   if (p) internal_memset(p, 0, count * size);
   return p;
@@ -200,18 +201,15 @@ void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) {
   low_level_alloc_callback = callback;
 }
 
-bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) {
-  if (!size) return false;
-  uptr max = (uptr)-1L;
-  return (max / size) < n;
-}
-
-static atomic_uint8_t reporting_out_of_memory = {0};
+static atomic_uint8_t allocator_out_of_memory = {0};
+static atomic_uint8_t allocator_may_return_null = {0};
 
-bool IsReportingOOM() { return atomic_load_relaxed(&reporting_out_of_memory); }
+bool IsAllocatorOutOfMemory() {
+  return atomic_load_relaxed(&allocator_out_of_memory);
+}
 
-void NORETURN ReportAllocatorCannotReturnNull(bool out_of_memory) {
-  if (out_of_memory) atomic_store_relaxed(&reporting_out_of_memory, 1);
+// Prints error message and kills the program.
+void NORETURN ReportAllocatorCannotReturnNull() {
   Report("%s's allocator is terminating the process instead of returning 0\n",
          SanitizerToolName);
   Report("If you don't like this behavior set allocator_may_return_null=1\n");
@@ -219,4 +217,35 @@ void NORETURN ReportAllocatorCannotReturnNull(bool out_of_memory) {
   Die();
 }
 
+bool AllocatorMayReturnNull() {
+  return atomic_load(&allocator_may_return_null, memory_order_relaxed);
+}
+
+void SetAllocatorMayReturnNull(bool may_return_null) {
+  atomic_store(&allocator_may_return_null, may_return_null,
+               memory_order_relaxed);
+}
+
+void *ReturnNullOrDieOnFailure::OnBadRequest() {
+  if (AllocatorMayReturnNull())
+    return nullptr;
+  ReportAllocatorCannotReturnNull();
+}
+
+void *ReturnNullOrDieOnFailure::OnOOM() {
+  atomic_store_relaxed(&allocator_out_of_memory, 1);
+  if (AllocatorMayReturnNull())
+    return nullptr;
+  ReportAllocatorCannotReturnNull();
+}
+
+void NORETURN *DieOnFailure::OnBadRequest() {
+  ReportAllocatorCannotReturnNull();
+}
+
+void NORETURN *DieOnFailure::OnOOM() {
+  atomic_store_relaxed(&allocator_out_of_memory, 1);
+  ReportAllocatorCannotReturnNull();
+}
+
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.h b/libsanitizer/sanitizer_common/sanitizer_allocator.h
index ba50acddbbf..523578a9d60 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator.h
@@ -22,12 +22,28 @@
 
 namespace __sanitizer {
 
-// Returns true if ReportAllocatorCannotReturnNull(true) was called.
-// Can be use to avoid memory hungry operations.
-bool IsReportingOOM();
+// Since flags are immutable and allocator behavior can be changed at runtime
+// (unit tests or ASan on Android are some examples), allocator_may_return_null
+// flag value is cached here and can be altered later.
+bool AllocatorMayReturnNull();
+void SetAllocatorMayReturnNull(bool may_return_null);
 
-// Prints error message and kills the program.
-void NORETURN ReportAllocatorCannotReturnNull(bool out_of_memory);
+// Allocator failure handling policies:
+// Implements AllocatorMayReturnNull policy, returns null when the flag is set,
+// dies otherwise.
+struct ReturnNullOrDieOnFailure {
+  static void *OnBadRequest();
+  static void *OnOOM();
+};
+// Always dies on the failure.
+struct DieOnFailure {
+  static void NORETURN *OnBadRequest();
+  static void NORETURN *OnOOM();
+};
+
+// Returns true if allocator detected OOM condition. Can be used to avoid memory
+// hungry operations. Set when AllocatorReturnNullOrDieOnOOM() is called.
+bool IsAllocatorOutOfMemory();
 
 // Allocators call these callbacks on mmap/munmap.
 struct NoOpMapUnmapCallback {
@@ -38,9 +54,6 @@ struct NoOpMapUnmapCallback {
 // Callback type for iterating over chunks.
 typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
 
-// Returns true if calloc(size, n) should return 0 due to overflow in size*n.
-bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n);
-
 #include "sanitizer_allocator_size_class_map.h"
 #include "sanitizer_allocator_stats.h"
 #include "sanitizer_allocator_primary64.h"
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_checks.cc b/libsanitizer/sanitizer_common/sanitizer_allocator_checks.cc
new file mode 100644
index 00000000000..3e6eb61a7d5
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_checks.cc
@@ -0,0 +1,21 @@
+//===-- sanitizer_allocator_checks.cc ---------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory
+// allocators.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_errno.h"
+
+namespace __sanitizer {
+
+void SetErrnoToENOMEM() {
+  errno = errno_ENOMEM;
+}
+
+} // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_checks.h b/libsanitizer/sanitizer_common/sanitizer_allocator_checks.h
new file mode 100644
index 00000000000..978813222b5
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_checks.h
@@ -0,0 +1,73 @@
+//===-- sanitizer_allocator_checks.h ----------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory
+// allocators.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_CHECKS_H
+#define SANITIZER_ALLOCATOR_CHECKS_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_common.h"
+#include "sanitizer_platform.h"
+
+namespace __sanitizer {
+
+// The following is defined in a separate compilation unit to avoid pulling in
+// sanitizer_errno.h in this header, which leads to conflicts when other system
+// headers include errno.h. This is usually the result of an unlikely event,
+// and as such we do not care as much about having it inlined.
+void SetErrnoToENOMEM();
+
+// A common errno setting logic shared by almost all sanitizer allocator APIs.
+INLINE void *SetErrnoOnNull(void *ptr) {
+  if (UNLIKELY(!ptr))
+    SetErrnoToENOMEM();
+  return ptr;
+}
+
+// In case of the check failure, the caller of the following Check... functions
+// should "return POLICY::OnBadRequest();" where POLICY is the current allocator
+// failure handling policy.
+
+// Checks aligned_alloc() parameters, verifies that the alignment is a power of
+// two and that the size is a multiple of alignment for POSIX implementation,
+// and a bit relaxed requirement for non-POSIX ones, that the size is a multiple
+// of alignment.
+INLINE bool CheckAlignedAllocAlignmentAndSize(uptr alignment, uptr size) {
+#if SANITIZER_POSIX
+  return IsPowerOfTwo(alignment) && (size & (alignment - 1)) == 0;
+#else
+  return size % alignment == 0;
+#endif
+}
+
+// Checks posix_memalign() parameters, verifies that alignment is a power of two
+// and a multiple of sizeof(void *).
+INLINE bool CheckPosixMemalignAlignment(uptr alignment) {
+  return IsPowerOfTwo(alignment) && (alignment % sizeof(void *)) == 0; // NOLINT
+}
+
+// Returns true if calloc(size, n) call overflows on size*n calculation.
+INLINE bool CheckForCallocOverflow(uptr size, uptr n) {
+  if (!size)
+    return false;
+  uptr max = (uptr)-1L;
+  return (max / size) < n;
+}
+
+// Returns true if the size passed to pvalloc overflows when rounded to the next
+// multiple of page_size.
+INLINE bool CheckForPvallocOverflow(uptr size, uptr page_size) {
+  return RoundUpTo(size, page_size) < size;
+}
+
+} // namespace __sanitizer
+
+#endif  // SANITIZER_ALLOCATOR_CHECKS_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
index 4dc9ca7401f..99e411f4378 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
@@ -22,72 +22,57 @@ template <class PrimaryAllocator, class AllocatorCache,
           class SecondaryAllocator>  // NOLINT
 class CombinedAllocator {
  public:
-  void InitCommon(bool may_return_null) {
-    primary_.Init();
-    atomic_store(&may_return_null_, may_return_null, memory_order_relaxed);
-  }
+  typedef typename SecondaryAllocator::FailureHandler FailureHandler;
 
-  void InitLinkerInitialized(bool may_return_null) {
-    secondary_.InitLinkerInitialized(may_return_null);
+  void InitLinkerInitialized(s32 release_to_os_interval_ms) {
+    primary_.Init(release_to_os_interval_ms);
+    secondary_.InitLinkerInitialized();
     stats_.InitLinkerInitialized();
-    InitCommon(may_return_null);
   }
 
-  void Init(bool may_return_null) {
-    secondary_.Init(may_return_null);
+  void Init(s32 release_to_os_interval_ms) {
+    primary_.Init(release_to_os_interval_ms);
+    secondary_.Init();
     stats_.Init();
-    InitCommon(may_return_null);
   }
 
-  void *Allocate(AllocatorCache *cache, uptr size, uptr alignment,
-                 bool cleared = false, bool check_rss_limit = false) {
+  void *Allocate(AllocatorCache *cache, uptr size, uptr alignment) {
     // Returning 0 on malloc(0) may break a lot of code.
     if (size == 0)
       size = 1;
-    if (size + alignment < size) return ReturnNullOrDieOnBadRequest();
-    if (check_rss_limit && RssLimitIsExceeded()) return ReturnNullOrDieOnOOM();
+    if (size + alignment < size)
+      return FailureHandler::OnBadRequest();
+    uptr original_size = size;
+    // If alignment requirements are to be fulfilled by the frontend allocator
+    // rather than by the primary or secondary, passing an alignment lower than
+    // or equal to 8 will prevent any further rounding up, as well as the later
+    // alignment check.
     if (alignment > 8)
       size = RoundUpTo(size, alignment);
+    // The primary allocator should return a 2^x aligned allocation when
+    // requested 2^x bytes, hence using the rounded up 'size' when being
+    // serviced by the primary (this is no longer true when the primary is
+    // using a non-fixed base address). The secondary takes care of the
+    // alignment without such requirement, and allocating 'size' would use
+    // extraneous memory, so we employ 'original_size'.
     void *res;
-    bool from_primary = primary_.CanAllocate(size, alignment);
-    if (from_primary)
+    if (primary_.CanAllocate(size, alignment))
       res = cache->Allocate(&primary_, primary_.ClassID(size));
     else
-      res = secondary_.Allocate(&stats_, size, alignment);
+      res = secondary_.Allocate(&stats_, original_size, alignment);
+    if (!res)
+      return FailureHandler::OnOOM();
     if (alignment > 8)
       CHECK_EQ(reinterpret_cast<uptr>(res) & (alignment - 1), 0);
-    if (cleared && res && from_primary)
-      internal_bzero_aligned16(res, RoundUpTo(size, 16));
     return res;
   }
 
-  bool MayReturnNull() const {
-    return atomic_load(&may_return_null_, memory_order_acquire);
-  }
-
-  void *ReturnNullOrDieOnBadRequest() {
-    if (MayReturnNull())
-      return nullptr;
-    ReportAllocatorCannotReturnNull(false);
-  }
-
-  void *ReturnNullOrDieOnOOM() {
-    if (MayReturnNull()) return nullptr;
-    ReportAllocatorCannotReturnNull(true);
+  s32 ReleaseToOSIntervalMs() const {
+    return primary_.ReleaseToOSIntervalMs();
   }
 
-  void SetMayReturnNull(bool may_return_null) {
-    secondary_.SetMayReturnNull(may_return_null);
-    atomic_store(&may_return_null_, may_return_null, memory_order_release);
-  }
-
-  bool RssLimitIsExceeded() {
-    return atomic_load(&rss_limit_is_exceeded_, memory_order_acquire);
-  }
-
-  void SetRssLimitIsExceeded(bool rss_limit_is_exceeded) {
-    atomic_store(&rss_limit_is_exceeded_, rss_limit_is_exceeded,
-                 memory_order_release);
+  void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
+    primary_.SetReleaseToOSIntervalMs(release_to_os_interval_ms);
   }
 
   void Deallocate(AllocatorCache *cache, void *p) {
@@ -191,8 +176,6 @@ class CombinedAllocator {
     primary_.ForceUnlock();
   }
 
-  void ReleaseToOS() { primary_.ReleaseToOS(); }
-
   // Iterate over all existing chunks.
   // The allocator must be locked when calling this function.
   void ForEachChunk(ForEachChunkCallback callback, void *arg) {
@@ -204,6 +187,4 @@ class CombinedAllocator {
   PrimaryAllocator primary_;
   SecondaryAllocator secondary_;
   AllocatorGlobalStats stats_;
-  atomic_uint8_t may_return_null_;
-  atomic_uint8_t rss_limit_is_exceeded_;
 };
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h b/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h
index 166f8c22049..fa74f8ca640 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h
@@ -32,13 +32,12 @@ SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_install_malloc_and_free_hooks(
     void (*free_hook)(const void *));
 
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-    /* OPTIONAL */ void __sanitizer_malloc_hook(void *ptr, uptr size);
+    void __sanitizer_malloc_hook(void *ptr, uptr size);
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-    /* OPTIONAL */ void __sanitizer_free_hook(void *ptr);
+    void __sanitizer_free_hook(void *ptr);
 
-
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-    void __sanitizer_print_memory_profile(int top_percent);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_print_memory_profile(uptr top_percent, uptr max_number_of_contexts);
 }  // extern "C"
 
 #endif  // SANITIZER_ALLOCATOR_INTERFACE_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h b/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h
index 6a8da8f3584..f1aed0f3bec 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h
@@ -21,27 +21,32 @@ namespace __sanitizer {
 // purposes.
 typedef CompactSizeClassMap InternalSizeClassMap;
 
-static const uptr kInternalAllocatorSpace = 0;
-static const u64 kInternalAllocatorSize = SANITIZER_MMAP_RANGE_SIZE;
 static const uptr kInternalAllocatorRegionSizeLog = 20;
-#if SANITIZER_WORDSIZE == 32
 static const uptr kInternalAllocatorNumRegions =
-    kInternalAllocatorSize >> kInternalAllocatorRegionSizeLog;
+    SANITIZER_MMAP_RANGE_SIZE >> kInternalAllocatorRegionSizeLog;
+#if SANITIZER_WORDSIZE == 32
 typedef FlatByteMap<kInternalAllocatorNumRegions> ByteMap;
 #else
-static const uptr kInternalAllocatorNumRegions =
-    kInternalAllocatorSize >> kInternalAllocatorRegionSizeLog;
 typedef TwoLevelByteMap<(kInternalAllocatorNumRegions >> 12), 1 << 12> ByteMap;
 #endif
-typedef SizeClassAllocator32<
-    kInternalAllocatorSpace, kInternalAllocatorSize, 0, InternalSizeClassMap,
-    kInternalAllocatorRegionSizeLog, ByteMap> PrimaryInternalAllocator;
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = 0;
+  typedef InternalSizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = kInternalAllocatorRegionSizeLog;
+  typedef __sanitizer::ByteMap ByteMap;
+  typedef NoOpMapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryInternalAllocator;
 
 typedef SizeClassAllocatorLocalCache<PrimaryInternalAllocator>
     InternalAllocatorCache;
 
 typedef CombinedAllocator<PrimaryInternalAllocator, InternalAllocatorCache,
-                          LargeMmapAllocator<> > InternalAllocator;
+                          LargeMmapAllocator<NoOpMapUnmapCallback, DieOnFailure>
+                         > InternalAllocator;
 
 void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr,
                     uptr alignment = 0);
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h
index 40ef0781adb..99013e37f8a 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h
@@ -24,9 +24,6 @@ struct SizeClassAllocatorLocalCache
 template <class SizeClassAllocator>
 struct SizeClassAllocator64LocalCache {
   typedef SizeClassAllocator Allocator;
-  static const uptr kNumClasses = SizeClassAllocator::kNumClasses;
-  typedef typename Allocator::SizeClassMapT SizeClassMap;
-  typedef typename Allocator::CompactPtrT CompactPtrT;
 
   void Init(AllocatorGlobalStats *s) {
     stats_.Init();
@@ -43,10 +40,12 @@ struct SizeClassAllocator64LocalCache {
   void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
-    stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
-    if (UNLIKELY(c->count == 0))
-      Refill(c, allocator, class_id);
+    if (UNLIKELY(c->count == 0)) {
+      if (UNLIKELY(!Refill(c, allocator, class_id)))
+        return nullptr;
+    }
+    stats_.Add(AllocatorStatAllocated, c->class_size);
     CHECK_GT(c->count, 0);
     CompactPtrT chunk = c->chunks[--c->count];
     void *res = reinterpret_cast<void *>(allocator->CompactPtrToPointer(
@@ -60,8 +59,8 @@ struct SizeClassAllocator64LocalCache {
     // If the first allocator call on a new thread is a deallocation, then
     // max_count will be zero, leading to check failure.
     InitCache();
-    stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
     CHECK_NE(c->max_count, 0UL);
     if (UNLIKELY(c->count == c->max_count))
       Drain(c, allocator, class_id, c->max_count / 2);
@@ -72,38 +71,46 @@ struct SizeClassAllocator64LocalCache {
   }
 
   void Drain(SizeClassAllocator *allocator) {
-    for (uptr class_id = 0; class_id < kNumClasses; class_id++) {
-      PerClass *c = &per_class_[class_id];
+    for (uptr i = 0; i < kNumClasses; i++) {
+      PerClass *c = &per_class_[i];
       while (c->count > 0)
-        Drain(c, allocator, class_id, c->count);
+        Drain(c, allocator, i, c->count);
     }
   }
 
-  // private:
+ private:
+  typedef typename Allocator::SizeClassMapT SizeClassMap;
+  static const uptr kNumClasses = SizeClassMap::kNumClasses;
+  typedef typename Allocator::CompactPtrT CompactPtrT;
+
   struct PerClass {
     u32 count;
     u32 max_count;
+    uptr class_size;
     CompactPtrT chunks[2 * SizeClassMap::kMaxNumCachedHint];
   };
   PerClass per_class_[kNumClasses];
   AllocatorStats stats_;
 
   void InitCache() {
-    if (per_class_[1].max_count)
+    if (LIKELY(per_class_[1].max_count))
       return;
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
       c->max_count = 2 * SizeClassMap::MaxCachedHint(i);
+      c->class_size = Allocator::ClassIdToSize(i);
     }
   }
 
-  NOINLINE void Refill(PerClass *c, SizeClassAllocator *allocator,
+  NOINLINE bool Refill(PerClass *c, SizeClassAllocator *allocator,
                        uptr class_id) {
     InitCache();
-    uptr num_requested_chunks = SizeClassMap::MaxCachedHint(class_id);
-    allocator->GetFromAllocator(&stats_, class_id, c->chunks,
-                                num_requested_chunks);
+    uptr num_requested_chunks = c->max_count / 2;
+    if (UNLIKELY(!allocator->GetFromAllocator(&stats_, class_id, c->chunks,
+                                              num_requested_chunks)))
+      return false;
     c->count = num_requested_chunks;
+    return true;
   }
 
   NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator, uptr class_id,
@@ -122,7 +129,6 @@ template <class SizeClassAllocator>
 struct SizeClassAllocator32LocalCache {
   typedef SizeClassAllocator Allocator;
   typedef typename Allocator::TransferBatch TransferBatch;
-  static const uptr kNumClasses = SizeClassAllocator::kNumClasses;
 
   void Init(AllocatorGlobalStats *s) {
     stats_.Init();
@@ -130,6 +136,21 @@ struct SizeClassAllocator32LocalCache {
       s->Register(&stats_);
   }
 
+  // Returns a TransferBatch suitable for class_id.
+  TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
+                             TransferBatch *b) {
+    if (uptr batch_class_id = per_class_[class_id].batch_class_id)
+      return (TransferBatch*)Allocate(allocator, batch_class_id);
+    return b;
+  }
+
+  // Destroys TransferBatch b.
+  void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
+                    TransferBatch *b) {
+    if (uptr batch_class_id = per_class_[class_id].batch_class_id)
+      Deallocate(allocator, batch_class_id, b);
+  }
+
   void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) {
     Drain(allocator);
     if (s)
@@ -139,10 +160,12 @@ struct SizeClassAllocator32LocalCache {
   void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
-    stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
-    if (UNLIKELY(c->count == 0))
-      Refill(allocator, class_id);
+    if (UNLIKELY(c->count == 0)) {
+      if (UNLIKELY(!Refill(allocator, class_id)))
+        return nullptr;
+    }
+    stats_.Add(AllocatorStatAllocated, c->class_size);
     void *res = c->batch[--c->count];
     PREFETCH(c->batch[c->count - 1]);
     return res;
@@ -154,8 +177,8 @@ struct SizeClassAllocator32LocalCache {
     // If the first allocator call on a new thread is a deallocation, then
     // max_count will be zero, leading to check failure.
     InitCache();
-    stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
     CHECK_NE(c->max_count, 0UL);
     if (UNLIKELY(c->count == c->max_count))
       Drain(allocator, class_id);
@@ -163,72 +186,68 @@ struct SizeClassAllocator32LocalCache {
   }
 
   void Drain(SizeClassAllocator *allocator) {
-    for (uptr class_id = 0; class_id < kNumClasses; class_id++) {
-      PerClass *c = &per_class_[class_id];
+    for (uptr i = 0; i < kNumClasses; i++) {
+      PerClass *c = &per_class_[i];
       while (c->count > 0)
-        Drain(allocator, class_id);
+        Drain(allocator, i);
     }
   }
 
-  // private:
-  typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap;
+ private:
+  typedef typename Allocator::SizeClassMapT SizeClassMap;
+  static const uptr kBatchClassID = SizeClassMap::kBatchClassID;
+  static const uptr kNumClasses = SizeClassMap::kNumClasses;
+  // If kUseSeparateSizeClassForBatch is true, all TransferBatch objects are
+  // allocated from kBatchClassID size class (except for those that are needed
+  // for kBatchClassID itself). The goal is to have TransferBatches in a totally
+  // different region of RAM to improve security.
+  static const bool kUseSeparateSizeClassForBatch =
+      Allocator::kUseSeparateSizeClassForBatch;
+
   struct PerClass {
     uptr count;
     uptr max_count;
+    uptr class_size;
+    uptr batch_class_id;
     void *batch[2 * TransferBatch::kMaxNumCached];
   };
   PerClass per_class_[kNumClasses];
   AllocatorStats stats_;
 
   void InitCache() {
-    if (per_class_[1].max_count)
+    if (LIKELY(per_class_[1].max_count))
       return;
+    const uptr batch_class_id = SizeClassMap::ClassID(sizeof(TransferBatch));
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
-      c->max_count = 2 * TransferBatch::MaxCached(i);
+      uptr max_cached = TransferBatch::MaxCached(i);
+      c->max_count = 2 * max_cached;
+      c->class_size = Allocator::ClassIdToSize(i);
+      // Precompute the class id to use to store batches for the current class
+      // id. 0 means the class size is large enough to store a batch within one
+      // of the chunks. If using a separate size class, it will always be
+      // kBatchClassID, except for kBatchClassID itself.
+      if (kUseSeparateSizeClassForBatch) {
+        c->batch_class_id = (i == kBatchClassID) ? 0 : kBatchClassID;
+      } else {
+        c->batch_class_id = (c->class_size <
+          TransferBatch::AllocationSizeRequiredForNElements(max_cached)) ?
+              batch_class_id : 0;
+      }
     }
   }
 
-  // TransferBatch class is declared in SizeClassAllocator.
-  // We transfer chunks between central and thread-local free lists in batches.
-  // For small size classes we allocate batches separately.
-  // For large size classes we may use one of the chunks to store the batch.
-  // sizeof(TransferBatch) must be a power of 2 for more efficient allocation.
-  static uptr SizeClassForTransferBatch(uptr class_id) {
-    if (Allocator::ClassIdToSize(class_id) <
-        TransferBatch::AllocationSizeRequiredForNElements(
-            TransferBatch::MaxCached(class_id)))
-      return SizeClassMap::ClassID(sizeof(TransferBatch));
-    return 0;
-  }
-
-  // Returns a TransferBatch suitable for class_id.
-  // For small size classes allocates the batch from the allocator.
-  // For large size classes simply returns b.
-  TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
-                             TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
-      return (TransferBatch*)Allocate(allocator, batch_class_id);
-    return b;
-  }
-
-  // Destroys TransferBatch b.
-  // For small size classes deallocates b to the allocator.
-  // Does notthing for large size classes.
-  void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
-                    TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
-      Deallocate(allocator, batch_class_id, b);
-  }
-
-  NOINLINE void Refill(SizeClassAllocator *allocator, uptr class_id) {
+  NOINLINE bool Refill(SizeClassAllocator *allocator, uptr class_id) {
     InitCache();
     PerClass *c = &per_class_[class_id];
     TransferBatch *b = allocator->AllocateBatch(&stats_, this, class_id);
+    if (UNLIKELY(!b))
+      return false;
     CHECK_GT(b->Count(), 0);
     b->CopyToArray(c->batch);
     c->count = b->Count();
     DestroyBatch(class_id, allocator, b);
+    return true;
   }
 
   NOINLINE void Drain(SizeClassAllocator *allocator, uptr class_id) {
@@ -238,6 +257,10 @@ struct SizeClassAllocator32LocalCache {
     uptr first_idx_to_drain = c->count - cnt;
     TransferBatch *b = CreateBatch(
         class_id, allocator, (TransferBatch *)c->batch[first_idx_to_drain]);
+    // Failure to allocate a batch while releasing memory is non recoverable.
+    // TODO(alekseys): Figure out how to do it without allocating a new batch.
+    if (UNLIKELY(!b))
+      DieOnFailure::OnOOM();
     b->SetFromArray(allocator->GetRegionBeginBySizeClass(class_id),
                     &c->batch[first_idx_to_drain], cnt);
     c->count -= cnt;
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
index 989b87a1923..90a57dbb6cf 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
@@ -22,7 +22,8 @@ template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache;
 // be returned by MmapOrDie().
 //
 // Region:
-//   a result of a single call to MmapAlignedOrDie(kRegionSize, kRegionSize).
+//   a result of a single call to MmapAlignedOrDieOnFatalError(kRegionSize,
+//                                                             kRegionSize).
 // Since the regions are aligned by kRegionSize, there are exactly
 // kNumPossibleRegions possible regions in the address space and so we keep
 // a ByteMap possible_regions to store the size classes of each Region.
@@ -34,13 +35,30 @@ template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache;
 //
 // In order to avoid false sharing the objects of this class should be
 // chache-line aligned.
-template <const uptr kSpaceBeg, const u64 kSpaceSize,
-          const uptr kMetadataSize, class SizeClassMap,
-          const uptr kRegionSizeLog,
-          class ByteMap,
-          class MapUnmapCallback = NoOpMapUnmapCallback>
+
+struct SizeClassAllocator32FlagMasks {  //  Bit masks.
+  enum {
+    kRandomShuffleChunks = 1,
+    kUseSeparateSizeClassForBatch = 2,
+  };
+};
+
+template <class Params>
 class SizeClassAllocator32 {
  public:
+  static const uptr kSpaceBeg = Params::kSpaceBeg;
+  static const u64 kSpaceSize = Params::kSpaceSize;
+  static const uptr kMetadataSize = Params::kMetadataSize;
+  typedef typename Params::SizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = Params::kRegionSizeLog;
+  typedef typename Params::ByteMap ByteMap;
+  typedef typename Params::MapUnmapCallback MapUnmapCallback;
+
+  static const bool kRandomShuffleChunks = Params::kFlags &
+      SizeClassAllocator32FlagMasks::kRandomShuffleChunks;
+  static const bool kUseSeparateSizeClassForBatch = Params::kFlags &
+      SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch;
+
   struct TransferBatch {
     static const uptr kMaxNumCached = SizeClassMap::kMaxNumCachedHint - 2;
     void SetFromArray(uptr region_beg_unused, void *batch[], uptr count) {
@@ -77,24 +95,30 @@ class SizeClassAllocator32 {
 
   static const uptr kBatchSize = sizeof(TransferBatch);
   COMPILER_CHECK((kBatchSize & (kBatchSize - 1)) == 0);
-  COMPILER_CHECK(sizeof(TransferBatch) ==
-                 SizeClassMap::kMaxNumCachedHint * sizeof(uptr));
+  COMPILER_CHECK(kBatchSize == SizeClassMap::kMaxNumCachedHint * sizeof(uptr));
 
   static uptr ClassIdToSize(uptr class_id) {
-    return SizeClassMap::Size(class_id);
+    return (class_id == SizeClassMap::kBatchClassID) ?
+        kBatchSize : SizeClassMap::Size(class_id);
   }
 
-  typedef SizeClassAllocator32<kSpaceBeg, kSpaceSize, kMetadataSize,
-      SizeClassMap, kRegionSizeLog, ByteMap, MapUnmapCallback> ThisT;
+  typedef SizeClassAllocator32<Params> ThisT;
   typedef SizeClassAllocator32LocalCache<ThisT> AllocatorCache;
 
-  void Init() {
+  void Init(s32 release_to_os_interval_ms) {
     possible_regions.TestOnlyInit();
     internal_memset(size_class_info_array, 0, sizeof(size_class_info_array));
   }
 
+  s32 ReleaseToOSIntervalMs() const {
+    return kReleaseToOSIntervalNever;
+  }
+
+  void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
+    // This is empty here. Currently only implemented in 64-bit allocator.
+  }
+
   void *MapWithCallback(uptr size) {
-    size = RoundUpTo(size, GetPageSizeCached());
     void *res = MmapOrDie(size, "SizeClassAllocator32");
     MapUnmapCallback().OnMap((uptr)res, size);
     return res;
@@ -126,8 +150,9 @@ class SizeClassAllocator32 {
     CHECK_LT(class_id, kNumClasses);
     SizeClassInfo *sci = GetSizeClassInfo(class_id);
     SpinMutexLock l(&sci->mutex);
-    if (sci->free_list.empty())
-      PopulateFreeList(stat, c, sci, class_id);
+    if (sci->free_list.empty() &&
+        UNLIKELY(!PopulateFreeList(stat, c, sci, class_id)))
+      return nullptr;
     CHECK(!sci->free_list.empty());
     TransferBatch *b = sci->free_list.front();
     sci->free_list.pop_front();
@@ -137,9 +162,9 @@ class SizeClassAllocator32 {
   NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id,
                                 TransferBatch *b) {
     CHECK_LT(class_id, kNumClasses);
+    CHECK_GT(b->Count(), 0);
     SizeClassInfo *sci = GetSizeClassInfo(class_id);
     SpinMutexLock l(&sci->mutex);
-    CHECK_GT(b->Count(), 0);
     sci->free_list.push_front(b);
   }
 
@@ -227,10 +252,6 @@ class SizeClassAllocator32 {
     return 0;
   }
 
-  // This is empty here. Currently only implemented in 64-bit allocator.
-  void ReleaseToOS() { }
-
-
   typedef SizeClassMap SizeClassMapT;
   static const uptr kNumClasses = SizeClassMap::kNumClasses;
 
@@ -258,11 +279,13 @@ class SizeClassAllocator32 {
 
   uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
     CHECK_LT(class_id, kNumClasses);
-    uptr res = reinterpret_cast<uptr>(MmapAlignedOrDie(kRegionSize, kRegionSize,
-                                      "SizeClassAllocator32"));
+    uptr res = reinterpret_cast<uptr>(MmapAlignedOrDieOnFatalError(
+        kRegionSize, kRegionSize, "SizeClassAllocator32"));
+    if (UNLIKELY(!res))
+      return 0;
     MapUnmapCallback().OnMap(res, kRegionSize);
     stat->Add(AllocatorStatMapped, kRegionSize);
-    CHECK_EQ(0U, (res & (kRegionSize - 1)));
+    CHECK(IsAligned(res, kRegionSize));
     possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
     return res;
   }
@@ -272,21 +295,25 @@ class SizeClassAllocator32 {
     return &size_class_info_array[class_id];
   }
 
-  void PopulateFreeList(AllocatorStats *stat, AllocatorCache *c,
+  bool PopulateFreeList(AllocatorStats *stat, AllocatorCache *c,
                         SizeClassInfo *sci, uptr class_id) {
     uptr size = ClassIdToSize(class_id);
     uptr reg = AllocateRegion(stat, class_id);
+    if (UNLIKELY(!reg))
+      return false;
     uptr n_chunks = kRegionSize / (size + kMetadataSize);
     uptr max_count = TransferBatch::MaxCached(class_id);
+    CHECK_GT(max_count, 0);
     TransferBatch *b = nullptr;
     for (uptr i = reg; i < reg + n_chunks * size; i += size) {
       if (!b) {
         b = c->CreateBatch(class_id, this, (TransferBatch*)i);
+        if (UNLIKELY(!b))
+          return false;
         b->Clear();
       }
       b->Add((void*)i);
       if (b->Count() == max_count) {
-        CHECK_GT(b->Count(), 0);
         sci->free_list.push_back(b);
         b = nullptr;
       }
@@ -295,6 +322,7 @@ class SizeClassAllocator32 {
       CHECK_GT(b->Count(), 0);
       sci->free_list.push_back(b);
     }
+    return true;
   }
 
   ByteMap possible_regions;
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
index 620639afbfd..4ae59c8b6b1 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
@@ -60,14 +60,14 @@ class SizeClassAllocator64 {
   // as a 4-byte integer (offset from the region start shifted right by 4).
   typedef u32 CompactPtrT;
   static const uptr kCompactPtrScale = 4;
-  CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) {
+  CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) const {
     return static_cast<CompactPtrT>((ptr - base) >> kCompactPtrScale);
   }
-  uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) {
+  uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) const {
     return base + (static_cast<uptr>(ptr32) << kCompactPtrScale);
   }
 
-  void Init() {
+  void Init(s32 release_to_os_interval_ms) {
     uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
     if (kUsingConstantSpaceBeg) {
       CHECK_EQ(kSpaceBeg, reinterpret_cast<uptr>(
@@ -77,17 +77,17 @@ class SizeClassAllocator64 {
           reinterpret_cast<uptr>(MmapNoAccess(TotalSpaceSize));
       CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
     }
-    MapWithCallback(SpaceEnd(), AdditionalSize());
+    SetReleaseToOSIntervalMs(release_to_os_interval_ms);
+    MapWithCallbackOrDie(SpaceEnd(), AdditionalSize());
   }
 
-  void MapWithCallback(uptr beg, uptr size) {
-    CHECK_EQ(beg, reinterpret_cast<uptr>(MmapFixedOrDie(beg, size)));
-    MapUnmapCallback().OnMap(beg, size);
+  s32 ReleaseToOSIntervalMs() const {
+    return atomic_load(&release_to_os_interval_ms_, memory_order_relaxed);
   }
 
-  void UnmapWithCallback(uptr beg, uptr size) {
-    MapUnmapCallback().OnUnmap(beg, size);
-    UnmapOrDie(reinterpret_cast<void *>(beg), size);
+  void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
+    atomic_store(&release_to_os_interval_ms_, release_to_os_interval_ms,
+                 memory_order_relaxed);
   }
 
   static bool CanAllocate(uptr size, uptr alignment) {
@@ -104,14 +104,20 @@ class SizeClassAllocator64 {
     BlockingMutexLock l(&region->mutex);
     uptr old_num_chunks = region->num_freed_chunks;
     uptr new_num_freed_chunks = old_num_chunks + n_chunks;
-    EnsureFreeArraySpace(region, region_beg, new_num_freed_chunks);
+    // Failure to allocate free array space while releasing memory is non
+    // recoverable.
+    if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg,
+                                       new_num_freed_chunks)))
+      DieOnFailure::OnOOM();
     for (uptr i = 0; i < n_chunks; i++)
       free_array[old_num_chunks + i] = chunks[i];
     region->num_freed_chunks = new_num_freed_chunks;
-    region->n_freed += n_chunks;
+    region->stats.n_freed += n_chunks;
+
+    MaybeReleaseToOS(class_id);
   }
 
-  NOINLINE void GetFromAllocator(AllocatorStats *stat, uptr class_id,
+  NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id,
                                  CompactPtrT *chunks, uptr n_chunks) {
     RegionInfo *region = GetRegionInfo(class_id);
     uptr region_beg = GetRegionBeginBySizeClass(class_id);
@@ -119,18 +125,19 @@ class SizeClassAllocator64 {
 
     BlockingMutexLock l(&region->mutex);
     if (UNLIKELY(region->num_freed_chunks < n_chunks)) {
-      PopulateFreeArray(stat, class_id, region,
-                        n_chunks - region->num_freed_chunks);
+      if (UNLIKELY(!PopulateFreeArray(stat, class_id, region,
+                                      n_chunks - region->num_freed_chunks)))
+        return false;
       CHECK_GE(region->num_freed_chunks, n_chunks);
     }
     region->num_freed_chunks -= n_chunks;
     uptr base_idx = region->num_freed_chunks;
     for (uptr i = 0; i < n_chunks; i++)
       chunks[i] = free_array[base_idx + i];
-    region->n_allocated += n_chunks;
+    region->stats.n_allocated += n_chunks;
+    return true;
   }
 
-
   bool PointerIsMine(const void *p) {
     uptr P = reinterpret_cast<uptr>(p);
     if (kUsingConstantSpaceBeg && (kSpaceBeg % kSpaceSize) == 0)
@@ -146,7 +153,7 @@ class SizeClassAllocator64 {
         space_beg;
   }
 
-  uptr GetRegionBeginBySizeClass(uptr class_id) {
+  uptr GetRegionBeginBySizeClass(uptr class_id) const {
     return SpaceBeg() + kRegionSize * class_id;
   }
 
@@ -197,7 +204,7 @@ class SizeClassAllocator64 {
 
   // Test-only.
   void TestOnlyUnmap() {
-    UnmapWithCallback(SpaceBeg(), kSpaceSize + AdditionalSize());
+    UnmapWithCallbackOrDie(SpaceBeg(), kSpaceSize + AdditionalSize());
   }
 
   static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats,
@@ -210,16 +217,18 @@ class SizeClassAllocator64 {
   void PrintStats(uptr class_id, uptr rss) {
     RegionInfo *region = GetRegionInfo(class_id);
     if (region->mapped_user == 0) return;
-    uptr in_use = region->n_allocated - region->n_freed;
+    uptr in_use = region->stats.n_allocated - region->stats.n_freed;
     uptr avail_chunks = region->allocated_user / ClassIdToSize(class_id);
     Printf(
-        "  %02zd (%zd): mapped: %zdK allocs: %zd frees: %zd inuse: %zd "
-        "num_freed_chunks %zd"
-        " avail: %zd rss: %zdK releases: %zd\n",
-        class_id, ClassIdToSize(class_id), region->mapped_user >> 10,
-        region->n_allocated, region->n_freed, in_use,
-        region->num_freed_chunks, avail_chunks, rss >> 10,
-        region->rtoi.num_releases);
+        "%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
+        "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
+        "last released: %6zdK region: 0x%zx\n",
+        region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
+        region->mapped_user >> 10, region->stats.n_allocated,
+        region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
+        rss >> 10, region->rtoi.num_releases,
+        region->rtoi.last_released_bytes >> 10,
+        SpaceBeg() + kRegionSize * class_id);
   }
 
   void PrintStats() {
@@ -229,8 +238,8 @@ class SizeClassAllocator64 {
     for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
       RegionInfo *region = GetRegionInfo(class_id);
       total_mapped += region->mapped_user;
-      n_allocated += region->n_allocated;
-      n_freed += region->n_freed;
+      n_allocated += region->stats.n_allocated;
+      n_freed += region->stats.n_freed;
     }
     Printf("Stats: SizeClassAllocator64: %zdM mapped in %zd allocations; "
            "remains %zd\n",
@@ -282,16 +291,244 @@ class SizeClassAllocator64 {
                      GetPageSizeCached());
   }
 
-  void ReleaseToOS() {
-    for (uptr class_id = 1; class_id < kNumClasses; class_id++)
-      ReleaseToOS(class_id);
-  }
-
   typedef SizeClassMap SizeClassMapT;
   static const uptr kNumClasses = SizeClassMap::kNumClasses;
   static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded;
 
+  // A packed array of counters. Each counter occupies 2^n bits, enough to store
+  // counter's max_value. Ctor will try to allocate the required buffer via
+  // mapper->MapPackedCounterArrayBuffer and the caller is expected to check
+  // whether the initialization was successful by checking IsAllocated() result.
+  // For the performance sake, none of the accessors check the validity of the
+  // arguments, it is assumed that index is always in [0, n) range and the value
+  // is not incremented past max_value.
+  template<class MemoryMapperT>
+  class PackedCounterArray {
+   public:
+    PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper)
+        : n(num_counters), memory_mapper(mapper) {
+      CHECK_GT(num_counters, 0);
+      CHECK_GT(max_value, 0);
+      constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL;
+      // Rounding counter storage size up to the power of two allows for using
+      // bit shifts calculating particular counter's index and offset.
+      uptr counter_size_bits =
+          RoundUpToPowerOfTwo(MostSignificantSetBitIndex(max_value) + 1);
+      CHECK_LE(counter_size_bits, kMaxCounterBits);
+      counter_size_bits_log = Log2(counter_size_bits);
+      counter_mask = ~0ULL >> (kMaxCounterBits - counter_size_bits);
+
+      uptr packing_ratio = kMaxCounterBits >> counter_size_bits_log;
+      CHECK_GT(packing_ratio, 0);
+      packing_ratio_log = Log2(packing_ratio);
+      bit_offset_mask = packing_ratio - 1;
+
+      buffer_size =
+          (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) *
+          sizeof(*buffer);
+      buffer = reinterpret_cast<u64*>(
+          memory_mapper->MapPackedCounterArrayBuffer(buffer_size));
+    }
+    ~PackedCounterArray() {
+      if (buffer) {
+        memory_mapper->UnmapPackedCounterArrayBuffer(
+            reinterpret_cast<uptr>(buffer), buffer_size);
+      }
+    }
+
+    bool IsAllocated() const {
+      return !!buffer;
+    }
+
+    u64 GetCount() const {
+      return n;
+    }
+
+    uptr Get(uptr i) const {
+      DCHECK_LT(i, n);
+      uptr index = i >> packing_ratio_log;
+      uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log;
+      return (buffer[index] >> bit_offset) & counter_mask;
+    }
+
+    void Inc(uptr i) const {
+      DCHECK_LT(Get(i), counter_mask);
+      uptr index = i >> packing_ratio_log;
+      uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log;
+      buffer[index] += 1ULL << bit_offset;
+    }
+
+    void IncRange(uptr from, uptr to) const {
+      DCHECK_LE(from, to);
+      for (uptr i = from; i <= to; i++)
+        Inc(i);
+    }
+
+   private:
+    const u64 n;
+    u64 counter_size_bits_log;
+    u64 counter_mask;
+    u64 packing_ratio_log;
+    u64 bit_offset_mask;
+
+    MemoryMapperT* const memory_mapper;
+    u64 buffer_size;
+    u64* buffer;
+  };
+
+  template<class MemoryMapperT>
+  class FreePagesRangeTracker {
+   public:
+    explicit FreePagesRangeTracker(MemoryMapperT* mapper)
+        : memory_mapper(mapper),
+          page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)),
+          in_the_range(false), current_page(0), current_range_start_page(0) {}
+
+    void NextPage(bool freed) {
+      if (freed) {
+        if (!in_the_range) {
+          current_range_start_page = current_page;
+          in_the_range = true;
+        }
+      } else {
+        CloseOpenedRange();
+      }
+      current_page++;
+    }
+
+    void Done() {
+      CloseOpenedRange();
+    }
+
+   private:
+    void CloseOpenedRange() {
+      if (in_the_range) {
+        memory_mapper->ReleasePageRangeToOS(
+            current_range_start_page << page_size_scaled_log,
+            current_page << page_size_scaled_log);
+        in_the_range = false;
+      }
+    }
+
+    MemoryMapperT* const memory_mapper;
+    const uptr page_size_scaled_log;
+    bool in_the_range;
+    uptr current_page;
+    uptr current_range_start_page;
+  };
+
+  // Iterates over the free_array to identify memory pages containing freed
+  // chunks only and returns these pages back to OS.
+  // allocated_pages_count is the total number of pages allocated for the
+  // current bucket.
+  template<class MemoryMapperT>
+  static void ReleaseFreeMemoryToOS(CompactPtrT *free_array,
+                                    uptr free_array_count, uptr chunk_size,
+                                    uptr allocated_pages_count,
+                                    MemoryMapperT *memory_mapper) {
+    const uptr page_size = GetPageSizeCached();
+
+    // Figure out the number of chunks per page and whether we can take a fast
+    // path (the number of chunks per page is the same for all pages).
+    uptr full_pages_chunk_count_max;
+    bool same_chunk_count_per_page;
+    if (chunk_size <= page_size && page_size % chunk_size == 0) {
+      // Same number of chunks per page, no cross overs.
+      full_pages_chunk_count_max = page_size / chunk_size;
+      same_chunk_count_per_page = true;
+    } else if (chunk_size <= page_size && page_size % chunk_size != 0 &&
+        chunk_size % (page_size % chunk_size) == 0) {
+      // Some chunks are crossing page boundaries, which means that the page
+      // contains one or two partial chunks, but all pages contain the same
+      // number of chunks.
+      full_pages_chunk_count_max = page_size / chunk_size + 1;
+      same_chunk_count_per_page = true;
+    } else if (chunk_size <= page_size) {
+      // Some chunks are crossing page boundaries, which means that the page
+      // contains one or two partial chunks.
+      full_pages_chunk_count_max = page_size / chunk_size + 2;
+      same_chunk_count_per_page = false;
+    } else if (chunk_size > page_size && chunk_size % page_size == 0) {
+      // One chunk covers multiple pages, no cross overs.
+      full_pages_chunk_count_max = 1;
+      same_chunk_count_per_page = true;
+    } else if (chunk_size > page_size) {
+      // One chunk covers multiple pages, Some chunks are crossing page
+      // boundaries. Some pages contain one chunk, some contain two.
+      full_pages_chunk_count_max = 2;
+      same_chunk_count_per_page = false;
+    } else {
+      UNREACHABLE("All chunk_size/page_size ratios must be handled.");
+    }
+
+    PackedCounterArray<MemoryMapperT> counters(allocated_pages_count,
+                                               full_pages_chunk_count_max,
+                                               memory_mapper);
+    if (!counters.IsAllocated())
+      return;
+
+    const uptr chunk_size_scaled = chunk_size >> kCompactPtrScale;
+    const uptr page_size_scaled = page_size >> kCompactPtrScale;
+    const uptr page_size_scaled_log = Log2(page_size_scaled);
+
+    // Iterate over free chunks and count how many free chunks affect each
+    // allocated page.
+    if (chunk_size <= page_size && page_size % chunk_size == 0) {
+      // Each chunk affects one page only.
+      for (uptr i = 0; i < free_array_count; i++)
+        counters.Inc(free_array[i] >> page_size_scaled_log);
+    } else {
+      // In all other cases chunks might affect more than one page.
+      for (uptr i = 0; i < free_array_count; i++) {
+        counters.IncRange(
+            free_array[i] >> page_size_scaled_log,
+            (free_array[i] + chunk_size_scaled - 1) >> page_size_scaled_log);
+      }
+    }
+
+    // Iterate over pages detecting ranges of pages with chunk counters equal
+    // to the expected number of chunks for the particular page.
+    FreePagesRangeTracker<MemoryMapperT> range_tracker(memory_mapper);
+    if (same_chunk_count_per_page) {
+      // Fast path, every page has the same number of chunks affecting it.
+      for (uptr i = 0; i < counters.GetCount(); i++)
+        range_tracker.NextPage(counters.Get(i) == full_pages_chunk_count_max);
+    } else {
+      // Show path, go through the pages keeping count how many chunks affect
+      // each page.
+      const uptr pn =
+          chunk_size < page_size ? page_size_scaled / chunk_size_scaled : 1;
+      const uptr pnc = pn * chunk_size_scaled;
+      // The idea is to increment the current page pointer by the first chunk
+      // size, middle portion size (the portion of the page covered by chunks
+      // except the first and the last one) and then the last chunk size, adding
+      // up the number of chunks on the current page and checking on every step
+      // whether the page boundary was crossed.
+      uptr prev_page_boundary = 0;
+      uptr current_boundary = 0;
+      for (uptr i = 0; i < counters.GetCount(); i++) {
+        uptr page_boundary = prev_page_boundary + page_size_scaled;
+        uptr chunks_per_page = pn;
+        if (current_boundary < page_boundary) {
+          if (current_boundary > prev_page_boundary)
+            chunks_per_page++;
+          current_boundary += pnc;
+          if (current_boundary < page_boundary) {
+            chunks_per_page++;
+            current_boundary += chunk_size_scaled;
+          }
+        }
+        prev_page_boundary = page_boundary;
+
+        range_tracker.NextPage(counters.Get(i) == chunks_per_page);
+      }
+    }
+    range_tracker.Done();
+  }
+
  private:
+  friend class MemoryMapper;
+
   static const uptr kRegionSize = kSpaceSize / kNumClassesRounded;
   // FreeArray is the array of free-d chunks (stored as 4-byte offsets).
   // In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize
@@ -315,12 +552,19 @@ class SizeClassAllocator64 {
   static const uptr kMetaMapSize = 1 << 16;
   // Call mmap for free array memory with at least this size.
   static const uptr kFreeArrayMapSize = 1 << 16;
-  // Granularity of ReleaseToOs (aka madvise).
-  static const uptr kReleaseToOsGranularity = 1 << 12;
+
+  atomic_sint32_t release_to_os_interval_ms_;
+
+  struct Stats {
+    uptr n_allocated;
+    uptr n_freed;
+  };
 
   struct ReleaseToOsInfo {
     uptr n_freed_at_last_release;
     uptr num_releases;
+    u64 last_release_at_ns;
+    u64 last_released_bytes;
   };
 
   struct RegionInfo {
@@ -331,8 +575,9 @@ class SizeClassAllocator64 {
     uptr allocated_meta;  // Bytes allocated for metadata.
     uptr mapped_user;  // Bytes mapped for user memory.
     uptr mapped_meta;  // Bytes mapped for metadata.
-    u32 rand_state; // Seed for random shuffle, used if kRandomShuffleChunks.
-    uptr n_allocated, n_freed;  // Just stats.
+    u32 rand_state;  // Seed for random shuffle, used if kRandomShuffleChunks.
+    bool exhausted;  // Whether region is out of space for new chunks.
+    Stats stats;
     ReleaseToOsInfo rtoi;
   };
   COMPILER_CHECK(sizeof(RegionInfo) >= kCacheLineSize);
@@ -349,18 +594,18 @@ class SizeClassAllocator64 {
       Swap(a[i], a[RandN(rand_state, i + 1)]);
   }
 
-  RegionInfo *GetRegionInfo(uptr class_id) {
+  RegionInfo *GetRegionInfo(uptr class_id) const {
     CHECK_LT(class_id, kNumClasses);
     RegionInfo *regions =
         reinterpret_cast<RegionInfo *>(SpaceBeg() + kSpaceSize);
     return &regions[class_id];
   }
 
-  uptr GetMetadataEnd(uptr region_beg) {
+  uptr GetMetadataEnd(uptr region_beg) const {
     return region_beg + kRegionSize - kFreeArraySize;
   }
 
-  uptr GetChunkIdx(uptr chunk, uptr size) {
+  uptr GetChunkIdx(uptr chunk, uptr size) const {
     if (!kUsingConstantSpaceBeg)
       chunk -= SpaceBeg();
 
@@ -372,132 +617,205 @@ class SizeClassAllocator64 {
     return (u32)offset / (u32)size;
   }
 
-  CompactPtrT *GetFreeArray(uptr region_beg) {
-    return reinterpret_cast<CompactPtrT *>(region_beg + kRegionSize -
-                                           kFreeArraySize);
+  CompactPtrT *GetFreeArray(uptr region_beg) const {
+    return reinterpret_cast<CompactPtrT *>(GetMetadataEnd(region_beg));
   }
 
-  void EnsureFreeArraySpace(RegionInfo *region, uptr region_beg,
+  bool MapWithCallback(uptr beg, uptr size) {
+    uptr mapped = reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(beg, size));
+    if (UNLIKELY(!mapped))
+      return false;
+    CHECK_EQ(beg, mapped);
+    MapUnmapCallback().OnMap(beg, size);
+    return true;
+  }
+
+  void MapWithCallbackOrDie(uptr beg, uptr size) {
+    CHECK_EQ(beg, reinterpret_cast<uptr>(MmapFixedOrDie(beg, size)));
+    MapUnmapCallback().OnMap(beg, size);
+  }
+
+  void UnmapWithCallbackOrDie(uptr beg, uptr size) {
+    MapUnmapCallback().OnUnmap(beg, size);
+    UnmapOrDie(reinterpret_cast<void *>(beg), size);
+  }
+
+  bool EnsureFreeArraySpace(RegionInfo *region, uptr region_beg,
                             uptr num_freed_chunks) {
     uptr needed_space = num_freed_chunks * sizeof(CompactPtrT);
     if (region->mapped_free_array < needed_space) {
-      CHECK_LE(needed_space, kFreeArraySize);
       uptr new_mapped_free_array = RoundUpTo(needed_space, kFreeArrayMapSize);
+      CHECK_LE(new_mapped_free_array, kFreeArraySize);
       uptr current_map_end = reinterpret_cast<uptr>(GetFreeArray(region_beg)) +
                              region->mapped_free_array;
       uptr new_map_size = new_mapped_free_array - region->mapped_free_array;
-      MapWithCallback(current_map_end, new_map_size);
+      if (UNLIKELY(!MapWithCallback(current_map_end, new_map_size)))
+        return false;
       region->mapped_free_array = new_mapped_free_array;
     }
+    return true;
   }
 
-
-  NOINLINE void PopulateFreeArray(AllocatorStats *stat, uptr class_id,
+  NOINLINE bool PopulateFreeArray(AllocatorStats *stat, uptr class_id,
                                   RegionInfo *region, uptr requested_count) {
     // region->mutex is held.
-    uptr size = ClassIdToSize(class_id);
-    uptr beg_idx = region->allocated_user;
-    uptr end_idx = beg_idx + requested_count * size;
-    uptr region_beg = GetRegionBeginBySizeClass(class_id);
-    if (end_idx > region->mapped_user) {
+    const uptr size = ClassIdToSize(class_id);
+    const uptr new_space_beg = region->allocated_user;
+    const uptr new_space_end = new_space_beg + requested_count * size;
+    const uptr region_beg = GetRegionBeginBySizeClass(class_id);
+
+    // Map more space for chunks, if necessary.
+    if (new_space_end > region->mapped_user) {
       if (!kUsingConstantSpaceBeg && region->mapped_user == 0)
         region->rand_state = static_cast<u32>(region_beg >> 12);  // From ASLR.
       // Do the mmap for the user memory.
       uptr map_size = kUserMapSize;
-      while (end_idx > region->mapped_user + map_size)
+      while (new_space_end > region->mapped_user + map_size)
         map_size += kUserMapSize;
-      CHECK_GE(region->mapped_user + map_size, end_idx);
-      MapWithCallback(region_beg + region->mapped_user, map_size);
+      CHECK_GE(region->mapped_user + map_size, new_space_end);
+      if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user,
+                                    map_size)))
+        return false;
       stat->Add(AllocatorStatMapped, map_size);
       region->mapped_user += map_size;
     }
-    CompactPtrT *free_array = GetFreeArray(region_beg);
-    uptr total_count = (region->mapped_user - beg_idx) / size;
-    uptr num_freed_chunks = region->num_freed_chunks;
-    EnsureFreeArraySpace(region, region_beg, num_freed_chunks + total_count);
-    for (uptr i = 0; i < total_count; i++) {
-      uptr chunk = beg_idx + i * size;
-      free_array[num_freed_chunks + total_count - 1 - i] =
-          PointerToCompactPtr(0, chunk);
+    const uptr new_chunks_count = (region->mapped_user - new_space_beg) / size;
+
+    // Calculate the required space for metadata.
+    const uptr requested_allocated_meta =
+        region->allocated_meta + new_chunks_count * kMetadataSize;
+    uptr requested_mapped_meta = region->mapped_meta;
+    while (requested_allocated_meta > requested_mapped_meta)
+      requested_mapped_meta += kMetaMapSize;
+    // Check whether this size class is exhausted.
+    if (region->mapped_user + requested_mapped_meta >
+        kRegionSize - kFreeArraySize) {
+      if (!region->exhausted) {
+        region->exhausted = true;
+        Printf("%s: Out of memory. ", SanitizerToolName);
+        Printf("The process has exhausted %zuMB for size class %zu.\n",
+               kRegionSize >> 20, size);
+      }
+      return false;
+    }
+    // Map more space for metadata, if necessary.
+    if (requested_mapped_meta > region->mapped_meta) {
+      if (UNLIKELY(!MapWithCallback(
+              GetMetadataEnd(region_beg) - requested_mapped_meta,
+              requested_mapped_meta - region->mapped_meta)))
+        return false;
+      region->mapped_meta = requested_mapped_meta;
     }
+
+    // If necessary, allocate more space for the free array and populate it with
+    // newly allocated chunks.
+    const uptr total_freed_chunks = region->num_freed_chunks + new_chunks_count;
+    if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg, total_freed_chunks)))
+      return false;
+    CompactPtrT *free_array = GetFreeArray(region_beg);
+    for (uptr i = 0, chunk = new_space_beg; i < new_chunks_count;
+         i++, chunk += size)
+      free_array[total_freed_chunks - 1 - i] = PointerToCompactPtr(0, chunk);
     if (kRandomShuffleChunks)
-      RandomShuffle(&free_array[num_freed_chunks], total_count,
+      RandomShuffle(&free_array[region->num_freed_chunks], new_chunks_count,
                     &region->rand_state);
-    region->num_freed_chunks += total_count;
-    region->allocated_user += total_count * size;
-    CHECK_LE(region->allocated_user, region->mapped_user);
 
-    region->allocated_meta += total_count * kMetadataSize;
-    if (region->allocated_meta > region->mapped_meta) {
-      uptr map_size = kMetaMapSize;
-      while (region->allocated_meta > region->mapped_meta + map_size)
-        map_size += kMetaMapSize;
-      // Do the mmap for the metadata.
-      CHECK_GE(region->mapped_meta + map_size, region->allocated_meta);
-      MapWithCallback(GetMetadataEnd(region_beg) -
-                      region->mapped_meta - map_size, map_size);
-      region->mapped_meta += map_size;
-    }
+    // All necessary memory is mapped and now it is safe to advance all
+    // 'allocated_*' counters.
+    region->num_freed_chunks += new_chunks_count;
+    region->allocated_user += new_chunks_count * size;
+    CHECK_LE(region->allocated_user, region->mapped_user);
+    region->allocated_meta = requested_allocated_meta;
     CHECK_LE(region->allocated_meta, region->mapped_meta);
-    if (region->mapped_user + region->mapped_meta >
-        kRegionSize - kFreeArraySize) {
-      Printf("%s: Out of memory. Dying. ", SanitizerToolName);
-      Printf("The process has exhausted %zuMB for size class %zu.\n",
-          kRegionSize / 1024 / 1024, size);
-      Die();
-    }
-  }
+    region->exhausted = false;
+
+    // TODO(alekseyshl): Consider bumping last_release_at_ns here to prevent
+    // MaybeReleaseToOS from releasing just allocated pages or protect these
+    // not yet used chunks some other way.
 
-  bool MaybeReleaseChunkRange(uptr region_beg, uptr chunk_size,
-                              CompactPtrT first, CompactPtrT last) {
-    uptr beg_ptr = CompactPtrToPointer(region_beg, first);
-    uptr end_ptr = CompactPtrToPointer(region_beg, last) + chunk_size;
-    CHECK_GE(end_ptr - beg_ptr, kReleaseToOsGranularity);
-    beg_ptr = RoundUpTo(beg_ptr, kReleaseToOsGranularity);
-    end_ptr = RoundDownTo(end_ptr, kReleaseToOsGranularity);
-    if (end_ptr == beg_ptr) return false;
-    ReleaseMemoryToOS(beg_ptr, end_ptr - beg_ptr);
     return true;
   }
 
-  // Releases some RAM back to OS.
-  // Algorithm:
-  // * Lock the region.
-  // * Sort the chunks.
-  // * Find ranges fully covered by free-d chunks
-  // * Release them to OS with madvise.
-  //
-  // TODO(kcc): make sure we don't do it too frequently.
-  void ReleaseToOS(uptr class_id) {
+  class MemoryMapper {
+   public:
+    MemoryMapper(const ThisT& base_allocator, uptr class_id)
+        : allocator(base_allocator),
+          region_base(base_allocator.GetRegionBeginBySizeClass(class_id)),
+          released_ranges_count(0),
+          released_bytes(0) {
+    }
+
+    uptr GetReleasedRangesCount() const {
+      return released_ranges_count;
+    }
+
+    uptr GetReleasedBytes() const {
+      return released_bytes;
+    }
+
+    uptr MapPackedCounterArrayBuffer(uptr buffer_size) {
+      // TODO(alekseyshl): The idea to explore is to check if we have enough
+      // space between num_freed_chunks*sizeof(CompactPtrT) and
+      // mapped_free_array to fit buffer_size bytes and use that space instead
+      // of mapping a temporary one.
+      return reinterpret_cast<uptr>(
+          MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters"));
+    }
+
+    void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {
+      UnmapOrDie(reinterpret_cast<void *>(buffer), buffer_size);
+    }
+
+    // Releases [from, to) range of pages back to OS.
+    void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) {
+      const uptr from_page = allocator.CompactPtrToPointer(region_base, from);
+      const uptr to_page = allocator.CompactPtrToPointer(region_base, to);
+      ReleaseMemoryPagesToOS(from_page, to_page);
+      released_ranges_count++;
+      released_bytes += to_page - from_page;
+    }
+
+   private:
+    const ThisT& allocator;
+    const uptr region_base;
+    uptr released_ranges_count;
+    uptr released_bytes;
+  };
+
+  // Attempts to release RAM occupied by freed chunks back to OS. The region is
+  // expected to be locked.
+  void MaybeReleaseToOS(uptr class_id) {
     RegionInfo *region = GetRegionInfo(class_id);
-    uptr region_beg = GetRegionBeginBySizeClass(class_id);
-    CompactPtrT *free_array = GetFreeArray(region_beg);
-    uptr chunk_size = ClassIdToSize(class_id);
-    uptr scaled_chunk_size = chunk_size >> kCompactPtrScale;
-    const uptr kScaledGranularity = kReleaseToOsGranularity >> kCompactPtrScale;
-    BlockingMutexLock l(&region->mutex);
+    const uptr chunk_size = ClassIdToSize(class_id);
+    const uptr page_size = GetPageSizeCached();
+
     uptr n = region->num_freed_chunks;
-    if (n * chunk_size < kReleaseToOsGranularity)
-      return;   // No chance to release anything.
-    if ((region->rtoi.n_freed_at_last_release - region->n_freed) * chunk_size <
-        kReleaseToOsGranularity)
+    if (n * chunk_size < page_size)
+      return;  // No chance to release anything.
+    if ((region->stats.n_freed -
+         region->rtoi.n_freed_at_last_release) * chunk_size < page_size) {
       return;  // Nothing new to release.
-    SortArray(free_array, n);
-    uptr beg = free_array[0];
-    uptr prev = free_array[0];
-    for (uptr i = 1; i < n; i++) {
-      uptr chunk = free_array[i];
-      CHECK_GT(chunk, prev);
-      if (chunk - prev != scaled_chunk_size) {
-        CHECK_GT(chunk - prev, scaled_chunk_size);
-        if (prev + scaled_chunk_size - beg >= kScaledGranularity) {
-          MaybeReleaseChunkRange(region_beg, chunk_size, beg, prev);
-          region->rtoi.n_freed_at_last_release = region->n_freed;
-          region->rtoi.num_releases++;
-        }
-        beg = chunk;
-      }
-      prev = chunk;
     }
+
+    s32 interval_ms = ReleaseToOSIntervalMs();
+    if (interval_ms < 0)
+      return;
+
+    if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > NanoTime())
+      return;  // Memory was returned recently.
+
+    MemoryMapper memory_mapper(*this, class_id);
+
+    ReleaseFreeMemoryToOS<MemoryMapper>(
+        GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size,
+        RoundUpTo(region->allocated_user, page_size) / page_size,
+        &memory_mapper);
+
+    if (memory_mapper.GetReleasedRangesCount() > 0) {
+      region->rtoi.n_freed_at_last_release = region->stats.n_freed;
+      region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
+      region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
+    }
+    region->rtoi.last_release_at_ns = NanoTime();
   }
 };
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
index 91c2ecc5b26..9a9b83a884f 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
@@ -15,17 +15,19 @@
 // This class can (de)allocate only large chunks of memory using mmap/unmap.
 // The main purpose of this allocator is to cover large and rare allocation
 // sizes not covered by more efficient allocators (e.g. SizeClassAllocator64).
-template <class MapUnmapCallback = NoOpMapUnmapCallback>
+template <class MapUnmapCallback = NoOpMapUnmapCallback,
+          class FailureHandlerT = ReturnNullOrDieOnFailure>
 class LargeMmapAllocator {
  public:
-  void InitLinkerInitialized(bool may_return_null) {
+  typedef FailureHandlerT FailureHandler;
+
+  void InitLinkerInitialized() {
     page_size_ = GetPageSizeCached();
-    atomic_store(&may_return_null_, may_return_null, memory_order_relaxed);
   }
 
-  void Init(bool may_return_null) {
+  void Init() {
     internal_memset(this, 0, sizeof(*this));
-    InitLinkerInitialized(may_return_null);
+    InitLinkerInitialized();
   }
 
   void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) {
@@ -34,9 +36,12 @@ class LargeMmapAllocator {
     if (alignment > page_size_)
       map_size += alignment;
     // Overflow.
-    if (map_size < size) return ReturnNullOrDieOnBadRequest();
+    if (map_size < size)
+      return FailureHandler::OnBadRequest();
     uptr map_beg = reinterpret_cast<uptr>(
-        MmapOrDie(map_size, "LargeMmapAllocator"));
+        MmapOrDieOnFatalError(map_size, "LargeMmapAllocator"));
+    if (!map_beg)
+      return FailureHandler::OnOOM();
     CHECK(IsAligned(map_beg, page_size_));
     MapUnmapCallback().OnMap(map_beg, map_size);
     uptr map_end = map_beg + map_size;
@@ -70,24 +75,6 @@ class LargeMmapAllocator {
     return reinterpret_cast<void*>(res);
   }
 
-  bool MayReturnNull() const {
-    return atomic_load(&may_return_null_, memory_order_acquire);
-  }
-
-  void *ReturnNullOrDieOnBadRequest() {
-    if (MayReturnNull()) return nullptr;
-    ReportAllocatorCannotReturnNull(false);
-  }
-
-  void *ReturnNullOrDieOnOOM() {
-    if (MayReturnNull()) return nullptr;
-    ReportAllocatorCannotReturnNull(true);
-  }
-
-  void SetMayReturnNull(bool may_return_null) {
-    atomic_store(&may_return_null_, may_return_null, memory_order_release);
-  }
-
   void Deallocate(AllocatorStats *stat, void *p) {
     Header *h = GetHeader(p);
     {
@@ -159,6 +146,14 @@ class LargeMmapAllocator {
     return GetUser(h);
   }
 
+  void EnsureSortedChunks() {
+    if (chunks_sorted_) return;
+    SortArray(reinterpret_cast<uptr*>(chunks_), n_chunks_);
+    for (uptr i = 0; i < n_chunks_; i++)
+      chunks_[i]->chunk_idx = i;
+    chunks_sorted_ = true;
+  }
+
   // This function does the same as GetBlockBegin, but is much faster.
   // Must be called with the allocator locked.
   void *GetBlockBeginFastLocked(void *ptr) {
@@ -166,16 +161,10 @@ class LargeMmapAllocator {
     uptr p = reinterpret_cast<uptr>(ptr);
     uptr n = n_chunks_;
     if (!n) return nullptr;
-    if (!chunks_sorted_) {
-      // Do one-time sort. chunks_sorted_ is reset in Allocate/Deallocate.
-      SortArray(reinterpret_cast<uptr*>(chunks_), n);
-      for (uptr i = 0; i < n; i++)
-        chunks_[i]->chunk_idx = i;
-      chunks_sorted_ = true;
-      min_mmap_ = reinterpret_cast<uptr>(chunks_[0]);
-      max_mmap_ = reinterpret_cast<uptr>(chunks_[n - 1]) +
-          chunks_[n - 1]->map_size;
-    }
+    EnsureSortedChunks();
+    auto min_mmap_ = reinterpret_cast<uptr>(chunks_[0]);
+    auto max_mmap_ =
+        reinterpret_cast<uptr>(chunks_[n - 1]) + chunks_[n - 1]->map_size;
     if (p < min_mmap_ || p >= max_mmap_)
       return nullptr;
     uptr beg = 0, end = n - 1;
@@ -228,8 +217,14 @@ class LargeMmapAllocator {
   // Iterate over all existing chunks.
   // The allocator must be locked when calling this function.
   void ForEachChunk(ForEachChunkCallback callback, void *arg) {
-    for (uptr i = 0; i < n_chunks_; i++)
+    EnsureSortedChunks();  // Avoid doing the sort while iterating.
+    for (uptr i = 0; i < n_chunks_; i++) {
+      auto t = chunks_[i];
       callback(reinterpret_cast<uptr>(GetUser(chunks_[i])), arg);
+      // Consistency check: verify that the array did not change.
+      CHECK_EQ(chunks_[i], t);
+      CHECK_EQ(chunks_[i]->chunk_idx, i);
+    }
   }
 
  private:
@@ -261,11 +256,9 @@ class LargeMmapAllocator {
   uptr page_size_;
   Header *chunks_[kMaxNumChunks];
   uptr n_chunks_;
-  uptr min_mmap_, max_mmap_;
   bool chunks_sorted_;
   struct Stats {
     uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64];
   } stats;
-  atomic_uint8_t may_return_null_;
   SpinMutex mutex_;
 };
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_size_class_map.h b/libsanitizer/sanitizer_common/sanitizer_allocator_size_class_map.h
index 4cda021a947..19af55138d1 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_size_class_map.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_size_class_map.h
@@ -132,8 +132,9 @@ class SizeClassMap {
 
   static const uptr kMaxSize = 1UL << kMaxSizeLog;
   static const uptr kNumClasses =
-      kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1;
+      kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1 + 1;
   static const uptr kLargestClassID = kNumClasses - 2;
+  static const uptr kBatchClassID = kNumClasses - 1;
   COMPILER_CHECK(kNumClasses >= 16 && kNumClasses <= 256);
   static const uptr kNumClassesRounded =
       kNumClasses <= 32  ? 32 :
@@ -141,6 +142,11 @@ class SizeClassMap {
       kNumClasses <= 128 ? 128 : 256;
 
   static uptr Size(uptr class_id) {
+    // Estimate the result for kBatchClassID because this class does not know
+    // the exact size of TransferBatch. It's OK since we are using the actual
+    // sizeof(TransferBatch) where it matters.
+    if (UNLIKELY(class_id == kBatchClassID))
+      return kMaxNumCachedHint * sizeof(uptr);
     if (class_id <= kMidClass)
       return kMinSize * class_id;
     class_id -= kMidClass;
@@ -149,9 +155,10 @@ class SizeClassMap {
   }
 
   static uptr ClassID(uptr size) {
+    if (UNLIKELY(size > kMaxSize))
+      return 0;
     if (size <= kMidSize)
       return (size + kMinSize - 1) >> kMinSizeLog;
-    if (size > kMaxSize) return 0;
     uptr l = MostSignificantSetBitIndex(size);
     uptr hbits = (size >> (l - S)) & M;
     uptr lbits = size & ((1 << (l - S)) - 1);
@@ -160,7 +167,13 @@ class SizeClassMap {
   }
 
   static uptr MaxCachedHint(uptr class_id) {
-    if (class_id == 0) return 0;
+    // Estimate the result for kBatchClassID because this class does not know
+    // the exact size of TransferBatch. We need to cache fewer batches than user
+    // chunks, so this number can be small.
+    if (UNLIKELY(class_id == kBatchClassID))
+      return 16;
+    if (UNLIKELY(class_id == 0))
+      return 0;
     uptr n = (1UL << kMaxBytesCachedLog) / Size(class_id);
     return Max<uptr>(1, Min(kMaxNumCachedHint, n));
   }
@@ -176,6 +189,8 @@ class SizeClassMap {
       uptr p = prev_s ? (d * 100 / prev_s) : 0;
       uptr l = s ? MostSignificantSetBitIndex(s) : 0;
       uptr cached = MaxCachedHint(i) * s;
+      if (i == kBatchClassID)
+        d = p = l = 0;
       Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd "
              "cached: %zd %zd; id %zd\n",
              i, Size(i), d, p, l, MaxCachedHint(i), cached, ClassID(s));
@@ -190,12 +205,13 @@ class SizeClassMap {
       // Printf("Validate: c%zd\n", c);
       uptr s = Size(c);
       CHECK_NE(s, 0U);
+      if (c == kBatchClassID)
+        continue;
       CHECK_EQ(ClassID(s), c);
-      if (c != kNumClasses - 1)
+      if (c < kLargestClassID)
         CHECK_EQ(ClassID(s + 1), c + 1);
       CHECK_EQ(ClassID(s - 1), c);
-      if (c)
-        CHECK_GT(Size(c), Size(c-1));
+      CHECK_GT(Size(c), Size(c - 1));
     }
     CHECK_EQ(ClassID(kMaxSize + 1), 0);
 
@@ -205,7 +221,7 @@ class SizeClassMap {
       CHECK_LT(c, kNumClasses);
       CHECK_GE(Size(c), s);
       if (c > 0)
-        CHECK_LT(Size(c-1), s);
+        CHECK_LT(Size(c - 1), s);
     }
   }
 };
diff --git a/libsanitizer/sanitizer_common/sanitizer_atomic.h b/libsanitizer/sanitizer_common/sanitizer_atomic.h
index 4973b7d4e88..82de0c6d446 100644
--- a/libsanitizer/sanitizer_common/sanitizer_atomic.h
+++ b/libsanitizer/sanitizer_common/sanitizer_atomic.h
@@ -35,6 +35,11 @@ struct atomic_uint16_t {
   volatile Type val_dont_use;
 };
 
+struct atomic_sint32_t {
+  typedef s32 Type;
+  volatile Type val_dont_use;
+};
+
 struct atomic_uint32_t {
   typedef u32 Type;
   volatile Type val_dont_use;
diff --git a/libsanitizer/sanitizer_common/sanitizer_atomic_clang.h b/libsanitizer/sanitizer_common/sanitizer_atomic_clang.h
index c600999e67a..dcdcd0ea4ad 100644
--- a/libsanitizer/sanitizer_common/sanitizer_atomic_clang.h
+++ b/libsanitizer/sanitizer_common/sanitizer_atomic_clang.h
@@ -69,16 +69,25 @@ INLINE typename T::Type atomic_exchange(volatile T *a,
   return v;
 }
 
-template<typename T>
-INLINE bool atomic_compare_exchange_strong(volatile T *a,
-                                           typename T::Type *cmp,
+template <typename T>
+INLINE bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
                                            typename T::Type xchg,
                                            memory_order mo) {
   typedef typename T::Type Type;
   Type cmpv = *cmp;
-  Type prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
-  if (prev == cmpv)
-    return true;
+  Type prev;
+#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32
+  if (sizeof(*a) == 8) {
+    Type volatile *val_ptr = const_cast<Type volatile *>(&a->val_dont_use);
+    prev = __mips_sync_val_compare_and_swap<u64>(
+        reinterpret_cast<u64 volatile *>(val_ptr), (u64)cmpv, (u64)xchg);
+  } else {
+    prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
+  }
+#else
+  prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
+#endif
+  if (prev == cmpv) return true;
   *cmp = prev;
   return false;
 }
diff --git a/libsanitizer/sanitizer_common/sanitizer_atomic_clang_other.h b/libsanitizer/sanitizer_common/sanitizer_atomic_clang_other.h
index c66c0992e1b..a0605bbbd82 100644
--- a/libsanitizer/sanitizer_common/sanitizer_atomic_clang_other.h
+++ b/libsanitizer/sanitizer_common/sanitizer_atomic_clang_other.h
@@ -15,6 +15,56 @@
 
 namespace __sanitizer {
 
+// MIPS32 does not support atomic > 4 bytes. To address this lack of
+// functionality, the sanitizer library provides helper methods which use an
+// internal spin lock mechanism to emulate atomic oprations when the size is
+// 8 bytes.
+#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32
+static void __spin_lock(volatile int *lock) {
+  while (__sync_lock_test_and_set(lock, 1))
+    while (*lock) {
+    }
+}
+
+static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); }
+
+
+// Make sure the lock is on its own cache line to prevent false sharing.
+// Put it inside a struct that is aligned and padded to the typical MIPS
+// cacheline which is 32 bytes.
+static struct {
+  int lock;
+  char pad[32 - sizeof(int)];
+} __attribute__((aligned(32))) lock = {0};
+
+template <class T>
+T __mips_sync_fetch_and_add(volatile T *ptr, T val) {
+  T ret;
+
+  __spin_lock(&lock.lock);
+
+  ret = *ptr;
+  *ptr = ret + val;
+
+  __spin_unlock(&lock.lock);
+
+  return ret;
+}
+
+template <class T>
+T __mips_sync_val_compare_and_swap(volatile T *ptr, T oldval, T newval) {
+  T ret;
+  __spin_lock(&lock.lock);
+
+  ret = *ptr;
+  if (ret == oldval) *ptr = newval;
+
+  __spin_unlock(&lock.lock);
+
+  return ret;
+}
+#endif
+
 INLINE void proc_yield(int cnt) {
   __asm__ __volatile__("" ::: "memory");
 }
@@ -51,8 +101,15 @@ INLINE typename T::Type atomic_load(
     // 64-bit load on 32-bit platform.
     // Gross, but simple and reliable.
     // Assume that it is not in read-only memory.
+#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32
+    typename T::Type volatile *val_ptr =
+        const_cast<typename T::Type volatile *>(&a->val_dont_use);
+    v = __mips_sync_fetch_and_add<u64>(
+        reinterpret_cast<u64 volatile *>(val_ptr), 0);
+#else
     v = __sync_fetch_and_add(
         const_cast<typename T::Type volatile *>(&a->val_dont_use), 0);
+#endif
   }
   return v;
 }
@@ -82,7 +139,14 @@ INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
     typename T::Type cmp = a->val_dont_use;
     typename T::Type cur;
     for (;;) {
+#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32
+      typename T::Type volatile *val_ptr =
+          const_cast<typename T::Type volatile *>(&a->val_dont_use);
+      cur = __mips_sync_val_compare_and_swap<u64>(
+          reinterpret_cast<u64 volatile *>(val_ptr), (u64)cmp, (u64)v);
+#else
       cur = __sync_val_compare_and_swap(&a->val_dont_use, cmp, v);
+#endif
       if (cmp == v)
         break;
       cmp = cur;
diff --git a/libsanitizer/sanitizer_common/sanitizer_common.cc b/libsanitizer/sanitizer_common/sanitizer_common.cc
index b445f613b85..8d7e9fae64e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_common.cc
@@ -25,72 +25,6 @@ const char *SanitizerToolName = "SanitizerTool";
 atomic_uint32_t current_verbosity;
 uptr PageSizeCached;
 
-StaticSpinMutex report_file_mu;
-ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0};
-
-void RawWrite(const char *buffer) {
-  report_file.Write(buffer, internal_strlen(buffer));
-}
-
-void ReportFile::ReopenIfNecessary() {
-  mu->CheckLocked();
-  if (fd == kStdoutFd || fd == kStderrFd) return;
-
-  uptr pid = internal_getpid();
-  // If in tracer, use the parent's file.
-  if (pid == stoptheworld_tracer_pid)
-    pid = stoptheworld_tracer_ppid;
-  if (fd != kInvalidFd) {
-    // If the report file is already opened by the current process,
-    // do nothing. Otherwise the report file was opened by the parent
-    // process, close it now.
-    if (fd_pid == pid)
-      return;
-    else
-      CloseFile(fd);
-  }
-
-  const char *exe_name = GetProcessName();
-  if (common_flags()->log_exe_name && exe_name) {
-    internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix,
-                      exe_name, pid);
-  } else {
-    internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid);
-  }
-  fd = OpenFile(full_path, WrOnly);
-  if (fd == kInvalidFd) {
-    const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
-    WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
-    WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
-    Die();
-  }
-  fd_pid = pid;
-}
-
-void ReportFile::SetReportPath(const char *path) {
-  if (!path)
-    return;
-  uptr len = internal_strlen(path);
-  if (len > sizeof(path_prefix) - 100) {
-    Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n",
-           path[0], path[1], path[2], path[3],
-           path[4], path[5], path[6], path[7]);
-    Die();
-  }
-
-  SpinMutexLock l(mu);
-  if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd)
-    CloseFile(fd);
-  fd = kInvalidFd;
-  if (internal_strcmp(path, "stdout") == 0) {
-    fd = kStdoutFd;
-  } else if (internal_strcmp(path, "stderr") == 0) {
-    fd = kStderrFd;
-  } else {
-    internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
-  }
-}
-
 // PID of the tracer task in StopTheWorld. It shares the address space with the
 // main process, but has a different PID and thus requires special handling.
 uptr stoptheworld_tracer_pid = 0;
@@ -98,6 +32,8 @@ uptr stoptheworld_tracer_pid = 0;
 // writing to the same log file.
 uptr stoptheworld_tracer_ppid = 0;
 
+StaticSpinMutex CommonSanitizerReportMutex;
+
 void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
                                       const char *mmap_type, error_t err,
                                       bool raw_report) {
@@ -118,42 +54,6 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
   UNREACHABLE("unable to mmap");
 }
 
-bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
-                      uptr *read_len, uptr max_len, error_t *errno_p) {
-  uptr PageSize = GetPageSizeCached();
-  uptr kMinFileLen = PageSize;
-  *buff = nullptr;
-  *buff_size = 0;
-  *read_len = 0;
-  // The files we usually open are not seekable, so try different buffer sizes.
-  for (uptr size = kMinFileLen; size <= max_len; size *= 2) {
-    fd_t fd = OpenFile(file_name, RdOnly, errno_p);
-    if (fd == kInvalidFd) return false;
-    UnmapOrDie(*buff, *buff_size);
-    *buff = (char*)MmapOrDie(size, __func__);
-    *buff_size = size;
-    *read_len = 0;
-    // Read up to one page at a time.
-    bool reached_eof = false;
-    while (*read_len + PageSize <= size) {
-      uptr just_read;
-      if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) {
-        UnmapOrDie(*buff, *buff_size);
-        return false;
-      }
-      if (just_read == 0) {
-        reached_eof = true;
-        break;
-      }
-      *read_len += just_read;
-    }
-    CloseFile(fd);
-    if (reached_eof)  // We've read the whole file.
-      break;
-  }
-  return true;
-}
-
 typedef bool UptrComparisonFunction(const uptr &a, const uptr &b);
 typedef bool U32ComparisonFunction(const u32 &a, const u32 &b);
 
@@ -197,23 +97,24 @@ const char *StripModuleName(const char *module) {
   return module;
 }
 
-void ReportErrorSummary(const char *error_message) {
+void ReportErrorSummary(const char *error_message, const char *alt_tool_name) {
   if (!common_flags()->print_summary)
     return;
   InternalScopedString buff(kMaxSummaryLength);
-  buff.append("SUMMARY: %s: %s", SanitizerToolName, error_message);
+  buff.append("SUMMARY: %s: %s",
+              alt_tool_name ? alt_tool_name : SanitizerToolName, error_message);
   __sanitizer_report_error_summary(buff.data());
 }
 
 #if !SANITIZER_GO
-void ReportErrorSummary(const char *error_type, const AddressInfo &info) {
-  if (!common_flags()->print_summary)
-    return;
+void ReportErrorSummary(const char *error_type, const AddressInfo &info,
+                        const char *alt_tool_name) {
+  if (!common_flags()->print_summary) return;
   InternalScopedString buff(kMaxSummaryLength);
   buff.append("%s ", error_type);
   RenderFrame(&buff, "%L %F", 0, info, common_flags()->symbolize_vs_style,
               common_flags()->strip_path_prefix);
-  ReportErrorSummary(buff.data());
+  ReportErrorSummary(buff.data(), alt_tool_name);
 }
 #endif
 
@@ -257,9 +158,23 @@ void LoadedModule::set(const char *module_name, uptr base_address) {
   base_address_ = base_address;
 }
 
+void LoadedModule::set(const char *module_name, uptr base_address,
+                       ModuleArch arch, u8 uuid[kModuleUUIDSize],
+                       bool instrumented) {
+  set(module_name, base_address);
+  arch_ = arch;
+  internal_memcpy(uuid_, uuid, sizeof(uuid_));
+  instrumented_ = instrumented;
+}
+
 void LoadedModule::clear() {
   InternalFree(full_name_);
+  base_address_ = 0;
+  max_executable_address_ = 0;
   full_name_ = nullptr;
+  arch_ = kModuleArchUnknown;
+  internal_memset(uuid_, 0, kModuleUUIDSize);
+  instrumented_ = false;
   while (!ranges_.empty()) {
     AddressRange *r = ranges_.front();
     ranges_.pop_front();
@@ -267,10 +182,14 @@ void LoadedModule::clear() {
   }
 }
 
-void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable) {
+void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable,
+                                   bool writable, const char *name) {
   void *mem = InternalAlloc(sizeof(AddressRange));
-  AddressRange *r = new(mem) AddressRange(beg, end, executable);
+  AddressRange *r =
+      new(mem) AddressRange(beg, end, executable, writable, name);
   ranges_.push_back(r);
+  if (executable && end > max_executable_address_)
+    max_executable_address_ = end;
 }
 
 bool LoadedModule::containsAddress(uptr address) const {
@@ -339,36 +258,6 @@ bool TemplateMatch(const char *templ, const char *str) {
   return true;
 }
 
-static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':';
-
-char *FindPathToBinary(const char *name) {
-  if (FileExists(name)) {
-    return internal_strdup(name);
-  }
-
-  const char *path = GetEnv("PATH");
-  if (!path)
-    return nullptr;
-  uptr name_len = internal_strlen(name);
-  InternalScopedBuffer<char> buffer(kMaxPathLength);
-  const char *beg = path;
-  while (true) {
-    const char *end = internal_strchrnul(beg, kPathSeparator);
-    uptr prefix_len = end - beg;
-    if (prefix_len + name_len + 2 <= kMaxPathLength) {
-      internal_memcpy(buffer.data(), beg, prefix_len);
-      buffer[prefix_len] = '/';
-      internal_memcpy(&buffer[prefix_len + 1], name, name_len);
-      buffer[prefix_len + 1 + name_len] = '\0';
-      if (FileExists(buffer.data()))
-        return internal_strdup(buffer.data());
-    }
-    if (*end == '\0') break;
-    beg = end + 1;
-  }
-  return nullptr;
-}
-
 static char binary_name_cache_str[kMaxPathLength];
 static char process_name_cache_str[kMaxPathLength];
 
@@ -462,16 +351,8 @@ static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr),
 using namespace __sanitizer;  // NOLINT
 
 extern "C" {
-void __sanitizer_set_report_path(const char *path) {
-  report_file.SetReportPath(path);
-}
-
-void __sanitizer_set_report_fd(void *fd) {
-  report_file.fd = (fd_t)reinterpret_cast<uptr>(fd);
-  report_file.fd_pid = internal_getpid();
-}
-
-void __sanitizer_report_error_summary(const char *error_summary) {
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_report_error_summary,
+                             const char *error_summary) {
   Printf("%s\n", error_summary);
 }
 
@@ -486,11 +367,4 @@ int __sanitizer_install_malloc_and_free_hooks(void (*malloc_hook)(const void *,
                                               void (*free_hook)(const void *)) {
   return InstallMallocFreeHooks(malloc_hook, free_hook);
 }
-
-#if !SANITIZER_GO && !SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_print_memory_profile(int top_percent) {
-  (void)top_percent;
-}
-#endif
 } // extern "C"
diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h
index 3e079ab7ad1..dd207d72e29 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common.h
+++ b/libsanitizer/sanitizer_common/sanitizer_common.h
@@ -27,8 +27,11 @@ extern "C" void _ReadWriteBarrier();
 #endif
 
 namespace __sanitizer {
-struct StackTrace;
+
 struct AddressInfo;
+struct BufferedStackTrace;
+struct SignalContext;
+struct StackTrace;
 
 // Constants.
 const uptr kWordSize = SANITIZER_WORDSIZE / 8;
@@ -70,7 +73,7 @@ INLINE uptr GetPageSizeCached() {
 uptr GetMmapGranularity();
 uptr GetMaxVirtualAddress();
 // Threads
-uptr GetTid();
+tid_t GetTid();
 uptr GetThreadSelf();
 void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
                                 uptr *stack_bottom);
@@ -83,25 +86,36 @@ INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) {
   return MmapOrDie(size, mem_type, /*raw_report*/ true);
 }
 void UnmapOrDie(void *addr, uptr size);
+// Behaves just like MmapOrDie, but tolerates out of memory condition, in that
+// case returns nullptr.
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type);
 void *MmapFixedNoReserve(uptr fixed_addr, uptr size,
                          const char *name = nullptr);
 void *MmapNoReserveOrDie(uptr size, const char *mem_type);
 void *MmapFixedOrDie(uptr fixed_addr, uptr size);
+// Behaves just like MmapFixedOrDie, but tolerates out of memory condition, in
+// that case returns nullptr.
+void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size);
 void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr);
 void *MmapNoAccess(uptr size);
 // Map aligned chunk of address space; size and alignment are powers of two.
-void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type);
+// Dies on all but out of memory errors, in the latter case returns nullptr.
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type);
 // Disallow access to a memory range.  Use MmapFixedNoAccess to allocate an
 // unaccessible memory.
 bool MprotectNoAccess(uptr addr, uptr size);
 bool MprotectReadOnly(uptr addr, uptr size);
 
 // Find an available address space.
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding);
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found);
 
 // Used to check if we can map shadow memory to a fixed location.
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
-void ReleaseMemoryToOS(uptr addr, uptr size);
+// Releases memory pages entirely within the [beg, end] address range. Noop if
+// the provided range does not contain at least one entire page.
+void ReleaseMemoryPagesToOS(uptr beg, uptr end);
 void IncreaseTotalMmap(uptr size);
 void DecreaseTotalMmap(uptr size);
 uptr GetRSS();
@@ -112,6 +126,14 @@ void CheckVMASize();
 void RunMallocHooks(const void *ptr, uptr size);
 void RunFreeHooks(const void *ptr);
 
+typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
+                               /*out*/uptr *stats, uptr stats_size);
+
+// Parse the contents of /proc/self/smaps and generate a memory profile.
+// |cb| is a tool-specific callback that fills the |stats| array containing
+// |stats_size| elements.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
+
 // InternalScopedBuffer can be used instead of large stack arrays to
 // keep frame size low.
 // FIXME: use InternalAlloc instead of MmapOrDie once
@@ -172,6 +194,7 @@ typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size);
 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback);
 
 // IO
+void CatastrophicErrorWrite(const char *buffer, uptr length);
 void RawWrite(const char *buffer);
 bool ColorizeReports();
 void RemoveANSIEscapeSequencesFromString(char *buffer);
@@ -188,66 +211,21 @@ void SetPrintfAndReportCallback(void (*callback)(const char *));
   } while (0)
 
 // Can be used to prevent mixing error reports from different sanitizers.
+// FIXME: Replace with ScopedErrorReportLock and hide.
 extern StaticSpinMutex CommonSanitizerReportMutex;
 
-struct ReportFile {
-  void Write(const char *buffer, uptr length);
-  bool SupportsColors();
-  void SetReportPath(const char *path);
-
-  // Don't use fields directly. They are only declared public to allow
-  // aggregate initialization.
-
-  // Protects fields below.
-  StaticSpinMutex *mu;
-  // Opened file descriptor. Defaults to stderr. It may be equal to
-  // kInvalidFd, in which case new file will be opened when necessary.
-  fd_t fd;
-  // Path prefix of report file, set via __sanitizer_set_report_path.
-  char path_prefix[kMaxPathLength];
-  // Full path to report, obtained as <path_prefix>.PID
-  char full_path[kMaxPathLength];
-  // PID of the process that opened fd. If a fork() occurs,
-  // the PID of child will be different from fd_pid.
-  uptr fd_pid;
+// Lock sanitizer error reporting and protects against nested errors.
+class ScopedErrorReportLock {
+ public:
+  ScopedErrorReportLock();
+  ~ScopedErrorReportLock();
 
- private:
-  void ReopenIfNecessary();
+  static void CheckLocked();
 };
-extern ReportFile report_file;
 
 extern uptr stoptheworld_tracer_pid;
 extern uptr stoptheworld_tracer_ppid;
 
-enum FileAccessMode {
-  RdOnly,
-  WrOnly,
-  RdWr
-};
-
-// Returns kInvalidFd on error.
-fd_t OpenFile(const char *filename, FileAccessMode mode,
-              error_t *errno_p = nullptr);
-void CloseFile(fd_t);
-
-// Return true on success, false on error.
-bool ReadFromFile(fd_t fd, void *buff, uptr buff_size,
-                  uptr *bytes_read = nullptr, error_t *error_p = nullptr);
-bool WriteToFile(fd_t fd, const void *buff, uptr buff_size,
-                 uptr *bytes_written = nullptr, error_t *error_p = nullptr);
-
-bool RenameFile(const char *oldpath, const char *newpath,
-                error_t *error_p = nullptr);
-
-// Scoped file handle closer.
-struct FileCloser {
-  explicit FileCloser(fd_t fd) : fd(fd) {}
-  ~FileCloser() { CloseFile(fd); }
-  fd_t fd;
-};
-
-bool SupportsColoredOutput(fd_t fd);
-
 // Opens the file 'file_name" and reads up to 'max_len' bytes.
 // The resulting buffer is mmaped and stored in '*buff'.
 // The size of the mmaped region is stored in '*buff_size'.
@@ -256,11 +234,6 @@ bool SupportsColoredOutput(fd_t fd);
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
                       uptr *read_len, uptr max_len = 1 << 26,
                       error_t *errno_p = nullptr);
-// Maps given file to virtual memory, and returns pointer to it
-// (or NULL if mapping fails). Stores the size of mmaped region
-// in '*buff_size'.
-void *MapFileToMemory(const char *file_name, uptr *buff_size);
-void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset);
 
 bool IsAccessibleMemoryRange(uptr beg, uptr size);
 
@@ -279,27 +252,9 @@ void UpdateProcessName();
 void CacheBinaryName();
 void DisableCoreDumperIfNecessary();
 void DumpProcessMap();
-bool FileExists(const char *filename);
+void PrintModuleMap();
 const char *GetEnv(const char *name);
 bool SetEnv(const char *name, const char *value);
-const char *GetPwd();
-char *FindPathToBinary(const char *name);
-bool IsPathSeparator(const char c);
-bool IsAbsolutePath(const char *path);
-// Starts a subprocess and returs its pid.
-// If *_fd parameters are not kInvalidFd their corresponding input/output
-// streams will be redirect to the file. The files will always be closed
-// in parent process even in case of an error.
-// The child process will close all fds after STDERR_FILENO
-// before passing control to a program.
-pid_t StartSubprocess(const char *filename, const char *const argv[],
-                      fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd,
-                      fd_t stderr_fd = kInvalidFd);
-// Checks if specified process is still running
-bool IsProcessRunning(pid_t pid);
-// Waits for the process to finish and returns its exit code.
-// Returns -1 in case of an error.
-int WaitForProcess(pid_t pid);
 
 u32 GetUid();
 void ReExec();
@@ -312,15 +267,9 @@ bool AddressSpaceIsUnlimited();
 void SetAddressSpaceUnlimited();
 void AdjustStackSize(void *attr);
 void PrepareForSandboxing(__sanitizer_sandbox_arguments *args);
-void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args);
 void SetSandboxingCallback(void (*f)());
 
-void CoverageUpdateMapping();
-void CovBeforeFork();
-void CovAfterFork(int child_pid);
-
 void InitializeCoverage(bool enabled, const char *coverage_dir);
-void ReInitializeCoverage(bool enabled, const char *coverage_dir);
 
 void InitTlsSize();
 uptr GetTlsSize();
@@ -373,16 +322,28 @@ void SetCheckFailedCallback(CheckFailedCallbackType callback);
 // The callback should be registered once at the tool init time.
 void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
 
-// Callback to be called when we want to try releasing unused allocator memory
-// back to the OS.
-typedef void (*AllocatorReleaseToOSCallback)();
-// The callback should be registered once at the tool init time.
-void SetAllocatorReleaseToOSCallback(AllocatorReleaseToOSCallback Callback);
-
 // Functions related to signal handling.
 typedef void (*SignalHandlerType)(int, void *, void *);
-bool IsHandledDeadlySignal(int signum);
+HandleSignalMode GetHandleSignalMode(int signum);
 void InstallDeadlySignalHandlers(SignalHandlerType handler);
+
+// Signal reporting.
+// Each sanitizer uses slightly different implementation of stack unwinding.
+typedef void (*UnwindSignalStackCallbackType)(const SignalContext &sig,
+                                              const void *callback_context,
+                                              BufferedStackTrace *stack);
+// Print deadly signal report and die.
+void HandleDeadlySignal(void *siginfo, void *context, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context);
+
+// Part of HandleDeadlySignal, exposed for asan.
+void StartReportDeadlySignal();
+// Part of HandleDeadlySignal, exposed for asan.
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context);
+
 // Alternative signal stack (POSIX-only).
 void SetAlternateSignalStack();
 void UnsetAlternateSignalStack();
@@ -392,12 +353,16 @@ const int kMaxSummaryLength = 1024;
 // Construct a one-line string:
 //   SUMMARY: SanitizerToolName: error_message
 // and pass it to __sanitizer_report_error_summary.
-void ReportErrorSummary(const char *error_message);
+// If alt_tool_name is provided, it's used in place of SanitizerToolName.
+void ReportErrorSummary(const char *error_message,
+                        const char *alt_tool_name = nullptr);
 // Same as above, but construct error_message as:
 //   error_type file:line[:column][ function]
-void ReportErrorSummary(const char *error_type, const AddressInfo &info);
+void ReportErrorSummary(const char *error_type, const AddressInfo &info,
+                        const char *alt_tool_name = nullptr);
 // Same as above, but obtains AddressInfo by symbolizing top stack trace frame.
-void ReportErrorSummary(const char *error_type, const StackTrace *trace);
+void ReportErrorSummary(const char *error_type, const StackTrace *trace,
+                        const char *alt_tool_name = nullptr);
 
 // Math
 #if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__)
@@ -549,6 +514,13 @@ class InternalMmapVectorNoCtor {
   uptr capacity() const {
     return capacity_;
   }
+  void resize(uptr new_size) {
+    Resize(new_size);
+    if (new_size > size_) {
+      internal_memset(&data_[size_], 0, sizeof(T) * (new_size - size_));
+    }
+    size_ = new_size;
+  }
 
   void clear() { size_ = 0; }
   bool empty() const { return size() == 0; }
@@ -633,43 +605,108 @@ void InternalSort(Container *v, uptr size, Compare comp) {
   }
 }
 
-template<class Container, class Value, class Compare>
-uptr InternalBinarySearch(const Container &v, uptr first, uptr last,
-                          const Value &val, Compare comp) {
-  uptr not_found = last + 1;
-  while (last >= first) {
+// Works like std::lower_bound: finds the first element that is not less
+// than the val.
+template <class Container, class Value, class Compare>
+uptr InternalLowerBound(const Container &v, uptr first, uptr last,
+                        const Value &val, Compare comp) {
+  while (last > first) {
     uptr mid = (first + last) / 2;
     if (comp(v[mid], val))
       first = mid + 1;
-    else if (comp(val, v[mid]))
-      last = mid - 1;
     else
-      return mid;
+      last = mid;
+  }
+  return first;
+}
+
+enum ModuleArch {
+  kModuleArchUnknown,
+  kModuleArchI386,
+  kModuleArchX86_64,
+  kModuleArchX86_64H,
+  kModuleArchARMV6,
+  kModuleArchARMV7,
+  kModuleArchARMV7S,
+  kModuleArchARMV7K,
+  kModuleArchARM64
+};
+
+// When adding a new architecture, don't forget to also update
+// script/asan_symbolize.py and sanitizer_symbolizer_libcdep.cc.
+inline const char *ModuleArchToString(ModuleArch arch) {
+  switch (arch) {
+    case kModuleArchUnknown:
+      return "";
+    case kModuleArchI386:
+      return "i386";
+    case kModuleArchX86_64:
+      return "x86_64";
+    case kModuleArchX86_64H:
+      return "x86_64h";
+    case kModuleArchARMV6:
+      return "armv6";
+    case kModuleArchARMV7:
+      return "armv7";
+    case kModuleArchARMV7S:
+      return "armv7s";
+    case kModuleArchARMV7K:
+      return "armv7k";
+    case kModuleArchARM64:
+      return "arm64";
   }
-  return not_found;
+  CHECK(0 && "Invalid module arch");
+  return "";
 }
 
+const uptr kModuleUUIDSize = 16;
+const uptr kMaxSegName = 16;
+
 // Represents a binary loaded into virtual memory (e.g. this can be an
 // executable or a shared object).
 class LoadedModule {
  public:
-  LoadedModule() : full_name_(nullptr), base_address_(0) { ranges_.clear(); }
+  LoadedModule()
+      : full_name_(nullptr),
+        base_address_(0),
+        max_executable_address_(0),
+        arch_(kModuleArchUnknown),
+        instrumented_(false) {
+    internal_memset(uuid_, 0, kModuleUUIDSize);
+    ranges_.clear();
+  }
   void set(const char *module_name, uptr base_address);
+  void set(const char *module_name, uptr base_address, ModuleArch arch,
+           u8 uuid[kModuleUUIDSize], bool instrumented);
   void clear();
-  void addAddressRange(uptr beg, uptr end, bool executable);
+  void addAddressRange(uptr beg, uptr end, bool executable, bool writable,
+                       const char *name = nullptr);
   bool containsAddress(uptr address) const;
 
   const char *full_name() const { return full_name_; }
   uptr base_address() const { return base_address_; }
+  uptr max_executable_address() const { return max_executable_address_; }
+  ModuleArch arch() const { return arch_; }
+  const u8 *uuid() const { return uuid_; }
+  bool instrumented() const { return instrumented_; }
 
   struct AddressRange {
     AddressRange *next;
     uptr beg;
     uptr end;
     bool executable;
-
-    AddressRange(uptr beg, uptr end, bool executable)
-        : next(nullptr), beg(beg), end(end), executable(executable) {}
+    bool writable;
+    char name[kMaxSegName];
+
+    AddressRange(uptr beg, uptr end, bool executable, bool writable,
+                 const char *name)
+        : next(nullptr),
+          beg(beg),
+          end(end),
+          executable(executable),
+          writable(writable) {
+      internal_strncpy(this->name, (name ? name : ""), ARRAY_SIZE(this->name));
+    }
   };
 
   const IntrusiveList<AddressRange> &ranges() const { return ranges_; }
@@ -677,6 +714,10 @@ class LoadedModule {
  private:
   char *full_name_;  // Owned.
   uptr base_address_;
+  uptr max_executable_address_;
+  ModuleArch arch_;
+  u8 uuid_[kModuleUUIDSize];
+  bool instrumented_;
   IntrusiveList<AddressRange> ranges_;
 };
 
@@ -684,9 +725,10 @@ class LoadedModule {
 // filling this information.
 class ListOfModules {
  public:
-  ListOfModules() : modules_(kInitialCapacity) {}
+  ListOfModules() : initialized(false) {}
   ~ListOfModules() { clear(); }
   void init();
+  void fallbackInit();  // Uses fallback init if available, otherwise clears
   const LoadedModule *begin() const { return modules_.begin(); }
   LoadedModule *begin() { return modules_.begin(); }
   const LoadedModule *end() const { return modules_.end(); }
@@ -702,10 +744,15 @@ class ListOfModules {
     for (auto &module : modules_) module.clear();
     modules_.clear();
   }
+  void clearOrInit() {
+    initialized ? clear() : modules_.Initialize(kInitialCapacity);
+    initialized = true;
+  }
 
-  InternalMmapVector<LoadedModule> modules_;
+  InternalMmapVectorNoCtor<LoadedModule> modules_;
   // We rarely have more than 16K loaded modules.
   static const uptr kInitialCapacity = 1 << 14;
+  bool initialized;
 };
 
 // Callback type for iterating over a set of memory ranges.
@@ -737,8 +784,11 @@ INLINE void LogMessageOnPrintf(const char *str) {}
 #if SANITIZER_LINUX
 // Initialize Android logging. Any writes before this are silently lost.
 void AndroidLogInit();
+void SetAbortMessage(const char *);
 #else
 INLINE void AndroidLogInit() {}
+// FIXME: MacOS implementation could use CRSetCrashLogMessage.
+INLINE void SetAbortMessage(const char *) {}
 #endif
 
 #if SANITIZER_ANDROID
@@ -778,33 +828,49 @@ static inline void SanitizerBreakOptimization(void *arg) {
 }
 
 struct SignalContext {
+  void *siginfo;
   void *context;
   uptr addr;
   uptr pc;
   uptr sp;
   uptr bp;
   bool is_memory_access;
-
   enum WriteFlag { UNKNOWN, READ, WRITE } write_flag;
 
-  SignalContext(void *context, uptr addr, uptr pc, uptr sp, uptr bp,
-                bool is_memory_access, WriteFlag write_flag)
-      : context(context),
-        addr(addr),
-        pc(pc),
-        sp(sp),
-        bp(bp),
-        is_memory_access(is_memory_access),
-        write_flag(write_flag) {}
+  // VS2013 doesn't implement unrestricted unions, so we need a trivial default
+  // constructor
+  SignalContext() = default;
 
   // Creates signal context in a platform-specific manner.
-  static SignalContext Create(void *siginfo, void *context);
+  // SignalContext is going to keep pointers to siginfo and context without
+  // owning them.
+  SignalContext(void *siginfo, void *context)
+      : siginfo(siginfo),
+        context(context),
+        addr(GetAddress()),
+        is_memory_access(IsMemoryAccess()),
+        write_flag(GetWriteFlag()) {
+    InitPcSpBp();
+  }
 
-  // Returns true if the "context" indicates a memory write.
-  static WriteFlag GetWriteFlag(void *context);
-};
+  static void DumpAllRegisters(void *context);
+
+  // Type of signal e.g. SIGSEGV or EXCEPTION_ACCESS_VIOLATION.
+  int GetType() const;
+
+  // String description of the signal.
+  const char *Describe() const;
+
+  // Returns true if signal is stack overflow.
+  bool IsStackOverflow() const;
 
-void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp);
+ private:
+  // Platform specific initialization.
+  void InitPcSpBp();
+  uptr GetAddress() const;
+  WriteFlag GetWriteFlag() const;
+  bool IsMemoryAccess() const;
+};
 
 void MaybeReexec();
 
@@ -840,6 +906,16 @@ struct StackDepotStats {
   uptr allocated;
 };
 
+// The default value for allocator_release_to_os_interval_ms common flag to
+// indicate that sanitizer allocator should not attempt to release memory to OS.
+const s32 kReleaseToOSIntervalNever = -1;
+
+void CheckNoDeepBind(const char *filename, int flag);
+
+// Returns the requested amount of random data (up to 256 bytes) that can then
+// be used to seed a PRNG. Defaults to blocking like the underlying syscall.
+bool GetRandom(void *buffer, uptr length, bool blocking = true);
+
 }  // namespace __sanitizer
 
 inline void *operator new(__sanitizer::operator_new_size_type size,
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
index 195014022a0..3f32b2f78ef 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
@@ -22,32 +22,34 @@
 //   COMMON_INTERCEPTOR_SET_THREAD_NAME
 //   COMMON_INTERCEPTOR_ON_DLOPEN
 //   COMMON_INTERCEPTOR_ON_EXIT
-//   COMMON_INTERCEPTOR_MUTEX_LOCK
+//   COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
+//   COMMON_INTERCEPTOR_MUTEX_POST_LOCK
 //   COMMON_INTERCEPTOR_MUTEX_UNLOCK
 //   COMMON_INTERCEPTOR_MUTEX_REPAIR
 //   COMMON_INTERCEPTOR_SET_PTHREAD_NAME
 //   COMMON_INTERCEPTOR_HANDLE_RECVMSG
 //   COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
+//   COMMON_INTERCEPTOR_MEMSET_IMPL
+//   COMMON_INTERCEPTOR_MEMMOVE_IMPL
+//   COMMON_INTERCEPTOR_MEMCPY_IMPL
+//   COMMON_INTERCEPTOR_COPY_STRING
+//   COMMON_INTERCEPTOR_STRNDUP_IMPL
 //===----------------------------------------------------------------------===//
 
 #include "interception/interception.h"
 #include "sanitizer_addrhashmap.h"
+#include "sanitizer_errno.h"
 #include "sanitizer_placement_new.h"
 #include "sanitizer_platform_interceptors.h"
+#include "sanitizer_symbolizer.h"
 #include "sanitizer_tls_get_addr.h"
 
 #include <stdarg.h>
 
 #if SANITIZER_INTERCEPTOR_HOOKS
-#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...)                                     \
-  do {                                                                         \
-    if (f)                                                                     \
-      f(__VA_ARGS__);                                                          \
-  } while (false);
-#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...)                                  \
-  extern "C" {                                                                 \
-  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void f(__VA_ARGS__);  \
-  } // extern "C"
+#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) f(__VA_ARGS__);
+#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \
+  SANITIZER_INTERFACE_WEAK_DEF(void, f, __VA_ARGS__) {}
 #else
 #define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...)
 #define CALL_WEAK_INTERCEPTOR_HOOK(f, ...)
@@ -65,6 +67,19 @@
 #define iconv __bsd_iconv
 #endif
 
+// Platform-specific options.
+#if SANITIZER_MAC
+namespace __sanitizer {
+bool PlatformHasDifferentMemcpyAndMemmove();
+}
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE \
+  (__sanitizer::PlatformHasDifferentMemcpyAndMemmove())
+#elif SANITIZER_WINDOWS64
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
+#else
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE true
+#endif  // SANITIZER_MAC
+
 #ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
 #define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}
 #endif
@@ -77,8 +92,12 @@
 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) {}
 #endif
 
-#ifndef COMMON_INTERCEPTOR_MUTEX_LOCK
-#define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) {}
+#ifndef COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
+#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MUTEX_POST_LOCK
+#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) {}
 #endif
 
 #ifndef COMMON_INTERCEPTOR_MUTEX_UNLOCK
@@ -122,15 +141,13 @@
 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (0)
 #endif
 
-#define COMMON_INTERCEPTOR_READ_STRING_OF_LEN(ctx, s, len, n)       \
-    COMMON_INTERCEPTOR_READ_RANGE((ctx), (s),                       \
-      common_flags()->strict_string_checks ? (len) + 1 : (n) )
-
 #define COMMON_INTERCEPTOR_READ_STRING(ctx, s, n)                   \
-    COMMON_INTERCEPTOR_READ_STRING_OF_LEN((ctx), (s), REAL(strlen)(s), (n))
+    COMMON_INTERCEPTOR_READ_RANGE((ctx), (s),                       \
+      common_flags()->strict_string_checks ? (REAL(strlen)(s)) + 1 : (n) )
 
 #ifndef COMMON_INTERCEPTOR_ON_DLOPEN
-#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) {}
+#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \
+  CheckNoDeepBind(filename, flag);
 #endif
 
 #ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE
@@ -161,6 +178,65 @@
     COMMON_INTERCEPT_FUNCTION(fn)
 #endif
 
+#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
+  {                                                       \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)        \
+      return internal_memset(dst, v, size);               \
+    COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size);  \
+    if (common_flags()->intercept_intrin)                 \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);     \
+    return REAL(memset)(dst, v, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
+  {                                                          \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)           \
+      return internal_memmove(dst, src, size);               \
+    COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size);  \
+    if (common_flags()->intercept_intrin) {                  \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);        \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);         \
+    }                                                        \
+    return REAL(memmove)(dst, src, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
+  {                                                         \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) {        \
+      return internal_memmove(dst, src, size);              \
+    }                                                       \
+    COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size);  \
+    if (common_flags()->intercept_intrin) {                 \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);       \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);        \
+    }                                                       \
+    return REAL(memcpy)(dst, src, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_COPY_STRING
+#define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_STRNDUP_IMPL
+#define COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size)                         \
+  COMMON_INTERCEPTOR_ENTER(ctx, strndup, s, size);                            \
+  uptr copy_length = internal_strnlen(s, size);                               \
+  char *new_mem = (char *)WRAP(malloc)(copy_length + 1);                      \
+  if (common_flags()->intercept_strndup) {                                    \
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1));       \
+  }                                                                           \
+  COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length);               \
+  internal_memcpy(new_mem, s, copy_length);                                   \
+  new_mem[copy_length] = '\0';                                                \
+  return new_mem;
+#endif
+
 struct FileMetadata {
   // For open_memstream().
   char **addr;
@@ -181,7 +257,7 @@ typedef AddrHashMap<CommonInterceptorMetadata, 31051> MetadataHashMap;
 
 static MetadataHashMap *interceptor_metadata_map;
 
-#if SI_NOT_WINDOWS
+#if SI_POSIX
 UNUSED static void SetInterceptorMetadata(__sanitizer_FILE *addr,
                                           const FileMetadata &file) {
   MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr);
@@ -208,7 +284,7 @@ UNUSED static void DeleteInterceptorMetadata(void *addr) {
   MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr, true);
   CHECK(h.exists());
 }
-#endif  // SI_NOT_WINDOWS
+#endif  // SI_POSIX
 
 #if SANITIZER_INTERCEPT_STRLEN
 INTERCEPTOR(SIZE_T, strlen, const char *s) {
@@ -244,11 +320,31 @@ INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T maxlen) {
 #define INIT_STRNLEN
 #endif
 
+#if SANITIZER_INTERCEPT_STRNDUP
+INTERCEPTOR(char*, strndup, const char *s, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
+}
+#define INIT_STRNDUP COMMON_INTERCEPT_FUNCTION(strndup)
+#else
+#define INIT_STRNDUP
+#endif // SANITIZER_INTERCEPT_STRNDUP
+
+#if SANITIZER_INTERCEPT___STRNDUP
+INTERCEPTOR(char*, __strndup, const char *s, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
+}
+#define INIT___STRNDUP COMMON_INTERCEPT_FUNCTION(__strndup)
+#else
+#define INIT___STRNDUP
+#endif // SANITIZER_INTERCEPT___STRNDUP
+
 #if SANITIZER_INTERCEPT_TEXTDOMAIN
 INTERCEPTOR(char*, textdomain, const char *domainname) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, textdomain, domainname);
-  COMMON_INTERCEPTOR_READ_STRING(ctx, domainname, 0);
+  if (domainname) COMMON_INTERCEPTOR_READ_STRING(ctx, domainname, 0);
   char *domain = REAL(textdomain)(domainname);
   if (domain) {
     COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, REAL(strlen)(domain) + 1);
@@ -302,8 +398,14 @@ INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) {
     c2 = (unsigned char)s2[i];
     if (c1 != c2 || c1 == '\0') break;
   }
-  COMMON_INTERCEPTOR_READ_STRING(ctx, s1, Min(i + 1, size));
-  COMMON_INTERCEPTOR_READ_STRING(ctx, s2, Min(i + 1, size));
+  uptr i1 = i;
+  uptr i2 = i;
+  if (common_flags()->strict_string_checks) {
+    for (; i1 < size && s1[i1]; i1++) {}
+    for (; i2 < size && s2[i2]; i2++) {}
+  }
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s1), Min(i1 + 1, size));
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s2), Min(i2 + 1, size));
   int result = CharCmpX(c1, c2);
   CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, GET_CALLER_PC(), s1,
                              s2, size, result);
@@ -346,24 +448,30 @@ INTERCEPTOR(int, strcasecmp, const char *s1, const char *s2) {
 }
 
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, uptr called_pc,
-                              const char *s1, const char *s2, uptr n,
+                              const char *s1, const char *s2, uptr size,
                               int result)
 
-INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T n) {
+INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T size) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, strncasecmp, s1, s2, n);
+  COMMON_INTERCEPTOR_ENTER(ctx, strncasecmp, s1, s2, size);
   unsigned char c1 = 0, c2 = 0;
   uptr i;
-  for (i = 0; i < n; i++) {
+  for (i = 0; i < size; i++) {
     c1 = (unsigned char)s1[i];
     c2 = (unsigned char)s2[i];
     if (CharCaseCmp(c1, c2) != 0 || c1 == '\0') break;
   }
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, Min(i + 1, n));
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, Min(i + 1, n));
+  uptr i1 = i;
+  uptr i2 = i;
+  if (common_flags()->strict_string_checks) {
+    for (; i1 < size && s1[i1]; i1++) {}
+    for (; i2 < size && s2[i2]; i2++) {}
+  }
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s1), Min(i1 + 1, size));
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s2), Min(i2 + 1, size));
   int result = CharCaseCmp(c1, c2);
   CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, GET_CALLER_PC(),
-                             s1, s2, n, result);
+                             s1, s2, size, result);
   return result;
 }
 
@@ -379,8 +487,7 @@ static inline void StrstrCheck(void *ctx, char *r, const char *s1,
                                const char *s2) {
     uptr len1 = REAL(strlen)(s1);
     uptr len2 = REAL(strlen)(s2);
-    COMMON_INTERCEPTOR_READ_STRING_OF_LEN(ctx, s1, len1,
-                                          r ? r - s1 + len2 : len1 + 1);
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r ? r - s1 + len2 : len1 + 1);
     COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2 + 1);
 }
 #endif
@@ -429,6 +536,52 @@ INTERCEPTOR(char*, strcasestr, const char *s1, const char *s2) {
 #define INIT_STRCASESTR
 #endif
 
+#if SANITIZER_INTERCEPT_STRTOK
+
+INTERCEPTOR(char*, strtok, char *str, const char *delimiters) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strtok, str, delimiters);
+  if (!common_flags()->intercept_strtok) {
+    return REAL(strtok)(str, delimiters);
+  }
+  if (common_flags()->strict_string_checks) {
+    // If strict_string_checks is enabled, we check the whole first argument
+    // string on the first call (strtok saves this string in a static buffer
+    // for subsequent calls). We do not need to check strtok's result.
+    // As the delimiters can change, we check them every call.
+    if (str != nullptr) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+    }
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters,
+                                  REAL(strlen)(delimiters) + 1);
+    return REAL(strtok)(str, delimiters);
+  } else {
+    // However, when strict_string_checks is disabled we cannot check the
+    // whole string on the first call. Instead, we check the result string
+    // which is guaranteed to be a NULL-terminated substring of the first
+    // argument. We also conservatively check one character of str and the
+    // delimiters.
+    if (str != nullptr) {
+      COMMON_INTERCEPTOR_READ_STRING(ctx, str, 1);
+    }
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters, 1);
+    char *result = REAL(strtok)(str, delimiters);
+    if (result != nullptr) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, result, REAL(strlen)(result) + 1);
+    } else if (str != nullptr) {
+      // No delimiter were found, it's safe to assume that the entire str was
+      // scanned.
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+    }
+    return result;
+  }
+}
+
+#define INIT_STRTOK COMMON_INTERCEPT_FUNCTION(strtok)
+#else
+#define INIT_STRTOK
+#endif
+
 #if SANITIZER_INTERCEPT_MEMMEM
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memmem, uptr called_pc,
                               const void *s1, SIZE_T len1, const void *s2,
@@ -460,10 +613,11 @@ INTERCEPTOR(char*, strchr, const char *s, int c) {
     return internal_strchr(s, c);
   COMMON_INTERCEPTOR_ENTER(ctx, strchr, s, c);
   char *result = REAL(strchr)(s, c);
-  uptr len = internal_strlen(s);
-  uptr n = result ? result - s + 1 : len + 1;
-  if (common_flags()->intercept_strchr)
-    COMMON_INTERCEPTOR_READ_STRING_OF_LEN(ctx, s, len, n);
+  if (common_flags()->intercept_strchr) {
+    // Keep strlen as macro argument, as macro may ignore it.
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s,
+      (result ? result - s : REAL(strlen)(s)) + 1);
+  }
   return result;
 }
 #define INIT_STRCHR COMMON_INTERCEPT_FUNCTION(strchr)
@@ -492,9 +646,8 @@ INTERCEPTOR(char*, strrchr, const char *s, int c) {
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return internal_strrchr(s, c);
   COMMON_INTERCEPTOR_ENTER(ctx, strrchr, s, c);
-  uptr len = internal_strlen(s);
   if (common_flags()->intercept_strchr)
-    COMMON_INTERCEPTOR_READ_STRING_OF_LEN(ctx, s, len, len + 1);
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
   return REAL(strrchr)(s, c);
 }
 #define INIT_STRRCHR COMMON_INTERCEPT_FUNCTION(strrchr)
@@ -551,14 +704,9 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) {
 #endif
 
 #if SANITIZER_INTERCEPT_MEMSET
-INTERCEPTOR(void*, memset, void *dst, int v, uptr size) {
-  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
-    return internal_memset(dst, v, size);
+INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size);
-  if (common_flags()->intercept_intrin)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);
-  return REAL(memset)(dst, v, size);
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
 }
 
 #define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
@@ -567,16 +715,9 @@ INTERCEPTOR(void*, memset, void *dst, int v, uptr size) {
 #endif
 
 #if SANITIZER_INTERCEPT_MEMMOVE
-INTERCEPTOR(void*, memmove, void *dst, const void *src, uptr size) {
-  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
-    return internal_memmove(dst, src, size);
+INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size);
-  if (common_flags()->intercept_intrin) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);
-  }
-  return REAL(memmove)(dst, src, size);
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
 }
 
 #define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
@@ -585,25 +726,30 @@ INTERCEPTOR(void*, memmove, void *dst, const void *src, uptr size) {
 #endif
 
 #if SANITIZER_INTERCEPT_MEMCPY
-INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) {
-  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) {
-    // On OS X, calling internal_memcpy here will cause memory corruptions,
-    // because memcpy and memmove are actually aliases of the same
-    // implementation.  We need to use internal_memmove here.
-    return internal_memmove(dst, src, size);
-  }
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size);
-  if (common_flags()->intercept_intrin) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);
-  }
+INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
+  // On OS X, calling internal_memcpy here will cause memory corruptions,
+  // because memcpy and memmove are actually aliases of the same
+  // implementation.  We need to use internal_memmove here.
   // N.B.: If we switch this to internal_ we'll have to use internal_memmove
   // due to memcpy being an alias of memmove on OS X.
-  return REAL(memcpy)(dst, src, size);
+  void *ctx;
+  if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) {
+    COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
+  } else {
+    COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+  }
 }
 
-#define INIT_MEMCPY COMMON_INTERCEPT_FUNCTION(memcpy)
+#define INIT_MEMCPY                                  \
+  do {                                               \
+    if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
+      COMMON_INTERCEPT_FUNCTION(memcpy);             \
+    } else {                                         \
+      ASSIGN_REAL(memcpy, memmove);                  \
+    }                                                \
+    CHECK(REAL(memcpy));                             \
+  } while (false)
+
 #else
 #define INIT_MEMCPY
 #endif
@@ -739,7 +885,7 @@ INTERCEPTOR(long double, frexpl, long double x, int *exp) {
 #define INIT_FREXPF_FREXPL
 #endif  // SANITIZER_INTERCEPT_FREXPF_FREXPL
 
-#if SI_NOT_WINDOWS
+#if SI_POSIX
 static void write_iovec(void *ctx, struct __sanitizer_iovec *iovec,
                         SIZE_T iovlen, SIZE_T maxlen) {
   for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
@@ -778,6 +924,23 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) {
 #define INIT_READ
 #endif
 
+#if SANITIZER_INTERCEPT_FREAD
+INTERCEPTOR(SIZE_T, fread, void *ptr, SIZE_T size, SIZE_T nmemb, void *file) {
+  // libc file streams can call user-supplied functions, see fopencookie.
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fread, ptr, size, nmemb, file);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(fread)(ptr, size, nmemb, file);
+  if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res * size);
+  return res;
+}
+#define INIT_FREAD COMMON_INTERCEPT_FUNCTION(fread)
+#else
+#define INIT_FREAD
+#endif
+
 #if SANITIZER_INTERCEPT_PREAD
 INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) {
   void *ctx;
@@ -878,6 +1041,20 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {
 #define INIT_WRITE
 #endif
 
+#if SANITIZER_INTERCEPT_FWRITE
+INTERCEPTOR(SIZE_T, fwrite, const void *p, uptr size, uptr nmemb, void *file) {
+  // libc file streams can call user-supplied functions, see fopencookie.
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fwrite, p, size, nmemb, file);
+  SIZE_T res = REAL(fwrite)(p, size, nmemb, file);
+  if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, p, res * size);
+  return res;
+}
+#define INIT_FWRITE COMMON_INTERCEPT_FUNCTION(fwrite)
+#else
+#define INIT_FWRITE
+#endif
+
 #if SANITIZER_INTERCEPT_PWRITE
 INTERCEPTOR(SSIZE_T, pwrite, int fd, void *ptr, SIZE_T count, OFF_T offset) {
   void *ctx;
@@ -1225,12 +1402,12 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
 
 #if SANITIZER_INTERCEPT_SCANF
 #define INIT_SCANF                    \
-  COMMON_INTERCEPT_FUNCTION(scanf);   \
-  COMMON_INTERCEPT_FUNCTION(sscanf);  \
-  COMMON_INTERCEPT_FUNCTION(fscanf);  \
-  COMMON_INTERCEPT_FUNCTION(vscanf);  \
-  COMMON_INTERCEPT_FUNCTION(vsscanf); \
-  COMMON_INTERCEPT_FUNCTION(vfscanf);
+  COMMON_INTERCEPT_FUNCTION_LDBL(scanf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(sscanf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(fscanf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vscanf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vsscanf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vfscanf);
 #else
 #define INIT_SCANF
 #endif
@@ -1403,16 +1580,16 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size,
 
 #if SANITIZER_INTERCEPT_PRINTF
 #define INIT_PRINTF                     \
-  COMMON_INTERCEPT_FUNCTION(printf);    \
-  COMMON_INTERCEPT_FUNCTION(sprintf);   \
-  COMMON_INTERCEPT_FUNCTION(snprintf);  \
-  COMMON_INTERCEPT_FUNCTION(asprintf);  \
-  COMMON_INTERCEPT_FUNCTION(fprintf);   \
-  COMMON_INTERCEPT_FUNCTION(vprintf);   \
-  COMMON_INTERCEPT_FUNCTION(vsprintf);  \
-  COMMON_INTERCEPT_FUNCTION(vsnprintf); \
-  COMMON_INTERCEPT_FUNCTION(vasprintf); \
-  COMMON_INTERCEPT_FUNCTION(vfprintf);
+  COMMON_INTERCEPT_FUNCTION_LDBL(printf);    \
+  COMMON_INTERCEPT_FUNCTION_LDBL(sprintf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(snprintf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(asprintf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(fprintf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vprintf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vsprintf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vsnprintf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vasprintf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vfprintf);
 #else
 #define INIT_PRINTF
 #endif
@@ -3187,6 +3364,30 @@ INTERCEPTOR(char *, strerror, int errnum) {
 #endif
 
 #if SANITIZER_INTERCEPT_STRERROR_R
+// There are 2 versions of strerror_r:
+//  * POSIX version returns 0 on success, negative error code on failure,
+//    writes message to buf.
+//  * GNU version returns message pointer, which points to either buf or some
+//    static storage.
+#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \
+    SANITIZER_MAC || SANITIZER_ANDROID
+// POSIX version. Spec is not clear on whether buf is NULL-terminated.
+// At least on OSX, buf contents are valid even when the call fails.
+INTERCEPTOR(int, strerror_r, int errnum, char *buf, SIZE_T buflen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strerror_r, errnum, buf, buflen);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(strerror_r)(errnum, buf, buflen);
+
+  SIZE_T sz = internal_strnlen(buf, buflen);
+  if (sz < buflen) ++sz;
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sz);
+  return res;
+}
+#else
+// GNU version.
 INTERCEPTOR(char *, strerror_r, int errnum, char *buf, SIZE_T buflen) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, strerror_r, errnum, buf, buflen);
@@ -3194,24 +3395,14 @@ INTERCEPTOR(char *, strerror_r, int errnum, char *buf, SIZE_T buflen) {
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
   char *res = REAL(strerror_r)(errnum, buf, buflen);
-  // There are 2 versions of strerror_r:
-  //  * POSIX version returns 0 on success, negative error code on failure,
-  //    writes message to buf.
-  //  * GNU version returns message pointer, which points to either buf or some
-  //    static storage.
-  SIZE_T posix_res = (SIZE_T)res;
-  if (posix_res < 1024 || posix_res > (SIZE_T) - 1024) {
-    // POSIX version. Spec is not clear on whether buf is NULL-terminated.
-    // At least on OSX, buf contents are valid even when the call fails.
-    SIZE_T sz = internal_strnlen(buf, buflen);
-    if (sz < buflen) ++sz;
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sz);
-  } else {
-    // GNU version.
+  if (res == buf)
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
-  }
+  else
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
   return res;
 }
+#endif //(_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE ||
+       //SANITIZER_MAC
 #define INIT_STRERROR_R COMMON_INTERCEPT_FUNCTION(strerror_r);
 #else
 #define INIT_STRERROR_R
@@ -3397,7 +3588,7 @@ INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds,
   if (fds && nfds) read_pollfd(ctx, fds, nfds);
   if (timeout_ts)
     COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout_ts, struct_timespec_sz);
-  // FIXME: read sigmask when all of sigemptyset, etc are intercepted.
+  if (sigmask) COMMON_INTERCEPTOR_READ_RANGE(ctx, sigmask, sizeof(*sigmask));
   int res =
       COMMON_INTERCEPTOR_BLOCK_REAL(ppoll)(fds, nfds, timeout_ts, sigmask);
   if (fds && nfds) write_pollfd(ctx, fds, nfds);
@@ -3438,7 +3629,7 @@ INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) {
 INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, sigwait, set, sig);
-  // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
   // FIXME: under ASan the call below may write to freed memory and corrupt
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
@@ -3455,7 +3646,7 @@ INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) {
 INTERCEPTOR(int, sigwaitinfo, __sanitizer_sigset_t *set, void *info) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, sigwaitinfo, set, info);
-  // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
   // FIXME: under ASan the call below may write to freed memory and corrupt
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
@@ -3474,7 +3665,7 @@ INTERCEPTOR(int, sigtimedwait, __sanitizer_sigset_t *set, void *info,
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, sigtimedwait, set, info, timeout);
   if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz);
-  // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
   // FIXME: under ASan the call below may write to freed memory and corrupt
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
@@ -3537,7 +3728,7 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set,
             __sanitizer_sigset_t *oldset) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, sigprocmask, how, set, oldset);
-  // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
   // FIXME: under ASan the call below may write to freed memory and corrupt
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
@@ -3606,11 +3797,12 @@ INTERCEPTOR(void, _exit, int status) {
 INTERCEPTOR(int, pthread_mutex_lock, void *m) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_lock, m);
+  COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
   int res = REAL(pthread_mutex_lock)(m);
   if (res == errno_EOWNERDEAD)
     COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
   if (res == 0 || res == errno_EOWNERDEAD)
-    COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m);
+    COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
   if (res == errno_EINVAL)
     COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
   return res;
@@ -4484,7 +4676,7 @@ INTERCEPTOR(SIZE_T, iconv, void *cd, char **inbuf, SIZE_T *inbytesleft,
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
   SIZE_T res = REAL(iconv)(cd, inbuf, inbytesleft, outbuf, outbytesleft);
-  if (res != (SIZE_T) - 1 && outbuf && *outbuf > outbuf_orig) {
+  if (outbuf && *outbuf > outbuf_orig) {
     SIZE_T sz = (char *)*outbuf - (char *)outbuf_orig;
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, outbuf_orig, sz);
   }
@@ -4839,47 +5031,67 @@ INTERCEPTOR(int, capset, void *hdrp, const void *datap) {
 #endif
 
 #if SANITIZER_INTERCEPT_AEABI_MEM
-DECLARE_REAL_AND_INTERCEPTOR(void *, memmove, void *, const void *, uptr)
-DECLARE_REAL_AND_INTERCEPTOR(void *, memcpy, void *, const void *, uptr)
-DECLARE_REAL_AND_INTERCEPTOR(void *, memset, void *, int, uptr)
-
 INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) {
-  return WRAP(memmove)(to, from, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) {
-  return WRAP(memmove)(to, from, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) {
-  return WRAP(memmove)(to, from, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) {
-  return WRAP(memcpy)(to, from, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) {
-  return WRAP(memcpy)(to, from, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) {
-  return WRAP(memcpy)(to, from, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
 }
+
 // Note the argument order.
 INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) {
-  return WRAP(memset)(block, c, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) {
-  return WRAP(memset)(block, c, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) {
-  return WRAP(memset)(block, c, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) {
-  return WRAP(memset)(block, 0, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) {
-  return WRAP(memset)(block, 0, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
+
 INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
-  return WRAP(memset)(block, 0, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
+
 #define INIT_AEABI_MEM                         \
   COMMON_INTERCEPT_FUNCTION(__aeabi_memmove);  \
   COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
@@ -4898,11 +5110,11 @@ INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
 #endif  // SANITIZER_INTERCEPT_AEABI_MEM
 
 #if SANITIZER_INTERCEPT___BZERO
-DECLARE_REAL_AND_INTERCEPTOR(void *, memset, void *, int, uptr);
-
 INTERCEPTOR(void *, __bzero, void *block, uptr size) {
-  return WRAP(memset)(block, 0, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
 }
+
 #define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
 #else
 #define INIT___BZERO
@@ -5362,6 +5574,7 @@ INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
   if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
   COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag);
   void *res = REAL(dlopen)(filename, flag);
+  Symbolizer::GetOrInit()->InvalidateModuleList();
   COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
   return res;
 }
@@ -5370,6 +5583,7 @@ INTERCEPTOR(int, dlclose, void *handle) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlclose, handle);
   int res = REAL(dlclose)(handle);
+  Symbolizer::GetOrInit()->InvalidateModuleList();
   COMMON_INTERCEPTOR_LIBRARY_UNLOADED();
   return res;
 }
@@ -5892,6 +6106,152 @@ INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) {
 
 // FIXME: add other *stat interceptor
 
+#if SANITIZER_INTERCEPT_UTMP
+INTERCEPTOR(void *, getutent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutent, dummy);
+  void *res = REAL(getutent)(dummy);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmp_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutid, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutid, ut);
+  void *res = REAL(getutid)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmp_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutline, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutline, ut);
+  void *res = REAL(getutline)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmp_sz);
+  return res;
+}
+#define INIT_UTMP                      \
+  COMMON_INTERCEPT_FUNCTION(getutent); \
+  COMMON_INTERCEPT_FUNCTION(getutid);  \
+  COMMON_INTERCEPT_FUNCTION(getutline);
+#else
+#define INIT_UTMP
+#endif
+
+#if SANITIZER_INTERCEPT_UTMPX
+INTERCEPTOR(void *, getutxent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutxent, dummy);
+  void *res = REAL(getutxent)(dummy);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutxid, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutxid, ut);
+  void *res = REAL(getutxid)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutxline, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutxline, ut);
+  void *res = REAL(getutxline)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz);
+  return res;
+}
+#define INIT_UTMPX                      \
+  COMMON_INTERCEPT_FUNCTION(getutxent); \
+  COMMON_INTERCEPT_FUNCTION(getutxid);  \
+  COMMON_INTERCEPT_FUNCTION(getutxline);
+#else
+#define INIT_UTMPX
+#endif
+
+#if SANITIZER_INTERCEPT_GETLOADAVG
+INTERCEPTOR(int, getloadavg, double *loadavg, int nelem) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getloadavg, loadavg, nelem);
+  int res = REAL(getloadavg)(loadavg, nelem);
+  if (res > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, loadavg, res * sizeof(*loadavg));
+  return res;
+}
+#define INIT_GETLOADAVG                      \
+  COMMON_INTERCEPT_FUNCTION(getloadavg);
+#else
+#define INIT_GETLOADAVG
+#endif
+
+#if SANITIZER_INTERCEPT_MCHECK_MPROBE
+INTERCEPTOR(int, mcheck, void (*abortfunc)(int mstatus)) {
+  return 0;
+}
+
+INTERCEPTOR(int, mcheck_pedantic, void (*abortfunc)(int mstatus)) {
+  return 0;
+}
+
+INTERCEPTOR(int, mprobe, void *ptr) {
+  return 0;
+}
+#endif
+
+INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s);
+  SIZE_T res = REAL(wcslen)(s);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (res + 1));
+  return res;
+}
+
+INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcsnlen, s, n);
+  SIZE_T res = REAL(wcsnlen)(s, n);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * Min(res + 1, n));
+  return res;
+}
+#define INIT_WCSLEN                  \
+  COMMON_INTERCEPT_FUNCTION(wcslen); \
+  COMMON_INTERCEPT_FUNCTION(wcsnlen);
+
+#if SANITIZER_INTERCEPT_WCSCAT
+INTERCEPTOR(wchar_t *, wcscat, wchar_t *dst, const wchar_t *src) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcscat, dst, src);
+  SIZE_T src_size = REAL(wcslen)(src);
+  SIZE_T dst_size = REAL(wcslen)(dst);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, src, (src_size + 1) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,
+                                 (src_size + 1) * sizeof(wchar_t));
+  return REAL(wcscat)(dst, src);  // NOLINT
+}
+
+INTERCEPTOR(wchar_t *, wcsncat, wchar_t *dst, const wchar_t *src, SIZE_T n) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcsncat, dst, src, n);
+  SIZE_T src_size = REAL(wcsnlen)(src, n);
+  SIZE_T dst_size = REAL(wcslen)(dst);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, src,
+                                Min(src_size + 1, n) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,
+                                 (src_size + 1) * sizeof(wchar_t));
+  return REAL(wcsncat)(dst, src, n);  // NOLINT
+}
+#define INIT_WCSCAT                  \
+  COMMON_INTERCEPT_FUNCTION(wcscat); \
+  COMMON_INTERCEPT_FUNCTION(wcsncat);
+#else
+#define INIT_WCSCAT
+#endif
+
 static void InitializeCommonInterceptors() {
   static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];
   interceptor_metadata_map = new((void *)&metadata_mem) MetadataHashMap();
@@ -5899,6 +6259,8 @@ static void InitializeCommonInterceptors() {
   INIT_TEXTDOMAIN;
   INIT_STRLEN;
   INIT_STRNLEN;
+  INIT_STRNDUP;
+  INIT___STRNDUP;
   INIT_STRCMP;
   INIT_STRNCMP;
   INIT_STRCASECMP;
@@ -5909,6 +6271,7 @@ static void InitializeCommonInterceptors() {
   INIT_STRCHRNUL;
   INIT_STRRCHR;
   INIT_STRSPN;
+  INIT_STRTOK;
   INIT_STRPBRK;
   INIT_MEMSET;
   INIT_MEMMOVE;
@@ -5918,12 +6281,14 @@ static void InitializeCommonInterceptors() {
   INIT_MEMRCHR;
   INIT_MEMMEM;
   INIT_READ;
+  INIT_FREAD;
   INIT_PREAD;
   INIT_PREAD64;
   INIT_READV;
   INIT_PREADV;
   INIT_PREADV64;
   INIT_WRITE;
+  INIT_FWRITE;
   INIT_PWRITE;
   INIT_PWRITE64;
   INIT_WRITEV;
@@ -6088,4 +6453,9 @@ static void InitializeCommonInterceptors() {
   INIT___LXSTAT;
   INIT___LXSTAT64;
   // FIXME: add other *stat interceptors.
+  INIT_UTMP;
+  INIT_UTMPX;
+  INIT_GETLOADAVG;
+  INIT_WCSLEN;
+  INIT_WCSCAT;
 }
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc
index 9133be7097d..30927d2a229 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc
@@ -323,8 +323,8 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
       continue;
     int size = scanf_get_value_size(&dir);
     if (size == FSS_INVALID) {
-      Report("WARNING: unexpected format specifier in scanf interceptor: "
-        "%.*s\n", dir.end - dir.begin, dir.begin);
+      Report("%s: WARNING: unexpected format specifier in scanf interceptor: ",
+             SanitizerToolName, "%.*s\n", dir.end - dir.begin, dir.begin);
       break;
     }
     void *argp = va_arg(aq, void *);
@@ -433,10 +433,6 @@ static const char *printf_parse_next(const char *p, PrintfDirective *dir) {
 }
 
 static int printf_get_value_size(PrintfDirective *dir) {
-  if (dir->convSpecifier == 'm') {
-    return sizeof(char *);
-  }
-
   if (char_is_one_of(dir->convSpecifier, "cCsS")) {
     unsigned charSize =
         format_get_char_size(dir->convSpecifier, dir->lengthModifier);
@@ -517,10 +513,17 @@ static void printf_common(void *ctx, const char *format, va_list aq) {
       // Dynamic precision
       SKIP_SCALAR_ARG(&aq, 'd', sizeof(int));
     }
+    // %m does not require an argument: strlen(errno).
+    if (dir.convSpecifier == 'm')
+      continue;
     int size = printf_get_value_size(&dir);
     if (size == FSS_INVALID) {
-      Report("WARNING: unexpected format specifier in printf "
-             "interceptor: %.*s\n", dir.end - dir.begin, dir.begin);
+      static int ReportedOnce;
+      if (!ReportedOnce++)
+        Report(
+            "%s: WARNING: unexpected format specifier in printf "
+            "interceptor: %.*s (reported once per process)\n",
+            SanitizerToolName, dir.end - dir.begin, dir.begin);
       break;
     }
     if (dir.convSpecifier == 'n') {
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interface.inc b/libsanitizer/sanitizer_common/sanitizer_common_interface.inc
new file mode 100644
index 00000000000..bd296f6dece
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_common_interface.inc
@@ -0,0 +1,37 @@
+//===-- sanitizer_common_interface.inc ------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Sanitizer Common interface list.
+//===----------------------------------------------------------------------===//
+INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
+INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
+INTERFACE_FUNCTION(__sanitizer_set_death_callback)
+INTERFACE_FUNCTION(__sanitizer_set_report_path)
+INTERFACE_FUNCTION(__sanitizer_set_report_fd)
+INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
+INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
+INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
+// Sanitizer weak hooks
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_memcmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strcmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strncmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strstr)
+// Stacktrace interface.
+INTERFACE_FUNCTION(__sanitizer_get_module_and_offset_for_pc)
+INTERFACE_FUNCTION(__sanitizer_symbolize_global)
+INTERFACE_FUNCTION(__sanitizer_symbolize_pc)
+// Allocator interface.
+INTERFACE_FUNCTION(__sanitizer_get_allocated_size)
+INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes)
+INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size)
+INTERFACE_FUNCTION(__sanitizer_get_free_bytes)
+INTERFACE_FUNCTION(__sanitizer_get_heap_size)
+INTERFACE_FUNCTION(__sanitizer_get_ownership)
+INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes)
+INTERFACE_FUNCTION(__sanitizer_install_malloc_and_free_hooks)
+INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
+INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
+INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interface_posix.inc b/libsanitizer/sanitizer_common/sanitizer_common_interface_posix.inc
new file mode 100644
index 00000000000..d3b72a8eeb7
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_common_interface_posix.inc
@@ -0,0 +1,12 @@
+//===-- sanitizer_common_interface_posix.inc ------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Sanitizer Common interface list only available for Posix systems.
+//===----------------------------------------------------------------------===//
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc
index 8c9fa98f835..a3f35319e23 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc
@@ -12,7 +12,10 @@
 #include "sanitizer_common.h"
 
 #include "sanitizer_allocator_interface.h"
+#include "sanitizer_file.h"
 #include "sanitizer_flags.h"
+#include "sanitizer_procmaps.h"
+#include "sanitizer_report_decorator.h"
 #include "sanitizer_stackdepot.h"
 #include "sanitizer_stacktrace.h"
 #include "sanitizer_symbolizer.h"
@@ -23,12 +26,25 @@
 
 namespace __sanitizer {
 
+#if !SANITIZER_FUCHSIA
+
 bool ReportFile::SupportsColors() {
   SpinMutexLock l(mu);
   ReopenIfNecessary();
   return SupportsColoredOutput(fd);
 }
 
+static INLINE bool ReportSupportsColors() {
+  return report_file.SupportsColors();
+}
+
+#else  // SANITIZER_FUCHSIA
+
+// Fuchsia's logs always go through post-processing that handles colorization.
+static INLINE bool ReportSupportsColors() { return true; }
+
+#endif  // !SANITIZER_FUCHSIA
+
 bool ColorizeReports() {
   // FIXME: Add proper Windows support to AnsiColorDecorator and re-enable color
   // printing on Windows.
@@ -37,7 +53,7 @@ bool ColorizeReports() {
 
   const char *flag = common_flags()->color;
   return internal_strcmp(flag, "always") == 0 ||
-         (internal_strcmp(flag, "auto") == 0 && report_file.SupportsColors());
+         (internal_strcmp(flag, "auto") == 0 && ReportSupportsColors());
 }
 
 static void (*sandboxing_callback)();
@@ -45,7 +61,8 @@ void SetSandboxingCallback(void (*f)()) {
   sandboxing_callback = f;
 }
 
-void ReportErrorSummary(const char *error_type, const StackTrace *stack) {
+void ReportErrorSummary(const char *error_type, const StackTrace *stack,
+                        const char *alt_tool_name) {
 #if !SANITIZER_GO
   if (!common_flags()->print_summary)
     return;
@@ -57,7 +74,7 @@ void ReportErrorSummary(const char *error_type, const StackTrace *stack) {
   // Maybe sometimes we need to choose another frame (e.g. skip memcpy/etc).
   uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);
   SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
-  ReportErrorSummary(error_type, frame->info);
+  ReportErrorSummary(error_type, frame->info, alt_tool_name);
   frame->ClearAll();
 #endif
 }
@@ -68,18 +85,11 @@ void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
   SoftRssLimitExceededCallback = Callback;
 }
 
-static AllocatorReleaseToOSCallback ReleseCallback;
-void SetAllocatorReleaseToOSCallback(AllocatorReleaseToOSCallback Callback) {
-  CHECK_EQ(ReleseCallback, nullptr);
-  ReleseCallback = Callback;
-}
-
 #if SANITIZER_LINUX && !SANITIZER_GO
 void BackgroundThread(void *arg) {
   uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
   uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
   bool heap_profile = common_flags()->heap_profile;
-  bool allocator_release_to_os = common_flags()->allocator_release_to_os;
   uptr prev_reported_rss = 0;
   uptr prev_reported_stack_depot_size = 0;
   bool reached_soft_rss_limit = false;
@@ -125,17 +135,137 @@ void BackgroundThread(void *arg) {
           SoftRssLimitExceededCallback(false);
       }
     }
-    if (allocator_release_to_os && ReleseCallback) ReleseCallback();
     if (heap_profile &&
         current_rss_mb > rss_during_last_reported_profile * 1.1) {
       Printf("\n\nHEAP PROFILE at RSS %zdMb\n", current_rss_mb);
-      __sanitizer_print_memory_profile(90);
+      __sanitizer_print_memory_profile(90, 20);
       rss_during_last_reported_profile = current_rss_mb;
     }
   }
 }
 #endif
 
+#if !SANITIZER_FUCHSIA && !SANITIZER_GO
+void StartReportDeadlySignal() {
+  // Write the first message using fd=2, just in case.
+  // It may actually fail to write in case stderr is closed.
+  CatastrophicErrorWrite(SanitizerToolName, internal_strlen(SanitizerToolName));
+  static const char kDeadlySignal[] = ":DEADLYSIGNAL\n";
+  CatastrophicErrorWrite(kDeadlySignal, sizeof(kDeadlySignal) - 1);
+}
+
+static void MaybeReportNonExecRegion(uptr pc) {
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+  MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
+      Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
+  }
+#endif
+}
+
+static void PrintMemoryByte(InternalScopedString *str, const char *before,
+                            u8 byte) {
+  SanitizerCommonDecorator d;
+  str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
+              d.Default());
+}
+
+static void MaybeDumpInstructionBytes(uptr pc) {
+  if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached()))
+    return;
+  InternalScopedString str(1024);
+  str.append("First 16 instruction bytes at pc: ");
+  if (IsAccessibleMemoryRange(pc, 16)) {
+    for (int i = 0; i < 16; ++i) {
+      PrintMemoryByte(&str, "", ((u8 *)pc)[i]);
+    }
+    str.append("\n");
+  } else {
+    str.append("unaccessible\n");
+  }
+  Report("%s", str.data());
+}
+
+static void MaybeDumpRegisters(void *context) {
+  if (!common_flags()->dump_registers) return;
+  SignalContext::DumpAllRegisters(context);
+}
+
+static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid,
+                                    UnwindSignalStackCallbackType unwind,
+                                    const void *unwind_context) {
+  SanitizerCommonDecorator d;
+  Printf("%s", d.Warning());
+  static const char kDescription[] = "stack-overflow";
+  Report("ERROR: %s: %s on address %p (pc %p bp %p sp %p T%d)\n",
+         SanitizerToolName, kDescription, (void *)sig.addr, (void *)sig.pc,
+         (void *)sig.bp, (void *)sig.sp, tid);
+  Printf("%s", d.Default());
+  InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+  BufferedStackTrace *stack = stack_buffer.data();
+  stack->Reset();
+  unwind(sig, unwind_context, stack);
+  stack->Print();
+  ReportErrorSummary(kDescription, stack);
+}
+
+static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
+                                   UnwindSignalStackCallbackType unwind,
+                                   const void *unwind_context) {
+  SanitizerCommonDecorator d;
+  Printf("%s", d.Warning());
+  const char *description = sig.Describe();
+  Report("ERROR: %s: %s on unknown address %p (pc %p bp %p sp %p T%d)\n",
+         SanitizerToolName, description, (void *)sig.addr, (void *)sig.pc,
+         (void *)sig.bp, (void *)sig.sp, tid);
+  Printf("%s", d.Default());
+  if (sig.pc < GetPageSizeCached())
+    Report("Hint: pc points to the zero page.\n");
+  if (sig.is_memory_access) {
+    const char *access_type =
+        sig.write_flag == SignalContext::WRITE
+            ? "WRITE"
+            : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
+    Report("The signal is caused by a %s memory access.\n", access_type);
+    if (sig.addr < GetPageSizeCached())
+      Report("Hint: address points to the zero page.\n");
+  }
+  MaybeReportNonExecRegion(sig.pc);
+  InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+  BufferedStackTrace *stack = stack_buffer.data();
+  stack->Reset();
+  unwind(sig, unwind_context, stack);
+  stack->Print();
+  MaybeDumpInstructionBytes(sig.pc);
+  MaybeDumpRegisters(sig.context);
+  Printf("%s can not provide additional info.\n", SanitizerToolName);
+  ReportErrorSummary(description, stack);
+}
+
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context) {
+  if (sig.IsStackOverflow())
+    ReportStackOverflowImpl(sig, tid, unwind, unwind_context);
+  else
+    ReportDeadlySignalImpl(sig, tid, unwind, unwind_context);
+}
+
+void HandleDeadlySignal(void *siginfo, void *context, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context) {
+  StartReportDeadlySignal();
+  ScopedErrorReportLock rl;
+  SignalContext sig(siginfo, context);
+  ReportDeadlySignal(sig, tid, unwind, unwind_context);
+  Report("ABORTING\n");
+  Die();
+}
+
+#endif  // !SANITIZER_FUCHSIA && !SANITIZER_GO
+
 void WriteToSyslog(const char *msg) {
   InternalScopedString msg_copy(kErrorMessageBufferSize);
   msg_copy.append("%s", msg);
@@ -160,17 +290,56 @@ void MaybeStartBackgroudThread() {
   // Start the background thread if one of the rss limits is given.
   if (!common_flags()->hard_rss_limit_mb &&
       !common_flags()->soft_rss_limit_mb &&
-      !common_flags()->allocator_release_to_os &&
       !common_flags()->heap_profile) return;
   if (!&real_pthread_create) return;  // Can't spawn the thread anyway.
   internal_start_thread(BackgroundThread, nullptr);
 #endif
 }
 
+static atomic_uintptr_t reporting_thread = {0};
+
+ScopedErrorReportLock::ScopedErrorReportLock() {
+  uptr current = GetThreadSelf();
+  for (;;) {
+    uptr expected = 0;
+    if (atomic_compare_exchange_strong(&reporting_thread, &expected, current,
+                                       memory_order_relaxed)) {
+      // We've claimed reporting_thread so proceed.
+      CommonSanitizerReportMutex.Lock();
+      return;
+    }
+
+    if (expected == current) {
+      // This is either asynch signal or nested error during error reporting.
+      // Fail simple to avoid deadlocks in Report().
+
+      // Can't use Report() here because of potential deadlocks in nested
+      // signal handlers.
+      CatastrophicErrorWrite(SanitizerToolName,
+                             internal_strlen(SanitizerToolName));
+      static const char msg[] = ": nested bug in the same thread, aborting.\n";
+      CatastrophicErrorWrite(msg, sizeof(msg) - 1);
+
+      internal__exit(common_flags()->exitcode);
+    }
+
+    internal_sched_yield();
+  }
+}
+
+ScopedErrorReportLock::~ScopedErrorReportLock() {
+  CommonSanitizerReportMutex.Unlock();
+  atomic_store_relaxed(&reporting_thread, 0);
+}
+
+void ScopedErrorReportLock::CheckLocked() {
+  CommonSanitizerReportMutex.CheckLocked();
+}
+
 }  // namespace __sanitizer
 
-void NOINLINE
-__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args) {
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
+                             __sanitizer_sandbox_arguments *args) {
   __sanitizer::PrepareForSandboxing(args);
   if (__sanitizer::sandboxing_callback)
     __sanitizer::sandboxing_callback();
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cc b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cc
new file mode 100644
index 00000000000..7397a011098
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cc
@@ -0,0 +1,34 @@
+//===-- sanitizer_common_nolibc.cc ----------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains stubs for libc function to facilitate optional use of
+// libc in no-libcdep sources.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+
+namespace __sanitizer {
+
+// The Windows implementations of these functions use the win32 API directly,
+// bypassing libc.
+#if !SANITIZER_WINDOWS
+#if SANITIZER_LINUX
+bool ShouldLogAfterPrintf() { return false; }
+void LogMessageOnPrintf(const char *str) {}
+#endif
+void WriteToSyslog(const char *buffer) {}
+void Abort() { internal__exit(1); }
+void SleepForSeconds(int seconds) { internal_sleep(seconds); }
+#endif // !SANITIZER_WINDOWS
+
+#if !SANITIZER_WINDOWS && !SANITIZER_MAC
+void ListOfModules::init() {}
+#endif
+
+}  // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_fuchsia.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_fuchsia.cc
new file mode 100644
index 00000000000..7f294512e9a
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_coverage_fuchsia.cc
@@ -0,0 +1,238 @@
+//===-- sanitizer_coverage_fuchsia.cc ------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// Sanitizer Coverage Controller for Trace PC Guard, Fuchsia-specific version.
+//
+// This Fuchsia-specific implementation uses the same basic scheme and the
+// same simple '.sancov' file format as the generic implementation.  The
+// difference is that we just produce a single blob of output for the whole
+// program, not a separate one per DSO.  We do not sort the PC table and do
+// not prune the zeros, so the resulting file is always as large as it
+// would be to report 100% coverage.  Implicit tracing information about
+// the address ranges of DSOs allows offline tools to split the one big
+// blob into separate files that the 'sancov' tool can understand.
+//
+// Unlike the traditional implementation that uses an atexit hook to write
+// out data files at the end, the results on Fuchsia do not go into a file
+// per se.  The 'coverage_dir' option is ignored.  Instead, they are stored
+// directly into a shared memory object (a Zircon VMO).  At exit, that VMO
+// is handed over to a system service that's responsible for getting the
+// data out to somewhere that it can be fed into the sancov tool (where and
+// how is not our problem).
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+
+#include <zircon/process.h>
+#include <zircon/sanitizer.h>
+#include <zircon/syscalls.h>
+
+using namespace __sanitizer;  // NOLINT
+
+namespace __sancov {
+namespace {
+
+// TODO(mcgrathr): Move the constant into a header shared with other impls.
+constexpr u64 Magic64 = 0xC0BFFFFFFFFFFF64ULL;
+static_assert(SANITIZER_WORDSIZE == 64, "Fuchsia is always LP64");
+
+constexpr const char kSancovSinkName[] = "sancov";
+
+// Collects trace-pc guard coverage.
+// This class relies on zero-initialization.
+class TracePcGuardController {
+ public:
+  // For each PC location being tracked, there is a u32 reserved in global
+  // data called the "guard".  At startup, we assign each guard slot a
+  // unique index into the big results array.  Later during runtime, the
+  // first call to TracePcGuard (below) will store the corresponding PC at
+  // that index in the array.  (Each later call with the same guard slot is
+  // presumed to be from the same PC.)  Then it clears the guard slot back
+  // to zero, which tells the compiler not to bother calling in again.  At
+  // the end of the run, we have a big array where each element is either
+  // zero or is a tracked PC location that was hit in the trace.
+
+  // This is called from global constructors.  Each translation unit has a
+  // contiguous array of guard slots, and a constructor that calls here
+  // with the bounds of its array.  Those constructors are allowed to call
+  // here more than once for the same array.  Usually all of these
+  // constructors run in the initial thread, but it's possible that a
+  // dlopen call on a secondary thread will run constructors that get here.
+  void InitTracePcGuard(u32 *start, u32 *end) {
+    if (end > start && *start == 0 && common_flags()->coverage) {
+      // Complete the setup before filling in any guards with indices.
+      // This avoids the possibility of code called from Setup reentering
+      // TracePcGuard.
+      u32 idx = Setup(end - start);
+      for (u32 *p = start; p < end; ++p) {
+        *p = idx++;
+      }
+    }
+  }
+
+  void TracePcGuard(u32 *guard, uptr pc) {
+    atomic_uint32_t *guard_ptr = reinterpret_cast<atomic_uint32_t *>(guard);
+    u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed);
+    if (idx > 0) array_[idx] = pc;
+  }
+
+  void Dump() {
+    BlockingMutexLock locked(&setup_lock_);
+    if (array_) {
+      CHECK_NE(vmo_, ZX_HANDLE_INVALID);
+
+      // Publish the VMO to the system, where it can be collected and
+      // analyzed after this process exits.  This always consumes the VMO
+      // handle.  Any failure is just logged and not indicated to us.
+      __sanitizer_publish_data(kSancovSinkName, vmo_);
+      vmo_ = ZX_HANDLE_INVALID;
+
+      // This will route to __sanitizer_log_write, which will ensure that
+      // information about shared libraries is written out.  This message
+      // uses the `dumpfile` symbolizer markup element to highlight the
+      // dump.  See the explanation for this in:
+      // https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
+      Printf("SanitizerCoverage: {{{dumpfile:%s:%s}}} with up to %u PCs\n",
+             kSancovSinkName, vmo_name_, next_index_ - 1);
+    }
+  }
+
+ private:
+  // We map in the largest possible view into the VMO: one word
+  // for every possible 32-bit index value.  This avoids the need
+  // to change the mapping when increasing the size of the VMO.
+  // We can always spare the 32G of address space.
+  static constexpr size_t MappingSize = sizeof(uptr) << 32;
+
+  BlockingMutex setup_lock_;
+  uptr *array_;
+  u32 next_index_;
+  zx_handle_t vmo_;
+  char vmo_name_[ZX_MAX_NAME_LEN];
+
+  size_t DataSize() const { return next_index_ * sizeof(uintptr_t); }
+
+  u32 Setup(u32 num_guards) {
+    BlockingMutexLock locked(&setup_lock_);
+    DCHECK(common_flags()->coverage);
+
+    if (next_index_ == 0) {
+      CHECK_EQ(vmo_, ZX_HANDLE_INVALID);
+      CHECK_EQ(array_, nullptr);
+
+      // The first sample goes at [1] to reserve [0] for the magic number.
+      next_index_ = 1 + num_guards;
+
+      zx_status_t status = _zx_vmo_create(DataSize(), 0, &vmo_);
+      CHECK_EQ(status, ZX_OK);
+
+      // Give the VMO a name including our process KOID so it's easy to spot.
+      internal_snprintf(vmo_name_, sizeof(vmo_name_), "%s.%zu", kSancovSinkName,
+                        internal_getpid());
+      _zx_object_set_property(vmo_, ZX_PROP_NAME, vmo_name_,
+                              internal_strlen(vmo_name_));
+
+      // Map the largest possible view we might need into the VMO.  Later
+      // we might need to increase the VMO's size before we can use larger
+      // indices, but we'll never move the mapping address so we don't have
+      // any multi-thread synchronization issues with that.
+      uintptr_t mapping;
+      status =
+          _zx_vmar_map(_zx_vmar_root_self(), 0, vmo_, 0, MappingSize,
+                       ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &mapping);
+      CHECK_EQ(status, ZX_OK);
+
+      // Hereafter other threads are free to start storing into
+      // elements [1, next_index_) of the big array.
+      array_ = reinterpret_cast<uptr *>(mapping);
+
+      // Store the magic number.
+      // Hereafter, the VMO serves as the contents of the '.sancov' file.
+      array_[0] = Magic64;
+
+      return 1;
+    } else {
+      // The VMO is already mapped in, but it's not big enough to use the
+      // new indices.  So increase the size to cover the new maximum index.
+
+      CHECK_NE(vmo_, ZX_HANDLE_INVALID);
+      CHECK_NE(array_, nullptr);
+
+      uint32_t first_index = next_index_;
+      next_index_ += num_guards;
+
+      zx_status_t status = _zx_vmo_set_size(vmo_, DataSize());
+      CHECK_EQ(status, ZX_OK);
+
+      return first_index;
+    }
+  }
+};
+
+static TracePcGuardController pc_guard_controller;
+
+}  // namespace
+}  // namespace __sancov
+
+namespace __sanitizer {
+void InitializeCoverage(bool enabled, const char *dir) {
+  CHECK_EQ(enabled, common_flags()->coverage);
+  CHECK_EQ(dir, common_flags()->coverage_dir);
+
+  static bool coverage_enabled = false;
+  if (!coverage_enabled) {
+    coverage_enabled = enabled;
+    Atexit(__sanitizer_cov_dump);
+    AddDieCallback(__sanitizer_cov_dump);
+  }
+}
+}  // namespace __sanitizer
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(  // NOLINT
+    const uptr *pcs, uptr len) {
+  UNIMPLEMENTED();
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *guard) {
+  if (!*guard) return;
+  __sancov::pc_guard_controller.TracePcGuard(guard, GET_CALLER_PC() - 1);
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init,
+                             u32 *start, u32 *end) {
+  if (start == end || *start) return;
+  __sancov::pc_guard_controller.InitTracePcGuard(start, end);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() {
+  __sancov::pc_guard_controller.Dump();
+}
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() {
+  __sanitizer_dump_trace_pc_guard_coverage();
+}
+// Default empty implementations (weak). Users should redefine them.
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
+}  // extern "C"
+
+#endif  // !SANITIZER_FUCHSIA
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_interface.inc b/libsanitizer/sanitizer_common/sanitizer_coverage_interface.inc
new file mode 100644
index 00000000000..fb78cc00f70
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_coverage_interface.inc
@@ -0,0 +1,31 @@
+//===-- sanitizer_coverage_interface.inc ----------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Sanitizer Coverage interface list.
+//===----------------------------------------------------------------------===//
+INTERFACE_FUNCTION(__sanitizer_cov_dump)
+INTERFACE_FUNCTION(__sanitizer_cov_reset)
+INTERFACE_FUNCTION(__sanitizer_dump_coverage)
+INTERFACE_FUNCTION(__sanitizer_dump_trace_pc_guard_coverage)
+INTERFACE_WEAK_FUNCTION(__sancov_default_options)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp1)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp2)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp1)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp2)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_pcs_init)
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc
deleted file mode 100644
index dd8620beaac..00000000000
--- a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc
+++ /dev/null
@@ -1,1043 +0,0 @@
-//===-- sanitizer_coverage.cc ---------------------------------------------===//
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Sanitizer Coverage.
-// This file implements run-time support for a poor man's coverage tool.
-//
-// Compiler instrumentation:
-// For every interesting basic block the compiler injects the following code:
-// if (Guard < 0) {
-//    __sanitizer_cov(&Guard);
-// }
-// At the module start up time __sanitizer_cov_module_init sets the guards
-// to consecutive negative numbers (-1, -2, -3, ...).
-// It's fine to call __sanitizer_cov more than once for a given block.
-//
-// Run-time:
-//  - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC).
-//    and atomically set Guard to -Guard.
-//  - __sanitizer_cov_dump: dump the coverage data to disk.
-//  For every module of the current process that has coverage data
-//  this will create a file module_name.PID.sancov.
-//
-// The file format is simple: the first 8 bytes is the magic,
-// one of 0xC0BFFFFFFFFFFF64 and 0xC0BFFFFFFFFFFF32. The last byte of the
-// magic defines the size of the following offsets.
-// The rest of the data is the offsets in the module.
-//
-// Eventually, this coverage implementation should be obsoleted by a more
-// powerful general purpose Clang/LLVM coverage instrumentation.
-// Consider this implementation as prototype.
-//
-// FIXME: support (or at least test with) dlclose.
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_allocator_internal.h"
-#include "sanitizer_common.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_mutex.h"
-#include "sanitizer_procmaps.h"
-#include "sanitizer_stacktrace.h"
-#include "sanitizer_symbolizer.h"
-#include "sanitizer_flags.h"
-
-using namespace __sanitizer;
-
-static const u64 kMagic64 = 0xC0BFFFFFFFFFFF64ULL;
-static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL;
-static const uptr kNumWordsForMagic = SANITIZER_WORDSIZE == 64 ? 1 : 2;
-static const u64 kMagic = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32;
-
-static atomic_uint32_t dump_once_guard;  // Ensure that CovDump runs only once.
-
-static atomic_uintptr_t coverage_counter;
-static atomic_uintptr_t caller_callee_counter;
-
-static void ResetGlobalCounters() {
-  return atomic_store(&coverage_counter, 0, memory_order_relaxed);
-  return atomic_store(&caller_callee_counter, 0, memory_order_relaxed);
-}
-
-// pc_array is the array containing the covered PCs.
-// To make the pc_array thread- and async-signal-safe it has to be large enough.
-// 128M counters "ought to be enough for anybody" (4M on 32-bit).
-
-// With coverage_direct=1 in ASAN_OPTIONS, pc_array memory is mapped to a file.
-// In this mode, __sanitizer_cov_dump does nothing, and CovUpdateMapping()
-// dump current memory layout to another file.
-
-static bool cov_sandboxed = false;
-static fd_t cov_fd = kInvalidFd;
-static unsigned int cov_max_block_size = 0;
-static bool coverage_enabled = false;
-static const char *coverage_dir;
-
-namespace __sanitizer {
-
-class CoverageData {
- public:
-  void Init();
-  void Enable();
-  void Disable();
-  void ReInit();
-  void BeforeFork();
-  void AfterFork(int child_pid);
-  void Extend(uptr npcs);
-  void Add(uptr pc, u32 *guard);
-  void IndirCall(uptr caller, uptr callee, uptr callee_cache[],
-                 uptr cache_size);
-  void DumpCallerCalleePairs();
-  void DumpTrace();
-  void DumpAsBitSet();
-  void DumpCounters();
-  void DumpOffsets();
-  void DumpAll();
-
-  ALWAYS_INLINE
-  void TraceBasicBlock(u32 *id);
-
-  void InitializeGuardArray(s32 *guards);
-  void InitializeGuards(s32 *guards, uptr n, const char *module_name,
-                        uptr caller_pc);
-  void InitializeCounters(u8 *counters, uptr n);
-  void ReinitializeGuards();
-  uptr GetNumberOf8bitCounters();
-  uptr Update8bitCounterBitsetAndClearCounters(u8 *bitset);
-
-  uptr *data();
-  uptr size() const;
-
- private:
-  struct NamedPcRange {
-    const char *copied_module_name;
-    uptr beg, end; // elements [beg,end) in pc_array.
-  };
-
-  void DirectOpen();
-  void UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end);
-  void GetRangeOffsets(const NamedPcRange& r, Symbolizer* s,
-      InternalMmapVector<uptr>* offsets) const;
-
-  // Maximal size pc array may ever grow.
-  // We MmapNoReserve this space to ensure that the array is contiguous.
-  static const uptr kPcArrayMaxSize =
-      FIRST_32_SECOND_64(1 << (SANITIZER_ANDROID ? 24 : 26), 1 << 27);
-  // The amount file mapping for the pc array is grown by.
-  static const uptr kPcArrayMmapSize = 64 * 1024;
-
-  // pc_array is allocated with MmapNoReserveOrDie and so it uses only as
-  // much RAM as it really needs.
-  uptr *pc_array;
-  // Index of the first available pc_array slot.
-  atomic_uintptr_t pc_array_index;
-  // Array size.
-  atomic_uintptr_t pc_array_size;
-  // Current file mapped size of the pc array.
-  uptr pc_array_mapped_size;
-  // Descriptor of the file mapped pc array.
-  fd_t pc_fd;
-
-  // Vector of coverage guard arrays, protected by mu.
-  InternalMmapVectorNoCtor<s32*> guard_array_vec;
-
-  // Vector of module and compilation unit pc ranges.
-  InternalMmapVectorNoCtor<NamedPcRange> comp_unit_name_vec;
-  InternalMmapVectorNoCtor<NamedPcRange> module_name_vec;
-
-  struct CounterAndSize {
-    u8 *counters;
-    uptr n;
-  };
-
-  InternalMmapVectorNoCtor<CounterAndSize> counters_vec;
-  uptr num_8bit_counters;
-
-  // Caller-Callee (cc) array, size and current index.
-  static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24);
-  uptr **cc_array;
-  atomic_uintptr_t cc_array_index;
-  atomic_uintptr_t cc_array_size;
-
-  // Tracing event array, size and current pointer.
-  // We record all events (basic block entries) in a global buffer of u32
-  // values. Each such value is the index in pc_array.
-  // So far the tracing is highly experimental:
-  //   - not thread-safe;
-  //   - does not support long traces;
-  //   - not tuned for performance.
-  static const uptr kTrEventArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 30);
-  u32 *tr_event_array;
-  uptr tr_event_array_size;
-  u32 *tr_event_pointer;
-  static const uptr kTrPcArrayMaxSize    = FIRST_32_SECOND_64(1 << 22, 1 << 27);
-
-  StaticSpinMutex mu;
-};
-
-static CoverageData coverage_data;
-
-void CovUpdateMapping(const char *path, uptr caller_pc = 0);
-
-void CoverageData::DirectOpen() {
-  InternalScopedString path(kMaxPathLength);
-  internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw",
-                    coverage_dir, internal_getpid());
-  pc_fd = OpenFile(path.data(), RdWr);
-  if (pc_fd == kInvalidFd) {
-    Report("Coverage: failed to open %s for reading/writing\n", path.data());
-    Die();
-  }
-
-  pc_array_mapped_size = 0;
-  CovUpdateMapping(coverage_dir);
-}
-
-void CoverageData::Init() {
-  pc_fd = kInvalidFd;
-}
-
-void CoverageData::Enable() {
-  if (pc_array)
-    return;
-  pc_array = reinterpret_cast<uptr *>(
-      MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit"));
-  atomic_store(&pc_array_index, 0, memory_order_relaxed);
-  if (common_flags()->coverage_direct) {
-    atomic_store(&pc_array_size, 0, memory_order_relaxed);
-  } else {
-    atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed);
-  }
-
-  cc_array = reinterpret_cast<uptr **>(MmapNoReserveOrDie(
-      sizeof(uptr *) * kCcArrayMaxSize, "CovInit::cc_array"));
-  atomic_store(&cc_array_size, kCcArrayMaxSize, memory_order_relaxed);
-  atomic_store(&cc_array_index, 0, memory_order_relaxed);
-
-  // Allocate tr_event_array with a guard page at the end.
-  tr_event_array = reinterpret_cast<u32 *>(MmapNoReserveOrDie(
-      sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(),
-      "CovInit::tr_event_array"));
-  MprotectNoAccess(
-      reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]),
-      GetMmapGranularity());
-  tr_event_array_size = kTrEventArrayMaxSize;
-  tr_event_pointer = tr_event_array;
-
-  num_8bit_counters = 0;
-}
-
-void CoverageData::InitializeGuardArray(s32 *guards) {
-  Enable();  // Make sure coverage is enabled at this point.
-  s32 n = guards[0];
-  for (s32 j = 1; j <= n; j++) {
-    uptr idx = atomic_load_relaxed(&pc_array_index);
-    atomic_store_relaxed(&pc_array_index, idx + 1);
-    guards[j] = -static_cast<s32>(idx + 1);
-  }
-}
-
-void CoverageData::Disable() {
-  if (pc_array) {
-    UnmapOrDie(pc_array, sizeof(uptr) * kPcArrayMaxSize);
-    pc_array = nullptr;
-  }
-  if (cc_array) {
-    UnmapOrDie(cc_array, sizeof(uptr *) * kCcArrayMaxSize);
-    cc_array = nullptr;
-  }
-  if (tr_event_array) {
-    UnmapOrDie(tr_event_array,
-               sizeof(tr_event_array[0]) * kTrEventArrayMaxSize +
-                   GetMmapGranularity());
-    tr_event_array = nullptr;
-    tr_event_pointer = nullptr;
-  }
-  if (pc_fd != kInvalidFd) {
-    CloseFile(pc_fd);
-    pc_fd = kInvalidFd;
-  }
-}
-
-void CoverageData::ReinitializeGuards() {
-  // Assuming single thread.
-  atomic_store(&pc_array_index, 0, memory_order_relaxed);
-  for (uptr i = 0; i < guard_array_vec.size(); i++)
-    InitializeGuardArray(guard_array_vec[i]);
-}
-
-void CoverageData::ReInit() {
-  Disable();
-  if (coverage_enabled) {
-    if (common_flags()->coverage_direct) {
-      // In memory-mapped mode we must extend the new file to the known array
-      // size.
-      uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
-      uptr npcs = size / sizeof(uptr);
-      Enable();
-      if (size) Extend(npcs);
-      if (coverage_enabled) CovUpdateMapping(coverage_dir);
-    } else {
-      Enable();
-    }
-  }
-  // Re-initialize the guards.
-  // We are single-threaded now, no need to grab any lock.
-  CHECK_EQ(atomic_load(&pc_array_index, memory_order_relaxed), 0);
-  ReinitializeGuards();
-}
-
-void CoverageData::BeforeFork() {
-  mu.Lock();
-}
-
-void CoverageData::AfterFork(int child_pid) {
-  // We are single-threaded so it's OK to release the lock early.
-  mu.Unlock();
-  if (child_pid == 0) ReInit();
-}
-
-// Extend coverage PC array to fit additional npcs elements.
-void CoverageData::Extend(uptr npcs) {
-  if (!common_flags()->coverage_direct) return;
-  SpinMutexLock l(&mu);
-
-  uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
-  size += npcs * sizeof(uptr);
-
-  if (coverage_enabled && size > pc_array_mapped_size) {
-    if (pc_fd == kInvalidFd) DirectOpen();
-    CHECK_NE(pc_fd, kInvalidFd);
-
-    uptr new_mapped_size = pc_array_mapped_size;
-    while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize;
-    CHECK_LE(new_mapped_size, sizeof(uptr) * kPcArrayMaxSize);
-
-    // Extend the file and map the new space at the end of pc_array.
-    uptr res = internal_ftruncate(pc_fd, new_mapped_size);
-    int err;
-    if (internal_iserror(res, &err)) {
-      Printf("failed to extend raw coverage file: %d\n", err);
-      Die();
-    }
-
-    uptr next_map_base = ((uptr)pc_array) + pc_array_mapped_size;
-    void *p = MapWritableFileToMemory((void *)next_map_base,
-                                      new_mapped_size - pc_array_mapped_size,
-                                      pc_fd, pc_array_mapped_size);
-    CHECK_EQ((uptr)p, next_map_base);
-    pc_array_mapped_size = new_mapped_size;
-  }
-
-  atomic_store(&pc_array_size, size, memory_order_release);
-}
-
-void CoverageData::InitializeCounters(u8 *counters, uptr n) {
-  if (!counters) return;
-  CHECK_EQ(reinterpret_cast<uptr>(counters) % 16, 0);
-  n = RoundUpTo(n, 16); // The compiler must ensure that counters is 16-aligned.
-  SpinMutexLock l(&mu);
-  counters_vec.push_back({counters, n});
-  num_8bit_counters += n;
-}
-
-void CoverageData::UpdateModuleNameVec(uptr caller_pc, uptr range_beg,
-                                       uptr range_end) {
-  auto sym = Symbolizer::GetOrInit();
-  if (!sym)
-    return;
-  const char *module_name = sym->GetModuleNameForPc(caller_pc);
-  if (!module_name) return;
-  if (module_name_vec.empty() ||
-      module_name_vec.back().copied_module_name != module_name)
-    module_name_vec.push_back({module_name, range_beg, range_end});
-  else
-    module_name_vec.back().end = range_end;
-}
-
-void CoverageData::InitializeGuards(s32 *guards, uptr n,
-                                    const char *comp_unit_name,
-                                    uptr caller_pc) {
-  // The array 'guards' has n+1 elements, we use the element zero
-  // to store 'n'.
-  CHECK_LT(n, 1 << 30);
-  guards[0] = static_cast<s32>(n);
-  InitializeGuardArray(guards);
-  SpinMutexLock l(&mu);
-  uptr range_end = atomic_load(&pc_array_index, memory_order_relaxed);
-  uptr range_beg = range_end - n;
-  comp_unit_name_vec.push_back({comp_unit_name, range_beg, range_end});
-  guard_array_vec.push_back(guards);
-  UpdateModuleNameVec(caller_pc, range_beg, range_end);
-}
-
-static const uptr kBundleCounterBits = 16;
-
-// When coverage_order_pcs==true and SANITIZER_WORDSIZE==64
-// we insert the global counter into the first 16 bits of the PC.
-uptr BundlePcAndCounter(uptr pc, uptr counter) {
-  if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
-    return pc;
-  static const uptr kMaxCounter = (1 << kBundleCounterBits) - 1;
-  if (counter > kMaxCounter)
-    counter = kMaxCounter;
-  CHECK_EQ(0, pc >> (SANITIZER_WORDSIZE - kBundleCounterBits));
-  return pc | (counter << (SANITIZER_WORDSIZE - kBundleCounterBits));
-}
-
-uptr UnbundlePc(uptr bundle) {
-  if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
-    return bundle;
-  return (bundle << kBundleCounterBits) >> kBundleCounterBits;
-}
-
-uptr UnbundleCounter(uptr bundle) {
-  if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
-    return 0;
-  return bundle >> (SANITIZER_WORDSIZE - kBundleCounterBits);
-}
-
-// If guard is negative, atomically set it to -guard and store the PC in
-// pc_array.
-void CoverageData::Add(uptr pc, u32 *guard) {
-  atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard);
-  s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed);
-  if (guard_value >= 0) return;
-
-  atomic_store(atomic_guard, -guard_value, memory_order_relaxed);
-  if (!pc_array) return;
-
-  uptr idx = -guard_value - 1;
-  if (idx >= atomic_load(&pc_array_index, memory_order_acquire))
-    return;  // May happen after fork when pc_array_index becomes 0.
-  CHECK_LT(idx * sizeof(uptr),
-           atomic_load(&pc_array_size, memory_order_acquire));
-  uptr counter = atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed);
-  pc_array[idx] = BundlePcAndCounter(pc, counter);
-}
-
-// Registers a pair caller=>callee.
-// When a given caller is seen for the first time, the callee_cache is added
-// to the global array cc_array, callee_cache[0] is set to caller and
-// callee_cache[1] is set to cache_size.
-// Then we are trying to add callee to callee_cache [2,cache_size) if it is
-// not there yet.
-// If the cache is full we drop the callee (may want to fix this later).
-void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[],
-                             uptr cache_size) {
-  if (!cc_array) return;
-  atomic_uintptr_t *atomic_callee_cache =
-      reinterpret_cast<atomic_uintptr_t *>(callee_cache);
-  uptr zero = 0;
-  if (atomic_compare_exchange_strong(&atomic_callee_cache[0], &zero, caller,
-                                     memory_order_seq_cst)) {
-    uptr idx = atomic_fetch_add(&cc_array_index, 1, memory_order_relaxed);
-    CHECK_LT(idx * sizeof(uptr),
-             atomic_load(&cc_array_size, memory_order_acquire));
-    callee_cache[1] = cache_size;
-    cc_array[idx] = callee_cache;
-  }
-  CHECK_EQ(atomic_load(&atomic_callee_cache[0], memory_order_relaxed), caller);
-  for (uptr i = 2; i < cache_size; i++) {
-    uptr was = 0;
-    if (atomic_compare_exchange_strong(&atomic_callee_cache[i], &was, callee,
-                                       memory_order_seq_cst)) {
-      atomic_fetch_add(&caller_callee_counter, 1, memory_order_relaxed);
-      return;
-    }
-    if (was == callee)  // Already have this callee.
-      return;
-  }
-}
-
-uptr CoverageData::GetNumberOf8bitCounters() {
-  return num_8bit_counters;
-}
-
-// Map every 8bit counter to a 8-bit bitset and clear the counter.
-uptr CoverageData::Update8bitCounterBitsetAndClearCounters(u8 *bitset) {
-  uptr num_new_bits = 0;
-  uptr cur = 0;
-  // For better speed we map 8 counters to 8 bytes of bitset at once.
-  static const uptr kBatchSize = 8;
-  CHECK_EQ(reinterpret_cast<uptr>(bitset) % kBatchSize, 0);
-  for (uptr i = 0, len = counters_vec.size(); i < len; i++) {
-    u8 *c = counters_vec[i].counters;
-    uptr n = counters_vec[i].n;
-    CHECK_EQ(n % 16, 0);
-    CHECK_EQ(cur % kBatchSize, 0);
-    CHECK_EQ(reinterpret_cast<uptr>(c) % kBatchSize, 0);
-    if (!bitset) {
-      internal_bzero_aligned16(c, n);
-      cur += n;
-      continue;
-    }
-    for (uptr j = 0; j < n; j += kBatchSize, cur += kBatchSize) {
-      CHECK_LT(cur, num_8bit_counters);
-      u64 *pc64 = reinterpret_cast<u64*>(c + j);
-      u64 *pb64 = reinterpret_cast<u64*>(bitset + cur);
-      u64 c64 = *pc64;
-      u64 old_bits_64 = *pb64;
-      u64 new_bits_64 = old_bits_64;
-      if (c64) {
-        *pc64 = 0;
-        for (uptr k = 0; k < kBatchSize; k++) {
-          u64 x = (c64 >> (8 * k)) & 0xff;
-          if (x) {
-            u64 bit = 0;
-            /**/ if (x >= 128) bit = 128;
-            else if (x >= 32) bit = 64;
-            else if (x >= 16) bit = 32;
-            else if (x >= 8) bit = 16;
-            else if (x >= 4) bit = 8;
-            else if (x >= 3) bit = 4;
-            else if (x >= 2) bit = 2;
-            else if (x >= 1) bit = 1;
-            u64 mask = bit << (8 * k);
-            if (!(new_bits_64 & mask)) {
-              num_new_bits++;
-              new_bits_64 |= mask;
-            }
-          }
-        }
-        *pb64 = new_bits_64;
-      }
-    }
-  }
-  CHECK_EQ(cur, num_8bit_counters);
-  return num_new_bits;
-}
-
-uptr *CoverageData::data() {
-  return pc_array;
-}
-
-uptr CoverageData::size() const {
-  return atomic_load(&pc_array_index, memory_order_relaxed);
-}
-
-// Block layout for packed file format: header, followed by module name (no
-// trailing zero), followed by data blob.
-struct CovHeader {
-  int pid;
-  unsigned int module_name_length;
-  unsigned int data_length;
-};
-
-static void CovWritePacked(int pid, const char *module, const void *blob,
-                           unsigned int blob_size) {
-  if (cov_fd == kInvalidFd) return;
-  unsigned module_name_length = internal_strlen(module);
-  CovHeader header = {pid, module_name_length, blob_size};
-
-  if (cov_max_block_size == 0) {
-    // Writing to a file. Just go ahead.
-    WriteToFile(cov_fd, &header, sizeof(header));
-    WriteToFile(cov_fd, module, module_name_length);
-    WriteToFile(cov_fd, blob, blob_size);
-  } else {
-    // Writing to a socket. We want to split the data into appropriately sized
-    // blocks.
-    InternalScopedBuffer<char> block(cov_max_block_size);
-    CHECK_EQ((uptr)block.data(), (uptr)(CovHeader *)block.data());
-    uptr header_size_with_module = sizeof(header) + module_name_length;
-    CHECK_LT(header_size_with_module, cov_max_block_size);
-    unsigned int max_payload_size =
-        cov_max_block_size - header_size_with_module;
-    char *block_pos = block.data();
-    internal_memcpy(block_pos, &header, sizeof(header));
-    block_pos += sizeof(header);
-    internal_memcpy(block_pos, module, module_name_length);
-    block_pos += module_name_length;
-    char *block_data_begin = block_pos;
-    const char *blob_pos = (const char *)blob;
-    while (blob_size > 0) {
-      unsigned int payload_size = Min(blob_size, max_payload_size);
-      blob_size -= payload_size;
-      internal_memcpy(block_data_begin, blob_pos, payload_size);
-      blob_pos += payload_size;
-      ((CovHeader *)block.data())->data_length = payload_size;
-      WriteToFile(cov_fd, block.data(), header_size_with_module + payload_size);
-    }
-  }
-}
-
-// If packed = false: <name>.<pid>.<sancov> (name = module name).
-// If packed = true and name == 0: <pid>.<sancov>.<packed>.
-// If packed = true and name != 0: <name>.<sancov>.<packed> (name is
-// user-supplied).
-static fd_t CovOpenFile(InternalScopedString *path, bool packed,
-                       const char *name, const char *extension = "sancov") {
-  path->clear();
-  if (!packed) {
-    CHECK(name);
-    path->append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(),
-                extension);
-  } else {
-    if (!name)
-      path->append("%s/%zd.%s.packed", coverage_dir, internal_getpid(),
-                  extension);
-    else
-      path->append("%s/%s.%s.packed", coverage_dir, name, extension);
-  }
-  error_t err;
-  fd_t fd = OpenFile(path->data(), WrOnly, &err);
-  if (fd == kInvalidFd)
-    Report("SanitizerCoverage: failed to open %s for writing (reason: %d)\n",
-           path->data(), err);
-  return fd;
-}
-
-// Dump trace PCs and trace events into two separate files.
-void CoverageData::DumpTrace() {
-  uptr max_idx = tr_event_pointer - tr_event_array;
-  if (!max_idx) return;
-  auto sym = Symbolizer::GetOrInit();
-  if (!sym)
-    return;
-  InternalScopedString out(32 << 20);
-  for (uptr i = 0, n = size(); i < n; i++) {
-    const char *module_name = "<unknown>";
-    uptr module_address = 0;
-    sym->GetModuleNameAndOffsetForPC(UnbundlePc(pc_array[i]), &module_name,
-                                     &module_address);
-    out.append("%s 0x%zx\n", module_name, module_address);
-  }
-  InternalScopedString path(kMaxPathLength);
-  fd_t fd = CovOpenFile(&path, false, "trace-points");
-  if (fd == kInvalidFd) return;
-  WriteToFile(fd, out.data(), out.length());
-  CloseFile(fd);
-
-  fd = CovOpenFile(&path, false, "trace-compunits");
-  if (fd == kInvalidFd) return;
-  out.clear();
-  for (uptr i = 0; i < comp_unit_name_vec.size(); i++)
-    out.append("%s\n", comp_unit_name_vec[i].copied_module_name);
-  WriteToFile(fd, out.data(), out.length());
-  CloseFile(fd);
-
-  fd = CovOpenFile(&path, false, "trace-events");
-  if (fd == kInvalidFd) return;
-  uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]);
-  u8 *event_bytes = reinterpret_cast<u8*>(tr_event_array);
-  // The trace file could be huge, and may not be written with a single syscall.
-  while (bytes_to_write) {
-    uptr actually_written;
-    if (WriteToFile(fd, event_bytes, bytes_to_write, &actually_written) &&
-        actually_written <= bytes_to_write) {
-      bytes_to_write -= actually_written;
-      event_bytes += actually_written;
-    } else {
-      break;
-    }
-  }
-  CloseFile(fd);
-  VReport(1, " CovDump: Trace: %zd PCs written\n", size());
-  VReport(1, " CovDump: Trace: %zd Events written\n", max_idx);
-}
-
-// This function dumps the caller=>callee pairs into a file as a sequence of
-// lines like "module_name offset".
-void CoverageData::DumpCallerCalleePairs() {
-  uptr max_idx = atomic_load(&cc_array_index, memory_order_relaxed);
-  if (!max_idx) return;
-  auto sym = Symbolizer::GetOrInit();
-  if (!sym)
-    return;
-  InternalScopedString out(32 << 20);
-  uptr total = 0;
-  for (uptr i = 0; i < max_idx; i++) {
-    uptr *cc_cache = cc_array[i];
-    CHECK(cc_cache);
-    uptr caller = cc_cache[0];
-    uptr n_callees = cc_cache[1];
-    const char *caller_module_name = "<unknown>";
-    uptr caller_module_address = 0;
-    sym->GetModuleNameAndOffsetForPC(caller, &caller_module_name,
-                                     &caller_module_address);
-    for (uptr j = 2; j < n_callees; j++) {
-      uptr callee = cc_cache[j];
-      if (!callee) break;
-      total++;
-      const char *callee_module_name = "<unknown>";
-      uptr callee_module_address = 0;
-      sym->GetModuleNameAndOffsetForPC(callee, &callee_module_name,
-                                       &callee_module_address);
-      out.append("%s 0x%zx\n%s 0x%zx\n", caller_module_name,
-                 caller_module_address, callee_module_name,
-                 callee_module_address);
-    }
-  }
-  InternalScopedString path(kMaxPathLength);
-  fd_t fd = CovOpenFile(&path, false, "caller-callee");
-  if (fd == kInvalidFd) return;
-  WriteToFile(fd, out.data(), out.length());
-  CloseFile(fd);
-  VReport(1, " CovDump: %zd caller-callee pairs written\n", total);
-}
-
-// Record the current PC into the event buffer.
-// Every event is a u32 value (index in tr_pc_array_index) so we compute
-// it once and then cache in the provided 'cache' storage.
-//
-// This function will eventually be inlined by the compiler.
-void CoverageData::TraceBasicBlock(u32 *id) {
-  // Will trap here if
-  //  1. coverage is not enabled at run-time.
-  //  2. The array tr_event_array is full.
-  *tr_event_pointer = *id - 1;
-  tr_event_pointer++;
-}
-
-void CoverageData::DumpCounters() {
-  if (!common_flags()->coverage_counters) return;
-  uptr n = coverage_data.GetNumberOf8bitCounters();
-  if (!n) return;
-  InternalScopedBuffer<u8> bitset(n);
-  coverage_data.Update8bitCounterBitsetAndClearCounters(bitset.data());
-  InternalScopedString path(kMaxPathLength);
-
-  for (uptr m = 0; m < module_name_vec.size(); m++) {
-    auto r = module_name_vec[m];
-    CHECK(r.copied_module_name);
-    CHECK_LE(r.beg, r.end);
-    CHECK_LE(r.end, size());
-    const char *base_name = StripModuleName(r.copied_module_name);
-    fd_t fd =
-        CovOpenFile(&path, /* packed */ false, base_name, "counters-sancov");
-    if (fd == kInvalidFd) return;
-    WriteToFile(fd, bitset.data() + r.beg, r.end - r.beg);
-    CloseFile(fd);
-    VReport(1, " CovDump: %zd counters written for '%s'\n", r.end - r.beg,
-            base_name);
-  }
-}
-
-void CoverageData::DumpAsBitSet() {
-  if (!common_flags()->coverage_bitset) return;
-  if (!size()) return;
-  InternalScopedBuffer<char> out(size());
-  InternalScopedString path(kMaxPathLength);
-  for (uptr m = 0; m < module_name_vec.size(); m++) {
-    uptr n_set_bits = 0;
-    auto r = module_name_vec[m];
-    CHECK(r.copied_module_name);
-    CHECK_LE(r.beg, r.end);
-    CHECK_LE(r.end, size());
-    for (uptr i = r.beg; i < r.end; i++) {
-      uptr pc = UnbundlePc(pc_array[i]);
-      out[i] = pc ? '1' : '0';
-      if (pc)
-        n_set_bits++;
-    }
-    const char *base_name = StripModuleName(r.copied_module_name);
-    fd_t fd = CovOpenFile(&path, /* packed */false, base_name, "bitset-sancov");
-    if (fd == kInvalidFd) return;
-    WriteToFile(fd, out.data() + r.beg, r.end - r.beg);
-    CloseFile(fd);
-    VReport(1,
-            " CovDump: bitset of %zd bits written for '%s', %zd bits are set\n",
-            r.end - r.beg, base_name, n_set_bits);
-  }
-}
-
-
-void CoverageData::GetRangeOffsets(const NamedPcRange& r, Symbolizer* sym,
-    InternalMmapVector<uptr>* offsets) const {
-  offsets->clear();
-  for (uptr i = 0; i < kNumWordsForMagic; i++)
-    offsets->push_back(0);
-  CHECK(r.copied_module_name);
-  CHECK_LE(r.beg, r.end);
-  CHECK_LE(r.end, size());
-  for (uptr i = r.beg; i < r.end; i++) {
-    uptr pc = UnbundlePc(pc_array[i]);
-    uptr counter = UnbundleCounter(pc_array[i]);
-    if (!pc) continue; // Not visited.
-    uptr offset = 0;
-    sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset);
-    offsets->push_back(BundlePcAndCounter(offset, counter));
-  }
-
-  CHECK_GE(offsets->size(), kNumWordsForMagic);
-  SortArray(offsets->data(), offsets->size());
-  for (uptr i = 0; i < offsets->size(); i++)
-    (*offsets)[i] = UnbundlePc((*offsets)[i]);
-}
-
-static void GenerateHtmlReport(const InternalMmapVector<char *> &cov_files) {
-  if (!common_flags()->html_cov_report) {
-    return;
-  }
-  char *sancov_path = FindPathToBinary(common_flags()->sancov_path);
-  if (sancov_path == nullptr) {
-    return;
-  }
-
-  InternalMmapVector<char *> sancov_argv(cov_files.size() * 2 + 3);
-  sancov_argv.push_back(sancov_path);
-  sancov_argv.push_back(internal_strdup("-html-report"));
-  auto argv_deleter = at_scope_exit([&] {
-    for (uptr i = 0; i < sancov_argv.size(); ++i) {
-      InternalFree(sancov_argv[i]);
-    }
-  });
-
-  for (const auto &cov_file : cov_files) {
-    sancov_argv.push_back(internal_strdup(cov_file));
-  }
-
-  {
-    ListOfModules modules;
-    modules.init();
-    for (const LoadedModule &module : modules) {
-      sancov_argv.push_back(internal_strdup(module.full_name()));
-    }
-  }
-
-  InternalScopedString report_path(kMaxPathLength);
-  fd_t report_fd =
-      CovOpenFile(&report_path, false /* packed */, GetProcessName(), "html");
-  int pid = StartSubprocess(sancov_argv[0], sancov_argv.data(),
-                            kInvalidFd /* stdin */, report_fd /* std_out */);
-  if (pid > 0) {
-    int result = WaitForProcess(pid);
-    if (result == 0)
-      Printf("coverage report generated to %s\n", report_path.data());
-  }
-}
-
-void CoverageData::DumpOffsets() {
-  auto sym = Symbolizer::GetOrInit();
-  if (!common_flags()->coverage_pcs) return;
-  CHECK_NE(sym, nullptr);
-  InternalMmapVector<uptr> offsets(0);
-  InternalScopedString path(kMaxPathLength);
-
-  InternalMmapVector<char *> cov_files(module_name_vec.size());
-  auto cov_files_deleter = at_scope_exit([&] {
-    for (uptr i = 0; i < cov_files.size(); ++i) {
-      InternalFree(cov_files[i]);
-    }
-  });
-
-  for (uptr m = 0; m < module_name_vec.size(); m++) {
-    auto r = module_name_vec[m];
-    GetRangeOffsets(r, sym, &offsets);
-
-    uptr num_offsets = offsets.size() - kNumWordsForMagic;
-    u64 *magic_p = reinterpret_cast<u64*>(offsets.data());
-    CHECK_EQ(*magic_p, 0ULL);
-    // FIXME: we may want to write 32-bit offsets even in 64-mode
-    // if all the offsets are small enough.
-    *magic_p = kMagic;
-
-    const char *module_name = StripModuleName(r.copied_module_name);
-    if (cov_sandboxed) {
-      if (cov_fd != kInvalidFd) {
-        CovWritePacked(internal_getpid(), module_name, offsets.data(),
-                       offsets.size() * sizeof(offsets[0]));
-        VReport(1, " CovDump: %zd PCs written to packed file\n", num_offsets);
-      }
-    } else {
-      // One file per module per process.
-      fd_t fd = CovOpenFile(&path, false /* packed */, module_name);
-      if (fd == kInvalidFd) continue;
-      WriteToFile(fd, offsets.data(), offsets.size() * sizeof(offsets[0]));
-      CloseFile(fd);
-      cov_files.push_back(internal_strdup(path.data()));
-      VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), num_offsets);
-    }
-  }
-  if (cov_fd != kInvalidFd)
-    CloseFile(cov_fd);
-
-  GenerateHtmlReport(cov_files);
-}
-
-void CoverageData::DumpAll() {
-  if (!coverage_enabled || common_flags()->coverage_direct) return;
-  if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
-    return;
-  DumpAsBitSet();
-  DumpCounters();
-  DumpTrace();
-  DumpOffsets();
-  DumpCallerCalleePairs();
-}
-
-void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
-  if (!args) return;
-  if (!coverage_enabled) return;
-  cov_sandboxed = args->coverage_sandboxed;
-  if (!cov_sandboxed) return;
-  cov_max_block_size = args->coverage_max_block_size;
-  if (args->coverage_fd >= 0) {
-    cov_fd = (fd_t)args->coverage_fd;
-  } else {
-    InternalScopedString path(kMaxPathLength);
-    // Pre-open the file now. The sandbox won't allow us to do it later.
-    cov_fd = CovOpenFile(&path, true /* packed */, nullptr);
-  }
-}
-
-fd_t MaybeOpenCovFile(const char *name) {
-  CHECK(name);
-  if (!coverage_enabled) return kInvalidFd;
-  InternalScopedString path(kMaxPathLength);
-  return CovOpenFile(&path, true /* packed */, name);
-}
-
-void CovBeforeFork() {
-  coverage_data.BeforeFork();
-}
-
-void CovAfterFork(int child_pid) {
-  coverage_data.AfterFork(child_pid);
-}
-
-static void MaybeDumpCoverage() {
-  if (common_flags()->coverage)
-    __sanitizer_cov_dump();
-}
-
-void InitializeCoverage(bool enabled, const char *dir) {
-  if (coverage_enabled)
-    return;  // May happen if two sanitizer enable coverage in the same process.
-  coverage_enabled = enabled;
-  coverage_dir = dir;
-  coverage_data.Init();
-  if (enabled) coverage_data.Enable();
-  if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump);
-  AddDieCallback(MaybeDumpCoverage);
-}
-
-void ReInitializeCoverage(bool enabled, const char *dir) {
-  coverage_enabled = enabled;
-  coverage_dir = dir;
-  coverage_data.ReInit();
-}
-
-void CoverageUpdateMapping() {
-  if (coverage_enabled)
-    CovUpdateMapping(coverage_dir);
-}
-
-} // namespace __sanitizer
-
-extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) {
-  coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()),
-                    guard);
-}
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) {
-  atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard);
-  if (static_cast<s32>(
-          __sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) < 0)
-    __sanitizer_cov(guard);
-}
-SANITIZER_INTERFACE_ATTRIBUTE void
-__sanitizer_cov_indir_call16(uptr callee, uptr callee_cache16[]) {
-  coverage_data.IndirCall(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()),
-                          callee, callee_cache16, 16);
-}
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() {
-  coverage_enabled = true;
-  coverage_dir = common_flags()->coverage_dir;
-  coverage_data.Init();
-}
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() {
-  coverage_data.DumpAll();
-}
-SANITIZER_INTERFACE_ATTRIBUTE void
-__sanitizer_cov_module_init(s32 *guards, uptr npcs, u8 *counters,
-                            const char *comp_unit_name) {
-  coverage_data.InitializeGuards(guards, npcs, comp_unit_name, GET_CALLER_PC());
-  coverage_data.InitializeCounters(counters, npcs);
-  if (!common_flags()->coverage_direct) return;
-  if (SANITIZER_ANDROID && coverage_enabled) {
-    // dlopen/dlclose interceptors do not work on Android, so we rely on
-    // Extend() calls to update .sancov.map.
-    CovUpdateMapping(coverage_dir, GET_CALLER_PC());
-  }
-  coverage_data.Extend(npcs);
-}
-SANITIZER_INTERFACE_ATTRIBUTE
-sptr __sanitizer_maybe_open_cov_file(const char *name) {
-  return (sptr)MaybeOpenCovFile(name);
-}
-SANITIZER_INTERFACE_ATTRIBUTE
-uptr __sanitizer_get_total_unique_coverage() {
-  return atomic_load(&coverage_counter, memory_order_relaxed);
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-uptr __sanitizer_get_total_unique_caller_callee_pairs() {
-  return atomic_load(&caller_callee_counter, memory_order_relaxed);
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_cov_trace_func_enter(u32 *id) {
-  __sanitizer_cov_with_check(id);
-  coverage_data.TraceBasicBlock(id);
-}
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_cov_trace_basic_block(u32 *id) {
-  __sanitizer_cov_with_check(id);
-  coverage_data.TraceBasicBlock(id);
-}
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_reset_coverage() {
-  ResetGlobalCounters();
-  coverage_data.ReinitializeGuards();
-  internal_bzero_aligned16(
-      coverage_data.data(),
-      RoundUpTo(coverage_data.size() * sizeof(coverage_data.data()[0]), 16));
-}
-SANITIZER_INTERFACE_ATTRIBUTE
-uptr __sanitizer_get_coverage_guards(uptr **data) {
-  *data = coverage_data.data();
-  return coverage_data.size();
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-uptr __sanitizer_get_number_of_counters() {
-  return coverage_data.GetNumberOf8bitCounters();
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) {
-  return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset);
-}
-// Default empty implementations (weak). Users should redefine them.
-#if !SANITIZER_WINDOWS  // weak does not work on Windows.
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_cmp() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_cmp1() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_cmp2() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_cmp4() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_cmp8() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_switch() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_div4() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_div8() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_gep() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_pc_guard() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_pc_indir() {}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_cov_trace_pc_guard_init() {}
-#endif  // !SANITIZER_WINDOWS
-} // extern "C"
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep_new.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep_new.cc
new file mode 100644
index 00000000000..25a5001bd3e
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep_new.cc
@@ -0,0 +1,218 @@
+//===-- sanitizer_coverage_libcdep_new.cc ---------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Sanitizer Coverage Controller for Trace PC Guard.
+
+#include "sanitizer_platform.h"
+
+#if !SANITIZER_FUCHSIA
+#include "sancov_flags.h"
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+#include "sanitizer_symbolizer.h"
+
+using namespace __sanitizer;
+
+using AddressRange = LoadedModule::AddressRange;
+
+namespace __sancov {
+namespace {
+
+static const u64 Magic64 = 0xC0BFFFFFFFFFFF64ULL;
+static const u64 Magic32 = 0xC0BFFFFFFFFFFF32ULL;
+static const u64 Magic = SANITIZER_WORDSIZE == 64 ? Magic64 : Magic32;
+
+static fd_t OpenFile(const char* path) {
+  error_t err;
+  fd_t fd = OpenFile(path, WrOnly, &err);
+  if (fd == kInvalidFd)
+    Report("SanitizerCoverage: failed to open %s for writing (reason: %d)\n",
+           path, err);
+  return fd;
+}
+
+static void GetCoverageFilename(char* path, const char* name,
+                                const char* extension) {
+  CHECK(name);
+  internal_snprintf(path, kMaxPathLength, "%s/%s.%zd.%s",
+                    common_flags()->coverage_dir, name, internal_getpid(),
+                    extension);
+}
+
+static void WriteModuleCoverage(char* file_path, const char* module_name,
+                                const uptr* pcs, uptr len) {
+  GetCoverageFilename(file_path, StripModuleName(module_name), "sancov");
+  fd_t fd = OpenFile(file_path);
+  WriteToFile(fd, &Magic, sizeof(Magic));
+  WriteToFile(fd, pcs, len * sizeof(*pcs));
+  CloseFile(fd);
+  Printf("SanitizerCoverage: %s: %zd PCs written\n", file_path, len);
+}
+
+static void SanitizerDumpCoverage(const uptr* unsorted_pcs, uptr len) {
+  if (!len) return;
+
+  char* file_path = static_cast<char*>(InternalAlloc(kMaxPathLength));
+  char* module_name = static_cast<char*>(InternalAlloc(kMaxPathLength));
+  uptr* pcs = static_cast<uptr*>(InternalAlloc(len * sizeof(uptr)));
+
+  internal_memcpy(pcs, unsorted_pcs, len * sizeof(uptr));
+  SortArray(pcs, len);
+
+  bool module_found = false;
+  uptr last_base = 0;
+  uptr module_start_idx = 0;
+
+  for (uptr i = 0; i < len; ++i) {
+    const uptr pc = pcs[i];
+    if (!pc) continue;
+
+    if (!__sanitizer_get_module_and_offset_for_pc(pc, nullptr, 0, &pcs[i])) {
+      Printf("ERROR: unknown pc 0x%x (may happen if dlclose is used)\n", pc);
+      continue;
+    }
+    uptr module_base = pc - pcs[i];
+
+    if (module_base != last_base || !module_found) {
+      if (module_found) {
+        WriteModuleCoverage(file_path, module_name, &pcs[module_start_idx],
+                            i - module_start_idx);
+      }
+
+      last_base = module_base;
+      module_start_idx = i;
+      module_found = true;
+      __sanitizer_get_module_and_offset_for_pc(pc, module_name, kMaxPathLength,
+                                               &pcs[i]);
+    }
+  }
+
+  if (module_found) {
+    WriteModuleCoverage(file_path, module_name, &pcs[module_start_idx],
+                        len - module_start_idx);
+  }
+
+  InternalFree(file_path);
+  InternalFree(module_name);
+  InternalFree(pcs);
+}
+
+// Collects trace-pc guard coverage.
+// This class relies on zero-initialization.
+class TracePcGuardController {
+ public:
+  void Initialize() {
+    CHECK(!initialized);
+
+    initialized = true;
+    InitializeSancovFlags();
+
+    pc_vector.Initialize(0);
+  }
+
+  void InitTracePcGuard(u32* start, u32* end) {
+    if (!initialized) Initialize();
+    CHECK(!*start);
+    CHECK_NE(start, end);
+
+    u32 i = pc_vector.size();
+    for (u32* p = start; p < end; p++) *p = ++i;
+    pc_vector.resize(i);
+  }
+
+  void TracePcGuard(u32* guard, uptr pc) {
+    u32 idx = *guard;
+    if (!idx) return;
+    // we start indices from 1.
+    atomic_uintptr_t* pc_ptr =
+        reinterpret_cast<atomic_uintptr_t*>(&pc_vector[idx - 1]);
+    if (atomic_load(pc_ptr, memory_order_relaxed) == 0)
+      atomic_store(pc_ptr, pc, memory_order_relaxed);
+  }
+
+  void Reset() {
+    internal_memset(&pc_vector[0], 0, sizeof(pc_vector[0]) * pc_vector.size());
+  }
+
+  void Dump() {
+    if (!initialized || !common_flags()->coverage) return;
+    __sanitizer_dump_coverage(pc_vector.data(), pc_vector.size());
+  }
+
+ private:
+  bool initialized;
+  InternalMmapVectorNoCtor<uptr> pc_vector;
+};
+
+static TracePcGuardController pc_guard_controller;
+
+}  // namespace
+}  // namespace __sancov
+
+namespace __sanitizer {
+void InitializeCoverage(bool enabled, const char *dir) {
+  static bool coverage_enabled = false;
+  if (coverage_enabled)
+    return;  // May happen if two sanitizer enable coverage in the same process.
+  coverage_enabled = enabled;
+  Atexit(__sanitizer_cov_dump);
+  AddDieCallback(__sanitizer_cov_dump);
+}
+} // namespace __sanitizer
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(  // NOLINT
+    const uptr* pcs, uptr len) {
+  return __sancov::SanitizerDumpCoverage(pcs, len);
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32* guard) {
+  if (!*guard) return;
+  __sancov::pc_guard_controller.TracePcGuard(guard, GET_CALLER_PC() - 1);
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init,
+                             u32* start, u32* end) {
+  if (start == end || *start) return;
+  __sancov::pc_guard_controller.InitTracePcGuard(start, end);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() {
+  __sancov::pc_guard_controller.Dump();
+}
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() {
+  __sanitizer_dump_trace_pc_guard_coverage();
+}
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_reset() {
+  __sancov::pc_guard_controller.Reset();
+}
+// Default empty implementations (weak). Users should redefine them.
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {}
+}  // extern "C"
+// Weak definition for code instrumented with -fsanitize-coverage=stack-depth
+// and later linked with code containing a strong definition.
+// E.g., -fsanitize=fuzzer-no-link
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE uptr __sancov_lowest_stack;
+
+#endif  // !SANITIZER_FUCHSIA
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc
deleted file mode 100644
index b2e724ab221..00000000000
--- a/libsanitizer/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc
+++ /dev/null
@@ -1,120 +0,0 @@
-//===-- sanitizer_coverage_mapping.cc -------------------------------------===//
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Mmap-based implementation of sanitizer coverage.
-//
-// This is part of the implementation of code coverage that does not require
-// __sanitizer_cov_dump() call. Data is stored in 2 files per process.
-//
-// $pid.sancov.map describes process memory layout in the following text-based
-// format:
-// <pointer size in bits>  // 1 line, 32 or 64
-// <mapping start> <mapping end> <base address> <dso name> // repeated
-// ...
-// Mapping lines are NOT sorted. This file is updated every time memory layout
-// is changed (i.e. in dlopen() and dlclose() interceptors).
-//
-// $pid.sancov.raw is a binary dump of PC values, sizeof(uptr) each. Again, not
-// sorted. This file is extended by 64Kb at a time and mapped into memory. It
-// contains one or more 0 words at the end, up to the next 64Kb aligned offset.
-//
-// To convert these 2 files to the usual .sancov format, run sancov.py rawunpack
-// $pid.sancov.raw.
-//
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_allocator_internal.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_procmaps.h"
-
-namespace __sanitizer {
-
-static const uptr kMaxTextSize = 64 * 1024;
-
-struct CachedMapping {
- public:
-  bool NeedsUpdate(uptr pc) {
-    int new_pid = internal_getpid();
-    if (last_pid == new_pid && pc && pc >= last_range_start &&
-        pc < last_range_end)
-      return false;
-    last_pid = new_pid;
-    return true;
-  }
-
-  void SetModuleRange(uptr start, uptr end) {
-    last_range_start = start;
-    last_range_end = end;
-  }
-
- private:
-  uptr last_range_start, last_range_end;
-  int last_pid;
-};
-
-static CachedMapping cached_mapping;
-static StaticSpinMutex mapping_mu;
-
-void CovUpdateMapping(const char *coverage_dir, uptr caller_pc) {
-  if (!common_flags()->coverage_direct) return;
-
-  SpinMutexLock l(&mapping_mu);
-
-  if (!cached_mapping.NeedsUpdate(caller_pc))
-    return;
-
-  InternalScopedString text(kMaxTextSize);
-
-  {
-    text.append("%d\n", sizeof(uptr) * 8);
-    ListOfModules modules;
-    modules.init();
-    for (const LoadedModule &module : modules) {
-      const char *module_name = StripModuleName(module.full_name());
-      uptr base = module.base_address();
-      for (const auto &range : module.ranges()) {
-        if (range.executable) {
-          uptr start = range.beg;
-          uptr end = range.end;
-          text.append("%zx %zx %zx %s\n", start, end, base, module_name);
-          if (caller_pc && caller_pc >= start && caller_pc < end)
-            cached_mapping.SetModuleRange(start, end);
-        }
-      }
-    }
-  }
-
-  error_t err;
-  InternalScopedString tmp_path(64 + internal_strlen(coverage_dir));
-  uptr res = internal_snprintf((char *)tmp_path.data(), tmp_path.size(),
-                               "%s/%zd.sancov.map.tmp", coverage_dir,
-                               internal_getpid());
-  CHECK_LE(res, tmp_path.size());
-  fd_t map_fd = OpenFile(tmp_path.data(), WrOnly, &err);
-  if (map_fd == kInvalidFd) {
-    Report("Coverage: failed to open %s for writing: %d\n", tmp_path.data(),
-           err);
-    Die();
-  }
-
-  if (!WriteToFile(map_fd, text.data(), text.length(), nullptr, &err)) {
-    Printf("sancov.map write failed: %d\n", err);
-    Die();
-  }
-  CloseFile(map_fd);
-
-  InternalScopedString path(64 + internal_strlen(coverage_dir));
-  res = internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.map",
-                          coverage_dir, internal_getpid());
-  CHECK_LE(res, path.size());
-  if (!RenameFile(tmp_path.data(), path.data(), &err)) {
-    Printf("sancov.map rename failed: %d\n", err);
-    Die();
-  }
-}
-
-} // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc
new file mode 100644
index 00000000000..5aea1203dbf
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_coverage_win_dll_thunk.cc
@@ -0,0 +1,19 @@
+//===-- sanitizer_coverage_win_dll_thunk.cc -------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a family of thunks that should be statically linked into
+// the DLLs that have instrumentation in order to delegate the calls to the
+// shared runtime that lives in the main binary.
+// See https://github.com/google/sanitizers/issues/209 for the details.
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DLL_THUNK
+#include "sanitizer_win_dll_thunk.h"
+// Sanitizer Coverage interface functions.
+#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "sanitizer_coverage_interface.inc"
+#endif // SANITIZER_DLL_THUNK
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc
new file mode 100644
index 00000000000..939f395071b
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cc
@@ -0,0 +1,19 @@
+//===-- sanitizer_coverage_win_dynamic_runtime_thunk.cc -------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines things that need to be present in the application modules
+// to interact with Sanitizer Coverage, when it is included in a dll.
+//
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
+#define SANITIZER_IMPORT_INTERFACE 1
+#include "sanitizer_win_defs.h"
+// Define weak alias for all weak functions imported from sanitizer coverage.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
+#include "sanitizer_coverage_interface.inc"
+#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_win_sections.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_win_sections.cc
new file mode 100644
index 00000000000..485dd45e14b
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_coverage_win_sections.cc
@@ -0,0 +1,20 @@
+//===-- sanitizer_coverage_win_sections.cc --------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines delimiters for Sanitizer Coverage's section.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+#include <stdint.h>
+#pragma section(".SCOV$A", read, write)  // NOLINT
+#pragma section(".SCOV$Z", read, write)  // NOLINT
+extern "C" {
+__declspec(allocate(".SCOV$A")) uint32_t __start___sancov_guards = 0;
+__declspec(allocate(".SCOV$Z")) uint32_t __stop___sancov_guards = 0;
+}
+#endif // SANITIZER_WINDOWS
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_win_weak_interception.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_win_weak_interception.cc
new file mode 100644
index 00000000000..64718dfc507
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_coverage_win_weak_interception.cc
@@ -0,0 +1,22 @@
+//===-- sanitizer_coverage_win_weak_interception.cc -----------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This module should be included in Sanitizer Coverage when it implemented as a
+// shared library on Windows (dll), in order to delegate the calls of weak
+// functions to the implementation in the main executable when a strong
+// definition is provided.
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DYNAMIC
+#include "sanitizer_win_weak_interception.h"
+#include "sanitizer_interface_internal.h"
+#include "sancov_flags.h"
+// Check if strong definitions for weak functions are present in the main
+// executable. If that is the case, override dll functions to point to strong
+// implementations.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "sanitizer_coverage_interface.inc"
+#endif // SANITIZER_DYNAMIC
diff --git a/libsanitizer/sanitizer_common/sanitizer_dbghelp.h b/libsanitizer/sanitizer_common/sanitizer_dbghelp.h
new file mode 100644
index 00000000000..bad17a91eb8
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_dbghelp.h
@@ -0,0 +1,40 @@
+//===-- sanitizer_dbghelp.h ------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Wrappers for lazy loaded dbghelp.dll. Provides function pointers and a
+// callback to initialize them.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_SYMBOLIZER_WIN_H
+#define SANITIZER_SYMBOLIZER_WIN_H
+
+#if !SANITIZER_WINDOWS
+#error "sanitizer_dbghelp.h is a Windows-only header"
+#endif
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <dbghelp.h>
+
+namespace __sanitizer {
+
+extern decltype(::StackWalk64) *StackWalk64;
+extern decltype(::SymCleanup) *SymCleanup;
+extern decltype(::SymFromAddr) *SymFromAddr;
+extern decltype(::SymFunctionTableAccess64) *SymFunctionTableAccess64;
+extern decltype(::SymGetLineFromAddr64) *SymGetLineFromAddr64;
+extern decltype(::SymGetModuleBase64) *SymGetModuleBase64;
+extern decltype(::SymGetSearchPathW) *SymGetSearchPathW;
+extern decltype(::SymInitialize) *SymInitialize;
+extern decltype(::SymSetOptions) *SymSetOptions;
+extern decltype(::SymSetSearchPathW) *SymSetSearchPathW;
+extern decltype(::UnDecorateSymbolName) *UnDecorateSymbolName;
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SYMBOLIZER_WIN_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_errno.cc b/libsanitizer/sanitizer_common/sanitizer_errno.cc
new file mode 100644
index 00000000000..b65f0e7ddf1
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_errno.cc
@@ -0,0 +1,33 @@
+//===-- sanitizer_errno.cc --------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into other
+// files (e.g. interceptors are not supposed to include any system headers).
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_internal_defs.h"
+
+#include <errno.h>
+
+namespace __sanitizer {
+
+COMPILER_CHECK(errno_ENOMEM == ENOMEM);
+COMPILER_CHECK(errno_EBUSY == EBUSY);
+COMPILER_CHECK(errno_EINVAL == EINVAL);
+
+// EOWNERDEAD is not present in some older platforms.
+#if defined(EOWNERDEAD)
+extern const int errno_EOWNERDEAD = EOWNERDEAD;
+#else
+extern const int errno_EOWNERDEAD = -1;
+#endif
+
+}  // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_errno.h b/libsanitizer/sanitizer_common/sanitizer_errno.h
new file mode 100644
index 00000000000..e9fc00f6c2a
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_errno.h
@@ -0,0 +1,35 @@
+//===-- sanitizer_errno.h ---------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into sensitive
+// files (e.g. interceptors are not supposed to include any system headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_H
+#define SANITIZER_ERRNO_H
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC
+#  define __errno_location __error
+#elif SANITIZER_ANDROID || SANITIZER_NETBSD
+#  define __errno_location __errno
+#elif SANITIZER_WINDOWS
+#  define __errno_location _errno
+#endif
+
+extern "C" int *__errno_location();
+
+#define errno (*__errno_location())
+
+#endif  // SANITIZER_ERRNO_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_errno_codes.h b/libsanitizer/sanitizer_common/sanitizer_errno_codes.h
new file mode 100644
index 00000000000..709f43b7689
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_errno_codes.h
@@ -0,0 +1,32 @@
+//===-- sanitizer_errno_codes.h ---------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno codes to avoid including errno.h and its dependencies into
+// sensitive files (e.g. interceptors are not supposed to include any system
+// headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_CODES_H
+#define SANITIZER_ERRNO_CODES_H
+
+namespace __sanitizer {
+
+#define errno_ENOMEM 12
+#define errno_EBUSY 16
+#define errno_EINVAL 22
+
+// Those might not present or their value differ on different platforms.
+extern const int errno_EOWNERDEAD;
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ERRNO_CODES_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_file.cc b/libsanitizer/sanitizer_common/sanitizer_file.cc
new file mode 100644
index 00000000000..8740dbb5b6f
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_file.cc
@@ -0,0 +1,175 @@
+//===-- sanitizer_file.cc ------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.  It defines filesystem-related interfaces.  This
+// is separate from sanitizer_common.cc so that it's simpler to disable
+// all the filesystem support code for a port that doesn't use it.
+//
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if !SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+
+namespace __sanitizer {
+
+void CatastrophicErrorWrite(const char *buffer, uptr length) {
+  WriteToFile(kStderrFd, buffer, length);
+}
+
+StaticSpinMutex report_file_mu;
+ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0};
+
+void RawWrite(const char *buffer) {
+  report_file.Write(buffer, internal_strlen(buffer));
+}
+
+void ReportFile::ReopenIfNecessary() {
+  mu->CheckLocked();
+  if (fd == kStdoutFd || fd == kStderrFd) return;
+
+  uptr pid = internal_getpid();
+  // If in tracer, use the parent's file.
+  if (pid == stoptheworld_tracer_pid)
+    pid = stoptheworld_tracer_ppid;
+  if (fd != kInvalidFd) {
+    // If the report file is already opened by the current process,
+    // do nothing. Otherwise the report file was opened by the parent
+    // process, close it now.
+    if (fd_pid == pid)
+      return;
+    else
+      CloseFile(fd);
+  }
+
+  const char *exe_name = GetProcessName();
+  if (common_flags()->log_exe_name && exe_name) {
+    internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix,
+                      exe_name, pid);
+  } else {
+    internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid);
+  }
+  fd = OpenFile(full_path, WrOnly);
+  if (fd == kInvalidFd) {
+    const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
+    WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
+    WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
+    Die();
+  }
+  fd_pid = pid;
+}
+
+void ReportFile::SetReportPath(const char *path) {
+  if (!path)
+    return;
+  uptr len = internal_strlen(path);
+  if (len > sizeof(path_prefix) - 100) {
+    Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n",
+           path[0], path[1], path[2], path[3],
+           path[4], path[5], path[6], path[7]);
+    Die();
+  }
+
+  SpinMutexLock l(mu);
+  if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd)
+    CloseFile(fd);
+  fd = kInvalidFd;
+  if (internal_strcmp(path, "stdout") == 0) {
+    fd = kStdoutFd;
+  } else if (internal_strcmp(path, "stderr") == 0) {
+    fd = kStderrFd;
+  } else {
+    internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
+  }
+}
+
+bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
+                      uptr *read_len, uptr max_len, error_t *errno_p) {
+  uptr PageSize = GetPageSizeCached();
+  uptr kMinFileLen = PageSize;
+  *buff = nullptr;
+  *buff_size = 0;
+  *read_len = 0;
+  // The files we usually open are not seekable, so try different buffer sizes.
+  for (uptr size = kMinFileLen; size <= max_len; size *= 2) {
+    fd_t fd = OpenFile(file_name, RdOnly, errno_p);
+    if (fd == kInvalidFd) return false;
+    UnmapOrDie(*buff, *buff_size);
+    *buff = (char*)MmapOrDie(size, __func__);
+    *buff_size = size;
+    *read_len = 0;
+    // Read up to one page at a time.
+    bool reached_eof = false;
+    while (*read_len + PageSize <= size) {
+      uptr just_read;
+      if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) {
+        UnmapOrDie(*buff, *buff_size);
+        return false;
+      }
+      if (just_read == 0) {
+        reached_eof = true;
+        break;
+      }
+      *read_len += just_read;
+    }
+    CloseFile(fd);
+    if (reached_eof)  // We've read the whole file.
+      break;
+  }
+  return true;
+}
+
+static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':';
+
+char *FindPathToBinary(const char *name) {
+  if (FileExists(name)) {
+    return internal_strdup(name);
+  }
+
+  const char *path = GetEnv("PATH");
+  if (!path)
+    return nullptr;
+  uptr name_len = internal_strlen(name);
+  InternalScopedBuffer<char> buffer(kMaxPathLength);
+  const char *beg = path;
+  while (true) {
+    const char *end = internal_strchrnul(beg, kPathSeparator);
+    uptr prefix_len = end - beg;
+    if (prefix_len + name_len + 2 <= kMaxPathLength) {
+      internal_memcpy(buffer.data(), beg, prefix_len);
+      buffer[prefix_len] = '/';
+      internal_memcpy(&buffer[prefix_len + 1], name, name_len);
+      buffer[prefix_len + 1 + name_len] = '\0';
+      if (FileExists(buffer.data()))
+        return internal_strdup(buffer.data());
+    }
+    if (*end == '\0') break;
+    beg = end + 1;
+  }
+  return nullptr;
+}
+
+} // namespace __sanitizer
+
+using namespace __sanitizer;  // NOLINT
+
+extern "C" {
+void __sanitizer_set_report_path(const char *path) {
+  report_file.SetReportPath(path);
+}
+
+void __sanitizer_set_report_fd(void *fd) {
+  report_file.fd = (fd_t)reinterpret_cast<uptr>(fd);
+  report_file.fd_pid = internal_getpid();
+}
+} // extern "C"
+
+#endif  // !SANITIZER_FUCHSIA
diff --git a/libsanitizer/sanitizer_common/sanitizer_file.h b/libsanitizer/sanitizer_common/sanitizer_file.h
new file mode 100644
index 00000000000..3f9e8ab2849
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_file.h
@@ -0,0 +1,108 @@
+//===-- sanitizer_file.h ---------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between run-time libraries of sanitizers.
+// It declares filesystem-related interfaces.  This is separate from
+// sanitizer_common.h so that it's simpler to disable all the filesystem
+// support code for a port that doesn't use it.
+//
+//===---------------------------------------------------------------------===//
+#ifndef SANITIZER_FILE_H
+#define SANITIZER_FILE_H
+
+#include "sanitizer_interface_internal.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+struct ReportFile {
+  void Write(const char *buffer, uptr length);
+  bool SupportsColors();
+  void SetReportPath(const char *path);
+
+  // Don't use fields directly. They are only declared public to allow
+  // aggregate initialization.
+
+  // Protects fields below.
+  StaticSpinMutex *mu;
+  // Opened file descriptor. Defaults to stderr. It may be equal to
+  // kInvalidFd, in which case new file will be opened when necessary.
+  fd_t fd;
+  // Path prefix of report file, set via __sanitizer_set_report_path.
+  char path_prefix[kMaxPathLength];
+  // Full path to report, obtained as <path_prefix>.PID
+  char full_path[kMaxPathLength];
+  // PID of the process that opened fd. If a fork() occurs,
+  // the PID of child will be different from fd_pid.
+  uptr fd_pid;
+
+ private:
+  void ReopenIfNecessary();
+};
+extern ReportFile report_file;
+
+enum FileAccessMode {
+  RdOnly,
+  WrOnly,
+  RdWr
+};
+
+// Returns kInvalidFd on error.
+fd_t OpenFile(const char *filename, FileAccessMode mode,
+              error_t *errno_p = nullptr);
+void CloseFile(fd_t);
+
+// Return true on success, false on error.
+bool ReadFromFile(fd_t fd, void *buff, uptr buff_size,
+                  uptr *bytes_read = nullptr, error_t *error_p = nullptr);
+bool WriteToFile(fd_t fd, const void *buff, uptr buff_size,
+                 uptr *bytes_written = nullptr, error_t *error_p = nullptr);
+
+bool RenameFile(const char *oldpath, const char *newpath,
+                error_t *error_p = nullptr);
+
+// Scoped file handle closer.
+struct FileCloser {
+  explicit FileCloser(fd_t fd) : fd(fd) {}
+  ~FileCloser() { CloseFile(fd); }
+  fd_t fd;
+};
+
+bool SupportsColoredOutput(fd_t fd);
+
+// OS
+const char *GetPwd();
+bool FileExists(const char *filename);
+char *FindPathToBinary(const char *name);
+bool IsPathSeparator(const char c);
+bool IsAbsolutePath(const char *path);
+// Starts a subprocess and returs its pid.
+// If *_fd parameters are not kInvalidFd their corresponding input/output
+// streams will be redirect to the file. The files will always be closed
+// in parent process even in case of an error.
+// The child process will close all fds after STDERR_FILENO
+// before passing control to a program.
+pid_t StartSubprocess(const char *filename, const char *const argv[],
+                      fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd,
+                      fd_t stderr_fd = kInvalidFd);
+// Checks if specified process is still running
+bool IsProcessRunning(pid_t pid);
+// Waits for the process to finish and returns its exit code.
+// Returns -1 in case of an error.
+int WaitForProcess(pid_t pid);
+
+// Maps given file to virtual memory, and returns pointer to it
+// (or NULL if mapping fails). Stores the size of mmaped region
+// in '*buff_size'.
+void *MapFileToMemory(const char *file_name, uptr *buff_size);
+void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FILE_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_flag_parser.h b/libsanitizer/sanitizer_common/sanitizer_flag_parser.h
index 7827d735770..13677c0ea68 100644
--- a/libsanitizer/sanitizer_common/sanitizer_flag_parser.h
+++ b/libsanitizer/sanitizer_common/sanitizer_flag_parser.h
@@ -32,27 +32,48 @@ class FlagHandler : public FlagHandlerBase {
   bool Parse(const char *value) final;
 };
 
-template <>
-inline bool FlagHandler<bool>::Parse(const char *value) {
+inline bool ParseBool(const char *value, bool *b) {
   if (internal_strcmp(value, "0") == 0 ||
       internal_strcmp(value, "no") == 0 ||
       internal_strcmp(value, "false") == 0) {
-    *t_ = false;
+    *b = false;
     return true;
   }
   if (internal_strcmp(value, "1") == 0 ||
       internal_strcmp(value, "yes") == 0 ||
       internal_strcmp(value, "true") == 0) {
-    *t_ = true;
+    *b = true;
     return true;
   }
+  return false;
+}
+
+template <>
+inline bool FlagHandler<bool>::Parse(const char *value) {
+  if (ParseBool(value, t_)) return true;
   Printf("ERROR: Invalid value for bool option: '%s'\n", value);
   return false;
 }
 
 template <>
+inline bool FlagHandler<HandleSignalMode>::Parse(const char *value) {
+  bool b;
+  if (ParseBool(value, &b)) {
+    *t_ = b ? kHandleSignalYes : kHandleSignalNo;
+    return true;
+  }
+  if (internal_strcmp(value, "2") == 0 ||
+      internal_strcmp(value, "exclusive") == 0) {
+    *t_ = kHandleSignalExclusive;
+    return true;
+  }
+  Printf("ERROR: Invalid value for signal handler option: '%s'\n", value);
+  return false;
+}
+
+template <>
 inline bool FlagHandler<const char *>::Parse(const char *value) {
-  *t_ = internal_strdup(value);
+  *t_ = value;
   return true;
 }
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.h b/libsanitizer/sanitizer_common/sanitizer_flags.h
index ff64af10ff5..2e3739e915d 100644
--- a/libsanitizer/sanitizer_common/sanitizer_flags.h
+++ b/libsanitizer/sanitizer_common/sanitizer_flags.h
@@ -16,6 +16,12 @@
 
 namespace __sanitizer {
 
+enum HandleSignalMode {
+  kHandleSignalNo,
+  kHandleSignalYes,
+  kHandleSignalExclusive,
+};
+
 struct CommonFlags {
 #define COMMON_FLAG(Type, Name, DefaultValue, Description) Type Name;
 #include "sanitizer_flags.inc"
diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.inc b/libsanitizer/sanitizer_common/sanitizer_flags.inc
index ccc0e416f4a..9e9b8a7daa6 100644
--- a/libsanitizer/sanitizer_common/sanitizer_flags.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_flags.inc
@@ -60,7 +60,7 @@ COMMON_FLAG(
 COMMON_FLAG(
     int, verbosity, 0,
     "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).")
-COMMON_FLAG(bool, detect_leaks, true, "Enable memory leak detection.")
+COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.")
 COMMON_FLAG(
     bool, leak_check_at_exit, true,
     "Invoke leak checking in an atexit handler. Has no effect if "
@@ -72,18 +72,28 @@ COMMON_FLAG(bool, allocator_may_return_null, false,
 COMMON_FLAG(bool, print_summary, true,
             "If false, disable printing error summaries in addition to error "
             "reports.")
+COMMON_FLAG(int, print_module_map, 0,
+            "OS X only (0 - don't print, 1 - print only once before process "
+            "exits, 2 - print after each report).")
 COMMON_FLAG(bool, check_printf, true, "Check printf arguments.")
-COMMON_FLAG(bool, handle_segv, true,
-            "If set, registers the tool's custom SIGSEGV/SIGBUS handler.")
-COMMON_FLAG(bool, handle_abort, false,
-            "If set, registers the tool's custom SIGABRT handler.")
-COMMON_FLAG(bool, handle_sigill, false,
-            "If set, registers the tool's custom SIGILL handler.")
-COMMON_FLAG(bool, handle_sigfpe, true,
-            "If set, registers the tool's custom SIGFPE handler.")
-COMMON_FLAG(bool, allow_user_segv_handler, false,
-            "If set, allows user to register a SEGV handler even if the tool "
-            "registers one.")
+#define COMMON_FLAG_HANDLE_SIGNAL_HELP(signal) \
+    "Controls custom tool's " #signal " handler (0 - do not registers the " \
+    "handler, 1 - register the handler and allow user to set own, " \
+    "2 - registers the handler and block user from changing it). "
+COMMON_FLAG(HandleSignalMode, handle_segv, kHandleSignalYes,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGSEGV))
+COMMON_FLAG(HandleSignalMode, handle_sigbus, kHandleSignalYes,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGBUS))
+COMMON_FLAG(HandleSignalMode, handle_abort, kHandleSignalNo,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGABRT))
+COMMON_FLAG(HandleSignalMode, handle_sigill, kHandleSignalNo,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGILL))
+COMMON_FLAG(HandleSignalMode, handle_sigfpe, kHandleSignalYes,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGFPE))
+#undef COMMON_FLAG_HANDLE_SIGNAL_HELP
+COMMON_FLAG(bool, allow_user_segv_handler, true,
+            "Deprecated. True has no effect, use handle_sigbus=1. If false, "
+            "handle_*=1 will be upgraded to handle_*=2.")
 COMMON_FLAG(bool, use_sigaltstack, true,
             "If set, uses alternate stack for signal handling.")
 COMMON_FLAG(bool, detect_deadlocks, false,
@@ -117,9 +127,11 @@ COMMON_FLAG(uptr, soft_rss_limit_mb, 0,
             " This limit does not affect memory allocations other than"
             " malloc/new.")
 COMMON_FLAG(bool, heap_profile, false, "Experimental heap profiler, asan-only")
-COMMON_FLAG(bool, allocator_release_to_os, false,
-            "Experimental. If true, try to periodically release unused"
-            " memory to the OS.\n")
+COMMON_FLAG(s32, allocator_release_to_os_interval_ms, kReleaseToOSIntervalNever,
+            "Experimental. Only affects a 64-bit allocator. If set, tries to "
+            "release unused memory to the OS, but not more often than this "
+            "interval (in milliseconds). Negative values mean do not attempt "
+            "to release memory to the OS.\n")
 COMMON_FLAG(bool, can_use_proc_maps_statm, true,
             "If false, do not attempt to read /proc/maps/statm."
             " Mostly useful for testing sanitizers.")
@@ -127,22 +139,6 @@ COMMON_FLAG(
     bool, coverage, false,
     "If set, coverage information will be dumped at program shutdown (if the "
     "coverage instrumentation was enabled at compile time).")
-COMMON_FLAG(bool, coverage_pcs, true,
-            "If set (and if 'coverage' is set too), the coverage information "
-            "will be dumped as a set of PC offsets for every module.")
-COMMON_FLAG(bool, coverage_order_pcs, false,
-             "If true, the PCs will be dumped in the order they've"
-             " appeared during the execution.")
-COMMON_FLAG(bool, coverage_bitset, false,
-            "If set (and if 'coverage' is set too), the coverage information "
-            "will also be dumped as a bitset to a separate file.")
-COMMON_FLAG(bool, coverage_counters, false,
-            "If set (and if 'coverage' is set too), the bitmap that corresponds"
-            " to coverage counters will be dumped.")
-COMMON_FLAG(bool, coverage_direct, SANITIZER_ANDROID,
-            "If set, coverage information will be dumped directly to a memory "
-            "mapped file. This way data is not lost even if the process is "
-            "suddenly killed.")
 COMMON_FLAG(const char *, coverage_dir, ".",
             "Target directory for coverage dumps. Defaults to the current "
             "directory.")
@@ -183,12 +179,18 @@ COMMON_FLAG(bool, intercept_strstr, true,
 COMMON_FLAG(bool, intercept_strspn, true,
             "If set, uses custom wrappers for strspn and strcspn function "
             "to find more errors.")
+COMMON_FLAG(bool, intercept_strtok, true,
+            "If set, uses a custom wrapper for the strtok function "
+            "to find more errors.")
 COMMON_FLAG(bool, intercept_strpbrk, true,
             "If set, uses custom wrappers for strpbrk function "
             "to find more errors.")
 COMMON_FLAG(bool, intercept_strlen, true,
             "If set, uses custom wrappers for strlen and strnlen functions "
             "to find more errors.")
+COMMON_FLAG(bool, intercept_strndup, true,
+            "If set, uses custom wrappers for strndup functions "
+            "to find more errors.")
 COMMON_FLAG(bool, intercept_strchr, true,
             "If set, uses custom wrappers for strchr, strchrnul, and strrchr "
             "functions to find more errors.")
@@ -225,3 +227,8 @@ COMMON_FLAG(bool, print_cmdline, false, "Print command line on crash "
             "(asan only).")
 COMMON_FLAG(bool, html_cov_report, false, "Generate html coverage report.")
 COMMON_FLAG(const char *, sancov_path, "sancov", "Sancov tool location.")
+COMMON_FLAG(bool, dump_instruction_bytes, false,
+          "If true, dump 16 bytes starting at the instruction that caused SEGV")
+COMMON_FLAG(bool, dump_registers, true,
+          "If true, dump values of CPU registers when SEGV happens. Only "
+          "available on OS X for now.")
diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cc b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cc
new file mode 100644
index 00000000000..da7018ca33d
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cc
@@ -0,0 +1,517 @@
+//===-- sanitizer_fuchsia.cc ---------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and other sanitizer
+// run-time libraries and implements Fuchsia-specific functions from
+// sanitizer_common.h.
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_stacktrace.h"
+
+#include <limits.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <unwind.h>
+#include <zircon/errors.h>
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+
+namespace __sanitizer {
+
+void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); }
+
+uptr internal_sched_yield() {
+  zx_status_t status = _zx_nanosleep(0);
+  CHECK_EQ(status, ZX_OK);
+  return 0;  // Why doesn't this return void?
+}
+
+static void internal_nanosleep(zx_time_t ns) {
+  zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns));
+  CHECK_EQ(status, ZX_OK);
+}
+
+unsigned int internal_sleep(unsigned int seconds) {
+  internal_nanosleep(ZX_SEC(seconds));
+  return 0;
+}
+
+u64 NanoTime() { return _zx_time_get(ZX_CLOCK_UTC); }
+
+uptr internal_getpid() {
+  zx_info_handle_basic_t info;
+  zx_status_t status =
+      _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &info,
+                          sizeof(info), NULL, NULL);
+  CHECK_EQ(status, ZX_OK);
+  uptr pid = static_cast<uptr>(info.koid);
+  CHECK_EQ(pid, info.koid);
+  return pid;
+}
+
+uptr GetThreadSelf() { return reinterpret_cast<uptr>(thrd_current()); }
+
+uptr GetTid() { return GetThreadSelf(); }
+
+void Abort() { abort(); }
+
+int Atexit(void (*function)(void)) { return atexit(function); }
+
+void SleepForSeconds(int seconds) { internal_sleep(seconds); }
+
+void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); }
+
+void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
+  pthread_attr_t attr;
+  CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
+  void *base;
+  size_t size;
+  CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0);
+  CHECK_EQ(pthread_attr_destroy(&attr), 0);
+
+  *stack_bottom = reinterpret_cast<uptr>(base);
+  *stack_top = *stack_bottom + size;
+}
+
+void MaybeReexec() {}
+void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+void DisableCoreDumperIfNecessary() {}
+void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
+void StartReportDeadlySignal() {}
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context) {}
+void SetAlternateSignalStack() {}
+void UnsetAlternateSignalStack() {}
+void InitTlsSize() {}
+
+void PrintModuleMap() {}
+
+bool SignalContext::IsStackOverflow() const { return false; }
+void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
+const char *SignalContext::Describe() const { UNIMPLEMENTED(); }
+
+struct UnwindTraceArg {
+  BufferedStackTrace *stack;
+  u32 max_depth;
+};
+
+_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
+  UnwindTraceArg *arg = static_cast<UnwindTraceArg *>(param);
+  CHECK_LT(arg->stack->size, arg->max_depth);
+  uptr pc = _Unwind_GetIP(ctx);
+  if (pc < PAGE_SIZE) return _URC_NORMAL_STOP;
+  arg->stack->trace_buffer[arg->stack->size++] = pc;
+  return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
+                                             : _URC_NO_REASON);
+}
+
+void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) {
+  CHECK_GE(max_depth, 2);
+  size = 0;
+  UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
+  _Unwind_Backtrace(Unwind_Trace, &arg);
+  CHECK_GT(size, 0);
+  // We need to pop a few frames so that pc is on top.
+  uptr to_pop = LocatePcInTrace(pc);
+  // trace_buffer[0] belongs to the current function so we always pop it,
+  // unless there is only 1 frame in the stack trace (1 frame is always better
+  // than 0!).
+  PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
+  trace_buffer[0] = pc;
+}
+
+void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context,
+                                                    u32 max_depth) {
+  CHECK_NE(context, nullptr);
+  UNREACHABLE("signal context doesn't exist");
+}
+
+enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
+
+BlockingMutex::BlockingMutex() {
+  // NOTE!  It's important that this use internal_memset, because plain
+  // memset might be intercepted (e.g., actually be __asan_memset).
+  // Defining this so the compiler initializes each field, e.g.:
+  //   BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {}
+  // might result in the compiler generating a call to memset, which would
+  // have the same problem.
+  internal_memset(this, 0, sizeof(*this));
+}
+
+void BlockingMutex::Lock() {
+  CHECK_EQ(owner_, 0);
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
+    return;
+  while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
+    zx_status_t status = _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m),
+                                        MtxSleeping, ZX_TIME_INFINITE);
+    if (status != ZX_ERR_BAD_STATE)  // Normal race.
+      CHECK_EQ(status, ZX_OK);
+  }
+}
+
+void BlockingMutex::Unlock() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
+  CHECK_NE(v, MtxUnlocked);
+  if (v == MtxSleeping) {
+    zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(m), 1);
+    CHECK_EQ(status, ZX_OK);
+  }
+}
+
+void BlockingMutex::CheckLocked() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
+}
+
+uptr GetPageSize() { return PAGE_SIZE; }
+
+uptr GetMmapGranularity() { return PAGE_SIZE; }
+
+sanitizer_shadow_bounds_t ShadowBounds;
+
+uptr GetMaxVirtualAddress() {
+  ShadowBounds = __sanitizer_shadow_bounds();
+  return ShadowBounds.memory_limit - 1;
+}
+
+static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
+                                  bool raw_report, bool die_for_nomem) {
+  size = RoundUpTo(size, PAGE_SIZE);
+
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status,
+                              raw_report);
+    return nullptr;
+  }
+  _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
+                          internal_strlen(mem_type));
+
+  // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
+  uintptr_t addr;
+  status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, size,
+                        ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr);
+  _zx_handle_close(vmo);
+
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status,
+                              raw_report);
+    return nullptr;
+  }
+
+  IncreaseTotalMmap(size);
+
+  return reinterpret_cast<void *>(addr);
+}
+
+void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
+  return DoAnonymousMmapOrDie(size, mem_type, raw_report, true);
+}
+
+void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
+  return MmapOrDie(size, mem_type);
+}
+
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  return DoAnonymousMmapOrDie(size, mem_type, false, false);
+}
+
+// MmapNoAccess and MmapFixedOrDie are used only by sanitizer_allocator.
+// Instead of doing exactly what they say, we make MmapNoAccess actually
+// just allocate a VMAR to reserve the address space.  Then MmapFixedOrDie
+// uses that VMAR instead of the root.
+
+zx_handle_t allocator_vmar = ZX_HANDLE_INVALID;
+uintptr_t allocator_vmar_base;
+size_t allocator_vmar_size;
+
+void *MmapNoAccess(uptr size) {
+  size = RoundUpTo(size, PAGE_SIZE);
+  CHECK_EQ(allocator_vmar, ZX_HANDLE_INVALID);
+  uintptr_t base;
+  zx_status_t status =
+      _zx_vmar_allocate(_zx_vmar_root_self(), 0, size,
+                        ZX_VM_FLAG_CAN_MAP_READ | ZX_VM_FLAG_CAN_MAP_WRITE |
+                            ZX_VM_FLAG_CAN_MAP_SPECIFIC,
+                        &allocator_vmar, &base);
+  if (status != ZX_OK)
+    ReportMmapFailureAndDie(size, "sanitizer allocator address space",
+                            "zx_vmar_allocate", status);
+
+  allocator_vmar_base = base;
+  allocator_vmar_size = size;
+  return reinterpret_cast<void *>(base);
+}
+
+constexpr const char kAllocatorVmoName[] = "sanitizer_allocator";
+
+static void *DoMmapFixedOrDie(uptr fixed_addr, uptr size, bool die_for_nomem) {
+  size = RoundUpTo(size, PAGE_SIZE);
+
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+      ReportMmapFailureAndDie(size, kAllocatorVmoName, "zx_vmo_create", status);
+    return nullptr;
+  }
+  _zx_object_set_property(vmo, ZX_PROP_NAME, kAllocatorVmoName,
+                          sizeof(kAllocatorVmoName) - 1);
+
+  DCHECK_GE(fixed_addr, allocator_vmar_base);
+  uintptr_t offset = fixed_addr - allocator_vmar_base;
+  DCHECK_LE(size, allocator_vmar_size);
+  DCHECK_GE(allocator_vmar_size - offset, size);
+
+  uintptr_t addr;
+  status = _zx_vmar_map(
+      allocator_vmar, offset, vmo, 0, size,
+      ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | ZX_VM_FLAG_SPECIFIC,
+      &addr);
+  _zx_handle_close(vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+      ReportMmapFailureAndDie(size, kAllocatorVmoName, "zx_vmar_map", status);
+    return nullptr;
+  }
+
+  IncreaseTotalMmap(size);
+
+  return reinterpret_cast<void *>(addr);
+}
+
+void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
+  return DoMmapFixedOrDie(fixed_addr, size, true);
+}
+
+void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) {
+  return DoMmapFixedOrDie(fixed_addr, size, false);
+}
+
+// This should never be called.
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
+  UNIMPLEMENTED();
+}
+
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type) {
+  CHECK_GE(size, PAGE_SIZE);
+  CHECK(IsPowerOfTwo(size));
+  CHECK(IsPowerOfTwo(alignment));
+
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status, false);
+    return nullptr;
+  }
+  _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
+                          internal_strlen(mem_type));
+
+  // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
+
+  // Map a larger size to get a chunk of address space big enough that
+  // it surely contains an aligned region of the requested size.  Then
+  // overwrite the aligned middle portion with a mapping from the
+  // beginning of the VMO, and unmap the excess before and after.
+  size_t map_size = size + alignment;
+  uintptr_t addr;
+  status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size,
+                        ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr);
+  if (status == ZX_OK) {
+    uintptr_t map_addr = addr;
+    uintptr_t map_end = map_addr + map_size;
+    addr = RoundUpTo(map_addr, alignment);
+    uintptr_t end = addr + size;
+    if (addr != map_addr) {
+      zx_info_vmar_t info;
+      status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info,
+                                   sizeof(info), NULL, NULL);
+      if (status == ZX_OK) {
+        uintptr_t new_addr;
+        status =
+            _zx_vmar_map(_zx_vmar_root_self(), addr - info.base, vmo, 0, size,
+                         ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE |
+                             ZX_VM_FLAG_SPECIFIC_OVERWRITE,
+                         &new_addr);
+        if (status == ZX_OK) CHECK_EQ(new_addr, addr);
+      }
+    }
+    if (status == ZX_OK && addr != map_addr)
+      status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr);
+    if (status == ZX_OK && end != map_end)
+      status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end);
+  }
+  _zx_handle_close(vmo);
+
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status, false);
+    return nullptr;
+  }
+
+  IncreaseTotalMmap(size);
+
+  return reinterpret_cast<void *>(addr);
+}
+
+void UnmapOrDie(void *addr, uptr size) {
+  if (!addr || !size) return;
+  size = RoundUpTo(size, PAGE_SIZE);
+
+  zx_status_t status = _zx_vmar_unmap(_zx_vmar_root_self(),
+                                      reinterpret_cast<uintptr_t>(addr), size);
+  if (status != ZX_OK) {
+    Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
+           SanitizerToolName, size, size, addr);
+    CHECK("unable to unmap" && 0);
+  }
+
+  DecreaseTotalMmap(size);
+}
+
+// This is used on the shadow mapping, which cannot be changed.
+// Zircon doesn't have anything like MADV_DONTNEED.
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {}
+
+void DumpProcessMap() {
+  UNIMPLEMENTED();  // TODO(mcgrathr): write it
+}
+
+bool IsAccessibleMemoryRange(uptr beg, uptr size) {
+  // TODO(mcgrathr): Figure out a better way.
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+  if (status == ZX_OK) {
+    while (size > 0) {
+      size_t wrote;
+      status = _zx_vmo_write(vmo, reinterpret_cast<const void *>(beg), 0, size,
+                             &wrote);
+      if (status != ZX_OK) break;
+      CHECK_GT(wrote, 0);
+      CHECK_LE(wrote, size);
+      beg += wrote;
+      size -= wrote;
+    }
+    _zx_handle_close(vmo);
+  }
+  return status == ZX_OK;
+}
+
+// FIXME implement on this platform.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {}
+
+bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
+                      uptr *read_len, uptr max_len, error_t *errno_p) {
+  zx_handle_t vmo;
+  zx_status_t status = __sanitizer_get_configuration(file_name, &vmo);
+  if (status == ZX_OK) {
+    uint64_t vmo_size;
+    status = _zx_vmo_get_size(vmo, &vmo_size);
+    if (status == ZX_OK) {
+      if (vmo_size < max_len) max_len = vmo_size;
+      size_t map_size = RoundUpTo(max_len, PAGE_SIZE);
+      uintptr_t addr;
+      status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size,
+                            ZX_VM_FLAG_PERM_READ, &addr);
+      if (status == ZX_OK) {
+        *buff = reinterpret_cast<char *>(addr);
+        *buff_size = map_size;
+        *read_len = max_len;
+      }
+    }
+    _zx_handle_close(vmo);
+  }
+  if (status != ZX_OK && errno_p) *errno_p = status;
+  return status == ZX_OK;
+}
+
+void RawWrite(const char *buffer) {
+  __sanitizer_log_write(buffer, internal_strlen(buffer));
+}
+
+void CatastrophicErrorWrite(const char *buffer, uptr length) {
+  __sanitizer_log_write(buffer, length);
+}
+
+char **StoredArgv;
+char **StoredEnviron;
+
+char **GetArgv() { return StoredArgv; }
+
+const char *GetEnv(const char *name) {
+  if (StoredEnviron) {
+    uptr NameLen = internal_strlen(name);
+    for (char **Env = StoredEnviron; *Env != 0; Env++) {
+      if (internal_strncmp(*Env, name, NameLen) == 0 && (*Env)[NameLen] == '=')
+        return (*Env) + NameLen + 1;
+    }
+  }
+  return nullptr;
+}
+
+uptr ReadBinaryName(/*out*/ char *buf, uptr buf_len) {
+  const char *argv0 = StoredArgv[0];
+  if (!argv0) argv0 = "<UNKNOWN>";
+  internal_strncpy(buf, argv0, buf_len);
+  return internal_strlen(buf);
+}
+
+uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
+  return ReadBinaryName(buf, buf_len);
+}
+
+uptr MainThreadStackBase, MainThreadStackSize;
+
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN);
+  size_t size;
+  CHECK_EQ(_zx_cprng_draw(buffer, length, &size), ZX_OK);
+  CHECK_EQ(size, length);
+  return true;
+}
+
+}  // namespace __sanitizer
+
+using namespace __sanitizer;  // NOLINT
+
+extern "C" {
+void __sanitizer_startup_hook(int argc, char **argv, char **envp,
+                              void *stack_base, size_t stack_size) {
+  __sanitizer::StoredArgv = argv;
+  __sanitizer::StoredEnviron = envp;
+  __sanitizer::MainThreadStackBase = reinterpret_cast<uintptr_t>(stack_base);
+  __sanitizer::MainThreadStackSize = stack_size;
+}
+
+void __sanitizer_set_report_path(const char *path) {
+  // Handle the initialization code in each sanitizer, but no other calls.
+  // This setting is never consulted on Fuchsia.
+  DCHECK_EQ(path, common_flags()->log_path);
+}
+
+void __sanitizer_set_report_fd(void *fd) {
+  UNREACHABLE("not available on Fuchsia");
+}
+}  // extern "C"
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.h b/libsanitizer/sanitizer_common/sanitizer_fuchsia.h
new file mode 100644
index 00000000000..59b679d6c7f
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.h
@@ -0,0 +1,29 @@
+//===-- sanitizer_fuchsia.h ------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// Fuchsia-specific sanitizer support.
+//
+//===---------------------------------------------------------------------===//
+#ifndef SANITIZER_FUCHSIA_H
+#define SANITIZER_FUCHSIA_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+
+#include <zircon/sanitizer.h>
+
+namespace __sanitizer {
+
+extern uptr MainThreadStackBase, MainThreadStackSize;
+extern sanitizer_shadow_bounds_t ShadowBounds;
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FUCHSIA
+#endif  // SANITIZER_FUCHSIA_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_interface_internal.h b/libsanitizer/sanitizer_common/sanitizer_interface_internal.h
index 34e80c3b50f..08c110c707c 100644
--- a/libsanitizer/sanitizer_common/sanitizer_interface_internal.h
+++ b/libsanitizer/sanitizer_common/sanitizer_interface_internal.h
@@ -45,7 +45,10 @@ extern "C" {
   void __sanitizer_report_error_summary(const char *error_summary);
 
   SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
-  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init();
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
+      const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();
+
   SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
   SANITIZER_INTERFACE_ATTRIBUTE
   void __sanitizer_annotate_contiguous_container(const void *beg,
@@ -58,6 +61,49 @@ extern "C" {
   SANITIZER_INTERFACE_ATTRIBUTE
   const void *__sanitizer_contiguous_container_find_bad_address(
       const void *beg, const void *mid, const void *end);
-  } // extern "C"
+
+  SANITIZER_INTERFACE_ATTRIBUTE
+  int __sanitizer_get_module_and_offset_for_pc(
+      __sanitizer::uptr pc, char *module_path,
+      __sanitizer::uptr module_path_len, __sanitizer::uptr *pc_offset);
+
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp1();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp2();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp4();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp8();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp1();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp2();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp4();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp8();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_switch();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_div4();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_div8();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_gep();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_pc_indir();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_pc_guard(__sanitizer::u32*);
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*,
+                                           __sanitizer::u32*);
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_8bit_counters_init();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_pcs_init();
+} // extern "C"
 
 #endif  // SANITIZER_INTERFACE_INTERNAL_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h
index 676ade143d8..ef405dec048 100644
--- a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h
+++ b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h
@@ -19,8 +19,11 @@
 
 // Only use SANITIZER_*ATTRIBUTE* before the function return type!
 #if SANITIZER_WINDOWS
+#if SANITIZER_IMPORT_INTERFACE
+# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllimport)
+#else
 # define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport)
-// FIXME find out what we need on Windows, if anything.
+#endif
 # define SANITIZER_WEAK_ATTRIBUTE
 #elif SANITIZER_GO
 # define SANITIZER_INTERFACE_ATTRIBUTE
@@ -30,11 +33,56 @@
 # define SANITIZER_WEAK_ATTRIBUTE  __attribute__((weak))
 #endif
 
-#if (SANITIZER_LINUX || SANITIZER_WINDOWS) && !SANITIZER_GO
+// TLS is handled differently on different platforms
+#if SANITIZER_LINUX
+# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
+    __attribute__((tls_model("initial-exec"))) thread_local
+#else
+# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE
+#endif
+
+//--------------------------- WEAK FUNCTIONS ---------------------------------//
+// When working with weak functions, to simplify the code and make it more
+// portable, when possible define a default implementation using this macro:
+//
+// SANITIZER_INTERFACE_WEAK_DEF(<return_type>, <name>, <parameter list>)
+//
+// For example:
+//   SANITIZER_INTERFACE_WEAK_DEF(bool, compare, int a, int b) { return a > b; }
+//
+#if SANITIZER_WINDOWS
+#include "sanitizer_win_defs.h"
+# define SANITIZER_INTERFACE_WEAK_DEF(ReturnType, Name, ...)                   \
+  WIN_WEAK_EXPORT_DEF(ReturnType, Name, __VA_ARGS__)
+#else
+# define SANITIZER_INTERFACE_WEAK_DEF(ReturnType, Name, ...)                   \
+  extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE            \
+  ReturnType Name(__VA_ARGS__)
+#endif
+
+// SANITIZER_SUPPORTS_WEAK_HOOKS means that we support real weak functions that
+// will evaluate to a null pointer when not defined.
+#ifndef SANITIZER_SUPPORTS_WEAK_HOOKS
+#if (SANITIZER_LINUX || SANITIZER_MAC) && !SANITIZER_GO
 # define SANITIZER_SUPPORTS_WEAK_HOOKS 1
 #else
 # define SANITIZER_SUPPORTS_WEAK_HOOKS 0
 #endif
+#endif // SANITIZER_SUPPORTS_WEAK_HOOKS
+// For some weak hooks that will be called very often and we want to avoid the
+// overhead of executing the default implementation when it is not necessary,
+// we can use the flag SANITIZER_SUPPORTS_WEAK_HOOKS to only define the default
+// implementation for platforms that doesn't support weak symbols. For example:
+//
+//   #if !SANITIZER_SUPPORT_WEAK_HOOKS
+//     SANITIZER_INTERFACE_WEAK_DEF(bool, compare_hook, int a, int b) {
+//       return a > b;
+//     }
+//   #endif
+//
+// And then use it as: if (compare_hook) compare_hook(a, b);
+//----------------------------------------------------------------------------//
+
 
 // We can use .preinit_array section on Linux to call sanitizer initialization
 // functions very early in the process startup (unless PIC macro is defined).
@@ -89,12 +137,8 @@ typedef int error_t;
 #endif
 typedef int pid_t;
 
-// WARNING: OFF_T may be different from OS type off_t, depending on the value of
-// _FILE_OFFSET_BITS. This definition of OFF_T matches the ABI of system calls
-// like pread and mmap, as opposed to pread64 and mmap64.
-// FreeBSD, Mac and Linux/x86-64 are special.
-#if SANITIZER_FREEBSD || SANITIZER_MAC || \
-  (SANITIZER_LINUX && defined(__x86_64__))
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC || \
+    (SANITIZER_LINUX && defined(__x86_64__))
 typedef u64 OFF_T;
 #else
 typedef uptr OFF_T;
@@ -112,6 +156,12 @@ typedef u32 operator_new_size_type;
 # endif
 #endif
 
+#if SANITIZER_MAC
+// On Darwin, thread IDs are 64-bit even on 32-bit systems.
+typedef u64 tid_t;
+#else
+typedef uptr tid_t;
+#endif
 
 // ----------- ATTENTION -------------
 // This header should NOT include any other headers to avoid portability issues.
@@ -211,8 +261,8 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond,
 
 #define CHECK_IMPL(c1, op, c2) \
   do { \
-    __sanitizer::u64 v1 = (u64)(c1); \
-    __sanitizer::u64 v2 = (u64)(c2); \
+    __sanitizer::u64 v1 = (__sanitizer::u64)(c1); \
+    __sanitizer::u64 v2 = (__sanitizer::u64)(c2); \
     if (UNLIKELY(!(v1 op v2))) \
       __sanitizer::CheckFailed(__FILE__, __LINE__, \
         "(" #c1 ") " #op " (" #c2 ")", v1, v2); \
@@ -287,13 +337,13 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond,
 enum LinkerInitialized { LINKER_INITIALIZED = 0 };
 
 #if !defined(_MSC_VER) || defined(__clang__)
-# if SANITIZER_S390_31
-#  define GET_CALLER_PC() \
-  (uptr)__builtin_extract_return_addr(__builtin_return_address(0))
-# else
-#  define GET_CALLER_PC() (uptr)__builtin_return_address(0)
-# endif
-# define GET_CURRENT_FRAME() (uptr)__builtin_frame_address(0)
+#if SANITIZER_S390_31
+#define GET_CALLER_PC() \
+  (__sanitizer::uptr) __builtin_extract_return_addr(__builtin_return_address(0))
+#else
+#define GET_CALLER_PC() (__sanitizer::uptr) __builtin_return_address(0)
+#endif
+#define GET_CURRENT_FRAME() (__sanitizer::uptr) __builtin_frame_address(0)
 inline void Trap() {
   __builtin_trap();
 }
@@ -302,9 +352,10 @@ extern "C" void* _ReturnAddress(void);
 extern "C" void* _AddressOfReturnAddress(void);
 # pragma intrinsic(_ReturnAddress)
 # pragma intrinsic(_AddressOfReturnAddress)
-# define GET_CALLER_PC() (uptr)_ReturnAddress()
+#define GET_CALLER_PC() (__sanitizer::uptr) _ReturnAddress()
 // CaptureStackBackTrace doesn't need to know BP on Windows.
-# define GET_CURRENT_FRAME() (((uptr)_AddressOfReturnAddress()) + sizeof(uptr))
+#define GET_CURRENT_FRAME() \
+  (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
 
 extern "C" void __ud2(void);
 # pragma intrinsic(__ud2)
@@ -322,11 +373,11 @@ inline void Trap() {
   }
 
 // Forces the compiler to generate a frame pointer in the function.
-#define ENABLE_FRAME_POINTER                                       \
-  do {                                                             \
-    volatile uptr enable_fp;                                       \
-    enable_fp = GET_CURRENT_FRAME();                               \
-    (void)enable_fp;                                               \
+#define ENABLE_FRAME_POINTER              \
+  do {                                    \
+    volatile __sanitizer::uptr enable_fp; \
+    enable_fp = GET_CURRENT_FRAME();      \
+    (void)enable_fp;                      \
   } while (0)
 
 }  // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_libignore.cc b/libsanitizer/sanitizer_common/sanitizer_libignore.cc
index 4b8cbed5ee3..e20c7559b8e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_libignore.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_libignore.cc
@@ -7,7 +7,7 @@
 
 #include "sanitizer_platform.h"
 
-#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD
 
 #include "sanitizer_libignore.h"
 #include "sanitizer_flags.h"
@@ -48,23 +48,23 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
   }
 
   // Scan suppressions list and find newly loaded and unloaded libraries.
-  MemoryMappingLayout proc_maps(/*cache_enabled*/false);
-  InternalScopedString module(kMaxPathLength);
+  ListOfModules modules;
+  modules.init();
   for (uptr i = 0; i < count_; i++) {
     Lib *lib = &libs_[i];
     bool loaded = false;
-    proc_maps.Reset();
-    uptr b, e, off, prot;
-    while (proc_maps.Next(&b, &e, &off, module.data(), module.size(), &prot)) {
-      if ((prot & MemoryMappingLayout::kProtectionExecute) == 0)
-        continue;
-      if (TemplateMatch(lib->templ, module.data()) ||
-          (lib->real_name &&
-          internal_strcmp(lib->real_name, module.data()) == 0)) {
+    for (const auto &mod : modules) {
+      for (const auto &range : mod.ranges()) {
+        if (!range.executable)
+          continue;
+        if (!TemplateMatch(lib->templ, mod.full_name()) &&
+            !(lib->real_name &&
+            internal_strcmp(lib->real_name, mod.full_name()) == 0))
+          continue;
         if (loaded) {
           Report("%s: called_from_lib suppression '%s' is matched against"
                  " 2 libraries: '%s' and '%s'\n",
-                 SanitizerToolName, lib->templ, lib->name, module.data());
+                 SanitizerToolName, lib->templ, lib->name, mod.full_name());
           Die();
         }
         loaded = true;
@@ -73,13 +73,16 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
         VReport(1,
                 "Matched called_from_lib suppression '%s' against library"
                 " '%s'\n",
-                lib->templ, module.data());
+                lib->templ, mod.full_name());
         lib->loaded = true;
-        lib->name = internal_strdup(module.data());
-        const uptr idx = atomic_load(&loaded_count_, memory_order_relaxed);
-        code_ranges_[idx].begin = b;
-        code_ranges_[idx].end = e;
-        atomic_store(&loaded_count_, idx + 1, memory_order_release);
+        lib->name = internal_strdup(mod.full_name());
+        const uptr idx =
+            atomic_load(&ignored_ranges_count_, memory_order_relaxed);
+        CHECK_LT(idx, kMaxLibs);
+        ignored_code_ranges_[idx].begin = range.beg;
+        ignored_code_ranges_[idx].end = range.end;
+        atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release);
+        break;
       }
     }
     if (lib->loaded && !loaded) {
@@ -89,6 +92,29 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
       Die();
     }
   }
+
+  // Track instrumented ranges.
+  if (track_instrumented_libs_) {
+    for (const auto &mod : modules) {
+      if (!mod.instrumented())
+        continue;
+      for (const auto &range : mod.ranges()) {
+        if (!range.executable)
+          continue;
+        if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
+          continue;
+        VReport(1, "Adding instrumented range %p-%p from library '%s'\n",
+                range.beg, range.end, mod.full_name());
+        const uptr idx =
+            atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
+        CHECK_LT(idx, kMaxLibs);
+        instrumented_code_ranges_[idx].begin = range.beg;
+        instrumented_code_ranges_[idx].end = range.end;
+        atomic_store(&instrumented_ranges_count_, idx + 1,
+                     memory_order_release);
+      }
+    }
+  }
 }
 
 void LibIgnore::OnLibraryUnloaded() {
@@ -97,4 +123,5 @@ void LibIgnore::OnLibraryUnloaded() {
 
 } // namespace __sanitizer
 
-#endif // #if SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC ||
+        // SANITIZER_NETBSD
diff --git a/libsanitizer/sanitizer_common/sanitizer_libignore.h b/libsanitizer/sanitizer_common/sanitizer_libignore.h
index 84419d14fed..e7627ee0256 100644
--- a/libsanitizer/sanitizer_common/sanitizer_libignore.h
+++ b/libsanitizer/sanitizer_common/sanitizer_libignore.h
@@ -28,6 +28,9 @@ class LibIgnore {
 
   // Must be called during initialization.
   void AddIgnoredLibrary(const char *name_templ);
+  void IgnoreNoninstrumentedModules(bool enable) {
+    track_instrumented_libs_ = enable;
+  }
 
   // Must be called after a new dynamic library is loaded.
   void OnLibraryLoaded(const char *name);
@@ -35,8 +38,14 @@ class LibIgnore {
   // Must be called after a dynamic library is unloaded.
   void OnLibraryUnloaded();
 
-  // Checks whether the provided PC belongs to one of the ignored libraries.
-  bool IsIgnored(uptr pc) const;
+  // Checks whether the provided PC belongs to one of the ignored libraries or
+  // the PC should be ignored because it belongs to an non-instrumented module
+  // (when ignore_noninstrumented_modules=1). Also returns true via
+  // "pc_in_ignored_lib" if the PC is in an ignored library, false otherwise.
+  bool IsIgnored(uptr pc, bool *pc_in_ignored_lib) const;
+
+  // Checks whether the provided PC belongs to an instrumented module.
+  bool IsPcInstrumented(uptr pc) const;
 
  private:
   struct Lib {
@@ -51,26 +60,48 @@ class LibIgnore {
     uptr end;
   };
 
+  inline bool IsInRange(uptr pc, const LibCodeRange &range) const {
+    return (pc >= range.begin && pc < range.end);
+  }
+
   static const uptr kMaxLibs = 128;
 
   // Hot part:
-  atomic_uintptr_t loaded_count_;
-  LibCodeRange code_ranges_[kMaxLibs];
+  atomic_uintptr_t ignored_ranges_count_;
+  LibCodeRange ignored_code_ranges_[kMaxLibs];
+
+  atomic_uintptr_t instrumented_ranges_count_;
+  LibCodeRange instrumented_code_ranges_[kMaxLibs];
 
   // Cold part:
   BlockingMutex mutex_;
   uptr count_;
   Lib libs_[kMaxLibs];
+  bool track_instrumented_libs_;
 
   // Disallow copying of LibIgnore objects.
   LibIgnore(const LibIgnore&);  // not implemented
   void operator = (const LibIgnore&);  // not implemented
 };
 
-inline bool LibIgnore::IsIgnored(uptr pc) const {
-  const uptr n = atomic_load(&loaded_count_, memory_order_acquire);
+inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
+  const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire);
+  for (uptr i = 0; i < n; i++) {
+    if (IsInRange(pc, ignored_code_ranges_[i])) {
+      *pc_in_ignored_lib = true;
+      return true;
+    }
+  }
+  *pc_in_ignored_lib = false;
+  if (track_instrumented_libs_ && !IsPcInstrumented(pc))
+    return true;
+  return false;
+}
+
+inline bool LibIgnore::IsPcInstrumented(uptr pc) const {
+  const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire);
   for (uptr i = 0; i < n; i++) {
-    if (pc >= code_ranges_[i].begin && pc < code_ranges_[i].end)
+    if (IsInRange(pc, instrumented_code_ranges_[i]))
       return true;
   }
   return false;
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.cc b/libsanitizer/sanitizer_common/sanitizer_linux.cc
index 5b6f18602e7..2826cc89e20 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_linux.cc
@@ -12,7 +12,7 @@
 
 #include "sanitizer_platform.h"
 
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 
 #include "sanitizer_common.h"
 #include "sanitizer_flags.h"
@@ -25,10 +25,14 @@
 #include "sanitizer_stacktrace.h"
 #include "sanitizer_symbolizer.h"
 
-#if !SANITIZER_FREEBSD
+#if SANITIZER_LINUX
 #include <asm/param.h>
 #endif
 
+#if SANITIZER_NETBSD
+#include <lwp.h>
+#endif
+
 // For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat'
 // format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To
 // access stat from asm/stat.h, without conflicting with definition in
@@ -57,11 +61,17 @@
 #include <ucontext.h>
 #include <unistd.h>
 
+#if SANITIZER_LINUX
+#include <sys/utsname.h>
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#include <sys/personality.h>
+#endif
+
 #if SANITIZER_FREEBSD
 #include <sys/exec.h>
 #include <sys/sysctl.h>
-#include <vm/vm_param.h>
-#include <vm/pmap.h>
 #include <machine/atomic.h>
 extern "C" {
 // <sys/umtx.h> must be included after <errno.h> and <sys/types.h> on
@@ -71,10 +81,30 @@ extern "C" {
 extern char **environ;  // provided by crt1
 #endif  // SANITIZER_FREEBSD
 
+#if SANITIZER_NETBSD
+#include <limits.h>  // For NAME_MAX
+#include <sys/sysctl.h>
+extern char **environ;  // provided by crt1
+#endif                  // SANITIZER_NETBSD
+
 #if !SANITIZER_ANDROID
 #include <sys/signal.h>
 #endif
 
+#ifndef __GLIBC_PREREQ
+#define __GLIBC_PREREQ(x, y) 0
+#endif
+
+#if SANITIZER_LINUX && __GLIBC_PREREQ(2, 16)
+# define SANITIZER_USE_GETAUXVAL 1
+#else
+# define SANITIZER_USE_GETAUXVAL 0
+#endif
+
+#if SANITIZER_USE_GETAUXVAL
+#include <sys/auxv.h>
+#endif
+
 #if SANITIZER_LINUX
 // <linux/time.h>
 struct kernel_timeval {
@@ -103,6 +133,15 @@ extern void internal_sigreturn();
 }
 #endif
 
+#if SANITIZER_LINUX && defined(__NR_getrandom)
+# if !defined(GRND_NONBLOCK)
+#  define GRND_NONBLOCK 1
+# endif
+# define SANITIZER_USE_GETRANDOM 1
+#else
+# define SANITIZER_USE_GETRANDOM 0
+#endif  // SANITIZER_LINUX && defined(__NR_getrandom)
+
 namespace __sanitizer {
 
 #if SANITIZER_LINUX && defined(__x86_64__)
@@ -117,7 +156,10 @@ namespace __sanitizer {
 #if !SANITIZER_S390
 uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
                    OFF_T offset) {
-#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
+#if SANITIZER_NETBSD
+  return internal_syscall_ptr(SYSCALL(mmap), addr, length, prot, flags, fd,
+                              (long)0, offset);
+#elif SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
   return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,
                           offset);
 #else
@@ -160,26 +202,38 @@ uptr internal_open(const char *filename, int flags, u32 mode) {
 
 uptr internal_read(fd_t fd, void *buf, uptr count) {
   sptr res;
+#if SANITIZER_NETBSD
+  HANDLE_EINTR(res, internal_syscall_ptr(SYSCALL(read), fd, buf, count));
+#else
   HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(read), fd, (uptr)buf,
                count));
+#endif
   return res;
 }
 
 uptr internal_write(fd_t fd, const void *buf, uptr count) {
   sptr res;
+#if SANITIZER_NETBSD
+  HANDLE_EINTR(res, internal_syscall_ptr(SYSCALL(write), fd, buf, count));
+#else
   HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(write), fd, (uptr)buf,
                count));
+#endif
   return res;
 }
 
 uptr internal_ftruncate(fd_t fd, uptr size) {
   sptr res;
+#if SANITIZER_NETBSD
+  HANDLE_EINTR(res, internal_syscall(SYSCALL(ftruncate), fd, 0, (s64)size));
+#else
   HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd,
                (OFF_T)size));
+#endif
   return res;
 }
 
-#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && !SANITIZER_FREEBSD
+#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX
 static void stat64_to_stat(struct stat64 *in, struct stat *out) {
   internal_memset(out, 0, sizeof(*out));
   out->st_dev = in->st_dev;
@@ -195,11 +249,25 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) {
   out->st_atime = in->st_atime;
   out->st_mtime = in->st_mtime;
   out->st_ctime = in->st_ctime;
-  out->st_ino = in->st_ino;
 }
 #endif
 
 #if defined(__mips64)
+// Undefine compatibility macros from <sys/stat.h>
+// so that they would not clash with the kernel_stat
+// st_[a|m|c]time fields
+#undef st_atime
+#undef st_mtime
+#undef st_ctime
+#if defined(SANITIZER_ANDROID)
+// Bionic sys/stat.h defines additional macros
+// for compatibility with the old NDKs and
+// they clash with the kernel_stat structure
+// st_[a|m|c]time_nsec fields.
+#undef st_atime_nsec
+#undef st_mtime_nsec
+#undef st_ctime_nsec
+#endif
 static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
   internal_memset(out, 0, sizeof(*out));
   out->st_dev = in->st_dev;
@@ -212,16 +280,30 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
   out->st_size = in->st_size;
   out->st_blksize = in->st_blksize;
   out->st_blocks = in->st_blocks;
-  out->st_atime = in->st_atime_nsec;
-  out->st_mtime = in->st_mtime_nsec;
-  out->st_ctime = in->st_ctime_nsec;
-  out->st_ino = in->st_ino;
+#if defined(__USE_MISC)     || \
+    defined(__USE_XOPEN2K8) || \
+    defined(SANITIZER_ANDROID)
+  out->st_atim.tv_sec = in->st_atime;
+  out->st_atim.tv_nsec = in->st_atime_nsec;
+  out->st_mtim.tv_sec = in->st_mtime;
+  out->st_mtim.tv_nsec = in->st_mtime_nsec;
+  out->st_ctim.tv_sec = in->st_ctime;
+  out->st_ctim.tv_nsec = in->st_ctime_nsec;
+#else
+  out->st_atime = in->st_atime;
+  out->st_atimensec = in->st_atime_nsec;
+  out->st_mtime = in->st_mtime;
+  out->st_mtimensec = in->st_mtime_nsec;
+  out->st_ctime = in->st_ctime;
+  out->st_atimensec = in->st_ctime_nsec;
+#endif
 }
 #endif
 
 uptr internal_stat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD
-  return internal_syscall(SYSCALL(stat), path, buf);
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+  return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path,
+                          (uptr)buf, 0);
 #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
   return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path,
                           (uptr)buf, 0);
@@ -244,8 +326,11 @@ uptr internal_stat(const char *path, void *buf) {
 }
 
 uptr internal_lstat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD
+#if SANITIZER_NETBSD
   return internal_syscall(SYSCALL(lstat), path, buf);
+#elif SANITIZER_FREEBSD
+  return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path,
+                          (uptr)buf, AT_SYMLINK_NOFOLLOW);
 #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
   return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path,
                          (uptr)buf, AT_SYMLINK_NOFOLLOW);
@@ -268,7 +353,7 @@ uptr internal_lstat(const char *path, void *buf) {
 }
 
 uptr internal_fstat(fd_t fd, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
+#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS || SANITIZER_NETBSD
 # if SANITIZER_MIPS64
   // For mips64, fstat syscall fills buffer in the format of kernel_stat
   struct kernel_stat kbuf;
@@ -302,7 +387,9 @@ uptr internal_dup2(int oldfd, int newfd) {
 }
 
 uptr internal_readlink(const char *path, char *buf, uptr bufsize) {
-#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+#if SANITIZER_NETBSD
+  return internal_syscall_ptr(SYSCALL(readlink), path, buf, bufsize);
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
   return internal_syscall(SYSCALL(readlinkat), AT_FDCWD,
                           (uptr)path, (uptr)buf, bufsize);
 #else
@@ -332,7 +419,7 @@ uptr internal_sched_yield() {
 }
 
 void internal__exit(int exitcode) {
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
   internal_syscall(SYSCALL(exit), exitcode);
 #else
   internal_syscall(SYSCALL(exit_group), exitcode);
@@ -368,22 +455,28 @@ bool FileExists(const char *filename) {
   return S_ISREG(st.st_mode);
 }
 
-uptr GetTid() {
+tid_t GetTid() {
 #if SANITIZER_FREEBSD
   return (uptr)pthread_self();
+#elif SANITIZER_NETBSD
+  return _lwp_self();
 #else
   return internal_syscall(SYSCALL(gettid));
 #endif
 }
 
 u64 NanoTime() {
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
   timeval tv;
 #else
   kernel_timeval tv;
 #endif
   internal_memset(&tv, 0, sizeof(tv));
+#if SANITIZER_NETBSD
+  internal_syscall_ptr(SYSCALL(gettimeofday), &tv, NULL);
+#else
   internal_syscall(SYSCALL(gettimeofday), (uptr)&tv, 0);
+#endif
   return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
 }
 
@@ -391,7 +484,7 @@ u64 NanoTime() {
 // 'environ' array (on FreeBSD) and does not use libc. This function should be
 // called first inside __asan_init.
 const char *GetEnv(const char *name) {
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
   if (::environ != 0) {
     uptr NameLen = internal_strlen(name);
     for (char **Env = ::environ; *Env != 0; Env++) {
@@ -525,6 +618,8 @@ void BlockingMutex::Lock() {
   while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
 #if SANITIZER_FREEBSD
     _umtx_op(m, UMTX_OP_WAIT_UINT, MtxSleeping, 0, 0);
+#elif SANITIZER_NETBSD
+    sched_yield(); /* No userspace futex-like synchromization */
 #else
     internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT, MtxSleeping, 0, 0, 0);
 #endif
@@ -533,11 +628,13 @@ void BlockingMutex::Lock() {
 
 void BlockingMutex::Unlock() {
   atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
-  u32 v = atomic_exchange(m, MtxUnlocked, memory_order_relaxed);
+  u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
   CHECK_NE(v, MtxUnlocked);
   if (v == MtxSleeping) {
 #if SANITIZER_FREEBSD
     _umtx_op(m, UMTX_OP_WAKE, 1, 0, 0);
+#elif SANITIZER_NETBSD
+                   /* No userspace futex-like synchromization */
 #else
     internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE, 1, 0, 0, 0);
 #endif
@@ -553,6 +650,17 @@ void BlockingMutex::CheckLocked() {
 // The actual size of this structure is specified by d_reclen.
 // Note that getdents64 uses a different structure format. We only provide the
 // 32-bit syscall here.
+#if SANITIZER_NETBSD
+// struct dirent is different for Linux and us. At this moment, we use only
+// d_fileno (Linux call this d_ino), d_reclen, and d_name.
+struct linux_dirent {
+  u64 d_ino;  // d_fileno
+  u16 d_reclen;
+  u16 d_namlen;  // not used
+  u8 d_type;     // not used
+  char d_name[NAME_MAX + 1];
+};
+#else
 struct linux_dirent {
 #if SANITIZER_X32 || defined(__aarch64__)
   u64 d_ino;
@@ -567,16 +675,34 @@ struct linux_dirent {
 #endif
   char               d_name[256];
 };
+#endif
 
 // Syscall wrappers.
 uptr internal_ptrace(int request, int pid, void *addr, void *data) {
+#if SANITIZER_NETBSD
+  // XXX We need additional work for ptrace:
+  //   - for request, we use PT_FOO whereas Linux uses PTRACE_FOO
+  //   - data is int for us, but void * for Linux
+  //   - Linux sometimes uses data in the case where we use addr instead
+  // At this moment, this function is used only within
+  // "#if SANITIZER_LINUX && defined(__x86_64__)" block in
+  // sanitizer_stoptheworld_linux_libcdep.cc.
+  return internal_syscall_ptr(SYSCALL(ptrace), request, pid, (uptr)addr,
+                              (uptr)data);
+#else
   return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr,
                           (uptr)data);
+#endif
 }
 
 uptr internal_waitpid(int pid, int *status, int options) {
+#if SANITIZER_NETBSD
+  return internal_syscall(SYSCALL(wait4), pid, status, options,
+                          NULL /* rusage */);
+#else
   return internal_syscall(SYSCALL(wait4), pid, (uptr)status, options,
                           0 /* rusage */);
+#endif
 }
 
 uptr internal_getpid() {
@@ -588,7 +714,11 @@ uptr internal_getppid() {
 }
 
 uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
-#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+#if SANITIZER_NETBSD
+  return internal_syscall(SYSCALL(getdents), fd, dirp, (uptr)count);
+#elif SANITIZER_FREEBSD
+  return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
   return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);
 #else
   return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count);
@@ -596,7 +726,11 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
 }
 
 uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
+#if SANITIZER_NETBSD
+  return internal_syscall64(SYSCALL(lseek), fd, 0, offset, whence);
+#else
   return internal_syscall(SYSCALL(lseek), fd, offset, whence);
+#endif
 }
 
 #if SANITIZER_LINUX
@@ -687,7 +821,7 @@ int internal_sigaction_syscall(int signum, const void *act, void *oldact) {
 
 uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
     __sanitizer_sigset_t *oldset) {
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
   return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);
 #else
   __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
@@ -796,20 +930,94 @@ bool ThreadLister::GetDirectoryEntries() {
   return true;
 }
 
+#if SANITIZER_WORDSIZE == 32
+// Take care of unusable kernel area in top gigabyte.
+static uptr GetKernelAreaSize() {
+#if SANITIZER_LINUX && !SANITIZER_X32
+  const uptr gbyte = 1UL << 30;
+
+  // Firstly check if there are writable segments
+  // mapped to top gigabyte (e.g. stack).
+  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0;
+  }
+
+#if !SANITIZER_ANDROID
+  // Even if nothing is mapped, top Gb may still be accessible
+  // if we are running on 64-bit kernel.
+  // Uname may report misleading results if personality type
+  // is modified (e.g. under schroot) so check this as well.
+  struct utsname uname_info;
+  int pers = personality(0xffffffffUL);
+  if (!(pers & PER_MASK)
+      && uname(&uname_info) == 0
+      && internal_strstr(uname_info.machine, "64"))
+    return 0;
+#endif  // SANITIZER_ANDROID
+
+  // Top gigabyte is reserved for kernel.
+  return gbyte;
+#else
+  return 0;
+#endif  // SANITIZER_LINUX && !SANITIZER_X32
+}
+#endif  // SANITIZER_WORDSIZE == 32
+
+uptr GetMaxVirtualAddress() {
+#if SANITIZER_NETBSD && defined(__x86_64__)
+  return 0x7f7ffffff000ULL;  // (0x00007f8000000000 - PAGE_SIZE)
+#elif SANITIZER_WORDSIZE == 64
+# if defined(__powerpc64__) || defined(__aarch64__)
+  // On PowerPC64 we have two different address space layouts: 44- and 46-bit.
+  // We somehow need to figure out which one we are using now and choose
+  // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
+  // Note that with 'ulimit -s unlimited' the stack is moved away from the top
+  // of the address space, so simply checking the stack address is not enough.
+  // This should (does) work for both PowerPC64 Endian modes.
+  // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
+  return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
+# elif defined(__mips64)
+  return (1ULL << 40) - 1;  // 0x000000ffffffffffUL;
+# elif defined(__s390x__)
+  return (1ULL << 53) - 1;  // 0x001fffffffffffffUL;
+# else
+  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
+# endif
+#else  // SANITIZER_WORDSIZE == 32
+# if defined(__s390__)
+  return (1ULL << 31) - 1;  // 0x7fffffff;
+# else
+  uptr res = (1ULL << 32) - 1;  // 0xffffffff;
+  if (!common_flags()->full_address_space)
+    res -= GetKernelAreaSize();
+  CHECK_LT(reinterpret_cast<uptr>(&res), res);
+  return res;
+# endif
+#endif  // SANITIZER_WORDSIZE
+}
+
 uptr GetPageSize() {
 // Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
 #if SANITIZER_ANDROID
   return 4096;
 #elif SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__))
   return EXEC_PAGESIZE;
+#elif SANITIZER_USE_GETAUXVAL
+  return getauxval(AT_PAGESZ);
 #else
   return sysconf(_SC_PAGESIZE);  // EXEC_PAGESIZE may not be trustworthy.
 #endif
 }
 
 uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
 #if SANITIZER_FREEBSD
-  const int Mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };
+  const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
+#else
+  const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME};
+#endif
   const char *default_module_name = "kern.proc.pathname";
   size_t Size = buf_len;
   bool IsErr = (sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0);
@@ -1094,36 +1302,50 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
 uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
                    int *parent_tidptr, void *newtls, int *child_tidptr) {
   long long res;
-/* Stack frame offsets.  */
-#if _CALL_ELF != 2
-#define FRAME_MIN_SIZE         112
-#define FRAME_TOC_SAVE         40
+// Stack frame structure.
+#if SANITIZER_PPC64V1
+//   Back chain == 0        (SP + 112)
+// Frame (112 bytes):
+//   Parameter save area    (SP + 48), 8 doublewords
+//   TOC save area          (SP + 40)
+//   Link editor doubleword (SP + 32)
+//   Compiler doubleword    (SP + 24)
+//   LR save area           (SP + 16)
+//   CR save area           (SP + 8)
+//   Back chain             (SP + 0)
+# define FRAME_SIZE 112
+# define FRAME_TOC_SAVE_OFFSET 40
+#elif SANITIZER_PPC64V2
+//   Back chain == 0        (SP + 32)
+// Frame (32 bytes):
+//   TOC save area          (SP + 24)
+//   LR save area           (SP + 16)
+//   CR save area           (SP + 8)
+//   Back chain             (SP + 0)
+# define FRAME_SIZE 32
+# define FRAME_TOC_SAVE_OFFSET 24
 #else
-#define FRAME_MIN_SIZE         32
-#define FRAME_TOC_SAVE         24
+# error "Unsupported PPC64 ABI"
 #endif
   if (!fn || !child_stack)
     return -EINVAL;
   CHECK_EQ(0, (uptr)child_stack % 16);
-  child_stack = (char *)child_stack - 2 * sizeof(unsigned long long);
-  ((unsigned long long *)child_stack)[0] = (uptr)fn;
-  ((unsigned long long *)child_stack)[1] = (uptr)arg;
 
   register int (*__fn)(void *) __asm__("r3") = fn;
   register void *__cstack      __asm__("r4") = child_stack;
   register int __flags         __asm__("r5") = flags;
-  register void * __arg        __asm__("r6") = arg;
-  register int * __ptidptr     __asm__("r7") = parent_tidptr;
-  register void * __newtls     __asm__("r8") = newtls;
-  register int * __ctidptr     __asm__("r9") = child_tidptr;
+  register void *__arg         __asm__("r6") = arg;
+  register int *__ptidptr      __asm__("r7") = parent_tidptr;
+  register void *__newtls      __asm__("r8") = newtls;
+  register int *__ctidptr      __asm__("r9") = child_tidptr;
 
  __asm__ __volatile__(
-           /* fn, arg, child_stack are saved acrVoss the syscall */
+           /* fn and arg are saved across the syscall */
            "mr 28, %5\n\t"
-           "mr 29, %6\n\t"
            "mr 27, %8\n\t"
 
            /* syscall
+             r0 == __NR_clone
              r3 == flags
              r4 == child_stack
              r5 == parent_tidptr
@@ -1141,15 +1363,21 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
            "crandc cr1*4+eq, cr1*4+eq, cr0*4+so\n\t"
            "bne-   cr1, 1f\n\t"
 
+           /* Set up stack frame */
+           "li    29, 0\n\t"
+           "stdu  29, -8(1)\n\t"
+           "stdu  1, -%12(1)\n\t"
            /* Do the function call */
            "std   2, %13(1)\n\t"
-#if _CALL_ELF != 2
+#if SANITIZER_PPC64V1
            "ld    0, 0(28)\n\t"
            "ld    2, 8(28)\n\t"
            "mtctr 0\n\t"
-#else
+#elif SANITIZER_PPC64V2
            "mr    12, 28\n\t"
            "mtctr 12\n\t"
+#else
+# error "Unsupported PPC64 ABI"
 #endif
            "mr    3, 27\n\t"
            "bctrl\n\t"
@@ -1163,13 +1391,151 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
            "1:\n\t"
            "mr %0, 3\n\t"
              : "=r" (res)
-             : "0" (-1), "i" (EINVAL),
-               "i" (__NR_clone), "i" (__NR_exit),
-               "r" (__fn), "r" (__cstack), "r" (__flags),
-               "r" (__arg), "r" (__ptidptr), "r" (__newtls),
-               "r" (__ctidptr), "i" (FRAME_MIN_SIZE), "i" (FRAME_TOC_SAVE)
-             : "cr0", "cr1", "memory", "ctr",
-               "r0", "r29", "r27", "r28");
+             : "0" (-1),
+               "i" (EINVAL),
+               "i" (__NR_clone),
+               "i" (__NR_exit),
+               "r" (__fn),
+               "r" (__cstack),
+               "r" (__flags),
+               "r" (__arg),
+               "r" (__ptidptr),
+               "r" (__newtls),
+               "r" (__ctidptr),
+               "i" (FRAME_SIZE),
+               "i" (FRAME_TOC_SAVE_OFFSET)
+             : "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
+  return res;
+}
+#elif defined(__i386__) && SANITIZER_LINUX
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  int res;
+  if (!fn || !child_stack)
+    return -EINVAL;
+  CHECK_EQ(0, (uptr)child_stack % 16);
+  child_stack = (char *)child_stack - 7 * sizeof(unsigned int);
+  ((unsigned int *)child_stack)[0] = (uptr)flags;
+  ((unsigned int *)child_stack)[1] = (uptr)0;
+  ((unsigned int *)child_stack)[2] = (uptr)fn;
+  ((unsigned int *)child_stack)[3] = (uptr)arg;
+  __asm__ __volatile__(
+                       /* %eax = syscall(%eax = SYSCALL(clone),
+                        *                %ebx = flags,
+                        *                %ecx = child_stack,
+                        *                %edx = parent_tidptr,
+                        *                %esi  = new_tls,
+                        *                %edi = child_tidptr)
+                        */
+
+                        /* Obtain flags */
+                        "movl    (%%ecx), %%ebx\n"
+                        /* Do the system call */
+                        "pushl   %%ebx\n"
+                        "pushl   %%esi\n"
+                        "pushl   %%edi\n"
+                        /* Remember the flag value.  */
+                        "movl    %%ebx, (%%ecx)\n"
+                        "int     $0x80\n"
+                        "popl    %%edi\n"
+                        "popl    %%esi\n"
+                        "popl    %%ebx\n"
+
+                        /* if (%eax != 0)
+                         *   return;
+                         */
+
+                        "test    %%eax,%%eax\n"
+                        "jnz    1f\n"
+
+                        /* terminate the stack frame */
+                        "xorl   %%ebp,%%ebp\n"
+                        /* Call FN. */
+                        "call    *%%ebx\n"
+#ifdef PIC
+                        "call    here\n"
+                        "here:\n"
+                        "popl    %%ebx\n"
+                        "addl    $_GLOBAL_OFFSET_TABLE_+[.-here], %%ebx\n"
+#endif
+                        /* Call exit */
+                        "movl    %%eax, %%ebx\n"
+                        "movl    %2, %%eax\n"
+                        "int     $0x80\n"
+                        "1:\n"
+                       : "=a" (res)
+                       : "a"(SYSCALL(clone)), "i"(SYSCALL(exit)),
+                         "c"(child_stack),
+                         "d"(parent_tidptr),
+                         "S"(newtls),
+                         "D"(child_tidptr)
+                       : "memory");
+  return res;
+}
+#elif defined(__arm__) && SANITIZER_LINUX
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  unsigned int res;
+  if (!fn || !child_stack)
+    return -EINVAL;
+  child_stack = (char *)child_stack - 2 * sizeof(unsigned int);
+  ((unsigned int *)child_stack)[0] = (uptr)fn;
+  ((unsigned int *)child_stack)[1] = (uptr)arg;
+  register int r0 __asm__("r0") = flags;
+  register void *r1 __asm__("r1") = child_stack;
+  register int *r2 __asm__("r2") = parent_tidptr;
+  register void *r3 __asm__("r3") = newtls;
+  register int *r4 __asm__("r4") = child_tidptr;
+  register int r7 __asm__("r7") = __NR_clone;
+
+#if __ARM_ARCH > 4 || defined (__ARM_ARCH_4T__)
+# define ARCH_HAS_BX
+#endif
+#if __ARM_ARCH > 4
+# define ARCH_HAS_BLX
+#endif
+
+#ifdef ARCH_HAS_BX
+# ifdef ARCH_HAS_BLX
+#  define BLX(R) "blx "  #R "\n"
+# else
+#  define BLX(R) "mov lr, pc; bx " #R "\n"
+# endif
+#else
+# define BLX(R)  "mov lr, pc; mov pc," #R "\n"
+#endif
+
+  __asm__ __volatile__(
+                       /* %r0 = syscall(%r7 = SYSCALL(clone),
+                        *               %r0 = flags,
+                        *               %r1 = child_stack,
+                        *               %r2 = parent_tidptr,
+                        *               %r3  = new_tls,
+                        *               %r4 = child_tidptr)
+                        */
+
+                       /* Do the system call */
+                       "swi 0x0\n"
+
+                       /* if (%r0 != 0)
+                        *   return %r0;
+                        */
+                       "cmp r0, #0\n"
+                       "bne 1f\n"
+
+                       /* In the child, now. Call "fn(arg)". */
+                       "ldr r0, [sp, #4]\n"
+                       "ldr ip, [sp], #8\n"
+                       BLX(ip)
+                       /* Call _exit(%r0). */
+                       "mov r7, %7\n"
+                       "swi 0x0\n"
+                       "1:\n"
+                       "mov %0, r0\n"
+                       : "=r"(res)
+                       : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r7),
+                         "i"(__NR_exit)
+                       : "memory");
   return res;
 }
 #endif  // defined(__x86_64__) && SANITIZER_LINUX
@@ -1217,14 +1583,27 @@ AndroidApiLevel AndroidGetApiLevel() {
 
 #endif
 
-bool IsHandledDeadlySignal(int signum) {
-  if (common_flags()->handle_abort && signum == SIGABRT)
-    return true;
-  if (common_flags()->handle_sigill && signum == SIGILL)
-    return true;
-  if (common_flags()->handle_sigfpe && signum == SIGFPE)
-    return true;
-  return (signum == SIGSEGV || signum == SIGBUS) && common_flags()->handle_segv;
+static HandleSignalMode GetHandleSignalModeImpl(int signum) {
+  switch (signum) {
+    case SIGABRT:
+      return common_flags()->handle_abort;
+    case SIGILL:
+      return common_flags()->handle_sigill;
+    case SIGFPE:
+      return common_flags()->handle_sigfpe;
+    case SIGSEGV:
+      return common_flags()->handle_segv;
+    case SIGBUS:
+      return common_flags()->handle_sigbus;
+  }
+  return kHandleSignalNo;
+}
+
+HandleSignalMode GetHandleSignalMode(int signum) {
+  HandleSignalMode result = GetHandleSignalModeImpl(signum);
+  if (result == kHandleSignalYes && !common_flags()->allow_user_segv_handler)
+    return kHandleSignalExclusive;
+  return result;
 }
 
 #if !SANITIZER_GO
@@ -1276,12 +1655,14 @@ static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) {
 }
 #endif
 
-SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
   ucontext_t *ucontext = (ucontext_t *)context;
 #if defined(__x86_64__) || defined(__i386__)
   static const uptr PF_WRITE = 1U << 1;
 #if SANITIZER_FREEBSD
   uptr err = ucontext->uc_mcontext.mc_err;
+#elif SANITIZER_NETBSD
+  uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
 #else
   uptr err = ucontext->uc_mcontext.gregs[REG_ERR];
 #endif
@@ -1301,7 +1682,11 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
 #endif
 }
 
-void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
+void SignalContext::DumpAllRegisters(void *context) {
+  // FIXME: Implement this.
+}
+
+static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
 #if defined(__arm__)
   ucontext_t *ucontext = (ucontext_t*)context;
   *pc = ucontext->uc_mcontext.arm_pc;
@@ -1324,6 +1709,11 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
   *pc = ucontext->uc_mcontext.mc_rip;
   *bp = ucontext->uc_mcontext.mc_rbp;
   *sp = ucontext->uc_mcontext.mc_rsp;
+#elif SANITIZER_NETBSD
+  ucontext_t *ucontext = (ucontext_t *)context;
+  *pc = ucontext->uc_mcontext.__gregs[_REG_RIP];
+  *bp = ucontext->uc_mcontext.__gregs[_REG_RBP];
+  *sp = ucontext->uc_mcontext.__gregs[_REG_RSP];
 # else
   ucontext_t *ucontext = (ucontext_t*)context;
   *pc = ucontext->uc_mcontext.gregs[REG_RIP];
@@ -1336,6 +1726,11 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
   *pc = ucontext->uc_mcontext.mc_eip;
   *bp = ucontext->uc_mcontext.mc_ebp;
   *sp = ucontext->uc_mcontext.mc_esp;
+#elif SANITIZER_NETBSD
+  ucontext_t *ucontext = (ucontext_t *)context;
+  *pc = ucontext->uc_mcontext.__gregs[_REG_EIP];
+  *bp = ucontext->uc_mcontext.__gregs[_REG_EBP];
+  *sp = ucontext->uc_mcontext.__gregs[_REG_ESP];
 # else
   ucontext_t *ucontext = (ucontext_t*)context;
   *pc = ucontext->uc_mcontext.gregs[REG_EIP];
@@ -1382,15 +1777,63 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
 #endif
 }
 
+void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
+
 void MaybeReexec() {
   // No need to re-exec on Linux.
 }
 
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) {
+void PrintModuleMap() { }
+
+void CheckNoDeepBind(const char *filename, int flag) {
+#ifdef RTLD_DEEPBIND
+  if (flag & RTLD_DEEPBIND) {
+    Report(
+        "You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"
+        " which is incompatibe with sanitizer runtime "
+        "(see https://github.com/google/sanitizers/issues/611 for details"
+        "). If you want to run %s library under sanitizers please remove "
+        "RTLD_DEEPBIND from dlopen flags.\n",
+        filename, filename);
+    Die();
+  }
+#endif
+}
+
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found) {
   UNREACHABLE("FindAvailableMemoryRange is not available");
   return 0;
 }
 
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  if (!buffer || !length || length > 256)
+    return false;
+#if SANITIZER_USE_GETRANDOM
+  static atomic_uint8_t skip_getrandom_syscall;
+  if (!atomic_load_relaxed(&skip_getrandom_syscall)) {
+    // Up to 256 bytes, getrandom will not be interrupted.
+    uptr res = internal_syscall(SYSCALL(getrandom), buffer, length,
+                                blocking ? 0 : GRND_NONBLOCK);
+    int rverrno = 0;
+    if (internal_iserror(res, &rverrno) && rverrno == ENOSYS)
+      atomic_store_relaxed(&skip_getrandom_syscall, 1);
+    else if (res == length)
+      return true;
+  }
+#endif  // SANITIZER_USE_GETRANDOM
+  // Up to 256 bytes, a read off /dev/urandom will not be interrupted.
+  // blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom.
+  uptr fd = internal_open("/dev/urandom", O_RDONLY);
+  if (internal_iserror(fd))
+    return false;
+  uptr res = internal_read(fd, buffer, length);
+  if (internal_iserror(res))
+    return false;
+  internal_close(fd);
+  return true;
+}
+
 } // namespace __sanitizer
 
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.h b/libsanitizer/sanitizer_common/sanitizer_linux.h
index a42df576405..910703d8b29 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux.h
+++ b/libsanitizer/sanitizer_common/sanitizer_linux.h
@@ -12,11 +12,12 @@
 #define SANITIZER_LINUX_H
 
 #include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 #include "sanitizer_common.h"
 #include "sanitizer_internal_defs.h"
-#include "sanitizer_posix.h"
+#include "sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_posix.h"
 
 struct link_map;  // Opaque type returned by dlopen().
 
@@ -25,6 +26,19 @@ namespace __sanitizer {
 // the one in <dirent.h>, which is used by readdir().
 struct linux_dirent;
 
+struct ProcSelfMapsBuff {
+  char *data;
+  uptr mmaped_size;
+  uptr len;
+};
+
+struct MemoryMappingLayoutData {
+  ProcSelfMapsBuff proc_self_maps;
+  const char *current;
+};
+
+void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
+
 // Syscall wrappers.
 uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
 uptr internal_sigaltstack(const void* ss, void* oss);
@@ -44,7 +58,8 @@ int internal_sigaction_syscall(int signum, const void *act, void *oldact);
 #endif
 void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
 #if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \
-  || defined(__powerpc64__) || defined(__s390__)
+  || defined(__powerpc64__) || defined(__s390__) || defined(__i386__) \
+  || defined(__arm__)
 uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
                     int *parent_tidptr, void *newtls, int *child_tidptr);
 #endif
@@ -83,7 +98,47 @@ bool LibraryNameIs(const char *full_name, const char *base_name);
 
 // Call cb for each region mapped by map.
 void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr));
+
+#if SANITIZER_ANDROID
+
+#if defined(__aarch64__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mrs %0, tpidr_el0" : "=r"(__v)); __v; })
+#elif defined(__arm__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); __v; })
+#elif defined(__mips__)
+// On mips32r1, this goes via a kernel illegal instruction trap that's
+// optimized for v1.
+# define __get_tls() \
+    ({ register void** __v asm("v1"); \
+       __asm__(".set    push\n" \
+               ".set    mips32r2\n" \
+               "rdhwr   %0,$29\n" \
+               ".set    pop\n" : "=r"(__v)); \
+       __v; })
+#elif defined(__i386__)
+# define __get_tls() \
+    ({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })
+#elif defined(__x86_64__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mov %%fs:0, %0" : "=r"(__v)); __v; })
+#else
+#error "Unsupported architecture."
+#endif
+
+// The Android Bionic team has allocated a TLS slot for TSan starting with N,
+// given that Android currently doesn't support ELF TLS. It is used to store
+// Sanitizers thread specific data.
+static const int TLS_SLOT_TSAN = 8;
+
+ALWAYS_INLINE uptr *get_android_tls_ptr() {
+  return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_TSAN]);
+}
+
+#endif  // SANITIZER_ANDROID
+
 }  // namespace __sanitizer
 
-#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 #endif  // SANITIZER_LINUX_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc
index 63e70660cf3..b279cf3135f 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc
@@ -12,11 +12,12 @@
 
 #include "sanitizer_platform.h"
 
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_atomic.h"
 #include "sanitizer_common.h"
+#include "sanitizer_file.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_freebsd.h"
 #include "sanitizer_linux.h"
@@ -24,10 +25,7 @@
 #include "sanitizer_procmaps.h"
 #include "sanitizer_stacktrace.h"
 
-#if SANITIZER_ANDROID || SANITIZER_FREEBSD
 #include <dlfcn.h>  // for dlsym()
-#endif
-
 #include <link.h>
 #include <pthread.h>
 #include <signal.h>
@@ -82,28 +80,25 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
 
     // Find the mapping that contains a stack variable.
     MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-    uptr start, end, offset;
+    MemoryMappedSegment segment;
     uptr prev_end = 0;
-    while (proc_maps.Next(&start, &end, &offset, nullptr, 0,
-          /* protection */nullptr)) {
-      if ((uptr)&rl < end)
-        break;
-      prev_end = end;
+    while (proc_maps.Next(&segment)) {
+      if ((uptr)&rl < segment.end) break;
+      prev_end = segment.end;
     }
-    CHECK((uptr)&rl >= start && (uptr)&rl < end);
+    CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
 
     // Get stacksize from rlimit, but clip it so that it does not overlap
     // with other mappings.
     uptr stacksize = rl.rlim_cur;
-    if (stacksize > end - prev_end)
-      stacksize = end - prev_end;
+    if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;
     // When running with unlimited stack size, we still want to set some limit.
     // The unlimited stack size is caused by 'ulimit -s unlimited'.
     // Also, for some reason, GNU make spawns subprocesses with unlimited stack.
     if (stacksize > kMaxThreadStackSize)
       stacksize = kMaxThreadStackSize;
-    *stack_top = end;
-    *stack_bottom = end - stacksize;
+    *stack_top = segment.end;
+    *stack_bottom = segment.end - stacksize;
     return;
   }
   pthread_attr_t attr;
@@ -114,7 +109,6 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
   my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
   pthread_attr_destroy(&attr);
 
-  CHECK_LE(stacksize, kMaxThreadStackSize);  // Sanity check.
   *stack_top = (uptr)stackaddr + stacksize;
   *stack_bottom = (uptr)stackaddr;
 }
@@ -153,7 +147,8 @@ bool SanitizerGetThreadName(char *name, int max_len) {
 #endif
 }
 
-#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO
+#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \
+    !SANITIZER_NETBSD
 static uptr g_tls_size;
 
 #ifdef __i386__
@@ -181,11 +176,12 @@ void InitTlsSize() {
 }
 #else
 void InitTlsSize() { }
-#endif  // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO
+#endif  // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&
+        // !SANITIZER_NETBSD
 
 #if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) \
-    || defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__)) \
-    && SANITIZER_LINUX && !SANITIZER_ANDROID
+    || defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) \
+    || defined(__arm__)) && SANITIZER_LINUX && !SANITIZER_ANDROID
 // sizeof(struct pthread) from glibc.
 static atomic_uintptr_t kThreadDescriptorSize;
 
@@ -193,14 +189,14 @@ uptr ThreadDescriptorSize() {
   uptr val = atomic_load(&kThreadDescriptorSize, memory_order_relaxed);
   if (val)
     return val;
-#if defined(__x86_64__) || defined(__i386__)
+#if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
 #ifdef _CS_GNU_LIBC_VERSION
   char buf[64];
   uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
   if (len < sizeof(buf) && internal_strncmp(buf, "glibc 2.", 8) == 0) {
     char *end;
     int minor = internal_simple_strtoll(buf + 8, &end, 10);
-    if (end != buf + 8 && (*end == '\0' || *end == '.')) {
+    if (end != buf + 8 && (*end == '\0' || *end == '.' || *end == '-')) {
       int patch = 0;
       if (*end == '.')
         // strtoll will return 0 if no valid conversion could be performed
@@ -209,6 +205,9 @@ uptr ThreadDescriptorSize() {
       /* sizeof(struct pthread) values from various glibc versions.  */
       if (SANITIZER_X32)
         val = 1728;  // Assume only one particular version for x32.
+      // For ARM sizeof(struct pthread) changed in Glibc 2.23.
+      else if (SANITIZER_ARM)
+        val = minor <= 22 ? 1120 : 1216;
       else if (minor <= 3)
         val = FIRST_32_SECOND_64(1104, 1696);
       else if (minor == 4)
@@ -271,9 +270,7 @@ static uptr TlsPreTcbSize() {
 # endif
   const uptr kTlsAlign = 16;
   const uptr kTlsPreTcbSize =
-    (ThreadDescriptorSize() + kTcbHead + kTlsAlign - 1) & ~(kTlsAlign - 1);
-  InitTlsSize();
-  g_tls_size = (g_tls_size + kTlsPreTcbSize + kTlsAlign -1) & ~(kTlsAlign - 1);
+      RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
   return kTlsPreTcbSize;
 }
 #endif
@@ -296,7 +293,7 @@ uptr ThreadSelf() {
                 rdhwr %0,$29;\
                 .set pop" : "=r" (thread_pointer));
   descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize();
-# elif defined(__aarch64__)
+# elif defined(__aarch64__) || defined(__arm__)
   descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
                                       ThreadDescriptorSize();
 # elif defined(__s390__)
@@ -335,7 +332,9 @@ static void **ThreadSelfSegbase() {
 uptr ThreadSelf() {
   return (uptr)ThreadSelfSegbase()[2];
 }
-#endif  // SANITIZER_FREEBSD
+#elif SANITIZER_NETBSD
+uptr ThreadSelf() { return (uptr)pthread_self(); }
+#endif  // SANITIZER_NETBSD
 
 #if !SANITIZER_GO
 static void GetTls(uptr *addr, uptr *size) {
@@ -345,7 +344,8 @@ static void GetTls(uptr *addr, uptr *size) {
   *size = GetTlsSize();
   *addr -= *size;
   *addr += ThreadDescriptorSize();
-# elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__)
+# elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__) \
+    || defined(__arm__)
   *addr = ThreadSelf();
   *size = GetTlsSize();
 # else
@@ -365,7 +365,7 @@ static void GetTls(uptr *addr, uptr *size) {
     *addr = (uptr) dtv[2];
     *size = (*addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]);
   }
-#elif SANITIZER_ANDROID
+#elif SANITIZER_ANDROID || SANITIZER_NETBSD
   *addr = 0;
   *size = 0;
 #else
@@ -376,10 +376,12 @@ static void GetTls(uptr *addr, uptr *size) {
 
 #if !SANITIZER_GO
 uptr GetTlsSize() {
-#if SANITIZER_FREEBSD || SANITIZER_ANDROID
+#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD
   uptr addr, size;
   GetTls(&addr, &size);
   return size;
+#elif defined(__mips__) || defined(__powerpc64__)
+  return RoundUpTo(g_tls_size + TlsPreTcbSize(), 16);
 #else
   return g_tls_size;
 #endif
@@ -420,7 +422,7 @@ typedef ElfW(Phdr) Elf_Phdr;
 # endif
 
 struct DlIteratePhdrData {
-  InternalMmapVector<LoadedModule> *modules;
+  InternalMmapVectorNoCtor<LoadedModule> *modules;
   bool first;
 };
 
@@ -444,7 +446,9 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
       uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
       uptr cur_end = cur_beg + phdr->p_memsz;
       bool executable = phdr->p_flags & PF_X;
-      cur_module.addAddressRange(cur_beg, cur_end, executable);
+      bool writable = phdr->p_flags & PF_W;
+      cur_module.addAddressRange(cur_beg, cur_end, executable,
+                                 writable);
     }
   }
   data->modules->push_back(cur_module);
@@ -456,21 +460,41 @@ extern "C" __attribute__((weak)) int dl_iterate_phdr(
     int (*)(struct dl_phdr_info *, size_t, void *), void *);
 #endif
 
-void ListOfModules::init() {
-  clear();
+static bool requiresProcmaps() {
 #if SANITIZER_ANDROID && __ANDROID_API__ <= 22
-  u32 api_level = AndroidGetApiLevel();
   // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
   // The runtime check allows the same library to work with
   // both K and L (and future) Android releases.
-  if (api_level <= ANDROID_LOLLIPOP_MR1) { // L or earlier
-    MemoryMappingLayout memory_mapping(false);
-    memory_mapping.DumpListOfModules(&modules_);
-    return;
-  }
+  return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
+#else
+  return false;
 #endif
-  DlIteratePhdrData data = {&modules_, true};
-  dl_iterate_phdr(dl_iterate_phdr_cb, &data);
+}
+
+static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
+  MemoryMappingLayout memory_mapping(/*cache_enabled*/true);
+  memory_mapping.DumpListOfModules(modules);
+}
+
+void ListOfModules::init() {
+  clearOrInit();
+  if (requiresProcmaps()) {
+    procmapsInit(&modules_);
+  } else {
+    DlIteratePhdrData data = {&modules_, true};
+    dl_iterate_phdr(dl_iterate_phdr_cb, &data);
+  }
+}
+
+// When a custom loader is used, dl_iterate_phdr may not contain the full
+// list of modules. Allow callers to fall back to using procmaps.
+void ListOfModules::fallbackInit() {
+  if (!requiresProcmaps()) {
+    clearOrInit();
+    procmapsInit(&modules_);
+  } else {
+    clear();
+  }
 }
 
 // getrusage does not give us the current RSS, only the max RSS.
@@ -547,6 +571,15 @@ void LogMessageOnPrintf(const char *str) {
     WriteToSyslog(str);
 }
 
+#if SANITIZER_ANDROID
+extern "C" __attribute__((weak)) void android_set_abort_message(const char *);
+void SetAbortMessage(const char *str) {
+  if (&android_set_abort_message) android_set_abort_message(str);
+}
+#else
+void SetAbortMessage(const char *str) {}
+#endif
+
 #endif // SANITIZER_LINUX
 
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_s390.cc b/libsanitizer/sanitizer_common/sanitizer_linux_s390.cc
index 3faaa1c26ac..b836447bd5d 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux_s390.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_linux_s390.cc
@@ -176,6 +176,13 @@ static bool FixedCVE_2016_2143() {
     // 4.4.6+ is OK.
     if (minor == 4 && patch >= 6)
       return true;
+    if (minor == 4 && patch == 0 && ptr[0] == '-' &&
+        internal_strstr(buf.version, "Ubuntu")) {
+      // Check Ubuntu 16.04
+      int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);
+      if (r1 >= 13) // 4.4.0-13 or later
+        return true;
+    }
     // Otherwise, OK if 4.5+.
     return minor >= 5;
   } else {
diff --git a/libsanitizer/sanitizer_common/sanitizer_list.h b/libsanitizer/sanitizer_common/sanitizer_list.h
index 190c7e67cf0..d7e8b501a6e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_list.h
+++ b/libsanitizer/sanitizer_common/sanitizer_list.h
@@ -68,6 +68,17 @@ struct IntrusiveList {
     size_--;
   }
 
+  void extract(Item *prev, Item *x) {
+    CHECK(!empty());
+    CHECK_NE(prev, nullptr);
+    CHECK_NE(x, nullptr);
+    CHECK_EQ(prev->next, x);
+    prev->next = x->next;
+    if (last_ == x)
+      last_ = prev;
+    size_--;
+  }
+
   Item *front() { return first_; }
   const Item *front() const { return first_; }
   Item *back() { return last_; }
diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.cc b/libsanitizer/sanitizer_common/sanitizer_mac.cc
index 2a05102e968..8c78494e81a 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mac.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_mac.cc
@@ -21,6 +21,7 @@
 #include <stdio.h>
 
 #include "sanitizer_common.h"
+#include "sanitizer_file.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_libc.h"
@@ -100,12 +101,12 @@ extern "C" int __munmap(void *, size_t) SANITIZER_WEAK_ATTRIBUTE;
 uptr internal_mmap(void *addr, size_t length, int prot, int flags,
                    int fd, u64 offset) {
   if (fd == -1) fd = VM_MAKE_TAG(VM_MEMORY_ANALYSIS_TOOL);
-  if (__mmap) return (uptr)__mmap(addr, length, prot, flags, fd, offset);
+  if (&__mmap) return (uptr)__mmap(addr, length, prot, flags, fd, offset);
   return (uptr)mmap(addr, length, prot, flags, fd, offset);
 }
 
 uptr internal_munmap(void *addr, uptr length) {
-  if (__munmap) return __munmap(addr, length);
+  if (&__munmap) return __munmap(addr, length);
   return munmap(addr, length);
 }
 
@@ -189,14 +190,15 @@ void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); }
 
 uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
                           __sanitizer_sigset_t *oldset) {
-  return sigprocmask(how, set, oldset);
+  // Don't use sigprocmask here, because it affects all threads.
+  return pthread_sigmask(how, set, oldset);
 }
 
 // Doesn't call pthread_atfork() handlers (but not available on 10.6).
 extern "C" pid_t __fork(void) SANITIZER_WEAK_ATTRIBUTE;
 
 int internal_fork() {
-  if (__fork)
+  if (&__fork)
     return __fork();
   return fork();
 }
@@ -250,9 +252,8 @@ bool FileExists(const char *filename) {
   return S_ISREG(st.st_mode);
 }
 
-uptr GetTid() {
-  // FIXME: This can potentially get truncated on 32-bit, where uptr is 4 bytes.
-  uint64_t tid;
+tid_t GetTid() {
+  tid_t tid;
   pthread_threadid_np(nullptr, &tid);
   return tid;
 }
@@ -346,20 +347,16 @@ BlockingMutex::BlockingMutex() {
 void BlockingMutex::Lock() {
   CHECK(sizeof(OSSpinLock) <= sizeof(opaque_storage_));
   CHECK_EQ(OS_SPINLOCK_INIT, 0);
-  CHECK_NE(owner_, (uptr)pthread_self());
+  CHECK_EQ(owner_, 0);
   OSSpinLockLock((OSSpinLock*)&opaque_storage_);
-  CHECK(!owner_);
-  owner_ = (uptr)pthread_self();
 }
 
 void BlockingMutex::Unlock() {
-  CHECK(owner_ == (uptr)pthread_self());
-  owner_ = 0;
   OSSpinLockUnlock((OSSpinLock*)&opaque_storage_);
 }
 
 void BlockingMutex::CheckLocked() {
-  CHECK_EQ((uptr)pthread_self(), owner_);
+  CHECK_NE(*(OSSpinLock*)&opaque_storage_, 0);
 }
 
 u64 NanoTime() {
@@ -373,6 +370,27 @@ uptr GetTlsSize() {
 void InitTlsSize() {
 }
 
+uptr TlsBaseAddr() {
+  uptr segbase = 0;
+#if defined(__x86_64__)
+  asm("movq %%gs:0,%0" : "=r"(segbase));
+#elif defined(__i386__)
+  asm("movl %%gs:0,%0" : "=r"(segbase));
+#endif
+  return segbase;
+}
+
+// The size of the tls on darwin does not appear to be well documented,
+// however the vm memory map suggests that it is 1024 uptrs in size,
+// with a size of 0x2000 bytes on x86_64 and 0x1000 bytes on i386.
+uptr TlsSize() {
+#if defined(__x86_64__) || defined(__i386__)
+  return 1024 * sizeof(uptr);
+#else
+  return 0;
+#endif
+}
+
 void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
                           uptr *tls_addr, uptr *tls_size) {
 #if !SANITIZER_GO
@@ -380,8 +398,8 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
   GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
   *stk_addr = stack_bottom;
   *stk_size = stack_top - stack_bottom;
-  *tls_addr = 0;
-  *tls_size = 0;
+  *tls_addr = TlsBaseAddr();
+  *tls_size = TlsSize();
 #else
   *stk_addr = 0;
   *stk_size = 0;
@@ -391,18 +409,37 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
 }
 
 void ListOfModules::init() {
-  clear();
+  clearOrInit();
   MemoryMappingLayout memory_mapping(false);
   memory_mapping.DumpListOfModules(&modules_);
 }
 
-bool IsHandledDeadlySignal(int signum) {
+void ListOfModules::fallbackInit() { clear(); }
+
+static HandleSignalMode GetHandleSignalModeImpl(int signum) {
+  switch (signum) {
+    case SIGABRT:
+      return common_flags()->handle_abort;
+    case SIGILL:
+      return common_flags()->handle_sigill;
+    case SIGFPE:
+      return common_flags()->handle_sigfpe;
+    case SIGSEGV:
+      return common_flags()->handle_segv;
+    case SIGBUS:
+      return common_flags()->handle_sigbus;
+  }
+  return kHandleSignalNo;
+}
+
+HandleSignalMode GetHandleSignalMode(int signum) {
+  // Handling fatal signals on watchOS and tvOS devices is disallowed.
   if ((SANITIZER_WATCHOS || SANITIZER_TVOS) && !(SANITIZER_IOSSIM))
-    // Handling fatal signals on watchOS and tvOS devices is disallowed.
-    return false;
-  if (common_flags()->handle_abort && signum == SIGABRT)
-    return true;
-  return (signum == SIGSEGV || signum == SIGBUS) && common_flags()->handle_segv;
+    return kHandleSignalNo;
+  HandleSignalMode result = GetHandleSignalModeImpl(signum);
+  if (result == kHandleSignalYes && !common_flags()->allow_user_segv_handler)
+    return kHandleSignalExclusive;
+  return result;
 }
 
 MacosVersion cached_macos_version = MACOS_VERSION_UNINITIALIZED;
@@ -448,6 +485,15 @@ MacosVersion GetMacosVersion() {
   return result;
 }
 
+bool PlatformHasDifferentMemcpyAndMemmove() {
+  // On OS X 10.7 memcpy() and memmove() are both resolved
+  // into memmove$VARIANT$sse42.
+  // See also https://github.com/google/sanitizers/issues/34.
+  // TODO(glider): need to check dynamically that memcpy() and memmove() are
+  // actually the same function.
+  return GetMacosVersion() == MACOS_VERSION_SNOW_LEOPARD;
+}
+
 uptr GetRSS() {
   struct task_basic_info info;
   unsigned count = TASK_BASIC_INFO_COUNT;
@@ -528,7 +574,7 @@ void LogFullErrorReport(const char *buffer) {
 #endif
 }
 
-SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
 #if defined(__x86_64__) || defined(__i386__)
   ucontext_t *ucontext = static_cast<ucontext_t*>(context);
   return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ;
@@ -537,7 +583,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
 #endif
 }
 
-void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
+static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
   ucontext_t *ucontext = (ucontext_t*)context;
 # if defined(__aarch64__)
   *pc = ucontext->uc_mcontext->__ss.__pc;
@@ -564,6 +610,8 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
 # endif
 }
 
+void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
+
 #if !SANITIZER_GO
 static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
 LowLevelAllocator allocator_for_env;
@@ -755,9 +803,69 @@ char **GetArgv() {
   return *_NSGetArgv();
 }
 
+#if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
+// The task_vm_info struct is normally provided by the macOS SDK, but we need
+// fields only available in 10.12+. Declare the struct manually to be able to
+// build against older SDKs.
+struct __sanitizer_task_vm_info {
+  mach_vm_size_t virtual_size;
+  integer_t region_count;
+  integer_t page_size;
+  mach_vm_size_t resident_size;
+  mach_vm_size_t resident_size_peak;
+  mach_vm_size_t device;
+  mach_vm_size_t device_peak;
+  mach_vm_size_t internal;
+  mach_vm_size_t internal_peak;
+  mach_vm_size_t external;
+  mach_vm_size_t external_peak;
+  mach_vm_size_t reusable;
+  mach_vm_size_t reusable_peak;
+  mach_vm_size_t purgeable_volatile_pmap;
+  mach_vm_size_t purgeable_volatile_resident;
+  mach_vm_size_t purgeable_volatile_virtual;
+  mach_vm_size_t compressed;
+  mach_vm_size_t compressed_peak;
+  mach_vm_size_t compressed_lifetime;
+  mach_vm_size_t phys_footprint;
+  mach_vm_address_t min_address;
+  mach_vm_address_t max_address;
+};
+#define __SANITIZER_TASK_VM_INFO_COUNT ((mach_msg_type_number_t) \
+    (sizeof(__sanitizer_task_vm_info) / sizeof(natural_t)))
+
+uptr GetTaskInfoMaxAddress() {
+  __sanitizer_task_vm_info vm_info = {};
+  mach_msg_type_number_t count = __SANITIZER_TASK_VM_INFO_COUNT;
+  int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count);
+  if (err == 0) {
+    return vm_info.max_address - 1;
+  } else {
+    // xnu cannot provide vm address limit
+    return 0x200000000 - 1;
+  }
+}
+#endif
+
+uptr GetMaxVirtualAddress() {
+#if SANITIZER_WORDSIZE == 64
+# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
+  // Get the maximum VM address
+  static uptr max_vm = GetTaskInfoMaxAddress();
+  CHECK(max_vm);
+  return max_vm;
+# else
+  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
+# endif
+#else  // SANITIZER_WORDSIZE == 32
+  return (1ULL << 32) - 1;  // 0xffffffff;
+#endif  // SANITIZER_WORDSIZE
+}
+
 uptr FindAvailableMemoryRange(uptr shadow_size,
                               uptr alignment,
-                              uptr left_padding) {
+                              uptr left_padding,
+                              uptr *largest_gap_found) {
   typedef vm_region_submap_short_info_data_64_t RegionInfo;
   enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
   // Start searching for available memory region past PAGEZERO, which is
@@ -768,6 +876,7 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
   mach_vm_address_t address = start_address;
   mach_vm_address_t free_begin = start_address;
   kern_return_t kr = KERN_SUCCESS;
+  if (largest_gap_found) *largest_gap_found = 0;
   while (kr == KERN_SUCCESS) {
     mach_vm_size_t vmsize = 0;
     natural_t depth = 0;
@@ -777,10 +886,15 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
                                 (vm_region_info_t)&vminfo, &count);
     if (free_begin != address) {
       // We found a free region [free_begin..address-1].
-      uptr shadow_address = RoundUpTo((uptr)free_begin + left_padding,
-                                      alignment);
-      if (shadow_address + shadow_size < (uptr)address) {
-        return shadow_address;
+      uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
+      uptr gap_end = RoundDownTo((uptr)address, alignment);
+      uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
+      if (shadow_size < gap_size) {
+        return gap_start;
+      }
+
+      if (largest_gap_found && *largest_gap_found < gap_size) {
+        *largest_gap_found = gap_size;
       }
     }
     // Move to the next region.
@@ -795,6 +909,95 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
 // FIXME implement on this platform.
 void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
 
+void SignalContext::DumpAllRegisters(void *context) {
+  Report("Register values:\n");
+
+  ucontext_t *ucontext = (ucontext_t*)context;
+# define DUMPREG64(r) \
+    Printf("%s = 0x%016llx  ", #r, ucontext->uc_mcontext->__ss.__ ## r);
+# define DUMPREG32(r) \
+    Printf("%s = 0x%08x  ", #r, ucontext->uc_mcontext->__ss.__ ## r);
+# define DUMPREG_(r)   Printf(" "); DUMPREG(r);
+# define DUMPREG__(r)  Printf("  "); DUMPREG(r);
+# define DUMPREG___(r) Printf("   "); DUMPREG(r);
+
+# if defined(__x86_64__)
+#  define DUMPREG(r) DUMPREG64(r)
+  DUMPREG(rax); DUMPREG(rbx); DUMPREG(rcx); DUMPREG(rdx); Printf("\n");
+  DUMPREG(rdi); DUMPREG(rsi); DUMPREG(rbp); DUMPREG(rsp); Printf("\n");
+  DUMPREG_(r8); DUMPREG_(r9); DUMPREG(r10); DUMPREG(r11); Printf("\n");
+  DUMPREG(r12); DUMPREG(r13); DUMPREG(r14); DUMPREG(r15); Printf("\n");
+# elif defined(__i386__)
+#  define DUMPREG(r) DUMPREG32(r)
+  DUMPREG(eax); DUMPREG(ebx); DUMPREG(ecx); DUMPREG(edx); Printf("\n");
+  DUMPREG(edi); DUMPREG(esi); DUMPREG(ebp); DUMPREG(esp); Printf("\n");
+# elif defined(__aarch64__)
+#  define DUMPREG(r) DUMPREG64(r)
+  DUMPREG_(x[0]); DUMPREG_(x[1]); DUMPREG_(x[2]); DUMPREG_(x[3]); Printf("\n");
+  DUMPREG_(x[4]); DUMPREG_(x[5]); DUMPREG_(x[6]); DUMPREG_(x[7]); Printf("\n");
+  DUMPREG_(x[8]); DUMPREG_(x[9]); DUMPREG(x[10]); DUMPREG(x[11]); Printf("\n");
+  DUMPREG(x[12]); DUMPREG(x[13]); DUMPREG(x[14]); DUMPREG(x[15]); Printf("\n");
+  DUMPREG(x[16]); DUMPREG(x[17]); DUMPREG(x[18]); DUMPREG(x[19]); Printf("\n");
+  DUMPREG(x[20]); DUMPREG(x[21]); DUMPREG(x[22]); DUMPREG(x[23]); Printf("\n");
+  DUMPREG(x[24]); DUMPREG(x[25]); DUMPREG(x[26]); DUMPREG(x[27]); Printf("\n");
+  DUMPREG(x[28]); DUMPREG___(fp); DUMPREG___(lr); DUMPREG___(sp); Printf("\n");
+# elif defined(__arm__)
+#  define DUMPREG(r) DUMPREG32(r)
+  DUMPREG_(r[0]); DUMPREG_(r[1]); DUMPREG_(r[2]); DUMPREG_(r[3]); Printf("\n");
+  DUMPREG_(r[4]); DUMPREG_(r[5]); DUMPREG_(r[6]); DUMPREG_(r[7]); Printf("\n");
+  DUMPREG_(r[8]); DUMPREG_(r[9]); DUMPREG(r[10]); DUMPREG(r[11]); Printf("\n");
+  DUMPREG(r[12]); DUMPREG___(sp); DUMPREG___(lr); DUMPREG___(pc); Printf("\n");
+# else
+# error "Unknown architecture"
+# endif
+
+# undef DUMPREG64
+# undef DUMPREG32
+# undef DUMPREG_
+# undef DUMPREG__
+# undef DUMPREG___
+# undef DUMPREG
+}
+
+static inline bool CompareBaseAddress(const LoadedModule &a,
+                                      const LoadedModule &b) {
+  return a.base_address() < b.base_address();
+}
+
+void FormatUUID(char *out, uptr size, const u8 *uuid) {
+  internal_snprintf(out, size,
+                    "<%02X%02X%02X%02X-%02X%02X-%02X%02X-%02X%02X-"
+                    "%02X%02X%02X%02X%02X%02X>",
+                    uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5],
+                    uuid[6], uuid[7], uuid[8], uuid[9], uuid[10], uuid[11],
+                    uuid[12], uuid[13], uuid[14], uuid[15]);
+}
+
+void PrintModuleMap() {
+  Printf("Process module map:\n");
+  MemoryMappingLayout memory_mapping(false);
+  InternalMmapVector<LoadedModule> modules(/*initial_capacity*/ 128);
+  memory_mapping.DumpListOfModules(&modules);
+  InternalSort(&modules, modules.size(), CompareBaseAddress);
+  for (uptr i = 0; i < modules.size(); ++i) {
+    char uuid_str[128];
+    FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
+    Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),
+           modules[i].max_executable_address(), modules[i].full_name(),
+           ModuleArchToString(modules[i].arch()), uuid_str);
+  }
+  Printf("End of module map.\n");
+}
+
+void CheckNoDeepBind(const char *filename, int flag) {
+  // Do nothing.
+}
+
+// FIXME: implement on this platform.
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  UNIMPLEMENTED();
+}
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_MAC
diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.h b/libsanitizer/sanitizer_common/sanitizer_mac.h
index 4bea069189f..4881b62ff57 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mac.h
+++ b/libsanitizer/sanitizer_common/sanitizer_mac.h
@@ -18,6 +18,17 @@
 
 namespace __sanitizer {
 
+struct MemoryMappingLayoutData {
+  int current_image;
+  u32 current_magic;
+  u32 current_filetype;
+  ModuleArch current_arch;
+  u8 current_uuid[kModuleUUIDSize];
+  int current_load_cmd_count;
+  char *current_load_cmd_addr;
+  bool current_instrumented;
+};
+
 enum MacosVersion {
   MACOS_VERSION_UNINITIALIZED = 0,
   MACOS_VERSION_UNKNOWN,
@@ -34,6 +45,8 @@ MacosVersion GetMacosVersion();
 
 char **GetEnviron();
 
+void RestrictMemoryToMaxAddress(uptr max_address);
+
 }  // namespace __sanitizer
 
 extern "C" {
diff --git a/libsanitizer/sanitizer_common/sanitizer_mac_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_mac_libcdep.cc
new file mode 100644
index 00000000000..b376a0710b9
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_mac_libcdep.cc
@@ -0,0 +1,28 @@
+//===-- sanitizer_mac_libcdep.cc ------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements OSX-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+#include "sanitizer_mac.h"
+
+#include <sys/mman.h>
+
+namespace __sanitizer {
+
+void RestrictMemoryToMaxAddress(uptr max_address) {
+  uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address;
+  void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
+  CHECK(res != MAP_FAILED);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
diff --git a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc
index 93fb19e922c..2ca4e061464 100644
--- a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc
@@ -44,9 +44,48 @@ INTERCEPTOR(malloc_zone_t *, malloc_create_zone,
     // This matches the behavior of malloc_create_zone() on OSX 10.7 and higher.
     mprotect(new_zone, allocated_size, PROT_READ);
   }
+  // We're explicitly *NOT* registering the zone.
   return new_zone;
 }
 
+INTERCEPTOR(void, malloc_destroy_zone, malloc_zone_t *zone) {
+  COMMON_MALLOC_ENTER();
+  // We don't need to do anything here.  We're not registering new zones, so we
+  // don't to unregister.  Just un-mprotect and free() the zone.
+  if (GetMacosVersion() >= MACOS_VERSION_LION) {
+    uptr page_size = GetPageSizeCached();
+    uptr allocated_size = RoundUpTo(sizeof(sanitizer_zone), page_size);
+    mprotect(zone, allocated_size, PROT_READ | PROT_WRITE);
+  }
+  if (zone->zone_name) {
+    COMMON_MALLOC_FREE((void *)zone->zone_name);
+  }
+  COMMON_MALLOC_FREE(zone);
+}
+
+extern unsigned malloc_num_zones;
+extern malloc_zone_t **malloc_zones;
+
+// We need to make sure that sanitizer_zone is registered as malloc_zones[0]. If
+// libmalloc tries to set up a different zone as malloc_zones[0], it will call
+// mprotect(malloc_zones, ..., PROT_READ).  This interceptor will catch that and
+// make sure we are still the first (default) zone.
+INTERCEPTOR(int, mprotect, void *addr, size_t len, int prot) {
+  if (addr == malloc_zones && prot == PROT_READ) {
+    if (malloc_num_zones > 1 && malloc_zones[0] != &sanitizer_zone) {
+      for (unsigned i = 1; i < malloc_num_zones; i++) {
+        if (malloc_zones[i] == &sanitizer_zone) {
+          // Swap malloc_zones[0] and malloc_zones[i].
+          malloc_zones[i] = malloc_zones[0];
+          malloc_zones[0] = &sanitizer_zone;
+          break;
+        }
+      }
+    }
+  }
+  return REAL(mprotect)(addr, len, prot);
+}
+
 INTERCEPTOR(malloc_zone_t *, malloc_default_zone, void) {
   COMMON_MALLOC_ENTER();
   return &sanitizer_zone;
diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.h b/libsanitizer/sanitizer_common/sanitizer_mutex.h
index 75f495a4d65..1ec409def41 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mutex.h
+++ b/libsanitizer/sanitizer_common/sanitizer_mutex.h
@@ -81,6 +81,14 @@ class BlockingMutex {
   BlockingMutex();
   void Lock();
   void Unlock();
+
+  // This function does not guarantee an explicit check that the calling thread
+  // is the thread which owns the mutex. This behavior, while more strictly
+  // correct, causes problems in cases like StopTheWorld, where a parent thread
+  // owns the mutex but a child checks that it is locked. Rather than
+  // maintaining complex state to work around those situations, the check only
+  // checks that the mutex is owned, and assumes callers to be generally
+  // well-behaved.
   void CheckLocked();
  private:
   uptr opaque_storage_[10];
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform.h b/libsanitizer/sanitizer_common/sanitizer_platform.h
index 428709d55ec..1eb4d0c61c6 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform.h
@@ -11,8 +11,8 @@
 #ifndef SANITIZER_PLATFORM_H
 #define SANITIZER_PLATFORM_H
 
-#if !defined(__linux__) && !defined(__FreeBSD__) && \
-  !defined(__APPLE__) && !defined(_WIN32)
+#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \
+  !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__)
 # error "This operating system is not supported"
 #endif
 
@@ -28,6 +28,12 @@
 # define SANITIZER_FREEBSD 0
 #endif
 
+#if defined(__NetBSD__)
+# define SANITIZER_NETBSD 1
+#else
+# define SANITIZER_NETBSD 0
+#endif
+
 #if defined(__APPLE__)
 # define SANITIZER_MAC     1
 # include <TargetConditionals.h>
@@ -77,7 +83,14 @@
 # define SANITIZER_ANDROID 0
 #endif
 
-#define SANITIZER_POSIX (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC)
+#if defined(__Fuchsia__)
+# define SANITIZER_FUCHSIA 1
+#else
+# define SANITIZER_FUCHSIA 0
+#endif
+
+#define SANITIZER_POSIX \
+  (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD)
 
 #if __LP64__ || defined(_WIN64)
 #  define SANITIZER_WORDSIZE 64
@@ -160,13 +173,19 @@
 # define SANITIZER_PPC64V2 0
 #endif
 
+#if defined(__arm__)
+# define SANITIZER_ARM 1
+#else
+# define SANITIZER_ARM 0
+#endif
+
 // By default we allow to use SizeClassAllocator64 on 64-bit platform.
 // But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64
 // does not work well and we need to fallback to SizeClassAllocator32.
 // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
 // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
 #ifndef SANITIZER_CAN_USE_ALLOCATOR64
-# if SANITIZER_ANDROID && defined(__aarch64__)
+# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA
 #  define SANITIZER_CAN_USE_ALLOCATOR64 1
 # elif defined(__mips64) || defined(__aarch64__)
 #  define SANITIZER_CAN_USE_ALLOCATOR64 0
@@ -251,4 +270,15 @@
 # define SANITIZER_GO 0
 #endif
 
+// On PowerPC and ARM Thumb, calling pthread_exit() causes LSan to detect leaks.
+// pthread_exit() performs unwinding that leads to dlopen'ing libgcc_s.so.
+// dlopen mallocs "libgcc_s.so" string which confuses LSan, it fails to realize
+// that this allocation happens in dynamic linker and should be ignored.
+#if SANITIZER_PPC || defined(__thumb__)
+# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1
+#else
+# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
+#endif
+
+
 #endif // SANITIZER_PLATFORM_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
index 6b2ba31a2bf..b9eb09ad3bd 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
@@ -14,11 +14,25 @@
 
 #include "sanitizer_internal_defs.h"
 
+#if SANITIZER_POSIX
+# define SI_POSIX 1
+#else
+# define SI_POSIX 0
+#endif
+
 #if !SANITIZER_WINDOWS
-# define SI_NOT_WINDOWS 1
-# include "sanitizer_platform_limits_posix.h"
+# define SI_WINDOWS 0
 #else
-# define SI_NOT_WINDOWS 0
+# define SI_WINDOWS 1
+#endif
+
+#if (SI_POSIX != 0) == (SI_WINDOWS != 0) && !SANITIZER_FUCHSIA
+# error "Windows is not POSIX!"
+#endif
+
+#if SI_POSIX
+# include "sanitizer_platform_limits_netbsd.h"
+# include "sanitizer_platform_limits_posix.h"
 #endif
 
 #if SANITIZER_LINUX && !SANITIZER_ANDROID
@@ -39,6 +53,12 @@
 # define SI_FREEBSD 0
 #endif
 
+#if SANITIZER_NETBSD
+# define SI_NETBSD 1
+#else
+# define SI_NETBSD 0
+#endif
+
 #if SANITIZER_LINUX
 # define SI_LINUX 1
 #else
@@ -59,28 +79,43 @@
 # define SI_IOS 0
 #endif
 
-#if !SANITIZER_WINDOWS && !SANITIZER_MAC
-# define SI_UNIX_NOT_MAC 1
+#if SANITIZER_FUCHSIA
+# define SI_NOT_FUCHSIA 0
+#else
+# define SI_NOT_FUCHSIA 1
+#endif
+
+#if SANITIZER_POSIX && !SANITIZER_MAC
+# define SI_POSIX_NOT_MAC 1
 #else
-# define SI_UNIX_NOT_MAC 0
+# define SI_POSIX_NOT_MAC 0
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_FREEBSD
+# define SI_LINUX_NOT_FREEBSD 1
+# else
+# define SI_LINUX_NOT_FREEBSD 0
 #endif
 
-#define SANITIZER_INTERCEPT_STRLEN 1
-#define SANITIZER_INTERCEPT_STRNLEN SI_NOT_MAC
-#define SANITIZER_INTERCEPT_STRCMP 1
-#define SANITIZER_INTERCEPT_STRSTR 1
-#define SANITIZER_INTERCEPT_STRCASESTR SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_STRCHR 1
-#define SANITIZER_INTERCEPT_STRCHRNUL SI_UNIX_NOT_MAC
-#define SANITIZER_INTERCEPT_STRRCHR 1
-#define SANITIZER_INTERCEPT_STRSPN 1
-#define SANITIZER_INTERCEPT_STRPBRK 1
+#define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRNLEN (SI_NOT_MAC && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_STRCMP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRSTR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCASESTR SI_POSIX
+#define SANITIZER_INTERCEPT_STRTOK SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCHRNUL SI_POSIX_NOT_MAC
+#define SANITIZER_INTERCEPT_STRRCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRSPN SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRPBRK SI_NOT_FUCHSIA
 #define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STRCASECMP SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_STRCASECMP SI_POSIX
 #define SANITIZER_INTERCEPT_MEMSET 1
 #define SANITIZER_INTERCEPT_MEMMOVE 1
 #define SANITIZER_INTERCEPT_MEMCPY 1
-#define SANITIZER_INTERCEPT_MEMCMP 1
+#define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRNDUP SI_POSIX
+#define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD
 #if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070
 # define SI_MAC_DEPLOYMENT_BELOW_10_7 1
@@ -89,78 +124,83 @@
 #endif
 // memmem on Darwin doesn't exist on 10.6
 // FIXME: enable memmem on Windows.
-#define SANITIZER_INTERCEPT_MEMMEM \
-  SI_NOT_WINDOWS && !SI_MAC_DEPLOYMENT_BELOW_10_7
-#define SANITIZER_INTERCEPT_MEMCHR 1
-#define SANITIZER_INTERCEPT_MEMRCHR SI_FREEBSD || SI_LINUX
+#define SANITIZER_INTERCEPT_MEMMEM (SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_7)
+#define SANITIZER_INTERCEPT_MEMCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_MEMRCHR (SI_FREEBSD || SI_LINUX || SI_NETBSD)
 
-#define SANITIZER_INTERCEPT_READ   SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PREAD  SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_WRITE  SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PWRITE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_READ SI_POSIX
+#define SANITIZER_INTERCEPT_PREAD SI_POSIX
+#define SANITIZER_INTERCEPT_WRITE SI_POSIX
+#define SANITIZER_INTERCEPT_PWRITE SI_POSIX
+
+#define SANITIZER_INTERCEPT_FREAD SI_POSIX
+#define SANITIZER_INTERCEPT_FWRITE SI_POSIX
 
 #define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID
 
-#define SANITIZER_INTERCEPT_READV SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_WRITEV SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_READV SI_POSIX
+#define SANITIZER_INTERCEPT_WRITEV SI_POSIX
 
-#define SANITIZER_INTERCEPT_PREADV SI_FREEBSD || SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PREADV \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_PWRITEV SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_PREADV64 SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_PWRITEV64 SI_LINUX_NOT_ANDROID
 
 #define SANITIZER_INTERCEPT_PRCTL   SI_LINUX
 
-#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_STRPTIME SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_POSIX
+#define SANITIZER_INTERCEPT_STRPTIME SI_POSIX
 
-#define SANITIZER_INTERCEPT_SCANF SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_SCANF SI_POSIX
 #define SANITIZER_INTERCEPT_ISOC99_SCANF SI_LINUX_NOT_ANDROID
 
 #ifndef SANITIZER_INTERCEPT_PRINTF
-# define SANITIZER_INTERCEPT_PRINTF SI_NOT_WINDOWS
-# define SANITIZER_INTERCEPT_PRINTF_L SI_FREEBSD
+# define SANITIZER_INTERCEPT_PRINTF SI_POSIX
+# define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD)
 # define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_LINUX_NOT_ANDROID
 #endif
 
-#define SANITIZER_INTERCEPT_FREXP 1
-#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_FREXP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_POSIX
 
-#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_POSIX
 #define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_GETPWENT \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_GETPWENT_R SI_FREEBSD || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SETPWENT SI_MAC || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_CLOCK_GETTIME SI_FREEBSD || SI_LINUX
-#define SANITIZER_INTERCEPT_GETITIMER SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_TIME SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_GETPWENT_R \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_SETPWENT (SI_MAC || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_CLOCK_GETTIME (SI_FREEBSD || SI_NETBSD || SI_LINUX)
+#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
+#define SANITIZER_INTERCEPT_TIME SI_POSIX
 #define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_WAIT SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_INET SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETADDRINFO SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETNAMEINFO SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETSOCKNAME SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R SI_FREEBSD || SI_LINUX
-#define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R SI_FREEBSD || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R SI_FREEBSD || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_GETHOSTENT_R SI_FREEBSD || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_GETSOCKOPT SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_ACCEPT SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_WAIT SI_POSIX
+#define SANITIZER_INTERCEPT_INET SI_POSIX
+#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX
+#define SANITIZER_INTERCEPT_GETADDRINFO SI_POSIX
+#define SANITIZER_INTERCEPT_GETNAMEINFO SI_POSIX
+#define SANITIZER_INTERCEPT_GETSOCKNAME SI_POSIX
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_POSIX
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R (SI_FREEBSD || SI_LINUX)
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R \
+  (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX
+#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX
 #define SANITIZER_INTERCEPT_ACCEPT4 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_MODF SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_RECVMSG SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SENDMSG SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETPEERNAME SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_IOCTL SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_INET_ATON SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_MODF SI_POSIX
+#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
+#define SANITIZER_INTERCEPT_SENDMSG SI_POSIX
+#define SANITIZER_INTERCEPT_GETPEERNAME SI_POSIX
+#define SANITIZER_INTERCEPT_IOCTL SI_POSIX
+#define SANITIZER_INTERCEPT_INET_ATON SI_POSIX
 #define SANITIZER_INTERCEPT_SYSINFO SI_LINUX
-#define SANITIZER_INTERCEPT_READDIR SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_READDIR SI_POSIX
 #define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID
 #if SI_LINUX_NOT_ANDROID && \
   (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
@@ -170,109 +210,116 @@
 #else
 #define SANITIZER_INTERCEPT_PTRACE 0
 #endif
-#define SANITIZER_INTERCEPT_SETLOCALE SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETCWD SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_SETLOCALE SI_POSIX
+#define SANITIZER_INTERCEPT_GETCWD SI_POSIX
 #define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STRTOIMAX SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_MBSTOWCS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_MBSNRTOWCS SI_MAC || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_WCSTOMBS SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_STRTOIMAX SI_POSIX
+#define SANITIZER_INTERCEPT_MBSTOWCS SI_POSIX
+#define SANITIZER_INTERCEPT_MBSNRTOWCS (SI_MAC || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_WCSTOMBS SI_POSIX
 #define SANITIZER_INTERCEPT_WCSNRTOMBS \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_WCRTOMB \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_REALPATH SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_REALPATH SI_POSIX
 #define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_CONFSTR \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STRERROR SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_STRERROR_R SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_STRERROR SI_POSIX
+#define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX
 #define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_SCANDIR \
-  SI_FREEBSD || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_GETGROUPS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_POLL SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
+#define SANITIZER_INTERCEPT_POLL SI_POSIX
 #define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_WORDEXP \
-  SI_FREEBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SIGWAIT SI_NOT_WINDOWS
+  (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX
 #define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_SIGSETOPS \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SIGPENDING SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SIGPROCMASK SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_BACKTRACE SI_FREEBSD || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX
+#define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX
+#define SANITIZER_INTERCEPT_BACKTRACE \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX
 #define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STATFS SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_STATFS \
+  (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_STATFS64 \
-  (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STATVFS SI_FREEBSD || SI_LINUX_NOT_ANDROID
+  ((SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_STATVFS \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_INITGROUPS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX
+#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX
 #define SANITIZER_INTERCEPT_ETHER_HOST \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_ETHER_R SI_FREEBSD || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SHMCTL \
-  ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64)
+  (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_SHMCTL                       \
+  (SI_NETBSD || ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && \
+                 SANITIZER_WORDSIZE == 64))  // NOLINT
 #define SANITIZER_INTERCEPT_RANDOM_R SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED SI_POSIX
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL \
-  SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING \
-  SI_MAC || SI_LINUX_NOT_ANDROID
+  (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_TMPNAM SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX
 #define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_TTYNAME_R SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_TEMPNAM SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX
+#define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX
 #define SANITIZER_INTERCEPT_SINCOS SI_LINUX
-#define SANITIZER_INTERCEPT_REMQUO SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_LGAMMA SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_LGAMMA_R SI_FREEBSD || SI_LINUX
+#define SANITIZER_INTERCEPT_REMQUO SI_POSIX
+#define SANITIZER_INTERCEPT_LGAMMA SI_POSIX
+#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX)
 #define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_RAND_R \
-  SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_ICONV SI_FREEBSD || SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_TIMES SI_NOT_WINDOWS
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_ICONV \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_TIMES SI_POSIX
 
 // FIXME: getline seems to be available on OSX 10.7
-#define SANITIZER_INTERCEPT_GETLINE SI_FREEBSD || SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_GETLINE \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 
-#define SANITIZER_INTERCEPT__EXIT SI_LINUX || SI_FREEBSD || SI_MAC
+#define SANITIZER_INTERCEPT__EXIT \
+  (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC)
 
-#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \
-  SI_FREEBSD || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 
 #define SANITIZER_INTERCEPT_TLS_GET_ADDR \
-  SI_FREEBSD || SI_LINUX_NOT_ANDROID
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 
 #define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX
 #define SANITIZER_INTERCEPT_GETXATTR SI_LINUX
 #define SANITIZER_INTERCEPT_GETRESID SI_LINUX
 #define SANITIZER_INTERCEPT_GETIFADDRS \
-  SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_MAC
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC)
 #define SANITIZER_INTERCEPT_IF_INDEXTONAME \
-  SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_MAC
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC)
 #define SANITIZER_INTERCEPT_CAPGET SI_LINUX_NOT_ANDROID
 #if SI_LINUX && defined(__arm__)
 #define SANITIZER_INTERCEPT_AEABI_MEM 1
@@ -280,42 +327,67 @@
 #define SANITIZER_INTERCEPT_AEABI_MEM 0
 #endif
 #define SANITIZER_INTERCEPT___BZERO SI_MAC
-#define SANITIZER_INTERCEPT_FTIME !SI_FREEBSD && SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_POSIX)
 #define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_TSEARCH SI_LINUX_NOT_ANDROID || SI_MAC
+#define SANITIZER_INTERCEPT_TSEARCH \
+  (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD)
 #define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_FOPEN SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_FOPEN SI_POSIX
 #define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM (SI_LINUX_NOT_ANDROID || SI_NETBSD)
 #define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_FFLUSH SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_FCLOSE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_FFLUSH SI_POSIX
+#define SANITIZER_INTERCEPT_FCLOSE SI_POSIX
 
 #ifndef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
 #define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE \
-    SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_MAC
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC)
 #endif
 
-#define SANITIZER_INTERCEPT_GETPASS SI_LINUX_NOT_ANDROID || SI_MAC
+#define SANITIZER_INTERCEPT_GETPASS \
+  (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD)
 #define SANITIZER_INTERCEPT_TIMERFD SI_LINUX_NOT_ANDROID
 
-#define SANITIZER_INTERCEPT_MLOCKX SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_MLOCKX SI_POSIX
 #define SANITIZER_INTERCEPT_FOPENCOOKIE SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SEM SI_LINUX || SI_FREEBSD
-#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_MINCORE SI_LINUX
+#define SANITIZER_INTERCEPT_SEM (SI_LINUX || SI_FREEBSD || SI_NETBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX
+#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD)
 #define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX
-#define SANITIZER_INTERCEPT_CTERMID SI_LINUX || SI_MAC || SI_FREEBSD
-#define SANITIZER_INTERCEPT_CTERMID_R SI_MAC || SI_FREEBSD
+#define SANITIZER_INTERCEPT_CTERMID \
+  (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD)
+#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD)
 
-#define SANITIZER_INTERCEPTOR_HOOKS SI_LINUX
-#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SEND_SENDTO SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPTOR_HOOKS (SI_LINUX || SI_MAC || SI_WINDOWS)
+#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX
+#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
 #define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
 
-#define SANITIZER_INTERCEPT_STAT (SI_FREEBSD || SI_MAC || SI_ANDROID)
-#define SANITIZER_INTERCEPT___XSTAT !SANITIZER_INTERCEPT_STAT && SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_STAT \
+  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD)
+#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
 #define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT
 #define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID
+
+#define SANITIZER_INTERCEPT_UTMP (SI_POSIX && !SI_MAC && !SI_FREEBSD)
+#define SANITIZER_INTERCEPT_UTMPX (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD)
+
+#define SANITIZER_INTERCEPT_GETLOADAVG \
+  (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
+
+#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO \
+  (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_PVALLOC \
+  (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_CFREE \
+  (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
+#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC)
+#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
+#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID
+
 #endif  // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_netbsd.cc
new file mode 100644
index 00000000000..3c18ca6ce25
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_netbsd.cc
@@ -0,0 +1,357 @@
+//===-- sanitizer_platform_limits_netbsd.cc -------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific NetBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_NETBSD
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <glob.h>
+#include <grp.h>
+#include <ifaddrs.h>
+#include <limits.h>
+#include <link_elf.h>
+#include <net/if.h>
+#include <net/if_ether.h>
+#include <net/ppp_defs.h>
+#include <net/route.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip_mroute.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stddef.h>
+#include <sys/filio.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/mqueue.h>
+#include <sys/msg.h>
+#include <sys/mtio.h>
+#include <sys/ptrace.h>
+#include <sys/resource.h>
+#include <sys/shm.h>
+#include <sys/signal.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/soundcard.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/timespec.h>
+#include <sys/timex.h>
+#include <sys/types.h>
+#include <sys/ucontext.h>
+#include <sys/utsname.h>
+#include <term.h>
+#include <termios.h>
+#include <time.h>
+#include <utime.h>
+#include <utmp.h>
+#include <utmpx.h>
+#include <wchar.h>
+#include <wordexp.h>
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_netbsd.h"
+
+namespace __sanitizer {
+unsigned struct_utsname_sz = sizeof(struct utsname);
+unsigned struct_stat_sz = sizeof(struct stat);
+unsigned struct_rusage_sz = sizeof(struct rusage);
+unsigned struct_tm_sz = sizeof(struct tm);
+unsigned struct_passwd_sz = sizeof(struct passwd);
+unsigned struct_group_sz = sizeof(struct group);
+unsigned siginfo_t_sz = sizeof(siginfo_t);
+unsigned struct_sigaction_sz = sizeof(struct sigaction);
+unsigned struct_itimerval_sz = sizeof(struct itimerval);
+unsigned pthread_t_sz = sizeof(pthread_t);
+unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+unsigned pid_t_sz = sizeof(pid_t);
+unsigned timeval_sz = sizeof(timeval);
+unsigned uid_t_sz = sizeof(uid_t);
+unsigned gid_t_sz = sizeof(gid_t);
+unsigned mbstate_t_sz = sizeof(mbstate_t);
+unsigned sigset_t_sz = sizeof(sigset_t);
+unsigned struct_timezone_sz = sizeof(struct timezone);
+unsigned struct_tms_sz = sizeof(struct tms);
+unsigned struct_sigevent_sz = sizeof(struct sigevent);
+unsigned struct_sched_param_sz = sizeof(struct sched_param);
+unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+unsigned ucontext_t_sz = sizeof(ucontext_t);
+unsigned struct_rlimit_sz = sizeof(struct rlimit);
+unsigned struct_timespec_sz = sizeof(struct timespec);
+unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
+unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
+unsigned struct_timex_sz = sizeof(struct timex);
+unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
+unsigned struct_mq_attr_sz = sizeof(struct mq_attr);
+unsigned struct_statvfs_sz = sizeof(struct statvfs);
+
+uptr sig_ign = (uptr)SIG_IGN;
+uptr sig_dfl = (uptr)SIG_DFL;
+uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+int shmctl_ipc_stat = (int)IPC_STAT;
+
+unsigned struct_utmp_sz = sizeof(struct utmp);
+unsigned struct_utmpx_sz = sizeof(struct utmpx);
+
+int map_fixed = MAP_FIXED;
+
+int af_inet = (int)AF_INET;
+int af_inet6 = (int)AF_INET6;
+
+uptr __sanitizer_in_addr_sz(int af) {
+  if (af == AF_INET)
+    return sizeof(struct in_addr);
+  else if (af == AF_INET6)
+    return sizeof(struct in6_addr);
+  else
+    return 0;
+}
+
+int glob_nomatch = GLOB_NOMATCH;
+int glob_altdirfunc = GLOB_ALTDIRFUNC;
+
+unsigned path_max = PATH_MAX;
+
+// ioctl arguments
+unsigned struct_ifreq_sz = sizeof(struct ifreq);
+unsigned struct_termios_sz = sizeof(struct termios);
+unsigned struct_winsize_sz = sizeof(struct winsize);
+unsigned struct_mtget_sz = sizeof(struct mtget);
+unsigned struct_mtop_sz = sizeof(struct mtop);
+unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
+unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
+unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
+unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+
+const unsigned IOCTL_NOT_PRESENT = 0;
+
+unsigned IOCTL_FIOASYNC = FIOASYNC;
+unsigned IOCTL_FIOCLEX = FIOCLEX;
+unsigned IOCTL_FIOGETOWN = FIOGETOWN;
+unsigned IOCTL_FIONBIO = FIONBIO;
+unsigned IOCTL_FIONCLEX = FIONCLEX;
+unsigned IOCTL_FIOSETOWN = FIOSETOWN;
+unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
+unsigned IOCTL_SIOCATMARK = SIOCATMARK;
+unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
+unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR;
+unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR;
+unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF;
+unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR;
+unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS;
+unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC;
+unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU;
+unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK;
+unsigned IOCTL_SIOCGPGRP = SIOCGPGRP;
+unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR;
+unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR;
+unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR;
+unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS;
+unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC;
+unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU;
+unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
+unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
+unsigned IOCTL_TIOCCONS = TIOCCONS;
+unsigned IOCTL_TIOCEXCL = TIOCEXCL;
+unsigned IOCTL_TIOCGETD = TIOCGETD;
+unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
+unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ;
+unsigned IOCTL_TIOCMBIC = TIOCMBIC;
+unsigned IOCTL_TIOCMBIS = TIOCMBIS;
+unsigned IOCTL_TIOCMGET = TIOCMGET;
+unsigned IOCTL_TIOCMSET = TIOCMSET;
+unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
+unsigned IOCTL_TIOCNXCL = TIOCNXCL;
+unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
+unsigned IOCTL_TIOCPKT = TIOCPKT;
+unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
+unsigned IOCTL_TIOCSETD = TIOCSETD;
+unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
+unsigned IOCTL_TIOCSTI = TIOCSTI;
+unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
+unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT;
+unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT;
+
+const int si_SEGV_MAPERR = SEGV_MAPERR;
+const int si_SEGV_ACCERR = SEGV_ACCERR;
+}  // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+// There are more undocumented fields in dl_phdr_info that we are not interested
+// in.
+COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info));
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_flags);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_fileno);
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+
+CHECK_TYPE_SIZE(wordexp_t);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
+CHECK_SIZE_AND_OFFSET(tm, tm_zone);
+
+CHECK_TYPE_SIZE(ether_addr);
+
+CHECK_TYPE_SIZE(ipc_perm);
+CHECK_SIZE_AND_OFFSET(ipc_perm, _key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, _seq);
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+
+CHECK_TYPE_SIZE(clock_t);
+
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+// Compare against the union, because we can't reach into the union in a
+// compliant way.
+#ifdef ifa_dstaddr
+#undef ifa_dstaddr
+#endif
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+
+CHECK_TYPE_SIZE(timeb);
+CHECK_SIZE_AND_OFFSET(timeb, time);
+CHECK_SIZE_AND_OFFSET(timeb, millitm);
+CHECK_SIZE_AND_OFFSET(timeb, timezone);
+CHECK_SIZE_AND_OFFSET(timeb, dstflag);
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+#endif  // SANITIZER_NETBSD
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_netbsd.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_netbsd.h
new file mode 100644
index 00000000000..e7034ed6b52
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_netbsd.h
@@ -0,0 +1,566 @@
+//===-- sanitizer_platform_limits_netbsd.h --------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific NetBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_NETBSD_H
+#define SANITIZER_PLATFORM_LIMITS_NETBSD_H
+
+#if SANITIZER_NETBSD
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+#define _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, shift) \
+  ((link_map *)((handle) == nullptr ? nullptr : ((char *)(handle) + (shift))))
+
+#if defined(__x86_64__)
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+  _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 608)
+#elif defined(__i386__)
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+  _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 324)
+#endif
+
+namespace __sanitizer {
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_sched_param_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+extern unsigned ucontext_t_sz;
+
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+extern unsigned struct_timespec_sz;
+
+struct __sanitizer_iocb {
+  u64 aio_offset;
+  uptr aio_buf;
+  long aio_nbytes;
+  u32 aio_fildes;
+  u32 aio_lio_opcode;
+  long aio_reqprio;
+#if SANITIZER_WORDSIZE == 64
+  u8 aio_sigevent[32];
+#else
+  u8 aio_sigevent[20];
+#endif
+  u32 _state;
+  u32 _errno;
+  long _retval;
+};
+
+struct __sanitizer___sysctl_args {
+  int *name;
+  int nlen;
+  void *oldval;
+  uptr *oldlenp;
+  void *newval;
+  uptr newlen;
+};
+
+struct __sanitizer_sem_t {
+  uptr data[5];
+};
+
+struct __sanitizer_ipc_perm {
+  u32 uid;
+  u32 gid;
+  u32 cuid;
+  u32 cgid;
+  u32 mode;
+  unsigned short _seq;
+  long _key;
+};
+
+struct __sanitizer_shmid_ds {
+  __sanitizer_ipc_perm shm_perm;
+  unsigned long shm_segsz;
+  u32 shm_lpid;
+  u32 shm_cpid;
+  unsigned int shm_nattch;
+  u64 shm_atime;
+  u64 shm_dtime;
+  u64 shm_ctime;
+  void *_shm_internal;
+};
+
+extern unsigned struct_msqid_ds_sz;
+extern unsigned struct_mq_attr_sz;
+extern unsigned struct_timex_sz;
+extern unsigned struct_statvfs_sz;
+
+struct __sanitizer_iovec {
+  void *iov_base;
+  uptr iov_len;
+};
+
+struct __sanitizer_ifaddrs {
+  struct __sanitizer_ifaddrs *ifa_next;
+  char *ifa_name;
+  unsigned int ifa_flags;
+  void *ifa_addr;     // (struct sockaddr *)
+  void *ifa_netmask;  // (struct sockaddr *)
+  void *ifa_dstaddr;  // (struct sockaddr *)
+  void *ifa_data;
+  unsigned int ifa_addrflags;
+};
+
+typedef unsigned __sanitizer_pthread_key_t;
+
+typedef long long __sanitizer_time_t;
+
+struct __sanitizer_passwd {
+  char *pw_name;
+  char *pw_passwd;
+  int pw_uid;
+  int pw_gid;
+  __sanitizer_time_t pw_change;
+  char *pw_class;
+  char *pw_gecos;
+  char *pw_dir;
+  char *pw_shell;
+  __sanitizer_time_t pw_expire;
+};
+
+struct __sanitizer_group {
+  char *gr_name;
+  char *gr_passwd;
+  int gr_gid;
+  char **gr_mem;
+};
+
+struct __sanitizer_timeb {
+  __sanitizer_time_t time;
+  unsigned short millitm;
+  short timezone;
+  short dstflag;
+};
+
+struct __sanitizer_ether_addr {
+  u8 octet[6];
+};
+
+struct __sanitizer_tm {
+  int tm_sec;
+  int tm_min;
+  int tm_hour;
+  int tm_mday;
+  int tm_mon;
+  int tm_year;
+  int tm_wday;
+  int tm_yday;
+  int tm_isdst;
+  long int tm_gmtoff;
+  const char *tm_zone;
+};
+
+struct __sanitizer_msghdr {
+  void *msg_name;
+  unsigned msg_namelen;
+  struct __sanitizer_iovec *msg_iov;
+  unsigned msg_iovlen;
+  void *msg_control;
+  unsigned msg_controllen;
+  int msg_flags;
+};
+struct __sanitizer_cmsghdr {
+  unsigned cmsg_len;
+  int cmsg_level;
+  int cmsg_type;
+};
+
+struct __sanitizer_dirent {
+  u64 d_fileno;
+  u16 d_reclen;
+  // more fields that we don't care about
+};
+
+typedef int __sanitizer_clock_t;
+typedef int __sanitizer_clockid_t;
+
+typedef u32 __sanitizer___kernel_uid_t;
+typedef u32 __sanitizer___kernel_gid_t;
+typedef u64 __sanitizer___kernel_off_t;
+typedef struct {
+  u32 fds_bits[8];
+} __sanitizer___kernel_fd_set;
+
+typedef struct {
+  unsigned int pta_magic;
+  int pta_flags;
+  void *pta_private;
+} __sanitizer_pthread_attr_t;
+
+struct __sanitizer_sigset_t {
+  // uint32_t * 4
+  unsigned int __bits[4];
+};
+
+struct __sanitizer_sigaction {
+  union {
+    void (*handler)(int sig);
+    void (*sigaction)(int sig, void *siginfo, void *uctx);
+  };
+  __sanitizer_sigset_t sa_mask;
+  int sa_flags;
+};
+
+typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
+
+struct __sanitizer_kernel_sigaction_t {
+  union {
+    void (*handler)(int signo);
+    void (*sigaction)(int signo, void *info, void *ctx);
+  };
+  unsigned long sa_flags;
+  void (*sa_restorer)(void);
+  __sanitizer_kernel_sigset_t sa_mask;
+};
+
+extern uptr sig_ign;
+extern uptr sig_dfl;
+extern uptr sa_siginfo;
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+struct __sanitizer_dl_phdr_info {
+  uptr dlpi_addr;
+  const char *dlpi_name;
+  const void *dlpi_phdr;
+  short dlpi_phnum;
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+
+struct __sanitizer_addrinfo {
+  int ai_flags;
+  int ai_family;
+  int ai_socktype;
+  int ai_protocol;
+  unsigned ai_addrlen;
+  char *ai_canonname;
+  void *ai_addr;
+  struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+  char *h_name;
+  char **h_aliases;
+  int h_addrtype;
+  int h_length;
+  char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+  int fd;
+  short events;
+  short revents;
+};
+
+typedef unsigned __sanitizer_nfds_t;
+
+struct __sanitizer_glob_t {
+  uptr gl_pathc;
+  uptr gl_matchc;
+  uptr gl_offs;
+  int gl_flags;
+  char **gl_pathv;
+  int (*gl_errfunc)(const char *, int);
+  void (*gl_closedir)(void *dirp);
+  struct dirent *(*gl_readdir)(void *dirp);
+  void *(*gl_opendir)(const char *);
+  int (*gl_lstat)(const char *, void * /* struct stat* */);
+  int (*gl_stat)(const char *, void * /* struct stat* */);
+};
+
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+
+extern unsigned path_max;
+
+struct __sanitizer_wordexp_t {
+  uptr we_wordc;
+  char **we_wordv;
+  uptr we_offs;
+  char *we_strings;
+  uptr we_nbytes;
+};
+
+typedef void __sanitizer_FILE;
+#define SANITIZER_HAS_STRUCT_FILE 0
+
+extern int shmctl_ipc_stat;
+
+// This simplifies generic code
+#define struct_shminfo_sz -1
+#define struct_shm_info_sz -1
+#define shmctl_shm_stat -1
+#define shmctl_ipc_info -1
+#define shmctl_shm_info -1
+
+extern unsigned struct_utmp_sz;
+extern unsigned struct_utmpx_sz;
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+  int ifc_len;
+  union {
+    void *ifcu_req;
+  } ifc_ifcu;
+};
+
+#define IOC_NRBITS 8
+#define IOC_TYPEBITS 8
+#define IOC_SIZEBITS 14
+#define IOC_DIRBITS 2
+#define IOC_NONE 0U
+#define IOC_WRITE 1U
+#define IOC_READ 2U
+#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+#undef IOC_DIRMASK
+#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+#define IOC_NRSHIFT 0
+#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+#define EVIOC_EV_MAX 0x1f
+#define EVIOC_ABS_MAX 0x3f
+
+#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+
+extern unsigned struct_ifreq_sz;
+extern unsigned struct_termios_sz;
+extern unsigned struct_winsize_sz;
+
+extern unsigned struct_arpreq_sz;
+
+extern unsigned struct_mtget_sz;
+extern unsigned struct_mtop_sz;
+extern unsigned struct_rtentry_sz;
+extern unsigned struct_sbi_instrument_sz;
+extern unsigned struct_seq_event_rec_sz;
+extern unsigned struct_synth_info_sz;
+extern unsigned struct_vt_mode_sz;
+extern unsigned struct_audio_buf_info_sz;
+extern unsigned struct_ppp_stats_sz;
+extern unsigned struct_sioc_sg_req_sz;
+extern unsigned struct_sioc_vif_req_sz;
+
+// ioctl request identifiers
+
+// A special value to mark ioctls that are not present on the target platform,
+// when it can not be determined without including any system headers.
+extern const unsigned IOCTL_NOT_PRESENT;
+
+extern unsigned IOCTL_FIOASYNC;
+extern unsigned IOCTL_FIOCLEX;
+extern unsigned IOCTL_FIOGETOWN;
+extern unsigned IOCTL_FIONBIO;
+extern unsigned IOCTL_FIONCLEX;
+extern unsigned IOCTL_FIOSETOWN;
+extern unsigned IOCTL_SIOCADDMULTI;
+extern unsigned IOCTL_SIOCATMARK;
+extern unsigned IOCTL_SIOCDELMULTI;
+extern unsigned IOCTL_SIOCGIFADDR;
+extern unsigned IOCTL_SIOCGIFBRDADDR;
+extern unsigned IOCTL_SIOCGIFCONF;
+extern unsigned IOCTL_SIOCGIFDSTADDR;
+extern unsigned IOCTL_SIOCGIFFLAGS;
+extern unsigned IOCTL_SIOCGIFMETRIC;
+extern unsigned IOCTL_SIOCGIFMTU;
+extern unsigned IOCTL_SIOCGIFNETMASK;
+extern unsigned IOCTL_SIOCGPGRP;
+extern unsigned IOCTL_SIOCSIFADDR;
+extern unsigned IOCTL_SIOCSIFBRDADDR;
+extern unsigned IOCTL_SIOCSIFDSTADDR;
+extern unsigned IOCTL_SIOCSIFFLAGS;
+extern unsigned IOCTL_SIOCSIFMETRIC;
+extern unsigned IOCTL_SIOCSIFMTU;
+extern unsigned IOCTL_SIOCSIFNETMASK;
+extern unsigned IOCTL_SIOCSPGRP;
+extern unsigned IOCTL_TIOCCONS;
+extern unsigned IOCTL_TIOCEXCL;
+extern unsigned IOCTL_TIOCGETD;
+extern unsigned IOCTL_TIOCGPGRP;
+extern unsigned IOCTL_TIOCGWINSZ;
+extern unsigned IOCTL_TIOCMBIC;
+extern unsigned IOCTL_TIOCMBIS;
+extern unsigned IOCTL_TIOCMGET;
+extern unsigned IOCTL_TIOCMSET;
+extern unsigned IOCTL_TIOCNOTTY;
+extern unsigned IOCTL_TIOCNXCL;
+extern unsigned IOCTL_TIOCOUTQ;
+extern unsigned IOCTL_TIOCPKT;
+extern unsigned IOCTL_TIOCSCTTY;
+extern unsigned IOCTL_TIOCSETD;
+extern unsigned IOCTL_TIOCSPGRP;
+extern unsigned IOCTL_TIOCSTI;
+extern unsigned IOCTL_TIOCSWINSZ;
+extern unsigned IOCTL_SIOCGETSGCNT;
+extern unsigned IOCTL_SIOCGETVIFCNT;
+extern unsigned IOCTL_MTIOCGET;
+extern unsigned IOCTL_MTIOCTOP;
+extern unsigned IOCTL_SIOCADDRT;
+extern unsigned IOCTL_SIOCDELRT;
+extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE;
+extern unsigned IOCTL_SNDCTL_DSP_GETFMTS;
+extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK;
+extern unsigned IOCTL_SNDCTL_DSP_POST;
+extern unsigned IOCTL_SNDCTL_DSP_RESET;
+extern unsigned IOCTL_SNDCTL_DSP_SETFMT;
+extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT;
+extern unsigned IOCTL_SNDCTL_DSP_SPEED;
+extern unsigned IOCTL_SNDCTL_DSP_STEREO;
+extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE;
+extern unsigned IOCTL_SNDCTL_DSP_SYNC;
+extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE;
+extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR;
+extern unsigned IOCTL_SNDCTL_MIDI_INFO;
+extern unsigned IOCTL_SNDCTL_MIDI_PRETIME;
+extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE;
+extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS;
+extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS;
+extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND;
+extern unsigned IOCTL_SNDCTL_SEQ_PANIC;
+extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE;
+extern unsigned IOCTL_SNDCTL_SEQ_RESET;
+extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES;
+extern unsigned IOCTL_SNDCTL_SEQ_SYNC;
+extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI;
+extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD;
+extern unsigned IOCTL_SNDCTL_SYNTH_INFO;
+extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL;
+extern unsigned IOCTL_SNDCTL_TMR_CONTINUE;
+extern unsigned IOCTL_SNDCTL_TMR_METRONOME;
+extern unsigned IOCTL_SNDCTL_TMR_SELECT;
+extern unsigned IOCTL_SNDCTL_TMR_SOURCE;
+extern unsigned IOCTL_SNDCTL_TMR_START;
+extern unsigned IOCTL_SNDCTL_TMR_STOP;
+extern unsigned IOCTL_SNDCTL_TMR_TEMPO;
+extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE;
+extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_BASS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CAPS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CD;
+extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE;
+extern unsigned IOCTL_SOUND_MIXER_READ_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_READ_MIC;
+extern unsigned IOCTL_SOUND_MIXER_READ_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_PCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS;
+extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_CD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME;
+extern unsigned IOCTL_SOUND_PCM_READ_BITS;
+extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_READ_FILTER;
+extern unsigned IOCTL_SOUND_PCM_READ_RATE;
+extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER;
+extern unsigned IOCTL_VT_ACTIVATE;
+extern unsigned IOCTL_VT_GETMODE;
+extern unsigned IOCTL_VT_OPENQRY;
+extern unsigned IOCTL_VT_RELDISP;
+extern unsigned IOCTL_VT_SETMODE;
+extern unsigned IOCTL_VT_WAITACTIVE;
+extern unsigned IOCTL_KDDISABIO;
+extern unsigned IOCTL_KDENABIO;
+extern unsigned IOCTL_KDGETLED;
+extern unsigned IOCTL_KDGKBMODE;
+extern unsigned IOCTL_KDGKBTYPE;
+extern unsigned IOCTL_KDMKTONE;
+extern unsigned IOCTL_KDSETLED;
+extern unsigned IOCTL_KDSETMODE;
+extern unsigned IOCTL_KDSKBMODE;
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+}  // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+  COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((struct CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(struct CLASS, MEMBER))
+
+#endif  // SANITIZER_NETBSD
+
+#endif
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
index 31a5e697eae..858bb218450 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
@@ -23,7 +23,6 @@
 #endif
 #include <arpa/inet.h>
 #include <dirent.h>
-#include <errno.h>
 #include <grp.h>
 #include <limits.h>
 #include <net/if.h>
@@ -44,6 +43,9 @@
 #include <termios.h>
 #include <time.h>
 #include <wchar.h>
+#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+#include <utmp.h>
+#endif
 
 #if !SANITIZER_IOS
 #include <net/route.h>
@@ -52,6 +54,7 @@
 #if !SANITIZER_ANDROID
 #include <sys/mount.h>
 #include <sys/timeb.h>
+#include <utmpx.h>
 #endif
 
 #if SANITIZER_LINUX
@@ -277,6 +280,13 @@ namespace __sanitizer {
   int shmctl_shm_stat = (int)SHM_STAT;
 #endif
 
+#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+  unsigned struct_utmp_sz = sizeof(struct utmp);
+#endif
+#if !SANITIZER_ANDROID
+  unsigned struct_utmpx_sz = sizeof(struct utmpx);
+#endif
+
   int map_fixed = MAP_FIXED;
 
   int af_inet = (int)AF_INET;
@@ -918,14 +928,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   unsigned IOCTL_SNDCTL_DSP_GETOSPACE = SNDCTL_DSP_GETOSPACE;
 #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
 
-  const int errno_EINVAL = EINVAL;
-// EOWNERDEAD is not present in some older platforms.
-#if defined(EOWNERDEAD)
-  const int errno_EOWNERDEAD = EOWNERDEAD;
-#else
-  const int errno_EOWNERDEAD = -1;
-#endif
-
   const int si_SEGV_MAPERR = SEGV_MAPERR;
   const int si_SEGV_ACCERR = SEGV_ACCERR;
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
index c139322839a..4d11d071776 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -13,6 +13,8 @@
 #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H
 #define SANITIZER_PLATFORM_LIMITS_POSIX_H
 
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC
+
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_platform.h"
 
@@ -21,6 +23,9 @@
 // incorporates the map structure.
 # define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
     ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 544)))
+// Get sys/_types.h, because that tells us whether 64-bit inodes are
+// used in struct dirent below.
+#include <sys/_types.h>
 #else
 # define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) ((link_map*)(handle))
 #endif  // !SANITIZER_FREEBSD
@@ -81,7 +86,7 @@ namespace __sanitizer {
 #elif defined(__mips__)
   const unsigned struct_kernel_stat_sz =
                  SANITIZER_ANDROID ? FIRST_32_SECOND_64(104, 128) :
-                                     FIRST_32_SECOND_64(144, 216);
+                                     FIRST_32_SECOND_64(160, 216);
   const unsigned struct_kernel_stat64_sz = 104;
 #elif defined(__s390__) && !defined(__s390x__)
   const unsigned struct_kernel_stat_sz = 64;
@@ -204,24 +209,24 @@ namespace __sanitizer {
     unsigned __seq;
     u64 __unused1;
     u64 __unused2;
-#elif defined(__mips__) || defined(__aarch64__) || defined(__s390x__)
-    unsigned int mode;
-    unsigned short __seq;
-    unsigned short __pad1;
-    unsigned long __unused1;
-    unsigned long __unused2;
 #elif defined(__sparc__)
-# if defined(__arch64__)
+#if defined(__arch64__)
     unsigned mode;
     unsigned short __pad1;
-# else
+#else
     unsigned short __pad1;
     unsigned short mode;
     unsigned short __pad2;
-# endif
+#endif
     unsigned short __seq;
     unsigned long long __unused1;
     unsigned long long __unused2;
+#elif defined(__mips__) || defined(__aarch64__) || defined(__s390x__)
+    unsigned int mode;
+    unsigned short __seq;
+    unsigned short __pad1;
+    unsigned long __unused1;
+    unsigned long __unused2;
 #else
     unsigned short mode;
     unsigned short __pad1;
@@ -240,17 +245,17 @@ namespace __sanitizer {
   struct __sanitizer_shmid_ds {
     __sanitizer_ipc_perm shm_perm;
   #if defined(__sparc__)
-  # if !defined(__arch64__)
+  #if !defined(__arch64__)
     u32 __pad1;
-  # endif
+  #endif
     long shm_atime;
-  # if !defined(__arch64__)
+  #if !defined(__arch64__)
     u32 __pad2;
-  # endif
+  #endif
     long shm_dtime;
-  # if !defined(__arch64__)
+  #if !defined(__arch64__)
     u32 __pad3;
-  # endif
+  #endif
     long shm_ctime;
     uptr shm_segsz;
     int shm_cpid;
@@ -483,7 +488,12 @@ namespace __sanitizer {
   };
 #elif SANITIZER_FREEBSD
   struct __sanitizer_dirent {
+#if defined(__INO64)
+    unsigned long long d_fileno;
+    unsigned long long d_off;
+#else
     unsigned int d_fileno;
+#endif
     unsigned short d_reclen;
     // more fields that we don't care about
   };
@@ -863,6 +873,13 @@ namespace __sanitizer {
   extern int shmctl_shm_stat;
 #endif
 
+#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+  extern unsigned struct_utmp_sz;
+#endif
+#if !SANITIZER_ANDROID
+  extern unsigned struct_utmpx_sz;
+#endif
+
   extern int map_fixed;
 
   // ioctl arguments
@@ -908,7 +925,8 @@ struct __sanitizer_cookie_io_functions_t {
 
 #define IOC_NRBITS 8
 #define IOC_TYPEBITS 8
-#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) || defined(__sparc__)
+#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) || \
+    defined(__sparc__)
 #define IOC_SIZEBITS 13
 #define IOC_DIRBITS 3
 #define IOC_NONE 1U
@@ -943,9 +961,8 @@ struct __sanitizer_cookie_io_functions_t {
 // In sparc the 14 bits SIZE field overlaps with the
 // least significant bit of DIR, so either IOC_READ or
 // IOC_WRITE shall be 1 in order to get a non-zero SIZE.
-# define IOC_SIZE(nr)                       \
-  ((((((nr) >> 29) & 0x7) & (4U|2U)) == 0)? \
-   0 : (((nr) >> 16) & 0x3fff))
+#define IOC_SIZE(nr) \
+  ((((((nr) >> 29) & 0x7) & (4U | 2U)) == 0) ? 0 : (((nr) >> 16) & 0x3fff))
 #else
 #define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
 #endif
@@ -1447,9 +1464,6 @@ struct __sanitizer_cookie_io_functions_t {
   extern unsigned IOCTL_PIO_SCRNMAP;
 #endif
 
-  extern const int errno_EINVAL;
-  extern const int errno_EOWNERDEAD;
-
   extern const int si_SEGV_MAPERR;
   extern const int si_SEGV_ACCERR;
 }  // namespace __sanitizer
@@ -1471,4 +1485,6 @@ struct __sanitizer_cookie_io_functions_t {
   COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) ==          \
                  offsetof(struct CLASS, MEMBER))
 
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC
+
 #endif
diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.cc b/libsanitizer/sanitizer_common/sanitizer_posix.cc
index d10213d917f..8f59deb23b3 100644
--- a/libsanitizer/sanitizer_common/sanitizer_posix.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_posix.cc
@@ -15,23 +15,17 @@
 #if SANITIZER_POSIX
 
 #include "sanitizer_common.h"
+#include "sanitizer_file.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_posix.h"
 #include "sanitizer_procmaps.h"
 #include "sanitizer_stacktrace.h"
 
+#include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <sys/mman.h>
 
-#if SANITIZER_LINUX
-#include <sys/utsname.h>
-#endif
-
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-#include <sys/personality.h>
-#endif
-
 #if SANITIZER_FREEBSD
 // The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
 // that, it was never implemented.  So just define it to zero.
@@ -46,87 +40,13 @@ uptr GetMmapGranularity() {
   return GetPageSize();
 }
 
-#if SANITIZER_WORDSIZE == 32
-// Take care of unusable kernel area in top gigabyte.
-static uptr GetKernelAreaSize() {
-#if SANITIZER_LINUX && !SANITIZER_X32
-  const uptr gbyte = 1UL << 30;
-
-  // Firstly check if there are writable segments
-  // mapped to top gigabyte (e.g. stack).
-  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr end, prot;
-  while (proc_maps.Next(/*start*/nullptr, &end,
-                        /*offset*/nullptr, /*filename*/nullptr,
-                        /*filename_size*/0, &prot)) {
-    if ((end >= 3 * gbyte)
-        && (prot & MemoryMappingLayout::kProtectionWrite) != 0)
-      return 0;
-  }
-
-#if !SANITIZER_ANDROID
-  // Even if nothing is mapped, top Gb may still be accessible
-  // if we are running on 64-bit kernel.
-  // Uname may report misleading results if personality type
-  // is modified (e.g. under schroot) so check this as well.
-  struct utsname uname_info;
-  int pers = personality(0xffffffffUL);
-  if (!(pers & PER_MASK)
-      && uname(&uname_info) == 0
-      && internal_strstr(uname_info.machine, "64"))
-    return 0;
-#endif  // SANITIZER_ANDROID
-
-  // Top gigabyte is reserved for kernel.
-  return gbyte;
-#else
-  return 0;
-#endif  // SANITIZER_LINUX && !SANITIZER_X32
-}
-#endif  // SANITIZER_WORDSIZE == 32
-
-uptr GetMaxVirtualAddress() {
-#if SANITIZER_WORDSIZE == 64
-# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
-  // Ideally, we would derive the upper bound from MACH_VM_MAX_ADDRESS. The
-  // upper bound can change depending on the device.
-  return 0x200000000 - 1;
-# elif defined(__powerpc64__) || defined(__aarch64__)
-  // On PowerPC64 we have two different address space layouts: 44- and 46-bit.
-  // We somehow need to figure out which one we are using now and choose
-  // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
-  // Note that with 'ulimit -s unlimited' the stack is moved away from the top
-  // of the address space, so simply checking the stack address is not enough.
-  // This should (does) work for both PowerPC64 Endian modes.
-  // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
-  return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
-# elif defined(__mips64)
-  return (1ULL << 40) - 1;  // 0x000000ffffffffffUL;
-# elif defined(__s390x__)
-  return (1ULL << 53) - 1;  // 0x001fffffffffffffUL;
-# else
-  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
-# endif
-#else  // SANITIZER_WORDSIZE == 32
-# if defined(__s390__)
-  return (1ULL << 31) - 1;  // 0x7fffffff;
-# else
-  uptr res = (1ULL << 32) - 1;  // 0xffffffff;
-  if (!common_flags()->full_address_space)
-    res -= GetKernelAreaSize();
-  CHECK_LT(reinterpret_cast<uptr>(&res), res);
-  return res;
-# endif
-#endif  // SANITIZER_WORDSIZE
-}
-
 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
   size = RoundUpTo(size, GetPageSizeCached());
   uptr res = internal_mmap(nullptr, size,
                            PROT_READ | PROT_WRITE,
                            MAP_PRIVATE | MAP_ANON, -1, 0);
   int reserrno;
-  if (internal_iserror(res, &reserrno))
+  if (UNLIKELY(internal_iserror(res, &reserrno)))
     ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno, raw_report);
   IncreaseTotalMmap(size);
   return (void *)res;
@@ -135,7 +55,7 @@ void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
 void UnmapOrDie(void *addr, uptr size) {
   if (!addr || !size) return;
   uptr res = internal_munmap(addr, size);
-  if (internal_iserror(res)) {
+  if (UNLIKELY(internal_iserror(res))) {
     Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
            SanitizerToolName, size, size, addr);
     CHECK("unable to unmap" && 0);
@@ -143,21 +63,39 @@ void UnmapOrDie(void *addr, uptr size) {
   DecreaseTotalMmap(size);
 }
 
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  uptr res = internal_mmap(nullptr, size,
+                           PROT_READ | PROT_WRITE,
+                           MAP_PRIVATE | MAP_ANON, -1, 0);
+  int reserrno;
+  if (UNLIKELY(internal_iserror(res, &reserrno))) {
+    if (reserrno == ENOMEM)
+      return nullptr;
+    ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
+  }
+  IncreaseTotalMmap(size);
+  return (void *)res;
+}
+
 // We want to map a chunk of address space aligned to 'alignment'.
-// We do it by maping a bit more and then unmaping redundant pieces.
+// We do it by mapping a bit more and then unmapping redundant pieces.
 // We probably can do it with fewer syscalls in some OS-dependent way.
-void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type) {
   CHECK(IsPowerOfTwo(size));
   CHECK(IsPowerOfTwo(alignment));
   uptr map_size = size + alignment;
-  uptr map_res = (uptr)MmapOrDie(map_size, mem_type);
+  uptr map_res = (uptr)MmapOrDieOnFatalError(map_size, mem_type);
+  if (UNLIKELY(!map_res))
+    return nullptr;
   uptr map_end = map_res + map_size;
   uptr res = map_res;
-  if (res & (alignment - 1))  // Not aligned.
-    res = (map_res + alignment) & ~(alignment - 1);
-  uptr end = res + size;
-  if (res != map_res)
+  if (!IsAligned(res, alignment)) {
+    res = (map_res + alignment - 1) & ~(alignment - 1);
     UnmapOrDie((void*)map_res, res - map_res);
+  }
+  uptr end = res + size;
   if (end != map_end)
     UnmapOrDie((void*)end, map_end - end);
   return (void*)res;
@@ -171,13 +109,13 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
                          MAP_PRIVATE | MAP_ANON | MAP_NORESERVE,
                          -1, 0);
   int reserrno;
-  if (internal_iserror(p, &reserrno))
+  if (UNLIKELY(internal_iserror(p, &reserrno)))
     ReportMmapFailureAndDie(size, mem_type, "allocate noreserve", reserrno);
   IncreaseTotalMmap(size);
   return (void *)p;
 }
 
-void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
+void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem) {
   uptr PageSize = GetPageSizeCached();
   uptr p = internal_mmap((void*)(fixed_addr & ~(PageSize - 1)),
       RoundUpTo(size, PageSize),
@@ -185,8 +123,10 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
       MAP_PRIVATE | MAP_ANON | MAP_FIXED,
       -1, 0);
   int reserrno;
-  if (internal_iserror(p, &reserrno)) {
-    char mem_type[30];
+  if (UNLIKELY(internal_iserror(p, &reserrno))) {
+    if (tolerate_enomem && reserrno == ENOMEM)
+      return nullptr;
+    char mem_type[40];
     internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
                       fixed_addr);
     ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
@@ -195,6 +135,14 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
   return (void *)p;
 }
 
+void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
+  return MmapFixedImpl(fixed_addr, size, false /*tolerate_enomem*/);
+}
+
+void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) {
+  return MmapFixedImpl(fixed_addr, size, true /*tolerate_enomem*/);
+}
+
 bool MprotectNoAccess(uptr addr, uptr size) {
   return 0 == internal_mprotect((void*)addr, size, PROT_NONE);
 }
@@ -282,13 +230,12 @@ static inline bool IntervalsAreSeparate(uptr start1, uptr end1,
 // memory).
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr start, end;
-  while (proc_maps.Next(&start, &end,
-                        /*offset*/nullptr, /*filename*/nullptr,
-                        /*filename_size*/0, /*protection*/nullptr)) {
-    if (start == end) continue;  // Empty range.
-    CHECK_NE(0, end);
-    if (!IntervalsAreSeparate(start, end - 1, range_start, range_end))
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (segment.start == segment.end) continue;  // Empty range.
+    CHECK_NE(0, segment.end);
+    if (!IntervalsAreSeparate(segment.start, segment.end - 1, range_start,
+                              range_end))
       return false;
   }
   return true;
@@ -296,13 +243,13 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
 
 void DumpProcessMap() {
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr start, end;
   const sptr kBufSize = 4095;
   char *filename = (char*)MmapOrDie(kBufSize, __func__);
+  MemoryMappedSegment segment(filename, kBufSize);
   Report("Process memory map follows:\n");
-  while (proc_maps.Next(&start, &end, /* file_offset */nullptr,
-                        filename, kBufSize, /* protection */nullptr)) {
-    Printf("\t%p-%p\t%s\n", (void*)start, (void*)end, filename);
+  while (proc_maps.Next(&segment)) {
+    Printf("\t%p-%p\t%s\n", (void *)segment.start, (void *)segment.end,
+           segment.filename);
   }
   Report("End of process memory map.\n");
   UnmapOrDie(filename, kBufSize);
@@ -332,28 +279,48 @@ void ReportFile::Write(const char *buffer, uptr length) {
 }
 
 bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) {
-  uptr s, e, off, prot;
-  InternalScopedString buff(kMaxPathLength);
   MemoryMappingLayout proc_maps(/*cache_enabled*/false);
-  while (proc_maps.Next(&s, &e, &off, buff.data(), buff.size(), &prot)) {
-    if ((prot & MemoryMappingLayout::kProtectionExecute) != 0
-        && internal_strcmp(module, buff.data()) == 0) {
-      *start = s;
-      *end = e;
+  InternalScopedString buff(kMaxPathLength);
+  MemoryMappedSegment segment(buff.data(), kMaxPathLength);
+  while (proc_maps.Next(&segment)) {
+    if (segment.IsExecutable() &&
+        internal_strcmp(module, segment.filename) == 0) {
+      *start = segment.start;
+      *end = segment.end;
       return true;
     }
   }
   return false;
 }
 
-SignalContext SignalContext::Create(void *siginfo, void *context) {
-  auto si = (siginfo_t *)siginfo;
-  uptr addr = (uptr)si->si_addr;
-  uptr pc, sp, bp;
-  GetPcSpBp(context, &pc, &sp, &bp);
-  WriteFlag write_flag = GetWriteFlag(context);
-  bool is_memory_access = si->si_signo == SIGSEGV;
-  return SignalContext(context, addr, pc, sp, bp, is_memory_access, write_flag);
+uptr SignalContext::GetAddress() const {
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  return (uptr)si->si_addr;
+}
+
+bool SignalContext::IsMemoryAccess() const {
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  return si->si_signo == SIGSEGV;
+}
+
+int SignalContext::GetType() const {
+  return static_cast<const siginfo_t *>(siginfo)->si_signo;
+}
+
+const char *SignalContext::Describe() const {
+  switch (GetType()) {
+    case SIGFPE:
+      return "FPE";
+    case SIGILL:
+      return "ILL";
+    case SIGABRT:
+      return "ABRT";
+    case SIGSEGV:
+      return "SEGV";
+    case SIGBUS:
+      return "BUS";
+  }
+  return "UNKNOWN SIGNAL";
 }
 
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.h b/libsanitizer/sanitizer_common/sanitizer_posix.h
index 68b34babdeb..9626654e53a 100644
--- a/libsanitizer/sanitizer_common/sanitizer_posix.h
+++ b/libsanitizer/sanitizer_common/sanitizer_posix.h
@@ -14,6 +14,7 @@
 // ----------- ATTENTION -------------
 // This header should NOT include any other headers from sanitizer runtime.
 #include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_platform_limits_posix.h"
 
 #if !SANITIZER_POSIX
@@ -85,6 +86,9 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum);
 
 uptr internal_execve(const char *filename, char *const argv[],
                      char *const envp[]);
+
+bool IsStateDetached(int state);
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_POSIX_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc
index 335aad1660e..1a37118c299 100644
--- a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc
@@ -16,6 +16,7 @@
 
 #include "sanitizer_common.h"
 #include "sanitizer_flags.h"
+#include "sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_platform_limits_posix.h"
 #include "sanitizer_posix.h"
 #include "sanitizer_procmaps.h"
@@ -54,8 +55,12 @@ uptr GetThreadSelf() {
   return (uptr)pthread_self();
 }
 
-void ReleaseMemoryToOS(uptr addr, uptr size) {
-  madvise((void*)addr, size, MADV_DONTNEED);
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
+  uptr page_size = GetPageSizeCached();
+  uptr beg_aligned = RoundUpTo(beg, page_size);
+  uptr end_aligned = RoundDownTo(end, page_size);
+  if (beg_aligned < end_aligned)
+    madvise((void*)beg_aligned, end_aligned - beg_aligned, MADV_DONTNEED);
 }
 
 void NoHugePagesInRegion(uptr addr, uptr size) {
@@ -128,7 +133,8 @@ void SleepForMillis(int millis) {
 void Abort() {
 #if !SANITIZER_GO
   // If we are handling SIGABRT, unhandle it first.
-  if (IsHandledDeadlySignal(SIGABRT)) {
+  // TODO(vitalybuka): Check if handler belongs to sanitizer.
+  if (GetHandleSignalMode(SIGABRT) != kHandleSignalNo) {
     struct sigaction sigact;
     internal_memset(&sigact, 0, sizeof(sigact));
     sigact.sa_sigaction = (sa_sigaction_t)SIG_DFL;
@@ -182,8 +188,8 @@ void UnsetAlternateSignalStack() {
 
 static void MaybeInstallSigaction(int signum,
                                   SignalHandlerType handler) {
-  if (!IsHandledDeadlySignal(signum))
-    return;
+  if (GetHandleSignalMode(signum) == kHandleSignalNo) return;
+
   struct sigaction sigact;
   internal_memset(&sigact, 0, sizeof(sigact));
   sigact.sa_sigaction = (sa_sigaction_t)handler;
@@ -206,6 +212,53 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) {
   MaybeInstallSigaction(SIGFPE, handler);
   MaybeInstallSigaction(SIGILL, handler);
 }
+
+bool SignalContext::IsStackOverflow() const {
+  // Access at a reasonable offset above SP, or slightly below it (to account
+  // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
+  // probably a stack overflow.
+#ifdef __s390__
+  // On s390, the fault address in siginfo points to start of the page, not
+  // to the precise word that was accessed.  Mask off the low bits of sp to
+  // take it into account.
+  bool IsStackAccess = addr >= (sp & ~0xFFF) && addr < sp + 0xFFFF;
+#else
+  bool IsStackAccess = addr + 512 > sp && addr < sp + 0xFFFF;
+#endif
+
+#if __powerpc__
+  // Large stack frames can be allocated with e.g.
+  //   lis r0,-10000
+  //   stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
+  // If the store faults then sp will not have been updated, so test above
+  // will not work, because the fault address will be more than just "slightly"
+  // below sp.
+  if (!IsStackAccess && IsAccessibleMemoryRange(pc, 4)) {
+    u32 inst = *(unsigned *)pc;
+    u32 ra = (inst >> 16) & 0x1F;
+    u32 opcd = inst >> 26;
+    u32 xo = (inst >> 1) & 0x3FF;
+    // Check for store-with-update to sp. The instructions we accept are:
+    //   stbu rs,d(ra)          stbux rs,ra,rb
+    //   sthu rs,d(ra)          sthux rs,ra,rb
+    //   stwu rs,d(ra)          stwux rs,ra,rb
+    //   stdu rs,ds(ra)         stdux rs,ra,rb
+    // where ra is r1 (the stack pointer).
+    if (ra == 1 &&
+        (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 ||
+         (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181))))
+      IsStackAccess = true;
+  }
+#endif  // __powerpc__
+
+  // We also check si_code to filter out SEGV caused by something else other
+  // then hitting the guard page or unmapped memory, like, for example,
+  // unaligned memory access.
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  return IsStackAccess &&
+         (si->si_code == si_SEGV_MAPERR || si->si_code == si_SEGV_ACCERR);
+}
+
 #endif  // SANITIZER_GO
 
 bool IsAccessibleMemoryRange(uptr beg, uptr size) {
@@ -239,7 +292,6 @@ void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
   // Same for /proc/self/exe in the symbolizer.
 #if !SANITIZER_GO
   Symbolizer::GetOrInit()->PrepareForSandboxing();
-  CovPrepareForSandboxing(args);
 #endif
 }
 
@@ -412,6 +464,10 @@ int WaitForProcess(pid_t pid) {
   return process_status;
 }
 
+bool IsStateDetached(int state) {
+  return state == PTHREAD_CREATE_DETACHED;
+}
+
 } // namespace __sanitizer
 
 #endif // SANITIZER_POSIX
diff --git a/libsanitizer/sanitizer_common/sanitizer_printf.cc b/libsanitizer/sanitizer_common/sanitizer_printf.cc
index c11113da244..1456c765b2c 100644
--- a/libsanitizer/sanitizer_common/sanitizer_printf.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_printf.cc
@@ -26,8 +26,6 @@
 
 namespace __sanitizer {
 
-StaticSpinMutex CommonSanitizerReportMutex;
-
 static int AppendChar(char **buff, const char *buff_end, char c) {
   if (*buff < buff_end) {
     **buff = c;
@@ -41,7 +39,7 @@ static int AppendChar(char **buff, const char *buff_end, char c) {
 // on the value of |pad_with_zero|.
 static int AppendNumber(char **buff, const char *buff_end, u64 absolute_value,
                         u8 base, u8 minimal_num_length, bool pad_with_zero,
-                        bool negative) {
+                        bool negative, bool uppercase) {
   uptr const kMaxLen = 30;
   RAW_CHECK(base == 10 || base == 16);
   RAW_CHECK(base == 10 || !negative);
@@ -74,23 +72,25 @@ static int AppendNumber(char **buff, const char *buff_end, u64 absolute_value,
   if (negative && !pad_with_zero) result += AppendChar(buff, buff_end, '-');
   for (; pos >= 0; pos--) {
     char digit = static_cast<char>(num_buffer[pos]);
-    result += AppendChar(buff, buff_end, (digit < 10) ? '0' + digit
-                                                      : 'a' + digit - 10);
+    digit = (digit < 10) ? '0' + digit : (uppercase ? 'A' : 'a') + digit - 10;
+    result += AppendChar(buff, buff_end, digit);
   }
   return result;
 }
 
 static int AppendUnsigned(char **buff, const char *buff_end, u64 num, u8 base,
-                          u8 minimal_num_length, bool pad_with_zero) {
+                          u8 minimal_num_length, bool pad_with_zero,
+                          bool uppercase) {
   return AppendNumber(buff, buff_end, num, base, minimal_num_length,
-                      pad_with_zero, false /* negative */);
+                      pad_with_zero, false /* negative */, uppercase);
 }
 
 static int AppendSignedDecimal(char **buff, const char *buff_end, s64 num,
                                u8 minimal_num_length, bool pad_with_zero) {
   bool negative = (num < 0);
   return AppendNumber(buff, buff_end, (u64)(negative ? -num : num), 10,
-                      minimal_num_length, pad_with_zero, negative);
+                      minimal_num_length, pad_with_zero, negative,
+                      false /* uppercase */);
 }
 
 static int AppendString(char **buff, const char *buff_end, int precision,
@@ -110,14 +110,16 @@ static int AppendPointer(char **buff, const char *buff_end, u64 ptr_value) {
   int result = 0;
   result += AppendString(buff, buff_end, -1, "0x");
   result += AppendUnsigned(buff, buff_end, ptr_value, 16,
-                           SANITIZER_POINTER_FORMAT_LENGTH, true);
+                           SANITIZER_POINTER_FORMAT_LENGTH,
+                           true /* pad_with_zero */, false /* uppercase */);
   return result;
 }
 
 int VSNPrintf(char *buff, int buff_length,
               const char *format, va_list args) {
   static const char *kPrintfFormatsHelp =
-    "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x}; %p; %(\\.\\*)?s; %c\n";
+      "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; %(\\.\\*)?s; "
+      "%c\n";
   RAW_CHECK(format);
   RAW_CHECK(buff_length > 0);
   const char *buff_end = &buff[buff_length - 1];
@@ -162,12 +164,14 @@ int VSNPrintf(char *buff, int buff_length,
         break;
       }
       case 'u':
-      case 'x': {
+      case 'x':
+      case 'X': {
         uval = have_ll ? va_arg(args, u64)
              : have_z ? va_arg(args, uptr)
              : va_arg(args, unsigned);
-        result += AppendUnsigned(&buff, buff_end, uval,
-                                 (*cur == 'u') ? 10 : 16, width, pad_with_zero);
+        bool uppercase = (*cur == 'X');
+        result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16,
+                                 width, pad_with_zero, uppercase);
         break;
       }
       case 'p': {
@@ -206,15 +210,11 @@ void SetPrintfAndReportCallback(void (*callback)(const char *)) {
 }
 
 // Can be overriden in frontend.
-#if SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void OnPrint(const char *str) {
-  (void)str;
-}
-#elif SANITIZER_GO && defined(TSAN_EXTERNAL_HOOKS)
-void OnPrint(const char *str);
+#if SANITIZER_GO && defined(TSAN_EXTERNAL_HOOKS)
+// Implementation must be defined in frontend.
+extern "C" void OnPrint(const char *str);
 #else
-void OnPrint(const char *str) {
+SANITIZER_INTERFACE_WEAK_DEF(void, OnPrint, const char *str) {
   (void)str;
 }
 #endif
@@ -225,19 +225,16 @@ static void CallPrintfAndReportCallback(const char *str) {
     PrintfAndReportCallback(str);
 }
 
-static void SharedPrintfCode(bool append_pid, const char *format,
-                             va_list args) {
+static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid,
+                                              char *local_buffer,
+                                              int buffer_size,
+                                              const char *format,
+                                              va_list args) {
   va_list args2;
   va_copy(args2, args);
   const int kLen = 16 * 1024;
-  // |local_buffer| is small enough not to overflow the stack and/or violate
-  // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other
-  // hand, the bigger the buffer is, the more the chance the error report will
-  // fit into it.
-  char local_buffer[400];
   int needed_length;
   char *buffer = local_buffer;
-  int buffer_size = ARRAY_SIZE(local_buffer);
   // First try to print a message using a local buffer, and then fall back to
   // mmaped buffer.
   for (int use_mmap = 0; use_mmap < 2; use_mmap++) {
@@ -255,7 +252,9 @@ static void SharedPrintfCode(bool append_pid, const char *format,
         RAW_CHECK_MSG(needed_length < kLen, \
                       "Buffer in Report is too short!\n"); \
       }
-    if (append_pid) {
+    // Fuchsia's logging infrastructure always keeps track of the logging
+    // process, thread, and timestamp, so never prepend such information.
+    if (!SANITIZER_FUCHSIA && append_pid) {
       int pid = internal_getpid();
       const char *exe_name = GetProcessName();
       if (common_flags()->log_exe_name && exe_name) {
@@ -263,9 +262,8 @@ static void SharedPrintfCode(bool append_pid, const char *format,
                                            "==%s", exe_name);
         CHECK_NEEDED_LENGTH
       }
-      needed_length += internal_snprintf(buffer + needed_length,
-                                         buffer_size - needed_length,
-                                         "==%d==", pid);
+      needed_length += internal_snprintf(
+          buffer + needed_length, buffer_size - needed_length, "==%d==", pid);
       CHECK_NEEDED_LENGTH
     }
     needed_length += VSNPrintf(buffer + needed_length,
@@ -288,6 +286,17 @@ static void SharedPrintfCode(bool append_pid, const char *format,
   va_end(args2);
 }
 
+static void NOINLINE SharedPrintfCode(bool append_pid, const char *format,
+                                      va_list args) {
+  // |local_buffer| is small enough not to overflow the stack and/or violate
+  // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other
+  // hand, the bigger the buffer is, the more the chance the error report will
+  // fit into it.
+  char local_buffer[400];
+  SharedPrintfCodeNoBuffer(append_pid, local_buffer, ARRAY_SIZE(local_buffer),
+                           format, args);
+}
+
 FORMAT(1, 2)
 void Printf(const char *format, ...) {
   va_list args;
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps.h b/libsanitizer/sanitizer_common/sanitizer_procmaps.h
index 0183a094111..040f6940f17 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps.h
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps.h
@@ -12,29 +12,61 @@
 #ifndef SANITIZER_PROCMAPS_H
 #define SANITIZER_PROCMAPS_H
 
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
+
 #include "sanitizer_common.h"
 #include "sanitizer_internal_defs.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mac.h"
 #include "sanitizer_mutex.h"
 
 namespace __sanitizer {
 
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
-struct ProcSelfMapsBuff {
-  char *data;
-  uptr mmaped_size;
-  uptr len;
-};
 
-// Reads process memory map in an OS-specific way.
-void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
-#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX
+// Memory protection masks.
+static const uptr kProtectionRead = 1;
+static const uptr kProtectionWrite = 2;
+static const uptr kProtectionExecute = 4;
+static const uptr kProtectionShared = 8;
+
+struct MemoryMappedSegmentData;
+
+class MemoryMappedSegment {
+ public:
+  MemoryMappedSegment(char *buff = nullptr, uptr size = 0)
+      : filename(buff), filename_size(size), data_(nullptr) {}
+  ~MemoryMappedSegment() {}
+
+  bool IsReadable() const { return protection & kProtectionRead; }
+  bool IsWritable() const { return protection & kProtectionWrite; }
+  bool IsExecutable() const { return protection & kProtectionExecute; }
+  bool IsShared() const { return protection & kProtectionShared; }
+
+  void AddAddressRanges(LoadedModule *module);
+
+  uptr start;
+  uptr end;
+  uptr offset;
+  char *filename;  // owned by caller
+  uptr filename_size;
+  uptr protection;
+  ModuleArch arch;
+  u8 uuid[kModuleUUIDSize];
+
+ private:
+  friend class MemoryMappingLayout;
+
+  // This field is assigned and owned by MemoryMappingLayout if needed
+  MemoryMappedSegmentData *data_;
+};
 
 class MemoryMappingLayout {
  public:
   explicit MemoryMappingLayout(bool cache_enabled);
   ~MemoryMappingLayout();
-  bool Next(uptr *start, uptr *end, uptr *offset,
-            char filename[], uptr filename_size, uptr *protection);
+  bool Next(MemoryMappedSegment *segment);
   void Reset();
   // In some cases, e.g. when running under a sandbox on Linux, ASan is unable
   // to obtain the memory mappings. It should fall back to pre-cached data
@@ -42,47 +74,14 @@ class MemoryMappingLayout {
   static void CacheMemoryMappings();
 
   // Adds all mapped objects into a vector.
-  void DumpListOfModules(InternalMmapVector<LoadedModule> *modules);
-
-  // Memory protection masks.
-  static const uptr kProtectionRead = 1;
-  static const uptr kProtectionWrite = 2;
-  static const uptr kProtectionExecute = 4;
-  static const uptr kProtectionShared = 8;
+  void DumpListOfModules(InternalMmapVectorNoCtor<LoadedModule> *modules);
 
  private:
   void LoadFromCache();
 
-  // FIXME: Hide implementation details for different platforms in
-  // platform-specific files.
-# if SANITIZER_FREEBSD || SANITIZER_LINUX
-  ProcSelfMapsBuff proc_self_maps_;
-  const char *current_;
-
-  // Static mappings cache.
-  static ProcSelfMapsBuff cached_proc_self_maps_;
-  static StaticSpinMutex cache_lock_;  // protects cached_proc_self_maps_.
-# elif SANITIZER_MAC
-  template<u32 kLCSegment, typename SegmentCommand>
-  bool NextSegmentLoad(uptr *start, uptr *end, uptr *offset,
-                       char filename[], uptr filename_size,
-                       uptr *protection);
-  int current_image_;
-  u32 current_magic_;
-  u32 current_filetype_;
-  int current_load_cmd_count_;
-  char *current_load_cmd_addr_;
-# endif
+  MemoryMappingLayoutData data_;
 };
 
-typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
-                               /*out*/uptr *stats, uptr stats_size);
-
-// Parse the contents of /proc/self/smaps and generate a memory profile.
-// |cb| is a tool-specific callback that fills the |stats| array containing
-// |stats_size| elements.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
-
 // Returns code range for the specified module.
 bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end);
 
@@ -93,4 +92,6 @@ uptr ParseHex(const char **p);
 
 }  // namespace __sanitizer
 
+#endif  // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+        // SANITIZER_MAC
 #endif  // SANITIZER_PROCMAPS_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc b/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc
index c725c2e7b66..36b97b1166e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc
@@ -10,7 +10,7 @@
 
 #include "sanitizer_platform.h"
 
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 
 #include "sanitizer_common.h"
 #include "sanitizer_placement_new.h"
@@ -18,9 +18,8 @@
 
 namespace __sanitizer {
 
-// Linker initialized.
-ProcSelfMapsBuff MemoryMappingLayout::cached_proc_self_maps_;
-StaticSpinMutex MemoryMappingLayout::cache_lock_;  // Linker initialized.
+static ProcSelfMapsBuff cached_proc_self_maps;
+static StaticSpinMutex cache_lock;
 
 static int TranslateDigit(char c) {
   if (c >= '0' && c <= '9')
@@ -62,15 +61,21 @@ uptr ParseHex(const char **p) {
   return ParseNumber(p, 16);
 }
 
+void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
+  // data_ should be unused on this platform
+  CHECK(!data_);
+  module->addAddressRange(start, end, IsExecutable(), IsWritable());
+}
+
 MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
-  ReadProcMaps(&proc_self_maps_);
+  ReadProcMaps(&data_.proc_self_maps);
   if (cache_enabled) {
-    if (proc_self_maps_.mmaped_size == 0) {
+    if (data_.proc_self_maps.mmaped_size == 0) {
       LoadFromCache();
-      CHECK_GT(proc_self_maps_.len, 0);
+      CHECK_GT(data_.proc_self_maps.len, 0);
     }
   } else {
-    CHECK_GT(proc_self_maps_.mmaped_size, 0);
+    CHECK_GT(data_.proc_self_maps.mmaped_size, 0);
   }
   Reset();
   // FIXME: in the future we may want to cache the mappings on demand only.
@@ -81,24 +86,22 @@ MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
 MemoryMappingLayout::~MemoryMappingLayout() {
   // Only unmap the buffer if it is different from the cached one. Otherwise
   // it will be unmapped when the cache is refreshed.
-  if (proc_self_maps_.data != cached_proc_self_maps_.data) {
-    UnmapOrDie(proc_self_maps_.data, proc_self_maps_.mmaped_size);
+  if (data_.proc_self_maps.data != cached_proc_self_maps.data) {
+    UnmapOrDie(data_.proc_self_maps.data, data_.proc_self_maps.mmaped_size);
   }
 }
 
-void MemoryMappingLayout::Reset() {
-  current_ = proc_self_maps_.data;
-}
+void MemoryMappingLayout::Reset() { data_.current = data_.proc_self_maps.data; }
 
 // static
 void MemoryMappingLayout::CacheMemoryMappings() {
-  SpinMutexLock l(&cache_lock_);
+  SpinMutexLock l(&cache_lock);
   // Don't invalidate the cache if the mappings are unavailable.
   ProcSelfMapsBuff old_proc_self_maps;
-  old_proc_self_maps = cached_proc_self_maps_;
-  ReadProcMaps(&cached_proc_self_maps_);
-  if (cached_proc_self_maps_.mmaped_size == 0) {
-    cached_proc_self_maps_ = old_proc_self_maps;
+  old_proc_self_maps = cached_proc_self_maps;
+  ReadProcMaps(&cached_proc_self_maps);
+  if (cached_proc_self_maps.mmaped_size == 0) {
+    cached_proc_self_maps = old_proc_self_maps;
   } else {
     if (old_proc_self_maps.mmaped_size) {
       UnmapOrDie(old_proc_self_maps.data,
@@ -108,21 +111,19 @@ void MemoryMappingLayout::CacheMemoryMappings() {
 }
 
 void MemoryMappingLayout::LoadFromCache() {
-  SpinMutexLock l(&cache_lock_);
-  if (cached_proc_self_maps_.data) {
-    proc_self_maps_ = cached_proc_self_maps_;
+  SpinMutexLock l(&cache_lock);
+  if (cached_proc_self_maps.data) {
+    data_.proc_self_maps = cached_proc_self_maps;
   }
 }
 
 void MemoryMappingLayout::DumpListOfModules(
-    InternalMmapVector<LoadedModule> *modules) {
+    InternalMmapVectorNoCtor<LoadedModule> *modules) {
   Reset();
-  uptr cur_beg, cur_end, cur_offset, prot;
   InternalScopedString module_name(kMaxPathLength);
-  for (uptr i = 0; Next(&cur_beg, &cur_end, &cur_offset, module_name.data(),
-                        module_name.size(), &prot);
-       i++) {
-    const char *cur_name = module_name.data();
+  MemoryMappedSegment segment(module_name.data(), module_name.size());
+  for (uptr i = 0; Next(&segment); i++) {
+    const char *cur_name = segment.filename;
     if (cur_name[0] == '\0')
       continue;
     // Don't subtract 'cur_beg' from the first entry:
@@ -136,10 +137,10 @@ void MemoryMappingLayout::DumpListOfModules(
     //   mapped high at address space (in particular, higher than
     //   shadow memory of the tool), so the module can't be the
     //   first entry.
-    uptr base_address = (i ? cur_beg : 0) - cur_offset;
+    uptr base_address = (i ? segment.start : 0) - segment.offset;
     LoadedModule cur_module;
     cur_module.set(cur_name, base_address);
-    cur_module.addAddressRange(cur_beg, cur_end, prot & kProtectionExecute);
+    segment.AddAddressRanges(&cur_module);
     modules->push_back(cur_module);
   }
 }
@@ -170,4 +171,4 @@ void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {
 
 } // namespace __sanitizer
 
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_freebsd.cc b/libsanitizer/sanitizer_common/sanitizer_procmaps_freebsd.cc
index fbc55203ab4..ba5d1449c0e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps_freebsd.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_freebsd.cc
@@ -5,18 +5,22 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Information about the process mappings (FreeBSD-specific parts).
+// Information about the process mappings (FreeBSD and NetBSD-specific parts).
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
 #include "sanitizer_common.h"
+#if SANITIZER_FREEBSD
 #include "sanitizer_freebsd.h"
+#endif
 #include "sanitizer_procmaps.h"
 
 #include <unistd.h>
 #include <sys/sysctl.h>
+#if SANITIZER_FREEBSD
 #include <sys/user.h>
+#endif
 
 // Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode.
 #if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
@@ -29,16 +33,30 @@
 namespace __sanitizer {
 
 void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
-  const int Mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid() };
+  const int Mib[] = {
+#if SANITIZER_FREEBSD
+    CTL_KERN,
+    KERN_PROC,
+    KERN_PROC_VMMAP,
+    getpid()
+#else
+    CTL_VM,
+    VM_PROC,
+    VM_PROC_MAP,
+    getpid(),
+    sizeof(struct kinfo_vmentry)
+#endif
+  };
+
   size_t Size = 0;
-  int Err = sysctl(Mib, 4, NULL, &Size, NULL, 0);
+  int Err = sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0);
   CHECK_EQ(Err, 0);
   CHECK_GT(Size, 0);
 
   size_t MmapedSize = Size * 4 / 3;
   void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()");
   Size = MmapedSize;
-  Err = sysctl(Mib, 4, VmMap, &Size, NULL, 0);
+  Err = sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0);
   CHECK_EQ(Err, 0);
 
   proc_maps->data = (char*)VmMap;
@@ -46,41 +64,38 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
   proc_maps->len = Size;
 }
 
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
-                               char filename[], uptr filename_size,
-                               uptr *protection) {
-  char *last = proc_self_maps_.data + proc_self_maps_.len;
-  if (current_ >= last) return false;
-  uptr dummy;
-  if (!start) start = &dummy;
-  if (!end) end = &dummy;
-  if (!offset) offset = &dummy;
-  if (!protection) protection = &dummy;
-  struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry*)current_;
-
-  *start = (uptr)VmEntry->kve_start;
-  *end = (uptr)VmEntry->kve_end;
-  *offset = (uptr)VmEntry->kve_offset;
-
-  *protection = 0;
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+  if (data_.current >= last) return false;
+  struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry *)data_.current;
+
+  segment->start = (uptr)VmEntry->kve_start;
+  segment->end = (uptr)VmEntry->kve_end;
+  segment->offset = (uptr)VmEntry->kve_offset;
+
+  segment->protection = 0;
   if ((VmEntry->kve_protection & KVME_PROT_READ) != 0)
-    *protection |= kProtectionRead;
+    segment->protection |= kProtectionRead;
   if ((VmEntry->kve_protection & KVME_PROT_WRITE) != 0)
-    *protection |= kProtectionWrite;
+    segment->protection |= kProtectionWrite;
   if ((VmEntry->kve_protection & KVME_PROT_EXEC) != 0)
-    *protection |= kProtectionExecute;
+    segment->protection |= kProtectionExecute;
 
-  if (filename != NULL && filename_size > 0) {
-    internal_snprintf(filename,
-                      Min(filename_size, (uptr)PATH_MAX),
-                      "%s", VmEntry->kve_path);
+  if (segment->filename != NULL && segment->filename_size > 0) {
+    internal_snprintf(segment->filename,
+                      Min(segment->filename_size, (uptr)PATH_MAX), "%s",
+                      VmEntry->kve_path);
   }
 
-  current_ += VmEntry->kve_structsize;
+#if SANITIZER_FREEBSD
+  data_.current += VmEntry->kve_structsize;
+#else
+  data_.current += sizeof(*VmEntry);
+#endif
 
   return true;
 }
 
 }  // namespace __sanitizer
 
-#endif  // SANITIZER_FREEBSD
+#endif  // SANITIZER_FREEBSD || SANITIZER_NETBSD
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc b/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc
index 10918e54398..b97d5f62dd2 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc
@@ -16,70 +16,57 @@
 namespace __sanitizer {
 
 void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
-  CHECK(ReadFileToBuffer("/proc/self/maps", &proc_maps->data,
-                         &proc_maps->mmaped_size, &proc_maps->len));
+  ReadFileToBuffer("/proc/self/maps", &proc_maps->data, &proc_maps->mmaped_size,
+                   &proc_maps->len);
 }
 
 static bool IsOneOf(char c, char c1, char c2) {
   return c == c1 || c == c2;
 }
 
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
-                               char filename[], uptr filename_size,
-                               uptr *protection) {
-  char *last = proc_self_maps_.data + proc_self_maps_.len;
-  if (current_ >= last) return false;
-  uptr dummy;
-  if (!start) start = &dummy;
-  if (!end) end = &dummy;
-  if (!offset) offset = &dummy;
-  if (!protection) protection = &dummy;
-  char *next_line = (char*)internal_memchr(current_, '\n', last - current_);
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+  if (data_.current >= last) return false;
+  char *next_line =
+      (char *)internal_memchr(data_.current, '\n', last - data_.current);
   if (next_line == 0)
     next_line = last;
   // Example: 08048000-08056000 r-xp 00000000 03:0c 64593   /foo/bar
-  *start = ParseHex(&current_);
-  CHECK_EQ(*current_++, '-');
-  *end = ParseHex(&current_);
-  CHECK_EQ(*current_++, ' ');
-  CHECK(IsOneOf(*current_, '-', 'r'));
-  *protection = 0;
-  if (*current_++ == 'r')
-    *protection |= kProtectionRead;
-  CHECK(IsOneOf(*current_, '-', 'w'));
-  if (*current_++ == 'w')
-    *protection |= kProtectionWrite;
-  CHECK(IsOneOf(*current_, '-', 'x'));
-  if (*current_++ == 'x')
-    *protection |= kProtectionExecute;
-  CHECK(IsOneOf(*current_, 's', 'p'));
-  if (*current_++ == 's')
-    *protection |= kProtectionShared;
-  CHECK_EQ(*current_++, ' ');
-  *offset = ParseHex(&current_);
-  CHECK_EQ(*current_++, ' ');
-  ParseHex(&current_);
-  CHECK_EQ(*current_++, ':');
-  ParseHex(&current_);
-  CHECK_EQ(*current_++, ' ');
-  while (IsDecimal(*current_))
-    current_++;
+  segment->start = ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, '-');
+  segment->end = ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ' ');
+  CHECK(IsOneOf(*data_.current, '-', 'r'));
+  segment->protection = 0;
+  if (*data_.current++ == 'r') segment->protection |= kProtectionRead;
+  CHECK(IsOneOf(*data_.current, '-', 'w'));
+  if (*data_.current++ == 'w') segment->protection |= kProtectionWrite;
+  CHECK(IsOneOf(*data_.current, '-', 'x'));
+  if (*data_.current++ == 'x') segment->protection |= kProtectionExecute;
+  CHECK(IsOneOf(*data_.current, 's', 'p'));
+  if (*data_.current++ == 's') segment->protection |= kProtectionShared;
+  CHECK_EQ(*data_.current++, ' ');
+  segment->offset = ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ' ');
+  ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ':');
+  ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ' ');
+  while (IsDecimal(*data_.current)) data_.current++;
   // Qemu may lack the trailing space.
   // https://github.com/google/sanitizers/issues/160
-  // CHECK_EQ(*current_++, ' ');
+  // CHECK_EQ(*data_.current++, ' ');
   // Skip spaces.
-  while (current_ < next_line && *current_ == ' ')
-    current_++;
+  while (data_.current < next_line && *data_.current == ' ') data_.current++;
   // Fill in the filename.
-  uptr i = 0;
-  while (current_ < next_line) {
-    if (filename && i < filename_size - 1)
-      filename[i++] = *current_;
-    current_++;
+  if (segment->filename) {
+    uptr len =
+        Min((uptr)(next_line - data_.current), segment->filename_size - 1);
+    internal_strncpy(segment->filename, data_.current, len);
+    segment->filename[len] = 0;
   }
-  if (filename && i < filename_size)
-    filename[i] = 0;
-  current_ = next_line + 1;
+
+  data_.current = next_line + 1;
   return true;
 }
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc
index 81829a7c284..34f0c207b09 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc
@@ -16,9 +16,69 @@
 
 #include <mach-o/dyld.h>
 #include <mach-o/loader.h>
+#include <mach/mach.h>
+
+// These are not available in older macOS SDKs.
+#ifndef CPU_SUBTYPE_X86_64_H
+#define CPU_SUBTYPE_X86_64_H  ((cpu_subtype_t)8)   /* Haswell */
+#endif
+#ifndef CPU_SUBTYPE_ARM_V7S
+#define CPU_SUBTYPE_ARM_V7S   ((cpu_subtype_t)11)  /* Swift */
+#endif
+#ifndef CPU_SUBTYPE_ARM_V7K
+#define CPU_SUBTYPE_ARM_V7K   ((cpu_subtype_t)12)
+#endif
+#ifndef CPU_TYPE_ARM64
+#define CPU_TYPE_ARM64        (CPU_TYPE_ARM | CPU_ARCH_ABI64)
+#endif
 
 namespace __sanitizer {
 
+// Contains information used to iterate through sections.
+struct MemoryMappedSegmentData {
+  char name[kMaxSegName];
+  uptr nsects;
+  char *current_load_cmd_addr;
+  u32 lc_type;
+  uptr base_virt_addr;
+  uptr addr_mask;
+};
+
+template <typename Section>
+static void NextSectionLoad(LoadedModule *module, MemoryMappedSegmentData *data,
+                            bool isWritable) {
+  const Section *sc = (const Section *)data->current_load_cmd_addr;
+  data->current_load_cmd_addr += sizeof(Section);
+
+  uptr sec_start = (sc->addr & data->addr_mask) + data->base_virt_addr;
+  uptr sec_end = sec_start + sc->size;
+  module->addAddressRange(sec_start, sec_end, /*executable=*/false, isWritable,
+                          sc->sectname);
+}
+
+void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
+  // Don't iterate over sections when the caller hasn't set up the
+  // data pointer, when there are no sections, or when the segment
+  // is executable. Avoid iterating over executable sections because
+  // it will confuse libignore, and because the extra granularity
+  // of information is not needed by any sanitizers.
+  if (!data_ || !data_->nsects || IsExecutable()) {
+    module->addAddressRange(start, end, IsExecutable(), IsWritable(),
+                            data_ ? data_->name : nullptr);
+    return;
+  }
+
+  do {
+    if (data_->lc_type == LC_SEGMENT) {
+      NextSectionLoad<struct section>(module, data_, IsWritable());
+#ifdef MH_MAGIC_64
+    } else if (data_->lc_type == LC_SEGMENT_64) {
+      NextSectionLoad<struct section_64>(module, data_, IsWritable());
+#endif
+    }
+  } while (--data_->nsects);
+}
+
 MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
   Reset();
 }
@@ -46,13 +106,22 @@ void MemoryMappingLayout::Reset() {
   // _dyld_image_count is thread-unsafe. We need to register callbacks for
   // adding and removing images which will invalidate the MemoryMappingLayout
   // state.
-  current_image_ = _dyld_image_count();
-  current_load_cmd_count_ = -1;
-  current_load_cmd_addr_ = 0;
-  current_magic_ = 0;
-  current_filetype_ = 0;
+  data_.current_image = _dyld_image_count();
+  data_.current_load_cmd_count = -1;
+  data_.current_load_cmd_addr = 0;
+  data_.current_magic = 0;
+  data_.current_filetype = 0;
+  data_.current_arch = kModuleArchUnknown;
+  internal_memset(data_.current_uuid, 0, kModuleUUIDSize);
 }
 
+// The dyld load address should be unchanged throughout process execution,
+// and it is expensive to compute once many libraries have been loaded,
+// so cache it here and do not reset.
+static mach_header *dyld_hdr = 0;
+static const char kDyldPath[] = "/usr/lib/dyld";
+static const int kDyldImageIdx = -1;
+
 // static
 void MemoryMappingLayout::CacheMemoryMappings() {
   // No-op on Mac for now.
@@ -62,6 +131,48 @@ void MemoryMappingLayout::LoadFromCache() {
   // No-op on Mac for now.
 }
 
+// _dyld_get_image_header() and related APIs don't report dyld itself.
+// We work around this by manually recursing through the memory map
+// until we hit a Mach header matching dyld instead. These recurse
+// calls are expensive, but the first memory map generation occurs
+// early in the process, when dyld is one of the only images loaded,
+// so it will be hit after only a few iterations.
+static mach_header *get_dyld_image_header() {
+  mach_port_name_t port;
+  if (task_for_pid(mach_task_self(), internal_getpid(), &port) !=
+      KERN_SUCCESS) {
+    return nullptr;
+  }
+
+  unsigned depth = 1;
+  vm_size_t size = 0;
+  vm_address_t address = 0;
+  kern_return_t err = KERN_SUCCESS;
+  mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
+
+  while (true) {
+    struct vm_region_submap_info_64 info;
+    err = vm_region_recurse_64(port, &address, &size, &depth,
+                               (vm_region_info_t)&info, &count);
+    if (err != KERN_SUCCESS) return nullptr;
+
+    if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {
+      mach_header *hdr = (mach_header *)address;
+      if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
+          hdr->filetype == MH_DYLINKER) {
+        return hdr;
+      }
+    }
+    address += size;
+  }
+}
+
+const mach_header *get_dyld_hdr() {
+  if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
+
+  return dyld_hdr;
+}
+
 // Next and NextSegmentLoad were inspired by base/sysinfo.cc in
 // Google Perftools, https://github.com/gperftools/gperftools.
 
@@ -69,79 +180,165 @@ void MemoryMappingLayout::LoadFromCache() {
 // and returns the start and end addresses and file offset of the corresponding
 // segment.
 // Note that the segment addresses are not necessarily sorted.
-template<u32 kLCSegment, typename SegmentCommand>
-bool MemoryMappingLayout::NextSegmentLoad(
-    uptr *start, uptr *end, uptr *offset,
-    char filename[], uptr filename_size, uptr *protection) {
-  const char* lc = current_load_cmd_addr_;
-  current_load_cmd_addr_ += ((const load_command *)lc)->cmdsize;
+template <u32 kLCSegment, typename SegmentCommand>
+static bool NextSegmentLoad(MemoryMappedSegment *segment,
+MemoryMappedSegmentData *seg_data, MemoryMappingLayoutData &layout_data) {
+  const char *lc = layout_data.current_load_cmd_addr;
+  layout_data.current_load_cmd_addr += ((const load_command *)lc)->cmdsize;
   if (((const load_command *)lc)->cmd == kLCSegment) {
-    const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_);
     const SegmentCommand* sc = (const SegmentCommand *)lc;
-    if (start) *start = sc->vmaddr + dlloff;
-    if (protection) {
-      // Return the initial protection.
-      *protection = sc->initprot;
+    uptr base_virt_addr, addr_mask;
+    if (layout_data.current_image == kDyldImageIdx) {
+      base_virt_addr = (uptr)get_dyld_hdr();
+      // vmaddr is masked with 0xfffff because on macOS versions < 10.12,
+      // it contains an absolute address rather than an offset for dyld.
+      // To make matters even more complicated, this absolute address
+      // isn't actually the absolute segment address, but the offset portion
+      // of the address is accurate when combined with the dyld base address,
+      // and the mask will give just this offset.
+      addr_mask = 0xfffff;
+    } else {
+      base_virt_addr =
+          (uptr)_dyld_get_image_vmaddr_slide(layout_data.current_image);
+      addr_mask = ~0;
     }
-    if (end) *end = sc->vmaddr + sc->vmsize + dlloff;
-    if (offset) {
-      if (current_filetype_ == /*MH_EXECUTE*/ 0x2) {
-        *offset = sc->vmaddr;
-      } else {
-        *offset = sc->fileoff;
-      }
+
+    segment->start = (sc->vmaddr & addr_mask) + base_virt_addr;
+    segment->end = segment->start + sc->vmsize;
+    // Most callers don't need section information, so only fill this struct
+    // when required.
+    if (seg_data) {
+      seg_data->nsects = sc->nsects;
+      seg_data->current_load_cmd_addr =
+          (char *)lc + sizeof(SegmentCommand);
+      seg_data->lc_type = kLCSegment;
+      seg_data->base_virt_addr = base_virt_addr;
+      seg_data->addr_mask = addr_mask;
+      internal_strncpy(seg_data->name, sc->segname,
+                       ARRAY_SIZE(seg_data->name));
     }
-    if (filename) {
-      internal_strncpy(filename, _dyld_get_image_name(current_image_),
-                       filename_size);
+
+    // Return the initial protection.
+    segment->protection = sc->initprot;
+    segment->offset = (layout_data.current_filetype ==
+                       /*MH_EXECUTE*/ 0x2)
+                          ? sc->vmaddr
+                          : sc->fileoff;
+    if (segment->filename) {
+      const char *src = (layout_data.current_image == kDyldImageIdx)
+                            ? kDyldPath
+                            : _dyld_get_image_name(layout_data.current_image);
+      internal_strncpy(segment->filename, src, segment->filename_size);
     }
+    segment->arch = layout_data.current_arch;
+    internal_memcpy(segment->uuid, layout_data.current_uuid, kModuleUUIDSize);
     return true;
   }
   return false;
 }
 
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
-                               char filename[], uptr filename_size,
-                               uptr *protection) {
-  for (; current_image_ >= 0; current_image_--) {
-    const mach_header* hdr = _dyld_get_image_header(current_image_);
+ModuleArch ModuleArchFromCpuType(cpu_type_t cputype, cpu_subtype_t cpusubtype) {
+  cpusubtype = cpusubtype & ~CPU_SUBTYPE_MASK;
+  switch (cputype) {
+    case CPU_TYPE_I386:
+      return kModuleArchI386;
+    case CPU_TYPE_X86_64:
+      if (cpusubtype == CPU_SUBTYPE_X86_64_ALL) return kModuleArchX86_64;
+      if (cpusubtype == CPU_SUBTYPE_X86_64_H) return kModuleArchX86_64H;
+      CHECK(0 && "Invalid subtype of x86_64");
+      return kModuleArchUnknown;
+    case CPU_TYPE_ARM:
+      if (cpusubtype == CPU_SUBTYPE_ARM_V6) return kModuleArchARMV6;
+      if (cpusubtype == CPU_SUBTYPE_ARM_V7) return kModuleArchARMV7;
+      if (cpusubtype == CPU_SUBTYPE_ARM_V7S) return kModuleArchARMV7S;
+      if (cpusubtype == CPU_SUBTYPE_ARM_V7K) return kModuleArchARMV7K;
+      CHECK(0 && "Invalid subtype of ARM");
+      return kModuleArchUnknown;
+    case CPU_TYPE_ARM64:
+      return kModuleArchARM64;
+    default:
+      CHECK(0 && "Invalid CPU type");
+      return kModuleArchUnknown;
+  }
+}
+
+static const load_command *NextCommand(const load_command *lc) {
+  return (const load_command *)((char *)lc + lc->cmdsize);
+}
+
+static void FindUUID(const load_command *first_lc, u8 *uuid_output) {
+  for (const load_command *lc = first_lc; lc->cmd != 0; lc = NextCommand(lc)) {
+    if (lc->cmd != LC_UUID) continue;
+
+    const uuid_command *uuid_lc = (const uuid_command *)lc;
+    const uint8_t *uuid = &uuid_lc->uuid[0];
+    internal_memcpy(uuid_output, uuid, kModuleUUIDSize);
+    return;
+  }
+}
+
+static bool IsModuleInstrumented(const load_command *first_lc) {
+  for (const load_command *lc = first_lc; lc->cmd != 0; lc = NextCommand(lc)) {
+    if (lc->cmd != LC_LOAD_DYLIB) continue;
+
+    const dylib_command *dylib_lc = (const dylib_command *)lc;
+    uint32_t dylib_name_offset = dylib_lc->dylib.name.offset;
+    const char *dylib_name = ((const char *)dylib_lc) + dylib_name_offset;
+    dylib_name = StripModuleName(dylib_name);
+    if (dylib_name != 0 && (internal_strstr(dylib_name, "libclang_rt."))) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  for (; data_.current_image >= kDyldImageIdx; data_.current_image--) {
+    const mach_header *hdr = (data_.current_image == kDyldImageIdx)
+                                 ? get_dyld_hdr()
+                                 : _dyld_get_image_header(data_.current_image);
     if (!hdr) continue;
-    if (current_load_cmd_count_ < 0) {
+    if (data_.current_load_cmd_count < 0) {
       // Set up for this image;
-      current_load_cmd_count_ = hdr->ncmds;
-      current_magic_ = hdr->magic;
-      current_filetype_ = hdr->filetype;
-      switch (current_magic_) {
+      data_.current_load_cmd_count = hdr->ncmds;
+      data_.current_magic = hdr->magic;
+      data_.current_filetype = hdr->filetype;
+      data_.current_arch = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype);
+      switch (data_.current_magic) {
 #ifdef MH_MAGIC_64
         case MH_MAGIC_64: {
-          current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header_64);
+          data_.current_load_cmd_addr = (char *)hdr + sizeof(mach_header_64);
           break;
         }
 #endif
         case MH_MAGIC: {
-          current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header);
+          data_.current_load_cmd_addr = (char *)hdr + sizeof(mach_header);
           break;
         }
         default: {
           continue;
         }
       }
+      FindUUID((const load_command *)data_.current_load_cmd_addr,
+               data_.current_uuid);
+      data_.current_instrumented = IsModuleInstrumented(
+          (const load_command *)data_.current_load_cmd_addr);
     }
 
-    for (; current_load_cmd_count_ >= 0; current_load_cmd_count_--) {
-      switch (current_magic_) {
-        // current_magic_ may be only one of MH_MAGIC, MH_MAGIC_64.
+    for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) {
+      switch (data_.current_magic) {
+        // data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64.
 #ifdef MH_MAGIC_64
         case MH_MAGIC_64: {
           if (NextSegmentLoad<LC_SEGMENT_64, struct segment_command_64>(
-                  start, end, offset, filename, filename_size, protection))
+          segment, segment->data_, data_))
             return true;
           break;
         }
 #endif
         case MH_MAGIC: {
           if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(
-                  start, end, offset, filename, filename_size, protection))
+          segment, segment->data_, data_))
             return true;
           break;
         }
@@ -154,26 +351,25 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
 }
 
 void MemoryMappingLayout::DumpListOfModules(
-    InternalMmapVector<LoadedModule> *modules) {
+    InternalMmapVectorNoCtor<LoadedModule> *modules) {
   Reset();
-  uptr cur_beg, cur_end, prot;
   InternalScopedString module_name(kMaxPathLength);
-  for (uptr i = 0; Next(&cur_beg, &cur_end, 0, module_name.data(),
-                        module_name.size(), &prot);
-       i++) {
-    const char *cur_name = module_name.data();
-    if (cur_name[0] == '\0')
-      continue;
+  MemoryMappedSegment segment(module_name.data(), kMaxPathLength);
+  MemoryMappedSegmentData data;
+  segment.data_ = &data;
+  while (Next(&segment)) {
+    if (segment.filename[0] == '\0') continue;
     LoadedModule *cur_module = nullptr;
     if (!modules->empty() &&
-        0 == internal_strcmp(cur_name, modules->back().full_name())) {
+        0 == internal_strcmp(segment.filename, modules->back().full_name())) {
       cur_module = &modules->back();
     } else {
       modules->push_back(LoadedModule());
       cur_module = &modules->back();
-      cur_module->set(cur_name, cur_beg);
+      cur_module->set(segment.filename, segment.start, segment.arch,
+                      segment.uuid, data_.current_instrumented);
     }
-    cur_module->addAddressRange(cur_beg, cur_end, prot & kProtectionExecute);
+    segment.AddAddressRanges(cur_module);
   }
 }
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_quarantine.h b/libsanitizer/sanitizer_common/sanitizer_quarantine.h
index 9e9268f2a5d..a90c8e3bdb8 100644
--- a/libsanitizer/sanitizer_common/sanitizer_quarantine.h
+++ b/libsanitizer/sanitizer_common/sanitizer_quarantine.h
@@ -29,6 +29,40 @@ struct QuarantineBatch {
   uptr size;
   uptr count;
   void *batch[kSize];
+
+  void init(void *ptr, uptr size) {
+    count = 1;
+    batch[0] = ptr;
+    this->size = size + sizeof(QuarantineBatch);  // Account for the batch size.
+  }
+
+  // The total size of quarantined nodes recorded in this batch.
+  uptr quarantined_size() const {
+    return size - sizeof(QuarantineBatch);
+  }
+
+  void push_back(void *ptr, uptr size) {
+    CHECK_LT(count, kSize);
+    batch[count++] = ptr;
+    this->size += size;
+  }
+
+  bool can_merge(const QuarantineBatch* const from) const {
+    return count + from->count <= kSize;
+  }
+
+  void merge(QuarantineBatch* const from) {
+    CHECK_LE(count + from->count, kSize);
+    CHECK_GE(size, sizeof(QuarantineBatch));
+
+    for (uptr i = 0; i < from->count; ++i)
+      batch[count + i] = from->batch[i];
+    count += from->count;
+    size += from->quarantined_size();
+
+    from->count = 0;
+    from->size = sizeof(QuarantineBatch);
+  }
 };
 
 COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13));  // 8Kb.
@@ -47,17 +81,31 @@ class Quarantine {
   }
 
   void Init(uptr size, uptr cache_size) {
-    atomic_store(&max_size_, size, memory_order_release);
+    // Thread local quarantine size can be zero only when global quarantine size
+    // is zero (it allows us to perform just one atomic read per Put() call).
+    CHECK((size == 0 && cache_size == 0) || cache_size != 0);
+
+    atomic_store(&max_size_, size, memory_order_relaxed);
     atomic_store(&min_size_, size / 10 * 9,
-                 memory_order_release); // 90% of max size.
-    max_cache_size_ = cache_size;
+                 memory_order_relaxed);  // 90% of max size.
+    atomic_store(&max_cache_size_, cache_size, memory_order_relaxed);
   }
 
-  uptr GetSize() const { return atomic_load(&max_size_, memory_order_acquire); }
+  uptr GetSize() const { return atomic_load(&max_size_, memory_order_relaxed); }
+  uptr GetCacheSize() const {
+    return atomic_load(&max_cache_size_, memory_order_relaxed);
+  }
 
   void Put(Cache *c, Callback cb, Node *ptr, uptr size) {
-    c->Enqueue(cb, ptr, size);
-    if (c->Size() > max_cache_size_)
+    uptr cache_size = GetCacheSize();
+    if (cache_size) {
+      c->Enqueue(cb, ptr, size);
+    } else {
+      // GetCacheSize() == 0 only when GetSize() == 0 (see Init).
+      cb.Recycle(ptr);
+    }
+    // Check cache size anyway to accommodate for runtime cache_size change.
+    if (c->Size() > cache_size)
       Drain(c, cb);
   }
 
@@ -70,12 +118,19 @@ class Quarantine {
       Recycle(cb);
   }
 
+  void PrintStats() const {
+    // It assumes that the world is stopped, just as the allocator's PrintStats.
+    Printf("Quarantine limits: global: %zdMb; thread local: %zdKb\n",
+           GetSize() >> 20, GetCacheSize() >> 10);
+    cache_.PrintStats();
+  }
+
  private:
   // Read-only data.
   char pad0_[kCacheLineSize];
   atomic_uintptr_t max_size_;
   atomic_uintptr_t min_size_;
-  uptr max_cache_size_;
+  atomic_uintptr_t max_cache_size_;
   char pad1_[kCacheLineSize];
   SpinMutex cache_mutex_;
   SpinMutex recycle_mutex_;
@@ -84,12 +139,30 @@ class Quarantine {
 
   void NOINLINE Recycle(Callback cb) {
     Cache tmp;
-    uptr min_size = atomic_load(&min_size_, memory_order_acquire);
+    uptr min_size = atomic_load(&min_size_, memory_order_relaxed);
     {
       SpinMutexLock l(&cache_mutex_);
+      // Go over the batches and merge partially filled ones to
+      // save some memory, otherwise batches themselves (since the memory used
+      // by them is counted against quarantine limit) can overcome the actual
+      // user's quarantined chunks, which diminishes the purpose of the
+      // quarantine.
+      uptr cache_size = cache_.Size();
+      uptr overhead_size = cache_.OverheadSize();
+      CHECK_GE(cache_size, overhead_size);
+      // Do the merge only when overhead exceeds this predefined limit (might
+      // require some tuning). It saves us merge attempt when the batch list
+      // quarantine is unlikely to contain batches suitable for merge.
+      const uptr kOverheadThresholdPercents = 100;
+      if (cache_size > overhead_size &&
+          overhead_size * (100 + kOverheadThresholdPercents) >
+              cache_size * kOverheadThresholdPercents) {
+        cache_.MergeBatches(&tmp);
+      }
+      // Extract enough chunks from the quarantine to get below the max
+      // quarantine size and leave some leeway for the newly quarantined chunks.
       while (cache_.Size() > min_size) {
-        QuarantineBatch *b = cache_.DequeueBatch();
-        tmp.EnqueueBatch(b);
+        tmp.EnqueueBatch(cache_.DequeueBatch());
       }
     }
     recycle_mutex_.Unlock();
@@ -124,26 +197,33 @@ class QuarantineCache {
     list_.clear();
   }
 
+  // Total memory used, including internal accounting.
   uptr Size() const {
     return atomic_load(&size_, memory_order_relaxed);
   }
 
+  // Memory used for internal accounting.
+  uptr OverheadSize() const {
+    return list_.size() * sizeof(QuarantineBatch);
+  }
+
   void Enqueue(Callback cb, void *ptr, uptr size) {
     if (list_.empty() || list_.back()->count == QuarantineBatch::kSize) {
-      AllocBatch(cb);
-      size += sizeof(QuarantineBatch);  // Count the batch in Quarantine size.
+      QuarantineBatch *b = (QuarantineBatch *)cb.Allocate(sizeof(*b));
+      CHECK(b);
+      b->init(ptr, size);
+      EnqueueBatch(b);
+    } else {
+      list_.back()->push_back(ptr, size);
+      SizeAdd(size);
     }
-    QuarantineBatch *b = list_.back();
-    CHECK(b);
-    b->batch[b->count++] = ptr;
-    b->size += size;
-    SizeAdd(size);
   }
 
-  void Transfer(QuarantineCache *c) {
-    list_.append_back(&c->list_);
-    SizeAdd(c->Size());
-    atomic_store(&c->size_, 0, memory_order_relaxed);
+  void Transfer(QuarantineCache *from_cache) {
+    list_.append_back(&from_cache->list_);
+    SizeAdd(from_cache->Size());
+
+    atomic_store(&from_cache->size_, 0, memory_order_relaxed);
   }
 
   void EnqueueBatch(QuarantineBatch *b) {
@@ -160,8 +240,57 @@ class QuarantineCache {
     return b;
   }
 
+  void MergeBatches(QuarantineCache *to_deallocate) {
+    uptr extracted_size = 0;
+    QuarantineBatch *current = list_.front();
+    while (current && current->next) {
+      if (current->can_merge(current->next)) {
+        QuarantineBatch *extracted = current->next;
+        // Move all the chunks into the current batch.
+        current->merge(extracted);
+        CHECK_EQ(extracted->count, 0);
+        CHECK_EQ(extracted->size, sizeof(QuarantineBatch));
+        // Remove the next batch from the list and account for its size.
+        list_.extract(current, extracted);
+        extracted_size += extracted->size;
+        // Add it to deallocation list.
+        to_deallocate->EnqueueBatch(extracted);
+      } else {
+        current = current->next;
+      }
+    }
+    SizeSub(extracted_size);
+  }
+
+  void PrintStats() const {
+    uptr batch_count = 0;
+    uptr total_overhead_bytes = 0;
+    uptr total_bytes = 0;
+    uptr total_quarantine_chunks = 0;
+    for (List::ConstIterator it = list_.begin(); it != list_.end(); ++it) {
+      batch_count++;
+      total_bytes += (*it).size;
+      total_overhead_bytes += (*it).size - (*it).quarantined_size();
+      total_quarantine_chunks += (*it).count;
+    }
+    uptr quarantine_chunks_capacity = batch_count * QuarantineBatch::kSize;
+    int chunks_usage_percent = quarantine_chunks_capacity == 0 ?
+        0 : total_quarantine_chunks * 100 / quarantine_chunks_capacity;
+    uptr total_quarantined_bytes = total_bytes - total_overhead_bytes;
+    int memory_overhead_percent = total_quarantined_bytes == 0 ?
+        0 : total_overhead_bytes * 100 / total_quarantined_bytes;
+    Printf("Global quarantine stats: batches: %zd; bytes: %zd (user: %zd); "
+           "chunks: %zd (capacity: %zd); %d%% chunks used; %d%% memory overhead"
+           "\n",
+           batch_count, total_bytes, total_quarantined_bytes,
+           total_quarantine_chunks, quarantine_chunks_capacity,
+           chunks_usage_percent, memory_overhead_percent);
+  }
+
  private:
-  IntrusiveList<QuarantineBatch> list_;
+  typedef IntrusiveList<QuarantineBatch> List;
+
+  List list_;
   atomic_uintptr_t size_;
 
   void SizeAdd(uptr add) {
@@ -170,16 +299,8 @@ class QuarantineCache {
   void SizeSub(uptr sub) {
     atomic_store(&size_, Size() - sub, memory_order_relaxed);
   }
-
-  NOINLINE QuarantineBatch* AllocBatch(Callback cb) {
-    QuarantineBatch *b = (QuarantineBatch *)cb.Allocate(sizeof(*b));
-    CHECK(b);
-    b->count = 0;
-    b->size = 0;
-    list_.push_back(b);
-    return b;
-  }
 };
+
 } // namespace __sanitizer
 
 #endif // SANITIZER_QUARANTINE_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_report_decorator.h b/libsanitizer/sanitizer_common/sanitizer_report_decorator.h
index e9be29fb3d5..c0edfcf7437 100644
--- a/libsanitizer/sanitizer_common/sanitizer_report_decorator.h
+++ b/libsanitizer/sanitizer_common/sanitizer_report_decorator.h
@@ -25,8 +25,9 @@ class SanitizerCommonDecorator {
   SanitizerCommonDecorator() : ansi_(ColorizeReports()) {}
   const char *Bold()    const { return ansi_ ? "\033[1m" : ""; }
   const char *Default() const { return ansi_ ? "\033[1m\033[0m"  : ""; }
-  const char *Warning()    { return Red(); }
-  const char *EndWarning() { return Default(); }
+  const char *Warning() const { return Red(); }
+  const char *MemoryByte() { return Magenta(); }
+
  protected:
   const char *Black()   const { return ansi_ ? "\033[1m\033[30m" : ""; }
   const char *Red()     const { return ansi_ ? "\033[1m\033[31m" : ""; }
diff --git a/libsanitizer/sanitizer_common/sanitizer_signal_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_signal_interceptors.inc
new file mode 100644
index 00000000000..b278f82ce03
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_signal_interceptors.inc
@@ -0,0 +1,65 @@
+//===-- sanitizer_signal_interceptors.inc -----------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Signal interceptors for sanitizers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_interceptors.h"
+
+using namespace __sanitizer;
+
+#if SANITIZER_INTERCEPT_BSD_SIGNAL
+INTERCEPTOR(void *, bsd_signal, int signum, void *handler) {
+  if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
+  return REAL(bsd_signal)(signum, handler);
+}
+#define INIT_BSD_SIGNAL COMMON_INTERCEPT_FUNCTION(bsd_signal)
+#else  // SANITIZER_INTERCEPT_BSD_SIGNAL
+#define INIT_BSD_SIGNAL
+#endif  // SANITIZER_INTERCEPT_BSD_SIGNAL
+
+#if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+INTERCEPTOR(void *, signal, int signum, void *handler) {
+  if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return nullptr;
+  return REAL(signal)(signum, handler);
+}
+#define INIT_SIGNAL COMMON_INTERCEPT_FUNCTION(signal)
+
+INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act,
+            struct sigaction *oldact) {
+  if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
+  return REAL(sigaction)(signum, act, oldact);
+}
+#define INIT_SIGACTION COMMON_INTERCEPT_FUNCTION(sigaction)
+
+namespace __sanitizer {
+int real_sigaction(int signum, const void *act, void *oldact) {
+  return REAL(sigaction)(signum, (const struct sigaction *)act,
+                         (struct sigaction *)oldact);
+}
+}  // namespace __sanitizer
+#else  // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+#define INIT_SIGNAL
+#define INIT_SIGACTION
+// We need to have defined REAL(sigaction) on other systems.
+DEFINE_REAL(int, sigaction, int signum, const struct sigaction *act,
+            struct sigaction *oldact)
+#endif  // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+
+static void InitializeSignalInterceptors() {
+  static bool was_called_once;
+  CHECK(!was_called_once);
+  was_called_once = true;
+
+  INIT_BSD_SIGNAL;
+  INIT_SIGNAL;
+  INIT_SIGACTION;
+}
diff --git a/libsanitizer/sanitizer_common/sanitizer_stackdepot.cc b/libsanitizer/sanitizer_common/sanitizer_stackdepot.cc
index 3c5313c3ae4..816a35cfb46 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stackdepot.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_stackdepot.cc
@@ -151,9 +151,9 @@ StackTrace StackDepotReverseMap::Get(u32 id) {
   if (!map_.size())
     return StackTrace();
   IdDescPair pair = {id, nullptr};
-  uptr idx = InternalBinarySearch(map_, 0, map_.size(), pair,
-                                  IdDescPair::IdComparator);
-  if (idx > map_.size())
+  uptr idx =
+      InternalLowerBound(map_, 0, map_.size(), pair, IdDescPair::IdComparator);
+  if (idx > map_.size() || map_[idx].id != id)
     return StackTrace();
   return map_[idx].desc->load();
 }
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc
index cbb3af270b6..83309d6ee38 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc
@@ -104,10 +104,6 @@ void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top,
   }
 }
 
-static bool MatchPc(uptr cur_pc, uptr trace_pc, uptr threshold) {
-  return cur_pc - trace_pc <= threshold || trace_pc - cur_pc <= threshold;
-}
-
 void BufferedStackTrace::PopStackFrames(uptr count) {
   CHECK_LT(count, size);
   size -= count;
@@ -116,15 +112,14 @@ void BufferedStackTrace::PopStackFrames(uptr count) {
   }
 }
 
+static uptr Distance(uptr a, uptr b) { return a < b ? b - a : a - b; }
+
 uptr BufferedStackTrace::LocatePcInTrace(uptr pc) {
-  // Use threshold to find PC in stack trace, as PC we want to unwind from may
-  // slightly differ from return address in the actual unwinded stack trace.
-  const int kPcThreshold = 350;
-  for (uptr i = 0; i < size; ++i) {
-    if (MatchPc(pc, trace[i], kPcThreshold))
-      return i;
+  uptr best = 0;
+  for (uptr i = 1; i < size; ++i) {
+    if (Distance(trace[i], pc) < Distance(trace[best], pc)) best = i;
   }
-  return 0;
+  return best;
 }
 
 }  // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
index c59dbd55883..31e99f6b9db 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
@@ -95,6 +95,11 @@ struct BufferedStackTrace : public StackTrace {
   void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top,
               uptr stack_bottom, bool request_fast_unwind);
 
+  void Reset() {
+    *static_cast<StackTrace *>(this) = StackTrace(trace_buffer, 0);
+    top_frame_bp = 0;
+  }
+
  private:
   void FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom,
                        u32 max_depth);
@@ -104,8 +109,8 @@ struct BufferedStackTrace : public StackTrace {
   void PopStackFrames(uptr count);
   uptr LocatePcInTrace(uptr pc);
 
-  BufferedStackTrace(const BufferedStackTrace &);
-  void operator=(const BufferedStackTrace &);
+  BufferedStackTrace(const BufferedStackTrace &) = delete;
+  void operator=(const BufferedStackTrace &) = delete;
 };
 
 // Check if given pointer points into allocated stack area.
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc
index ac3ee3a019c..f4c0f31b2af 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc
@@ -41,7 +41,8 @@ void StackTrace::Print() const {
       if (dedup_frames-- > 0) {
         if (dedup_token.length())
           dedup_token.append("--");
-        dedup_token.append(cur->info.function);
+        if (cur->info.function != nullptr)
+          dedup_token.append(cur->info.function);
       }
     }
     frames->ClearAll();
@@ -80,6 +81,21 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
   }
 }
 
+static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
+                                   uptr module_name_len, uptr *pc_offset) {
+  const char *found_module_name = nullptr;
+  bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(
+      pc, &found_module_name, pc_offset);
+
+  if (!ok) return false;
+
+  if (module_name && module_name_len) {
+    internal_strncpy(module_name, found_module_name, module_name_len);
+    module_name[module_name_len - 1] = '\x00';
+  }
+  return true;
+}
+
 }  // namespace __sanitizer
 using namespace __sanitizer;
 
@@ -115,4 +131,11 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
   internal_strncpy(out_buf, data_desc.data(), out_buf_size);
   out_buf[out_buf_size - 1] = 0;
 }
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_get_module_and_offset_for_pc( // NOLINT
+    uptr pc, char *module_name, uptr module_name_len, uptr *pc_offset) {
+  return __sanitizer::GetModuleAndOffsetForPc(pc, module_name, module_name_len,
+                                              pc_offset);
+}
 }  // extern "C"
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc
index de78c7ac8d1..3c5bed3d75a 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc
@@ -10,9 +10,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_stacktrace_printer.h"
+#include "sanitizer_file.h"
+#include "sanitizer_fuchsia.h"
 
 namespace __sanitizer {
 
+// sanitizer_symbolizer_fuchsia.cc implements these differently for Fuchsia.
+#if !SANITIZER_FUCHSIA
+
 static const char *StripFunctionName(const char *function, const char *prefix) {
   if (!function) return nullptr;
   if (!prefix) return function;
@@ -91,7 +96,7 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
                              vs_style, strip_path_prefix);
       } else if (info.module) {
         RenderModuleLocation(buffer, info.module, info.module_offset,
-                             strip_path_prefix);
+                             info.module_arch, strip_path_prefix);
       } else {
         buffer->append("(<unknown module>)");
       }
@@ -101,8 +106,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
       if (info.address & kExternalPCBit)
         {} // There PCs are not meaningful.
       else if (info.module)
-        buffer->append("(%s+%p)", StripModuleName(info.module),
-                       (void *)info.module_offset);
+        // Always strip the module name for %M.
+        RenderModuleLocation(buffer, StripModuleName(info.module),
+                             info.module_offset, info.module_arch, "");
       else
         buffer->append("(%p)", (void *)info.address);
       break;
@@ -143,6 +149,8 @@ void RenderData(InternalScopedString *buffer, const char *format,
   }
 }
 
+#endif  // !SANITIZER_FUCHSIA
+
 void RenderSourceLocation(InternalScopedString *buffer, const char *file,
                           int line, int column, bool vs_style,
                           const char *strip_path_prefix) {
@@ -163,9 +171,13 @@ void RenderSourceLocation(InternalScopedString *buffer, const char *file,
 }
 
 void RenderModuleLocation(InternalScopedString *buffer, const char *module,
-                          uptr offset, const char *strip_path_prefix) {
-  buffer->append("(%s+0x%zx)", StripPathPrefix(module, strip_path_prefix),
-                 offset);
+                          uptr offset, ModuleArch arch,
+                          const char *strip_path_prefix) {
+  buffer->append("(%s", StripPathPrefix(module, strip_path_prefix));
+  if (arch != kModuleArchUnknown) {
+    buffer->append(":%s", ModuleArchToString(arch));
+  }
+  buffer->append("+0x%zx)", offset);
 }
 
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h
index 6726f141f22..cf3cd4292dd 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h
@@ -55,7 +55,8 @@ void RenderSourceLocation(InternalScopedString *buffer, const char *file,
                           const char *strip_path_prefix);
 
 void RenderModuleLocation(InternalScopedString *buffer, const char *module,
-                          uptr offset, const char *strip_path_prefix);
+                          uptr offset, ModuleArch arch,
+                          const char *strip_path_prefix);
 
 // Same as RenderFrame, but for data section (global variables).
 // Accepts %s, %l from above.
diff --git a/libsanitizer/sanitizer_common/sanitizer_stoptheworld.h b/libsanitizer/sanitizer_common/sanitizer_stoptheworld.h
index c3245266fb9..8c3d2c05557 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stoptheworld.h
+++ b/libsanitizer/sanitizer_common/sanitizer_stoptheworld.h
@@ -16,36 +16,32 @@
 #include "sanitizer_common.h"
 
 namespace __sanitizer {
-typedef int SuspendedThreadID;
+
+enum PtraceRegistersStatus {
+  REGISTERS_UNAVAILABLE_FATAL = -1,
+  REGISTERS_UNAVAILABLE = 0,
+  REGISTERS_AVAILABLE = 1
+};
 
 // Holds the list of suspended threads and provides an interface to dump their
 // register contexts.
 class SuspendedThreadsList {
  public:
-  SuspendedThreadsList()
-    : thread_ids_(1024) {}
-  SuspendedThreadID GetThreadID(uptr index) const {
-    CHECK_LT(index, thread_ids_.size());
-    return thread_ids_[index];
+  SuspendedThreadsList() = default;
+
+  // Can't declare pure virtual functions in sanitizer runtimes:
+  // __cxa_pure_virtual might be unavailable. Use UNIMPLEMENTED() instead.
+  virtual PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+                                                  uptr *sp) const {
+    UNIMPLEMENTED();
   }
-  int GetRegistersAndSP(uptr index, uptr *buffer, uptr *sp) const;
+
   // The buffer in GetRegistersAndSP should be at least this big.
-  static uptr RegisterCount();
-  uptr thread_count() const { return thread_ids_.size(); }
-  bool Contains(SuspendedThreadID thread_id) const {
-    for (uptr i = 0; i < thread_ids_.size(); i++) {
-      if (thread_ids_[i] == thread_id)
-        return true;
-    }
-    return false;
-  }
-  void Append(SuspendedThreadID thread_id) {
-    thread_ids_.push_back(thread_id);
-  }
+  virtual uptr RegisterCount() const { UNIMPLEMENTED(); }
+  virtual uptr ThreadCount() const { UNIMPLEMENTED(); }
+  virtual tid_t GetThreadID(uptr index) const { UNIMPLEMENTED(); }
 
  private:
-  InternalMmapVector<SuspendedThreadID> thread_ids_;
-
   // Prohibit copy and assign.
   SuspendedThreadsList(const SuspendedThreadsList&);
   void operator=(const SuspendedThreadsList&);
diff --git a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
index 234e8c652c6..d746fa5403a 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
@@ -14,7 +14,8 @@
 
 #if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) || \
                         defined(__aarch64__) || defined(__powerpc64__) || \
-                        defined(__s390__))
+                        defined(__s390__) || defined(__i386__) || \
+                        defined(__arm__))
 
 #include "sanitizer_stoptheworld.h"
 
@@ -29,17 +30,13 @@
 #include <sys/types.h> // for pid_t
 #include <sys/uio.h> // for iovec
 #include <elf.h> // for NT_PRSTATUS
-#if SANITIZER_ANDROID && defined(__arm__)
-# include <linux/user.h>  // for pt_regs
-#else
-# ifdef __aarch64__
+#if defined(__aarch64__) && !SANITIZER_ANDROID
 // GLIBC 2.20+ sys/user does not include asm/ptrace.h
-#  include <asm/ptrace.h>
-# endif
-# include <sys/user.h>  // for user_regs_struct
-# if SANITIZER_ANDROID && SANITIZER_MIPS
-#   include <asm/reg.h>  // for mips SP register in sys/user.h
-# endif
+# include <asm/ptrace.h>
+#endif
+#include <sys/user.h>  // for user_regs_struct
+#if SANITIZER_ANDROID && SANITIZER_MIPS
+# include <asm/reg.h>  // for mips SP register in sys/user.h
 #endif
 #include <sys/wait.h> // for signal-related stuff
 
@@ -79,7 +76,22 @@
 
 namespace __sanitizer {
 
-COMPILER_CHECK(sizeof(SuspendedThreadID) == sizeof(pid_t));
+class SuspendedThreadsListLinux : public SuspendedThreadsList {
+ public:
+  SuspendedThreadsListLinux() : thread_ids_(1024) {}
+
+  tid_t GetThreadID(uptr index) const;
+  uptr ThreadCount() const;
+  bool ContainsTid(tid_t thread_id) const;
+  void Append(tid_t tid);
+
+  PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+                                          uptr *sp) const;
+  uptr RegisterCount() const;
+
+ private:
+  InternalMmapVector<tid_t> thread_ids_;
+};
 
 // Structure for passing arguments into the tracer thread.
 struct TracerThreadArgument {
@@ -104,31 +116,31 @@ class ThreadSuspender {
   bool SuspendAllThreads();
   void ResumeAllThreads();
   void KillAllThreads();
-  SuspendedThreadsList &suspended_threads_list() {
+  SuspendedThreadsListLinux &suspended_threads_list() {
     return suspended_threads_list_;
   }
   TracerThreadArgument *arg;
  private:
-  SuspendedThreadsList suspended_threads_list_;
+  SuspendedThreadsListLinux suspended_threads_list_;
   pid_t pid_;
-  bool SuspendThread(SuspendedThreadID thread_id);
+  bool SuspendThread(tid_t thread_id);
 };
 
-bool ThreadSuspender::SuspendThread(SuspendedThreadID tid) {
+bool ThreadSuspender::SuspendThread(tid_t tid) {
   // Are we already attached to this thread?
   // Currently this check takes linear time, however the number of threads is
   // usually small.
-  if (suspended_threads_list_.Contains(tid))
-    return false;
+  if (suspended_threads_list_.ContainsTid(tid)) return false;
   int pterrno;
   if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr),
                        &pterrno)) {
     // Either the thread is dead, or something prevented us from attaching.
     // Log this event and move on.
-    VReport(1, "Could not attach to thread %d (errno %d).\n", tid, pterrno);
+    VReport(1, "Could not attach to thread %zu (errno %d).\n", (uptr)tid,
+            pterrno);
     return false;
   } else {
-    VReport(2, "Attached to thread %d.\n", tid);
+    VReport(2, "Attached to thread %zu.\n", (uptr)tid);
     // The thread is not guaranteed to stop before ptrace returns, so we must
     // wait on it. Note: if the thread receives a signal concurrently,
     // we can get notification about the signal before notification about stop.
@@ -146,8 +158,8 @@ bool ThreadSuspender::SuspendThread(SuspendedThreadID tid) {
       if (internal_iserror(waitpid_status, &wperrno)) {
         // Got a ECHILD error. I don't think this situation is possible, but it
         // doesn't hurt to report it.
-        VReport(1, "Waiting on thread %d failed, detaching (errno %d).\n",
-                tid, wperrno);
+        VReport(1, "Waiting on thread %zu failed, detaching (errno %d).\n",
+                (uptr)tid, wperrno);
         internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr);
         return false;
       }
@@ -164,7 +176,7 @@ bool ThreadSuspender::SuspendThread(SuspendedThreadID tid) {
 }
 
 void ThreadSuspender::ResumeAllThreads() {
-  for (uptr i = 0; i < suspended_threads_list_.thread_count(); i++) {
+  for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++) {
     pid_t tid = suspended_threads_list_.GetThreadID(i);
     int pterrno;
     if (!internal_iserror(internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr),
@@ -180,7 +192,7 @@ void ThreadSuspender::ResumeAllThreads() {
 }
 
 void ThreadSuspender::KillAllThreads() {
-  for (uptr i = 0; i < suspended_threads_list_.thread_count(); i++)
+  for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++)
     internal_ptrace(PTRACE_KILL, suspended_threads_list_.GetThreadID(i),
                     nullptr, nullptr);
 }
@@ -232,7 +244,7 @@ static void TracerThreadDieCallback() {
 
 // Signal handler to wake up suspended threads when the tracer thread dies.
 static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) {
-  SignalContext ctx = SignalContext::Create(siginfo, uctx);
+  SignalContext ctx(siginfo, uctx);
   Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
          ctx.addr, ctx.pc, ctx.sp);
   ThreadSuspender *inst = thread_suspender_instance;
@@ -249,7 +261,7 @@ static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) {
 }
 
 // Size of alternative stack for signal handlers in the tracer thread.
-static const int kHandlerStackSize = 4096;
+static const int kHandlerStackSize = 8192;
 
 // This function will be run as a cloned task.
 static int TracerThread(void* argument) {
@@ -491,9 +503,28 @@ typedef _user_regs_struct regs_struct;
 #error "Unsupported architecture"
 #endif // SANITIZER_ANDROID && defined(__arm__)
 
-int SuspendedThreadsList::GetRegistersAndSP(uptr index,
-                                            uptr *buffer,
-                                            uptr *sp) const {
+tid_t SuspendedThreadsListLinux::GetThreadID(uptr index) const {
+  CHECK_LT(index, thread_ids_.size());
+  return thread_ids_[index];
+}
+
+uptr SuspendedThreadsListLinux::ThreadCount() const {
+  return thread_ids_.size();
+}
+
+bool SuspendedThreadsListLinux::ContainsTid(tid_t thread_id) const {
+  for (uptr i = 0; i < thread_ids_.size(); i++) {
+    if (thread_ids_[i] == thread_id) return true;
+  }
+  return false;
+}
+
+void SuspendedThreadsListLinux::Append(tid_t tid) {
+  thread_ids_.push_back(tid);
+}
+
+PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
+    uptr index, uptr *buffer, uptr *sp) const {
   pid_t tid = GetThreadID(index);
   regs_struct regs;
   int pterrno;
@@ -511,19 +542,23 @@ int SuspendedThreadsList::GetRegistersAndSP(uptr index,
   if (isErr) {
     VReport(1, "Could not get registers from thread %d (errno %d).\n", tid,
             pterrno);
-    return -1;
+    // ESRCH means that the given thread is not suspended or already dead.
+    // Therefore it's unsafe to inspect its data (e.g. walk through stack) and
+    // we should notify caller about this.
+    return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL
+                            : REGISTERS_UNAVAILABLE;
   }
 
   *sp = regs.REG_SP;
   internal_memcpy(buffer, &regs, sizeof(regs));
-  return 0;
+  return REGISTERS_AVAILABLE;
 }
 
-uptr SuspendedThreadsList::RegisterCount() {
+uptr SuspendedThreadsListLinux::RegisterCount() const {
   return sizeof(regs_struct) / sizeof(uptr);
 }
 } // namespace __sanitizer
 
 #endif  // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)
         // || defined(__aarch64__) || defined(__powerpc64__)
-        // || defined(__s390__)
+        // || defined(__s390__) || defined(__i386__) || defined(__arm__)
diff --git a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cc b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cc
new file mode 100644
index 00000000000..6282694fa8c
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cc
@@ -0,0 +1,176 @@
+//===-- sanitizer_stoptheworld_mac.cc -------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__) || \
+                      defined(__i386))
+
+#include <mach/mach.h>
+#include <mach/thread_info.h>
+#include <pthread.h>
+
+#include "sanitizer_stoptheworld.h"
+
+namespace __sanitizer {
+typedef struct {
+  tid_t tid;
+  thread_t thread;
+} SuspendedThreadInfo;
+
+class SuspendedThreadsListMac : public SuspendedThreadsList {
+ public:
+  SuspendedThreadsListMac() : threads_(1024) {}
+
+  tid_t GetThreadID(uptr index) const;
+  thread_t GetThread(uptr index) const;
+  uptr ThreadCount() const;
+  bool ContainsThread(thread_t thread) const;
+  void Append(thread_t thread);
+
+  PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+                                          uptr *sp) const;
+  uptr RegisterCount() const;
+
+ private:
+  InternalMmapVector<SuspendedThreadInfo> threads_;
+};
+
+struct RunThreadArgs {
+  StopTheWorldCallback callback;
+  void *argument;
+};
+
+void RunThread(void *arg) {
+  struct RunThreadArgs *run_args = (struct RunThreadArgs *)arg;
+  SuspendedThreadsListMac suspended_threads_list;
+
+  thread_array_t threads;
+  mach_msg_type_number_t num_threads;
+  kern_return_t err = task_threads(mach_task_self(), &threads, &num_threads);
+  if (err != KERN_SUCCESS) {
+    VReport(1, "Failed to get threads for task (errno %d).\n", err);
+    return;
+  }
+
+  thread_t thread_self = mach_thread_self();
+  for (unsigned int i = 0; i < num_threads; ++i) {
+    if (threads[i] == thread_self) continue;
+
+    thread_suspend(threads[i]);
+    suspended_threads_list.Append(threads[i]);
+  }
+
+  run_args->callback(suspended_threads_list, run_args->argument);
+
+  uptr num_suspended = suspended_threads_list.ThreadCount();
+  for (unsigned int i = 0; i < num_suspended; ++i) {
+    thread_resume(suspended_threads_list.GetThread(i));
+  }
+}
+
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+  struct RunThreadArgs arg = {callback, argument};
+  pthread_t run_thread = (pthread_t)internal_start_thread(RunThread, &arg);
+  internal_join_thread(run_thread);
+}
+
+#if defined(__x86_64__)
+typedef x86_thread_state64_t regs_struct;
+
+#define SP_REG __rsp
+
+#elif defined(__aarch64__)
+typedef arm_thread_state64_t regs_struct;
+
+# if __DARWIN_UNIX03
+#  define SP_REG __sp
+# else
+#  define SP_REG sp
+# endif
+
+#elif defined(__i386)
+typedef x86_thread_state32_t regs_struct;
+
+#define SP_REG __esp
+
+#else
+#error "Unsupported architecture"
+#endif
+
+tid_t SuspendedThreadsListMac::GetThreadID(uptr index) const {
+  CHECK_LT(index, threads_.size());
+  return threads_[index].tid;
+}
+
+thread_t SuspendedThreadsListMac::GetThread(uptr index) const {
+  CHECK_LT(index, threads_.size());
+  return threads_[index].thread;
+}
+
+uptr SuspendedThreadsListMac::ThreadCount() const {
+  return threads_.size();
+}
+
+bool SuspendedThreadsListMac::ContainsThread(thread_t thread) const {
+  for (uptr i = 0; i < threads_.size(); i++) {
+    if (threads_[i].thread == thread) return true;
+  }
+  return false;
+}
+
+void SuspendedThreadsListMac::Append(thread_t thread) {
+  thread_identifier_info_data_t info;
+  mach_msg_type_number_t info_count = THREAD_IDENTIFIER_INFO_COUNT;
+  kern_return_t err = thread_info(thread, THREAD_IDENTIFIER_INFO,
+                                  (thread_info_t)&info, &info_count);
+  if (err != KERN_SUCCESS) {
+    VReport(1, "Error - unable to get thread ident for a thread\n");
+    return;
+  }
+  threads_.push_back({info.thread_id, thread});
+}
+
+PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
+    uptr index, uptr *buffer, uptr *sp) const {
+  thread_t thread = GetThread(index);
+  regs_struct regs;
+  int err;
+  mach_msg_type_number_t reg_count = MACHINE_THREAD_STATE_COUNT;
+  err = thread_get_state(thread, MACHINE_THREAD_STATE, (thread_state_t)&regs,
+                         &reg_count);
+  if (err != KERN_SUCCESS) {
+    VReport(1, "Error - unable to get registers for a thread\n");
+    // KERN_INVALID_ARGUMENT indicates that either the flavor is invalid,
+    // or the thread does not exist. The other possible error case,
+    // MIG_ARRAY_TOO_LARGE, means that the state is too large, but it's
+    // still safe to proceed.
+    return err == KERN_INVALID_ARGUMENT ? REGISTERS_UNAVAILABLE_FATAL
+                                        : REGISTERS_UNAVAILABLE;
+  }
+
+  internal_memcpy(buffer, &regs, sizeof(regs));
+  *sp = regs.SP_REG;
+
+  // On x86_64 and aarch64, we must account for the stack redzone, which is 128
+  // bytes.
+  if (SANITIZER_WORDSIZE == 64) *sp -= 128;
+
+  return REGISTERS_AVAILABLE;
+}
+
+uptr SuspendedThreadsListMac::RegisterCount() const {
+  return MACHINE_THREAD_STATE_COUNT;
+}
+} // namespace __sanitizer
+
+#endif  // SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__)) ||
+        //                   defined(__i386))
diff --git a/libsanitizer/sanitizer_common/sanitizer_suppressions.cc b/libsanitizer/sanitizer_common/sanitizer_suppressions.cc
index bfdff59a35c..4095a0942a3 100644
--- a/libsanitizer/sanitizer_common/sanitizer_suppressions.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_suppressions.cc
@@ -14,6 +14,7 @@
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_common.h"
 #include "sanitizer_flags.h"
+#include "sanitizer_file.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_placement_new.h"
 
@@ -48,6 +49,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
   if (filename[0] == '\0')
     return;
 
+#if !SANITIZER_FUCHSIA
   // If we cannot find the file, check if its location is relative to
   // the location of the executable.
   InternalScopedString new_file_path(kMaxPathLength);
@@ -56,6 +58,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
                                           new_file_path.size())) {
     filename = new_file_path.data();
   }
+#endif  // !SANITIZER_FUCHSIA
 
   // Read the file.
   VPrintf(1, "%s: reading suppressions file at %s\n",
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc
index 3557415aeab..9d3e0115eb2 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc
@@ -31,9 +31,11 @@ void AddressInfo::Clear() {
   function_offset = kUnknown;
 }
 
-void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset) {
+void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
+                                 ModuleArch mod_arch) {
   module = internal_strdup(mod_name);
   module_offset = mod_offset;
+  module_arch = mod_arch;
 }
 
 SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
@@ -67,6 +69,10 @@ Symbolizer *Symbolizer::symbolizer_;
 StaticSpinMutex Symbolizer::init_mu_;
 LowLevelAllocator Symbolizer::symbolizer_allocator_;
 
+void Symbolizer::InvalidateModuleList() {
+  modules_fresh_ = false;
+}
+
 void Symbolizer::AddHooks(Symbolizer::StartSymbolizationHook start_hook,
                           Symbolizer::EndSymbolizationHook end_hook) {
   CHECK(start_hook_ == 0 && end_hook_ == 0);
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer.h
index 2b90b42e2ba..5e2c843d86e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer.h
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer.h
@@ -29,6 +29,7 @@ struct AddressInfo {
 
   char *module;
   uptr module_offset;
+  ModuleArch module_arch;
 
   static const uptr kUnknown = ~(uptr)0;
   char *function;
@@ -41,7 +42,7 @@ struct AddressInfo {
   AddressInfo();
   // Deletes all strings and resets all fields.
   void Clear();
-  void FillModuleInfo(const char *mod_name, uptr mod_offset);
+  void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch);
 };
 
 // Linked list of symbolized frames (each frame is described by AddressInfo).
@@ -63,6 +64,8 @@ struct DataInfo {
   // (de)allocated using sanitizer internal allocator.
   char *module;
   uptr module_offset;
+  ModuleArch module_arch;
+
   char *file;
   uptr line;
   char *name;
@@ -114,8 +117,11 @@ class Symbolizer final {
   void AddHooks(StartSymbolizationHook start_hook,
                 EndSymbolizationHook end_hook);
 
+  void RefreshModules();
   const LoadedModule *FindModuleForAddress(uptr address);
 
+  void InvalidateModuleList();
+
  private:
   // GetModuleNameAndOffsetForPC has to return a string to the caller.
   // Since the corresponding module might get unloaded later, we should create
@@ -141,8 +147,10 @@ class Symbolizer final {
   static Symbolizer *PlatformInit();
 
   bool FindModuleNameAndOffsetForAddress(uptr address, const char **module_name,
-                                         uptr *module_offset);
+                                         uptr *module_offset,
+                                         ModuleArch *module_arch);
   ListOfModules modules_;
+  ListOfModules fallback_modules_;
   // If stale, need to reload the modules before looking up addresses.
   bool modules_fresh_;
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_fuchsia.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_fuchsia.cc
new file mode 100644
index 00000000000..04263205e16
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_fuchsia.cc
@@ -0,0 +1,105 @@
+//===-- sanitizer_symbolizer_fuchsia.cc -----------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries.
+//
+// Implementation of Fuchsia-specific symbolizer.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_fuchsia.h"
+#include "sanitizer_symbolizer.h"
+
+namespace __sanitizer {
+
+// For Fuchsia we don't do any actual symbolization per se.
+// Instead, we emit text containing raw addresses and raw linkage
+// symbol names, embedded in Fuchsia's symbolization markup format.
+// Fuchsia's logging infrastructure emits enough information about
+// process memory layout that a post-processing filter can do the
+// symbolization and pretty-print the markup.  See the spec at:
+// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
+
+// This is used by UBSan for type names, and by ASan for global variable names.
+constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
+constexpr uptr kFormatDemangleMax = 1024;  // Arbitrary.
+
+// Function name or equivalent from PC location.
+constexpr const char *kFormatFunction = "{{{pc:%p}}}";
+constexpr uptr kFormatFunctionMax = 64;  // More than big enough for 64-bit hex.
+
+// Global variable name or equivalent from data memory address.
+constexpr const char *kFormatData = "{{{data:%p}}}";
+
+// One frame in a backtrace (printed on a line by itself).
+constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}";
+
+// This is used by UBSan for type names, and by ASan for global variable names.
+// It's expected to return a static buffer that will be reused on each call.
+const char *Symbolizer::Demangle(const char *name) {
+  static char buffer[kFormatDemangleMax];
+  internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
+  return buffer;
+}
+
+// This is used mostly for suppression matching.  Making it work
+// would enable "interceptor_via_lib" suppressions.  It's also used
+// once in UBSan to say "in module ..." in a message that also
+// includes an address in the module, so post-processing can already
+// pretty-print that so as to indicate the module.
+bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
+                                             uptr *module_address) {
+  return false;
+}
+
+// This is used in some places for suppression checking, which we
+// don't really support for Fuchsia.  It's also used in UBSan to
+// identify a PC location to a function name, so we always fill in
+// the function member with a string containing markup around the PC
+// value.
+// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
+// to render stack frames, but that should be changed to use
+// RenderStackFrame.
+SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
+  SymbolizedStack *s = SymbolizedStack::New(addr);
+  char buffer[kFormatFunctionMax];
+  internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
+  s->info.function = internal_strdup(buffer);
+  return s;
+}
+
+// Always claim we succeeded, so that RenderDataInfo will be called.
+bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+  info->Clear();
+  info->start = addr;
+  return true;
+}
+
+// We ignore the format argument to __sanitizer_symbolize_global.
+void RenderData(InternalScopedString *buffer, const char *format,
+                const DataInfo *DI, const char *strip_path_prefix) {
+  buffer->append(kFormatData, DI->start);
+}
+
+// We don't support the stack_trace_format flag at all.
+void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
+                 const AddressInfo &info, bool vs_style,
+                 const char *strip_path_prefix, const char *strip_func_prefix) {
+  buffer->append(kFormatFrame, frame_no, info.address);
+}
+
+Symbolizer *Symbolizer::PlatformInit() {
+  return new (symbolizer_allocator_) Symbolizer({});
+}
+
+void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h
index 119cb688463..eae7509576e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h
@@ -13,6 +13,7 @@
 #define SANITIZER_SYMBOLIZER_INTERNAL_H
 
 #include "sanitizer_symbolizer.h"
+#include "sanitizer_file.h"
 
 namespace __sanitizer {
 
@@ -122,8 +123,8 @@ class LLVMSymbolizer : public SymbolizerTool {
   bool SymbolizeData(uptr addr, DataInfo *info) override;
 
  private:
-  const char *SendCommand(bool is_data, const char *module_name,
-                          uptr module_offset);
+  const char *FormatAndSendCommand(bool is_data, const char *module_name,
+                                   uptr module_offset, ModuleArch arch);
 
   LLVMSymbolizerProcess *symbolizer_process_;
   static const uptr kBufferSize = 16 * 1024;
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc
index b5bcfdbc089..eebc30b124d 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc
@@ -93,7 +93,8 @@ struct SymbolizeCodeCallbackArg {
     if (frames_symbolized > 0) {
       SymbolizedStack *cur = SymbolizedStack::New(addr);
       AddressInfo *info = &cur->info;
-      info->FillModuleInfo(first->info.module, first->info.module_offset);
+      info->FillModuleInfo(first->info.module, first->info.module_offset,
+                           first->info.module_arch);
       last->next = cur;
       last = cur;
     }
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc
index 45eb11a4d3b..df934412146 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc
@@ -15,6 +15,18 @@
 
 namespace __sanitizer {
 
+Symbolizer *Symbolizer::GetOrInit() {
+  SpinMutexLock l(&init_mu_);
+  if (symbolizer_)
+    return symbolizer_;
+  symbolizer_ = PlatformInit();
+  CHECK(symbolizer_);
+  return symbolizer_;
+}
+
+// See sanitizer_symbolizer_fuchsia.cc.
+#if !SANITIZER_FUCHSIA
+
 const char *ExtractToken(const char *str, const char *delims, char **result) {
   uptr prefix_len = internal_strcspn(str, delims);
   *result = (char*)InternalAlloc(prefix_len + 1);
@@ -62,11 +74,13 @@ SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
   BlockingMutexLock l(&mu_);
   const char *module_name;
   uptr module_offset;
+  ModuleArch arch;
   SymbolizedStack *res = SymbolizedStack::New(addr);
-  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset))
+  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
+                                         &arch))
     return res;
   // Always fill data about module name and offset.
-  res->info.FillModuleInfo(module_name, module_offset);
+  res->info.FillModuleInfo(module_name, module_offset, arch);
   for (auto &tool : tools_) {
     SymbolizerScope sym_scope(this);
     if (tool.SymbolizePC(addr, res)) {
@@ -80,11 +94,14 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
   BlockingMutexLock l(&mu_);
   const char *module_name;
   uptr module_offset;
-  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset))
+  ModuleArch arch;
+  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
+                                         &arch))
     return false;
   info->Clear();
   info->module = internal_strdup(module_name);
   info->module_offset = module_offset;
+  info->module_arch = arch;
   for (auto &tool : tools_) {
     SymbolizerScope sym_scope(this);
     if (tool.SymbolizeData(addr, info)) {
@@ -98,8 +115,9 @@ bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
                                              uptr *module_address) {
   BlockingMutexLock l(&mu_);
   const char *internal_module_name = nullptr;
+  ModuleArch arch;
   if (!FindModuleNameAndOffsetForAddress(pc, &internal_module_name,
-                                         module_address))
+                                         module_address, &arch))
     return false;
 
   if (module_name)
@@ -132,46 +150,58 @@ void Symbolizer::PrepareForSandboxing() {
 
 bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address,
                                                    const char **module_name,
-                                                   uptr *module_offset) {
+                                                   uptr *module_offset,
+                                                   ModuleArch *module_arch) {
   const LoadedModule *module = FindModuleForAddress(address);
   if (module == nullptr)
     return false;
   *module_name = module->full_name();
   *module_offset = address - module->base_address();
+  *module_arch = module->arch();
   return true;
 }
 
+void Symbolizer::RefreshModules() {
+  modules_.init();
+  fallback_modules_.fallbackInit();
+  RAW_CHECK(modules_.size() > 0);
+  modules_fresh_ = true;
+}
+
+static const LoadedModule *SearchForModule(const ListOfModules &modules,
+                                           uptr address) {
+  for (uptr i = 0; i < modules.size(); i++) {
+    if (modules[i].containsAddress(address)) {
+      return &modules[i];
+    }
+  }
+  return nullptr;
+}
+
 const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) {
   bool modules_were_reloaded = false;
   if (!modules_fresh_) {
-    modules_.init();
-    RAW_CHECK(modules_.size() > 0);
-    modules_fresh_ = true;
+    RefreshModules();
     modules_were_reloaded = true;
   }
-  for (uptr i = 0; i < modules_.size(); i++) {
-    if (modules_[i].containsAddress(address)) {
-      return &modules_[i];
-    }
-  }
-  // Reload the modules and look up again, if we haven't tried it yet.
+  const LoadedModule *module = SearchForModule(modules_, address);
+  if (module) return module;
+
+  // dlopen/dlclose interceptors invalidate the module list, but when
+  // interception is disabled, we need to retry if the lookup fails in
+  // case the module list changed.
+#if !SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
   if (!modules_were_reloaded) {
-    // FIXME: set modules_fresh_ from dlopen()/dlclose() interceptors.
-    // It's too aggressive to reload the list of modules each time we fail
-    // to find a module for a given address.
-    modules_fresh_ = false;
-    return FindModuleForAddress(address);
+    RefreshModules();
+    module = SearchForModule(modules_, address);
+    if (module) return module;
   }
-  return 0;
-}
+#endif
 
-Symbolizer *Symbolizer::GetOrInit() {
-  SpinMutexLock l(&init_mu_);
-  if (symbolizer_)
-    return symbolizer_;
-  symbolizer_ = PlatformInit();
-  CHECK(symbolizer_);
-  return symbolizer_;
+  if (fallback_modules_.size()) {
+    module = SearchForModule(fallback_modules_, address);
+  }
+  return module;
 }
 
 // For now we assume the following protocol:
@@ -195,6 +225,8 @@ class LLVMSymbolizerProcess : public SymbolizerProcess {
            buffer[length - 2] == '\n';
   }
 
+  // When adding a new architecture, don't forget to also update
+  // script/asan_symbolize.py and sanitizer_common.h.
   void GetArgV(const char *path_to_binary,
                const char *(&argv)[kArgVMax]) const override {
 #if defined(__x86_64h__)
@@ -240,25 +272,21 @@ static const char *ParseFileLineInfo(AddressInfo *info, const char *str) {
   char *file_line_info = 0;
   str = ExtractToken(str, "\n", &file_line_info);
   CHECK(file_line_info);
-  // Parse the last :<int>, which must be there.
-  char *last_colon = internal_strrchr(file_line_info, ':');
-  CHECK(last_colon);
-  int line_or_column = internal_atoll(last_colon + 1);
-  // Truncate the string at the last colon and find the next-to-last colon.
-  *last_colon = '\0';
-  last_colon = internal_strrchr(file_line_info, ':');
-  if (last_colon && IsDigit(last_colon[1])) {
-    // If the second-to-last colon is followed by a digit, it must be the line
-    // number, and the previous parsed number was a column.
-    info->line = internal_atoll(last_colon + 1);
-    info->column = line_or_column;
-    *last_colon = '\0';
-  } else {
-    // Otherwise, we have line info but no column info.
-    info->line = line_or_column;
-    info->column = 0;
+
+  if (uptr size = internal_strlen(file_line_info)) {
+    char *back = file_line_info + size - 1;
+    for (int i = 0; i < 2; ++i) {
+      while (back > file_line_info && IsDigit(*back)) --back;
+      if (*back != ':' || !IsDigit(back[1])) break;
+      info->column = info->line;
+      info->line = internal_atoll(back + 1);
+      // Truncate the string at the colon to keep only filename.
+      *back = '\0';
+      --back;
+    }
+    ExtractToken(file_line_info, "", &info->file);
   }
-  ExtractToken(file_line_info, "", &info->file);
+
   InternalFree(file_line_info);
   return str;
 }
@@ -286,7 +314,8 @@ void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res) {
       top_frame = false;
     } else {
       cur = SymbolizedStack::New(res->info.address);
-      cur->info.FillModuleInfo(res->info.module, res->info.module_offset);
+      cur->info.FillModuleInfo(res->info.module, res->info.module_offset,
+                               res->info.module_arch);
       last->next = cur;
       last = cur;
     }
@@ -319,8 +348,10 @@ void ParseSymbolizeDataOutput(const char *str, DataInfo *info) {
 }
 
 bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
-  if (const char *buf = SendCommand(/*is_data*/ false, stack->info.module,
-                                    stack->info.module_offset)) {
+  AddressInfo *info = &stack->info;
+  const char *buf = FormatAndSendCommand(
+      /*is_data*/ false, info->module, info->module_offset, info->module_arch);
+  if (buf) {
     ParseSymbolizePCOutput(buf, stack);
     return true;
   }
@@ -328,8 +359,9 @@ bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
 }
 
 bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
-  if (const char *buf =
-          SendCommand(/*is_data*/ true, info->module, info->module_offset)) {
+  const char *buf = FormatAndSendCommand(
+      /*is_data*/ true, info->module, info->module_offset, info->module_arch);
+  if (buf) {
     ParseSymbolizeDataOutput(buf, info);
     info->start += (addr - info->module_offset); // Add the base address.
     return true;
@@ -337,11 +369,27 @@ bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
   return false;
 }
 
-const char *LLVMSymbolizer::SendCommand(bool is_data, const char *module_name,
-                                        uptr module_offset) {
+const char *LLVMSymbolizer::FormatAndSendCommand(bool is_data,
+                                                 const char *module_name,
+                                                 uptr module_offset,
+                                                 ModuleArch arch) {
   CHECK(module_name);
-  internal_snprintf(buffer_, kBufferSize, "%s\"%s\" 0x%zx\n",
-                    is_data ? "DATA " : "", module_name, module_offset);
+  const char *is_data_str = is_data ? "DATA " : "";
+  if (arch == kModuleArchUnknown) {
+    if (internal_snprintf(buffer_, kBufferSize, "%s\"%s\" 0x%zx\n", is_data_str,
+                          module_name,
+                          module_offset) >= static_cast<int>(kBufferSize)) {
+      Report("WARNING: Command buffer too small");
+      return nullptr;
+    }
+  } else {
+    if (internal_snprintf(buffer_, kBufferSize, "%s\"%s:%s\" 0x%zx\n",
+                          is_data_str, module_name, ModuleArchToString(arch),
+                          module_offset) >= static_cast<int>(kBufferSize)) {
+      Report("WARNING: Command buffer too small");
+      return nullptr;
+    }
+  }
   return symbolizer_process_->SendCommand(buffer_);
 }
 
@@ -357,7 +405,23 @@ SymbolizerProcess::SymbolizerProcess(const char *path, bool use_forkpty)
   CHECK_NE(path_[0], '\0');
 }
 
+static bool IsSameModule(const char* path) {
+  if (const char* ProcessName = GetProcessName()) {
+    if (const char* SymbolizerName = StripModuleName(path)) {
+      return !internal_strcmp(ProcessName, SymbolizerName);
+    }
+  }
+  return false;
+}
+
 const char *SymbolizerProcess::SendCommand(const char *command) {
+  if (failed_to_start_)
+    return nullptr;
+  if (IsSameModule(path_)) {
+    Report("WARNING: Symbolizer was blocked from starting itself!\n");
+    failed_to_start_ = true;
+    return nullptr;
+  }
   for (; times_restarted_ < kMaxTimesRestarted; times_restarted_++) {
     // Start or restart symbolizer if we failed to send command to it.
     if (const char *res = SendCommandImpl(command))
@@ -406,6 +470,11 @@ bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
     read_len += just_read;
     if (ReachedEndOfOutput(buffer, read_len))
       break;
+    if (read_len + 1 == max_length) {
+      Report("WARNING: Symbolizer buffer too small\n");
+      read_len = 0;
+      break;
+    }
   }
   buffer[read_len] = '\0';
   return true;
@@ -423,4 +492,6 @@ bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {
   return true;
 }
 
+#endif  // !SANITIZER_FUCHSIA
+
 }  // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc
index 3fcd7d04880..afd6bbe7fbf 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc
@@ -14,6 +14,7 @@
 #if SANITIZER_POSIX
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_common.h"
+#include "sanitizer_file.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_linux.h"
@@ -53,7 +54,7 @@ const char *DemangleCXXABI(const char *name) {
   // own demangler (libc++abi's implementation could be adapted so that
   // it does not allocate). For now, we just call it anyway, and we leak
   // the returned value.
-  if (__cxxabiv1::__cxa_demangle)
+  if (&__cxxabiv1::__cxa_demangle)
     if (const char *demangled_name =
           __cxxabiv1::__cxa_demangle(name, 0, 0, 0))
       return demangled_name;
@@ -99,6 +100,46 @@ const char *DemangleSwiftAndCXX(const char *name) {
   return DemangleCXXABI(name);
 }
 
+static bool CreateTwoHighNumberedPipes(int *infd_, int *outfd_) {
+  int *infd = NULL;
+  int *outfd = NULL;
+  // The client program may close its stdin and/or stdout and/or stderr
+  // thus allowing socketpair to reuse file descriptors 0, 1 or 2.
+  // In this case the communication between the forked processes may be
+  // broken if either the parent or the child tries to close or duplicate
+  // these descriptors. The loop below produces two pairs of file
+  // descriptors, each greater than 2 (stderr).
+  int sock_pair[5][2];
+  for (int i = 0; i < 5; i++) {
+    if (pipe(sock_pair[i]) == -1) {
+      for (int j = 0; j < i; j++) {
+        internal_close(sock_pair[j][0]);
+        internal_close(sock_pair[j][1]);
+      }
+      return false;
+    } else if (sock_pair[i][0] > 2 && sock_pair[i][1] > 2) {
+      if (infd == NULL) {
+        infd = sock_pair[i];
+      } else {
+        outfd = sock_pair[i];
+        for (int j = 0; j < i; j++) {
+          if (sock_pair[j] == infd) continue;
+          internal_close(sock_pair[j][0]);
+          internal_close(sock_pair[j][1]);
+        }
+        break;
+      }
+    }
+  }
+  CHECK(infd);
+  CHECK(outfd);
+  infd_[0] = infd[0];
+  infd_[1] = infd[1];
+  outfd_[0] = outfd[0];
+  outfd_[1] = outfd[1];
+  return true;
+}
+
 bool SymbolizerProcess::StartSymbolizerSubprocess() {
   if (!FileExists(path_)) {
     if (!reported_invalid_path_) {
@@ -108,7 +149,18 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
     return false;
   }
 
-  int pid;
+  int pid = -1;
+
+  int infd[2];
+  internal_memset(&infd, 0, sizeof(infd));
+  int outfd[2];
+  internal_memset(&outfd, 0, sizeof(outfd));
+  if (!CreateTwoHighNumberedPipes(infd, outfd)) {
+    Report("WARNING: Can't create a socket pair to start "
+           "external symbolizer (errno: %d)\n", errno);
+    return false;
+  }
+
   if (use_forkpty_) {
 #if SANITIZER_MAC
     fd_t fd = kInvalidFd;
@@ -119,6 +171,10 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
     int saved_stderr = dup(STDERR_FILENO);
     CHECK_GE(saved_stderr, 0);
 
+    // We only need one pipe, for stdin of the child.
+    close(outfd[0]);
+    close(outfd[1]);
+
     // Use forkpty to disable buffering in the new terminal.
     pid = internal_forkpty(&fd);
     if (pid == -1) {
@@ -129,6 +185,13 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
     } else if (pid == 0) {
       // Child subprocess.
 
+      // infd[0] is the child's reading end.
+      close(infd[1]);
+
+      // Set up stdin to read from the pipe.
+      CHECK_GE(dup2(infd[0], STDIN_FILENO), 0);
+      close(infd[0]);
+
       // Restore stderr.
       CHECK_GE(dup2(saved_stderr, STDERR_FILENO), 0);
       close(saved_stderr);
@@ -139,8 +202,12 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
       internal__exit(1);
     }
 
+    // Input for the child, infd[1] is our writing end.
+    output_fd_ = infd[1];
+    close(infd[0]);
+
     // Continue execution in parent process.
-    input_fd_ = output_fd_ = fd;
+    input_fd_ = fd;
 
     close(saved_stderr);
 
@@ -154,41 +221,6 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
     UNIMPLEMENTED();
 #endif  // SANITIZER_MAC
   } else {
-    int *infd = NULL;
-    int *outfd = NULL;
-    // The client program may close its stdin and/or stdout and/or stderr
-    // thus allowing socketpair to reuse file descriptors 0, 1 or 2.
-    // In this case the communication between the forked processes may be
-    // broken if either the parent or the child tries to close or duplicate
-    // these descriptors. The loop below produces two pairs of file
-    // descriptors, each greater than 2 (stderr).
-    int sock_pair[5][2];
-    for (int i = 0; i < 5; i++) {
-      if (pipe(sock_pair[i]) == -1) {
-        for (int j = 0; j < i; j++) {
-          internal_close(sock_pair[j][0]);
-          internal_close(sock_pair[j][1]);
-        }
-        Report("WARNING: Can't create a socket pair to start "
-               "external symbolizer (errno: %d)\n", errno);
-        return false;
-      } else if (sock_pair[i][0] > 2 && sock_pair[i][1] > 2) {
-        if (infd == NULL) {
-          infd = sock_pair[i];
-        } else {
-          outfd = sock_pair[i];
-          for (int j = 0; j < i; j++) {
-            if (sock_pair[j] == infd) continue;
-            internal_close(sock_pair[j][0]);
-            internal_close(sock_pair[j][1]);
-          }
-          break;
-        }
-      }
-    }
-    CHECK(infd);
-    CHECK(outfd);
-
     const char *argv[kArgVMax];
     GetArgV(path_, argv);
     pid = StartSubprocess(path_, argv, /* stdin */ outfd[0],
@@ -203,6 +235,8 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
     output_fd_ = outfd[1];
   }
 
+  CHECK_GT(pid, 0);
+
   // Check that symbolizer subprocess started successfully.
   SleepForMillis(kSymbolizerStartupTimeMillis);
   if (!IsProcessRunning(pid)) {
@@ -236,6 +270,10 @@ class Addr2LineProcess : public SymbolizerProcess {
   bool ReadFromSymbolizer(char *buffer, uptr max_length) override {
     if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))
       return false;
+    // The returned buffer is empty when output is valid, but exceeds
+    // max_length.
+    if (*buffer == '\0')
+      return true;
     // We should cut out output_terminator_ at the end of given buffer,
     // appended by addr2line to mark the end of its meaningful output.
     // We cannot scan buffer from it's beginning, because it is legal for it
@@ -389,7 +427,6 @@ class InternalSymbolizer : public SymbolizerTool {
   InternalSymbolizer() { }
 
   static const int kBufferSize = 16 * 1024;
-  static const int kMaxDemangledNameSize = 1024;
   char buffer_[kBufferSize];
 };
 #else  // SANITIZER_SUPPORTS_WEAK_HOOKS
@@ -436,16 +473,16 @@ static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) {
 
   // Otherwise symbolizer program is unknown, let's search $PATH
   CHECK(path == nullptr);
-  if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
-    VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
-    return new(*allocator) LLVMSymbolizer(found_path, allocator);
-  }
 #if SANITIZER_MAC
   if (const char *found_path = FindPathToBinary("atos")) {
     VReport(2, "Using atos found at: %s\n", found_path);
     return new(*allocator) AtosSymbolizer(found_path, allocator);
   }
 #endif  // SANITIZER_MAC
+  if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
+    VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
+    return new(*allocator) LLVMSymbolizer(found_path, allocator);
+  }
   if (common_flags()->allow_addr2line) {
     if (const char *found_path = FindPathToBinary("addr2line")) {
       VReport(2, "Using addr2line found at: %s\n", found_path);
@@ -461,7 +498,7 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
     VReport(2, "Symbolizer is disabled.\n");
     return;
   }
-  if (IsReportingOOM()) {
+  if (IsAllocatorOutOfMemory()) {
     VReport(2, "Cannot use internal symbolizer: out of memory\n");
   } else if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) {
     VReport(2, "Using internal symbolizer.\n");
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc
index 95fda7ec42e..06375fc2f4e 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc
@@ -12,15 +12,24 @@
 
 #include "sanitizer_platform.h"
 #if SANITIZER_WINDOWS
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <dbghelp.h>
-#pragma comment(lib, "dbghelp.lib")
 
+#include "sanitizer_dbghelp.h"
 #include "sanitizer_symbolizer_internal.h"
 
 namespace __sanitizer {
 
+decltype(::StackWalk64) *StackWalk64;
+decltype(::SymCleanup) *SymCleanup;
+decltype(::SymFromAddr) *SymFromAddr;
+decltype(::SymFunctionTableAccess64) *SymFunctionTableAccess64;
+decltype(::SymGetLineFromAddr64) *SymGetLineFromAddr64;
+decltype(::SymGetModuleBase64) *SymGetModuleBase64;
+decltype(::SymGetSearchPathW) *SymGetSearchPathW;
+decltype(::SymInitialize) *SymInitialize;
+decltype(::SymSetOptions) *SymSetOptions;
+decltype(::SymSetSearchPathW) *SymSetSearchPathW;
+decltype(::UnDecorateSymbolName) *UnDecorateSymbolName;
+
 namespace {
 
 class WinSymbolizerTool : public SymbolizerTool {
@@ -48,6 +57,29 @@ bool TrySymInitialize() {
 void InitializeDbgHelpIfNeeded() {
   if (is_dbghelp_initialized)
     return;
+
+  HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
+  CHECK(dbghelp && "failed to load dbghelp.dll");
+
+#define DBGHELP_IMPORT(name)                                                  \
+  do {                                                                        \
+    name =                                                                    \
+        reinterpret_cast<decltype(::name) *>(GetProcAddress(dbghelp, #name)); \
+    CHECK(name != nullptr);                                                   \
+  } while (0)
+  DBGHELP_IMPORT(StackWalk64);
+  DBGHELP_IMPORT(SymCleanup);
+  DBGHELP_IMPORT(SymFromAddr);
+  DBGHELP_IMPORT(SymFunctionTableAccess64);
+  DBGHELP_IMPORT(SymGetLineFromAddr64);
+  DBGHELP_IMPORT(SymGetModuleBase64);
+  DBGHELP_IMPORT(SymGetSearchPathW);
+  DBGHELP_IMPORT(SymInitialize);
+  DBGHELP_IMPORT(SymSetOptions);
+  DBGHELP_IMPORT(SymSetSearchPathW);
+  DBGHELP_IMPORT(UnDecorateSymbolName);
+#undef DBGHELP_IMPORT
+
   if (!TrySymInitialize()) {
     // OK, maybe the client app has called SymInitialize already.
     // That's a bit unfortunate for us as all the DbgHelp functions are
diff --git a/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc b/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc
index 3bc50d972a4..7c9a2719eea 100644
--- a/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc
@@ -5,17 +5,41 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Generic implementations of internal_syscall and internal_iserror.
+// Generic implementations of internal_syscall* and internal_iserror.
 //
 //===----------------------------------------------------------------------===//
 
-#if SANITIZER_FREEBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
 # define SYSCALL(name) SYS_ ## name
 #else
 # define SYSCALL(name) __NR_ ## name
 #endif
 
-#if (SANITIZER_FREEBSD || SANITIZER_MAC) && defined(__x86_64__)
+#if SANITIZER_NETBSD
+// We use 3 kinds of internal_syscall's for different types of retval in order
+// to address differences in calling conventions (e.g. registers to place the
+// return value in).
+//   - internal_syscall     for 32-bit length (int, pid_t)
+//   - internal_syscall64   for 64-bit length (off_t)
+//   - internal_syscall_ptr for pointer and (s)size_t
+# define  internal_syscall      syscall
+# define  internal_syscall64    __syscall
+// Handle syscall renames manually
+# define SYS_stat SYS___stat50
+# define SYS_lstat SYS___lstat50
+# define SYS_fstat SYS___fstat50
+# define SYS_gettimeofday SYS___gettimeofday50
+# define SYS_wait4 SYS___wait450
+# define SYS_getdents SYS___getdents30
+# define SYS_sigaltstack SYS___sigaltstack14
+# define SYS_sigprocmask SYS___sigprocmask14
+# define SYS_nanosleep SYS___nanosleep50
+# if SANITIZER_WORDSIZE == 64
+#  define internal_syscall_ptr  __syscall
+# else
+#  define internal_syscall_ptr  syscall
+# endif
+#elif defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC)
 # define internal_syscall __syscall
 # else
 # define internal_syscall syscall
diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc
index c865d2cad84..d58c93983c5 100644
--- a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc
@@ -17,7 +17,7 @@ namespace __sanitizer {
 ThreadContextBase::ThreadContextBase(u32 tid)
     : tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0),
       status(ThreadStatusInvalid),
-      detached(false), parent_tid(0), next(0) {
+      detached(false), workerthread(false), parent_tid(0), next(0) {
   name[0] = '\0';
 }
 
@@ -52,14 +52,19 @@ void ThreadContextBase::SetJoined(void *arg) {
 }
 
 void ThreadContextBase::SetFinished() {
-  if (!detached)
-    status = ThreadStatusFinished;
+  // ThreadRegistry::FinishThread calls here in ThreadStatusCreated state
+  // for a thread that never actually started.  In that case the thread
+  // should go to ThreadStatusFinished regardless of whether it was created
+  // as detached.
+  if (!detached || status == ThreadStatusCreated) status = ThreadStatusFinished;
   OnFinished();
 }
 
-void ThreadContextBase::SetStarted(uptr _os_id, void *arg) {
+void ThreadContextBase::SetStarted(tid_t _os_id, bool _workerthread,
+                                   void *arg) {
   status = ThreadStatusRunning;
   os_id = _os_id;
+  workerthread = _workerthread;
   OnStarted(arg);
 }
 
@@ -190,7 +195,7 @@ static bool FindThreadContextByOsIdCallback(ThreadContextBase *tctx,
       tctx->status != ThreadStatusDead);
 }
 
-ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(uptr os_id) {
+ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {
   return FindThreadContextLocked(FindThreadContextByOsIdCallback,
                                  (void *)os_id);
 }
@@ -200,7 +205,8 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
   CHECK_LT(tid, n_contexts_);
   ThreadContextBase *tctx = threads_[tid];
   CHECK_NE(tctx, 0);
-  CHECK_EQ(ThreadStatusRunning, tctx->status);
+  CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
+           tctx->status);
   tctx->SetName(name);
 }
 
@@ -247,31 +253,43 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
   QuarantinePush(tctx);
 }
 
+// Normally this is called when the thread is about to exit.  If
+// called in ThreadStatusCreated state, then this thread was never
+// really started.  We just did CreateThread for a prospective new
+// thread before trying to create it, and then failed to actually
+// create it, and so never called StartThread.
 void ThreadRegistry::FinishThread(u32 tid) {
   BlockingMutexLock l(&mtx_);
   CHECK_GT(alive_threads_, 0);
   alive_threads_--;
-  CHECK_GT(running_threads_, 0);
-  running_threads_--;
   CHECK_LT(tid, n_contexts_);
   ThreadContextBase *tctx = threads_[tid];
   CHECK_NE(tctx, 0);
-  CHECK_EQ(ThreadStatusRunning, tctx->status);
+  bool dead = tctx->detached;
+  if (tctx->status == ThreadStatusRunning) {
+    CHECK_GT(running_threads_, 0);
+    running_threads_--;
+  } else {
+    // The thread never really existed.
+    CHECK_EQ(tctx->status, ThreadStatusCreated);
+    dead = true;
+  }
   tctx->SetFinished();
-  if (tctx->detached) {
+  if (dead) {
     tctx->SetDead();
     QuarantinePush(tctx);
   }
 }
 
-void ThreadRegistry::StartThread(u32 tid, uptr os_id, void *arg) {
+void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread,
+                                 void *arg) {
   BlockingMutexLock l(&mtx_);
   running_threads_++;
   CHECK_LT(tid, n_contexts_);
   ThreadContextBase *tctx = threads_[tid];
   CHECK_NE(tctx, 0);
   CHECK_EQ(ThreadStatusCreated, tctx->status);
-  tctx->SetStarted(os_id, arg);
+  tctx->SetStarted(os_id, workerthread, arg);
 }
 
 void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) {
diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_registry.h b/libsanitizer/sanitizer_common/sanitizer_thread_registry.h
index 8d5f1ea3988..16c2f86e1f9 100644
--- a/libsanitizer/sanitizer_common/sanitizer_thread_registry.h
+++ b/libsanitizer/sanitizer_common/sanitizer_thread_registry.h
@@ -37,12 +37,13 @@ class ThreadContextBase {
   const u32 tid;  // Thread ID. Main thread should have tid = 0.
   u64 unique_id;  // Unique thread ID.
   u32 reuse_count;  // Number of times this tid was reused.
-  uptr os_id;     // PID (used for reporting).
+  tid_t os_id;     // PID (used for reporting).
   uptr user_id;   // Some opaque user thread id (e.g. pthread_t).
   char name[64];  // As annotated by user.
 
   ThreadStatus status;
   bool detached;
+  bool workerthread;
 
   u32 parent_tid;
   ThreadContextBase *next;  // For storing thread contexts in a list.
@@ -52,7 +53,7 @@ class ThreadContextBase {
   void SetDead();
   void SetJoined(void *arg);
   void SetFinished();
-  void SetStarted(uptr _os_id, void *arg);
+  void SetStarted(tid_t _os_id, bool _workerthread, void *arg);
   void SetCreated(uptr _user_id, u64 _unique_id, bool _detached,
                   u32 _parent_tid, void *arg);
   void Reset();
@@ -106,14 +107,14 @@ class ThreadRegistry {
   // is found.
   ThreadContextBase *FindThreadContextLocked(FindThreadCallback cb,
                                              void *arg);
-  ThreadContextBase *FindThreadContextByOsIDLocked(uptr os_id);
+  ThreadContextBase *FindThreadContextByOsIDLocked(tid_t os_id);
 
   void SetThreadName(u32 tid, const char *name);
   void SetThreadNameByUserId(uptr user_id, const char *name);
   void DetachThread(u32 tid, void *arg);
   void JoinThread(u32 tid, void *arg);
   void FinishThread(u32 tid);
-  void StartThread(u32 tid, uptr os_id, void *arg);
+  void StartThread(u32 tid, tid_t os_id, bool workerthread, void *arg);
 
  private:
   const ThreadContextFactory context_factory_;
diff --git a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc
index 229225d95dd..ebf5ec09444 100644
--- a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc
@@ -43,7 +43,7 @@ static const uptr kDestroyedThread = -1;
 
 static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) {
   if (!size) return;
-  VPrintf(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
+  VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
   UnmapOrDie(dtv, size * sizeof(DTLS::DTV));
   atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
 }
@@ -56,7 +56,7 @@ static inline void DTLS_Resize(uptr new_size) {
       (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize");
   uptr num_live_dtls =
       atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
-  VPrintf(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
+  VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
   CHECK_LT(num_live_dtls, 1 << 20);
   uptr old_dtv_size = dtls.dtv_size;
   DTLS::DTV *old_dtv = dtls.dtv;
@@ -70,7 +70,7 @@ static inline void DTLS_Resize(uptr new_size) {
 
 void DTLS_Destroy() {
   if (!common_flags()->intercept_tls_get_addr) return;
-  VPrintf(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
+  VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
   uptr s = dtls.dtv_size;
   dtls.dtv_size = kDestroyedThread;  // Do this before unmap for AS-safety.
   DTLS_Deallocate(dtls.dtv, s);
@@ -95,28 +95,28 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   if (dtls.dtv[dso_id].beg) return 0;
   uptr tls_size = 0;
   uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
-  VPrintf(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
+  VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
              "num_live_dtls %zd\n",
           arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg,
           atomic_load(&number_of_live_dtls, memory_order_relaxed));
   if (dtls.last_memalign_ptr == tls_beg) {
     tls_size = dtls.last_memalign_size;
-    VPrintf(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
+    VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
         tls_beg, tls_size);
   } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
     // This is the static TLS block which was initialized / unpoisoned at thread
     // creation.
-    VPrintf(2, "__tls_get_addr: static tls: %p\n", tls_beg);
+    VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg);
     tls_size = 0;
   } else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) {
     // We may want to check gnu_get_libc_version().
     Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1;
     tls_size = header->size;
     tls_beg = header->start;
-    VPrintf(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
+    VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
         tls_beg, tls_size);
   } else {
-    VPrintf(2, "__tls_get_addr: Can't guess glibc version\n");
+    VReport(2, "__tls_get_addr: Can't guess glibc version\n");
     // This may happen inside the DTOR of main thread, so just ignore it.
     tls_size = 0;
   }
@@ -127,18 +127,26 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
 
 void DTLS_on_libc_memalign(void *ptr, uptr size) {
   if (!common_flags()->intercept_tls_get_addr) return;
-  VPrintf(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
+  VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
   dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
   dtls.last_memalign_size = size;
 }
 
 DTLS *DTLS_Get() { return &dtls; }
 
+bool DTLSInDestruction(DTLS *dtls) {
+  return dtls->dtv_size == kDestroyedThread;
+}
+
 #else
 void DTLS_on_libc_memalign(void *ptr, uptr size) {}
 DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res) { return 0; }
 DTLS *DTLS_Get() { return 0; }
 void DTLS_Destroy() {}
+bool DTLSInDestruction(DTLS *dtls) {
+  UNREACHABLE("dtls is unsupported on this platform!");
+}
+
 #endif  // SANITIZER_INTERCEPT_TLS_GET_ADDR
 
 }  // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.h b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.h
index e4f8c0c535e..19c8472c481 100644
--- a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.h
+++ b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.h
@@ -53,6 +53,8 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res, uptr static_tls_begin,
 void DTLS_on_libc_memalign(void *ptr, uptr size);
 DTLS *DTLS_Get();
 void DTLS_Destroy();  // Make sure to call this before the thread is destroyed.
+// Returns true if DTLS of suspended thread is in destruction process.
+bool DTLSInDestruction(DTLS *dtls);
 
 }  // namespace __sanitizer
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc
index eb1c133e4f4..7dba9e7ccd2 100644
--- a/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc
@@ -6,11 +6,11 @@
 //===----------------------------------------------------------------------===//
 //
 // This file contains the unwind.h-based (aka "slow") stack unwinding routines
-// available to the tools on Linux, Android, and FreeBSD.
+// available to the tools on Linux, Android, NetBSD and FreeBSD.
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 #include "sanitizer_common.h"
 #include "sanitizer_stacktrace.h"
 
@@ -76,7 +76,8 @@ void SanitizerInitializeUnwinder() {
 }
 #endif
 
-#ifdef __arm__
+#if defined(__arm__) && !SANITIZER_NETBSD
+// NetBSD uses dwarf EH
 #define UNWIND_STOP _URC_END_OF_STACK
 #define UNWIND_CONTINUE _URC_NO_REASON
 #else
@@ -163,4 +164,4 @@ void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context,
 
 }  // namespace __sanitizer
 
-#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cc b/libsanitizer/sanitizer_common/sanitizer_win.cc
index 785883ce767..84e66b43c99 100644
--- a/libsanitizer/sanitizer_common/sanitizer_win.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_win.cc
@@ -16,18 +16,34 @@
 #define WIN32_LEAN_AND_MEAN
 #define NOGDI
 #include <windows.h>
-#include <dbghelp.h>
 #include <io.h>
 #include <psapi.h>
 #include <stdlib.h>
 
 #include "sanitizer_common.h"
+#include "sanitizer_dbghelp.h"
+#include "sanitizer_file.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_mutex.h"
 #include "sanitizer_placement_new.h"
-#include "sanitizer_procmaps.h"
 #include "sanitizer_stacktrace.h"
 #include "sanitizer_symbolizer.h"
+#include "sanitizer_win_defs.h"
+
+// A macro to tell the compiler that this part of the code cannot be reached,
+// if the compiler supports this feature. Since we're using this in
+// code that is called when terminating the process, the expansion of the
+// macro should not terminate the process to avoid infinite recursion.
+#if defined(__clang__)
+# define BUILTIN_UNREACHABLE() __builtin_unreachable()
+#elif defined(__GNUC__) && \
+    (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))
+# define BUILTIN_UNREACHABLE() __builtin_unreachable()
+#elif defined(_MSC_VER)
+# define BUILTIN_UNREACHABLE() __assume(0)
+#else
+# define BUILTIN_UNREACHABLE()
+#endif
 
 namespace __sanitizer {
 
@@ -62,7 +78,7 @@ uptr internal_getpid() {
 
 // In contrast to POSIX, on Windows GetCurrentThreadId()
 // returns a system-unique identifier.
-uptr GetTid() {
+tid_t GetTid() {
   return GetCurrentThreadId();
 }
 
@@ -113,8 +129,24 @@ void UnmapOrDie(void *addr, uptr size) {
   }
 }
 
+static void *ReturnNullptrOnOOMOrDie(uptr size, const char *mem_type,
+                                     const char *mmap_type) {
+  error_t last_error = GetLastError();
+  if (last_error == ERROR_NOT_ENOUGH_MEMORY)
+    return nullptr;
+  ReportMmapFailureAndDie(size, mem_type, mmap_type, last_error);
+}
+
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  if (rv == 0)
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate");
+  return rv;
+}
+
 // We want to map a chunk of address space aligned to 'alignment'.
-void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type) {
   CHECK(IsPowerOfTwo(size));
   CHECK(IsPowerOfTwo(alignment));
 
@@ -124,7 +156,7 @@ void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
   uptr mapped_addr =
       (uptr)VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
   if (!mapped_addr)
-    ReportMmapFailureAndDie(size, mem_type, "allocate aligned", GetLastError());
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate aligned");
 
   // If we got it right on the first try, return. Otherwise, unmap it and go to
   // the slow path.
@@ -144,8 +176,7 @@ void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
     mapped_addr =
         (uptr)VirtualAlloc(0, size + alignment, MEM_RESERVE, PAGE_NOACCESS);
     if (!mapped_addr)
-      ReportMmapFailureAndDie(size, mem_type, "allocate aligned",
-                              GetLastError());
+      return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate aligned");
 
     // Find the aligned address.
     uptr aligned_addr = RoundUpTo(mapped_addr, alignment);
@@ -163,7 +194,7 @@ void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
 
   // Fail if we can't make this work quickly.
   if (retries == kMaxRetries && mapped_addr == 0)
-    ReportMmapFailureAndDie(size, mem_type, "allocate aligned", GetLastError());
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate aligned");
 
   return (void *)mapped_addr;
 }
@@ -202,6 +233,18 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
   return p;
 }
 
+void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) {
+  void *p = VirtualAlloc((LPVOID)fixed_addr, size,
+      MEM_COMMIT, PAGE_READWRITE);
+  if (p == 0) {
+    char mem_type[30];
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
+                      fixed_addr);
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate");
+  }
+  return p;
+}
+
 void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
   // FIXME: make this really NoReserve?
   return MmapOrDie(size, mem_type);
@@ -232,8 +275,7 @@ bool MprotectNoAccess(uptr addr, uptr size) {
   return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection);
 }
 
-
-void ReleaseMemoryToOS(uptr addr, uptr size) {
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
   // This is almost useless on 32-bits.
   // FIXME: add madvise-analog when we move to 64-bits.
 }
@@ -247,7 +289,8 @@ void DontDumpShadowMemory(uptr addr, uptr length) {
   // FIXME: add madvise-analog when we move to 64-bits.
 }
 
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) {
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found) {
   uptr address = 0;
   while (true) {
     MEMORY_BASIC_INFORMATION info;
@@ -372,6 +415,8 @@ void DumpProcessMap() {
 }
 #endif
 
+void PrintModuleMap() { }
+
 void DisableCoreDumperIfNecessary() {
   // Do nothing.
 }
@@ -381,9 +426,6 @@ void ReExec() {
 }
 
 void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
-#if !SANITIZER_GO
-  CovPrepareForSandboxing(args);
-#endif
 }
 
 bool StackSizeIsUnlimited() {
@@ -423,8 +465,6 @@ u64 NanoTime() {
 }
 
 void Abort() {
-  if (::IsDebuggerPresent())
-    __debugbreak();
   internal__exit(3);
 }
 
@@ -482,7 +522,7 @@ static uptr GetPreferredBase(const char *modname) {
 }
 
 void ListOfModules::init() {
-  clear();
+  clearOrInit();
   HANDLE cur_process = GetCurrentProcess();
 
   // Query the list of modules.  Start by assuming there are no more than 256
@@ -536,11 +576,14 @@ void ListOfModules::init() {
     LoadedModule cur_module;
     cur_module.set(module_name, adjusted_base);
     // We add the whole module as one single address range.
-    cur_module.addAddressRange(base_address, end_address, /*executable*/ true);
+    cur_module.addAddressRange(base_address, end_address, /*executable*/ true,
+                               /*writable*/ true);
     modules_.push_back(cur_module);
   }
   UnmapOrDie(hmodules, modules_buffer_size);
-};
+}
+
+void ListOfModules::fallbackInit() { clear(); }
 
 // We can't use atexit() directly at __asan_init time as the CRT is not fully
 // initialized at this point.  Place the functions into a vector and use
@@ -654,7 +697,13 @@ uptr internal_sched_yield() {
 }
 
 void internal__exit(int exitcode) {
-  ExitProcess(exitcode);
+  // ExitProcess runs some finalizers, so use TerminateProcess to avoid that.
+  // The debugger doesn't stop on TerminateProcess like it does on ExitProcess,
+  // so add our own breakpoint here.
+  if (::IsDebuggerPresent())
+    __debugbreak();
+  TerminateProcess(GetCurrentProcess(), exitcode);
+  BUILTIN_UNREACHABLE();
 }
 
 uptr internal_ftruncate(fd_t fd, uptr size) {
@@ -776,8 +825,8 @@ void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context,
   stack_frame.AddrFrame.Mode = AddrModeFlat;
   stack_frame.AddrStack.Mode = AddrModeFlat;
   while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
-                     &stack_frame, &ctx, NULL, &SymFunctionTableAccess64,
-                     &SymGetModuleBase64, NULL) &&
+                     &stack_frame, &ctx, NULL, SymFunctionTableAccess64,
+                     SymGetModuleBase64, NULL) &&
          size < Min(max_depth, kStackTraceMax)) {
     trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
   }
@@ -808,8 +857,35 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) {
   // FIXME: Decide what to do on Windows.
 }
 
-bool IsHandledDeadlySignal(int signum) {
+HandleSignalMode GetHandleSignalMode(int signum) {
   // FIXME: Decide what to do on Windows.
+  return kHandleSignalNo;
+}
+
+// Check based on flags if we should handle this exception.
+bool IsHandledDeadlyException(DWORD exceptionCode) {
+  switch (exceptionCode) {
+    case EXCEPTION_ACCESS_VIOLATION:
+    case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
+    case EXCEPTION_STACK_OVERFLOW:
+    case EXCEPTION_DATATYPE_MISALIGNMENT:
+    case EXCEPTION_IN_PAGE_ERROR:
+      return common_flags()->handle_segv;
+    case EXCEPTION_ILLEGAL_INSTRUCTION:
+    case EXCEPTION_PRIV_INSTRUCTION:
+    case EXCEPTION_BREAKPOINT:
+      return common_flags()->handle_sigill;
+    case EXCEPTION_FLT_DENORMAL_OPERAND:
+    case EXCEPTION_FLT_DIVIDE_BY_ZERO:
+    case EXCEPTION_FLT_INEXACT_RESULT:
+    case EXCEPTION_FLT_INVALID_OPERATION:
+    case EXCEPTION_FLT_OVERFLOW:
+    case EXCEPTION_FLT_STACK_CHECK:
+    case EXCEPTION_FLT_UNDERFLOW:
+    case EXCEPTION_INT_DIVIDE_BY_ZERO:
+    case EXCEPTION_INT_OVERFLOW:
+      return common_flags()->handle_sigfpe;
+  }
   return false;
 }
 
@@ -838,33 +914,99 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
   return true;
 }
 
-SignalContext SignalContext::Create(void *siginfo, void *context) {
+bool SignalContext::IsStackOverflow() const {
+  return GetType() == EXCEPTION_STACK_OVERFLOW;
+}
+
+void SignalContext::InitPcSpBp() {
   EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
   CONTEXT *context_record = (CONTEXT *)context;
 
-  uptr pc = (uptr)exception_record->ExceptionAddress;
+  pc = (uptr)exception_record->ExceptionAddress;
 #ifdef _WIN64
-  uptr bp = (uptr)context_record->Rbp;
-  uptr sp = (uptr)context_record->Rsp;
+  bp = (uptr)context_record->Rbp;
+  sp = (uptr)context_record->Rsp;
 #else
-  uptr bp = (uptr)context_record->Ebp;
-  uptr sp = (uptr)context_record->Esp;
+  bp = (uptr)context_record->Ebp;
+  sp = (uptr)context_record->Esp;
 #endif
-  uptr access_addr = exception_record->ExceptionInformation[1];
+}
 
+uptr SignalContext::GetAddress() const {
+  EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
+  return exception_record->ExceptionInformation[1];
+}
+
+bool SignalContext::IsMemoryAccess() const {
+  return GetWriteFlag() != SignalContext::UNKNOWN;
+}
+
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
+  EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
   // The contents of this array are documented at
   // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx
   // The first element indicates read as 0, write as 1, or execute as 8.  The
   // second element is the faulting address.
-  WriteFlag write_flag = SignalContext::UNKNOWN;
   switch (exception_record->ExceptionInformation[0]) {
-  case 0: write_flag = SignalContext::READ; break;
-  case 1: write_flag = SignalContext::WRITE; break;
-  case 8: write_flag = SignalContext::UNKNOWN; break;
+    case 0:
+      return SignalContext::READ;
+    case 1:
+      return SignalContext::WRITE;
+    case 8:
+      return SignalContext::UNKNOWN;
   }
-  bool is_memory_access = write_flag != SignalContext::UNKNOWN;
-  return SignalContext(context, access_addr, pc, sp, bp, is_memory_access,
-                       write_flag);
+  return SignalContext::UNKNOWN;
+}
+
+void SignalContext::DumpAllRegisters(void *context) {
+  // FIXME: Implement this.
+}
+
+int SignalContext::GetType() const {
+  return static_cast<const EXCEPTION_RECORD *>(siginfo)->ExceptionCode;
+}
+
+const char *SignalContext::Describe() const {
+  unsigned code = GetType();
+  // Get the string description of the exception if this is a known deadly
+  // exception.
+  switch (code) {
+    case EXCEPTION_ACCESS_VIOLATION:
+      return "access-violation";
+    case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
+      return "array-bounds-exceeded";
+    case EXCEPTION_STACK_OVERFLOW:
+      return "stack-overflow";
+    case EXCEPTION_DATATYPE_MISALIGNMENT:
+      return "datatype-misalignment";
+    case EXCEPTION_IN_PAGE_ERROR:
+      return "in-page-error";
+    case EXCEPTION_ILLEGAL_INSTRUCTION:
+      return "illegal-instruction";
+    case EXCEPTION_PRIV_INSTRUCTION:
+      return "priv-instruction";
+    case EXCEPTION_BREAKPOINT:
+      return "breakpoint";
+    case EXCEPTION_FLT_DENORMAL_OPERAND:
+      return "flt-denormal-operand";
+    case EXCEPTION_FLT_DIVIDE_BY_ZERO:
+      return "flt-divide-by-zero";
+    case EXCEPTION_FLT_INEXACT_RESULT:
+      return "flt-inexact-result";
+    case EXCEPTION_FLT_INVALID_OPERATION:
+      return "flt-invalid-operation";
+    case EXCEPTION_FLT_OVERFLOW:
+      return "flt-overflow";
+    case EXCEPTION_FLT_STACK_CHECK:
+      return "flt-stack-check";
+    case EXCEPTION_FLT_UNDERFLOW:
+      return "flt-underflow";
+    case EXCEPTION_INT_DIVIDE_BY_ZERO:
+      return "int-divide-by-zero";
+    case EXCEPTION_INT_OVERFLOW:
+      return "int-overflow";
+  }
+  return "unknown exception";
 }
 
 uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
@@ -910,21 +1052,15 @@ int WaitForProcess(pid_t pid) { return -1; }
 // FIXME implement on this platform.
 void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
 
+void CheckNoDeepBind(const char *filename, int flag) {
+  // Do nothing.
+}
 
-}  // namespace __sanitizer
-
-#if !SANITIZER_GO
-// Workaround to implement weak hooks on Windows. COFF doesn't directly support
-// weak symbols, but it does support /alternatename, which is similar. If the
-// user does not override the hook, we will use this default definition instead
-// of null.
-extern "C" void __sanitizer_print_memory_profile(int top_percent) {}
+// FIXME: implement on this platform.
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  UNIMPLEMENTED();
+}
 
-#ifdef _WIN64
-#pragma comment(linker, "/alternatename:__sanitizer_print_memory_profile=__sanitizer_default_print_memory_profile") // NOLINT
-#else
-#pragma comment(linker, "/alternatename:___sanitizer_print_memory_profile=___sanitizer_default_print_memory_profile") // NOLINT
-#endif
-#endif
+}  // namespace __sanitizer
 
 #endif  // _WIN32
diff --git a/libsanitizer/sanitizer_common/sanitizer_win.h b/libsanitizer/sanitizer_common/sanitizer_win.h
new file mode 100644
index 00000000000..c2d53a46fed
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_win.h
@@ -0,0 +1,24 @@
+//===-- sanitizer_win.h -----------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Windows-specific declarations.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_H
+#define SANITIZER_WIN_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+// Check based on flags if we should handle the exception.
+bool IsHandledDeadlyException(DWORD exceptionCode);
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_WINDOWS
+#endif  // SANITIZER_WIN_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_win_defs.h b/libsanitizer/sanitizer_common/sanitizer_win_defs.h
new file mode 100644
index 00000000000..e71081db575
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_win_defs.h
@@ -0,0 +1,151 @@
+//===-- sanitizer_win_defs.h ------------------------------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Common definitions for Windows-specific code.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_DEFS_H
+#define SANITIZER_WIN_DEFS_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+
+#ifndef WINAPI
+#ifdef _M_IX86
+#define WINAPI __stdcall
+#else
+#define WINAPI
+#endif
+#endif
+
+#if defined(_WIN64)
+#define WIN_SYM_PREFIX
+#else
+#define WIN_SYM_PREFIX "_"
+#endif
+
+// Intermediate macro to ensure the parameter is expanded before stringified.
+#define STRINGIFY_(A) #A
+#define STRINGIFY(A) STRINGIFY_(A)
+
+// ----------------- A workaround for the absence of weak symbols --------------
+// We don't have a direct equivalent of weak symbols when using MSVC, but we can
+// use the /alternatename directive to tell the linker to default a specific
+// symbol to a specific value.
+// Take into account that this is a pragma directive for the linker, so it will
+// be ignored by the compiler and the function will be marked as UNDEF in the
+// symbol table of the resulting object file. The linker won't find the default
+// implementation until it links with that object file.
+// So, suppose we provide a default implementation "fundef" for "fun", and this
+// is compiled into the object file "test.obj" including the pragma directive.
+// If we have some code with references to "fun" and we link that code with
+// "test.obj", it will work because the linker always link object files.
+// But, if "test.obj" is included in a static library, like "test.lib", then the
+// liker will only link to "test.obj" if necessary. If we only included the
+// definition of "fun", it won't link to "test.obj" (from test.lib) because
+// "fun" appears as UNDEF, so it doesn't resolve the symbol "fun", and will
+// result in a link error (the linker doesn't find the pragma directive).
+// So, a workaround is to force linkage with the modules that include weak
+// definitions, with the following macro: WIN_FORCE_LINK()
+
+#define WIN_WEAK_ALIAS(Name, Default)                                          \
+  __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY(Name) "="\
+                                             WIN_SYM_PREFIX STRINGIFY(Default)))
+
+#define WIN_FORCE_LINK(Name)                                                   \
+  __pragma(comment(linker, "/include:" WIN_SYM_PREFIX STRINGIFY(Name)))
+
+#define WIN_EXPORT(ExportedName, Name)                                         \
+  __pragma(comment(linker, "/export:" WIN_SYM_PREFIX STRINGIFY(ExportedName)   \
+                                  "=" WIN_SYM_PREFIX STRINGIFY(Name)))
+
+// We cannot define weak functions on Windows, but we can use WIN_WEAK_ALIAS()
+// which defines an alias to a default implementation, and only works when
+// linking statically.
+// So, to define a weak function "fun", we define a default implementation with
+// a different name "fun__def" and we create a "weak alias" fun = fun__def.
+// Then, users can override it just defining "fun".
+// We impose "extern "C"" because otherwise WIN_WEAK_ALIAS() will fail because
+// of name mangling.
+
+// Dummy name for default implementation of weak function.
+# define WEAK_DEFAULT_NAME(Name) Name##__def
+// Name for exported implementation of weak function.
+# define WEAK_EXPORT_NAME(Name) Name##__dll
+
+// Use this macro when you need to define and export a weak function from a
+// library. For example:
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+# define WIN_WEAK_EXPORT_DEF(ReturnType, Name, ...)                            \
+  WIN_WEAK_ALIAS(Name, WEAK_DEFAULT_NAME(Name))                                \
+  WIN_EXPORT(WEAK_EXPORT_NAME(Name), Name)                                     \
+  extern "C" ReturnType Name(__VA_ARGS__);                                     \
+  extern "C" ReturnType WEAK_DEFAULT_NAME(Name)(__VA_ARGS__)
+
+// Use this macro when you need to import a weak function from a library. It
+// defines a weak alias to the imported function from the dll. For example:
+//   WIN_WEAK_IMPORT_DEF(compare)
+# define WIN_WEAK_IMPORT_DEF(Name)                                             \
+  WIN_WEAK_ALIAS(Name, WEAK_EXPORT_NAME(Name))
+
+// So, for Windows we provide something similar to weak symbols in Linux, with
+// some differences:
+// + A default implementation must always be provided.
+//
+// + When linking statically it works quite similarly. For example:
+//
+//   // libExample.cc
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+//
+//   // client.cc
+//   // We can use the default implementation from the library:
+//   compare(1, 2);
+//   // Or we can override it:
+//   extern "C" bool compare (int a, int b) { return a >= b; }
+//
+//  And it will work fine. If we don't override the function, we need to ensure
+//  that the linker includes the object file with the default implementation.
+//  We can do so with the linker option "-wholearchive:".
+//
+// + When linking dynamically with a library (dll), weak functions are exported
+//  with "__dll" suffix. Clients can use the macro WIN_WEAK_IMPORT_DEF(fun)
+//  which defines a "weak alias" fun = fun__dll.
+//
+//   // libExample.cc
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+//
+//   // client.cc
+//   WIN_WEAK_IMPORT_DEF(compare)
+//   // We can use the default implementation from the library:
+//   compare(1, 2);
+//   // Or we can override it:
+//   extern "C" bool compare (int a, int b) { return a >= b; }
+//
+//  But if we override the function, the dlls don't have access to it (which
+//  is different in linux). If that is desired, the strong definition must be
+//  exported and interception can be used from the rest of the dlls.
+//
+//   // libExample.cc
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+//   // When initialized, check if the main executable defined "compare".
+//   int libExample_init() {
+//     uptr fnptr = __interception::InternalGetProcAddress(
+//         (void *)GetModuleHandleA(0), "compare");
+//     if (fnptr && !__interception::OverrideFunction((uptr)compare, fnptr, 0))
+//       abort();
+//     return 0;
+//   }
+//
+//   // client.cc
+//   WIN_WEAK_IMPORT_DEF(compare)
+//   // We override and export compare:
+//   extern "C" __declspec(dllexport) bool compare (int a, int b) {
+//     return a >= b;
+//   }
+//
+#endif // SANITIZER_WINDOWS
+#endif // SANITIZER_WIN_DEFS_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.cc b/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.cc
new file mode 100644
index 00000000000..6577a3606d1
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.cc
@@ -0,0 +1,100 @@
+//===-- sanitizer_win_dll_thunk.cc ----------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This file defines a family of thunks that should be statically linked into
+// the DLLs that have instrumentation in order to delegate the calls to the
+// shared runtime that lives in the main binary.
+// See https://github.com/google/sanitizers/issues/209 for the details.
+//===----------------------------------------------------------------------===//
+
+#ifdef SANITIZER_DLL_THUNK
+#include "sanitizer_win_defs.h"
+#include "sanitizer_win_dll_thunk.h"
+#include "interception/interception.h"
+
+extern "C" {
+void *WINAPI GetModuleHandleA(const char *module_name);
+void abort();
+}
+
+namespace __sanitizer {
+uptr dllThunkGetRealAddrOrDie(const char *name) {
+  uptr ret =
+      __interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name);
+  if (!ret)
+    abort();
+  return ret;
+}
+
+int dllThunkIntercept(const char* main_function, uptr dll_function) {
+  uptr wrapper = dllThunkGetRealAddrOrDie(main_function);
+  if (!__interception::OverrideFunction(dll_function, wrapper, 0))
+    abort();
+  return 0;
+}
+
+int dllThunkInterceptWhenPossible(const char* main_function,
+    const char* default_function, uptr dll_function) {
+  uptr wrapper = __interception::InternalGetProcAddress(
+    (void *)GetModuleHandleA(0), main_function);
+  if (!wrapper)
+    wrapper = dllThunkGetRealAddrOrDie(default_function);
+  if (!__interception::OverrideFunction(dll_function, wrapper, 0))
+    abort();
+  return 0;
+}
+} // namespace __sanitizer
+
+// Include Sanitizer Common interface.
+#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "sanitizer_common_interface.inc"
+
+#pragma section(".DLLTH$A", read)  // NOLINT
+#pragma section(".DLLTH$Z", read)  // NOLINT
+
+typedef void (*DllThunkCB)();
+extern "C" {
+__declspec(allocate(".DLLTH$A")) DllThunkCB __start_dll_thunk;
+__declspec(allocate(".DLLTH$Z")) DllThunkCB __stop_dll_thunk;
+}
+
+// Disable compiler warnings that show up if we declare our own version
+// of a compiler intrinsic (e.g. strlen).
+#pragma warning(disable: 4391)
+#pragma warning(disable: 4392)
+
+extern "C" int __dll_thunk_init() {
+  static bool flag = false;
+  // __dll_thunk_init is expected to be called by only one thread.
+  if (flag) return 0;
+  flag = true;
+
+  for (DllThunkCB *it = &__start_dll_thunk; it < &__stop_dll_thunk; ++it)
+    if (*it)
+      (*it)();
+
+  // In DLLs, the callbacks are expected to return 0,
+  // otherwise CRT initialization fails.
+  return 0;
+}
+
+// We want to call dll_thunk_init before C/C++ initializers / constructors are
+// executed, otherwise functions like memset might be invoked.
+#pragma section(".CRT$XIB", long, read)  // NOLINT
+__declspec(allocate(".CRT$XIB")) int (*__dll_thunk_preinit)() =
+    __dll_thunk_init;
+
+static void WINAPI dll_thunk_thread_init(void *mod, unsigned long reason,
+                                         void *reserved) {
+  if (reason == /*DLL_PROCESS_ATTACH=*/1) __dll_thunk_init();
+}
+
+#pragma section(".CRT$XLAB", long, read)  // NOLINT
+__declspec(allocate(".CRT$XLAB")) void (WINAPI *__dll_thunk_tls_init)(void *,
+    unsigned long, void *) = dll_thunk_thread_init;
+
+#endif // SANITIZER_DLL_THUNK
diff --git a/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.h b/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.h
new file mode 100644
index 00000000000..5a475e0798e
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.h
@@ -0,0 +1,180 @@
+//===-- sanitizer_win_dll_thunk.h -----------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This header provide helper macros to delegate calls to the shared runtime
+// that lives in the main executable. It should be included to dll_thunks that
+// will be linked to the dlls, when the sanitizer is a static library included
+// in the main executable.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_DLL_THUNK_H
+#define SANITIZER_WIN_DLL_THUNK_H
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+uptr dllThunkGetRealAddrOrDie(const char *name);
+
+int dllThunkIntercept(const char* main_function, uptr dll_function);
+
+int dllThunkInterceptWhenPossible(const char* main_function,
+    const char* default_function, uptr dll_function);
+}
+
+extern "C" int __dll_thunk_init();
+
+// ----------------- Function interception helper macros -------------------- //
+// Override dll_function with main_function from main executable.
+#define INTERCEPT_OR_DIE(main_function, dll_function)                          \
+  static int intercept_##dll_function() {                                      \
+    return __sanitizer::dllThunkIntercept(main_function, (__sanitizer::uptr)   \
+        dll_function);                                                         \
+  }                                                                            \
+  __pragma(section(".DLLTH$M", long, read))                                    \
+  __declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() =       \
+    intercept_##dll_function;
+
+// Try to override dll_function with main_function from main executable.
+// If main_function is not present, override dll_function with default_function.
+#define INTERCEPT_WHEN_POSSIBLE(main_function, default_function, dll_function) \
+  static int intercept_##dll_function() {                                      \
+    return __sanitizer::dllThunkInterceptWhenPossible(main_function,           \
+        default_function, (__sanitizer::uptr)dll_function);                    \
+  }                                                                            \
+  __pragma(section(".DLLTH$M", long, read))                                    \
+  __declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() =       \
+    intercept_##dll_function;
+
+// -------------------- Function interception macros ------------------------ //
+// Special case of hooks -- ASan own interface functions.  Those are only called
+// after __asan_init, thus an empty implementation is sufficient.
+#define INTERCEPT_SANITIZER_FUNCTION(name)                                     \
+  extern "C" __declspec(noinline) void name() {                                \
+    volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__;                  \
+    static const char function_name[] = #name;                                 \
+    for (const char* ptr = &function_name[0]; *ptr; ++ptr)                     \
+      prevent_icf ^= *ptr;                                                     \
+    (void)prevent_icf;                                                         \
+    __debugbreak();                                                            \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name)
+
+// Special case of hooks -- Weak functions, could be redefined in the main
+// executable, but that is not necessary, so we shouldn't die if we can not find
+// a reference. Instead, when the function is not present in the main executable
+// we consider the default impl provided by asan library.
+#define INTERCEPT_SANITIZER_WEAK_FUNCTION(name)                                \
+  extern "C" __declspec(noinline) void name() {                                \
+    volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__;                  \
+    static const char function_name[] = #name;                                 \
+    for (const char* ptr = &function_name[0]; *ptr; ++ptr)                     \
+      prevent_icf ^= *ptr;                                                     \
+    (void)prevent_icf;                                                         \
+    __debugbreak();                                                            \
+  }                                                                            \
+  INTERCEPT_WHEN_POSSIBLE(#name, STRINGIFY(WEAK_EXPORT_NAME(name)), name)
+
+// We can't define our own version of strlen etc. because that would lead to
+// link-time or even type mismatch errors.  Instead, we can declare a function
+// just to be able to get its address.  Me may miss the first few calls to the
+// functions since it can be called before __dll_thunk_init, but that would lead
+// to false negatives in the startup code before user's global initializers,
+// which isn't a big deal.
+#define INTERCEPT_LIBRARY_FUNCTION(name)                                       \
+  extern "C" void name();                                                      \
+  INTERCEPT_OR_DIE(WRAPPER_NAME(name), name)
+
+// Use these macros for functions that could be called before __dll_thunk_init()
+// is executed and don't lead to errors if defined (free, malloc, etc).
+#define INTERCEPT_WRAP_V_V(name)                                               \
+  extern "C" void name() {                                                     \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn();                                                                      \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_V_W(name)                                               \
+  extern "C" void name(void *arg) {                                            \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn(arg);                                                                   \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_V_WW(name)                                              \
+  extern "C" void name(void *arg1, void *arg2) {                               \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn(arg1, arg2);                                                            \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_V_WWW(name)                                             \
+  extern "C" void name(void *arg1, void *arg2, void *arg3) {                   \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn(arg1, arg2, arg3);                                                      \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_V(name)                                               \
+  extern "C" void *name() {                                                    \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn();                                                               \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_W(name)                                               \
+  extern "C" void *name(void *arg) {                                           \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg);                                                            \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WW(name)                                              \
+  extern "C" void *name(void *arg1, void *arg2) {                              \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2);                                                     \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWW(name)                                             \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3) {                  \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3);                                               \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWWW(name)                                            \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4) {      \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3, arg4);                                         \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWWWW(name)                                           \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4,        \
+                        void *arg5) {                                          \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3, arg4, arg5);                                   \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWWWWW(name)                                          \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4,        \
+                        void *arg5, void *arg6) {                              \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3, arg4, arg5, arg6);                             \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#endif // SANITIZER_WIN_DLL_THUNK_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc b/libsanitizer/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc
new file mode 100644
index 00000000000..f3b3037b276
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cc
@@ -0,0 +1,19 @@
+//===-- santizer_win_dynamic_runtime_thunk.cc -----------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines things that need to be present in the application modules
+// to interact with Sanitizer Common, when it is included in a dll.
+//
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
+#define SANITIZER_IMPORT_INTERFACE 1
+#include "sanitizer_win_defs.h"
+// Define weak alias for all weak functions imported from sanitizer common.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
+#include "sanitizer_common_interface.inc"
+#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
diff --git a/libsanitizer/sanitizer_common/sanitizer_win_weak_interception.cc b/libsanitizer/sanitizer_common/sanitizer_win_weak_interception.cc
new file mode 100644
index 00000000000..dffec0c7943
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_win_weak_interception.cc
@@ -0,0 +1,92 @@
+//===-- sanitizer_win_weak_interception.cc --------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This module should be included in the sanitizer when it is implemented as a
+// shared library on Windows (dll), in order to delegate the calls of weak
+// functions to the implementation in the main executable when a strong
+// definition is provided.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_WINDOWS && SANITIZER_DYNAMIC
+#include "sanitizer_win_weak_interception.h"
+#include "sanitizer_allocator_interface.h"
+#include "sanitizer_interface_internal.h"
+#include "sanitizer_win_defs.h"
+#include "interception/interception.h"
+
+extern "C" {
+void *WINAPI GetModuleHandleA(const char *module_name);
+void abort();
+}
+
+namespace __sanitizer {
+// Try to get a pointer to real_function in the main module and override
+// dll_function with that pointer. If the function isn't found, nothing changes.
+int interceptWhenPossible(uptr dll_function, const char *real_function) {
+  uptr real = __interception::InternalGetProcAddress(
+      (void *)GetModuleHandleA(0), real_function);
+  if (real && !__interception::OverrideFunction((uptr)dll_function, real, 0))
+    abort();
+  return 0;
+}
+} // namespace __sanitizer
+
+// Declare weak hooks.
+extern "C" {
+void __sanitizer_weak_hook_memcmp(uptr called_pc, const void *s1,
+                                  const void *s2, uptr n, int result);
+void __sanitizer_weak_hook_strcmp(uptr called_pc, const char *s1,
+                                  const char *s2, int result);
+void __sanitizer_weak_hook_strncmp(uptr called_pc, const char *s1,
+                                   const char *s2, uptr n, int result);
+void __sanitizer_weak_hook_strstr(uptr called_pc, const char *s1,
+                                  const char *s2, char *result);
+}
+
+// Include Sanitizer Common interface.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "sanitizer_common_interface.inc"
+
+#pragma section(".WEAK$A", read)  // NOLINT
+#pragma section(".WEAK$Z", read)  // NOLINT
+
+typedef void (*InterceptCB)();
+extern "C" {
+__declspec(allocate(".WEAK$A")) InterceptCB __start_weak_list;
+__declspec(allocate(".WEAK$Z")) InterceptCB __stop_weak_list;
+}
+
+static int weak_intercept_init() {
+  static bool flag = false;
+  // weak_interception_init is expected to be called by only one thread.
+  if (flag) return 0;
+  flag = true;
+
+  for (InterceptCB *it = &__start_weak_list; it < &__stop_weak_list; ++it)
+    if (*it)
+      (*it)();
+
+  // In DLLs, the callbacks are expected to return 0,
+  // otherwise CRT initialization fails.
+  return 0;
+}
+
+#pragma section(".CRT$XIB", long, read)  // NOLINT
+__declspec(allocate(".CRT$XIB")) int (*__weak_intercept_preinit)() =
+    weak_intercept_init;
+
+static void WINAPI weak_intercept_thread_init(void *mod, unsigned long reason,
+                                              void *reserved) {
+  if (reason == /*DLL_PROCESS_ATTACH=*/1) weak_intercept_init();
+}
+
+#pragma section(".CRT$XLAB", long, read)  // NOLINT
+__declspec(allocate(".CRT$XLAB")) void(WINAPI *__weak_intercept_tls_init)(
+    void *, unsigned long, void *) = weak_intercept_thread_init;
+
+#endif // SANITIZER_WINDOWS && SANITIZER_DYNAMIC
diff --git a/libsanitizer/sanitizer_common/sanitizer_win_weak_interception.h b/libsanitizer/sanitizer_common/sanitizer_win_weak_interception.h
new file mode 100644
index 00000000000..873f9b822ab
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_win_weak_interception.h
@@ -0,0 +1,31 @@
+//===-- sanitizer_win_weak_interception.h ---------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This header provide helper macros to delegate calls of weak functions to the
+// implementation in the main executable when a strong definition is present.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_WEAK_INTERCEPTION_H
+#define SANITIZER_WIN_WEAK_INTERCEPTION_H
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+int interceptWhenPossible(uptr dll_function, const char *real_function);
+}
+
+// ----------------- Function interception helper macros -------------------- //
+// Weak functions, could be redefined in the main executable, but that is not
+// necessary, so we shouldn't die if we can not find a reference.
+#define INTERCEPT_WEAK(Name) interceptWhenPossible((uptr) Name, #Name);
+
+#define INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)                                \
+  static int intercept_##Name() {                                              \
+    return __sanitizer::interceptWhenPossible((__sanitizer::uptr) Name, #Name);\
+  }                                                                            \
+  __pragma(section(".WEAK$M", long, read))                                     \
+  __declspec(allocate(".WEAK$M")) int (*__weak_intercept_##Name)() =           \
+      intercept_##Name;
+
+#endif // SANITIZER_WIN_WEAK_INTERCEPTION_H
diff --git a/libsanitizer/tsan/Makefile.am b/libsanitizer/tsan/Makefile.am
index 53a8d1cb0cd..c218983e2fe 100644
--- a/libsanitizer/tsan/Makefile.am
+++ b/libsanitizer/tsan/Makefile.am
@@ -15,6 +15,7 @@ nodist_toolexeclib_HEADERS = libtsan_preinit.o
 tsan_files = \
 	tsan_clock.cc \
 	tsan_debugging.cc \
+	tsan_external.cc \
 	tsan_fd.cc \
 	tsan_flags.cc \
 	tsan_ignoreset.cc \
diff --git a/libsanitizer/tsan/Makefile.in b/libsanitizer/tsan/Makefile.in
index 770c053e64f..e32e9c0bfa1 100644
--- a/libsanitizer/tsan/Makefile.in
+++ b/libsanitizer/tsan/Makefile.in
@@ -106,18 +106,19 @@ am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \
 	"$(DESTDIR)$(toolexeclibdir)"
 LTLIBRARIES = $(toolexeclib_LTLIBRARIES)
 am__DEPENDENCIES_1 =
-am__objects_1 = tsan_clock.lo tsan_debugging.lo tsan_fd.lo \
-	tsan_flags.lo tsan_ignoreset.lo tsan_interceptors.lo \
-	tsan_interceptors_mac.lo tsan_interface_ann.lo \
-	tsan_interface_atomic.lo tsan_interface.lo \
-	tsan_interface_java.lo tsan_libdispatch_mac.lo \
-	tsan_malloc_mac.lo tsan_md5.lo tsan_mman.lo tsan_mutex.lo \
-	tsan_mutexset.lo tsan_new_delete.lo tsan_platform_linux.lo \
-	tsan_platform_mac.lo tsan_platform_posix.lo \
-	tsan_platform_windows.lo tsan_report.lo tsan_rtl.lo \
-	tsan_rtl_mutex.lo tsan_rtl_proc.lo tsan_rtl_report.lo \
-	tsan_rtl_thread.lo tsan_stack_trace.lo tsan_stat.lo \
-	tsan_suppressions.lo tsan_symbolize.lo tsan_sync.lo
+am__objects_1 = tsan_clock.lo tsan_debugging.lo tsan_external.lo \
+	tsan_fd.lo tsan_flags.lo tsan_ignoreset.lo \
+	tsan_interceptors.lo tsan_interceptors_mac.lo \
+	tsan_interface_ann.lo tsan_interface_atomic.lo \
+	tsan_interface.lo tsan_interface_java.lo \
+	tsan_libdispatch_mac.lo tsan_malloc_mac.lo tsan_md5.lo \
+	tsan_mman.lo tsan_mutex.lo tsan_mutexset.lo tsan_new_delete.lo \
+	tsan_platform_linux.lo tsan_platform_mac.lo \
+	tsan_platform_posix.lo tsan_platform_windows.lo tsan_report.lo \
+	tsan_rtl.lo tsan_rtl_mutex.lo tsan_rtl_proc.lo \
+	tsan_rtl_report.lo tsan_rtl_thread.lo tsan_stack_trace.lo \
+	tsan_stat.lo tsan_suppressions.lo tsan_symbolize.lo \
+	tsan_sync.lo
 am_libtsan_la_OBJECTS = $(am__objects_1)
 libtsan_la_OBJECTS = $(am_libtsan_la_OBJECTS)
 libtsan_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \
@@ -319,6 +320,7 @@ nodist_toolexeclib_HEADERS = libtsan_preinit.o
 tsan_files = \
 	tsan_clock.cc \
 	tsan_debugging.cc \
+	tsan_external.cc \
 	tsan_fd.cc \
 	tsan_flags.cc \
 	tsan_ignoreset.cc \
@@ -480,6 +482,7 @@ distclean-compile:
 
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_clock.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_debugging.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_external.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_fd.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_flags.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_ignoreset.Plo@am__quote@
diff --git a/libsanitizer/tsan/tsan_clock.cc b/libsanitizer/tsan/tsan_clock.cc
index 23f9228a672..c2b5b582ca3 100644
--- a/libsanitizer/tsan/tsan_clock.cc
+++ b/libsanitizer/tsan/tsan_clock.cc
@@ -59,20 +59,13 @@
 // an exclusive lock; ThreadClock's are private to respective threads and so
 // do not need any protection.
 //
-// Description of ThreadClock state:
-// clk_ - fixed size vector clock.
-// nclk_ - effective size of the vector clock (the rest is zeros).
-// tid_ - index of the thread associated with he clock ("current thread").
-// last_acquire_ - current thread time when it acquired something from
-//   other threads.
-//
 // Description of SyncClock state:
 // clk_ - variable size vector clock, low kClkBits hold timestamp,
 //   the remaining bits hold "acquired" flag (the actual value is thread's
 //   reused counter);
 //   if acquried == thr->reused_, then the respective thread has already
-//   acquired this clock (except possibly dirty_tids_).
-// dirty_tids_ - holds up to two indeces in the vector clock that other threads
+//   acquired this clock (except possibly for dirty elements).
+// dirty_ - holds up to two indeces in the vector clock that other threads
 //   need to acquire regardless of "acquired" flag value;
 // release_store_tid_ - denotes that the clock state is a result of
 //   release-store operation by the thread with release_store_tid_ index.
@@ -88,18 +81,51 @@
 
 namespace __tsan {
 
+static atomic_uint32_t *ref_ptr(ClockBlock *cb) {
+  return reinterpret_cast<atomic_uint32_t *>(&cb->table[ClockBlock::kRefIdx]);
+}
+
+// Drop reference to the first level block idx.
+static void UnrefClockBlock(ClockCache *c, u32 idx, uptr blocks) {
+  ClockBlock *cb = ctx->clock_alloc.Map(idx);
+  atomic_uint32_t *ref = ref_ptr(cb);
+  u32 v = atomic_load(ref, memory_order_acquire);
+  for (;;) {
+    CHECK_GT(v, 0);
+    if (v == 1)
+      break;
+    if (atomic_compare_exchange_strong(ref, &v, v - 1, memory_order_acq_rel))
+      return;
+  }
+  // First level block owns second level blocks, so them as well.
+  for (uptr i = 0; i < blocks; i++)
+    ctx->clock_alloc.Free(c, cb->table[ClockBlock::kBlockIdx - i]);
+  ctx->clock_alloc.Free(c, idx);
+}
+
 ThreadClock::ThreadClock(unsigned tid, unsigned reused)
     : tid_(tid)
-    , reused_(reused + 1) {  // 0 has special meaning
+    , reused_(reused + 1)  // 0 has special meaning
+    , cached_idx_()
+    , cached_size_()
+    , cached_blocks_() {
   CHECK_LT(tid, kMaxTidInClock);
   CHECK_EQ(reused_, ((u64)reused_ << kClkBits) >> kClkBits);
   nclk_ = tid_ + 1;
   last_acquire_ = 0;
   internal_memset(clk_, 0, sizeof(clk_));
-  clk_[tid_].reused = reused_;
 }
 
-void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
+void ThreadClock::ResetCached(ClockCache *c) {
+  if (cached_idx_) {
+    UnrefClockBlock(c, cached_idx_, cached_blocks_);
+    cached_idx_ = 0;
+    cached_size_ = 0;
+    cached_blocks_ = 0;
+  }
+}
+
+void ThreadClock::acquire(ClockCache *c, SyncClock *src) {
   DCHECK_LE(nclk_, kMaxTid);
   DCHECK_LE(src->size_, kMaxTid);
   CPP_STAT_INC(StatClockAcquire);
@@ -111,52 +137,46 @@ void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
     return;
   }
 
-  // Check if we've already acquired src after the last release operation on src
   bool acquired = false;
-  if (nclk > tid_) {
-    CPP_STAT_INC(StatClockAcquireLarge);
-    if (src->elem(tid_).reused == reused_) {
-      CPP_STAT_INC(StatClockAcquireRepeat);
-      for (unsigned i = 0; i < kDirtyTids; i++) {
-        unsigned tid = src->dirty_tids_[i];
-        if (tid != kInvalidTid) {
-          u64 epoch = src->elem(tid).epoch;
-          if (clk_[tid].epoch < epoch) {
-            clk_[tid].epoch = epoch;
-            acquired = true;
-          }
-        }
-      }
-      if (acquired) {
-        CPP_STAT_INC(StatClockAcquiredSomething);
-        last_acquire_ = clk_[tid_].epoch;
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    SyncClock::Dirty dirty = src->dirty_[i];
+    unsigned tid = dirty.tid;
+    if (tid != kInvalidTid) {
+      if (clk_[tid] < dirty.epoch) {
+        clk_[tid] = dirty.epoch;
+        acquired = true;
       }
-      return;
     }
   }
 
-  // O(N) acquire.
-  CPP_STAT_INC(StatClockAcquireFull);
-  nclk_ = max(nclk_, nclk);
-  for (uptr i = 0; i < nclk; i++) {
-    u64 epoch = src->elem(i).epoch;
-    if (clk_[i].epoch < epoch) {
-      clk_[i].epoch = epoch;
-      acquired = true;
+  // Check if we've already acquired src after the last release operation on src
+  if (tid_ >= nclk || src->elem(tid_).reused != reused_) {
+    // O(N) acquire.
+    CPP_STAT_INC(StatClockAcquireFull);
+    nclk_ = max(nclk_, nclk);
+    u64 *dst_pos = &clk_[0];
+    for (ClockElem &src_elem : *src) {
+      u64 epoch = src_elem.epoch;
+      if (*dst_pos < epoch) {
+        *dst_pos = epoch;
+        acquired = true;
+      }
+      dst_pos++;
     }
-  }
 
-  // Remember that this thread has acquired this clock.
-  if (nclk > tid_)
-    src->elem(tid_).reused = reused_;
+    // Remember that this thread has acquired this clock.
+    if (nclk > tid_)
+      src->elem(tid_).reused = reused_;
+  }
 
   if (acquired) {
     CPP_STAT_INC(StatClockAcquiredSomething);
-    last_acquire_ = clk_[tid_].epoch;
+    last_acquire_ = clk_[tid_];
+    ResetCached(c);
   }
 }
 
-void ThreadClock::release(ClockCache *c, SyncClock *dst) const {
+void ThreadClock::release(ClockCache *c, SyncClock *dst) {
   DCHECK_LE(nclk_, kMaxTid);
   DCHECK_LE(dst->size_, kMaxTid);
 
@@ -176,7 +196,7 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) const {
   // since the last release on dst. If so, we need to update
   // only dst->elem(tid_).
   if (dst->elem(tid_).epoch > last_acquire_) {
-    UpdateCurrentThread(dst);
+    UpdateCurrentThread(c, dst);
     if (dst->release_store_tid_ != tid_ ||
         dst->release_store_reused_ != reused_)
       dst->release_store_tid_ = kInvalidTid;
@@ -185,23 +205,24 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) const {
 
   // O(N) release.
   CPP_STAT_INC(StatClockReleaseFull);
+  dst->Unshare(c);
   // First, remember whether we've acquired dst.
   bool acquired = IsAlreadyAcquired(dst);
   if (acquired)
     CPP_STAT_INC(StatClockReleaseAcquired);
   // Update dst->clk_.
-  for (uptr i = 0; i < nclk_; i++) {
-    ClockElem &ce = dst->elem(i);
-    ce.epoch = max(ce.epoch, clk_[i].epoch);
+  dst->FlushDirty();
+  uptr i = 0;
+  for (ClockElem &ce : *dst) {
+    ce.epoch = max(ce.epoch, clk_[i]);
     ce.reused = 0;
+    i++;
   }
   // Clear 'acquired' flag in the remaining elements.
   if (nclk_ < dst->size_)
     CPP_STAT_INC(StatClockReleaseClearTail);
   for (uptr i = nclk_; i < dst->size_; i++)
     dst->elem(i).reused = 0;
-  for (unsigned i = 0; i < kDirtyTids; i++)
-    dst->dirty_tids_[i] = kInvalidTid;
   dst->release_store_tid_ = kInvalidTid;
   dst->release_store_reused_ = 0;
   // If we've acquired dst, remember this fact,
@@ -210,11 +231,37 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) const {
     dst->elem(tid_).reused = reused_;
 }
 
-void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) const {
+void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) {
   DCHECK_LE(nclk_, kMaxTid);
   DCHECK_LE(dst->size_, kMaxTid);
   CPP_STAT_INC(StatClockStore);
 
+  if (dst->size_ == 0 && cached_idx_ != 0) {
+    // Reuse the cached clock.
+    // Note: we could reuse/cache the cached clock in more cases:
+    // we could update the existing clock and cache it, or replace it with the
+    // currently cached clock and release the old one. And for a shared
+    // existing clock, we could replace it with the currently cached;
+    // or unshare, update and cache. But, for simplicity, we currnetly reuse
+    // cached clock only when the target clock is empty.
+    dst->tab_ = ctx->clock_alloc.Map(cached_idx_);
+    dst->tab_idx_ = cached_idx_;
+    dst->size_ = cached_size_;
+    dst->blocks_ = cached_blocks_;
+    CHECK_EQ(dst->dirty_[0].tid, kInvalidTid);
+    // The cached clock is shared (immutable),
+    // so this is where we store the current clock.
+    dst->dirty_[0].tid = tid_;
+    dst->dirty_[0].epoch = clk_[tid_];
+    dst->release_store_tid_ = tid_;
+    dst->release_store_reused_ = reused_;
+    // Rememeber that we don't need to acquire it in future.
+    dst->elem(tid_).reused = reused_;
+    // Grab a reference.
+    atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
+    return;
+  }
+
   // Check if we need to resize dst.
   if (dst->size_ < nclk_)
     dst->Resize(c, nclk_);
@@ -223,32 +270,41 @@ void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) const {
       dst->release_store_reused_ == reused_ &&
       dst->elem(tid_).epoch > last_acquire_) {
     CPP_STAT_INC(StatClockStoreFast);
-    UpdateCurrentThread(dst);
+    UpdateCurrentThread(c, dst);
     return;
   }
 
   // O(N) release-store.
   CPP_STAT_INC(StatClockStoreFull);
-  for (uptr i = 0; i < nclk_; i++) {
-    ClockElem &ce = dst->elem(i);
-    ce.epoch = clk_[i].epoch;
+  dst->Unshare(c);
+  // Note: dst can be larger than this ThreadClock.
+  // This is fine since clk_ beyond size is all zeros.
+  uptr i = 0;
+  for (ClockElem &ce : *dst) {
+    ce.epoch = clk_[i];
     ce.reused = 0;
+    i++;
   }
-  // Clear the tail of dst->clk_.
-  if (nclk_ < dst->size_) {
-    for (uptr i = nclk_; i < dst->size_; i++) {
-      ClockElem &ce = dst->elem(i);
-      ce.epoch = 0;
-      ce.reused = 0;
-    }
-    CPP_STAT_INC(StatClockStoreTail);
-  }
-  for (unsigned i = 0; i < kDirtyTids; i++)
-    dst->dirty_tids_[i] = kInvalidTid;
+  for (uptr i = 0; i < kDirtyTids; i++)
+    dst->dirty_[i].tid = kInvalidTid;
   dst->release_store_tid_ = tid_;
   dst->release_store_reused_ = reused_;
   // Rememeber that we don't need to acquire it in future.
   dst->elem(tid_).reused = reused_;
+
+  // If the resulting clock is cachable, cache it for future release operations.
+  // The clock is always cachable if we released to an empty sync object.
+  if (cached_idx_ == 0 && dst->Cachable()) {
+    // Grab a reference to the ClockBlock.
+    atomic_uint32_t *ref = ref_ptr(dst->tab_);
+    if (atomic_load(ref, memory_order_acquire) == 1)
+      atomic_store_relaxed(ref, 2);
+    else
+      atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
+    cached_idx_ = dst->tab_idx_;
+    cached_size_ = dst->size_;
+    cached_blocks_ = dst->blocks_;
+  }
 }
 
 void ThreadClock::acq_rel(ClockCache *c, SyncClock *dst) {
@@ -258,157 +314,248 @@ void ThreadClock::acq_rel(ClockCache *c, SyncClock *dst) {
 }
 
 // Updates only single element related to the current thread in dst->clk_.
-void ThreadClock::UpdateCurrentThread(SyncClock *dst) const {
+void ThreadClock::UpdateCurrentThread(ClockCache *c, SyncClock *dst) const {
   // Update the threads time, but preserve 'acquired' flag.
-  dst->elem(tid_).epoch = clk_[tid_].epoch;
-
   for (unsigned i = 0; i < kDirtyTids; i++) {
-    if (dst->dirty_tids_[i] == tid_) {
-      CPP_STAT_INC(StatClockReleaseFast1);
-      return;
-    }
-    if (dst->dirty_tids_[i] == kInvalidTid) {
-      CPP_STAT_INC(StatClockReleaseFast2);
-      dst->dirty_tids_[i] = tid_;
+    SyncClock::Dirty *dirty = &dst->dirty_[i];
+    const unsigned tid = dirty->tid;
+    if (tid == tid_ || tid == kInvalidTid) {
+      CPP_STAT_INC(StatClockReleaseFast);
+      dirty->tid = tid_;
+      dirty->epoch = clk_[tid_];
       return;
     }
   }
   // Reset all 'acquired' flags, O(N).
+  // We are going to touch dst elements, so we need to unshare it.
+  dst->Unshare(c);
   CPP_STAT_INC(StatClockReleaseSlow);
+  dst->elem(tid_).epoch = clk_[tid_];
   for (uptr i = 0; i < dst->size_; i++)
     dst->elem(i).reused = 0;
-  for (unsigned i = 0; i < kDirtyTids; i++)
-    dst->dirty_tids_[i] = kInvalidTid;
+  dst->FlushDirty();
 }
 
-// Checks whether the current threads has already acquired src.
+// Checks whether the current thread has already acquired src.
 bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const {
   if (src->elem(tid_).reused != reused_)
     return false;
   for (unsigned i = 0; i < kDirtyTids; i++) {
-    unsigned tid = src->dirty_tids_[i];
-    if (tid != kInvalidTid) {
-      if (clk_[tid].epoch < src->elem(tid).epoch)
+    SyncClock::Dirty dirty = src->dirty_[i];
+    if (dirty.tid != kInvalidTid) {
+      if (clk_[dirty.tid] < dirty.epoch)
         return false;
     }
   }
   return true;
 }
 
+// Sets a single element in the vector clock.
+// This function is called only from weird places like AcquireGlobal.
+void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) {
+  DCHECK_LT(tid, kMaxTid);
+  DCHECK_GE(v, clk_[tid]);
+  clk_[tid] = v;
+  if (nclk_ <= tid)
+    nclk_ = tid + 1;
+  last_acquire_ = clk_[tid_];
+  ResetCached(c);
+}
+
+void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) {
+  printf("clock=[");
+  for (uptr i = 0; i < nclk_; i++)
+    printf("%s%llu", i == 0 ? "" : ",", clk_[i]);
+  printf("] tid=%u/%u last_acq=%llu", tid_, reused_, last_acquire_);
+}
+
+SyncClock::SyncClock() {
+  ResetImpl();
+}
+
+SyncClock::~SyncClock() {
+  // Reset must be called before dtor.
+  CHECK_EQ(size_, 0);
+  CHECK_EQ(blocks_, 0);
+  CHECK_EQ(tab_, 0);
+  CHECK_EQ(tab_idx_, 0);
+}
+
+void SyncClock::Reset(ClockCache *c) {
+  if (size_)
+    UnrefClockBlock(c, tab_idx_, blocks_);
+  ResetImpl();
+}
+
+void SyncClock::ResetImpl() {
+  tab_ = 0;
+  tab_idx_ = 0;
+  size_ = 0;
+  blocks_ = 0;
+  release_store_tid_ = kInvalidTid;
+  release_store_reused_ = 0;
+  for (uptr i = 0; i < kDirtyTids; i++)
+    dirty_[i].tid = kInvalidTid;
+}
+
 void SyncClock::Resize(ClockCache *c, uptr nclk) {
   CPP_STAT_INC(StatClockReleaseResize);
-  if (RoundUpTo(nclk, ClockBlock::kClockCount) <=
-      RoundUpTo(size_, ClockBlock::kClockCount)) {
-    // Growing within the same block.
+  Unshare(c);
+  if (nclk <= capacity()) {
     // Memory is already allocated, just increase the size.
     size_ = nclk;
     return;
   }
-  if (nclk <= ClockBlock::kClockCount) {
+  if (size_ == 0) {
     // Grow from 0 to one-level table.
     CHECK_EQ(size_, 0);
+    CHECK_EQ(blocks_, 0);
     CHECK_EQ(tab_, 0);
     CHECK_EQ(tab_idx_, 0);
-    size_ = nclk;
     tab_idx_ = ctx->clock_alloc.Alloc(c);
     tab_ = ctx->clock_alloc.Map(tab_idx_);
     internal_memset(tab_, 0, sizeof(*tab_));
-    return;
-  }
-  // Growing two-level table.
-  if (size_ == 0) {
-    // Allocate first level table.
-    tab_idx_ = ctx->clock_alloc.Alloc(c);
-    tab_ = ctx->clock_alloc.Map(tab_idx_);
-    internal_memset(tab_, 0, sizeof(*tab_));
-  } else if (size_ <= ClockBlock::kClockCount) {
-    // Transform one-level table to two-level table.
-    u32 old = tab_idx_;
-    tab_idx_ = ctx->clock_alloc.Alloc(c);
-    tab_ = ctx->clock_alloc.Map(tab_idx_);
-    internal_memset(tab_, 0, sizeof(*tab_));
-    tab_->table[0] = old;
+    atomic_store_relaxed(ref_ptr(tab_), 1);
+    size_ = 1;
+  } else if (size_ > blocks_ * ClockBlock::kClockCount) {
+    u32 idx = ctx->clock_alloc.Alloc(c);
+    ClockBlock *new_cb = ctx->clock_alloc.Map(idx);
+    uptr top = size_ - blocks_ * ClockBlock::kClockCount;
+    CHECK_LT(top, ClockBlock::kClockCount);
+    const uptr move = top * sizeof(tab_->clock[0]);
+    internal_memcpy(&new_cb->clock[0], tab_->clock, move);
+    internal_memset(&new_cb->clock[top], 0, sizeof(*new_cb) - move);
+    internal_memset(tab_->clock, 0, move);
+    append_block(idx);
   }
-  // At this point we have first level table allocated.
+  // At this point we have first level table allocated and all clock elements
+  // are evacuated from it to a second level block.
   // Add second level tables as necessary.
-  for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount);
-      i < nclk; i += ClockBlock::kClockCount) {
+  while (nclk > capacity()) {
     u32 idx = ctx->clock_alloc.Alloc(c);
     ClockBlock *cb = ctx->clock_alloc.Map(idx);
     internal_memset(cb, 0, sizeof(*cb));
-    CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0);
-    tab_->table[i/ClockBlock::kClockCount] = idx;
+    append_block(idx);
   }
   size_ = nclk;
 }
 
-// Sets a single element in the vector clock.
-// This function is called only from weird places like AcquireGlobal.
-void ThreadClock::set(unsigned tid, u64 v) {
-  DCHECK_LT(tid, kMaxTid);
-  DCHECK_GE(v, clk_[tid].epoch);
-  clk_[tid].epoch = v;
-  if (nclk_ <= tid)
-    nclk_ = tid + 1;
-  last_acquire_ = clk_[tid_].epoch;
-}
-
-void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) {
-  printf("clock=[");
-  for (uptr i = 0; i < nclk_; i++)
-    printf("%s%llu", i == 0 ? "" : ",", clk_[i].epoch);
-  printf("] reused=[");
-  for (uptr i = 0; i < nclk_; i++)
-    printf("%s%llu", i == 0 ? "" : ",", clk_[i].reused);
-  printf("] tid=%u/%u last_acq=%llu",
-      tid_, reused_, last_acquire_);
+// Flushes all dirty elements into the main clock array.
+void SyncClock::FlushDirty() {
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    Dirty *dirty = &dirty_[i];
+    if (dirty->tid != kInvalidTid) {
+      CHECK_LT(dirty->tid, size_);
+      elem(dirty->tid).epoch = dirty->epoch;
+      dirty->tid = kInvalidTid;
+    }
+  }
 }
 
-SyncClock::SyncClock()
-    : release_store_tid_(kInvalidTid)
-    , release_store_reused_()
-    , tab_()
-    , tab_idx_()
-    , size_() {
-  for (uptr i = 0; i < kDirtyTids; i++)
-    dirty_tids_[i] = kInvalidTid;
+bool SyncClock::IsShared() const {
+  if (size_ == 0)
+    return false;
+  atomic_uint32_t *ref = ref_ptr(tab_);
+  u32 v = atomic_load(ref, memory_order_acquire);
+  CHECK_GT(v, 0);
+  return v > 1;
 }
 
-SyncClock::~SyncClock() {
-  // Reset must be called before dtor.
-  CHECK_EQ(size_, 0);
-  CHECK_EQ(tab_, 0);
-  CHECK_EQ(tab_idx_, 0);
+// Unshares the current clock if it's shared.
+// Shared clocks are immutable, so they need to be unshared before any updates.
+// Note: this does not apply to dirty entries as they are not shared.
+void SyncClock::Unshare(ClockCache *c) {
+  if (!IsShared())
+    return;
+  // First, copy current state into old.
+  SyncClock old;
+  old.tab_ = tab_;
+  old.tab_idx_ = tab_idx_;
+  old.size_ = size_;
+  old.blocks_ = blocks_;
+  old.release_store_tid_ = release_store_tid_;
+  old.release_store_reused_ = release_store_reused_;
+  for (unsigned i = 0; i < kDirtyTids; i++)
+    old.dirty_[i] = dirty_[i];
+  // Then, clear current object.
+  ResetImpl();
+  // Allocate brand new clock in the current object.
+  Resize(c, old.size_);
+  // Now copy state back into this object.
+  Iter old_iter(&old);
+  for (ClockElem &ce : *this) {
+    ce = *old_iter;
+    ++old_iter;
+  }
+  release_store_tid_ = old.release_store_tid_;
+  release_store_reused_ = old.release_store_reused_;
+  for (unsigned i = 0; i < kDirtyTids; i++)
+    dirty_[i] = old.dirty_[i];
+  // Drop reference to old and delete if necessary.
+  old.Reset(c);
 }
 
-void SyncClock::Reset(ClockCache *c) {
-  if (size_ == 0) {
-    // nothing
-  } else if (size_ <= ClockBlock::kClockCount) {
-    // One-level table.
-    ctx->clock_alloc.Free(c, tab_idx_);
-  } else {
-    // Two-level table.
-    for (uptr i = 0; i < size_; i += ClockBlock::kClockCount)
-      ctx->clock_alloc.Free(c, tab_->table[i / ClockBlock::kClockCount]);
-    ctx->clock_alloc.Free(c, tab_idx_);
+// Can we cache this clock for future release operations?
+ALWAYS_INLINE bool SyncClock::Cachable() const {
+  if (size_ == 0)
+    return false;
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    if (dirty_[i].tid != kInvalidTid)
+      return false;
   }
-  tab_ = 0;
-  tab_idx_ = 0;
-  size_ = 0;
-  release_store_tid_ = kInvalidTid;
-  release_store_reused_ = 0;
-  for (uptr i = 0; i < kDirtyTids; i++)
-    dirty_tids_[i] = kInvalidTid;
+  return atomic_load_relaxed(ref_ptr(tab_)) == 1;
 }
 
-ClockElem &SyncClock::elem(unsigned tid) const {
+// elem linearizes the two-level structure into linear array.
+// Note: this is used only for one time accesses, vector operations use
+// the iterator as it is much faster.
+ALWAYS_INLINE ClockElem &SyncClock::elem(unsigned tid) const {
   DCHECK_LT(tid, size_);
-  if (size_ <= ClockBlock::kClockCount)
+  const uptr block = tid / ClockBlock::kClockCount;
+  DCHECK_LE(block, blocks_);
+  tid %= ClockBlock::kClockCount;
+  if (block == blocks_)
     return tab_->clock[tid];
-  u32 idx = tab_->table[tid / ClockBlock::kClockCount];
+  u32 idx = get_block(block);
   ClockBlock *cb = ctx->clock_alloc.Map(idx);
-  return cb->clock[tid % ClockBlock::kClockCount];
+  return cb->clock[tid];
+}
+
+ALWAYS_INLINE uptr SyncClock::capacity() const {
+  if (size_ == 0)
+    return 0;
+  uptr ratio = sizeof(ClockBlock::clock[0]) / sizeof(ClockBlock::table[0]);
+  // How many clock elements we can fit into the first level block.
+  // +1 for ref counter.
+  uptr top = ClockBlock::kClockCount - RoundUpTo(blocks_ + 1, ratio) / ratio;
+  return blocks_ * ClockBlock::kClockCount + top;
+}
+
+ALWAYS_INLINE u32 SyncClock::get_block(uptr bi) const {
+  DCHECK(size_);
+  DCHECK_LT(bi, blocks_);
+  return tab_->table[ClockBlock::kBlockIdx - bi];
+}
+
+ALWAYS_INLINE void SyncClock::append_block(u32 idx) {
+  uptr bi = blocks_++;
+  CHECK_EQ(get_block(bi), 0);
+  tab_->table[ClockBlock::kBlockIdx - bi] = idx;
+}
+
+// Used only by tests.
+u64 SyncClock::get(unsigned tid) const {
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    Dirty dirty = dirty_[i];
+    if (dirty.tid == tid)
+      return dirty.epoch;
+  }
+  return elem(tid).epoch;
+}
+
+// Used only by Iter test.
+u64 SyncClock::get_clean(unsigned tid) const {
+  return elem(tid).epoch;
 }
 
 void SyncClock::DebugDump(int(*printf)(const char *s, ...)) {
@@ -418,8 +565,32 @@ void SyncClock::DebugDump(int(*printf)(const char *s, ...)) {
   printf("] reused=[");
   for (uptr i = 0; i < size_; i++)
     printf("%s%llu", i == 0 ? "" : ",", elem(i).reused);
-  printf("] release_store_tid=%d/%d dirty_tids=%d/%d",
+  printf("] release_store_tid=%d/%d dirty_tids=%d[%llu]/%d[%llu]",
       release_store_tid_, release_store_reused_,
-      dirty_tids_[0], dirty_tids_[1]);
+      dirty_[0].tid, dirty_[0].epoch,
+      dirty_[1].tid, dirty_[1].epoch);
+}
+
+void SyncClock::Iter::Next() {
+  // Finished with the current block, move on to the next one.
+  block_++;
+  if (block_ < parent_->blocks_) {
+    // Iterate over the next second level block.
+    u32 idx = parent_->get_block(block_);
+    ClockBlock *cb = ctx->clock_alloc.Map(idx);
+    pos_ = &cb->clock[0];
+    end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
+        ClockBlock::kClockCount);
+    return;
+  }
+  if (block_ == parent_->blocks_ &&
+      parent_->size_ > parent_->blocks_ * ClockBlock::kClockCount) {
+    // Iterate over elements in the first level block.
+    pos_ = &parent_->tab_->clock[0];
+    end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
+        ClockBlock::kClockCount);
+    return;
+  }
+  parent_ = nullptr;  // denotes end
 }
 }  // namespace __tsan
diff --git a/libsanitizer/tsan/tsan_clock.h b/libsanitizer/tsan/tsan_clock.h
index 3deb7f5198c..c8eb8ee2d1a 100644
--- a/libsanitizer/tsan/tsan_clock.h
+++ b/libsanitizer/tsan/tsan_clock.h
@@ -16,25 +16,6 @@
 
 namespace __tsan {
 
-struct ClockElem {
-  u64 epoch  : kClkBits;
-  u64 reused : 64 - kClkBits;
-};
-
-struct ClockBlock {
-  static const uptr kSize = 512;
-  static const uptr kTableSize = kSize / sizeof(u32);
-  static const uptr kClockCount = kSize / sizeof(ClockElem);
-
-  union {
-    u32       table[kTableSize];
-    ClockElem clock[kClockCount];
-  };
-
-  ClockBlock() {
-  }
-};
-
 typedef DenseSlabAlloc<ClockBlock, 1<<16, 1<<10> ClockAlloc;
 typedef DenseSlabAllocCache ClockCache;
 
@@ -44,84 +25,200 @@ class SyncClock {
   SyncClock();
   ~SyncClock();
 
-  uptr size() const {
-    return size_;
-  }
+  uptr size() const;
 
-  u64 get(unsigned tid) const {
-    return elem(tid).epoch;
-  }
+  // These are used only in tests.
+  u64 get(unsigned tid) const;
+  u64 get_clean(unsigned tid) const;
 
   void Resize(ClockCache *c, uptr nclk);
   void Reset(ClockCache *c);
 
   void DebugDump(int(*printf)(const char *s, ...));
 
+  // Clock element iterator.
+  // Note: it iterates only over the table without regard to dirty entries.
+  class Iter {
+   public:
+    explicit Iter(SyncClock* parent);
+    Iter& operator++();
+    bool operator!=(const Iter& other);
+    ClockElem &operator*();
+
+   private:
+    SyncClock *parent_;
+    // [pos_, end_) is the current continuous range of clock elements.
+    ClockElem *pos_;
+    ClockElem *end_;
+    int block_;  // Current number of second level block.
+
+    NOINLINE void Next();
+  };
+
+  Iter begin();
+  Iter end();
+
  private:
-  friend struct ThreadClock;
+  friend class ThreadClock;
+  friend class Iter;
   static const uptr kDirtyTids = 2;
 
+  struct Dirty {
+    u64 epoch  : kClkBits;
+    u64 tid : 64 - kClkBits;  // kInvalidId if not active
+  };
+
   unsigned release_store_tid_;
   unsigned release_store_reused_;
-  unsigned dirty_tids_[kDirtyTids];
-  // tab_ contains indirect pointer to a 512b block using DenseSlabAlloc.
-  // If size_ <= 64, then tab_ points to an array with 64 ClockElem's.
-  // Otherwise, tab_ points to an array with 128 u32 elements,
+  Dirty dirty_[kDirtyTids];
+  // If size_ is 0, tab_ is nullptr.
+  // If size <= 64 (kClockCount), tab_ contains pointer to an array with
+  // 64 ClockElem's (ClockBlock::clock).
+  // Otherwise, tab_ points to an array with up to 127 u32 elements,
   // each pointing to the second-level 512b block with 64 ClockElem's.
+  // Unused space in the first level ClockBlock is used to store additional
+  // clock elements.
+  // The last u32 element in the first level ClockBlock is always used as
+  // reference counter.
+  //
+  // See the following scheme for details.
+  // All memory blocks are 512 bytes (allocated from ClockAlloc).
+  // Clock (clk) elements are 64 bits.
+  // Idx and ref are 32 bits.
+  //
+  // tab_
+  //    |
+  //    \/
+  //    +----------------------------------------------------+
+  //    | clk128 | clk129 | ...unused... | idx1 | idx0 | ref |
+  //    +----------------------------------------------------+
+  //                                        |      |
+  //                                        |      \/
+  //                                        |      +----------------+
+  //                                        |      | clk0 ... clk63 |
+  //                                        |      +----------------+
+  //                                        \/
+  //                                        +------------------+
+  //                                        | clk64 ... clk127 |
+  //                                        +------------------+
+  //
+  // Note: dirty entries, if active, always override what's stored in the clock.
   ClockBlock *tab_;
   u32 tab_idx_;
-  u32 size_;
-
+  u16 size_;
+  u16 blocks_;  // Number of second level blocks.
+
+  void Unshare(ClockCache *c);
+  bool IsShared() const;
+  bool Cachable() const;
+  void ResetImpl();
+  void FlushDirty();
+  uptr capacity() const;
+  u32 get_block(uptr bi) const;
+  void append_block(u32 idx);
   ClockElem &elem(unsigned tid) const;
 };
 
 // The clock that lives in threads.
-struct ThreadClock {
+class ThreadClock {
  public:
   typedef DenseSlabAllocCache Cache;
 
   explicit ThreadClock(unsigned tid, unsigned reused = 0);
 
-  u64 get(unsigned tid) const {
-    DCHECK_LT(tid, kMaxTidInClock);
-    return clk_[tid].epoch;
-  }
-
-  void set(unsigned tid, u64 v);
-
-  void set(u64 v) {
-    DCHECK_GE(v, clk_[tid_].epoch);
-    clk_[tid_].epoch = v;
-  }
+  u64 get(unsigned tid) const;
+  void set(ClockCache *c, unsigned tid, u64 v);
+  void set(u64 v);
+  void tick();
+  uptr size() const;
 
-  void tick() {
-    clk_[tid_].epoch++;
-  }
-
-  uptr size() const {
-    return nclk_;
-  }
-
-  void acquire(ClockCache *c, const SyncClock *src);
-  void release(ClockCache *c, SyncClock *dst) const;
+  void acquire(ClockCache *c, SyncClock *src);
+  void release(ClockCache *c, SyncClock *dst);
   void acq_rel(ClockCache *c, SyncClock *dst);
-  void ReleaseStore(ClockCache *c, SyncClock *dst) const;
+  void ReleaseStore(ClockCache *c, SyncClock *dst);
+  void ResetCached(ClockCache *c);
 
   void DebugReset();
   void DebugDump(int(*printf)(const char *s, ...));
 
  private:
   static const uptr kDirtyTids = SyncClock::kDirtyTids;
+  // Index of the thread associated with he clock ("current thread").
   const unsigned tid_;
-  const unsigned reused_;
+  const unsigned reused_;  // tid_ reuse count.
+  // Current thread time when it acquired something from other threads.
   u64 last_acquire_;
+
+  // Cached SyncClock (without dirty entries and release_store_tid_).
+  // We reuse it for subsequent store-release operations without intervening
+  // acquire operations. Since it is shared (and thus constant), clock value
+  // for the current thread is then stored in dirty entries in the SyncClock.
+  // We host a refernece to the table while it is cached here.
+  u32 cached_idx_;
+  u16 cached_size_;
+  u16 cached_blocks_;
+
+  // Number of active elements in the clk_ table (the rest is zeros).
   uptr nclk_;
-  ClockElem clk_[kMaxTidInClock];
+  u64 clk_[kMaxTidInClock];  // Fixed size vector clock.
 
   bool IsAlreadyAcquired(const SyncClock *src) const;
-  void UpdateCurrentThread(SyncClock *dst) const;
+  void UpdateCurrentThread(ClockCache *c, SyncClock *dst) const;
 };
 
+ALWAYS_INLINE u64 ThreadClock::get(unsigned tid) const {
+  DCHECK_LT(tid, kMaxTidInClock);
+  return clk_[tid];
+}
+
+ALWAYS_INLINE void ThreadClock::set(u64 v) {
+  DCHECK_GE(v, clk_[tid_]);
+  clk_[tid_] = v;
+}
+
+ALWAYS_INLINE void ThreadClock::tick() {
+  clk_[tid_]++;
+}
+
+ALWAYS_INLINE uptr ThreadClock::size() const {
+  return nclk_;
+}
+
+ALWAYS_INLINE SyncClock::Iter SyncClock::begin() {
+  return Iter(this);
+}
+
+ALWAYS_INLINE SyncClock::Iter SyncClock::end() {
+  return Iter(nullptr);
+}
+
+ALWAYS_INLINE uptr SyncClock::size() const {
+  return size_;
+}
+
+ALWAYS_INLINE SyncClock::Iter::Iter(SyncClock* parent)
+    : parent_(parent)
+    , pos_(nullptr)
+    , end_(nullptr)
+    , block_(-1) {
+  if (parent)
+    Next();
+}
+
+ALWAYS_INLINE SyncClock::Iter& SyncClock::Iter::operator++() {
+  pos_++;
+  if (UNLIKELY(pos_ >= end_))
+    Next();
+  return *this;
+}
+
+ALWAYS_INLINE bool SyncClock::Iter::operator!=(const SyncClock::Iter& other) {
+  return parent_ != other.parent_;
+}
+
+ALWAYS_INLINE ClockElem &SyncClock::Iter::operator*() {
+  return *pos_;
+}
 }  // namespace __tsan
 
 #endif  // TSAN_CLOCK_H
diff --git a/libsanitizer/tsan/tsan_debugging.cc b/libsanitizer/tsan/tsan_debugging.cc
index d26d4828b15..9a9c67fc42e 100644
--- a/libsanitizer/tsan/tsan_debugging.cc
+++ b/libsanitizer/tsan/tsan_debugging.cc
@@ -13,6 +13,8 @@
 #include "tsan_report.h"
 #include "tsan_rtl.h"
 
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
 using namespace __tsan;
 
 static const char *ReportTypeDescription(ReportType typ) {
@@ -20,6 +22,7 @@ static const char *ReportTypeDescription(ReportType typ) {
   if (typ == ReportTypeVptrRace) return "data-race-vptr";
   if (typ == ReportTypeUseAfterFree) return "heap-use-after-free";
   if (typ == ReportTypeVptrUseAfterFree) return "heap-use-after-free-vptr";
+  if (typ == ReportTypeExternalRace) return "external-race";
   if (typ == ReportTypeThreadLeak) return "thread-leak";
   if (typ == ReportTypeMutexDestroyLocked) return "locked-mutex-destroy";
   if (typ == ReportTypeMutexDoubleLock) return "mutex-double-lock";
@@ -123,6 +126,16 @@ int __tsan_get_report_loc(void *report, uptr idx, const char **type,
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc_object_type(void *report, uptr idx,
+                                      const char **object_type) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->locs.Size());
+  ReportLocation *loc = rep->locs[idx];
+  *object_type = GetObjectTypeFromTag(loc->external_tag);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
                             int *destroyed, void **trace, uptr trace_size) {
   const ReportDesc *rep = (ReportDesc *)report;
@@ -136,7 +149,7 @@ int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
-int __tsan_get_report_thread(void *report, uptr idx, int *tid, uptr *os_id,
+int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id,
                              int *running, const char **name, int *parent_tid,
                              void **trace, uptr trace_size) {
   const ReportDesc *rep = (ReportDesc *)report;
@@ -158,3 +171,78 @@ int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid) {
   *tid = rep->unique_tids[idx];
   return 1;
 }
+
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__tsan_locate_address(uptr addr, char *name, uptr name_size,
+                                  uptr *region_address_ptr,
+                                  uptr *region_size_ptr) {
+  uptr region_address = 0;
+  uptr region_size = 0;
+  const char *region_kind = nullptr;
+  if (name && name_size > 0) name[0] = 0;
+
+  if (IsMetaMem(addr)) {
+    region_kind = "meta shadow";
+  } else if (IsShadowMem(addr)) {
+    region_kind = "shadow";
+  } else {
+    bool is_stack = false;
+    MBlock *b = 0;
+    Allocator *a = allocator();
+    if (a->PointerIsMine((void *)addr)) {
+      void *block_begin = a->GetBlockBegin((void *)addr);
+      if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+    }
+
+    if (b != 0) {
+      region_address = (uptr)allocator()->GetBlockBegin((void *)addr);
+      region_size = b->siz;
+      region_kind = "heap";
+    } else {
+      // TODO(kuba.brecka): We should not lock. This is supposed to be called
+      // from within the debugger when other threads are stopped.
+      ctx->thread_registry->Lock();
+      ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack);
+      ctx->thread_registry->Unlock();
+      if (tctx) {
+        region_kind = is_stack ? "stack" : "tls";
+      } else {
+        region_kind = "global";
+        DataInfo info;
+        if (Symbolizer::GetOrInit()->SymbolizeData(addr, &info)) {
+          internal_strncpy(name, info.name, name_size);
+          region_address = info.start;
+          region_size = info.size;
+        }
+      }
+    }
+  }
+
+  CHECK(region_kind);
+  if (region_address_ptr) *region_address_ptr = region_address;
+  if (region_size_ptr) *region_size_ptr = region_size;
+  return region_kind;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id,
+                           tid_t *os_id) {
+  MBlock *b = 0;
+  Allocator *a = allocator();
+  if (a->PointerIsMine((void *)addr)) {
+    void *block_begin = a->GetBlockBegin((void *)addr);
+    if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+  }
+  if (b == 0) return 0;
+
+  *thread_id = b->tid;
+  // No locking.  This is supposed to be called from within the debugger when
+  // other threads are stopped.
+  ThreadContextBase *tctx = ctx->thread_registry->GetThreadLocked(b->tid);
+  *os_id = tctx->os_id;
+
+  StackTrace stack = StackDepotGet(b->stk);
+  size = Min(size, (uptr)stack.size);
+  for (uptr i = 0; i < size; i++) trace[i] = stack.trace[stack.size - i - 1];
+  return size;
+}
diff --git a/libsanitizer/tsan/tsan_defs.h b/libsanitizer/tsan/tsan_defs.h
index e540526caa2..2c7eda69f92 100644
--- a/libsanitizer/tsan/tsan_defs.h
+++ b/libsanitizer/tsan/tsan_defs.h
@@ -36,15 +36,40 @@
 
 namespace __tsan {
 
+const int kClkBits = 42;
+const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
+
+struct ClockElem {
+  u64 epoch  : kClkBits;
+  u64 reused : 64 - kClkBits;  // tid reuse count
+};
+
+struct ClockBlock {
+  static const uptr kSize = 512;
+  static const uptr kTableSize = kSize / sizeof(u32);
+  static const uptr kClockCount = kSize / sizeof(ClockElem);
+  static const uptr kRefIdx = kTableSize - 1;
+  static const uptr kBlockIdx = kTableSize - 2;
+
+  union {
+    u32       table[kTableSize];
+    ClockElem clock[kClockCount];
+  };
+
+  ClockBlock() {
+  }
+};
+
 const int kTidBits = 13;
-const unsigned kMaxTid = 1 << kTidBits;
+// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is
+// occupied by reference counter, so total number of elements we can store
+// in SyncClock is kClockCount * (kTableSize - 1).
+const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount;
 #if !SANITIZER_GO
 const unsigned kMaxTidInClock = kMaxTid * 2;  // This includes msb 'freed' bit.
 #else
 const unsigned kMaxTidInClock = kMaxTid;  // Go does not track freed memory.
 #endif
-const int kClkBits = 42;
-const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
 const uptr kShadowStackSize = 64 * 1024;
 
 // Count of shadow values in a shadow cell.
@@ -72,7 +97,7 @@ const bool kCollectHistory = false;
 const bool kCollectHistory = true;
 #endif
 
-const unsigned kInvalidTid = (unsigned)-1;
+const u16 kInvalidTid = kMaxTid + 1;
 
 // The following "build consistency" machinery ensures that all source files
 // are built in the same configuration. Inconsistent builds lead to
@@ -147,13 +172,23 @@ class RegionAlloc;
 
 // Descriptor of user's memory block.
 struct MBlock {
-  u64  siz;
+  u64  siz : 48;
+  u64  tag : 16;
   u32  stk;
   u16  tid;
 };
 
 COMPILER_CHECK(sizeof(MBlock) == 16);
 
+enum ExternalTag : uptr {
+  kExternalTagNone = 0,
+  kExternalTagSwiftModifyingAccess = 1,
+  kExternalTagFirstUserAvailable = 2,
+  kExternalTagMax = 1024,
+  // Don't set kExternalTagMax over 65,536, since MBlock only stores tags
+  // as 16-bit values, see tsan_defs.h.
+};
+
 }  // namespace __tsan
 
 #endif  // TSAN_DEFS_H
diff --git a/libsanitizer/tsan/tsan_dense_alloc.h b/libsanitizer/tsan/tsan_dense_alloc.h
index 780ff6f4c8e..197b96fdeed 100644
--- a/libsanitizer/tsan/tsan_dense_alloc.h
+++ b/libsanitizer/tsan/tsan_dense_alloc.h
@@ -37,7 +37,7 @@ class DenseSlabAlloc {
   typedef DenseSlabAllocCache Cache;
   typedef typename Cache::IndexT IndexT;
 
-  DenseSlabAlloc() {
+  explicit DenseSlabAlloc(const char *name) {
     // Check that kL1Size and kL2Size are sane.
     CHECK_EQ(kL1Size & (kL1Size - 1), 0);
     CHECK_EQ(kL2Size & (kL2Size - 1), 0);
@@ -47,6 +47,7 @@ class DenseSlabAlloc {
     internal_memset(map_, 0, sizeof(map_));
     freelist_ = 0;
     fillpos_ = 0;
+    name_ = name;
   }
 
   ~DenseSlabAlloc() {
@@ -94,15 +95,19 @@ class DenseSlabAlloc {
   SpinMutex mtx_;
   IndexT freelist_;
   uptr fillpos_;
+  const char *name_;
 
   void Refill(Cache *c) {
     SpinMutexLock lock(&mtx_);
     if (freelist_ == 0) {
       if (fillpos_ == kL1Size) {
-        Printf("ThreadSanitizer: DenseSlabAllocator overflow. Dying.\n");
+        Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
+            name_, kL1Size, kL2Size);
         Die();
       }
-      T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), "DenseSlabAllocator");
+      VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
+          name_, fillpos_, kL1Size, kL2Size);
+      T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
       // Reserve 0 as invalid index.
       IndexT start = fillpos_ == 0 ? 1 : 0;
       for (IndexT i = start; i < kL2Size; i++) {
diff --git a/libsanitizer/tsan/tsan_external.cc b/libsanitizer/tsan/tsan_external.cc
new file mode 100644
index 00000000000..3dddc3a4f64
--- /dev/null
+++ b/libsanitizer/tsan/tsan_external.cc
@@ -0,0 +1,123 @@
+//===-- tsan_external.cc --------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_rtl.h"
+#include "tsan_interceptors.h"
+
+namespace __tsan {
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+struct TagData {
+  const char *object_type;
+  const char *header;
+};
+
+static TagData registered_tags[kExternalTagMax] = {
+  {},
+  {"Swift variable", "Swift access race"},
+};
+static atomic_uint32_t used_tags{kExternalTagFirstUserAvailable};  // NOLINT.
+static TagData *GetTagData(uptr tag) {
+  // Invalid/corrupted tag?  Better return NULL and let the caller deal with it.
+  if (tag >= atomic_load(&used_tags, memory_order_relaxed)) return nullptr;
+  return &registered_tags[tag];
+}
+
+const char *GetObjectTypeFromTag(uptr tag) {
+  TagData *tag_data = GetTagData(tag);
+  return tag_data ? tag_data->object_type : nullptr;
+}
+
+const char *GetReportHeaderFromTag(uptr tag) {
+  TagData *tag_data = GetTagData(tag);
+  return tag_data ? tag_data->header : nullptr;
+}
+
+void InsertShadowStackFrameForTag(ThreadState *thr, uptr tag) {
+  FuncEntry(thr, (uptr)&registered_tags[tag]);
+}
+
+uptr TagFromShadowStackFrame(uptr pc) {
+  uptr tag_count = atomic_load(&used_tags, memory_order_relaxed);
+  void *pc_ptr = (void *)pc;
+  if (pc_ptr < GetTagData(0) || pc_ptr > GetTagData(tag_count - 1))
+    return 0;
+  return (TagData *)pc_ptr - GetTagData(0);
+}
+
+#if !SANITIZER_GO
+
+typedef void(*AccessFunc)(ThreadState *, uptr, uptr, int);
+void ExternalAccess(void *addr, void *caller_pc, void *tag, AccessFunc access) {
+  CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
+  ThreadState *thr = cur_thread();
+  if (caller_pc) FuncEntry(thr, (uptr)caller_pc);
+  InsertShadowStackFrameForTag(thr, (uptr)tag);
+  bool in_ignored_lib;
+  if (!caller_pc || !libignore()->IsIgnored((uptr)caller_pc, &in_ignored_lib)) {
+    access(thr, CALLERPC, (uptr)addr, kSizeLog1);
+  }
+  FuncExit(thr);
+  if (caller_pc) FuncExit(thr);
+}
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_external_register_tag(const char *object_type) {
+  uptr new_tag = atomic_fetch_add(&used_tags, 1, memory_order_relaxed);
+  CHECK_LT(new_tag, kExternalTagMax);
+  GetTagData(new_tag)->object_type = internal_strdup(object_type);
+  char header[127] = {0};
+  internal_snprintf(header, sizeof(header), "race on %s", object_type);
+  GetTagData(new_tag)->header = internal_strdup(header);
+  return (void *)new_tag;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_register_header(void *tag, const char *header) {
+  CHECK_GE((uptr)tag, kExternalTagFirstUserAvailable);
+  CHECK_LT((uptr)tag, kExternalTagMax);
+  atomic_uintptr_t *header_ptr =
+      (atomic_uintptr_t *)&GetTagData((uptr)tag)->header;
+  header = internal_strdup(header);
+  char *old_header =
+      (char *)atomic_exchange(header_ptr, (uptr)header, memory_order_seq_cst);
+  if (old_header) internal_free(old_header);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_assign_tag(void *addr, void *tag) {
+  CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
+  Allocator *a = allocator();
+  MBlock *b = nullptr;
+  if (a->PointerIsMine((void *)addr)) {
+    void *block_begin = a->GetBlockBegin((void *)addr);
+    if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+  }
+  if (b) {
+    b->tag = (uptr)tag;
+  }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_read(void *addr, void *caller_pc, void *tag) {
+  ExternalAccess(addr, caller_pc, tag, MemoryRead);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_write(void *addr, void *caller_pc, void *tag) {
+  ExternalAccess(addr, caller_pc, tag, MemoryWrite);
+}
+}  // extern "C"
+
+#endif  // !SANITIZER_GO
+
+}  // namespace __tsan
diff --git a/libsanitizer/tsan/tsan_fd.cc b/libsanitizer/tsan/tsan_fd.cc
index 8f75a28200d..effa35ddeb5 100644
--- a/libsanitizer/tsan/tsan_fd.cc
+++ b/libsanitizer/tsan/tsan_fd.cc
@@ -46,8 +46,8 @@ static bool bogusfd(int fd) {
 }
 
 static FdSync *allocsync(ThreadState *thr, uptr pc) {
-  FdSync *s = (FdSync*)user_alloc(thr, pc, sizeof(FdSync), kDefaultAlignment,
-      false);
+  FdSync *s = (FdSync*)user_alloc_internal(thr, pc, sizeof(FdSync),
+      kDefaultAlignment, false);
   atomic_store(&s->rc, 1, memory_order_relaxed);
   return s;
 }
@@ -77,7 +77,7 @@ static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) {
   if (l1 == 0) {
     uptr size = kTableSizeL2 * sizeof(FdDesc);
     // We need this to reside in user memory to properly catch races on it.
-    void *p = user_alloc(thr, pc, size, kDefaultAlignment, false);
+    void *p = user_alloc_internal(thr, pc, size, kDefaultAlignment, false);
     internal_memset(p, 0, size);
     MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size);
     if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel))
diff --git a/libsanitizer/tsan/tsan_flags.cc b/libsanitizer/tsan/tsan_flags.cc
index 5064a0fe653..4217691658e 100644
--- a/libsanitizer/tsan/tsan_flags.cc
+++ b/libsanitizer/tsan/tsan_flags.cc
@@ -19,10 +19,6 @@
 
 namespace __tsan {
 
-Flags *flags() {
-  return &ctx->flags;
-}
-
 // Can be overriden in frontend.
 #ifdef TSAN_EXTERNAL_HOOKS
 extern "C" const char* __tsan_default_options();
diff --git a/libsanitizer/tsan/tsan_flags.h b/libsanitizer/tsan/tsan_flags.h
index 3d58ff3793e..35b0efc1c6b 100644
--- a/libsanitizer/tsan/tsan_flags.h
+++ b/libsanitizer/tsan/tsan_flags.h
@@ -26,7 +26,6 @@ struct Flags : DDFlags {
   void ParseFromString(const char *str);
 };
 
-Flags *flags();
 void InitializeFlags(Flags *flags, const char *env);
 }  // namespace __tsan
 
diff --git a/libsanitizer/tsan/tsan_flags.inc b/libsanitizer/tsan/tsan_flags.inc
index 78f5e80fd5c..e9873f16ce4 100644
--- a/libsanitizer/tsan/tsan_flags.inc
+++ b/libsanitizer/tsan/tsan_flags.inc
@@ -77,5 +77,8 @@ TSAN_FLAG(bool, die_after_fork, true,
 TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
 TSAN_FLAG(bool, ignore_interceptors_accesses, false,
           "Ignore reads and writes from all interceptors.")
+TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false,
+          "Interceptors should only detect races when called from instrumented "
+          "modules.")
 TSAN_FLAG(bool, shared_ptr_interceptor, true,
           "Track atomic reference counting in libc++ shared_ptr and weak_ptr.")
diff --git a/libsanitizer/tsan/tsan_interceptors.cc b/libsanitizer/tsan/tsan_interceptors.cc
index bf5f2d5b66c..15f20d4b668 100644
--- a/libsanitizer/tsan/tsan_interceptors.cc
+++ b/libsanitizer/tsan/tsan_interceptors.cc
@@ -12,10 +12,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_posix.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 #include "interception/interception.h"
@@ -27,30 +30,25 @@
 #include "tsan_mman.h"
 #include "tsan_fd.h"
 
-#if SANITIZER_POSIX
-#include "sanitizer_common/sanitizer_posix.h"
-#endif
 
 using namespace __tsan;  // NOLINT
 
 #if SANITIZER_FREEBSD || SANITIZER_MAC
-#define __errno_location __error
 #define stdout __stdoutp
 #define stderr __stderrp
 #endif
 
-#if SANITIZER_ANDROID
-#define __errno_location __errno
-#define mallopt(a, b)
+#if SANITIZER_NETBSD
+#define dirfd(dirp) (*(int *)(dirp))
+#define fileno_unlocked fileno
+#define stdout __sF[1]
+#define stderr __sF[2]
 #endif
 
-#if SANITIZER_LINUX || SANITIZER_FREEBSD
-#define PTHREAD_CREATE_DETACHED 1
-#elif SANITIZER_MAC
-#define PTHREAD_CREATE_DETACHED 2
+#if SANITIZER_ANDROID
+#define mallopt(a, b)
 #endif
 
-
 #ifdef __mips__
 const int kSigCount = 129;
 #else
@@ -91,24 +89,26 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 extern "C" void *pthread_self();
 extern "C" void _exit(int status);
-extern "C" int *__errno_location();
 extern "C" int fileno_unlocked(void *stream);
+#if !SANITIZER_NETBSD
 extern "C" int dirfd(void *dirp);
-#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD
 extern "C" int mallopt(int param, int value);
 #endif
+#if SANITIZER_NETBSD
+extern __sanitizer_FILE **__sF;
+#else
 extern __sanitizer_FILE *stdout, *stderr;
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
 const int PTHREAD_MUTEX_RECURSIVE = 1;
 const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
 #else
 const int PTHREAD_MUTEX_RECURSIVE = 2;
 const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
 #endif
-const int EINVAL = 22;
-const int EBUSY = 16;
-const int EOWNERDEAD = 130;
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
 const int EPOLL_CTL_ADD = 1;
 #endif
 const int SIGILL = 4;
@@ -117,7 +117,7 @@ const int SIGFPE = 8;
 const int SIGSEGV = 11;
 const int SIGPIPE = 13;
 const int SIGTERM = 15;
-#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC
+#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
 const int SIGBUS = 10;
 const int SIGSYS = 12;
 #else
@@ -125,7 +125,9 @@ const int SIGBUS = 7;
 const int SIGSYS = 31;
 #endif
 void *const MAP_FAILED = (void*)-1;
-#if !SANITIZER_MAC
+#if SANITIZER_NETBSD
+const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
+#elif !SANITIZER_MAC
 const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
 #endif
 const int MAP_FIXED = 0x10;
@@ -137,8 +139,6 @@ typedef long long_t;  // NOLINT
 # define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
 # define F_TEST  3      /* Test a region for other processes locks.  */
 
-#define errno (*__errno_location())
-
 typedef void (*sighandler_t)(int sig);
 typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx);
 
@@ -152,6 +152,15 @@ struct sigaction_t {
   __sanitizer_sigset_t sa_mask;
   void (*sa_restorer)();
 };
+#elif SANITIZER_NETBSD
+struct sigaction_t {
+  union {
+    sighandler_t sa_handler;
+    sigactionhandler_t sa_sigaction;
+  };
+  __sanitizer_sigset_t sa_mask;
+  int sa_flags;
+};
 #else
 struct sigaction_t {
 #ifdef __mips__
@@ -180,7 +189,7 @@ struct sigaction_t {
 const sighandler_t SIG_DFL = (sighandler_t)0;
 const sighandler_t SIG_IGN = (sighandler_t)1;
 const sighandler_t SIG_ERR = (sighandler_t)-1;
-#if SANITIZER_FREEBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
 const int SA_SIGINFO = 0x40;
 const int SIG_SETMASK = 3;
 #elif defined(__mips__)
@@ -217,7 +226,7 @@ struct ThreadSignalContext {
 // The object is 64-byte aligned, because we want hot data to be located in
 // a single cache line if possible (it's accessed in every interceptor).
 static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)];
-static LibIgnore *libignore() {
+LibIgnore *libignore() {
   return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]);
 }
 
@@ -229,6 +238,8 @@ void InitializeLibIgnore() {
     if (0 == internal_strcmp(s->type, kSuppressionLib))
       libignore()->AddIgnoredLibrary(s->templ);
   }
+  if (flags()->ignore_noninstrumented_modules)
+    libignore()->IgnoreNoninstrumentedModules(true);
   libignore()->OnLibraryLoaded(0);
 }
 
@@ -250,31 +261,20 @@ static unsigned g_thread_finalize_key;
 
 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
                                      uptr pc)
-    : thr_(thr)
-    , pc_(pc)
-    , in_ignored_lib_(false) {
+    : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) {
   Initialize(thr);
-  if (!thr_->is_inited)
-    return;
-  if (!thr_->ignore_interceptors)
-    FuncEntry(thr, pc);
+  if (!thr_->is_inited) return;
+  if (!thr_->ignore_interceptors) FuncEntry(thr, pc);
   DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
-  if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) {
-    in_ignored_lib_ = true;
-    thr_->in_ignored_lib = true;
-    ThreadIgnoreBegin(thr_, pc_);
-  }
-  if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_);
+  ignoring_ =
+      !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses ||
+                                libignore()->IsIgnored(pc, &in_ignored_lib_));
+  EnableIgnores();
 }
 
 ScopedInterceptor::~ScopedInterceptor() {
-  if (!thr_->is_inited)
-    return;
-  if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_);
-  if (in_ignored_lib_) {
-    thr_->in_ignored_lib = false;
-    ThreadIgnoreEnd(thr_, pc_);
-  }
+  if (!thr_->is_inited) return;
+  DisableIgnores();
   if (!thr_->ignore_interceptors) {
     ProcessPendingSignals(thr_);
     FuncExit(thr_);
@@ -282,24 +282,30 @@ ScopedInterceptor::~ScopedInterceptor() {
   }
 }
 
-void ScopedInterceptor::UserCallbackStart() {
-  if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_);
-  if (in_ignored_lib_) {
-    thr_->in_ignored_lib = false;
-    ThreadIgnoreEnd(thr_, pc_);
+void ScopedInterceptor::EnableIgnores() {
+  if (ignoring_) {
+    ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false);
+    if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;
+    if (in_ignored_lib_) {
+      DCHECK(!thr_->in_ignored_lib);
+      thr_->in_ignored_lib = true;
+    }
   }
 }
 
-void ScopedInterceptor::UserCallbackEnd() {
-  if (in_ignored_lib_) {
-    thr_->in_ignored_lib = true;
-    ThreadIgnoreBegin(thr_, pc_);
+void ScopedInterceptor::DisableIgnores() {
+  if (ignoring_) {
+    ThreadIgnoreEnd(thr_, pc_);
+    if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--;
+    if (in_ignored_lib_) {
+      DCHECK(thr_->in_ignored_lib);
+      thr_->in_ignored_lib = false;
+    }
   }
-  if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_);
 }
 
 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
 #else
 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
@@ -363,6 +369,11 @@ TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
   return res;
 }
 
+TSAN_INTERCEPTOR(int, pause) {
+  SCOPED_TSAN_INTERCEPTOR(pause);
+  return BLOCK_REAL(pause)();
+}
+
 // The sole reason tsan wraps atexit callbacks is to establish synchronization
 // between callback setup and callback execution.
 struct AtExitCtx {
@@ -476,8 +487,14 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
 static void LongJmp(ThreadState *thr, uptr *env) {
 #ifdef __powerpc__
   uptr mangled_sp = env[0];
-#elif SANITIZER_FREEBSD || SANITIZER_MAC
+#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
   uptr mangled_sp = env[2];
+#elif SANITIZER_MAC
+# ifdef __aarch64__
+    uptr mangled_sp = env[13];
+# else
+    uptr mangled_sp = env[2];
+# endif
 #elif defined(SANITIZER_LINUX)
 # ifdef __aarch64__
   uptr mangled_sp = env[13];
@@ -595,7 +612,7 @@ TSAN_INTERCEPTOR(void*, malloc, uptr size) {
 
 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
   SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
-  return user_alloc(thr, pc, sz, align);
+  return user_memalign(thr, pc, align, sz);
 }
 
 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
@@ -675,7 +692,7 @@ static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
   if (*addr) {
     if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
       if (flags & MAP_FIXED) {
-        errno = EINVAL;
+        errno = errno_EINVAL;
         return false;
       } else {
         *addr = 0;
@@ -741,7 +758,7 @@ TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
 #if SANITIZER_LINUX
 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
   SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
-  return user_alloc(thr, pc, sz, align);
+  return user_memalign(thr, pc, align, sz);
 }
 #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
 #else
@@ -750,21 +767,20 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
 
 #if !SANITIZER_MAC
 TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
-  SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
-  return user_alloc(thr, pc, sz, align);
+  SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz);
+  return user_aligned_alloc(thr, pc, align, sz);
 }
 
 TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
   SCOPED_INTERCEPTOR_RAW(valloc, sz);
-  return user_alloc(thr, pc, sz, GetPageSizeCached());
+  return user_valloc(thr, pc, sz);
 }
 #endif
 
 #if SANITIZER_LINUX
 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
   SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
-  sz = RoundUp(sz, GetPageSizeCached());
-  return user_alloc(thr, pc, sz, GetPageSizeCached());
+  return user_pvalloc(thr, pc, sz);
 }
 #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
 #else
@@ -774,8 +790,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
 #if !SANITIZER_MAC
 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
   SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
-  *memptr = user_alloc(thr, pc, sz, align);
-  return 0;
+  return user_posix_memalign(thr, pc, memptr, align, sz);
 }
 #endif
 
@@ -884,7 +899,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) {
       internal_sched_yield();
     Processor *proc = ProcCreate();
     ProcWire(proc, thr);
-    ThreadStart(thr, tid, GetTid());
+    ThreadStart(thr, tid, GetTid(), /*workerthread*/ false);
     atomic_store(&p->tid, 0, memory_order_release);
   }
   void *res = callback(param);
@@ -931,8 +946,7 @@ TSAN_INTERCEPTOR(int, pthread_create,
     ThreadIgnoreEnd(thr, pc);
   }
   if (res == 0) {
-    int tid = ThreadCreate(thr, pc, *(uptr*)th,
-                           detached == PTHREAD_CREATE_DETACHED);
+    int tid = ThreadCreate(thr, pc, *(uptr*)th, IsStateDetached(detached));
     CHECK_NE(tid, 0);
     // Synchronization on p.tid serves two purposes:
     // 1. ThreadCreate must finish before the new thread starts.
@@ -1028,7 +1042,7 @@ static void cond_mutex_unlock(CondMutexUnlockCtx *arg) {
   ThreadSignalContext *ctx = SigCtx(arg->thr);
   CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);
   atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
-  MutexLock(arg->thr, arg->pc, (uptr)arg->m);
+  MutexPostLock(arg->thr, arg->pc, (uptr)arg->m, MutexFlagDoPreLockOnPostLock);
   // Undo BlockingCall ctor effects.
   arg->thr->ignore_interceptors--;
   arg->si->~ScopedInterceptor();
@@ -1057,7 +1071,7 @@ static int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si,
         fn, c, m, t, (void (*)(void *arg))cond_mutex_unlock, &arg);
   }
   if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m);
-  MutexLock(thr, pc, (uptr)m);
+  MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
   return res;
 }
 
@@ -1117,14 +1131,15 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
   int res = REAL(pthread_mutex_init)(m, a);
   if (res == 0) {
-    bool recursive = false;
+    u32 flagz = 0;
     if (a) {
       int type = 0;
       if (REAL(pthread_mutexattr_gettype)(a, &type) == 0)
-        recursive = (type == PTHREAD_MUTEX_RECURSIVE
-            || type == PTHREAD_MUTEX_RECURSIVE_NP);
+        if (type == PTHREAD_MUTEX_RECURSIVE ||
+            type == PTHREAD_MUTEX_RECURSIVE_NP)
+          flagz |= MutexFlagWriteReentrant;
     }
-    MutexCreate(thr, pc, (uptr)m, false, recursive, false);
+    MutexCreate(thr, pc, (uptr)m, flagz);
   }
   return res;
 }
@@ -1132,7 +1147,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
   int res = REAL(pthread_mutex_destroy)(m);
-  if (res == 0 || res == EBUSY) {
+  if (res == 0 || res == errno_EBUSY) {
     MutexDestroy(thr, pc, (uptr)m);
   }
   return res;
@@ -1141,10 +1156,10 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
   int res = REAL(pthread_mutex_trylock)(m);
-  if (res == EOWNERDEAD)
+  if (res == errno_EOWNERDEAD)
     MutexRepair(thr, pc, (uptr)m);
-  if (res == 0 || res == EOWNERDEAD)
-    MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
+  if (res == 0 || res == errno_EOWNERDEAD)
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
   return res;
 }
 
@@ -1153,7 +1168,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
   int res = REAL(pthread_mutex_timedlock)(m, abstime);
   if (res == 0) {
-    MutexLock(thr, pc, (uptr)m);
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
   }
   return res;
 }
@@ -1164,7 +1179,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
   SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
   int res = REAL(pthread_spin_init)(m, pshared);
   if (res == 0) {
-    MutexCreate(thr, pc, (uptr)m, false, false, false);
+    MutexCreate(thr, pc, (uptr)m);
   }
   return res;
 }
@@ -1180,9 +1195,10 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
 
 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
+  MutexPreLock(thr, pc, (uptr)m);
   int res = REAL(pthread_spin_lock)(m);
   if (res == 0) {
-    MutexLock(thr, pc, (uptr)m);
+    MutexPostLock(thr, pc, (uptr)m);
   }
   return res;
 }
@@ -1191,7 +1207,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
   int res = REAL(pthread_spin_trylock)(m);
   if (res == 0) {
-    MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
   }
   return res;
 }
@@ -1208,7 +1224,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
   int res = REAL(pthread_rwlock_init)(m, a);
   if (res == 0) {
-    MutexCreate(thr, pc, (uptr)m, true, false, false);
+    MutexCreate(thr, pc, (uptr)m);
   }
   return res;
 }
@@ -1224,9 +1240,10 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
 
 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
+  MutexPreReadLock(thr, pc, (uptr)m);
   int res = REAL(pthread_rwlock_rdlock)(m);
   if (res == 0) {
-    MutexReadLock(thr, pc, (uptr)m);
+    MutexPostReadLock(thr, pc, (uptr)m);
   }
   return res;
 }
@@ -1235,7 +1252,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
   int res = REAL(pthread_rwlock_tryrdlock)(m);
   if (res == 0) {
-    MutexReadLock(thr, pc, (uptr)m, /*try_lock=*/true);
+    MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock);
   }
   return res;
 }
@@ -1245,7 +1262,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
   int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
   if (res == 0) {
-    MutexReadLock(thr, pc, (uptr)m);
+    MutexPostReadLock(thr, pc, (uptr)m);
   }
   return res;
 }
@@ -1253,9 +1270,10 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
 
 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
+  MutexPreLock(thr, pc, (uptr)m);
   int res = REAL(pthread_rwlock_wrlock)(m);
   if (res == 0) {
-    MutexLock(thr, pc, (uptr)m);
+    MutexPostLock(thr, pc, (uptr)m);
   }
   return res;
 }
@@ -1264,7 +1282,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
   int res = REAL(pthread_rwlock_trywrlock)(m);
   if (res == 0) {
-    MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
   }
   return res;
 }
@@ -1274,7 +1292,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
   int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
   if (res == 0) {
-    MutexLock(thr, pc, (uptr)m);
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
   }
   return res;
 }
@@ -1318,7 +1336,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
   SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
   if (o == 0 || f == 0)
-    return EINVAL;
+    return errno_EINVAL;
   atomic_uint32_t *a;
   if (!SANITIZER_MAC)
     a = static_cast<atomic_uint32_t*>(o);
@@ -1355,7 +1373,7 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
 #endif
 
 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
   SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
   if (fd > 0)
     FdAccess(thr, pc, fd);
@@ -1647,24 +1665,6 @@ TSAN_INTERCEPTOR(void*, tmpfile64, int fake) {
 #define TSAN_MAYBE_INTERCEPT_TMPFILE64
 #endif
 
-TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) {
-  // libc file streams can call user-supplied functions, see fopencookie.
-  {
-    SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f);
-    MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true);
-  }
-  return REAL(fread)(ptr, size, nmemb, f);
-}
-
-TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) {
-  // libc file streams can call user-supplied functions, see fopencookie.
-  {
-    SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f);
-    MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false);
-  }
-  return REAL(fwrite)(p, size, nmemb, f);
-}
-
 static void FlushStreams() {
   // Flushing all the streams here may freeze the process if a child thread is
   // performing file stream operations at the same time.
@@ -1954,7 +1954,7 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
   sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags;
   internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
       sizeof(sigactions[sig].sa_mask));
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
   sigactions[sig].sa_restorer = act->sa_restorer;
 #endif
   sigaction_t newact;
@@ -2254,8 +2254,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
   OnExit(((TsanInterceptorContext *) ctx)->thr)
 
-#define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \
-  MutexLock(((TsanInterceptorContext *)ctx)->thr, \
+#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
+  MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
+  MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
 
 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
@@ -2312,7 +2316,7 @@ struct ScopedSyscall {
   }
 };
 
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
 static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
   TSAN_SYSCALL();
   MemoryAccessRange(thr, pc, p, s, write);
@@ -2582,6 +2586,7 @@ void InitializeInterceptors() {
   TSAN_INTERCEPT(sleep);
   TSAN_INTERCEPT(usleep);
   TSAN_INTERCEPT(nanosleep);
+  TSAN_INTERCEPT(pause);
   TSAN_INTERCEPT(gettimeofday);
   TSAN_INTERCEPT(getaddrinfo);
 
diff --git a/libsanitizer/tsan/tsan_interceptors.h b/libsanitizer/tsan/tsan_interceptors.h
index a0f9a0753a6..de47466501d 100644
--- a/libsanitizer/tsan/tsan_interceptors.h
+++ b/libsanitizer/tsan/tsan_interceptors.h
@@ -10,14 +10,17 @@ class ScopedInterceptor {
  public:
   ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
   ~ScopedInterceptor();
-  void UserCallbackStart();
-  void UserCallbackEnd();
+  void DisableIgnores();
+  void EnableIgnores();
  private:
   ThreadState *const thr_;
   const uptr pc_;
   bool in_ignored_lib_;
+  bool ignoring_;
 };
 
+LibIgnore *libignore();
+
 }  // namespace __tsan
 
 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
@@ -39,10 +42,10 @@ class ScopedInterceptor {
 /**/
 
 #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
-    si.UserCallbackStart();
+    si.DisableIgnores();
 
 #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \
-    si.UserCallbackEnd();
+    si.EnableIgnores();
 
 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
 
diff --git a/libsanitizer/tsan/tsan_interceptors_mac.cc b/libsanitizer/tsan/tsan_interceptors_mac.cc
index eaf866d6c75..913e9ed0e14 100644
--- a/libsanitizer/tsan/tsan_interceptors_mac.cc
+++ b/libsanitizer/tsan/tsan_interceptors_mac.cc
@@ -19,7 +19,10 @@
 #include "tsan_interface_ann.h"
 
 #include <libkern/OSAtomic.h>
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
 #include <xpc/xpc.h>
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
 
 typedef long long_t;  // NOLINT
 
@@ -233,6 +236,8 @@ TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
   REAL(os_lock_unlock)(lock);
 }
 
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
                  xpc_connection_t connection, xpc_handler_t handler) {
   SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
@@ -279,6 +284,14 @@ TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
   (connection, message, replyq, new_handler);
 }
 
+TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
+  Release(thr, pc, (uptr)connection);
+  REAL(xpc_connection_cancel)(connection);
+}
+
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
 // On macOS, libc++ is always linked dynamically, so intercepting works the
 // usual way.
 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
@@ -295,18 +308,20 @@ struct fake_shared_weak_count {
 };
 }  // namespace
 
-// This adds a libc++ interceptor for:
+// The following code adds libc++ interceptors for:
 //     void __shared_weak_count::__release_shared() _NOEXCEPT;
+//     bool __shared_count::__release_shared() _NOEXCEPT;
 // Shared and weak pointers in C++ maintain reference counts via atomics in
 // libc++.dylib, which are TSan-invisible, and this leads to false positives in
-// destructor code.  This interceptor re-implements the whole function so that
+// destructor code. These interceptors re-implements the whole functions so that
 // the mo_acq_rel semantics of the atomic decrement are visible.
 //
-// Unfortunately, this interceptor cannot simply Acquire/Release some sync
+// Unfortunately, the interceptors cannot simply Acquire/Release some sync
 // object and call the original function, because it would have a race between
 // the sync and the destruction of the object.  Calling both under a lock will
 // not work because the destructor can invoke this interceptor again (and even
 // in a different thread, so recursive locks don't help).
+
 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
                    fake_shared_weak_count *o) {
   if (!flags()->shared_ptr_interceptor)
@@ -325,6 +340,20 @@ STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
   }
 }
 
+STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
+                   fake_shared_weak_count *o) {
+  if (!flags()->shared_ptr_interceptor)
+    return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
+
+  SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
+  if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
+    Acquire(thr, pc, (uptr)&o->shared_owners);
+    o->on_zero_shared();
+    return true;
+  }
+  return false;
+}
+
 namespace {
 struct call_once_callback_args {
   void (*orig_func)(void *arg);
diff --git a/libsanitizer/tsan/tsan_interface.cc b/libsanitizer/tsan/tsan_interface.cc
index ee9a627076b..d98ff15f562 100644
--- a/libsanitizer/tsan/tsan_interface.cc
+++ b/libsanitizer/tsan/tsan_interface.cc
@@ -26,6 +26,10 @@ void __tsan_init() {
   Initialize(cur_thread());
 }
 
+void __tsan_flush_memory() {
+  FlushShadowMemory();
+}
+
 void __tsan_read16(void *addr) {
   MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
   MemoryRead(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
diff --git a/libsanitizer/tsan/tsan_interface.h b/libsanitizer/tsan/tsan_interface.h
index 066dde6f543..7dc67655276 100644
--- a/libsanitizer/tsan/tsan_interface.h
+++ b/libsanitizer/tsan/tsan_interface.h
@@ -16,6 +16,7 @@
 
 #include <sanitizer_common/sanitizer_internal_defs.h>
 using __sanitizer::uptr;
+using __sanitizer::tid_t;
 
 // This header should NOT include any other headers.
 // All functions in this header are extern "C" and start with __tsan_.
@@ -30,6 +31,8 @@ extern "C" {
 // before any instrumented code is executed and before any call to malloc.
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init();
 
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_flush_memory();
+
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1(void *addr);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2(void *addr);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4(void *addr);
@@ -71,6 +74,20 @@ void __tsan_vptr_update(void **vptr_p, void *new_val);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit();
 
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_begin();
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_end();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_external_register_tag(const char *object_type);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_register_header(void *tag, const char *header);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_assign_tag(void *addr, void *tag);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_read(void *addr, void *caller_pc, void *tag);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_write(void *addr, void *caller_pc, void *tag);
+
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_read_range(void *addr, unsigned long size);  // NOLINT
 SANITIZER_INTERFACE_ATTRIBUTE
@@ -116,6 +133,10 @@ int __tsan_get_report_loc(void *report, uptr idx, const char **type,
                           int *fd, int *suppressable, void **trace,
                           uptr trace_size);
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc_object_type(void *report, uptr idx,
+                                      const char **object_type);
+
 // Returns information about mutexes included in the report.
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
@@ -123,7 +144,7 @@ int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
 
 // Returns information about threads included in the report.
 SANITIZER_INTERFACE_ATTRIBUTE
-int __tsan_get_report_thread(void *report, uptr idx, int *tid, uptr *os_id,
+int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id,
                              int *running, const char **name, int *parent_tid,
                              void **trace, uptr trace_size);
 
@@ -131,6 +152,17 @@ int __tsan_get_report_thread(void *report, uptr idx, int *tid, uptr *os_id,
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid);
 
+// Returns the type of the pointer (heap, stack, global, ...) and if possible
+// also the starting address (e.g. of a heap allocation) and size.
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__tsan_locate_address(uptr addr, char *name, uptr name_size,
+                                  uptr *region_address, uptr *region_size);
+
+// Returns the allocation stack for a heap pointer.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id,
+                           tid_t *os_id);
+
 #endif  // SANITIZER_GO
 
 #ifdef __cplusplus
diff --git a/libsanitizer/tsan/tsan_interface_ann.cc b/libsanitizer/tsan/tsan_interface_ann.cc
index 19ff4050abc..083138fc046 100644
--- a/libsanitizer/tsan/tsan_interface_ann.cc
+++ b/libsanitizer/tsan/tsan_interface_ann.cc
@@ -29,11 +29,10 @@ namespace __tsan {
 
 class ScopedAnnotation {
  public:
-  ScopedAnnotation(ThreadState *thr, const char *aname, const char *f, int l,
-                   uptr pc)
+  ScopedAnnotation(ThreadState *thr, const char *aname, uptr pc)
       : thr_(thr) {
     FuncEntry(thr_, pc);
-    DPrintf("#%d: annotation %s() %s:%d\n", thr_->tid, aname, f, l);
+    DPrintf("#%d: annotation %s()\n", thr_->tid, aname);
   }
 
   ~ScopedAnnotation() {
@@ -44,18 +43,20 @@ class ScopedAnnotation {
   ThreadState *const thr_;
 };
 
-#define SCOPED_ANNOTATION(typ) \
+#define SCOPED_ANNOTATION_RET(typ, ret) \
     if (!flags()->enable_annotations) \
-      return; \
+      return ret; \
     ThreadState *thr = cur_thread(); \
     const uptr caller_pc = (uptr)__builtin_return_address(0); \
     StatInc(thr, StatAnnotation); \
     StatInc(thr, Stat##typ); \
-    ScopedAnnotation sa(thr, __func__, f, l, caller_pc); \
+    ScopedAnnotation sa(thr, __func__, caller_pc); \
     const uptr pc = StackTrace::GetCurrentPc(); \
     (void)pc; \
 /**/
 
+#define SCOPED_ANNOTATION(typ) SCOPED_ANNOTATION_RET(typ, )
+
 static const int kMaxDescLen = 128;
 
 struct ExpectRace {
@@ -250,12 +251,12 @@ void INTERFACE_ATTRIBUTE AnnotateCondVarWait(char *f, int l, uptr cv,
 
 void INTERFACE_ATTRIBUTE AnnotateRWLockCreate(char *f, int l, uptr m) {
   SCOPED_ANNOTATION(AnnotateRWLockCreate);
-  MutexCreate(thr, pc, m, true, true, false);
+  MutexCreate(thr, pc, m, MutexFlagWriteReentrant);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateRWLockCreateStatic(char *f, int l, uptr m) {
   SCOPED_ANNOTATION(AnnotateRWLockCreateStatic);
-  MutexCreate(thr, pc, m, true, true, true);
+  MutexCreate(thr, pc, m, MutexFlagWriteReentrant | MutexFlagLinkerInit);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateRWLockDestroy(char *f, int l, uptr m) {
@@ -267,9 +268,9 @@ void INTERFACE_ATTRIBUTE AnnotateRWLockAcquired(char *f, int l, uptr m,
                                                 uptr is_w) {
   SCOPED_ANNOTATION(AnnotateRWLockAcquired);
   if (is_w)
-    MutexLock(thr, pc, m);
+    MutexPostLock(thr, pc, m, MutexFlagDoPreLockOnPostLock);
   else
-    MutexReadLock(thr, pc, m);
+    MutexPostReadLock(thr, pc, m, MutexFlagDoPreLockOnPostLock);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateRWLockReleased(char *f, int l, uptr m,
@@ -456,4 +457,95 @@ void INTERFACE_ATTRIBUTE
 AnnotateMemoryIsInitialized(char *f, int l, uptr mem, uptr sz) {}
 void INTERFACE_ATTRIBUTE
 AnnotateMemoryIsUninitialized(char *f, int l, uptr mem, uptr sz) {}
+
+// Note: the parameter is called flagz, because flags is already taken
+// by the global function that returns flags.
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_create(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_create);
+  MutexCreate(thr, pc, (uptr)m, flagz & MutexCreationFlagMask);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_destroy(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_destroy);
+  MutexDestroy(thr, pc, (uptr)m, flagz);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_lock);
+  if (!(flagz & MutexFlagTryLock)) {
+    if (flagz & MutexFlagReadLock)
+      MutexPreReadLock(thr, pc, (uptr)m);
+    else
+      MutexPreLock(thr, pc, (uptr)m);
+  }
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_lock(void *m, unsigned flagz, int rec) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_lock);
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+  if (!(flagz & MutexFlagTryLockFailed)) {
+    if (flagz & MutexFlagReadLock)
+      MutexPostReadLock(thr, pc, (uptr)m, flagz);
+    else
+      MutexPostLock(thr, pc, (uptr)m, flagz, rec);
+  }
+}
+
+INTERFACE_ATTRIBUTE
+int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION_RET(__tsan_mutex_pre_unlock, 0);
+  int ret = 0;
+  if (flagz & MutexFlagReadLock) {
+    CHECK(!(flagz & MutexFlagRecursiveUnlock));
+    MutexReadUnlock(thr, pc, (uptr)m);
+  } else {
+    ret = MutexUnlock(thr, pc, (uptr)m, flagz);
+  }
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+  return ret;
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_unlock);
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_signal(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_signal);
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_divert);
+  // Exit from ignore region started in __tsan_mutex_pre_lock/unlock/signal.
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_divert);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+}
 }  // extern "C"
diff --git a/libsanitizer/tsan/tsan_interface_atomic.cc b/libsanitizer/tsan/tsan_interface_atomic.cc
index deb4206a624..c175d614764 100644
--- a/libsanitizer/tsan/tsan_interface_atomic.cc
+++ b/libsanitizer/tsan/tsan_interface_atomic.cc
@@ -218,8 +218,7 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
 #endif
 
 template<typename T>
-static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
-    morder mo) {
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
   CHECK(IsLoadOrder(mo));
   // This fast-path is critical for performance.
   // Assume the access is atomic.
@@ -227,10 +226,17 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
     MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
     return NoTsanAtomicLoad(a, mo);
   }
-  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false);
-  AcquireImpl(thr, pc, &s->clock);
+  // Don't create sync object if it does not exist yet. For example, an atomic
+  // pointer is initialized to nullptr and then periodically acquire-loaded.
   T v = NoTsanAtomicLoad(a, mo);
-  s->mtx.ReadUnlock();
+  SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
+  if (s) {
+    AcquireImpl(thr, pc, &s->clock);
+    // Re-read under sync mutex because we need a consistent snapshot
+    // of the value and the clock we acquire.
+    v = NoTsanAtomicLoad(a, mo);
+    s->mtx.ReadUnlock();
+  }
   MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
   return v;
 }
@@ -265,7 +271,7 @@ static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
   thr->fast_state.IncrementEpoch();
   // Can't increment epoch w/o writing to the trace as well.
   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-  ReleaseImpl(thr, pc, &s->clock);
+  ReleaseStoreImpl(thr, pc, &s->clock);
   NoTsanAtomicStore(a, v, mo);
   s->mtx.Unlock();
 }
@@ -448,7 +454,7 @@ static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
 
 // C/C++
 
-static morder covert_morder(morder mo) {
+static morder convert_morder(morder mo) {
   if (flags()->force_seq_cst_atomics)
     return (morder)mo_seq_cst;
 
@@ -466,12 +472,14 @@ static morder covert_morder(morder mo) {
 }
 
 #define SCOPED_ATOMIC(func, ...) \
-    const uptr callpc = (uptr)__builtin_return_address(0); \
-    uptr pc = StackTrace::GetCurrentPc(); \
-    mo = covert_morder(mo); \
     ThreadState *const thr = cur_thread(); \
-    if (thr->ignore_interceptors) \
+    if (thr->ignore_sync || thr->ignore_interceptors) { \
+      ProcessPendingSignals(thr); \
       return NoTsanAtomic##func(__VA_ARGS__); \
+    } \
+    const uptr callpc = (uptr)__builtin_return_address(0); \
+    uptr pc = StackTrace::GetCurrentPc(); \
+    mo = convert_morder(mo); \
     AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
     ScopedAtomic sa(thr, callpc, a, mo, __func__); \
     return Atomic##func(thr, pc, __VA_ARGS__); \
diff --git a/libsanitizer/tsan/tsan_interface_inl.h b/libsanitizer/tsan/tsan_interface_inl.h
index 67a91aa341a..bf70cdce34b 100644
--- a/libsanitizer/tsan/tsan_interface_inl.h
+++ b/libsanitizer/tsan/tsan_interface_inl.h
@@ -106,6 +106,14 @@ void __tsan_func_exit() {
   FuncExit(cur_thread());
 }
 
+void __tsan_ignore_thread_begin() {
+  ThreadIgnoreBegin(cur_thread(), CALLERPC);
+}
+
+void __tsan_ignore_thread_end() {
+  ThreadIgnoreEnd(cur_thread(), CALLERPC);
+}
+
 void __tsan_read_range(void *addr, uptr size) {
   MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
 }
diff --git a/libsanitizer/tsan/tsan_interface_java.cc b/libsanitizer/tsan/tsan_interface_java.cc
index d1638f2aa9d..d3f35a9bc82 100644
--- a/libsanitizer/tsan/tsan_interface_java.cc
+++ b/libsanitizer/tsan/tsan_interface_java.cc
@@ -148,6 +148,23 @@ void __tsan_java_move(jptr src, jptr dst, jptr size) {
   }
 }
 
+jptr __tsan_java_find(jptr *from_ptr, jptr to) {
+  SCOPED_JAVA_FUNC(__tsan_java_find);
+  DPrintf("#%d: java_find(&%p, %p)\n", *from_ptr, to);
+  CHECK_EQ((*from_ptr) % kHeapAlignment, 0);
+  CHECK_EQ(to % kHeapAlignment, 0);
+  CHECK_GE(*from_ptr, jctx->heap_begin);
+  CHECK_LE(to, jctx->heap_begin + jctx->heap_size);
+  for (uptr from = *from_ptr; from < to; from += kHeapAlignment) {
+    MBlock *b = ctx->metamap.GetBlock(from);
+    if (b) {
+      *from_ptr = from;
+      return b->siz;
+    }
+  }
+  return 0;
+}
+
 void __tsan_java_finalize() {
   SCOPED_JAVA_FUNC(__tsan_java_finalize);
   DPrintf("#%d: java_mutex_finalize()\n", thr->tid);
@@ -161,8 +178,8 @@ void __tsan_java_mutex_lock(jptr addr) {
   CHECK_GE(addr, jctx->heap_begin);
   CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
 
-  MutexCreate(thr, pc, addr, true, true, true);
-  MutexLock(thr, pc, addr);
+  MutexPostLock(thr, pc, addr, MutexFlagLinkerInit | MutexFlagWriteReentrant |
+      MutexFlagDoPreLockOnPostLock);
 }
 
 void __tsan_java_mutex_unlock(jptr addr) {
@@ -182,8 +199,8 @@ void __tsan_java_mutex_read_lock(jptr addr) {
   CHECK_GE(addr, jctx->heap_begin);
   CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
 
-  MutexCreate(thr, pc, addr, true, true, true);
-  MutexReadLock(thr, pc, addr);
+  MutexPostReadLock(thr, pc, addr, MutexFlagLinkerInit |
+      MutexFlagWriteReentrant | MutexFlagDoPreLockOnPostLock);
 }
 
 void __tsan_java_mutex_read_unlock(jptr addr) {
@@ -204,8 +221,8 @@ void __tsan_java_mutex_lock_rec(jptr addr, int rec) {
   CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
   CHECK_GT(rec, 0);
 
-  MutexCreate(thr, pc, addr, true, true, true);
-  MutexLock(thr, pc, addr, rec);
+  MutexPostLock(thr, pc, addr, MutexFlagLinkerInit | MutexFlagWriteReentrant |
+      MutexFlagDoPreLockOnPostLock | MutexFlagRecursiveLock, rec);
 }
 
 int __tsan_java_mutex_unlock_rec(jptr addr) {
@@ -215,7 +232,7 @@ int __tsan_java_mutex_unlock_rec(jptr addr) {
   CHECK_GE(addr, jctx->heap_begin);
   CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
 
-  return MutexUnlock(thr, pc, addr, true);
+  return MutexUnlock(thr, pc, addr, MutexFlagRecursiveUnlock);
 }
 
 void __tsan_java_acquire(jptr addr) {
diff --git a/libsanitizer/tsan/tsan_interface_java.h b/libsanitizer/tsan/tsan_interface_java.h
index 04e52031dd3..2dd49f05c68 100644
--- a/libsanitizer/tsan/tsan_interface_java.h
+++ b/libsanitizer/tsan/tsan_interface_java.h
@@ -55,6 +55,10 @@ void __tsan_java_move(jptr src, jptr dst, jptr size) INTERFACE_ATTRIBUTE;
 // It ensures necessary synchronization between
 // java object creation and finalization.
 void __tsan_java_finalize() INTERFACE_ATTRIBUTE;
+// Finds the first allocated memory block in the [*from_ptr, to) range, saves
+// its address in *from_ptr and returns its size. Returns 0 if there are no
+// allocated memory blocks in the range.
+jptr __tsan_java_find(jptr *from_ptr, jptr to) INTERFACE_ATTRIBUTE;
 
 // Mutex lock.
 // Addr is any unique address associated with the mutex.
diff --git a/libsanitizer/tsan/tsan_libdispatch_mac.cc b/libsanitizer/tsan/tsan_libdispatch_mac.cc
index 10c70a831c6..5200a791fc2 100644
--- a/libsanitizer/tsan/tsan_libdispatch_mac.cc
+++ b/libsanitizer/tsan/tsan_libdispatch_mac.cc
@@ -66,13 +66,17 @@ static bool IsQueueSerial(dispatch_queue_t q) {
   return width == 1;
 }
 
-static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
+static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
   CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
-  dispatch_queue_t target_queue =
-      *(dispatch_queue_t *)(((uptr)source) +
-                            dispatch_queue_offsets.dqo_target_queue);
-  CHECK_NE(target_queue, 0);
-  return target_queue;
+  dispatch_queue_t tq = *(
+      dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
+  return tq;
+}
+
+static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
+  dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
+  CHECK_NE(tq, 0);
+  return tq;
 }
 
 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
@@ -80,37 +84,66 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
                                           void *orig_context,
                                           dispatch_function_t orig_work) {
   tsan_block_context_t *new_context =
-      (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
+      (tsan_block_context_t *)user_alloc_internal(thr, pc,
+                                                  sizeof(tsan_block_context_t));
   new_context->queue = queue;
   new_context->orig_context = orig_context;
   new_context->orig_work = orig_work;
   new_context->free_context_in_callback = true;
   new_context->submitted_synchronously = false;
   new_context->is_barrier_block = false;
+  new_context->non_queue_sync_object = 0;
   return new_context;
 }
 
+#define GET_QUEUE_SYNC_VARS(context, q)                                  \
+  bool is_queue_serial = q && IsQueueSerial(q);                          \
+  uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object;             \
+  uptr serial_sync = (uptr)sync_ptr;                                     \
+  uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
+  bool serial_task = context->is_barrier_block || is_queue_serial
+
+static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
+                                      tsan_block_context_t *context) {
+  uptr submit_sync = (uptr)context;
+  Acquire(thr, pc, submit_sync);
+
+  dispatch_queue_t q = context->queue;
+  do {
+    GET_QUEUE_SYNC_VARS(context, q);
+    if (serial_sync) Acquire(thr, pc, serial_sync);
+    if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
+
+    if (q) q = GetTargetQueueFromQueue(q);
+  } while (q);
+}
+
+static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
+                                       tsan_block_context_t *context) {
+  uptr submit_sync = (uptr)context;
+  if (context->submitted_synchronously) Release(thr, pc, submit_sync);
+
+  dispatch_queue_t q = context->queue;
+  do {
+    GET_QUEUE_SYNC_VARS(context, q);
+    if (serial_task && serial_sync) Release(thr, pc, serial_sync);
+    if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
+
+    if (q) q = GetTargetQueueFromQueue(q);
+  } while (q);
+}
+
 static void dispatch_callback_wrap(void *param) {
   SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
   tsan_block_context_t *context = (tsan_block_context_t *)param;
-  bool is_queue_serial = context->queue && IsQueueSerial(context->queue);
-  uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object;
-
-  uptr serial_sync = (uptr)sync_ptr;
-  uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr);
-  uptr submit_sync = (uptr)context;
-  bool serial_task = context->is_barrier_block || is_queue_serial;
 
-  Acquire(thr, pc, submit_sync);
-  Acquire(thr, pc, serial_sync);
-  if (serial_task) Acquire(thr, pc, concurrent_sync);
+  dispatch_sync_pre_execute(thr, pc, context);
 
   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
   context->orig_work(context->orig_context);
   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 
-  Release(thr, pc, serial_task ? serial_sync : concurrent_sync);
-  if (context->submitted_synchronously) Release(thr, pc, submit_sync);
+  dispatch_sync_post_execute(thr, pc, context);
 
   if (context->free_context_in_callback) user_free(thr, pc, context);
 }
@@ -142,7 +175,8 @@ static void invoke_and_release_block(void *param) {
   }
 
 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
-  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
+  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q,                           \
+                   DISPATCH_NOESCAPE dispatch_block_t block) {               \
     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
     dispatch_block_t heap_block = Block_copy(block);                         \
@@ -232,7 +266,7 @@ TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
 // need to undefine the macro.
 #undef dispatch_once
 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
-                 dispatch_block_t block) {
+                 DISPATCH_NOESCAPE dispatch_block_t block) {
   SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
   atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
   u32 v = atomic_load(a, memory_order_acquire);
@@ -442,7 +476,8 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
 }
 
 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
-                 dispatch_queue_t queue, void (^block)(size_t)) {
+                 dispatch_queue_t queue,
+                 DISPATCH_NOESCAPE void (^block)(size_t)) {
   SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
 
   void *parent_to_child_sync = nullptr;
@@ -674,6 +709,15 @@ TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
   return REAL(dispatch_io_close)(channel, flags);
 }
 
+// Resuming a suspended queue needs to synchronize with all subsequent
+// executions of blocks in that queue.
+TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
+  Release(thr, pc, (uptr)o);  // Synchronizes with the Acquire() on serial_sync
+                              // in dispatch_sync_pre_execute
+  return REAL(dispatch_resume)(o);
+}
+
 }  // namespace __tsan
 
 #endif  // SANITIZER_MAC
diff --git a/libsanitizer/tsan/tsan_malloc_mac.cc b/libsanitizer/tsan/tsan_malloc_mac.cc
index cc1031a584f..b418a211b58 100644
--- a/libsanitizer/tsan/tsan_malloc_mac.cc
+++ b/libsanitizer/tsan/tsan_malloc_mac.cc
@@ -24,7 +24,7 @@ using namespace __tsan;
 #define COMMON_MALLOC_FORCE_UNLOCK()
 #define COMMON_MALLOC_MEMALIGN(alignment, size) \
   void *p =                                     \
-      user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment)
+      user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size)
 #define COMMON_MALLOC_MALLOC(size)                             \
   if (cur_thread()->in_symbolizer) return InternalAlloc(size); \
   SCOPED_INTERCEPTOR_RAW(malloc, size);                        \
@@ -41,7 +41,7 @@ using namespace __tsan;
   if (cur_thread()->in_symbolizer)                            \
     return InternalAlloc(size, nullptr, GetPageSizeCached()); \
   SCOPED_INTERCEPTOR_RAW(valloc, size);                       \
-  void *p = user_alloc(thr, pc, size, GetPageSizeCached())
+  void *p = user_valloc(thr, pc, size)
 #define COMMON_MALLOC_FREE(ptr)                              \
   if (cur_thread()->in_symbolizer) return InternalFree(ptr); \
   SCOPED_INTERCEPTOR_RAW(free, ptr);                         \
diff --git a/libsanitizer/tsan/tsan_mman.cc b/libsanitizer/tsan/tsan_mman.cc
index 152c2de28d8..18505aca70e 100644
--- a/libsanitizer/tsan/tsan_mman.cc
+++ b/libsanitizer/tsan/tsan_mman.cc
@@ -8,8 +8,10 @@
 // This file is a part of ThreadSanitizer (TSan), a race detector.
 //
 //===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "tsan_mman.h"
 #include "tsan_rtl.h"
@@ -52,7 +54,8 @@ struct MapUnmapCallback {
     diff = p + size - RoundDown(p + size, kPageSize);
     if (diff != 0)
       size -= diff;
-    ReleaseMemoryToOS((uptr)MemToMeta(p), size / kMetaRatio);
+    uptr p_meta = (uptr)MemToMeta(p);
+    ReleaseMemoryPagesToOS(p_meta, p_meta + size / kMetaRatio);
   }
 };
 
@@ -109,7 +112,8 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {
 }
 
 void InitializeAllocator() {
-  allocator()->Init(common_flags()->allocator_may_return_null);
+  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+  allocator()->Init(common_flags()->allocator_release_to_os_interval_ms);
 }
 
 void InitializeAllocatorLate() {
@@ -144,11 +148,12 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   OutputReport(thr, rep);
 }
 
-void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) {
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
+                          bool signal) {
   if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
-    return allocator()->ReturnNullOrDieOnBadRequest();
+    return Allocator::FailureHandler::OnBadRequest();
   void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
-  if (p == 0)
+  if (UNLIKELY(p == 0))
     return 0;
   if (ctx && ctx->initialized)
     OnUserAlloc(thr, pc, (uptr)p, sz, true);
@@ -157,15 +162,6 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) {
   return p;
 }
 
-void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
-  if (CallocShouldReturnNullDueToOverflow(size, n))
-    return allocator()->ReturnNullOrDieOnBadRequest();
-  void *p = user_alloc(thr, pc, n * size);
-  if (p)
-    internal_memset(p, 0, n * size);
-  return p;
-}
-
 void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
   ScopedGlobalProcessor sgp;
   if (ctx && ctx->initialized)
@@ -175,6 +171,19 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
     SignalUnsafeCall(thr, pc);
 }
 
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment));
+}
+
+void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
+  if (UNLIKELY(CheckForCallocOverflow(size, n)))
+    return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest());
+  void *p = user_alloc_internal(thr, pc, n * size);
+  if (p)
+    internal_memset(p, 0, n * size);
+  return SetErrnoOnNull(p);
+}
+
 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
   DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
   ctx->metamap.AllocBlock(thr, pc, p, sz);
@@ -195,15 +204,64 @@ void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
 void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
   // FIXME: Handle "shrinking" more efficiently,
   // it seems that some software actually does this.
-  void *p2 = user_alloc(thr, pc, sz);
-  if (p2 == 0)
-    return 0;
-  if (p) {
-    uptr oldsz = user_alloc_usable_size(p);
-    internal_memcpy(p2, p, min(oldsz, sz));
+  if (!p)
+    return SetErrnoOnNull(user_alloc_internal(thr, pc, sz));
+  if (!sz) {
     user_free(thr, pc, p);
+    return nullptr;
+  }
+  void *new_p = user_alloc_internal(thr, pc, sz);
+  if (new_p) {
+    uptr old_sz = user_alloc_usable_size(p);
+    internal_memcpy(new_p, p, min(old_sz, sz));
+    user_free(thr, pc, p);
+  }
+  return SetErrnoOnNull(new_p);
+}
+
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+  if (UNLIKELY(!IsPowerOfTwo(align))) {
+    errno = errno_EINVAL;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+                        uptr sz) {
+  if (UNLIKELY(!CheckPosixMemalignAlignment(align))) {
+    Allocator::FailureHandler::OnBadRequest();
+    return errno_EINVAL;
+  }
+  void *ptr = user_alloc_internal(thr, pc, sz, align);
+  if (UNLIKELY(!ptr))
+    return errno_ENOMEM;
+  CHECK(IsAligned((uptr)ptr, align));
+  *memptr = ptr;
+  return 0;
+}
+
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) {
+    errno = errno_EINVAL;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz) {
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached()));
+}
+
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
+  uptr PageSize = GetPageSizeCached();
+  if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) {
+    errno = errno_ENOMEM;
+    return Allocator::FailureHandler::OnBadRequest();
   }
-  return p2;
+  // pvalloc(0) should allocate one page.
+  sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
 }
 
 uptr user_alloc_usable_size(const void *p) {
@@ -290,6 +348,8 @@ uptr __sanitizer_get_allocated_size(const void *p) {
 
 void __tsan_on_thread_idle() {
   ThreadState *thr = cur_thread();
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
   allocator()->SwallowCache(&thr->proc()->alloc_cache);
   internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
   ctx->metamap.OnProcIdle(thr->proc());
diff --git a/libsanitizer/tsan/tsan_mman.h b/libsanitizer/tsan/tsan_mman.h
index d49c9a9e9f2..3443cb0d8f9 100644
--- a/libsanitizer/tsan/tsan_mman.h
+++ b/libsanitizer/tsan/tsan_mman.h
@@ -25,13 +25,20 @@ void AllocatorProcFinish(Processor *proc);
 void AllocatorPrintStats();
 
 // For user allocations.
-void *user_alloc(ThreadState *thr, uptr pc, uptr sz,
-                 uptr align = kDefaultAlignment, bool signal = true);
-void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz,
+                          uptr align = kDefaultAlignment, bool signal = true);
 // Does not accept NULL.
 void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true);
+// Interceptor implementations.
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
 void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz);
-void *user_alloc_aligned(ThreadState *thr, uptr pc, uptr sz, uptr align);
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz);
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+                        uptr sz);
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz);
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz);
 uptr user_alloc_usable_size(const void *p);
 
 // Invoking malloc/free hooks that may be installed by the user.
diff --git a/libsanitizer/tsan/tsan_new_delete.cc b/libsanitizer/tsan/tsan_new_delete.cc
index 606cdd659f4..65ae61b8164 100644
--- a/libsanitizer/tsan/tsan_new_delete.cc
+++ b/libsanitizer/tsan/tsan_new_delete.cc
@@ -10,6 +10,7 @@
 // Interceptors for operators new and delete.
 //===----------------------------------------------------------------------===//
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "tsan_interceptors.h"
 
@@ -22,13 +23,15 @@ struct nothrow_t {};
 DECLARE_REAL(void *, malloc, uptr size)
 DECLARE_REAL(void, free, void *ptr)
 
-#define OPERATOR_NEW_BODY(mangled_name) \
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(mangled_name, nothrow) \
   if (cur_thread()->in_symbolizer) \
     return InternalAlloc(size); \
   void *p = 0; \
   {  \
     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
     p = user_alloc(thr, pc, size); \
+    if (!nothrow && UNLIKELY(!p)) DieOnFailure::OnOOM(); \
   }  \
   invoke_malloc_hook(p, size);  \
   return p;
@@ -36,25 +39,25 @@ DECLARE_REAL(void, free, void *ptr)
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new(__sanitizer::uptr size);
 void *operator new(__sanitizer::uptr size) {
-  OPERATOR_NEW_BODY(_Znwm);
+  OPERATOR_NEW_BODY(_Znwm, false /*nothrow*/);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new[](__sanitizer::uptr size);
 void *operator new[](__sanitizer::uptr size) {
-  OPERATOR_NEW_BODY(_Znam);
+  OPERATOR_NEW_BODY(_Znam, false /*nothrow*/);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
 void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t);
+  OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t, true /*nothrow*/);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
+  OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t, true /*nothrow*/);
 }
 
 #define OPERATOR_DELETE_BODY(mangled_name) \
diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h
index 368edc21ce4..ddf4b13d05a 100644
--- a/libsanitizer/tsan/tsan_platform.h
+++ b/libsanitizer/tsan/tsan_platform.h
@@ -98,6 +98,37 @@ struct Mapping {
 };
 
 #define TSAN_MID_APP_RANGE 1
+#elif defined(__aarch64__) && defined(__APPLE__)
+/*
+C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
+0000 0000 00 - 0100 0000 00: -                                    (4 GB)
+0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
+0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
+0300 0000 00 - 0400 0000 00: -                                    (4 GB)
+0400 0000 00 - 0c00 0000 00: shadow memory                       (32 GB)
+0c00 0000 00 - 0d00 0000 00: -                                    (4 GB)
+0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
+0e00 0000 00 - 0f00 0000 00: -                                    (4 GB)
+0f00 0000 00 - 1000 0000 00: traces                               (4 GB)
+*/
+struct Mapping {
+  static const uptr kLoAppMemBeg   = 0x0100000000ull;
+  static const uptr kLoAppMemEnd   = 0x0200000000ull;
+  static const uptr kHeapMemBeg    = 0x0200000000ull;
+  static const uptr kHeapMemEnd    = 0x0300000000ull;
+  static const uptr kShadowBeg     = 0x0400000000ull;
+  static const uptr kShadowEnd     = 0x0c00000000ull;
+  static const uptr kMetaShadowBeg = 0x0d00000000ull;
+  static const uptr kMetaShadowEnd = 0x0e00000000ull;
+  static const uptr kTraceMemBeg   = 0x0f00000000ull;
+  static const uptr kTraceMemEnd   = 0x1000000000ull;
+  static const uptr kHiAppMemBeg   = 0x1000000000ull;
+  static const uptr kHiAppMemEnd   = 0x1000000000ull;
+  static const uptr kAppMemMsk     =          0x0ull;
+  static const uptr kAppMemXor     =          0x0ull;
+  static const uptr kVdsoBeg       = 0x7000000000000000ull;
+};
+
 #elif defined(__aarch64__)
 // AArch64 supports multiple VMA which leads to multiple address transformation
 // functions.  To support these multiple VMAS transformations and mappings TSAN
@@ -387,7 +418,7 @@ uptr MappingImpl(void) {
 
 template<int Type>
 uptr MappingArchImpl(void) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return MappingImpl<Mapping39, Type>();
     case 42: return MappingImpl<Mapping42, Type>();
@@ -540,7 +571,7 @@ bool IsAppMemImpl(uptr mem) {
 
 ALWAYS_INLINE
 bool IsAppMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return IsAppMemImpl<Mapping39>(mem);
     case 42: return IsAppMemImpl<Mapping42>(mem);
@@ -567,7 +598,7 @@ bool IsShadowMemImpl(uptr mem) {
 
 ALWAYS_INLINE
 bool IsShadowMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return IsShadowMemImpl<Mapping39>(mem);
     case 42: return IsShadowMemImpl<Mapping42>(mem);
@@ -594,7 +625,7 @@ bool IsMetaMemImpl(uptr mem) {
 
 ALWAYS_INLINE
 bool IsMetaMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return IsMetaMemImpl<Mapping39>(mem);
     case 42: return IsMetaMemImpl<Mapping42>(mem);
@@ -631,7 +662,7 @@ uptr MemToShadowImpl(uptr x) {
 
 ALWAYS_INLINE
 uptr MemToShadow(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return MemToShadowImpl<Mapping39>(x);
     case 42: return MemToShadowImpl<Mapping42>(x);
@@ -670,7 +701,7 @@ u32 *MemToMetaImpl(uptr x) {
 
 ALWAYS_INLINE
 u32 *MemToMeta(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return MemToMetaImpl<Mapping39>(x);
     case 42: return MemToMetaImpl<Mapping42>(x);
@@ -722,7 +753,7 @@ uptr ShadowToMemImpl(uptr s) {
 
 ALWAYS_INLINE
 uptr ShadowToMem(uptr s) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return ShadowToMemImpl<Mapping39>(s);
     case 42: return ShadowToMemImpl<Mapping42>(s);
@@ -757,7 +788,7 @@ uptr GetThreadTraceImpl(int tid) {
 
 ALWAYS_INLINE
 uptr GetThreadTrace(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return GetThreadTraceImpl<Mapping39>(tid);
     case 42: return GetThreadTraceImpl<Mapping42>(tid);
@@ -787,7 +818,7 @@ uptr GetThreadTraceHeaderImpl(int tid) {
 
 ALWAYS_INLINE
 uptr GetThreadTraceHeader(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
     case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
@@ -814,6 +845,7 @@ void FlushShadowMemory();
 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
 int ExtractResolvFDs(void *state, int *fds, int nfd);
 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
 
 int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
     void *abstime), void *c, void *m, void *abstime,
diff --git a/libsanitizer/tsan/tsan_platform_linux.cc b/libsanitizer/tsan/tsan_platform_linux.cc
index 6f972ab0dd6..f8ae2567478 100644
--- a/libsanitizer/tsan/tsan_platform_linux.cc
+++ b/libsanitizer/tsan/tsan_platform_linux.cc
@@ -45,7 +45,6 @@
 #include <sys/resource.h>
 #include <sys/stat.h>
 #include <unistd.h>
-#include <errno.h>
 #include <sched.h>
 #include <dlfcn.h>
 #if SANITIZER_LINUX
@@ -132,7 +131,7 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
 void FlushShadowMemoryCallback(
     const SuspendedThreadsList &suspended_threads_list,
     void *argument) {
-  ReleaseMemoryToOS(ShadowBeg(), ShadowEnd() - ShadowBeg());
+  ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
 }
 #endif
 
@@ -180,17 +179,15 @@ static void MapRodata() {
   }
   // Map the file into shadow of .rodata sections.
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr start, end, offset, prot;
   // Reusing the buffer 'name'.
-  while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) {
-    if (name[0] != 0 && name[0] != '['
-        && (prot & MemoryMappingLayout::kProtectionRead)
-        && (prot & MemoryMappingLayout::kProtectionExecute)
-        && !(prot & MemoryMappingLayout::kProtectionWrite)
-        && IsAppMem(start)) {
+  MemoryMappedSegment segment(name, ARRAY_SIZE(name));
+  while (proc_maps.Next(&segment)) {
+    if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
+        segment.IsReadable() && segment.IsExecutable() &&
+        !segment.IsWritable() && IsAppMem(segment.start)) {
       // Assume it's .rodata
-      char *shadow_start = (char*)MemToShadow(start);
-      char *shadow_end = (char*)MemToShadow(end);
+      char *shadow_start = (char *)MemToShadow(segment.start);
+      char *shadow_end = (char *)MemToShadow(segment.end);
       for (char *p = shadow_start; p < shadow_end; p += marker.size()) {
         internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p),
                       PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
@@ -318,6 +315,20 @@ int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) {
   return res;
 }
 
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
+  // Check that the thr object is in tls;
+  const uptr thr_beg = (uptr)thr;
+  const uptr thr_end = (uptr)thr + sizeof(*thr);
+  CHECK_GE(thr_beg, tls_addr);
+  CHECK_LE(thr_beg, tls_addr + tls_size);
+  CHECK_GE(thr_end, tls_addr);
+  CHECK_LE(thr_end, tls_addr + tls_size);
+  // Since the thr object is huge, skip it.
+  MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, thr_beg - tls_addr);
+  MemoryRangeImitateWrite(thr, /*pc=*/2, thr_end,
+                          tls_addr + tls_size - thr_end);
+}
+
 // Note: this function runs with async signals enabled,
 // so it must not touch any tsan state.
 int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
@@ -339,36 +350,22 @@ void ReplaceSystemMalloc() { }
 
 #if !SANITIZER_GO
 #if SANITIZER_ANDROID
-
-#if defined(__aarch64__)
-# define __get_tls() \
-    ({ void** __val; __asm__("mrs %0, tpidr_el0" : "=r"(__val)); __val; })
-#elif defined(__x86_64__)
-# define __get_tls() \
-    ({ void** __val; __asm__("mov %%fs:0, %0" : "=r"(__val)); __val; })
-#else
-#error unsupported architecture
-#endif
-
-// On Android, __thread is not supported. So we store the pointer to ThreadState
-// in TLS_SLOT_TSAN, which is the tls slot allocated by Android bionic for tsan.
-static const int TLS_SLOT_TSAN = 8;
 // On Android, one thread can call intercepted functions after
 // DestroyThreadState(), so add a fake thread state for "dead" threads.
 static ThreadState *dead_thread_state = nullptr;
 
 ThreadState *cur_thread() {
-  ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN];
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
   if (thr == nullptr) {
     __sanitizer_sigset_t emptyset;
     internal_sigfillset(&emptyset);
     __sanitizer_sigset_t oldset;
     CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
-    thr = reinterpret_cast<ThreadState*>(__get_tls()[TLS_SLOT_TSAN]);
+    thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
     if (thr == nullptr) {
       thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState),
                                                      "ThreadState"));
-      __get_tls()[TLS_SLOT_TSAN] = thr;
+      *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
       if (dead_thread_state == nullptr) {
         dead_thread_state = reinterpret_cast<ThreadState*>(
             MmapOrDie(sizeof(ThreadState), "ThreadState"));
@@ -390,9 +387,9 @@ void cur_thread_finalize() {
   internal_sigfillset(&emptyset);
   __sanitizer_sigset_t oldset;
   CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
-  ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN];
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
   if (thr != dead_thread_state) {
-    __get_tls()[TLS_SLOT_TSAN] = dead_thread_state;
+    *get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state);
     UnmapOrDie(thr, sizeof(ThreadState));
   }
   CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
diff --git a/libsanitizer/tsan/tsan_platform_mac.cc b/libsanitizer/tsan/tsan_platform_mac.cc
index ff5131e75bb..8eb22fa58ea 100644
--- a/libsanitizer/tsan/tsan_platform_mac.cc
+++ b/libsanitizer/tsan/tsan_platform_mac.cc
@@ -18,10 +18,12 @@
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_posix.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
 #include "tsan_platform.h"
 #include "tsan_rtl.h"
 #include "tsan_flags.h"
 
+#include <mach/mach.h>
 #include <pthread.h>
 #include <signal.h>
 #include <stdio.h>
@@ -71,12 +73,18 @@ static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) {
 static uptr main_thread_identity = 0;
 ALIGNED(64) static char main_thread_state[sizeof(ThreadState)];
 
+ThreadState **cur_thread_location() {
+  ThreadState **thread_identity = (ThreadState **)pthread_self();
+  return ((uptr)thread_identity == main_thread_identity) ? nullptr
+                                                         : thread_identity;
+}
+
 ThreadState *cur_thread() {
-  uptr thread_identity = (uptr)pthread_self();
-  if (thread_identity == main_thread_identity || main_thread_identity == 0) {
+  ThreadState **thr_state_loc = cur_thread_location();
+  if (thr_state_loc == nullptr || main_thread_identity == 0) {
     return (ThreadState *)&main_thread_state;
   }
-  ThreadState **fake_tls = (ThreadState **)MemToShadow(thread_identity);
+  ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc);
   ThreadState *thr = (ThreadState *)SignalSafeGetOrAllocate(
       (uptr *)fake_tls, sizeof(ThreadState));
   return thr;
@@ -86,26 +94,92 @@ ThreadState *cur_thread() {
 // munmap first and then clear `fake_tls`; if we receive a signal in between,
 // handler will try to access the unmapped ThreadState.
 void cur_thread_finalize() {
-  uptr thread_identity = (uptr)pthread_self();
-  if (thread_identity == main_thread_identity) {
+  ThreadState **thr_state_loc = cur_thread_location();
+  if (thr_state_loc == nullptr) {
     // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
     // exit the main thread. Let's keep the main thread's ThreadState.
     return;
   }
-  ThreadState **fake_tls = (ThreadState **)MemToShadow(thread_identity);
+  ThreadState **fake_tls = (ThreadState **)MemToShadow((uptr)thr_state_loc);
   internal_munmap(*fake_tls, sizeof(ThreadState));
   *fake_tls = nullptr;
 }
 #endif
 
-uptr GetShadowMemoryConsumption() {
-  return 0;
+void FlushShadowMemory() {
 }
 
-void FlushShadowMemory() {
+static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) {
+  vm_address_t address = start;
+  vm_address_t end_address = end;
+  uptr resident_pages = 0;
+  uptr dirty_pages = 0;
+  while (address < end_address) {
+    vm_size_t vm_region_size;
+    mach_msg_type_number_t count = VM_REGION_EXTENDED_INFO_COUNT;
+    vm_region_extended_info_data_t vm_region_info;
+    mach_port_t object_name;
+    kern_return_t ret = vm_region_64(
+        mach_task_self(), &address, &vm_region_size, VM_REGION_EXTENDED_INFO,
+        (vm_region_info_t)&vm_region_info, &count, &object_name);
+    if (ret != KERN_SUCCESS) break;
+
+    resident_pages += vm_region_info.pages_resident;
+    dirty_pages += vm_region_info.pages_dirtied;
+
+    address += vm_region_size;
+  }
+  *res = resident_pages * GetPageSizeCached();
+  *dirty = dirty_pages * GetPageSizeCached();
 }
 
 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
+  uptr shadow_res, shadow_dirty;
+  uptr meta_res, meta_dirty;
+  uptr trace_res, trace_dirty;
+  RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
+  RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
+  RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
+
+#if !SANITIZER_GO
+  uptr low_res, low_dirty;
+  uptr high_res, high_dirty;
+  uptr heap_res, heap_dirty;
+  RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &low_res, &low_dirty);
+  RegionMemUsage(HiAppMemBeg(), HiAppMemEnd(), &high_res, &high_dirty);
+  RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
+#else  // !SANITIZER_GO
+  uptr app_res, app_dirty;
+  RegionMemUsage(AppMemBeg(), AppMemEnd(), &app_res, &app_dirty);
+#endif
+
+  StackDepotStats *stacks = StackDepotGetStats();
+  internal_snprintf(buf, buf_size,
+    "shadow   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "meta     (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "traces   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#if !SANITIZER_GO
+    "low app  (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "heap     (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#else  // !SANITIZER_GO
+    "app      (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#endif
+    "stacks: %zd unique IDs, %zd kB allocated\n"
+    "threads: %zd total, %zd live\n"
+    "------------------------------\n",
+    ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
+    MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
+    TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
+#if !SANITIZER_GO
+    LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
+    HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
+    HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
+#else  // !SANITIZER_GO
+    AppMemBeg(), AppMemEnd(), app_res / 1024, app_dirty / 1024,
+#endif
+    stacks->n_uniq_ids, stacks->allocated / 1024,
+    nthread, nlive);
 }
 
 #if !SANITIZER_GO
@@ -137,7 +211,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
       ThreadState *parent_thread_state = nullptr;  // No parent.
       int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
       CHECK_NE(tid, 0);
-      ThreadStart(thr, tid, GetTid());
+      ThreadStart(thr, tid, GetTid(), /*workerthread*/ true);
     }
   } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
     if (thread == pthread_self()) {
@@ -154,6 +228,14 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
 #endif
 
 void InitializePlatformEarly() {
+#if defined(__aarch64__)
+  uptr max_vm = GetMaxVirtualAddress() + 1;
+  if (max_vm != Mapping::kHiAppMemEnd) {
+    Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
+           max_vm, Mapping::kHiAppMemEnd);
+    Die();
+  }
+#endif
 }
 
 void InitializePlatform() {
@@ -170,6 +252,29 @@ void InitializePlatform() {
 }
 
 #if !SANITIZER_GO
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
+  // The pointer to the ThreadState object is stored in the shadow memory
+  // of the tls.
+  uptr tls_end = tls_addr + tls_size;
+  ThreadState **thr_state_loc = cur_thread_location();
+  if (thr_state_loc == nullptr) {
+    MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, tls_size);
+  } else {
+    uptr thr_state_start = (uptr)thr_state_loc;
+    uptr thr_state_end = thr_state_start + sizeof(uptr);
+    CHECK_GE(thr_state_start, tls_addr);
+    CHECK_LE(thr_state_start, tls_addr + tls_size);
+    CHECK_GE(thr_state_end, tls_addr);
+    CHECK_LE(thr_state_end, tls_addr + tls_size);
+    MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr,
+                            thr_state_start - tls_addr);
+    MemoryRangeImitateWrite(thr, /*pc=*/2, thr_state_end,
+                            tls_end - thr_state_end);
+  }
+}
+#endif
+
+#if !SANITIZER_GO
 // Note: this function runs with async signals enabled,
 // so it must not touch any tsan state.
 int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
diff --git a/libsanitizer/tsan/tsan_platform_posix.cc b/libsanitizer/tsan/tsan_platform_posix.cc
index 0a3b61a08dc..6e62575f1ec 100644
--- a/libsanitizer/tsan/tsan_platform_posix.cc
+++ b/libsanitizer/tsan/tsan_platform_posix.cc
@@ -44,6 +44,9 @@ void InitializeShadowMemory() {
 #elif defined(__mips64)
   const uptr kMadviseRangeBeg  = 0xff00000000ull;
   const uptr kMadviseRangeSize = 0x0100000000ull;
+#elif defined(__aarch64__) && defined(__APPLE__)
+  uptr kMadviseRangeBeg = LoAppMemBeg();
+  uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
 #elif defined(__aarch64__)
   uptr kMadviseRangeBeg = 0;
   uptr kMadviseRangeSize = 0;
@@ -113,21 +116,24 @@ static void ProtectRange(uptr beg, uptr end) {
 void CheckAndProtect() {
   // Ensure that the binary is indeed compiled with -pie.
   MemoryMappingLayout proc_maps(true);
-  uptr p, end, prot;
-  while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) {
-    if (IsAppMem(p))
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (IsAppMem(segment.start)) continue;
+    if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
+    if (segment.protection == 0)  // Zero page or mprotected.
       continue;
-    if (p >= HeapMemEnd() &&
-        p < HeapEnd())
-      continue;
-    if (prot == 0)  // Zero page or mprotected.
-      continue;
-    if (p >= VdsoBeg())  // vdso
+    if (segment.start >= VdsoBeg())  // vdso
       break;
-    Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end);
+    Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
+           segment.start, segment.end);
     Die();
   }
 
+#if defined(__aarch64__) && defined(__APPLE__)
+  ProtectRange(HeapMemEnd(), ShadowBeg());
+  ProtectRange(ShadowEnd(), MetaShadowBeg());
+  ProtectRange(MetaShadowEnd(), TraceMemBeg());
+#else
   ProtectRange(LoAppMemEnd(), ShadowBeg());
   ProtectRange(ShadowEnd(), MetaShadowBeg());
 #ifdef TSAN_MID_APP_RANGE
@@ -141,6 +147,7 @@ void CheckAndProtect() {
   ProtectRange(TraceMemBeg(), TraceMemEnd());
   ProtectRange(TraceMemEnd(), HeapMemBeg());
   ProtectRange(HeapEnd(), HiAppMemBeg());
+#endif
 }
 #endif
 
diff --git a/libsanitizer/tsan/tsan_platform_windows.cc b/libsanitizer/tsan/tsan_platform_windows.cc
index 54ec6a655df..76883caabf1 100644
--- a/libsanitizer/tsan/tsan_platform_windows.cc
+++ b/libsanitizer/tsan/tsan_platform_windows.cc
@@ -19,10 +19,6 @@
 
 namespace __tsan {
 
-uptr GetShadowMemoryConsumption() {
-  return 0;
-}
-
 void FlushShadowMemory() {
 }
 
diff --git a/libsanitizer/tsan/tsan_report.cc b/libsanitizer/tsan/tsan_report.cc
index ba3e34fbfa5..f8cb3e73ca4 100644
--- a/libsanitizer/tsan/tsan_report.cc
+++ b/libsanitizer/tsan/tsan_report.cc
@@ -11,6 +11,7 @@
 #include "tsan_report.h"
 #include "tsan_platform.h"
 #include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_file.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_report_decorator.h"
 #include "sanitizer_common/sanitizer_stacktrace_printer.h"
@@ -36,22 +37,16 @@ ReportLocation *ReportLocation::New(ReportLocationType type) {
 class Decorator: public __sanitizer::SanitizerCommonDecorator {
  public:
   Decorator() : SanitizerCommonDecorator() { }
-  const char *Warning()    { return Red(); }
-  const char *EndWarning() { return Default(); }
   const char *Access()     { return Blue(); }
-  const char *EndAccess()  { return Default(); }
   const char *ThreadDescription()    { return Cyan(); }
-  const char *EndThreadDescription() { return Default(); }
   const char *Location()   { return Green(); }
-  const char *EndLocation() { return Default(); }
   const char *Sleep()   { return Yellow(); }
-  const char *EndSleep() { return Default(); }
   const char *Mutex()   { return Magenta(); }
-  const char *EndMutex() { return Default(); }
 };
 
 ReportDesc::ReportDesc()
-    : stacks(MBlockReportStack)
+    : tag(kExternalTagNone)
+    , stacks(MBlockReportStack)
     , mops(MBlockReportMop)
     , locs(MBlockReportLoc)
     , mutexes(MBlockReportMutex)
@@ -79,7 +74,7 @@ const char *thread_name(char *buf, int tid) {
   return buf;
 }
 
-static const char *ReportTypeString(ReportType typ) {
+static const char *ReportTypeString(ReportType typ, uptr tag) {
   if (typ == ReportTypeRace)
     return "data race";
   if (typ == ReportTypeVptrRace)
@@ -88,6 +83,10 @@ static const char *ReportTypeString(ReportType typ) {
     return "heap-use-after-free";
   if (typ == ReportTypeVptrUseAfterFree)
     return "heap-use-after-free (virtual call vs free)";
+  if (typ == ReportTypeExternalRace) {
+    const char *str = GetReportHeaderFromTag(tag);
+    return str ? str : "race on external object";
+  }
   if (typ == ReportTypeThreadLeak)
     return "thread leak";
   if (typ == ReportTypeMutexDestroyLocked)
@@ -150,17 +149,30 @@ static const char *MopDesc(bool first, bool write, bool atomic) {
                 : (write ? "Previous write" : "Previous read"));
 }
 
+static const char *ExternalMopDesc(bool first, bool write) {
+  return first ? (write ? "Modifying" : "Read-only")
+               : (write ? "Previous modifying" : "Previous read-only");
+}
+
 static void PrintMop(const ReportMop *mop, bool first) {
   Decorator d;
   char thrbuf[kThreadBufSize];
   Printf("%s", d.Access());
-  Printf("  %s of size %d at %p by %s",
-      MopDesc(first, mop->write, mop->atomic),
-      mop->size, (void*)mop->addr,
-      thread_name(thrbuf, mop->tid));
+  if (mop->external_tag == kExternalTagNone) {
+    Printf("  %s of size %d at %p by %s",
+           MopDesc(first, mop->write, mop->atomic), mop->size,
+           (void *)mop->addr, thread_name(thrbuf, mop->tid));
+  } else {
+    const char *object_type = GetObjectTypeFromTag(mop->external_tag);
+    if (object_type == nullptr)
+        object_type = "external object";
+    Printf("  %s access of %s at %p by %s",
+           ExternalMopDesc(first, mop->write), object_type,
+           (void *)mop->addr, thread_name(thrbuf, mop->tid));
+  }
   PrintMutexSet(mop->mset);
   Printf(":\n");
-  Printf("%s", d.EndAccess());
+  Printf("%s", d.Default());
   PrintStack(mop->stack);
 }
 
@@ -181,9 +193,16 @@ static void PrintLocation(const ReportLocation *loc) {
              global.module_offset);
   } else if (loc->type == ReportLocationHeap) {
     char thrbuf[kThreadBufSize];
-    Printf("  Location is heap block of size %zu at %p allocated by %s:\n",
-           loc->heap_chunk_size, loc->heap_chunk_start,
-           thread_name(thrbuf, loc->tid));
+    const char *object_type = GetObjectTypeFromTag(loc->external_tag);
+    if (!object_type) {
+      Printf("  Location is heap block of size %zu at %p allocated by %s:\n",
+             loc->heap_chunk_size, loc->heap_chunk_start,
+             thread_name(thrbuf, loc->tid));
+    } else {
+      Printf("  Location is %s of size %zu at %p allocated by %s:\n",
+             object_type, loc->heap_chunk_size, loc->heap_chunk_start,
+             thread_name(thrbuf, loc->tid));
+    }
     print_stack = true;
   } else if (loc->type == ReportLocationStack) {
     Printf("  Location is stack of %s.\n\n", thread_name(thrbuf, loc->tid));
@@ -194,20 +213,20 @@ static void PrintLocation(const ReportLocation *loc) {
         loc->fd, thread_name(thrbuf, loc->tid));
     print_stack = true;
   }
-  Printf("%s", d.EndLocation());
+  Printf("%s", d.Default());
   if (print_stack)
     PrintStack(loc->stack);
 }
 
 static void PrintMutexShort(const ReportMutex *rm, const char *after) {
   Decorator d;
-  Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.EndMutex(), after);
+  Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after);
 }
 
 static void PrintMutexShortWithAddress(const ReportMutex *rm,
                                        const char *after) {
   Decorator d;
-  Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.EndMutex(), after);
+  Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after);
 }
 
 static void PrintMutex(const ReportMutex *rm) {
@@ -215,11 +234,11 @@ static void PrintMutex(const ReportMutex *rm) {
   if (rm->destroyed) {
     Printf("%s", d.Mutex());
     Printf("  Mutex M%llu is already destroyed.\n\n", rm->id);
-    Printf("%s", d.EndMutex());
+    Printf("%s", d.Default());
   } else {
     Printf("%s", d.Mutex());
     Printf("  Mutex M%llu (%p) created at:\n", rm->id, rm->addr);
-    Printf("%s", d.EndMutex());
+    Printf("%s", d.Default());
     PrintStack(rm->stack);
   }
 }
@@ -233,13 +252,19 @@ static void PrintThread(const ReportThread *rt) {
   if (rt->name && rt->name[0] != '\0')
     Printf(" '%s'", rt->name);
   char thrbuf[kThreadBufSize];
-  Printf(" (tid=%zu, %s) created by %s",
-    rt->os_id, rt->running ? "running" : "finished",
-    thread_name(thrbuf, rt->parent_tid));
+  const char *thread_status = rt->running ? "running" : "finished";
+  if (rt->workerthread) {
+    Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status);
+    Printf("\n");
+    Printf("%s", d.Default());
+    return;
+  }
+  Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status,
+         thread_name(thrbuf, rt->parent_tid));
   if (rt->stack)
     Printf(" at:");
   Printf("\n");
-  Printf("%s", d.EndThreadDescription());
+  Printf("%s", d.Default());
   PrintStack(rt->stack);
 }
 
@@ -247,7 +272,7 @@ static void PrintSleep(const ReportStack *s) {
   Decorator d;
   Printf("%s", d.Sleep());
   Printf("  As if synchronized via sleep:\n");
-  Printf("%s", d.EndSleep());
+  Printf("%s", d.Default());
   PrintStack(s);
 }
 
@@ -287,11 +312,11 @@ static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) {
 void PrintReport(const ReportDesc *rep) {
   Decorator d;
   Printf("==================\n");
-  const char *rep_typ_str = ReportTypeString(rep->typ);
+  const char *rep_typ_str = ReportTypeString(rep->typ, rep->tag);
   Printf("%s", d.Warning());
   Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str,
          (int)internal_getpid());
-  Printf("%s", d.EndWarning());
+  Printf("%s", d.Default());
 
   if (rep->typ == ReportTypeDeadlock) {
     char thrbuf[kThreadBufSize];
@@ -309,7 +334,7 @@ void PrintReport(const ReportDesc *rep) {
       PrintMutexShort(rep->mutexes[i], " in ");
       Printf("%s", d.ThreadDescription());
       Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i]));
-      Printf("%s", d.EndThreadDescription());
+      Printf("%s", d.Default());
       if (flags()->second_deadlock_stack) {
         PrintStack(rep->stacks[2*i]);
         Printf("  Mutex ");
@@ -356,6 +381,8 @@ void PrintReport(const ReportDesc *rep) {
       ReportErrorSummary(rep_typ_str, frame->info);
   }
 
+  if (common_flags()->print_module_map == 2) PrintModuleMap();
+
   Printf("==================\n");
 }
 
diff --git a/libsanitizer/tsan/tsan_report.h b/libsanitizer/tsan/tsan_report.h
index e51ed4f50f8..6eb043fc866 100644
--- a/libsanitizer/tsan/tsan_report.h
+++ b/libsanitizer/tsan/tsan_report.h
@@ -22,6 +22,7 @@ enum ReportType {
   ReportTypeVptrRace,
   ReportTypeUseAfterFree,
   ReportTypeVptrUseAfterFree,
+  ReportTypeExternalRace,
   ReportTypeThreadLeak,
   ReportTypeMutexDestroyLocked,
   ReportTypeMutexDoubleLock,
@@ -54,6 +55,7 @@ struct ReportMop {
   int size;
   bool write;
   bool atomic;
+  uptr external_tag;
   Vector<ReportMopMutex> mset;
   ReportStack *stack;
 
@@ -73,6 +75,7 @@ struct ReportLocation {
   DataInfo global;
   uptr heap_chunk_start;
   uptr heap_chunk_size;
+  uptr external_tag;
   int tid;
   int fd;
   bool suppressable;
@@ -85,10 +88,11 @@ struct ReportLocation {
 
 struct ReportThread {
   int id;
-  uptr os_id;
+  tid_t os_id;
   bool running;
+  bool workerthread;
   char *name;
-  int parent_tid;
+  u32 parent_tid;
   ReportStack *stack;
 };
 
@@ -102,6 +106,7 @@ struct ReportMutex {
 class ReportDesc {
  public:
   ReportType typ;
+  uptr tag;
   Vector<ReportStack*> stacks;
   Vector<ReportMop*> mops;
   Vector<ReportLocation*> locs;
diff --git a/libsanitizer/tsan/tsan_rtl.cc b/libsanitizer/tsan/tsan_rtl.cc
index 5be28ce5502..4a1f50061a6 100644
--- a/libsanitizer/tsan/tsan_rtl.cc
+++ b/libsanitizer/tsan/tsan_rtl.cc
@@ -12,6 +12,7 @@
 
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_file.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
@@ -43,7 +44,7 @@ extern "C" void __tsan_resume() {
 namespace __tsan {
 
 #if !SANITIZER_GO && !SANITIZER_MAC
-  __attribute__((tls_model("initial-exec")))
+__attribute__((tls_model("initial-exec")))
 THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
 #endif
 static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
@@ -102,7 +103,8 @@ Context::Context()
   , racy_stacks(MBlockRacyStacks)
   , racy_addresses(MBlockRacyAddresses)
   , fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
-  , fired_suppressions(8) {
+  , fired_suppressions(8)
+  , clock_alloc("clock allocator") {
 }
 
 // The objects are allocated in TLS, so one may rely on zero-initialization.
@@ -232,9 +234,7 @@ static void StopBackgroundThread() {
 #endif
 
 void DontNeedShadowFor(uptr addr, uptr size) {
-  uptr shadow_beg = MemToShadow(addr);
-  uptr shadow_end = MemToShadow(addr + size);
-  ReleaseMemoryToOS(shadow_beg, shadow_end - shadow_beg);
+  ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
 }
 
 void MapShadow(uptr addr, uptr size) {
@@ -381,7 +381,7 @@ void Initialize(ThreadState *thr) {
   // Initialize thread 0.
   int tid = ThreadCreate(thr, 0, 0, true);
   CHECK_EQ(tid, 0);
-  ThreadStart(thr, tid, internal_getpid());
+  ThreadStart(thr, tid, GetTid(), /*workerthread*/ false);
 #if TSAN_CONTAINS_UBSAN
   __ubsan::InitAsPlugin();
 #endif
@@ -404,6 +404,8 @@ void Initialize(ThreadState *thr) {
 int Finalize(ThreadState *thr) {
   bool failed = false;
 
+  if (common_flags()->print_module_map == 1) PrintModuleMap();
+
   if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
     SleepForMillis(flags()->atexit_sleep_ms);
 
@@ -864,9 +866,8 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
   // Don't want to touch lots of shadow memory.
   // If a program maps 10MB stack, there is no need reset the whole range.
   size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
-  // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
-  // so we do it only for C/C++.
-  if (SANITIZER_GO || size < common_flags()->clear_shadow_mmap_threshold) {
+  // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
+  if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
     u64 *p = (u64*)MemToShadow(addr);
     CHECK(IsShadowMem((uptr)p));
     CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
@@ -978,21 +979,21 @@ void FuncExit(ThreadState *thr) {
   thr->shadow_stack_pos--;
 }
 
-void ThreadIgnoreBegin(ThreadState *thr, uptr pc) {
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack) {
   DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
   thr->ignore_reads_and_writes++;
   CHECK_GT(thr->ignore_reads_and_writes, 0);
   thr->fast_state.SetIgnoreBit();
 #if !SANITIZER_GO
-  if (!ctx->after_multithreaded_fork)
+  if (save_stack && !ctx->after_multithreaded_fork)
     thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
 #endif
 }
 
 void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
+  CHECK_GT(thr->ignore_reads_and_writes, 0);
   thr->ignore_reads_and_writes--;
-  CHECK_GE(thr->ignore_reads_and_writes, 0);
   if (thr->ignore_reads_and_writes == 0) {
     thr->fast_state.ClearIgnoreBit();
 #if !SANITIZER_GO
@@ -1001,20 +1002,28 @@ void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
   }
 }
 
-void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
+#if !SANITIZER_GO
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+uptr __tsan_testonly_shadow_stack_current_size() {
+  ThreadState *thr = cur_thread();
+  return thr->shadow_stack_pos - thr->shadow_stack;
+}
+#endif
+
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack) {
   DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
   thr->ignore_sync++;
   CHECK_GT(thr->ignore_sync, 0);
 #if !SANITIZER_GO
-  if (!ctx->after_multithreaded_fork)
+  if (save_stack && !ctx->after_multithreaded_fork)
     thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
 #endif
 }
 
 void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
+  CHECK_GT(thr->ignore_sync, 0);
   thr->ignore_sync--;
-  CHECK_GE(thr->ignore_sync, 0);
 #if !SANITIZER_GO
   if (thr->ignore_sync == 0)
     thr->sync_ignore_set.Reset();
diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h
index 522c76002a1..7dd9779e42b 100644
--- a/libsanitizer/tsan/tsan_rtl.h
+++ b/libsanitizer/tsan/tsan_rtl.h
@@ -53,16 +53,22 @@ namespace __tsan {
 #if !SANITIZER_GO
 struct MapUnmapCallback;
 #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
-static const uptr kAllocatorSpace = 0;
-static const uptr kAllocatorSize = SANITIZER_MMAP_RANGE_SIZE;
 static const uptr kAllocatorRegionSizeLog = 20;
 static const uptr kAllocatorNumRegions =
-    kAllocatorSize >> kAllocatorRegionSizeLog;
+    SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog;
 typedef TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12,
     MapUnmapCallback> ByteMap;
-typedef SizeClassAllocator32<kAllocatorSpace, kAllocatorSize, 0,
-    CompactSizeClassMap, kAllocatorRegionSizeLog, ByteMap,
-    MapUnmapCallback> PrimaryAllocator;
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = 0;
+  typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = kAllocatorRegionSizeLog;
+  typedef __tsan::ByteMap ByteMap;
+  typedef __tsan::MapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
 #else
 struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
   static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
@@ -379,6 +385,7 @@ struct ThreadState {
   // for better performance.
   int ignore_reads_and_writes;
   int ignore_sync;
+  int suppress_reports;
   // Go does not support ignores.
 #if !SANITIZER_GO
   IgnoreSet mop_ignore_set;
@@ -543,6 +550,10 @@ struct Context {
 
 extern Context *ctx;  // The one and the only global runtime context.
 
+ALWAYS_INLINE Flags *flags() {
+  return &ctx->flags;
+}
+
 struct ScopedIgnoreInterceptors {
   ScopedIgnoreInterceptors() {
 #if !SANITIZER_GO
@@ -557,12 +568,16 @@ struct ScopedIgnoreInterceptors {
   }
 };
 
+const char *GetObjectTypeFromTag(uptr tag);
+const char *GetReportHeaderFromTag(uptr tag);
+uptr TagFromShadowStackFrame(uptr pc);
+
 class ScopedReport {
  public:
-  explicit ScopedReport(ReportType typ);
+  explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
   ~ScopedReport();
 
-  void AddMemoryAccess(uptr addr, Shadow s, StackTrace stack,
+  void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
                        const MutexSet *mset);
   void AddStack(StackTrace stack, bool suppressable = false);
   void AddThread(const ThreadContext *tctx, bool suppressable = false);
@@ -588,11 +603,28 @@ class ScopedReport {
   void operator = (const ScopedReport&);
 };
 
+ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
 void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
-                  MutexSet *mset);
+                  MutexSet *mset, uptr *tag = nullptr);
+
+// The stack could look like:
+//   <start> | <main> | <foo> | tag | <bar>
+// This will extract the tag and keep:
+//   <start> | <main> | <foo> | <bar>
+template<typename StackTraceTy>
+void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
+  if (stack->size < 2) return;
+  uptr possible_tag_pc = stack->trace[stack->size - 2];
+  uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
+  if (possible_tag == kExternalTagNone) return;
+  stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
+  stack->size -= 1;
+  if (tag) *tag = possible_tag;
+}
 
 template<typename StackTraceTy>
-void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) {
+void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
+                        uptr *tag = nullptr) {
   uptr size = thr->shadow_stack_pos - thr->shadow_stack;
   uptr start = 0;
   if (size + !!toppc > kStackTraceMax) {
@@ -600,6 +632,7 @@ void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) {
     size = kStackTraceMax - !!toppc;
   }
   stack->Init(&thr->shadow_stack[start], size, toppc);
+  ExtractTagFromStack(stack, tag);
 }
 
 
@@ -701,16 +734,16 @@ void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
 
-void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);
 void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
-void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);
 void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
 
 void FuncEntry(ThreadState *thr, uptr pc);
 void FuncExit(ThreadState *thr);
 
 int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
-void ThreadStart(ThreadState *thr, int tid, uptr os_id);
+void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread);
 void ThreadFinish(ThreadState *thr);
 int ThreadTid(ThreadState *thr, uptr pc, uptr uid);
 void ThreadJoin(ThreadState *thr, uptr pc, int tid);
@@ -725,13 +758,16 @@ void ProcDestroy(Processor *proc);
 void ProcWire(Processor *proc, ThreadState *thr);
 void ProcUnwire(Processor *proc, ThreadState *thr);
 
-void MutexCreate(ThreadState *thr, uptr pc, uptr addr,
-                 bool rw, bool recursive, bool linker_init);
-void MutexDestroy(ThreadState *thr, uptr pc, uptr addr);
-void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1,
-               bool try_lock = false);
-int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all = false);
-void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool try_lock = false);
+// Note: the parameter is called flagz, because flags is already taken
+// by the global function that returns flags.
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
+    int rec = 1);
+int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
 void MutexRepair(ThreadState *thr, uptr pc, uptr addr);  // call on EOWNERDEAD
@@ -787,7 +823,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
     return;
   DCHECK_GE((int)typ, 0);
   DCHECK_LE((int)typ, 7);
-  DCHECK_EQ(GetLsb(addr, 61), addr);
+  DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
   StatInc(thr, StatEvents);
   u64 pos = fs.GetTracePos();
   if (UNLIKELY((pos % kTracePartSize) == 0)) {
@@ -799,7 +835,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
   }
   Event *trace = (Event*)GetThreadTrace(fs.tid());
   Event *evp = &trace[pos];
-  Event ev = (u64)addr | ((u64)typ << 61);
+  Event ev = (u64)addr | ((u64)typ << kEventPCBits);
   *evp = ev;
 }
 
diff --git a/libsanitizer/tsan/tsan_rtl_aarch64.S b/libsanitizer/tsan/tsan_rtl_aarch64.S
index ef06f0444ae..61171d635c1 100644
--- a/libsanitizer/tsan/tsan_rtl_aarch64.S
+++ b/libsanitizer/tsan/tsan_rtl_aarch64.S
@@ -1,13 +1,46 @@
+// The content of this file is AArch64-only:
+#if defined(__aarch64__)
+
 #include "sanitizer_common/sanitizer_asm.h"
 
+#if !defined(__APPLE__)
 .section .bss
 .type	__tsan_pointer_chk_guard, %object
-.size	__tsan_pointer_chk_guard, 8
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard))
 __tsan_pointer_chk_guard:
 .zero	8
+#endif
+
+#if defined(__APPLE__)
+.align  2
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _setjmp$non_lazy_ptr
+_setjmp$non_lazy_ptr:
+.indirect_symbol _setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long __setjmp$non_lazy_ptr
+__setjmp$non_lazy_ptr:
+.indirect_symbol __setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _sigsetjmp$non_lazy_ptr
+_sigsetjmp$non_lazy_ptr:
+.indirect_symbol _sigsetjmp
+.long 0
+#endif
 
+#if !defined(__APPLE__)
 .section .text
+#else
+.section __TEXT,__text
+.align 3
+#endif
 
+#if !defined(__APPLE__)
 // GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
 // functions) by XORing them with a random guard pointer.  For AArch64 it is a
 // global variable rather than a TCB one (as for x86_64/powerpc) and althought
@@ -16,9 +49,9 @@ __tsan_pointer_chk_guard:
 // not stable). So InitializeGuardPtr obtains the pointer guard value by
 // issuing a setjmp and checking the resulting pointers values against the
 // original ones.
-.hidden _Z18InitializeGuardPtrv
+ASM_HIDDEN(_Z18InitializeGuardPtrv)
 .global _Z18InitializeGuardPtrv
-.type _Z18InitializeGuardPtrv, @function
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
 _Z18InitializeGuardPtrv:
   CFI_STARTPROC
   // Allocates a jmp_buf for the setjmp call.
@@ -55,12 +88,14 @@ _Z18InitializeGuardPtrv:
   CFI_DEF_CFA (31, 0)
   ret
   CFI_ENDPROC
-.size _Z18InitializeGuardPtrv, .-_Z18InitializeGuardPtrv
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
+#endif
 
-.hidden __tsan_setjmp
+ASM_HIDDEN(__tsan_setjmp)
 .comm _ZN14__interception11real_setjmpE,8,8
-.type setjmp, @function
-setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -78,14 +113,19 @@ setjmp:
   CFI_OFFSET (19, -16)
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#else
+  add     x0, x29, 32
+  mov     x1, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   // restore env parameter
   mov     x0, x19
@@ -96,18 +136,24 @@ setjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc setjmp
+#if !defined(__APPLE__)
   adrp    x1, :got:_ZN14__interception11real_setjmpE
   ldr     x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
   ldr     x1, [x1]
+#else
+  adrp    x1, _setjmp$non_lazy_ptr@page
+  add     x1, x1, _setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
   br      x1
 
   CFI_ENDPROC
-.size setjmp, .-setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
 
 .comm _ZN14__interception12real__setjmpE,8,8
-.globl _setjmp
-.type _setjmp, @function
-_setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -125,14 +171,19 @@ _setjmp:
   CFI_OFFSET (19, -16)
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#else
+  add     x0, x29, 32
+  mov     x1, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   // Restore jmp_buf parameter
   mov     x0, x19
@@ -143,18 +194,24 @@ _setjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc setjmp
+#if !defined(__APPLE__)
   adrp    x1, :got:_ZN14__interception12real__setjmpE
   ldr     x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
   ldr     x1, [x1]
+#else
+  adrp    x1, __setjmp$non_lazy_ptr@page
+  add     x1, x1, __setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
   br      x1
 
   CFI_ENDPROC
-.size _setjmp, .-_setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
 
 .comm _ZN14__interception14real_sigsetjmpE,8,8
-.globl sigsetjmp
-.type sigsetjmp, @function
-sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -174,14 +231,19 @@ sigsetjmp:
   mov     w20, w1
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#else
+  add     x0, x29, 32
+  mov     x1, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   // restore env parameter
   mov     w1, w20
@@ -195,17 +257,24 @@ sigsetjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc sigsetjmp
+#if !defined(__APPLE__)
   adrp    x2, :got:_ZN14__interception14real_sigsetjmpE
   ldr     x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
   ldr     x2, [x2]
+#else
+  adrp    x2, _sigsetjmp$non_lazy_ptr@page
+  add     x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
+  ldr     x2, [x2]
+#endif
   br      x2
   CFI_ENDPROC
-.size sigsetjmp, .-sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
 
+#if !defined(__APPLE__)
 .comm _ZN14__interception16real___sigsetjmpE,8,8
-.globl __sigsetjmp
-.type __sigsetjmp, @function
-__sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -225,14 +294,16 @@ __sigsetjmp:
   mov     w20, w1
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   mov     w1, w20
   mov     x0, x19
@@ -245,14 +316,22 @@ __sigsetjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc __sigsetjmp
+#if !defined(__APPLE__)
   adrp    x2, :got:_ZN14__interception16real___sigsetjmpE
   ldr     x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
   ldr     x2, [x2]
+#else
+  adrp    x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page
+  add     x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff
+#endif
   br      x2
   CFI_ENDPROC
-.size __sigsetjmp, .-__sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif
 
 #if defined(__linux__)
 /* We do not need executable stack.  */
 .section        .note.GNU-stack,"",@progbits
 #endif
+
+#endif
diff --git a/libsanitizer/tsan/tsan_rtl_amd64.S b/libsanitizer/tsan/tsan_rtl_amd64.S
index caa832375e5..98947fd2a1b 100644
--- a/libsanitizer/tsan/tsan_rtl_amd64.S
+++ b/libsanitizer/tsan/tsan_rtl_amd64.S
@@ -1,4 +1,8 @@
+// The content of this file is x86_64-only:
+#if defined(__x86_64__)
+
 #include "sanitizer_common/sanitizer_asm.h"
+
 #if !defined(__APPLE__)
 .section .text
 #else
@@ -357,3 +361,5 @@ ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
 /* We do not need executable stack.  */
 .section        .note.GNU-stack,"",@progbits
 #endif
+
+#endif
diff --git a/libsanitizer/tsan/tsan_rtl_mutex.cc b/libsanitizer/tsan/tsan_rtl_mutex.cc
index e575bbfb7e9..d10a87374ba 100644
--- a/libsanitizer/tsan/tsan_rtl_mutex.cc
+++ b/libsanitizer/tsan/tsan_rtl_mutex.cc
@@ -60,32 +60,29 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
   OutputReport(thr, rep);
 }
 
-void MutexCreate(ThreadState *thr, uptr pc, uptr addr,
-                 bool rw, bool recursive, bool linker_init) {
-  DPrintf("#%d: MutexCreate %zx\n", thr->tid, addr);
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
   StatInc(thr, StatMutexCreate);
-  if (!linker_init && IsAppMem(addr)) {
+  if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
     CHECK(!thr->is_freeing);
     thr->is_freeing = true;
     MemoryWrite(thr, pc, addr, kSizeLog1);
     thr->is_freeing = false;
   }
   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
-  s->is_rw = rw;
-  s->is_recursive = recursive;
-  s->is_linker_init = linker_init;
+  s->SetFlags(flagz & MutexCreationFlagMask);
   if (!SANITIZER_GO && s->creation_stack_id == 0)
     s->creation_stack_id = CurrentStackId(thr, pc);
   s->mtx.Unlock();
 }
 
-void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) {
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
   StatInc(thr, StatMutexDestroy);
   SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
   if (s == 0)
     return;
-  if (s->is_linker_init) {
+  if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit)) {
     // Destroy is no-op for linker-initialized mutexes.
     s->mtx.Unlock();
     return;
@@ -98,8 +95,8 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) {
   bool unlock_locked = false;
   if (flags()->report_destroy_locked
       && s->owner_tid != SyncVar::kInvalidTid
-      && !s->is_broken) {
-    s->is_broken = true;
+      && !s->IsFlagSet(MutexFlagBroken)) {
+    s->SetFlags(MutexFlagBroken);
     unlock_locked = true;
   }
   u64 mid = s->GetId();
@@ -139,12 +136,33 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) {
   // s will be destroyed and freed in MetaMap::FreeBlock.
 }
 
-void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec, bool try_lock) {
-  DPrintf("#%d: MutexLock %zx rec=%d\n", thr->tid, addr, rec);
-  CHECK_GT(rec, 0);
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
+    SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+    s->UpdateFlags(flagz);
+    if (s->owner_tid != thr->tid) {
+      Callback cb(thr, pc);
+      ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
+      s->mtx.ReadUnlock();
+      ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+    } else {
+      s->mtx.ReadUnlock();
+    }
+  }
+}
+
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
+  DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
+      thr->tid, addr, flagz, rec);
+  if (flagz & MutexFlagRecursiveLock)
+    CHECK_GT(rec, 0);
+  else
+    rec = 1;
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  s->UpdateFlags(flagz);
   thr->fast_state.IncrementEpoch();
   TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
   bool report_double_lock = false;
@@ -154,38 +172,43 @@ void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec, bool try_lock) {
     s->last_lock = thr->fast_state.raw();
   } else if (s->owner_tid == thr->tid) {
     CHECK_GT(s->recursion, 0);
-  } else if (flags()->report_mutex_bugs && !s->is_broken) {
-    s->is_broken = true;
+  } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+    s->SetFlags(MutexFlagBroken);
     report_double_lock = true;
   }
-  if (s->recursion == 0) {
+  const bool first = s->recursion == 0;
+  s->recursion += rec;
+  if (first) {
     StatInc(thr, StatMutexLock);
     AcquireImpl(thr, pc, &s->clock);
     AcquireImpl(thr, pc, &s->read_clock);
-  } else if (!s->is_recursive) {
+  } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
     StatInc(thr, StatMutexRecLock);
   }
-  s->recursion += rec;
   thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
-  if (common_flags()->detect_deadlocks && (s->recursion - rec) == 0) {
+  bool pre_lock = false;
+  if (first && common_flags()->detect_deadlocks) {
+    pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+        !(flagz & MutexFlagTryLock);
     Callback cb(thr, pc);
-    if (!try_lock)
+    if (pre_lock)
       ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
-    ctx->dd->MutexAfterLock(&cb, &s->dd, true, try_lock);
+    ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
   }
   u64 mid = s->GetId();
   s->mtx.Unlock();
   // Can't touch s after this point.
+  s = 0;
   if (report_double_lock)
     ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid);
-  if (common_flags()->detect_deadlocks) {
+  if (first && pre_lock && common_flags()->detect_deadlocks) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
   }
 }
 
-int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) {
-  DPrintf("#%d: MutexUnlock %zx all=%d\n", thr->tid, addr, all);
+int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
@@ -194,12 +217,12 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) {
   int rec = 0;
   bool report_bad_unlock = false;
   if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
-    if (flags()->report_mutex_bugs && !s->is_broken) {
-      s->is_broken = true;
+    if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
       report_bad_unlock = true;
     }
   } else {
-    rec = all ? s->recursion : 1;
+    rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
     s->recursion -= rec;
     if (s->recursion == 0) {
       StatInc(thr, StatMutexUnlock);
@@ -227,36 +250,53 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) {
   return rec;
 }
 
-void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool trylock) {
-  DPrintf("#%d: MutexReadLock %zx\n", thr->tid, addr);
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
+    SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+    s->UpdateFlags(flagz);
+    Callback cb(thr, pc);
+    ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
+    s->mtx.ReadUnlock();
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
   StatInc(thr, StatMutexReadLock);
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+  s->UpdateFlags(flagz);
   thr->fast_state.IncrementEpoch();
   TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
   bool report_bad_lock = false;
   if (s->owner_tid != SyncVar::kInvalidTid) {
-    if (flags()->report_mutex_bugs && !s->is_broken) {
-      s->is_broken = true;
+    if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
       report_bad_lock = true;
     }
   }
   AcquireImpl(thr, pc, &s->clock);
   s->last_lock = thr->fast_state.raw();
   thr->mset.Add(s->GetId(), false, thr->fast_state.epoch());
-  if (common_flags()->detect_deadlocks && s->recursion == 0) {
+  bool pre_lock = false;
+  if (common_flags()->detect_deadlocks) {
+    pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+        !(flagz & MutexFlagTryLock);
     Callback cb(thr, pc);
-    if (!trylock)
+    if (pre_lock)
       ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
-    ctx->dd->MutexAfterLock(&cb, &s->dd, false, trylock);
+    ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
   }
   u64 mid = s->GetId();
   s->mtx.ReadUnlock();
   // Can't touch s after this point.
+  s = 0;
   if (report_bad_lock)
     ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid);
-  if (common_flags()->detect_deadlocks) {
+  if (pre_lock  && common_flags()->detect_deadlocks) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
   }
@@ -272,8 +312,8 @@ void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
   TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
   bool report_bad_unlock = false;
   if (s->owner_tid != SyncVar::kInvalidTid) {
-    if (flags()->report_mutex_bugs && !s->is_broken) {
-      s->is_broken = true;
+    if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
       report_bad_unlock = true;
     }
   }
@@ -321,8 +361,8 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
     } else {
       StatInc(thr, StatMutexRecUnlock);
     }
-  } else if (!s->is_broken) {
-    s->is_broken = true;
+  } else if (!s->IsFlagSet(MutexFlagBroken)) {
+    s->SetFlags(MutexFlagBroken);
     report_bad_unlock = true;
   }
   thr->mset.Del(s->GetId(), write);
@@ -371,10 +411,10 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) {
 static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
   ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
   ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
   if (tctx->status == ThreadStatusRunning)
-    thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch());
-  else
-    thr->clock.set(tctx->tid, tctx->epoch1);
+    epoch = tctx->thr->fast_state.epoch();
+  thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
 }
 
 void AcquireGlobal(ThreadState *thr, uptr pc) {
@@ -414,10 +454,10 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
 static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
   ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
   ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
   if (tctx->status == ThreadStatusRunning)
-    thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch());
-  else
-    thr->last_sleep_clock.set(tctx->tid, tctx->epoch1);
+    epoch = tctx->thr->fast_state.epoch();
+  thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
 }
 
 void AfterSleep(ThreadState *thr, uptr pc) {
diff --git a/libsanitizer/tsan/tsan_rtl_report.cc b/libsanitizer/tsan/tsan_rtl_report.cc
index 8f2882485e6..a961139b61d 100644
--- a/libsanitizer/tsan/tsan_rtl_report.cc
+++ b/libsanitizer/tsan/tsan_rtl_report.cc
@@ -141,11 +141,12 @@ static ReportStack *SymbolizeStack(StackTrace trace) {
   return stack;
 }
 
-ScopedReport::ScopedReport(ReportType typ) {
+ScopedReport::ScopedReport(ReportType typ, uptr tag) {
   ctx->thread_registry->CheckLocked();
   void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));
   rep_ = new(mem) ReportDesc;
   rep_->typ = typ;
+  rep_->tag = tag;
   ctx->report_mtx.Lock();
   CommonSanitizerReportMutex.Lock();
 }
@@ -162,8 +163,8 @@ void ScopedReport::AddStack(StackTrace stack, bool suppressable) {
   (*rs)->suppressable = suppressable;
 }
 
-void ScopedReport::AddMemoryAccess(uptr addr, Shadow s, StackTrace stack,
-                                   const MutexSet *mset) {
+void ScopedReport::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
+                                   StackTrace stack, const MutexSet *mset) {
   void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop));
   ReportMop *mop = new(mem) ReportMop;
   rep_->mops.PushBack(mop);
@@ -173,6 +174,7 @@ void ScopedReport::AddMemoryAccess(uptr addr, Shadow s, StackTrace stack,
   mop->write = s.IsWrite();
   mop->atomic = s.IsAtomic();
   mop->stack = SymbolizeStack(stack);
+  mop->external_tag = external_tag;
   if (mop->stack)
     mop->stack->suppressable = true;
   for (uptr i = 0; i < mset->Size(); i++) {
@@ -200,6 +202,7 @@ void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) {
   rt->running = (tctx->status == ThreadStatusRunning);
   rt->name = internal_strdup(tctx->name);
   rt->parent_tid = tctx->parent_tid;
+  rt->workerthread = tctx->workerthread;
   rt->stack = 0;
   rt->stack = SymbolizeStackId(tctx->creation_stack_id);
   if (rt->stack)
@@ -309,7 +312,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
     return;
 #if !SANITIZER_GO
   int fd = -1;
-  int creat_tid = -1;
+  int creat_tid = kInvalidTid;
   u32 creat_stack = 0;
   if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
     ReportLocation *loc = ReportLocation::New(ReportLocationFD);
@@ -334,6 +337,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
     ReportLocation *loc = ReportLocation::New(ReportLocationHeap);
     loc->heap_chunk_start = (uptr)allocator()->GetBlockBegin((void *)addr);
     loc->heap_chunk_size = b->siz;
+    loc->external_tag = b->tag;
     loc->tid = tctx ? tctx->tid : b->tid;
     loc->stack = SymbolizeStackId(b->stk);
     rep_->locs.PushBack(loc);
@@ -372,7 +376,7 @@ const ReportDesc *ScopedReport::GetReport() const {
 }
 
 void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
-                  MutexSet *mset) {
+                  MutexSet *mset, uptr *tag) {
   // This function restores stack trace and mutex set for the thread/epoch.
   // It does so by getting stack trace and mutex set at the beginning of
   // trace part, and then replaying the trace till the given epoch.
@@ -400,8 +404,8 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
   Event *events = (Event*)GetThreadTrace(tid);
   for (uptr i = ebegin; i <= eend; i++) {
     Event ev = events[i];
-    EventType typ = (EventType)(ev >> 61);
-    uptr pc = (uptr)(ev & ((1ull << 61) - 1));
+    EventType typ = (EventType)(ev >> kEventPCBits);
+    uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
     DPrintf2("  %zu typ=%d pc=%zx\n", i, typ, pc);
     if (typ == EventTypeMop) {
       stack[pos] = pc;
@@ -431,6 +435,7 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
     return;
   pos++;
   stk->Init(&stack[0], pos);
+  ExtractTagFromStack(stk, tag);
 }
 
 static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2],
@@ -495,7 +500,7 @@ static void AddRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2],
 }
 
 bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
-  if (!flags()->report_bugs)
+  if (!flags()->report_bugs || thr->suppress_reports)
     return false;
   atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime());
   const ReportDesc *rep = srep.GetReport();
@@ -626,8 +631,29 @@ void ReportRace(ThreadState *thr) {
 
   const uptr kMop = 2;
   VarSizeStackTrace traces[kMop];
-  const uptr toppc = TraceTopPC(thr);
-  ObtainCurrentStack(thr, toppc, &traces[0]);
+  uptr tags[kMop] = {kExternalTagNone};
+  uptr toppc = TraceTopPC(thr);
+  if (toppc >> kEventPCBits) {
+    // This is a work-around for a known issue.
+    // The scenario where this happens is rather elaborate and requires
+    // an instrumented __sanitizer_report_error_summary callback and
+    // a __tsan_symbolize_external callback and a race during a range memory
+    // access larger than 8 bytes. MemoryAccessRange adds the current PC to
+    // the trace and starts processing memory accesses. A first memory access
+    // triggers a race, we report it and call the instrumented
+    // __sanitizer_report_error_summary, which adds more stuff to the trace
+    // since it is intrumented. Then a second memory access in MemoryAccessRange
+    // also triggers a race and we get here and call TraceTopPC to get the
+    // current PC, however now it contains some unrelated events from the
+    // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
+    // event. Later we subtract -1 from it (in GetPreviousInstructionPc)
+    // and the resulting PC has kExternalPCBit set, so we pass it to
+    // __tsan_symbolize_external. __tsan_symbolize_external is within its rights
+    // to crash since the PC is completely bogus.
+    // test/tsan/double_race.cc contains a test case for this.
+    toppc = 0;
+  }
+  ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
   if (IsFiredSuppression(ctx, typ, traces[0]))
     return;
 
@@ -637,18 +663,29 @@ void ReportRace(ThreadState *thr) {
   MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
 
   Shadow s2(thr->racy_state[1]);
-  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2);
+  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
   if (IsFiredSuppression(ctx, typ, traces[1]))
     return;
 
   if (HandleRacyStacks(thr, traces, addr_min, addr_max))
     return;
 
+  // If any of the accesses has a tag, treat this as an "external" race.
+  uptr tag = kExternalTagNone;
+  for (uptr i = 0; i < kMop; i++) {
+    if (tags[i] != kExternalTagNone) {
+      typ = ReportTypeExternalRace;
+      tag = tags[i];
+      break;
+    }
+  }
+
   ThreadRegistryLock l0(ctx->thread_registry);
-  ScopedReport rep(typ);
+  ScopedReport rep(typ, tag);
   for (uptr i = 0; i < kMop; i++) {
     Shadow s(thr->racy_state[i]);
-    rep.AddMemoryAccess(addr, s, traces[i], i == 0 ? &thr->mset : mset2);
+    rep.AddMemoryAccess(addr, tags[i], s, traces[i],
+                        i == 0 ? &thr->mset : mset2);
   }
 
   for (uptr i = 0; i < kMop; i++) {
diff --git a/libsanitizer/tsan/tsan_rtl_thread.cc b/libsanitizer/tsan/tsan_rtl_thread.cc
index 6c4b74e2a76..e81669d3b77 100644
--- a/libsanitizer/tsan/tsan_rtl_thread.cc
+++ b/libsanitizer/tsan/tsan_rtl_thread.cc
@@ -66,7 +66,8 @@ void ThreadContext::OnCreated(void *arg) {
 
 void ThreadContext::OnReset() {
   CHECK_EQ(sync.size(), 0);
-  ReleaseMemoryToOS(GetThreadTrace(tid), TraceSize() * sizeof(Event));
+  uptr trace_p = GetThreadTrace(tid);
+  ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event));
   //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace));
 }
 
@@ -139,6 +140,10 @@ void ThreadContext::OnFinished() {
 
   if (common_flags()->detect_deadlocks)
     ctx->dd->DestroyLogicalThread(thr->dd_lt);
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+#if !SANITIZER_GO
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+#endif
   thr->~ThreadState();
 #if TSAN_COLLECT_STATS
   StatAggregate(ctx->stat, thr->stat);
@@ -233,7 +238,7 @@ int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
   return tid;
 }
 
-void ThreadStart(ThreadState *thr, int tid, uptr os_id) {
+void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) {
   uptr stk_addr = 0;
   uptr stk_size = 0;
   uptr tls_addr = 0;
@@ -245,25 +250,13 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) {
     if (stk_addr && stk_size)
       MemoryRangeImitateWrite(thr, /*pc=*/ 1, stk_addr, stk_size);
 
-    if (tls_addr && tls_size) {
-      // Check that the thr object is in tls;
-      const uptr thr_beg = (uptr)thr;
-      const uptr thr_end = (uptr)thr + sizeof(*thr);
-      CHECK_GE(thr_beg, tls_addr);
-      CHECK_LE(thr_beg, tls_addr + tls_size);
-      CHECK_GE(thr_end, tls_addr);
-      CHECK_LE(thr_end, tls_addr + tls_size);
-      // Since the thr object is huge, skip it.
-      MemoryRangeImitateWrite(thr, /*pc=*/ 2, tls_addr, thr_beg - tls_addr);
-      MemoryRangeImitateWrite(thr, /*pc=*/ 2,
-          thr_end, tls_addr + tls_size - thr_end);
-    }
+    if (tls_addr && tls_size) ImitateTlsWrite(thr, tls_addr, tls_size);
   }
 #endif
 
   ThreadRegistry *tr = ctx->thread_registry;
   OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };
-  tr->StartThread(tid, os_id, &args);
+  tr->StartThread(tid, os_id, workerthread, &args);
 
   tr->Lock();
   thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid);
@@ -354,6 +347,7 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
   StatInc(thr, StatMopRange);
 
   if (*shadow_mem == kShadowRodata) {
+    DCHECK(!is_write);
     // Access to .rodata section, no races here.
     // Measurements show that it can be 10-20% of all memory accesses.
     StatInc(thr, StatMopRangeRodata);
diff --git a/libsanitizer/tsan/tsan_stat.cc b/libsanitizer/tsan/tsan_stat.cc
index 740cb86c69e..decb7a20b84 100644
--- a/libsanitizer/tsan/tsan_stat.cc
+++ b/libsanitizer/tsan/tsan_stat.cc
@@ -73,14 +73,11 @@ void StatOutput(u64 *stat) {
   name[StatClockAcquire]                 = "Clock acquire                     ";
   name[StatClockAcquireEmpty]            = "  empty clock                     ";
   name[StatClockAcquireFastRelease]      = "  fast from release-store         ";
-  name[StatClockAcquireLarge]            = "  contains my tid                 ";
-  name[StatClockAcquireRepeat]           = "  repeated (fast)                 ";
   name[StatClockAcquireFull]             = "  full (slow)                     ";
   name[StatClockAcquiredSomething]       = "  acquired something              ";
   name[StatClockRelease]                 = "Clock release                     ";
   name[StatClockReleaseResize]           = "  resize                          ";
-  name[StatClockReleaseFast1]            = "  fast1                           ";
-  name[StatClockReleaseFast2]            = "  fast2                           ";
+  name[StatClockReleaseFast]             = "  fast                            ";
   name[StatClockReleaseSlow]             = "  dirty overflow (slow)           ";
   name[StatClockReleaseFull]             = "  full (slow)                     ";
   name[StatClockReleaseAcquired]         = "  was acquired                    ";
@@ -151,6 +148,16 @@ void StatOutput(u64 *stat) {
   name[StatAnnotatePublishMemoryRange]   = "  PublishMemoryRange              ";
   name[StatAnnotateUnpublishMemoryRange] = "  UnpublishMemoryRange            ";
   name[StatAnnotateThreadName]           = "  ThreadName                      ";
+  name[Stat__tsan_mutex_create]          = "  __tsan_mutex_create             ";
+  name[Stat__tsan_mutex_destroy]         = "  __tsan_mutex_destroy            ";
+  name[Stat__tsan_mutex_pre_lock]        = "  __tsan_mutex_pre_lock           ";
+  name[Stat__tsan_mutex_post_lock]       = "  __tsan_mutex_post_lock          ";
+  name[Stat__tsan_mutex_pre_unlock]      = "  __tsan_mutex_pre_unlock         ";
+  name[Stat__tsan_mutex_post_unlock]     = "  __tsan_mutex_post_unlock        ";
+  name[Stat__tsan_mutex_pre_signal]      = "  __tsan_mutex_pre_signal         ";
+  name[Stat__tsan_mutex_post_signal]     = "  __tsan_mutex_post_signal        ";
+  name[Stat__tsan_mutex_pre_divert]      = "  __tsan_mutex_pre_divert         ";
+  name[Stat__tsan_mutex_post_divert]     = "  __tsan_mutex_post_divert        ";
 
   name[StatMtxTotal]                     = "Contentionz                       ";
   name[StatMtxTrace]                     = "  Trace                           ";
diff --git a/libsanitizer/tsan/tsan_stat.h b/libsanitizer/tsan/tsan_stat.h
index 788adeb56ea..c4859df52b0 100644
--- a/libsanitizer/tsan/tsan_stat.h
+++ b/libsanitizer/tsan/tsan_stat.h
@@ -72,15 +72,12 @@ enum StatType {
   StatClockAcquire,
   StatClockAcquireEmpty,
   StatClockAcquireFastRelease,
-  StatClockAcquireLarge,
-  StatClockAcquireRepeat,
   StatClockAcquireFull,
   StatClockAcquiredSomething,
   // Clocks - release.
   StatClockRelease,
   StatClockReleaseResize,
-  StatClockReleaseFast1,
-  StatClockReleaseFast2,
+  StatClockReleaseFast,
   StatClockReleaseSlow,
   StatClockReleaseFull,
   StatClockReleaseAcquired,
@@ -155,6 +152,16 @@ enum StatType {
   StatAnnotatePublishMemoryRange,
   StatAnnotateUnpublishMemoryRange,
   StatAnnotateThreadName,
+  Stat__tsan_mutex_create,
+  Stat__tsan_mutex_destroy,
+  Stat__tsan_mutex_pre_lock,
+  Stat__tsan_mutex_post_lock,
+  Stat__tsan_mutex_pre_unlock,
+  Stat__tsan_mutex_post_unlock,
+  Stat__tsan_mutex_pre_signal,
+  Stat__tsan_mutex_post_signal,
+  Stat__tsan_mutex_pre_divert,
+  Stat__tsan_mutex_post_divert,
 
   // Internal mutex contentionz.
   StatMtxTotal,
diff --git a/libsanitizer/tsan/tsan_suppressions.cc b/libsanitizer/tsan/tsan_suppressions.cc
index dc862b1b9ad..e40eb70ceec 100644
--- a/libsanitizer/tsan/tsan_suppressions.cc
+++ b/libsanitizer/tsan/tsan_suppressions.cc
@@ -72,6 +72,8 @@ static const char *conv(ReportType typ) {
     return kSuppressionRace;
   else if (typ == ReportTypeVptrUseAfterFree)
     return kSuppressionRace;
+  else if (typ == ReportTypeExternalRace)
+    return kSuppressionRace;
   else if (typ == ReportTypeThreadLeak)
     return kSuppressionThread;
   else if (typ == ReportTypeMutexDestroyLocked)
diff --git a/libsanitizer/tsan/tsan_sync.cc b/libsanitizer/tsan/tsan_sync.cc
index 0ee2295e69e..0f840de1f11 100644
--- a/libsanitizer/tsan/tsan_sync.cc
+++ b/libsanitizer/tsan/tsan_sync.cc
@@ -40,10 +40,7 @@ void SyncVar::Reset(Processor *proc) {
   owner_tid = kInvalidTid;
   last_lock = 0;
   recursion = 0;
-  is_rw = 0;
-  is_recursive = 0;
-  is_broken = 0;
-  is_linker_init = 0;
+  atomic_store_relaxed(&flags, 0);
 
   if (proc == 0) {
     CHECK_EQ(clock.size(), 0);
@@ -54,7 +51,9 @@ void SyncVar::Reset(Processor *proc) {
   }
 }
 
-MetaMap::MetaMap() {
+MetaMap::MetaMap()
+    : block_alloc_("heap block allocator")
+    , sync_alloc_("sync allocator") {
   atomic_store(&uid_gen_, 0, memory_order_relaxed);
 }
 
@@ -62,6 +61,7 @@ void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
   u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
   MBlock *b = block_alloc_.Map(idx);
   b->siz = sz;
+  b->tag = 0;
   b->tid = thr->tid;
   b->stk = CurrentStackId(thr, pc);
   u32 *meta = MemToMeta(p);
diff --git a/libsanitizer/tsan/tsan_sync.h b/libsanitizer/tsan/tsan_sync.h
index 48d7027b428..195279fc1f2 100644
--- a/libsanitizer/tsan/tsan_sync.h
+++ b/libsanitizer/tsan/tsan_sync.h
@@ -21,6 +21,29 @@
 
 namespace __tsan {
 
+// These need to match __tsan_mutex_* flags defined in tsan_interface.h.
+// See documentation there as well.
+enum MutexFlags {
+  MutexFlagLinkerInit          = 1 << 0, // __tsan_mutex_linker_init
+  MutexFlagWriteReentrant      = 1 << 1, // __tsan_mutex_write_reentrant
+  MutexFlagReadReentrant       = 1 << 2, // __tsan_mutex_read_reentrant
+  MutexFlagReadLock            = 1 << 3, // __tsan_mutex_read_lock
+  MutexFlagTryLock             = 1 << 4, // __tsan_mutex_try_lock
+  MutexFlagTryLockFailed       = 1 << 5, // __tsan_mutex_try_lock_failed
+  MutexFlagRecursiveLock       = 1 << 6, // __tsan_mutex_recursive_lock
+  MutexFlagRecursiveUnlock     = 1 << 7, // __tsan_mutex_recursive_unlock
+
+  // The following flags are runtime private.
+  // Mutex API misuse was detected, so don't report any more.
+  MutexFlagBroken              = 1 << 30,
+  // We did not intercept pre lock event, so handle it on post lock.
+  MutexFlagDoPreLockOnPostLock = 1 << 29,
+  // Must list all mutex creation flags.
+  MutexCreationFlagMask        = MutexFlagLinkerInit |
+                                 MutexFlagWriteReentrant |
+                                 MutexFlagReadReentrant,
+};
+
 struct SyncVar {
   SyncVar();
 
@@ -33,10 +56,7 @@ struct SyncVar {
   int owner_tid;  // Set only by exclusive owners.
   u64 last_lock;
   int recursion;
-  bool is_rw;
-  bool is_recursive;
-  bool is_broken;
-  bool is_linker_init;
+  atomic_uint32_t flags;
   u32 next;  // in MetaMap
   DDMutex dd;
   SyncClock read_clock;  // Used for rw mutexes only.
@@ -59,6 +79,26 @@ struct SyncVar {
     *uid = id >> 48;
     return (uptr)GetLsb(id, 48);
   }
+
+  bool IsFlagSet(u32 f) const {
+    return atomic_load_relaxed(&flags) & f;
+  }
+
+  void SetFlags(u32 f) {
+    atomic_store_relaxed(&flags, atomic_load_relaxed(&flags) | f);
+  }
+
+  void UpdateFlags(u32 flagz) {
+    // Filter out operation flags.
+    if (!(flagz & MutexCreationFlagMask))
+      return;
+    u32 current = atomic_load_relaxed(&flags);
+    if (current & MutexCreationFlagMask)
+      return;
+    // Note: this can be called from MutexPostReadLock which holds only read
+    // lock on the SyncVar.
+    atomic_store_relaxed(&flags, current | (flagz & MutexCreationFlagMask));
+  }
 };
 
 /* MetaMap allows to map arbitrary user pointers onto various descriptors.
diff --git a/libsanitizer/tsan/tsan_trace.h b/libsanitizer/tsan/tsan_trace.h
index 1ea733bfd07..5cc3f8f3498 100644
--- a/libsanitizer/tsan/tsan_trace.h
+++ b/libsanitizer/tsan/tsan_trace.h
@@ -39,6 +39,8 @@ enum EventType {
 // u64 addr : 61;  // Associated pc.
 typedef u64 Event;
 
+const uptr kEventPCBits = 61;
+
 struct TraceHeader {
 #if !SANITIZER_GO
   BufferedStackTrace stack0;  // Start stack for the trace.
diff --git a/libsanitizer/ubsan/Makefile.am b/libsanitizer/ubsan/Makefile.am
index cb27091137a..cce728ceeca 100644
--- a/libsanitizer/ubsan/Makefile.am
+++ b/libsanitizer/ubsan/Makefile.am
@@ -3,7 +3,7 @@ AM_CPPFLAGS = -I $(top_srcdir) -I $(top_srcdir)/include
 # May be used by toolexeclibdir.
 gcc_version := $(shell @get_gcc_base_ver@ $(top_srcdir)/../gcc/BASE-VER)
 
-DEFS = -D_GNU_SOURCE -D_DEBUG -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -DPIC -DCAN_SANITIZE_UB=1
+DEFS = -D_GNU_SOURCE -D_DEBUG -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -DPIC -DCAN_SANITIZE_UB=1 -DUBSAN_CAN_USE_CXXABI=1
 AM_CXXFLAGS = -Wall -W -Wno-unused-parameter -Wwrite-strings -pedantic -Wno-long-long  -fPIC -fno-builtin -fno-exceptions -fno-rtti -fomit-frame-pointer -funwind-tables -fvisibility=hidden -Wno-variadic-macros
 AM_CXXFLAGS += $(LIBSTDCXX_RAW_CXX_CXXFLAGS)
 AM_CXXFLAGS += -std=gnu++11
@@ -24,7 +24,8 @@ ubsan_plugin_files = \
 
 ubsan_files = \
 	$(ubsan_plugin_files) \
-	ubsan_init_standalone.cc
+	ubsan_init_standalone.cc \
+	ubsan_signals_standalone.cc
 
 libubsan_la_SOURCES = $(ubsan_files) 
 libubsan_la_LIBADD = $(top_builddir)/sanitizer_common/libsanitizer_common.la 
diff --git a/libsanitizer/ubsan/Makefile.in b/libsanitizer/ubsan/Makefile.in
index 1664ce9497e..9552ec1425d 100644
--- a/libsanitizer/ubsan/Makefile.in
+++ b/libsanitizer/ubsan/Makefile.in
@@ -111,7 +111,8 @@ am__objects_1 = ubsan_diag.lo ubsan_flags.lo ubsan_handlers.lo \
 	ubsan_handlers_cxx.lo ubsan_init.lo ubsan_type_hash.lo \
 	ubsan_type_hash_itanium.lo ubsan_type_hash_win.lo \
 	ubsan_value.lo
-am__objects_2 = $(am__objects_1) ubsan_init_standalone.lo
+am__objects_2 = $(am__objects_1) ubsan_init_standalone.lo \
+	ubsan_signals_standalone.lo
 am_libubsan_la_OBJECTS = $(am__objects_2)
 libubsan_la_OBJECTS = $(am_libubsan_la_OBJECTS)
 libubsan_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \
@@ -162,7 +163,7 @@ CXXCPP = @CXXCPP@
 CXXDEPMODE = @CXXDEPMODE@
 CXXFLAGS = @CXXFLAGS@
 CYGPATH_W = @CYGPATH_W@
-DEFS = -D_GNU_SOURCE -D_DEBUG -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -DPIC -DCAN_SANITIZE_UB=1
+DEFS = -D_GNU_SOURCE -D_DEBUG -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -DPIC -DCAN_SANITIZE_UB=1 -DUBSAN_CAN_USE_CXXABI=1
 DEPDIR = @DEPDIR@
 DSYMUTIL = @DSYMUTIL@
 DUMPBIN = @DUMPBIN@
@@ -307,7 +308,8 @@ ubsan_plugin_files = \
 
 ubsan_files = \
 	$(ubsan_plugin_files) \
-	ubsan_init_standalone.cc
+	ubsan_init_standalone.cc \
+	ubsan_signals_standalone.cc
 
 libubsan_la_SOURCES = $(ubsan_files) 
 libubsan_la_LIBADD =  \
@@ -435,6 +437,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ubsan_handlers_cxx.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ubsan_init.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ubsan_init_standalone.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ubsan_signals_standalone.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ubsan_type_hash.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ubsan_type_hash_itanium.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ubsan_type_hash_win.Plo@am__quote@
diff --git a/libsanitizer/ubsan/libtool-version b/libsanitizer/ubsan/libtool-version
index 204fdd2d8e5..4b494765130 100644
--- a/libsanitizer/ubsan/libtool-version
+++ b/libsanitizer/ubsan/libtool-version
@@ -3,4 +3,4 @@
 # a separate file so that version updates don't involve re-running
 # automake.
 # CURRENT:REVISION:AGE
-0:0:0
+1:0:0
diff --git a/libsanitizer/ubsan/ubsan_checks.inc b/libsanitizer/ubsan/ubsan_checks.inc
index 31e9495e301..c4459245f3e 100644
--- a/libsanitizer/ubsan/ubsan_checks.inc
+++ b/libsanitizer/ubsan/ubsan_checks.inc
@@ -27,6 +27,7 @@ UBSAN_CHECK(UnsignedIntegerOverflow, "unsigned-integer-overflow",
 UBSAN_CHECK(IntegerDivideByZero, "integer-divide-by-zero",
             "integer-divide-by-zero")
 UBSAN_CHECK(FloatDivideByZero, "float-divide-by-zero", "float-divide-by-zero")
+UBSAN_CHECK(InvalidBuiltin, "invalid-builtin-use", "invalid-builtin-use")
 UBSAN_CHECK(InvalidShiftBase, "invalid-shift-base", "shift-base")
 UBSAN_CHECK(InvalidShiftExponent, "invalid-shift-exponent", "shift-exponent")
 UBSAN_CHECK(OutOfBoundsIndex, "out-of-bounds-index", "bounds")
diff --git a/libsanitizer/ubsan/ubsan_diag.cc b/libsanitizer/ubsan/ubsan_diag.cc
index b6e982a170e..978d966f1c3 100644
--- a/libsanitizer/ubsan/ubsan_diag.cc
+++ b/libsanitizer/ubsan/ubsan_diag.cc
@@ -24,20 +24,25 @@
 
 using namespace __ubsan;
 
+void __ubsan::GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack,
+                                              uptr max_depth, uptr pc, uptr bp,
+                                              void *context, bool fast) {
+  uptr top = 0;
+  uptr bottom = 0;
+  if (fast)
+    GetThreadStackTopAndBottom(false, &top, &bottom);
+  stack->Unwind(max_depth, pc, bp, context, top, bottom, fast);
+}
+
 static void MaybePrintStackTrace(uptr pc, uptr bp) {
   // We assume that flags are already parsed, as UBSan runtime
   // will definitely be called when we print the first diagnostics message.
   if (!flags()->print_stacktrace)
     return;
-  // We can only use slow unwind, as we don't have any information about stack
-  // top/bottom.
-  // FIXME: It's better to respect "fast_unwind_on_fatal" runtime flag and
-  // fetch stack top/bottom information if we have it (e.g. if we're running
-  // under ASan).
-  if (StackTrace::WillUseFastUnwind(false))
-    return;
+
   BufferedStackTrace stack;
-  stack.Unwind(kStackTraceMax, pc, bp, 0, 0, 0, false);
+  GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, nullptr,
+                                  common_flags()->fast_unwind_on_fatal);
   stack.Print();
 }
 
@@ -77,16 +82,16 @@ static void MaybeReportErrorSummary(Location Loc, ErrorType Type) {
       AI.line = SLoc.getLine();
       AI.column = SLoc.getColumn();
       AI.function = internal_strdup("");  // Avoid printing ?? as function name.
-      ReportErrorSummary(ErrorKind, AI);
+      ReportErrorSummary(ErrorKind, AI, GetSanititizerToolName());
       AI.Clear();
       return;
     }
   } else if (Loc.isSymbolizedStack()) {
     const AddressInfo &AI = Loc.getSymbolizedStack()->info;
-    ReportErrorSummary(ErrorKind, AI);
+    ReportErrorSummary(ErrorKind, AI, GetSanititizerToolName());
     return;
   }
-  ReportErrorSummary(ErrorKind);
+  ReportErrorSummary(ErrorKind, GetSanititizerToolName());
 }
 
 namespace {
@@ -94,9 +99,7 @@ class Decorator : public SanitizerCommonDecorator {
  public:
   Decorator() : SanitizerCommonDecorator() {}
   const char *Highlight() const { return Green(); }
-  const char *EndHighlight() const { return Default(); }
   const char *Note() const { return Black(); }
-  const char *EndNote() const { return Default(); }
 };
 }
 
@@ -155,7 +158,7 @@ static void RenderLocation(InternalScopedString *Buffer, Location Loc) {
                            common_flags()->strip_path_prefix);
     else if (Info.module)
       RenderModuleLocation(Buffer, Info.module, Info.module_offset,
-                           common_flags()->strip_path_prefix);
+                           Info.module_arch, common_flags()->strip_path_prefix);
     else
       Buffer->append("%p", Info.address);
     return;
@@ -292,7 +295,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc,
     Buffer.append("%c", P == Loc ? '^' : Byte);
     Buffer.append("%c", Byte);
   }
-  Buffer.append("%s\n", Decor.EndHighlight());
+  Buffer.append("%s\n", Decor.Default());
 
   // Go over the line again, and print names for the ranges.
   InRange = 0;
@@ -332,7 +335,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc,
 
 Diag::~Diag() {
   // All diagnostics should be printed under report mutex.
-  CommonSanitizerReportMutex.CheckLocked();
+  ScopedReport::CheckLocked();
   Decorator Decor;
   InternalScopedString Buffer(1024);
 
@@ -342,12 +345,12 @@ Diag::~Diag() {
 
   switch (Level) {
   case DL_Error:
-    Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.EndWarning(),
+    Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.Default(),
                   Decor.Bold());
     break;
 
   case DL_Note:
-    Buffer.append("%s note: %s", Decor.Note(), Decor.EndNote());
+    Buffer.append("%s note: %s", Decor.Note(), Decor.Default());
     break;
   }
 
@@ -360,17 +363,15 @@ Diag::~Diag() {
     PrintMemorySnippet(Decor, Loc.getMemoryLocation(), Ranges, NumRanges, Args);
 }
 
+ScopedReport::Initializer::Initializer() { InitAsStandaloneIfNecessary(); }
+
 ScopedReport::ScopedReport(ReportOptions Opts, Location SummaryLoc,
                            ErrorType Type)
-    : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) {
-  InitAsStandaloneIfNecessary();
-  CommonSanitizerReportMutex.Lock();
-}
+    : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) {}
 
 ScopedReport::~ScopedReport() {
   MaybePrintStackTrace(Opts.pc, Opts.bp);
   MaybeReportErrorSummary(SummaryLoc, Type);
-  CommonSanitizerReportMutex.Unlock();
   if (flags()->halt_on_error)
     Die();
 }
diff --git a/libsanitizer/ubsan/ubsan_diag.h b/libsanitizer/ubsan/ubsan_diag.h
index 3456aaa72c5..e9e5ab67e3e 100644
--- a/libsanitizer/ubsan/ubsan_diag.h
+++ b/libsanitizer/ubsan/ubsan_diag.h
@@ -229,10 +229,20 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET);
     GET_CALLER_PC_BP; \
     ReportOptions Opts = {unrecoverable_handler, pc, bp}
 
+void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth,
+                                     uptr pc, uptr bp, void *context,
+                                     bool fast);
+
 /// \brief Instantiate this class before printing diagnostics in the error
 /// report. This class ensures that reports from different threads and from
 /// different sanitizers won't be mixed.
 class ScopedReport {
+  struct Initializer {
+    Initializer();
+  };
+  Initializer initializer_;
+  ScopedErrorReportLock report_lock_;
+
   ReportOptions Opts;
   Location SummaryLoc;
   ErrorType Type;
@@ -240,6 +250,8 @@ class ScopedReport {
 public:
   ScopedReport(ReportOptions Opts, Location SummaryLoc, ErrorType Type);
   ~ScopedReport();
+
+  static void CheckLocked() { ScopedErrorReportLock::CheckLocked(); }
 };
 
 void InitializeSuppressions();
diff --git a/libsanitizer/ubsan/ubsan_diag_standalone.cc b/libsanitizer/ubsan/ubsan_diag_standalone.cc
new file mode 100644
index 00000000000..ddc1be7018c
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_diag_standalone.cc
@@ -0,0 +1,36 @@
+//===-- ubsan_diag_standalone.cc ------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Diagnostic reporting for the standalone UBSan runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ubsan_platform.h"
+#if CAN_SANITIZE_UB
+#include "ubsan_diag.h"
+
+using namespace __ubsan;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+  uptr top = 0;
+  uptr bottom = 0;
+  bool request_fast_unwind = common_flags()->fast_unwind_on_fatal;
+  if (request_fast_unwind)
+    __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom);
+
+  GET_CURRENT_PC_BP_SP;
+  (void)sp;
+  BufferedStackTrace stack;
+  stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom,
+               request_fast_unwind);
+  stack.Print();
+}
+} // extern "C"
+
+#endif  // CAN_SANITIZE_UB
diff --git a/libsanitizer/ubsan/ubsan_flags.cc b/libsanitizer/ubsan/ubsan_flags.cc
index 19f5de1727e..9a3ea4b51e5 100644
--- a/libsanitizer/ubsan/ubsan_flags.cc
+++ b/libsanitizer/ubsan/ubsan_flags.cc
@@ -43,6 +43,7 @@ void InitializeFlags() {
     CommonFlags cf;
     cf.CopyFrom(*common_flags());
     cf.print_summary = false;
+    cf.external_symbolizer_path = GetEnv("UBSAN_SYMBOLIZER_PATH");
     OverrideCommonFlags(cf);
   }
 
@@ -65,22 +66,8 @@ void InitializeFlags() {
 
 }  // namespace __ubsan
 
-extern "C" {
-
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char *__ubsan_default_options() { return ""; }
-#endif
-
-#if SANITIZER_WINDOWS
-const char *__ubsan_default_default_options() { return ""; }
-# ifdef _WIN64
-#  pragma comment(linker, "/alternatename:__ubsan_default_options=__ubsan_default_default_options")
-# else
-#  pragma comment(linker, "/alternatename:___ubsan_default_options=___ubsan_default_default_options")
-# endif
-#endif
-
-}  // extern "C"
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __ubsan_default_options, void) {
+  return "";
+}
 
 #endif  // CAN_SANITIZE_UB
diff --git a/libsanitizer/ubsan/ubsan_handlers.cc b/libsanitizer/ubsan/ubsan_handlers.cc
index 761ccef63f3..4d29832ae0d 100644
--- a/libsanitizer/ubsan/ubsan_handlers.cc
+++ b/libsanitizer/ubsan/ubsan_handlers.cc
@@ -36,17 +36,18 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET) {
 const char *TypeCheckKinds[] = {
     "load of", "store to", "reference binding to", "member access within",
     "member call on", "constructor call on", "downcast of", "downcast of",
-    "upcast of", "cast to virtual base of"};
+    "upcast of", "cast to virtual base of", "_Nonnull binding to"};
 }
 
 static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer,
                                    ReportOptions Opts) {
   Location Loc = Data->Loc.acquire();
 
+  uptr Alignment = (uptr)1 << Data->LogAlignment;
   ErrorType ET;
   if (!Pointer)
     ET = ErrorType::NullPointerUse;
-  else if (Data->Alignment && (Pointer & (Data->Alignment - 1)))
+  else if (Pointer & (Alignment - 1))
     ET = ErrorType::MisalignedPointerUse;
   else
     ET = ErrorType::InsufficientObjectSize;
@@ -72,8 +73,8 @@ static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer,
   case ErrorType::MisalignedPointerUse:
     Diag(Loc, DL_Error, "%0 misaligned address %1 for type %3, "
                         "which requires %2 byte alignment")
-        << TypeCheckKinds[Data->TypeCheckKind] << (void *)Pointer
-        << Data->Alignment << Data->Type;
+        << TypeCheckKinds[Data->TypeCheckKind] << (void *)Pointer << Alignment
+        << Data->Type;
     break;
   case ErrorType::InsufficientObjectSize:
     Diag(Loc, DL_Error, "%0 address %1 with insufficient space "
@@ -88,13 +89,13 @@ static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer,
     Diag(Pointer, DL_Note, "pointer points here");
 }
 
-void __ubsan::__ubsan_handle_type_mismatch(TypeMismatchData *Data,
-                                           ValueHandle Pointer) {
+void __ubsan::__ubsan_handle_type_mismatch_v1(TypeMismatchData *Data,
+                                              ValueHandle Pointer) {
   GET_REPORT_OPTIONS(false);
   handleTypeMismatchImpl(Data, Pointer, Opts);
 }
-void __ubsan::__ubsan_handle_type_mismatch_abort(TypeMismatchData *Data,
-                                                 ValueHandle Pointer) {
+void __ubsan::__ubsan_handle_type_mismatch_v1_abort(TypeMismatchData *Data,
+                                                    ValueHandle Pointer) {
   GET_REPORT_OPTIONS(true);
   handleTypeMismatchImpl(Data, Pointer, Opts);
   Die();
@@ -387,7 +388,7 @@ static void handleFloatCastOverflow(void *DataPtr, ValueHandle From,
   ScopedReport R(Opts, Loc, ET);
 
   Diag(Loc, DL_Error,
-       "value %0 is outside the range of representable values of type %2")
+       "%0 is outside the range of representable values of type %2")
       << Value(*FromType, From) << *FromType << *ToType;
 }
 
@@ -407,7 +408,8 @@ static void handleLoadInvalidValue(InvalidValueData *Data, ValueHandle Val,
   SourceLocation Loc = Data->Loc.acquire();
   // This check could be more precise if we used different handlers for
   // -fsanitize=bool and -fsanitize=enum.
-  bool IsBool = (0 == internal_strcmp(Data->Type.getTypeName(), "'bool'"));
+  bool IsBool = (0 == internal_strcmp(Data->Type.getTypeName(), "'bool'")) ||
+                (0 == internal_strncmp(Data->Type.getTypeName(), "'BOOL'", 6));
   ErrorType ET =
       IsBool ? ErrorType::InvalidBoolLoad : ErrorType::InvalidEnumLoad;
 
@@ -433,6 +435,30 @@ void __ubsan::__ubsan_handle_load_invalid_value_abort(InvalidValueData *Data,
   Die();
 }
 
+static void handleInvalidBuiltin(InvalidBuiltinData *Data, ReportOptions Opts) {
+  SourceLocation Loc = Data->Loc.acquire();
+  ErrorType ET = ErrorType::InvalidBuiltin;
+
+  if (ignoreReport(Loc, Opts, ET))
+    return;
+
+  ScopedReport R(Opts, Loc, ET);
+
+  Diag(Loc, DL_Error,
+       "passing zero to %0, which is not a valid argument")
+    << ((Data->Kind == BCK_CTZPassedZero) ? "ctz()" : "clz()");
+}
+
+void __ubsan::__ubsan_handle_invalid_builtin(InvalidBuiltinData *Data) {
+  GET_REPORT_OPTIONS(true);
+  handleInvalidBuiltin(Data, Opts);
+}
+void __ubsan::__ubsan_handle_invalid_builtin_abort(InvalidBuiltinData *Data) {
+  GET_REPORT_OPTIONS(true);
+  handleInvalidBuiltin(Data, Opts);
+  Die();
+}
+
 static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data,
                                        ValueHandle Function,
                                        ReportOptions Opts) {
@@ -469,8 +495,12 @@ void __ubsan::__ubsan_handle_function_type_mismatch_abort(
   Die();
 }
 
-static void handleNonNullReturn(NonNullReturnData *Data, ReportOptions Opts) {
-  SourceLocation Loc = Data->Loc.acquire();
+static void handleNonNullReturn(NonNullReturnData *Data, SourceLocation *LocPtr,
+                                ReportOptions Opts, bool IsAttr) {
+  if (!LocPtr)
+    UNREACHABLE("source location pointer is null!");
+
+  SourceLocation Loc = LocPtr->acquire();
   ErrorType ET = ErrorType::InvalidNullReturn;
 
   if (ignoreReport(Loc, Opts, ET))
@@ -481,21 +511,39 @@ static void handleNonNullReturn(NonNullReturnData *Data, ReportOptions Opts) {
   Diag(Loc, DL_Error, "null pointer returned from function declared to never "
                       "return null");
   if (!Data->AttrLoc.isInvalid())
-    Diag(Data->AttrLoc, DL_Note, "returns_nonnull attribute specified here");
+    Diag(Data->AttrLoc, DL_Note, "%0 specified here")
+        << (IsAttr ? "returns_nonnull attribute"
+                   : "_Nonnull return type annotation");
+}
+
+void __ubsan::__ubsan_handle_nonnull_return_v1(NonNullReturnData *Data,
+                                               SourceLocation *LocPtr) {
+  GET_REPORT_OPTIONS(false);
+  handleNonNullReturn(Data, LocPtr, Opts, true);
 }
 
-void __ubsan::__ubsan_handle_nonnull_return(NonNullReturnData *Data) {
+void __ubsan::__ubsan_handle_nonnull_return_v1_abort(NonNullReturnData *Data,
+                                                     SourceLocation *LocPtr) {
+  GET_REPORT_OPTIONS(true);
+  handleNonNullReturn(Data, LocPtr, Opts, true);
+  Die();
+}
+
+void __ubsan::__ubsan_handle_nullability_return_v1(NonNullReturnData *Data,
+                                                   SourceLocation *LocPtr) {
   GET_REPORT_OPTIONS(false);
-  handleNonNullReturn(Data, Opts);
+  handleNonNullReturn(Data, LocPtr, Opts, false);
 }
 
-void __ubsan::__ubsan_handle_nonnull_return_abort(NonNullReturnData *Data) {
+void __ubsan::__ubsan_handle_nullability_return_v1_abort(
+    NonNullReturnData *Data, SourceLocation *LocPtr) {
   GET_REPORT_OPTIONS(true);
-  handleNonNullReturn(Data, Opts);
+  handleNonNullReturn(Data, LocPtr, Opts, false);
   Die();
 }
 
-static void handleNonNullArg(NonNullArgData *Data, ReportOptions Opts) {
+static void handleNonNullArg(NonNullArgData *Data, ReportOptions Opts,
+                             bool IsAttr) {
   SourceLocation Loc = Data->Loc.acquire();
   ErrorType ET = ErrorType::InvalidNullArgument;
 
@@ -504,20 +552,34 @@ static void handleNonNullArg(NonNullArgData *Data, ReportOptions Opts) {
 
   ScopedReport R(Opts, Loc, ET);
 
-  Diag(Loc, DL_Error, "null pointer passed as argument %0, which is declared to "
-       "never be null") << Data->ArgIndex;
+  Diag(Loc, DL_Error,
+       "null pointer passed as argument %0, which is declared to "
+       "never be null")
+      << Data->ArgIndex;
   if (!Data->AttrLoc.isInvalid())
-    Diag(Data->AttrLoc, DL_Note, "nonnull attribute specified here");
+    Diag(Data->AttrLoc, DL_Note, "%0 specified here")
+        << (IsAttr ? "nonnull attribute" : "_Nonnull type annotation");
 }
 
 void __ubsan::__ubsan_handle_nonnull_arg(NonNullArgData *Data) {
   GET_REPORT_OPTIONS(false);
-  handleNonNullArg(Data, Opts);
+  handleNonNullArg(Data, Opts, true);
 }
 
 void __ubsan::__ubsan_handle_nonnull_arg_abort(NonNullArgData *Data) {
   GET_REPORT_OPTIONS(true);
-  handleNonNullArg(Data, Opts);
+  handleNonNullArg(Data, Opts, true);
+  Die();
+}
+
+void __ubsan::__ubsan_handle_nullability_arg(NonNullArgData *Data) {
+  GET_REPORT_OPTIONS(false);
+  handleNonNullArg(Data, Opts, false);
+}
+
+void __ubsan::__ubsan_handle_nullability_arg_abort(NonNullArgData *Data) {
+  GET_REPORT_OPTIONS(true);
+  handleNonNullArg(Data, Opts, false);
   Die();
 }
 
@@ -533,8 +595,19 @@ static void handlePointerOverflowImpl(PointerOverflowData *Data,
 
   ScopedReport R(Opts, Loc, ET);
 
-  Diag(Loc, DL_Error, "pointer index expression with base %0 overflowed to %1")
-    << (void *)Base << (void*)Result;
+  if ((sptr(Base) >= 0) == (sptr(Result) >= 0)) {
+    if (Base > Result)
+      Diag(Loc, DL_Error, "addition of unsigned offset to %0 overflowed to %1")
+          << (void *)Base << (void *)Result;
+    else
+      Diag(Loc, DL_Error,
+           "subtraction of unsigned offset from %0 overflowed to %1")
+          << (void *)Base << (void *)Result;
+  } else {
+    Diag(Loc, DL_Error,
+         "pointer index expression with base %0 overflowed to %1")
+        << (void *)Base << (void *)Result;
+  }
 }
 
 void __ubsan::__ubsan_handle_pointer_overflow(PointerOverflowData *Data,
@@ -577,32 +650,33 @@ static void handleCFIBadIcall(CFICheckFailData *Data, ValueHandle Function,
 }
 
 namespace __ubsan {
+
 #ifdef UBSAN_CAN_USE_CXXABI
-SANITIZER_WEAK_ATTRIBUTE
-void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
-                      bool ValidVtable, ReportOptions Opts);
-#else
-static void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
-                             bool ValidVtable, ReportOptions Opts) {
+
+#ifdef _WIN32
+
+extern "C" void __ubsan_handle_cfi_bad_type_default(CFICheckFailData *Data,
+                                                    ValueHandle Vtable,
+                                                    bool ValidVtable,
+                                                    ReportOptions Opts) {
   Die();
 }
-#endif
-}  // namespace __ubsan
 
-void __ubsan::__ubsan_handle_cfi_bad_icall(CFIBadIcallData *CallData,
-                                           ValueHandle Function) {
-  GET_REPORT_OPTIONS(false);
-  CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type};
-  handleCFIBadIcall(&Data, Function, Opts);
-}
+WIN_WEAK_ALIAS(__ubsan_handle_cfi_bad_type, __ubsan_handle_cfi_bad_type_default)
+#else
+SANITIZER_WEAK_ATTRIBUTE
+#endif
+void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
+                                 bool ValidVtable, ReportOptions Opts);
 
-void __ubsan::__ubsan_handle_cfi_bad_icall_abort(CFIBadIcallData *CallData,
-                                                 ValueHandle Function) {
-  GET_REPORT_OPTIONS(true);
-  CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type};
-  handleCFIBadIcall(&Data, Function, Opts);
+#else
+void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
+                                 bool ValidVtable, ReportOptions Opts) {
   Die();
 }
+#endif
+
+}  // namespace __ubsan
 
 void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data,
                                             ValueHandle Value,
@@ -611,7 +685,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data,
   if (Data->CheckKind == CFITCK_ICall)
     handleCFIBadIcall(Data, Value, Opts);
   else
-    HandleCFIBadType(Data, Value, ValidVtable, Opts);
+    __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts);
 }
 
 void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data,
@@ -621,7 +695,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data,
   if (Data->CheckKind == CFITCK_ICall)
     handleCFIBadIcall(Data, Value, Opts);
   else
-    HandleCFIBadType(Data, Value, ValidVtable, Opts);
+    __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts);
   Die();
 }
 
diff --git a/libsanitizer/ubsan/ubsan_handlers.h b/libsanitizer/ubsan/ubsan_handlers.h
index d04554acee3..c5e499cd68e 100644
--- a/libsanitizer/ubsan/ubsan_handlers.h
+++ b/libsanitizer/ubsan/ubsan_handlers.h
@@ -18,7 +18,7 @@ namespace __ubsan {
 struct TypeMismatchData {
   SourceLocation Loc;
   const TypeDescriptor &Type;
-  uptr Alignment;
+  unsigned char LogAlignment;
   unsigned char TypeCheckKind;
 };
 
@@ -35,7 +35,7 @@ struct TypeMismatchData {
 /// \brief Handle a runtime type check failure, caused by either a misaligned
 /// pointer, a null pointer, or a pointer to insufficient storage for the
 /// type.
-RECOVERABLE(type_mismatch, TypeMismatchData *Data, ValueHandle Pointer)
+RECOVERABLE(type_mismatch_v1, TypeMismatchData *Data, ValueHandle Pointer)
 
 struct OverflowData {
   SourceLocation Loc;
@@ -120,6 +120,21 @@ struct InvalidValueData {
 /// \brief Handle a load of an invalid value for the type.
 RECOVERABLE(load_invalid_value, InvalidValueData *Data, ValueHandle Val)
 
+/// Known builtin check kinds.
+/// Keep in sync with the enum of the same name in CodeGenFunction.h
+enum BuiltinCheckKind : unsigned char {
+  BCK_CTZPassedZero,
+  BCK_CLZPassedZero,
+};
+
+struct InvalidBuiltinData {
+  SourceLocation Loc;
+  unsigned char Kind;
+};
+
+/// Handle a builtin called in an invalid way.
+RECOVERABLE(invalid_builtin, InvalidBuiltinData *Data)
+
 struct FunctionTypeMismatchData {
   SourceLocation Loc;
   const TypeDescriptor &Type;
@@ -130,12 +145,13 @@ RECOVERABLE(function_type_mismatch,
             ValueHandle Val)
 
 struct NonNullReturnData {
-  SourceLocation Loc;
   SourceLocation AttrLoc;
 };
 
-/// \brief Handle returning null from function with returns_nonnull attribute.
-RECOVERABLE(nonnull_return, NonNullReturnData *Data)
+/// \brief Handle returning null from function with the returns_nonnull
+/// attribute, or a return type annotated with _Nonnull.
+RECOVERABLE(nonnull_return_v1, NonNullReturnData *Data, SourceLocation *Loc)
+RECOVERABLE(nullability_return_v1, NonNullReturnData *Data, SourceLocation *Loc)
 
 struct NonNullArgData {
   SourceLocation Loc;
@@ -143,8 +159,10 @@ struct NonNullArgData {
   int ArgIndex;
 };
 
-/// \brief Handle passing null pointer to function with nonnull attribute.
+/// \brief Handle passing null pointer to a function parameter with the nonnull
+/// attribute, or a _Nonnull type annotation.
 RECOVERABLE(nonnull_arg, NonNullArgData *Data)
+RECOVERABLE(nullability_arg, NonNullArgData *Data)
 
 struct PointerOverflowData {
   SourceLocation Loc;
@@ -163,23 +181,22 @@ enum CFITypeCheckKind : unsigned char {
   CFITCK_ICall,
 };
 
-struct CFIBadIcallData {
-  SourceLocation Loc;
-  const TypeDescriptor &Type;
-};
-
 struct CFICheckFailData {
   CFITypeCheckKind CheckKind;
   SourceLocation Loc;
   const TypeDescriptor &Type;
 };
 
-/// \brief Handle control flow integrity failure for indirect function calls.
-RECOVERABLE(cfi_bad_icall, CFIBadIcallData *Data, ValueHandle Function)
-
 /// \brief Handle control flow integrity failures.
 RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function,
             uptr VtableIsValid)
+
+struct ReportOptions;
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_cfi_bad_type(
+    CFICheckFailData *Data, ValueHandle Vtable, bool ValidVtable,
+    ReportOptions Opts);
+
 }
 
 #endif // UBSAN_HANDLERS_H
diff --git a/libsanitizer/ubsan/ubsan_handlers_cxx.cc b/libsanitizer/ubsan/ubsan_handlers_cxx.cc
index 007a1d6d0c2..bf729db6ce0 100644
--- a/libsanitizer/ubsan/ubsan_handlers_cxx.cc
+++ b/libsanitizer/ubsan/ubsan_handlers_cxx.cc
@@ -93,8 +93,8 @@ void __ubsan::__ubsan_handle_dynamic_type_cache_miss_abort(
 }
 
 namespace __ubsan {
-void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
-                      bool ValidVtable, ReportOptions Opts) {
+void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
+                                 bool ValidVtable, ReportOptions Opts) {
   SourceLocation Loc = Data->Loc.acquire();
   ErrorType ET = ErrorType::CFIBadType;
 
@@ -121,6 +121,7 @@ void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
     CheckKindStr = "cast to unrelated type";
     break;
   case CFITCK_ICall:
+  default:
     Die();
   }
 
@@ -142,22 +143,4 @@ void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
 }
 }  // namespace __ubsan
 
-void __ubsan::__ubsan_handle_cfi_bad_type(CFIBadTypeData *TypeData,
-                                          ValueHandle Vtable) {
-  GET_REPORT_OPTIONS(false);
-  CFITypeCheckKind TypeCheckKind
-    = static_cast<CFITypeCheckKind> (TypeData->TypeCheckKind);
-  CFICheckFailData Data = {TypeCheckKind, TypeData->Loc, TypeData->Type};
-  HandleCFIBadType(&Data, Vtable, false, Opts);
-}
-
-void __ubsan::__ubsan_handle_cfi_bad_type_abort(CFIBadTypeData *TypeData,
-                                                ValueHandle Vtable) {
-  GET_REPORT_OPTIONS(true);
-  CFITypeCheckKind TypeCheckKind
-    = static_cast<CFITypeCheckKind> (TypeData->TypeCheckKind);
-  CFICheckFailData Data = {TypeCheckKind, TypeData->Loc, TypeData->Type};
-  HandleCFIBadType(&Data, Vtable, false, Opts);
-}
-
 #endif // CAN_SANITIZE_UB
diff --git a/libsanitizer/ubsan/ubsan_handlers_cxx.h b/libsanitizer/ubsan/ubsan_handlers_cxx.h
index 6ace2b3472b..37382359b1e 100644
--- a/libsanitizer/ubsan/ubsan_handlers_cxx.h
+++ b/libsanitizer/ubsan/ubsan_handlers_cxx.h
@@ -23,12 +23,6 @@ struct DynamicTypeCacheMissData {
   unsigned char TypeCheckKind;
 };
 
-struct CFIBadTypeData {
-  SourceLocation Loc;
-  const TypeDescriptor &Type;
-  unsigned char TypeCheckKind;
-};
-
 /// \brief Handle a runtime type check failure, caused by an incorrect vptr.
 /// When this handler is called, all we know is that the type was not in the
 /// cache; this does not necessarily imply the existence of a bug.
@@ -38,13 +32,6 @@ void __ubsan_handle_dynamic_type_cache_miss(
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 void __ubsan_handle_dynamic_type_cache_miss_abort(
   DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash);
-
-/// \brief Handle a control flow integrity check failure by printing a
-/// diagnostic.
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
-__ubsan_handle_cfi_bad_type(CFIBadTypeData *Data, ValueHandle Vtable);
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
-__ubsan_handle_cfi_bad_type_abort(CFIBadTypeData *Data, ValueHandle Vtable);
 }
 
 #endif // UBSAN_HANDLERS_H
diff --git a/libsanitizer/ubsan/ubsan_init.cc b/libsanitizer/ubsan/ubsan_init.cc
index 07f748109e7..9ae17f59ea1 100644
--- a/libsanitizer/ubsan/ubsan_init.cc
+++ b/libsanitizer/ubsan/ubsan_init.cc
@@ -21,11 +21,11 @@
 
 using namespace __ubsan;
 
-static enum {
-  UBSAN_MODE_UNKNOWN = 0,
-  UBSAN_MODE_STANDALONE,
-  UBSAN_MODE_PLUGIN
-} ubsan_mode;
+const char *__ubsan::GetSanititizerToolName() {
+  return "UndefinedBehaviorSanitizer";
+}
+
+static bool ubsan_initialized;
 static StaticSpinMutex ubsan_init_mu;
 
 static void CommonInit() {
@@ -33,45 +33,31 @@ static void CommonInit() {
 }
 
 static void CommonStandaloneInit() {
-  SanitizerToolName = "UndefinedBehaviorSanitizer";
-  InitializeFlags();
+  SanitizerToolName = GetSanititizerToolName();
   CacheBinaryName();
+  InitializeFlags();
   __sanitizer_set_report_path(common_flags()->log_path);
   AndroidLogInit();
   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
   CommonInit();
-  ubsan_mode = UBSAN_MODE_STANDALONE;
 }
 
 void __ubsan::InitAsStandalone() {
-  if (SANITIZER_CAN_USE_PREINIT_ARRAY) {
-    CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode);
-    CommonStandaloneInit();
-    return;
-  }
   SpinMutexLock l(&ubsan_init_mu);
-  CHECK_NE(UBSAN_MODE_PLUGIN, ubsan_mode);
-  if (ubsan_mode == UBSAN_MODE_UNKNOWN)
+  if (!ubsan_initialized) {
     CommonStandaloneInit();
-}
-
-void __ubsan::InitAsStandaloneIfNecessary() {
-  if (SANITIZER_CAN_USE_PREINIT_ARRAY) {
-    CHECK_NE(UBSAN_MODE_UNKNOWN, ubsan_mode);
-    return;
+    ubsan_initialized = true;
   }
-  SpinMutexLock l(&ubsan_init_mu);
-  if (ubsan_mode == UBSAN_MODE_UNKNOWN)
-    CommonStandaloneInit();
 }
 
+void __ubsan::InitAsStandaloneIfNecessary() { return InitAsStandalone(); }
+
 void __ubsan::InitAsPlugin() {
-#if !SANITIZER_CAN_USE_PREINIT_ARRAY
   SpinMutexLock l(&ubsan_init_mu);
-#endif
-  CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode);
-  CommonInit();
-  ubsan_mode = UBSAN_MODE_PLUGIN;
+  if (!ubsan_initialized) {
+    CommonInit();
+    ubsan_initialized = true;
+  }
 }
 
 #endif  // CAN_SANITIZE_UB
diff --git a/libsanitizer/ubsan/ubsan_init.h b/libsanitizer/ubsan/ubsan_init.h
index 6a8366f82bf..73bd3f3580f 100644
--- a/libsanitizer/ubsan/ubsan_init.h
+++ b/libsanitizer/ubsan/ubsan_init.h
@@ -13,6 +13,9 @@
 
 namespace __ubsan {
 
+// Get the full tool name for UBSan.
+const char *GetSanititizerToolName();
+
 // Initialize UBSan as a standalone tool. Typically should be called early
 // during initialization.
 void InitAsStandalone();
diff --git a/libsanitizer/ubsan/ubsan_init_standalone.cc b/libsanitizer/ubsan/ubsan_init_standalone.cc
index 1630fd724fb..67223bea629 100644
--- a/libsanitizer/ubsan/ubsan_init_standalone.cc
+++ b/libsanitizer/ubsan/ubsan_init_standalone.cc
@@ -16,17 +16,17 @@
 
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "ubsan_init.h"
+#include "ubsan_signals_standalone.h"
+
+namespace __ubsan {
 
-#if SANITIZER_CAN_USE_PREINIT_ARRAY
-__attribute__((section(".preinit_array"), used))
-void (*__local_ubsan_preinit)(void) = __ubsan::InitAsStandalone;
-#else
-// Use a dynamic initializer.
 class UbsanStandaloneInitializer {
  public:
   UbsanStandaloneInitializer() {
-    __ubsan::InitAsStandalone();
+    InitAsStandalone();
+    InitializeDeadlySignals();
   }
 };
 static UbsanStandaloneInitializer ubsan_standalone_initializer;
-#endif  // SANITIZER_CAN_USE_PREINIT_ARRAY
+
+} // namespace __ubsan
diff --git a/libsanitizer/ubsan/ubsan_init_standalone_preinit.cc b/libsanitizer/ubsan/ubsan_init_standalone_preinit.cc
new file mode 100644
index 00000000000..80b2f3c3fd4
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_init_standalone_preinit.cc
@@ -0,0 +1,35 @@
+//===-- ubsan_init_standalone_preinit.cc
+//------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Initialization of standalone UBSan runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ubsan_platform.h"
+#if !CAN_SANITIZE_UB
+#error "UBSan is not supported on this platform!"
+#endif
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "ubsan_init.h"
+#include "ubsan_signals_standalone.h"
+
+#if SANITIZER_CAN_USE_PREINIT_ARRAY
+
+namespace __ubsan {
+
+static void PreInitAsStandalone() {
+  InitAsStandalone();
+  InitializeDeadlySignals();
+}
+
+} // namespace __ubsan
+
+__attribute__((section(".preinit_array"), used)) void (*__local_ubsan_preinit)(
+    void) = __ubsan::PreInitAsStandalone;
+#endif // SANITIZER_CAN_USE_PREINIT_ARRAY
diff --git a/libsanitizer/ubsan/ubsan_interface.inc b/libsanitizer/ubsan/ubsan_interface.inc
new file mode 100644
index 00000000000..1b0bc425da7
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_interface.inc
@@ -0,0 +1,52 @@
+//===-- ubsan_interface.inc -----------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Ubsan interface list.
+//===----------------------------------------------------------------------===//
+INTERFACE_FUNCTION(__ubsan_handle_add_overflow)
+INTERFACE_FUNCTION(__ubsan_handle_add_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_builtin_unreachable)
+INTERFACE_FUNCTION(__ubsan_handle_cfi_bad_type)
+INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail)
+INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail_abort)
+INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow)
+INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss)
+INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort)
+INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow)
+INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch)
+INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort)
+INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin)
+INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin_abort)
+INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value)
+INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value_abort)
+INTERFACE_FUNCTION(__ubsan_handle_missing_return)
+INTERFACE_FUNCTION(__ubsan_handle_mul_overflow)
+INTERFACE_FUNCTION(__ubsan_handle_mul_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_negate_overflow)
+INTERFACE_FUNCTION(__ubsan_handle_negate_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_nonnull_arg)
+INTERFACE_FUNCTION(__ubsan_handle_nonnull_arg_abort)
+INTERFACE_FUNCTION(__ubsan_handle_nonnull_return_v1)
+INTERFACE_FUNCTION(__ubsan_handle_nonnull_return_v1_abort)
+INTERFACE_FUNCTION(__ubsan_handle_nullability_arg)
+INTERFACE_FUNCTION(__ubsan_handle_nullability_arg_abort)
+INTERFACE_FUNCTION(__ubsan_handle_nullability_return_v1)
+INTERFACE_FUNCTION(__ubsan_handle_nullability_return_v1_abort)
+INTERFACE_FUNCTION(__ubsan_handle_out_of_bounds)
+INTERFACE_FUNCTION(__ubsan_handle_out_of_bounds_abort)
+INTERFACE_FUNCTION(__ubsan_handle_pointer_overflow)
+INTERFACE_FUNCTION(__ubsan_handle_pointer_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_shift_out_of_bounds)
+INTERFACE_FUNCTION(__ubsan_handle_shift_out_of_bounds_abort)
+INTERFACE_FUNCTION(__ubsan_handle_sub_overflow)
+INTERFACE_FUNCTION(__ubsan_handle_sub_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_type_mismatch_v1)
+INTERFACE_FUNCTION(__ubsan_handle_type_mismatch_v1_abort)
+INTERFACE_FUNCTION(__ubsan_handle_vla_bound_not_positive)
+INTERFACE_FUNCTION(__ubsan_handle_vla_bound_not_positive_abort)
+INTERFACE_WEAK_FUNCTION(__ubsan_default_options)
diff --git a/libsanitizer/ubsan/ubsan_platform.h b/libsanitizer/ubsan/ubsan_platform.h
index fa4e1a191aa..e73df632302 100644
--- a/libsanitizer/ubsan/ubsan_platform.h
+++ b/libsanitizer/ubsan/ubsan_platform.h
@@ -13,12 +13,13 @@
 
 #ifndef CAN_SANITIZE_UB
 // Other platforms should be easy to add, and probably work as-is.
-#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)) && \
-    (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \
-     defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) || \
+#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) ||       \
+     defined(__NetBSD__)) &&                                                   \
+    (defined(__x86_64__) || defined(__i386__) || defined(__arm__) ||           \
+     defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) ||    \
      defined(__s390__))
 # define CAN_SANITIZE_UB 1
-#elif defined(_WIN32)
+#elif defined(_WIN32) || defined(__Fuchsia__)
 # define CAN_SANITIZE_UB 1
 #else
 # define CAN_SANITIZE_UB 0
diff --git a/libsanitizer/ubsan/ubsan_signals_standalone.cc b/libsanitizer/ubsan/ubsan_signals_standalone.cc
new file mode 100644
index 00000000000..eb9b99ce9ac
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_signals_standalone.cc
@@ -0,0 +1,52 @@
+//=-- ubsan_signals_standalone.cc
+//------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Installs signal handlers and related interceptors for UBSan standalone.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ubsan_platform.h"
+#if CAN_SANITIZE_UB
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "ubsan_diag.h"
+#include "ubsan_init.h"
+
+#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+namespace __ubsan {
+
+#if SANITIZER_FUCHSIA
+void InitializeDeadlySignals() {}
+#else
+static void OnStackUnwind(const SignalContext &sig, const void *,
+                          BufferedStackTrace *stack) {
+  GetStackTraceWithPcBpAndContext(stack, kStackTraceMax, sig.pc, sig.bp,
+                                  sig.context,
+                                  common_flags()->fast_unwind_on_fatal);
+}
+
+static void UBsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+  HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
+static bool is_initialized = false;
+
+void InitializeDeadlySignals() {
+  if (is_initialized)
+    return;
+  is_initialized = true;
+  InitializeSignalInterceptors();
+  InstallDeadlySignalHandlers(&UBsanOnDeadlySignal);
+}
+#endif
+
+} // namespace __ubsan
+
+#endif // CAN_SANITIZE_UB
diff --git a/libsanitizer/ubsan/ubsan_signals_standalone.h b/libsanitizer/ubsan/ubsan_signals_standalone.h
new file mode 100644
index 00000000000..65e64f205ca
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_signals_standalone.h
@@ -0,0 +1,23 @@
+//=-- ubsan_signals_standalone.h
+//------------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Installs signal handlers and related interceptors for UBSan standalone.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef UBSAN_SIGNALS_STANDALONE_H
+#define UBSAN_SIGNALS_STANDALONE_H
+
+namespace __ubsan {
+
+// Initializes signal handlers and interceptors.
+void InitializeDeadlySignals();
+
+} // namespace __ubsan
+
+#endif // UBSAN_SIGNALS_STANDALONE_H
diff --git a/libsanitizer/ubsan/ubsan_type_hash_itanium.cc b/libsanitizer/ubsan/ubsan_type_hash_itanium.cc
index 790b126815d..9df316e14be 100644
--- a/libsanitizer/ubsan/ubsan_type_hash_itanium.cc
+++ b/libsanitizer/ubsan/ubsan_type_hash_itanium.cc
@@ -195,9 +195,9 @@ struct VtablePrefix {
 };
 VtablePrefix *getVtablePrefix(void *Vtable) {
   VtablePrefix *Vptr = reinterpret_cast<VtablePrefix*>(Vtable);
-  if (!Vptr)
-    return nullptr;
   VtablePrefix *Prefix = Vptr - 1;
+  if (!IsAccessibleMemoryRange((uptr)Prefix, sizeof(VtablePrefix)))
+    return nullptr;
   if (!Prefix->TypeInfo)
     // This can't possibly be a valid vtable.
     return nullptr;
diff --git a/libsanitizer/ubsan/ubsan_win_dll_thunk.cc b/libsanitizer/ubsan/ubsan_win_dll_thunk.cc
new file mode 100644
index 00000000000..1091ac0351a
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_win_dll_thunk.cc
@@ -0,0 +1,19 @@
+//===-- ubsan_win_dll_thunk.cc --------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a family of thunks that should be statically linked into
+// the DLLs that have instrumentation in order to delegate the calls to the
+// shared runtime that lives in the main binary.
+// See https://github.com/google/sanitizers/issues/209 for the details.
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DLL_THUNK
+#include "sanitizer_common/sanitizer_win_dll_thunk.h"
+// Ubsan interface functions.
+#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "ubsan_interface.inc"
+#endif // SANITIZER_DLL_THUNK
diff --git a/libsanitizer/ubsan/ubsan_win_dynamic_runtime_thunk.cc b/libsanitizer/ubsan/ubsan_win_dynamic_runtime_thunk.cc
new file mode 100644
index 00000000000..6ab5ae388a3
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_win_dynamic_runtime_thunk.cc
@@ -0,0 +1,19 @@
+//===-- ubsan_win_dynamic_runtime_thunk.cc --------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines things that need to be present in the application modules
+// to interact with Ubsan, when it is included in a dll.
+//
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
+#define SANITIZER_IMPORT_INTERFACE 1
+#include "sanitizer_common/sanitizer_win_defs.h"
+// Define weak alias for all weak functions imported from ubsan.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
+#include "ubsan_interface.inc"
+#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
diff --git a/libsanitizer/ubsan/ubsan_win_weak_interception.cc b/libsanitizer/ubsan/ubsan_win_weak_interception.cc
new file mode 100644
index 00000000000..98c8c27b2fa
--- /dev/null
+++ b/libsanitizer/ubsan/ubsan_win_weak_interception.cc
@@ -0,0 +1,21 @@
+//===-- ubsan_win_weak_interception.cc ------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This module should be included in Ubsan when it is implemented as a shared
+// library on Windows (dll), in order to delegate the calls of weak functions to
+// the implementation in the main executable when a strong definition is
+// provided.
+//===----------------------------------------------------------------------===//
+#ifdef SANITIZER_DYNAMIC
+#include "sanitizer_common/sanitizer_win_weak_interception.h"
+#include "ubsan_flags.h"
+// Check if strong definitions for weak functions are present in the main
+// executable. If that is the case, override dll functions to point to strong
+// implementations.
+#define INTERFACE_FUNCTION(Name)
+#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
+#include "ubsan_interface.inc"
+#endif // SANITIZER_DYNAMIC
diff --git a/libstdc++-v3/ChangeLog b/libstdc++-v3/ChangeLog
index 9a512b15b10..b12fa76551d 100644
--- a/libstdc++-v3/ChangeLog
+++ b/libstdc++-v3/ChangeLog
@@ -1,3 +1,491 @@
+2017-11-03  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/type_traits (endian): Define new enumeration type.
+	* testsuite/20_util/endian/1.cc: New test.
+
+	* include/bits/node_handle.h (_Node_insert_return::get): Remove, as
+	per P0508R0.
+
+2017-11-01  François Dumont  <fdumont@gcc.gnu.org>
+
+	* python/libstdcxx/v6/printers.py (StdExpAnyPrinter.__init__): Strip
+	typename versioned namespace before the substitution.
+	(StdExpOptionalPrinter.__init__): Likewise.
+	(StdVariantPrinter.__init__): Likewise.
+	(Printer.add_version): Inject versioned namespace after std or
+	__gnu_cxx.
+	(build_libstdcxx_dictionary): Adapt add_version usages, always pass
+	namespace first and symbol second.
+
+2017-11-01  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/82777
+	* src/filesystem/std-path.cc (path::lexically_normal): Remove dot-dot
+	elements correctly.
+	* testsuite/27_io/filesystem/path/generation/normal.cc: Add testcase.
+	* testsuite/util/testsuite_fs.h (compare_paths): Improve exception
+	text.
+
+2017-10-30  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/Makefile.am (stamp-bits-sup): Do not create broken symlink
+	to stamp-bits.
+	* include/Makefile.in: Regenerate.
+
+	* include/std/fstream (basic_ifstream, basic_ofstream, basic_fstream):
+	Remove outdated comments about calling c_str() to create a file stream
+	from a std::string.
+	(basic_ofstream::basic_ofstream, basic_ofstream::open): Remove
+	redundant ios_mode::trunc bits from default arguments and comments.
+
+	* doc/xml/faq.xml: Adjust "What is libstdc++?" answer slightly.
+
+	* include/bits/hashtable_policy.h: Include <tuple>.
+	* include/std/unordered_map: Only include <bits/stl_pair.h> instead
+	of <utility> and <tuple>.
+	* include/std/unordered_set: Likewise.
+
+2017-10-30  Ville Voutilainen  <ville.voutilainen@gmail.com>
+
+	Implement LWG 2485
+	* include/debug/array (get(const array<_Tp, _Nm>&&)): New.
+	* include/std/array (get(const array<_Tp, _Nm>&&)): Likewise.
+	* include/std/tuple (get(const tuple<_Elements...>&&)): Likewise.
+	(get(const tuple<_Types...>&&)): Likewise.
+	* include/std/utility
+	(__pair_get::__const_move_get(const std::pair<_Tp1, _Tp2>&&)):
+	Likewise.
+	(get(const std::pair<_Tp1, _Tp2>&&)): Likewise.
+	(get(const pair<_Tp, _Up>&&)): Likewise.
+	(get(const pair<_Up, _Tp>&&)): Likewise.
+	* testsuite/20_util/pair/astuple/get.cc: Add tests for
+	new overloads.
+	* testsuite/20_util/pair/astuple/get_by_type.cc: Likewise.
+	* testsuite/20_util/tuple/element_access/get2.cc: Likewise.
+	* testsuite/20_util/tuple/element_access/get2_by_type.cc: Likewise.
+	* testsuite/23_containers/array/tuple_interface/get.cc: Likewise.
+	* testsuite/23_containers/array/tuple_interface/tuple_element_debug_neg.cc:
+	Adjust.
+	* testsuite/23_containers/array/tuple_interface/tuple_element_neg.cc:
+	Likewise.
+
+2017-10-27  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/node_handle.h (_Node_insert_return::get): Avoid
+	use of std::tie and std::get.
+
+	* include/Makefile.am: Put headers in alphabetical order.
+	* include/Makefile.in: Regenerate.
+
+	* include/bits/fs_dir.h (directory_iterator): Remove noexcept from
+	constructors and increment member (LWG 3013).
+	(recursive_directory_iterator): Likewise.
+	* include/bits/fs_ops.h (copy, copy_file, create_directories)
+	(is_empty, remove_all): Remove noexcept (LWG 3013 and LWG 3014).
+	* src/filesystem/std-dir.cc (directory_iterator::increment)
+	(recursive_directory_iterator::increment): Remove noexcept.
+	* src/filesystem/std-ops.cc (copy, copy_file, create_directories)
+	(is_empty, remove_all): Remove noexcept
+
+	* include/bits/fs_dir.h (file_status): Make default constructor
+	non-explicit (LWG 2787).
+	* testsuite/27_io/filesystem/file_status/1.cc: New test.
+	* testsuite/experimental/filesystem/file_status/1.cc: New test.
+
+	* include/std/fstream (basic_filebuf::_If_path): New SFINAE helper.
+	(basic_filebuf::open<Path>(const Path&, const ios_base::openmode&))
+	(basic_ifstream<Path>(const Path&, const ios_base::openmode&))
+	(basic_ifstream::open<Path>(const Path&, const ios_base::openmode&))
+	(basic_ofstream<Path>(const Path&, const ios_base::openmode&))
+	(basic_ofstream::open<Path>(const Path&, const ios_base::openmode&))
+	(basic_fstream<Path>(const Path&, const ios_base::openmode&))
+	(basic_fstream::open<Path>(const Path&, const ios_base::openmode&)):
+	New constructors and member functions.
+	* testsuite/27_io/basic_filebuf/open/char/path.cc: New test.
+	* testsuite/27_io/basic_fstream/cons/char/path.cc: New test.
+	* testsuite/27_io/basic_fstream/open/char/path.cc: New test.
+	* testsuite/27_io/basic_ifstream/cons/char/path.cc: New test.
+	* testsuite/27_io/basic_ifstream/open/char/path.cc: New test.
+	* testsuite/27_io/basic_ofstream/cons/char/path.cc: New test.
+	* testsuite/27_io/basic_ofstream/open/char/path.cc: New test.
+
+	* include/bits/fs_path.h (path::format): Define new enumeration type.
+	(path(string_type&&), path<Source>(const Source&))
+	(path<InputIterator>(InputIterator, InputIterator))
+	(path<Source>(const Source&, const locale&))
+	(path<InputIterator>(InputIterator, InputIterator, const locale&)):
+	Add format parameter.
+	* testsuite/27_io/filesystem/path/construct/format.cc: New test.
+
+	* include/bits/stl_algo.h (__find_if_not_n, generate_n): Cast to void
+	to ensure overloaded comma not used.
+	* include/bits/stl_algobase.h (__fill_n_a, equal): Likewise.
+	* include/bits/stl_uninitialized.h (__uninitialized_fill_n)
+	(__uninitialized_fill_n_a, __uninitialized_default_n_1)
+	(__uninitialized_default_n_a, __uninitialized_copy_n)
+	(__uninitialized_copy_n_pair): Likewise
+	* testsuite/20_util/specialized_algorithms/memory_management_tools/1.cc:
+	Use test iterator wrappers with overloaded comma operator.
+	* testsuite/25_algorithms/fill_n/1.cc: Likewise.
+	* testsuite/25_algorithms/generate_n/1.cc: New test.
+	* testsuite/25_algorithms/stable_partition/1.cc: New test.
+	* testsuite/util/testsuite_iterators.h (operator,): Add deleted
+	non-member comma operator with iterator wrappers as right operand.
+
+2017-10-26  Ville Voutilainen  <ville.voutilainen@gmail.com>
+
+	Deduction guides for associative containers, debug mode deduction
+	guide fixes.
+	* include/bits/stl_iterator.h (__iter_key_t)
+	(__iter_val_t, __iter_to_alloc_t): New.
+	* include/bits/stl_map.h: Add deduction guides.
+	* include/bits/stl_multimap.h: Likewise.
+	* include/bits/stl_multiset.h: Likewise.
+	* include/bits/stl_set.h: Likewise.
+	* include/bits/unordered_map.h: Likewise.
+	* include/bits/unordered_set.h: Likewise.
+	* include/debug/deque: Likewise.
+	* include/debug/forward_list: Likewise.
+	* include/debug/list: Likewise.
+	* include/debug/map.h: Likewise.
+	* include/debug/multimap.h: Likewise.
+	* include/debug/multiset.h: Likewise.
+	* include/debug/set.h: Likewise.
+	* include/debug/unordered_map: Likewise.
+	* include/debug/unordered_set: Likewise.
+	* include/debug/vector: Likewise.
+	* testsuite/23_containers/map/cons/deduction.cc: New.
+	* testsuite/23_containers/multimap/cons/deduction.cc: Likewise.
+	* testsuite/23_containers/multiset/cons/deduction.cc: Likewise.
+	* testsuite/23_containers/set/cons/deduction.cc: Likewise.
+	* testsuite/23_containers/unordered_map/cons/deduction.cc: Likewise.
+	* testsuite/23_containers/unordered_multimap/cons/deduction.cc:
+	Likewise.
+	* testsuite/23_containers/unordered_multiset/cons/deduction.cc:
+	Likewise.
+	* testsuite/23_containers/unordered_set/cons/deduction.cc: Likewise.
+
+2017-10-25  Jonathan Wakely  <jwakely@redhat.com>
+
+	* doc/xml/manual/status_cxx2017.xml: Update C++17 status, and
+	information on feature-test macros.
+	* doc/html/*: Regenerate.
+
+	PR libstdc++/82716
+	* include/std/array (tuple_size, tuple_element): Change class-key
+	from class to struct, to avoid annoying Clang warnings.
+
+	PR libstdc++/79283
+	* src/filesystem/ops.cc (read_symlink): Handle st_size being zero.
+	* src/filesystem/std-ops.cc (read_symlink): Likewise.
+	(do_copy_file) [!NEED_DO_COPY_FILE]: Avoid multiple definitions.
+
+	* src/filesystem/std-path.cc (path::lexically_normal): Add missing
+	step to algorithm, for removing dot-dot elements after root-directory.
+	* testsuite/27_io/filesystem/operations/canonical.cc: Use
+	compare_paths for more exhaustive checks.
+	* testsuite/27_io/filesystem/operations/proximate.cc: Likewise.
+	* testsuite/27_io/filesystem/path/append/path.cc: Likewise.
+	* testsuite/27_io/filesystem/path/concat/path.cc: Likewise.
+	* testsuite/27_io/filesystem/path/concat/strings.cc: Fix comment.
+	* testsuite/27_io/filesystem/path/construct/locale.cc: Likewise.
+	* testsuite/27_io/filesystem/path/decompose/root_directory.cc:
+	Likewise.
+	* testsuite/27_io/filesystem/path/generation/normal.cc: Use
+	compare_paths for more exhaustive checks. Add extra testcases.
+	* testsuite/27_io/filesystem/path/generation/proximate.cc: Use
+	compare_paths for more exhaustive checks.
+	* testsuite/27_io/filesystem/path/generation/relative.cc: Likewise.
+	* testsuite/27_io/filesystem/path/generic/generic_string.cc: Remove
+	unused header.
+	* testsuite/27_io/filesystem/path/modifiers/make_preferred.cc: Fix
+	comment.
+	* testsuite/27_io/filesystem/path/modifiers/remove_filename.cc: Use
+	compare_paths for more exhaustive checks.
+	* testsuite/27_io/filesystem/path/modifiers/replace_extension.cc:
+	Likewise.
+	* testsuite/27_io/filesystem/path/modifiers/replace_filename.cc:
+	Likewise.
+	* testsuite/util/testsuite_fs.h (compare_paths): Also compare native
+	strings.
+
+	PR libstdc++/82706
+	* testsuite/27_io/filesystem/operations/permissions.cc: Fix test.
+
+2017-10-24  François Dumont  <fdumont@gcc.gnu.org>
+
+	* testsuite/lib/libstdc++.exp (check_v3_target_normal_mode): Add
+	c++config.h include.
+	(check_v3_target_normal_namespace): New.
+	* testsuite/lib/dg-options.exp (dg-require-normal-namespace): New.
+	* testsuite/18_support/headers/limits/synopsis.cc: Add latter.
+	* testsuite/20_util/from_chars/requirements.cc: Likewise.
+	* testsuite/20_util/headers/functional/synopsis.cc: Likewise.
+	* testsuite/20_util/headers/memory/synopsis.cc: Likewise.
+	* testsuite/20_util/headers/utility/synopsis.cc: Likewise.
+	* testsuite/20_util/to_chars/requirements.cc: Likewise.
+	* testsuite/21_strings/headers/string/synopsis.cc: Likewise.
+	* testsuite/22_locale/headers/locale/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/bitset/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/deque/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/forward_list/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/list/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/map/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/queue/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/set/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/stack/synopsis.cc: Likewise.
+	* testsuite/23_containers/headers/vector/synopsis.cc: Likewise.
+	* testsuite/23_containers/map/modifiers/erase/abi_tag.cc: Likewise.
+	* testsuite/23_containers/multimap/modifiers/erase/abi_tag.cc: Likewise.
+	* testsuite/23_containers/multiset/modifiers/erase/abi_tag.cc: Likewise.
+	* testsuite/23_containers/set/modifiers/erase/abi_tag.cc: Likewise.
+	* testsuite/24_iterators/headers/iterator/synopsis.cc: Likewise.
+	* testsuite/24_iterators/headers/iterator/synopsis_c++11.cc: Likewise.
+	* testsuite/24_iterators/headers/iterator/synopsis_c++14.cc: Likewise.
+	* testsuite/24_iterators/headers/iterator/synopsis_c++17.cc: Likewise.
+	* testsuite/26_numerics/complex/abi_tag.cc: Likewise.
+	* testsuite/26_numerics/headers/complex/synopsis.cc: Likewise.
+	* testsuite/26_numerics/headers/valarray/synopsis.cc: Likewise.
+	* testsuite/27_io/headers/fstream/synopsis.cc: Likewise.
+	* testsuite/27_io/headers/ios/synopsis.cc: Likewise.
+	* testsuite/27_io/headers/istream/synopsis.cc: Likewise.
+	* testsuite/27_io/headers/ostream/synopsis.cc: Likewise.
+	* testsuite/27_io/headers/sstream/synopsis.cc: Likewise.
+	* testsuite/27_io/headers/streambuf/synopsis.cc: Likewise.
+
+2017-10-24  Jonathan Wakely  <jwakely@redhat.com>
+
+	* config/locale/gnu/c_locale.cc [_GLIBCXX_LONG_DOUBLE_COMPAT]: Ignore
+	-Wattribute-alias warnings.
+	* src/c++11/istream-inst.cc: Likewise.
+	* src/c++11/locale-inst.cc: Likewise.
+	* src/c++11/ostream-inst.cc: Likewise.
+	* src/c++11/wlocale-inst.cc: Likewise.
+	* src/c++98/hash-long-double-tr1-aux.cc: Likewise.
+
+	* include/bits/string_view.tcc (find_first_of, find_last_of)
+	(find_first_not_of, find_last_not_of): Add noexcept.
+	* include/std/string_view (basic_string_view(const _CharT*))
+	(basic_string_view(const _CharT*, size_type))
+	(front, back, remove_prefix, remove_suffix, find, rfind)
+	(find_first_of, find_first_not_of): Add noexcept.
+	(at(size_type), _S_compare(size_type, size_type)): Replace conditional
+	expressions with if statements.
+	(copy(_CharT*, size_type, size_type), substr(size_type, size_type)):
+	Use _M_check for length checks.
+	(compare(basic_string_view)): Reformat.
+	(_M_check(size_type, const char)): Add noexcept(false).
+	(_M_limit(size_type, size_type)): Use noexcept not _GLIBCXX_NOEXCEPT.
+
+	PR libstdc++/82685
+	* include/experimental/string_view (operator""sv): Add noexcept.
+	* include/std/string_view (operator""sv): Likewise.
+
+2017-10-23  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/Makefile.am: Add new headers for C++17 filesystem library.
+	* include/Makefile.in: Regenerate.
+	* include/bits/fs_dir.h: New header, based on Filesystem TS code in
+	include/experimental/bits directory.
+	* include/bits/fs_fwd.h: Likewise.
+	* include/bits/fs_ops.h: Likewise.
+	* include/bits/fs_path.h: Likewise.
+	* include/experimental/bits/fs_dir.h: Rename Doxygen group.
+	* include/experimental/bits/fs_fwd.h: Likewise.
+	* include/experimental/bits/fs_ops.h: Likewise.
+	* include/experimental/bits/fs_path.h: Likewise.
+	* include/experimental/filesystem (filesystem_error::_M_gen_what):
+	Remove inline definition.
+	* include/precompiled/stdc++.h: Add <filesystem> to precompiled
+	header.
+	* include/std/filesystem: New header.
+	* python/libstdcxx/v6/printers.py: Enable printer for std::filesystem
+	paths.
+	* src/filesystem/Makefile.am: Add new files. Compile as C++17.
+	* src/filesystem/Makefile.in: Regenerate.
+	* src/filesystem/cow-dir.cc: Update comment.
+	* src/filesystem/cow-ops.cc: Likewise.
+	* src/filesystem/cow-path.cc: Likewise.
+	* src/filesystem/cow-std-dir.cc: New file.
+	* src/filesystem/cow-std-ops.cc: New file.
+	* src/filesystem/cow-std-path.cc: New file.
+	* src/filesystem/dir-common.h (_Dir_base, get_file_type): New header
+	for common code.
+	* src/filesystem/dir.cc (_Dir): Derive from _Dir_base.
+	(open_dir): Move to _Dir_base constructor.
+	(get_file_type): Move to dir-common.h.
+	(recurse): Move to _Dir_base::should_recurse.
+	* src/filesystem/ops-common.h: New header for common code.
+	* src/filesystem/ops.cc (is_set, make_file_type, make_file_status)
+	(is_not_found_errno, file_time, do_copy_file): Move to ops-common.h.
+	* src/filesystem/path.cc (filesystem_error::_M_gen_what): Define.
+	* src/filesystem/std-dir.cc: New file, based on Filesystem TS code.
+	* src/filesystem/std-ops.cc: Likewise.
+	* src/filesystem/std-dir.cc: Likewise.
+	* testsuite/27_io/filesystem/iterators/directory_iterator.cc: New
+	test.
+	* testsuite/27_io/filesystem/iterators/pop.cc: New test.
+	* testsuite/27_io/filesystem/iterators/recursive_directory_iterator.cc:
+	New test.
+	* testsuite/27_io/filesystem/operations/absolute.cc: New test.
+	* testsuite/27_io/filesystem/operations/canonical.cc: New test.
+	* testsuite/27_io/filesystem/operations/copy.cc: New test.
+	* testsuite/27_io/filesystem/operations/copy_file.cc: New test.
+	* testsuite/27_io/filesystem/operations/create_directories.cc: New
+	test.
+	* testsuite/27_io/filesystem/operations/create_directory.cc: New test.
+	* testsuite/27_io/filesystem/operations/create_symlink.cc: New test.
+	* testsuite/27_io/filesystem/operations/current_path.cc: New test.
+	* testsuite/27_io/filesystem/operations/equivalent.cc: New test.
+	* testsuite/27_io/filesystem/operations/exists.cc: New test.
+	* testsuite/27_io/filesystem/operations/file_size.cc: New test.
+	* testsuite/27_io/filesystem/operations/is_empty.cc: New test.
+	* testsuite/27_io/filesystem/operations/last_write_time.cc: New test.
+	* testsuite/27_io/filesystem/operations/permissions.cc: New test.
+	* testsuite/27_io/filesystem/operations/proximate.cc: New test.
+	* testsuite/27_io/filesystem/operations/read_symlink.cc: New test.
+	* testsuite/27_io/filesystem/operations/relative.cc: New test.
+	* testsuite/27_io/filesystem/operations/remove_all.cc: New test.
+	* testsuite/27_io/filesystem/operations/space.cc: New test.
+	* testsuite/27_io/filesystem/operations/status.cc: New test.
+	* testsuite/27_io/filesystem/operations/symlink_status.cc: New test.
+	* testsuite/27_io/filesystem/operations/temp_directory_path.cc: New
+	test.
+	* testsuite/27_io/filesystem/operations/weakly_canonical.cc: New test.
+	* testsuite/27_io/filesystem/path/append/path.cc: New test.
+	* testsuite/27_io/filesystem/path/assign/assign.cc: New test.
+	* testsuite/27_io/filesystem/path/assign/copy.cc: New test.
+	* testsuite/27_io/filesystem/path/compare/compare.cc: New test.
+	* testsuite/27_io/filesystem/path/compare/path.cc: New test.
+	* testsuite/27_io/filesystem/path/compare/strings.cc: New test.
+	* testsuite/27_io/filesystem/path/concat/path.cc: New test.
+	* testsuite/27_io/filesystem/path/concat/strings.cc: New test.
+	* testsuite/27_io/filesystem/path/construct/copy.cc: New test.
+	* testsuite/27_io/filesystem/path/construct/default.cc: New test.
+	* testsuite/27_io/filesystem/path/construct/locale.cc: New test.
+	* testsuite/27_io/filesystem/path/construct/range.cc: New test.
+	* testsuite/27_io/filesystem/path/construct/string_view.cc: New test.
+	* testsuite/27_io/filesystem/path/decompose/extension.cc: New test.
+	* testsuite/27_io/filesystem/path/decompose/filename.cc: New test.
+	* testsuite/27_io/filesystem/path/decompose/parent_path.cc: New test.
+	* testsuite/27_io/filesystem/path/decompose/relative_path.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/decompose/root_directory.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/decompose/root_name.cc: New test.
+	* testsuite/27_io/filesystem/path/decompose/root_path.cc: New test.
+	* testsuite/27_io/filesystem/path/decompose/stem.cc: New test.
+	* testsuite/27_io/filesystem/path/generation/normal.cc: New test.
+	* testsuite/27_io/filesystem/path/generation/proximate.cc: New test.
+	* testsuite/27_io/filesystem/path/generation/relative.cc: New test.
+	* testsuite/27_io/filesystem/path/generic/generic_string.cc: New test.
+	* testsuite/27_io/filesystem/path/itr/traversal.cc: New test.
+	* testsuite/27_io/filesystem/path/modifiers/clear.cc: New test.
+	* testsuite/27_io/filesystem/path/modifiers/make_preferred.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/modifiers/remove_filename.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/modifiers/replace_extension.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/modifiers/replace_filename.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/modifiers/swap.cc: New test.
+	* testsuite/27_io/filesystem/path/native/string.cc: New test.
+	* testsuite/27_io/filesystem/path/nonmember/hash_value.cc: New test.
+	* testsuite/27_io/filesystem/path/query/empty.cc: New test.
+	* testsuite/27_io/filesystem/path/query/has_extension.cc: New test.
+	* testsuite/27_io/filesystem/path/query/has_filename.cc: New test.
+	* testsuite/27_io/filesystem/path/query/has_parent_path.cc: New test.
+	* testsuite/27_io/filesystem/path/query/has_relative_path.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/query/has_root_directory.cc: New
+	test.
+	* testsuite/27_io/filesystem/path/query/has_root_name.cc: New test.
+	* testsuite/27_io/filesystem/path/query/has_root_path.cc: New test.
+	* testsuite/27_io/filesystem/path/query/has_stem.cc: New test.
+	* testsuite/27_io/filesystem/path/query/is_relative.cc: New test.
+	* testsuite/experimental/filesystem/path/construct/string_view.cc:
+	Define USE_FILESYSTEM_TS.
+	* testsuite/util/testsuite_fs.h: Allow use with C++17 paths as well
+	as Filesystem TS.
+
+	PR libstdc++/82644
+	* doc/xml/manual/intro.xml: Include new section.
+	* doc/xml/manual/status_cxxis29124.xml: New section on IS 29124
+	status.
+	* include/bits/specfun.h [__STRICT_ANSI__] (hyperg, hypergf, hypergl)
+	(conf_hyperg, conf_hypergf, conf_hypergl): Don't declare.
+	* include/c_compatibility/math.h: Import special functions into
+	global namespace for C++17.
+	* testsuite/26_numerics/headers/cmath/82644.cc: New test.
+	* testsuite/26_numerics/headers/cmath/functions_global_c++17.cc: New
+	test.
+
+2017-10-21  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/experimental/filesystem/path/itr/traversal.cc: Do not
+	increment past-the-end iterators.
+
+2017-10-20  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/chrono (__cpp_lib_chrono): Update macro value to
+	indicate support for P0505R0.
+	* testsuite/20_util/duration/arithmetic/constexpr_c++17.cc: Check
+	for updated macro.
+
+	* include/c_global/cstddef: Define __cpp_lib_byte feature-test macro.
+	* testsuite/18_support/byte/requirements.cc: Check macro.
+
+2017-10-19  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/experimental/filesystem/iterators/
+	recursive_directory_iterator.cc: Ensure that error_code arguments are
+	cleared when required.
+	* testsuite/experimental/filesystem/operations/create_directory.cc:
+	Remove redundant check.
+	* testsuite/experimental/filesystem/operations/temp_directory_path.cc:
+	Ensure that error_code argument is cleared when required.
+
+	* include/experimental/bits/fs_path.h (path::iterator++(int))
+	(path::iterator--(int)): Fix for paths with only one component.
+	* testsuite/experimental/filesystem/path/itr/traversal.cc: Test
+	post-increment and post-decrement.
+
+	* doc/xml/manual/status_cxx2017.xml: Update references to C++17
+	section numbers.
+
+	* testsuite/decimal/conversion-to-integral.cc: Use predefined macro
+	instead of non-standard glibc one.
+
+2017-10-16  François Dumont  <fdumont@gcc.gnu.org>
+
+	PR libstdc++/82558
+	* include/bits/stl_bvector.h (fill): Add iterator offset check.
+	* testsuite/23_containers/vector/bool/82558.cc: New.
+
+2017-10-13  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/82522
+	* doc/xml/manual/intro.xml: Document LWG 2354 changes.
+	* include/bits/stl_map.h (map::insert(value_type&&))
+	(map::insert(const_iterator, value_type&&)): Add overload for rvalues.
+	* include/bits/stl_multimap.h (multimap::insert(value_type&&))
+	(multimap::insert(const_iterator, value_type&&)): Likewise.
+	* include/bits/unordered_map.h (unordered_map::insert(value_type&&))
+	(unordered_map::insert(const_iterator, value_type&&))
+	(unordered_multimap::insert(value_type&&))
+	(unordered_multimap::insert(const_iterator, value_type&&)): Likewise.
+	* testsuite/23_containers/map/modifiers/insert/dr2354.cc: New test.
+	* testsuite/23_containers/multimap/modifiers/insert/dr2354.cc: New
+	test.
+	* testsuite/23_containers/unordered_map/insert/dr2354.cc: New test.
+	* testsuite/23_containers/unordered_multimap/insert/dr2354.cc: New
+	test.
+
+	PR libstdc++/82481
+	* include/std/mutex (call_once): Suppress clang-tidy warnings about
+	dangling references.
+
 2017-10-10  Jonathan Wakely  <jwakely@redhat.com>
 
 	* include/bits/streambuf_iterator.h (istreambuf_iterator::equal):
diff --git a/libstdc++-v3/config/locale/gnu/c_locale.cc b/libstdc++-v3/config/locale/gnu/c_locale.cc
index 7c334e3a3e4..9aba3c2892f 100644
--- a/libstdc++-v3/config/locale/gnu/c_locale.cc
+++ b/libstdc++-v3/config/locale/gnu/c_locale.cc
@@ -289,6 +289,7 @@ _GLIBCXX_END_NAMESPACE_VERSION
 
 // XXX GLIBCXX_ABI Deprecated
 #ifdef _GLIBCXX_LONG_DOUBLE_COMPAT
+#pragma GCC diagnostic ignored "-Wattribute-alias"
 #define _GLIBCXX_LDBL_COMPAT(dbl, ldbl) \
   extern "C" void ldbl (void) __attribute__ ((alias (#dbl)))
 _GLIBCXX_LDBL_COMPAT(_ZSt14__convert_to_vIdEvPKcRT_RSt12_Ios_IostateRKP15__locale_struct, _ZSt14__convert_to_vIeEvPKcRT_RSt12_Ios_IostateRKP15__locale_struct);
diff --git a/libstdc++-v3/doc/html/api.html b/libstdc++-v3/doc/html/api.html
index 9b5fedf5b13..04fcb83966a 100644
--- a/libstdc++-v3/doc/html/api.html
+++ b/libstdc++-v3/doc/html/api.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>The GNU C++ Library API Reference</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="bk02.html" title="" /><link rel="prev" href="bk02.html" title="" /><link rel="next" href="bk03.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">The GNU C++ Library API Reference</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bk02.html">Prev</a> </td><th width="60%" align="center"></th><td width="20%" align="right"> <a accesskey="n" href="bk03.html">Next</a></td></tr></table><hr /></div><div class="article"><div class="titlepage"><div><div><h1 class="title"><a id="api"></a>The GNU C++ Library API Reference</h1></div><div><p class="copyright">Copyright © 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>The GNU C++ Library API Reference</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="bk02.html" title="" /><link rel="prev" href="bk02.html" title="" /><link rel="next" href="bk03.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">The GNU C++ Library API Reference</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bk02.html">Prev</a> </td><th width="60%" align="center"></th><td width="20%" align="right"> <a accesskey="n" href="bk03.html">Next</a></td></tr></table><hr /></div><div class="article"><div class="titlepage"><div><div><h1 class="title"><a id="api"></a>The GNU C++ Library API Reference</h1></div><div><p class="copyright">Copyright © 
       2008
     , 
       2010
diff --git a/libstdc++-v3/doc/html/bk02.html b/libstdc++-v3/doc/html/bk02.html
index 4529fb8a230..7c47759f223 100644
--- a/libstdc++-v3/doc/html/bk02.html
+++ b/libstdc++-v3/doc/html/bk02.html
@@ -1,2 +1,2 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title></title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library" /><link rel="prev" href="manual/backwards.html" title="Backwards Compatibility" /><link rel="next" href="api.html" title="The GNU C++ Library API Reference" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center"></th></tr><tr><td width="20%" align="left"><a accesskey="p" href="manual/backwards.html">Prev</a> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="api.html">Next</a></td></tr></table><hr /></div><div class="book"><div class="titlepage"><hr /></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="article"><a href="api.html">The GNU C++ Library API Reference</a></span></dt></dl></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="manual/backwards.html">Prev</a> </td><td width="20%" align="center"> </td><td width="40%" align="right"> <a accesskey="n" href="api.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Backwards Compatibility </td><td width="20%" align="center"><a accesskey="h" href="index.html">Home</a></td><td width="40%" align="right" valign="top"> The GNU C++ Library API Reference</td></tr></table></div></body></html>
\ No newline at end of file
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title></title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library" /><link rel="prev" href="manual/backwards.html" title="Backwards Compatibility" /><link rel="next" href="api.html" title="The GNU C++ Library API Reference" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center"></th></tr><tr><td width="20%" align="left"><a accesskey="p" href="manual/backwards.html">Prev</a> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="api.html">Next</a></td></tr></table><hr /></div><div class="book"><div class="titlepage"><hr /></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="article"><a href="api.html">The GNU C++ Library API Reference</a></span></dt></dl></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="manual/backwards.html">Prev</a> </td><td width="20%" align="center"> </td><td width="40%" align="right"> <a accesskey="n" href="api.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Backwards Compatibility </td><td width="20%" align="center"><a accesskey="h" href="index.html">Home</a></td><td width="40%" align="right" valign="top"> The GNU C++ Library API Reference</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/bk03.html b/libstdc++-v3/doc/html/bk03.html
index 072e7faa59f..4bcf75adea9 100644
--- a/libstdc++-v3/doc/html/bk03.html
+++ b/libstdc++-v3/doc/html/bk03.html
@@ -1,2 +1,2 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title></title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library" /><link rel="prev" href="api.html" title="The GNU C++ Library API Reference" /><link rel="next" href="faq.html" title="Frequently Asked Questions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center"></th></tr><tr><td width="20%" align="left"><a accesskey="p" href="api.html">Prev</a> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="faq.html">Next</a></td></tr></table><hr /></div><div class="book"><div class="titlepage"><hr /></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="article"><a href="faq.html">Frequently Asked Questions</a></span></dt></dl></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="api.html">Prev</a> </td><td width="20%" align="center"> </td><td width="40%" align="right"> <a accesskey="n" href="faq.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">The GNU C++ Library API Reference </td><td width="20%" align="center"><a accesskey="h" href="index.html">Home</a></td><td width="40%" align="right" valign="top"> Frequently Asked Questions</td></tr></table></div></body></html>
\ No newline at end of file
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title></title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library" /><link rel="prev" href="api.html" title="The GNU C++ Library API Reference" /><link rel="next" href="faq.html" title="Frequently Asked Questions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center"></th></tr><tr><td width="20%" align="left"><a accesskey="p" href="api.html">Prev</a> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="faq.html">Next</a></td></tr></table><hr /></div><div class="book"><div class="titlepage"><hr /></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="article"><a href="faq.html">Frequently Asked Questions</a></span></dt></dl></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="api.html">Prev</a> </td><td width="20%" align="center"> </td><td width="40%" align="right"> <a accesskey="n" href="faq.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">The GNU C++ Library API Reference </td><td width="20%" align="center"><a accesskey="h" href="index.html">Home</a></td><td width="40%" align="right" valign="top"> Frequently Asked Questions</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/faq.html b/libstdc++-v3/doc/html/faq.html
index 34a9b2bce60..2300868af1e 100644
--- a/libstdc++-v3/doc/html/faq.html
+++ b/libstdc++-v3/doc/html/faq.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Frequently Asked Questions</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="bk03.html" title="" /><link rel="prev" href="bk03.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Frequently Asked Questions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bk03.html">Prev</a> </td><th width="60%" align="center"></th><td width="20%" align="right"> </td></tr></table><hr /></div><div class="article"><div class="titlepage"><div><div><h1 class="title"><a id="faq"></a>Frequently Asked Questions</h1></div><div><p class="copyright">Copyright © 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Frequently Asked Questions</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="up" href="bk03.html" title="" /><link rel="prev" href="bk03.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Frequently Asked Questions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bk03.html">Prev</a> </td><th width="60%" align="center"></th><td width="20%" align="right"> </td></tr></table><hr /></div><div class="article"><div class="titlepage"><div><div><h1 class="title"><a id="faq"></a>Frequently Asked Questions</h1></div><div><p class="copyright">Copyright © 
       2008-2014
      
       <a class="link" href="http://www.fsf.org" target="_top">FSF</a>
diff --git a/libstdc++-v3/doc/html/index.html b/libstdc++-v3/doc/html/index.html
index 15dc624e7f8..576c088c5a0 100644
--- a/libstdc++-v3/doc/html/index.html
+++ b/libstdc++-v3/doc/html/index.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>The GNU C++ Library</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="description" content="Short Contents Copyright (C) 2008-2017 FSF Permission is granted to copy, distribute and/or modify this document under the terms of the GNU Free Documentation License, Version 1.2 or any later version published by the Free Software Foundation; with no Invariant Sections, with no Front-Cover Texts, and with no Back-Cover Texts. This is the top level of the libstdc++ documentation set. The documentation is divided into the following three sections. Manual Frequently Asked Questions API and Source Documentation" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="next" href="manual/index.html" title="The GNU C++ Library Manual" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">The GNU C++ Library</th></tr><tr><td width="20%" align="left"> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="manual/index.html">Next</a></td></tr></table><hr /></div><div class="set" lang="en" xml:lang="en"><div class="titlepage"><div><div><h1 class="title"><a id="set-index"></a>The GNU C++ Library</h1></div><div><div class="abstract"><a id="contents"></a><p class="title"><strong>Short Contents</strong></p><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>The GNU C++ Library</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="description" content="Short Contents Copyright (C) 2008-2017 FSF Permission is granted to copy, distribute and/or modify this document under the terms of the GNU Free Documentation License, Version 1.2 or any later version published by the Free Software Foundation; with no Invariant Sections, with no Front-Cover Texts, and with no Back-Cover Texts. This is the top level of the libstdc++ documentation set. The documentation is divided into the following three sections. Manual Frequently Asked Questions API and Source Documentation" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="index.html" title="The GNU C++ Library" /><link rel="next" href="manual/index.html" title="The GNU C++ Library Manual" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">The GNU C++ Library</th></tr><tr><td width="20%" align="left"> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="manual/index.html">Next</a></td></tr></table><hr /></div><div class="set" lang="en" xml:lang="en"><div class="titlepage"><div><div><h1 class="title"><a id="set-index"></a>The GNU C++ Library</h1></div><div><div class="abstract"><a id="contents"></a><p class="title"><strong>Short Contents</strong></p><p>
       Copyright (C) 2008-2017
       <a class="link" href="http://www.fsf.org/" target="_top">FSF
       </a>
@@ -23,7 +23,7 @@
     </p></div></div></div><hr /></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="book"><a href="manual/index.html">The GNU C++ Library Manual</a></span></dt><dd><dl><dt><span class="part"><a href="manual/intro.html">I. 
   Introduction
   
-</a></span></dt><dd><dl><dt><span class="chapter"><a href="manual/status.html">1. Status</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="manual/status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="manual/status.html#status.iso.201z">C++ 201z</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.tr24733">C++ TR 24733</a></span></dt></dl></dd><dt><span class="section"><a href="manual/license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="manual/license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="manual/license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="manual/bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="manual/bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="manual/bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></dd><dt><span class="chapter"><a href="manual/setup.html">2. Setup</a></span></dt><dd><dl><dt><span class="section"><a href="manual/setup.html#manual.intro.setup.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="manual/configure.html">Configure</a></span></dt><dt><span class="section"><a href="manual/make.html">Make</a></span></dt></dl></dd><dt><span class="chapter"><a href="manual/using.html">3. Using</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using.html#manual.intro.using.flags">Command Options</a></span></dt><dt><span class="section"><a href="manual/using_headers.html">Headers</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.all">Header Files</a></span></dt><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.mixing">Mixing Headers</a></span></dt><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.cheaders">The C Headers and <code class="code">namespace std</code></a></span></dt><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.pre">Precompiled Headers</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_macros.html">Macros</a></span></dt><dt><span class="section"><a href="manual/using_dual_abi.html">Dual ABI</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_dual_abi.html#manual.intro.using.abi.trouble">Troubleshooting</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_namespaces.html">Namespaces</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_namespaces.html#manual.intro.using.namespaces.all">Available Namespaces</a></span></dt><dt><span class="section"><a href="manual/using_namespaces.html#manual.intro.using.namespaces.std">namespace std</a></span></dt><dt><span class="section"><a href="manual/using_namespaces.html#manual.intro.using.namespaces.comp">Using Namespace Composition</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_dynamic_or_shared.html">Linking</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_dynamic_or_shared.html#manual.intro.using.linkage.freestanding">Almost Nothing</a></span></dt><dt><span class="section"><a href="manual/using_dynamic_or_shared.html#manual.intro.using.linkage.dynamic">Finding Dynamic or Shared Libraries</a></span></dt><dt><span class="section"><a href="manual/using_dynamic_or_shared.html#manual.intro.using.linkage.experimental">Experimental Library Extensions</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_concurrency.html">Concurrency</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.thread_safety">Thread Safety</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.atomics">Atomics</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.io">IO</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.structure">Structure</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.defaults">Defaults</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.future">Future</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.alt">Alternatives</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.containers">Containers</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_exceptions.html">Exceptions</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.safety">Exception Safety</a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.propagating">Exception Neutrality</a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.no">Doing without</a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.compat">Compatibility</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_exceptions.html#using.exception.compat.c">With <code class="literal">C</code></a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#using.exception.compat.posix">With <code class="literal">POSIX</code> thread cancellation</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="manual/debug.html">Debugging Support</a></span></dt><dd><dl><dt><span class="section"><a href="manual/debug.html#debug.compiler">Using <span class="command"><strong>g++</strong></span></a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.req">Debug Versions of Library Binary Files</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.memory">Memory Leak Hunting</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.races">Data Race Hunting</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.gdb">Using <span class="command"><strong>gdb</strong></span></a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.exceptions">Tracking uncaught exceptions</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.debug_mode">Debug Mode</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.compile_time_checks">Compile Time Checking</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.profile_mode">Profile-based Performance Analysis</a></span></dt></dl></dd></dl></dd></dl></dd><dt><span class="part"><a href="manual/std_contents.html">II. 
+</a></span></dt><dd><dl><dt><span class="chapter"><a href="manual/status.html">1. Status</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="manual/status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="manual/status.html#status.iso.2017">C++ 2017</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="manual/status.html#status.iso.tr24733">C++ TR 24733</a></span></dt><dt><span class="section"><a href="manual/status.html#status.iso.specfun">C++ IS 29124</a></span></dt><dd><dl><dt><span class="section"><a href="manual/status.html#iso.specfun.specific">Implementation Specific Behavior</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="manual/license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="manual/license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="manual/license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="manual/bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="manual/bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="manual/bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></dd><dt><span class="chapter"><a href="manual/setup.html">2. Setup</a></span></dt><dd><dl><dt><span class="section"><a href="manual/setup.html#manual.intro.setup.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="manual/configure.html">Configure</a></span></dt><dt><span class="section"><a href="manual/make.html">Make</a></span></dt></dl></dd><dt><span class="chapter"><a href="manual/using.html">3. Using</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using.html#manual.intro.using.flags">Command Options</a></span></dt><dt><span class="section"><a href="manual/using_headers.html">Headers</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.all">Header Files</a></span></dt><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.mixing">Mixing Headers</a></span></dt><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.cheaders">The C Headers and <code class="code">namespace std</code></a></span></dt><dt><span class="section"><a href="manual/using_headers.html#manual.intro.using.headers.pre">Precompiled Headers</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_macros.html">Macros</a></span></dt><dt><span class="section"><a href="manual/using_dual_abi.html">Dual ABI</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_dual_abi.html#manual.intro.using.abi.trouble">Troubleshooting</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_namespaces.html">Namespaces</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_namespaces.html#manual.intro.using.namespaces.all">Available Namespaces</a></span></dt><dt><span class="section"><a href="manual/using_namespaces.html#manual.intro.using.namespaces.std">namespace std</a></span></dt><dt><span class="section"><a href="manual/using_namespaces.html#manual.intro.using.namespaces.comp">Using Namespace Composition</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_dynamic_or_shared.html">Linking</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_dynamic_or_shared.html#manual.intro.using.linkage.freestanding">Almost Nothing</a></span></dt><dt><span class="section"><a href="manual/using_dynamic_or_shared.html#manual.intro.using.linkage.dynamic">Finding Dynamic or Shared Libraries</a></span></dt><dt><span class="section"><a href="manual/using_dynamic_or_shared.html#manual.intro.using.linkage.experimental">Experimental Library Extensions</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_concurrency.html">Concurrency</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.thread_safety">Thread Safety</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.atomics">Atomics</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.io">IO</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.structure">Structure</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.defaults">Defaults</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.future">Future</a></span></dt><dt><span class="section"><a href="manual/using_concurrency.html#concurrency.io.alt">Alternatives</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_concurrency.html#manual.intro.using.concurrency.containers">Containers</a></span></dt></dl></dd><dt><span class="section"><a href="manual/using_exceptions.html">Exceptions</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.safety">Exception Safety</a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.propagating">Exception Neutrality</a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.no">Doing without</a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#intro.using.exception.compat">Compatibility</a></span></dt><dd><dl><dt><span class="section"><a href="manual/using_exceptions.html#using.exception.compat.c">With <code class="literal">C</code></a></span></dt><dt><span class="section"><a href="manual/using_exceptions.html#using.exception.compat.posix">With <code class="literal">POSIX</code> thread cancellation</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="manual/debug.html">Debugging Support</a></span></dt><dd><dl><dt><span class="section"><a href="manual/debug.html#debug.compiler">Using <span class="command"><strong>g++</strong></span></a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.req">Debug Versions of Library Binary Files</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.memory">Memory Leak Hunting</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.races">Data Race Hunting</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.gdb">Using <span class="command"><strong>gdb</strong></span></a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.exceptions">Tracking uncaught exceptions</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.debug_mode">Debug Mode</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.compile_time_checks">Compile Time Checking</a></span></dt><dt><span class="section"><a href="manual/debug.html#debug.profile_mode">Profile-based Performance Analysis</a></span></dt></dl></dd></dl></dd></dl></dd><dt><span class="part"><a href="manual/std_contents.html">II. 
     Standard Contents
   </a></span></dt><dd><dl><dt><span class="chapter"><a href="manual/support.html">4. 
   Support
diff --git a/libstdc++-v3/doc/html/manual/abi.html b/libstdc++-v3/doc/html/manual/abi.html
index a611f7b4014..1d217452aef 100644
--- a/libstdc++-v3/doc/html/manual/abi.html
+++ b/libstdc++-v3/doc/html/manual/abi.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>ABI Policy and Guidelines</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, ABI, version, dynamic, shared, compatibility" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="test.html" title="Testing" /><link rel="next" href="api.html" title="API Evolution and Deprecation History" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">ABI Policy and Guidelines</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="test.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>ABI Policy and Guidelines</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, ABI, version, dynamic, shared, compatibility" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="test.html" title="Testing" /><link rel="next" href="api.html" title="API Evolution and Deprecation History" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">ABI Policy and Guidelines</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="test.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
   Porting and Maintenance
   
 </th><td width="20%" align="right"> <a accesskey="n" href="api.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="appendix.porting.abi"></a>ABI Policy and Guidelines</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/algorithms.html b/libstdc++-v3/doc/html/manual/algorithms.html
index 3a04d8360ee..ecb7c15e5e3 100644
--- a/libstdc++-v3/doc/html/manual/algorithms.html
+++ b/libstdc++-v3/doc/html/manual/algorithms.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 11.  Algorithms</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library, algorithm" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="iterators.html" title="Chapter 10.  Iterators" /><link rel="next" href="numerics.html" title="Chapter 12.  Numerics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 11. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 11.  Algorithms</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library, algorithm" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="iterators.html" title="Chapter 10.  Iterators" /><link rel="next" href="numerics.html" title="Chapter 12.  Numerics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 11. 
   Algorithms
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="iterators.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/api.html b/libstdc++-v3/doc/html/manual/api.html
index 566f686b853..3745fdc3d59 100644
--- a/libstdc++-v3/doc/html/manual/api.html
+++ b/libstdc++-v3/doc/html/manual/api.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>API Evolution and Deprecation History</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, api, evolution, deprecation, history" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="abi.html" title="ABI Policy and Guidelines" /><link rel="next" href="backwards.html" title="Backwards Compatibility" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">API Evolution and Deprecation History</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="abi.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>API Evolution and Deprecation History</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, api, evolution, deprecation, history" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="abi.html" title="ABI Policy and Guidelines" /><link rel="next" href="backwards.html" title="Backwards Compatibility" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">API Evolution and Deprecation History</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="abi.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
   Porting and Maintenance
   
 </th><td width="20%" align="right"> <a accesskey="n" href="backwards.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="appendix.porting.api"></a>API Evolution and Deprecation History</h2></div></div></div><p>
@@ -77,11 +77,11 @@ _Alloc_traits</code> have been removed.
    <span class="type">__alloc</span> to select an underlying allocator that
    satisfied memory allocation requests. The selection of this
    underlying allocator was not user-configurable.
-   </p><div class="table"><a id="table.extension_allocators"></a><p class="title"><strong>Table B.6. Extension Allocators</strong></p><div class="table-contents"><table summary="Extension Allocators" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Allocator (3.4)</th><th align="left">Header (3.4)</th><th align="left">Allocator (3.[0-3])</th><th align="left">Header (3.[0-3])</th></tr></thead><tbody><tr><td align="left"><code class="classname">__gnu_cxx::new_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/new_allocator.h</code></td><td align="left"><code class="classname">std::__new_alloc</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::malloc_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/malloc_allocator.h</code></td><td align="left"><code class="classname">std::__malloc_alloc_template&lt;int&gt;</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::debug_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/debug_allocator.h</code></td><td align="left"><code class="classname">std::debug_alloc&lt;T&gt;</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::__pool_alloc&lt;T&gt;</code></td><td align="left"><code class="filename">ext/pool_allocator.h</code></td><td align="left"><code class="classname">std::__default_alloc_template&lt;bool,int&gt;</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::__mt_alloc&lt;T&gt;</code></td><td align="left"><code class="filename">ext/mt_allocator.h</code></td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left"><code class="classname">__gnu_cxx::bitmap_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/bitmap_allocator.h</code></td><td align="left"> </td><td align="left"> </td></tr></tbody></table></div></div><br class="table-break" /><p> Releases after gcc-3.4 have continued to add to the collection
+   </p><div class="table"><a id="table.extension_allocators"></a><p class="title"><strong>Table B.6. Extension Allocators</strong></p><div class="table-contents"><table class="table" summary="Extension Allocators" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Allocator (3.4)</th><th align="left">Header (3.4)</th><th align="left">Allocator (3.[0-3])</th><th align="left">Header (3.[0-3])</th></tr></thead><tbody><tr><td align="left"><code class="classname">__gnu_cxx::new_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/new_allocator.h</code></td><td align="left"><code class="classname">std::__new_alloc</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::malloc_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/malloc_allocator.h</code></td><td align="left"><code class="classname">std::__malloc_alloc_template&lt;int&gt;</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::debug_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/debug_allocator.h</code></td><td align="left"><code class="classname">std::debug_alloc&lt;T&gt;</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::__pool_alloc&lt;T&gt;</code></td><td align="left"><code class="filename">ext/pool_allocator.h</code></td><td align="left"><code class="classname">std::__default_alloc_template&lt;bool,int&gt;</code></td><td align="left"><code class="filename">memory</code></td></tr><tr><td align="left"><code class="classname">__gnu_cxx::__mt_alloc&lt;T&gt;</code></td><td align="left"><code class="filename">ext/mt_allocator.h</code></td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left"><code class="classname">__gnu_cxx::bitmap_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/bitmap_allocator.h</code></td><td align="left"> </td><td align="left"> </td></tr></tbody></table></div></div><br class="table-break" /><p> Releases after gcc-3.4 have continued to add to the collection
    of available allocators. All of these new allocators are
    standard-style. The following table includes details, along with
    the first released version of GCC that included the extension allocator.
-   </p><div class="table"><a id="table.extension_allocators2"></a><p class="title"><strong>Table B.7. Extension Allocators Continued</strong></p><div class="table-contents"><table summary="Extension Allocators Continued" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left">Allocator</th><th align="left">Include</th><th align="left">Version</th></tr></thead><tbody><tr><td align="left"><code class="classname">__gnu_cxx::array_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/array_allocator.h</code></td><td align="left">4.0.0</td></tr><tr><td align="left"><code class="classname">__gnu_cxx::throw_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/throw_allocator.h</code></td><td align="left">4.2.0</td></tr></tbody></table></div></div><br class="table-break" /><p>
+   </p><div class="table"><a id="table.extension_allocators2"></a><p class="title"><strong>Table B.7. Extension Allocators Continued</strong></p><div class="table-contents"><table class="table" summary="Extension Allocators Continued" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left">Allocator</th><th align="left">Include</th><th align="left">Version</th></tr></thead><tbody><tr><td align="left"><code class="classname">__gnu_cxx::array_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/array_allocator.h</code></td><td align="left">4.0.0</td></tr><tr><td align="left"><code class="classname">__gnu_cxx::throw_allocator&lt;T&gt;</code></td><td align="left"><code class="filename">ext/throw_allocator.h</code></td><td align="left">4.2.0</td></tr></tbody></table></div></div><br class="table-break" /><p>
 Debug mode first appears.
 </p><p>
 Precompiled header support <acronym class="acronym">PCH</acronym> support.
diff --git a/libstdc++-v3/doc/html/manual/appendix.html b/libstdc++-v3/doc/html/manual/appendix.html
index b190d7bec8b..f71165e106b 100644
--- a/libstdc++-v3/doc/html/manual/appendix.html
+++ b/libstdc++-v3/doc/html/manual/appendix.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part IV.  Appendices</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="ext_concurrency_use.html" title="Use" /><link rel="next" href="appendix_contributing.html" title="Appendix A.  Contributing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part IV. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part IV.  Appendices</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="ext_concurrency_use.html" title="Use" /><link rel="next" href="appendix_contributing.html" title="Appendix A.  Contributing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part IV. 
   Appendices
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_concurrency_use.html">Prev</a> </td><th width="60%" align="center">The GNU C++ Library Manual</th><td width="20%" align="right"> <a accesskey="n" href="appendix_contributing.html">Next</a></td></tr></table><hr /></div><div class="part"><div class="titlepage"><div><div><h1 class="title"><a id="appendix"></a>Part IV. 
   Appendices
diff --git a/libstdc++-v3/doc/html/manual/appendix_contributing.html b/libstdc++-v3/doc/html/manual/appendix_contributing.html
index 4bbae64554b..ca8ae873b9e 100644
--- a/libstdc++-v3/doc/html/manual/appendix_contributing.html
+++ b/libstdc++-v3/doc/html/manual/appendix_contributing.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix A.  Contributing</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="appendix.html" title="Part IV.  Appendices" /><link rel="next" href="source_organization.html" title="Directory Layout and Source Conventions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix A. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix A.  Contributing</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="appendix.html" title="Part IV.  Appendices" /><link rel="next" href="source_organization.html" title="Directory Layout and Source Conventions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix A. 
   Contributing
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix.html">Prev</a> </td><th width="60%" align="center">Part IV. 
diff --git a/libstdc++-v3/doc/html/manual/appendix_free.html b/libstdc++-v3/doc/html/manual/appendix_free.html
index 92b159adcd4..f7d0e5efdaa 100644
--- a/libstdc++-v3/doc/html/manual/appendix_free.html
+++ b/libstdc++-v3/doc/html/manual/appendix_free.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix C.  Free Software Needs Free Documentation</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="backwards.html" title="Backwards Compatibility" /><link rel="next" href="appendix_gpl.html" title="Appendix D.  GNU General Public License version 3" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix C. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix C.  Free Software Needs Free Documentation</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="backwards.html" title="Backwards Compatibility" /><link rel="next" href="appendix_gpl.html" title="Appendix D.  GNU General Public License version 3" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix C. 
   Free Software Needs Free Documentation
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="backwards.html">Prev</a> </td><th width="60%" align="center">Part IV. 
diff --git a/libstdc++-v3/doc/html/manual/appendix_gfdl.html b/libstdc++-v3/doc/html/manual/appendix_gfdl.html
index d0531b67dbb..6e72325f147 100644
--- a/libstdc++-v3/doc/html/manual/appendix_gfdl.html
+++ b/libstdc++-v3/doc/html/manual/appendix_gfdl.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix E. GNU Free Documentation License</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="appendix_gpl.html" title="Appendix D.  GNU General Public License version 3" /><link rel="next" href="../bk02.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix E. GNU Free Documentation License</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix_gpl.html">Prev</a> </td><th width="60%" align="center">Part IV. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix E. GNU Free Documentation License</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="appendix_gpl.html" title="Appendix D.  GNU General Public License version 3" /><link rel="next" href="../bk02.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix E. GNU Free Documentation License</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix_gpl.html">Prev</a> </td><th width="60%" align="center">Part IV. 
   Appendices
 </th><td width="20%" align="right"> <a accesskey="n" href="../bk02.html">Next</a></td></tr></table><hr /></div><div class="appendix"><div class="titlepage"><div><div><h1 class="title"><a id="appendix.gfdl-1.3"></a>GNU Free Documentation License</h1></div></div></div><p>Version 1.3, 3 November 2008</p><p>
     Copyright © 2000, 2001, 2002, 2007, 2008
diff --git a/libstdc++-v3/doc/html/manual/appendix_gpl.html b/libstdc++-v3/doc/html/manual/appendix_gpl.html
index 182545d625e..43094e87371 100644
--- a/libstdc++-v3/doc/html/manual/appendix_gpl.html
+++ b/libstdc++-v3/doc/html/manual/appendix_gpl.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix D.  GNU General Public License version 3</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="appendix_free.html" title="Appendix C.  Free Software Needs Free Documentation" /><link rel="next" href="appendix_gfdl.html" title="Appendix E. GNU Free Documentation License" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix D. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix D.  GNU General Public License version 3</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="appendix_free.html" title="Appendix C.  Free Software Needs Free Documentation" /><link rel="next" href="appendix_gfdl.html" title="Appendix E. GNU Free Documentation License" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix D. 
     <acronym class="acronym">GNU</acronym> General Public License version 3
   </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix_free.html">Prev</a> </td><th width="60%" align="center">Part IV. 
   Appendices
diff --git a/libstdc++-v3/doc/html/manual/appendix_porting.html b/libstdc++-v3/doc/html/manual/appendix_porting.html
index 97e354c9616..b3ee6517b19 100644
--- a/libstdc++-v3/doc/html/manual/appendix_porting.html
+++ b/libstdc++-v3/doc/html/manual/appendix_porting.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix B.  Porting and Maintenance</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="source_design_notes.html" title="Design Notes" /><link rel="next" href="documentation_hacking.html" title="Writing and Generating Documentation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix B. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Appendix B.  Porting and Maintenance</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix.html" title="Part IV.  Appendices" /><link rel="prev" href="source_design_notes.html" title="Design Notes" /><link rel="next" href="documentation_hacking.html" title="Writing and Generating Documentation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Appendix B. 
   Porting and Maintenance
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="source_design_notes.html">Prev</a> </td><th width="60%" align="center">Part IV. 
diff --git a/libstdc++-v3/doc/html/manual/associative.html b/libstdc++-v3/doc/html/manual/associative.html
index ddf6116ad84..88250da60d5 100644
--- a/libstdc++-v3/doc/html/manual/associative.html
+++ b/libstdc++-v3/doc/html/manual/associative.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Associative</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="containers.html" title="Chapter 9.  Containers" /><link rel="prev" href="containers.html" title="Chapter 9.  Containers" /><link rel="next" href="unordered_associative.html" title="Unordered Associative" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Associative</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="containers.html">Prev</a> </td><th width="60%" align="center">Chapter 9. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Associative</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="containers.html" title="Chapter 9.  Containers" /><link rel="prev" href="containers.html" title="Chapter 9.  Containers" /><link rel="next" href="unordered_associative.html" title="Unordered Associative" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Associative</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="containers.html">Prev</a> </td><th width="60%" align="center">Chapter 9. 
   Containers
   
 </th><td width="20%" align="right"> <a accesskey="n" href="unordered_associative.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.containers.associative"></a>Associative</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="containers.associative.insert_hints"></a>Insertion Hints</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/atomics.html b/libstdc++-v3/doc/html/manual/atomics.html
index d81b0f85cff..416bf0acc6a 100644
--- a/libstdc++-v3/doc/html/manual/atomics.html
+++ b/libstdc++-v3/doc/html/manual/atomics.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 14.  Atomics</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library, atomic" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="io_and_c.html" title="Interacting with C" /><link rel="next" href="concurrency.html" title="Chapter 15.  Concurrency" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 14. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 14.  Atomics</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library, atomic" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="io_and_c.html" title="Interacting with C" /><link rel="next" href="concurrency.html" title="Chapter 15.  Concurrency" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 14. 
   Atomics
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="io_and_c.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/backwards.html b/libstdc++-v3/doc/html/manual/backwards.html
index e69451ee693..31cbe01e6e0 100644
--- a/libstdc++-v3/doc/html/manual/backwards.html
+++ b/libstdc++-v3/doc/html/manual/backwards.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Backwards Compatibility</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, backwards" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="api.html" title="API Evolution and Deprecation History" /><link rel="next" href="appendix_free.html" title="Appendix C.  Free Software Needs Free Documentation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Backwards Compatibility</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="api.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Backwards Compatibility</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, backwards" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="api.html" title="API Evolution and Deprecation History" /><link rel="next" href="appendix_free.html" title="Appendix C.  Free Software Needs Free Documentation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Backwards Compatibility</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="api.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
   Porting and Maintenance
   
 </th><td width="20%" align="right"> <a accesskey="n" href="appendix_free.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.appendix.porting.backwards"></a>Backwards Compatibility</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="backwards.first"></a>First</h3></div></div></div><p>The first generation GNU C++ library was called libg++.  It was a
diff --git a/libstdc++-v3/doc/html/manual/bitmap_allocator.html b/libstdc++-v3/doc/html/manual/bitmap_allocator.html
index fc48f65dc5b..265bba92764 100644
--- a/libstdc++-v3/doc/html/manual/bitmap_allocator.html
+++ b/libstdc++-v3/doc/html/manual/bitmap_allocator.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 21. The bitmap_allocator</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="mt_allocator_ex_multi.html" title="Multiple Thread Example" /><link rel="next" href="bitmap_allocator_impl.html" title="Implementation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 21. The bitmap_allocator</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_ex_multi.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 21. The bitmap_allocator</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="mt_allocator_ex_multi.html" title="Multiple Thread Example" /><link rel="next" href="bitmap_allocator_impl.html" title="Implementation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 21. The bitmap_allocator</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_ex_multi.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="bitmap_allocator_impl.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.allocator.bitmap"></a>Chapter 21. The bitmap_allocator</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="bitmap_allocator.html#allocator.bitmap.design">Design</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html">Implementation</a></span></dt><dd><dl><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.free_list_store">Free List Store</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.super_block">Super Block</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.super_block_data">Super Block Data Layout</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.max_wasted">Maximum Wasted Percentage</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.allocate"><code class="function">allocate</code></a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.deallocate"><code class="function">deallocate</code></a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.questions">Questions</a></span></dt><dd><dl><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.question.1">1</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.question.2">2</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.question.3">3</a></span></dt></dl></dd><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.locality">Locality</a></span></dt><dt><span class="section"><a href="bitmap_allocator_impl.html#bitmap.impl.grow_policy">Overhead and Grow Policy</a></span></dt></dl></dd></dl></div><p>
diff --git a/libstdc++-v3/doc/html/manual/bitmap_allocator_impl.html b/libstdc++-v3/doc/html/manual/bitmap_allocator_impl.html
index 44bd192131b..44b5728e808 100644
--- a/libstdc++-v3/doc/html/manual/bitmap_allocator_impl.html
+++ b/libstdc++-v3/doc/html/manual/bitmap_allocator_impl.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="bitmap_allocator.html" title="Chapter 21. The bitmap_allocator" /><link rel="prev" href="bitmap_allocator.html" title="Chapter 21. The bitmap_allocator" /><link rel="next" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bitmap_allocator.html">Prev</a> </td><th width="60%" align="center">Chapter 21. The bitmap_allocator</th><td width="20%" align="right"> <a accesskey="n" href="policy_data_structures.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.bitmap.impl"></a>Implementation</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="bitmap.impl.free_list_store"></a>Free List Store</h3></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="bitmap_allocator.html" title="Chapter 21. The bitmap_allocator" /><link rel="prev" href="bitmap_allocator.html" title="Chapter 21. The bitmap_allocator" /><link rel="next" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bitmap_allocator.html">Prev</a> </td><th width="60%" align="center">Chapter 21. The bitmap_allocator</th><td width="20%" align="right"> <a accesskey="n" href="policy_data_structures.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.bitmap.impl"></a>Implementation</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="bitmap.impl.free_list_store"></a>Free List Store</h3></div></div></div><p>
     The Free List Store (referred to as FLS for the remaining part of this
     document) is the Global memory pool that is shared by all instances of
     the bitmapped allocator instantiated for any type. This maintains a
@@ -75,7 +75,7 @@ else return false.</p></li></ol></div><p>
   </p><p>
     Consider a block of size 64 ints. In memory, it would look like this:
     (assume a 32-bit system where, size_t is a 32-bit entity).
-  </p><div class="table"><a id="table.bitmap_alloc"></a><p class="title"><strong>Table 21.1. Bitmap Allocator Memory Map</strong></p><div class="table-contents"><table summary="Bitmap Allocator Memory Map" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left">268</td><td align="left">0</td><td align="left">4294967295</td><td align="left">4294967295</td><td align="left">Data -&gt; Space for 64 ints</td></tr></tbody></table></div></div><br class="table-break" /><p>
+  </p><div class="table"><a id="table.bitmap_alloc"></a><p class="title"><strong>Table 21.1. Bitmap Allocator Memory Map</strong></p><div class="table-contents"><table class="table" summary="Bitmap Allocator Memory Map" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left">268</td><td align="left">0</td><td align="left">4294967295</td><td align="left">4294967295</td><td align="left">Data -&gt; Space for 64 ints</td></tr></tbody></table></div></div><br class="table-break" /><p>
     The first Column(268) represents the size of the Block in bytes as
     seen by the Bitmap Allocator. Internally, a global free list is
     used to keep track of the free blocks used and given back by the
diff --git a/libstdc++-v3/doc/html/manual/bugs.html b/libstdc++-v3/doc/html/manual/bugs.html
index 1ed199e432e..a98fcc77e69 100644
--- a/libstdc++-v3/doc/html/manual/bugs.html
+++ b/libstdc++-v3/doc/html/manual/bugs.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Bugs</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="status.html" title="Chapter 1. Status" /><link rel="prev" href="license.html" title="License" /><link rel="next" href="setup.html" title="Chapter 2. Setup" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Bugs</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="license.html">Prev</a> </td><th width="60%" align="center">Chapter 1. Status</th><td width="20%" align="right"> <a accesskey="n" href="setup.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.status.bugs"></a>Bugs</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.status.bugs.impl"></a>Implementation Bugs</h3></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Bugs</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="status.html" title="Chapter 1. Status" /><link rel="prev" href="license.html" title="License" /><link rel="next" href="setup.html" title="Chapter 2. Setup" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Bugs</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="license.html">Prev</a> </td><th width="60%" align="center">Chapter 1. Status</th><td width="20%" align="right"> <a accesskey="n" href="setup.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.status.bugs"></a>Bugs</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.status.bugs.impl"></a>Implementation Bugs</h3></div></div></div><p>
       Information on known bugs, details on efforts to fix them, and
       fixed bugs are all available as part of the <a class="link" href="https://gcc.gnu.org/bugs/" target="_top">GCC bug tracking system</a>,
       under the component <span class="quote">“<span class="quote">libstdc++</span>�</span>.
@@ -430,6 +430,9 @@
     </p></dd><dt><a id="manual.bugs.dr2332"></a><span class="term"><a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2332" target="_top">2332</a>:
        <span class="emphasis"><em><code class="code">regex_iterator/regex_token_iterator</code> should forbid temporary regexes</em></span>
     </span></dt><dd><p>Add deleted constructors.
+    </p></dd><dt><a id="manual.bugs.dr2354"></a><span class="term"><a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2354" target="_top">2332</a>:
+       <span class="emphasis"><em>Unnecessary copying when inserting into maps with braced-init syntax</em></span>
+    </span></dt><dd><p>Add overloads of <code class="code">insert</code> taking <code class="code">value_type&amp;&amp;</code> rvalues.
     </p></dd><dt><a id="manual.bugs.dr2399"></a><span class="term"><a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2399" target="_top">2399</a>:
 	<span class="emphasis"><em><code class="code">shared_ptr</code>'s constructor from <code class="code">unique_ptr</code> should be constrained</em></span>
     </span></dt><dd><p>Constrain the constructor to require convertibility.
diff --git a/libstdc++-v3/doc/html/manual/concept_checking.html b/libstdc++-v3/doc/html/manual/concept_checking.html
index a01957d5e27..f2ec9735f06 100644
--- a/libstdc++-v3/doc/html/manual/concept_checking.html
+++ b/libstdc++-v3/doc/html/manual/concept_checking.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Concept Checking</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="diagnostics.html" title="Chapter 5.  Diagnostics" /><link rel="prev" href="errno.html" title="Use of errno by the library" /><link rel="next" href="utilities.html" title="Chapter 6.  Utilities" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Concept Checking</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="errno.html">Prev</a> </td><th width="60%" align="center">Chapter 5. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Concept Checking</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="diagnostics.html" title="Chapter 5.  Diagnostics" /><link rel="prev" href="errno.html" title="Use of errno by the library" /><link rel="next" href="utilities.html" title="Chapter 6.  Utilities" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Concept Checking</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="errno.html">Prev</a> </td><th width="60%" align="center">Chapter 5. 
   Diagnostics
   
 </th><td width="20%" align="right"> <a accesskey="n" href="utilities.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.diagnostics.concept_checking"></a>Concept Checking</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/concurrency.html b/libstdc++-v3/doc/html/manual/concurrency.html
index dbeb49066d7..e0c83759094 100644
--- a/libstdc++-v3/doc/html/manual/concurrency.html
+++ b/libstdc++-v3/doc/html/manual/concurrency.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 15.  Concurrency</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library, mutex, thread, future, condition_variable" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="atomics.html" title="Chapter 14.  Atomics" /><link rel="next" href="extensions.html" title="Part III.  Extensions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 15. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 15.  Concurrency</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library, mutex, thread, future, condition_variable" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="atomics.html" title="Chapter 14.  Atomics" /><link rel="next" href="extensions.html" title="Part III.  Extensions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 15. 
   Concurrency
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="atomics.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/configure.html b/libstdc++-v3/doc/html/manual/configure.html
index a54582d7a79..31187ddb869 100644
--- a/libstdc++-v3/doc/html/manual/configure.html
+++ b/libstdc++-v3/doc/html/manual/configure.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Configure</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, configure, options" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="setup.html" title="Chapter 2. Setup" /><link rel="prev" href="setup.html" title="Chapter 2. Setup" /><link rel="next" href="make.html" title="Make" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Configure</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="setup.html">Prev</a> </td><th width="60%" align="center">Chapter 2. Setup</th><td width="20%" align="right"> <a accesskey="n" href="make.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.setup.configure"></a>Configure</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Configure</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, configure, options" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="setup.html" title="Chapter 2. Setup" /><link rel="prev" href="setup.html" title="Chapter 2. Setup" /><link rel="next" href="make.html" title="Make" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Configure</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="setup.html">Prev</a> </td><th width="60%" align="center">Chapter 2. Setup</th><td width="20%" align="right"> <a accesskey="n" href="make.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.setup.configure"></a>Configure</h2></div></div></div><p>
   When configuring libstdc++, you'll have to configure the entire
   <span class="emphasis"><em>gccsrcdir</em></span> directory. Consider using the
   toplevel gcc configuration option
diff --git a/libstdc++-v3/doc/html/manual/containers.html b/libstdc++-v3/doc/html/manual/containers.html
index c668e61883d..a30aef2740e 100644
--- a/libstdc++-v3/doc/html/manual/containers.html
+++ b/libstdc++-v3/doc/html/manual/containers.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 9.  Containers</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="facets.html" title="Facets" /><link rel="next" href="associative.html" title="Associative" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 9. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 9.  Containers</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="facets.html" title="Facets" /><link rel="next" href="associative.html" title="Associative" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 9. 
   Containers
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="facets.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/containers_and_c.html b/libstdc++-v3/doc/html/manual/containers_and_c.html
index c8f44defba3..7afdab501d4 100644
--- a/libstdc++-v3/doc/html/manual/containers_and_c.html
+++ b/libstdc++-v3/doc/html/manual/containers_and_c.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Interacting with C</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="containers.html" title="Chapter 9.  Containers" /><link rel="prev" href="unordered_associative.html" title="Unordered Associative" /><link rel="next" href="iterators.html" title="Chapter 10.  Iterators" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Interacting with C</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="unordered_associative.html">Prev</a> </td><th width="60%" align="center">Chapter 9. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Interacting with C</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="containers.html" title="Chapter 9.  Containers" /><link rel="prev" href="unordered_associative.html" title="Unordered Associative" /><link rel="next" href="iterators.html" title="Chapter 10.  Iterators" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Interacting with C</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="unordered_associative.html">Prev</a> </td><th width="60%" align="center">Chapter 9. 
   Containers
   
 </th><td width="20%" align="right"> <a accesskey="n" href="iterators.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.containers.c"></a>Interacting with C</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="containers.c.vs_array"></a>Containers vs. Arrays</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/debug.html b/libstdc++-v3/doc/html/manual/debug.html
index c0dc2d29b0a..5e37b0e2476 100644
--- a/libstdc++-v3/doc/html/manual/debug.html
+++ b/libstdc++-v3/doc/html/manual/debug.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Debugging Support</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_exceptions.html" title="Exceptions" /><link rel="next" href="std_contents.html" title="Part II.  Standard Contents" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Debugging Support</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_exceptions.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="std_contents.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.debug"></a>Debugging Support</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Debugging Support</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_exceptions.html" title="Exceptions" /><link rel="next" href="std_contents.html" title="Part II.  Standard Contents" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Debugging Support</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_exceptions.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="std_contents.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.debug"></a>Debugging Support</h2></div></div></div><p>
   There are numerous things that can be done to improve the ease with
   which C++ binaries are debugged when using the GNU tool chain. Here
   are some of them.
diff --git a/libstdc++-v3/doc/html/manual/debug_mode.html b/libstdc++-v3/doc/html/manual/debug_mode.html
index 35d9c809a80..5e3882b935c 100644
--- a/libstdc++-v3/doc/html/manual/debug_mode.html
+++ b/libstdc++-v3/doc/html/manual/debug_mode.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 17. Debug Mode</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_compile_checks.html" title="Chapter 16. Compile Time Checks" /><link rel="next" href="debug_mode_semantics.html" title="Semantics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 17. Debug Mode</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_compile_checks.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 17. Debug Mode</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_compile_checks.html" title="Chapter 16. Compile Time Checks" /><link rel="next" href="debug_mode_semantics.html" title="Semantics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 17. Debug Mode</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_compile_checks.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="debug_mode_semantics.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.debug_mode"></a>Chapter 17. Debug Mode</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="debug_mode.html#manual.ext.debug_mode.intro">Intro</a></span></dt><dt><span class="section"><a href="debug_mode_semantics.html">Semantics</a></span></dt><dt><span class="section"><a href="debug_mode_using.html">Using</a></span></dt><dd><dl><dt><span class="section"><a href="debug_mode_using.html#debug_mode.using.mode">Using the Debug Mode</a></span></dt><dt><span class="section"><a href="debug_mode_using.html#debug_mode.using.specific">Using a Specific Debug Container</a></span></dt></dl></dd><dt><span class="section"><a href="debug_mode_design.html">Design</a></span></dt><dd><dl><dt><span class="section"><a href="debug_mode_design.html#debug_mode.design.goals">Goals</a></span></dt><dt><span class="section"><a href="debug_mode_design.html#debug_mode.design.methods">Methods</a></span></dt><dd><dl><dt><span class="section"><a href="debug_mode_design.html#debug_mode.design.methods.wrappers">The Wrapper Model</a></span></dt><dd><dl><dt><span class="section"><a href="debug_mode_design.html#debug_mode.design.methods.safe_iter">Safe Iterators</a></span></dt><dt><span class="section"><a href="debug_mode_design.html#debug_mode.design.methods.safe_seq">Safe Sequences (Containers)</a></span></dt></dl></dd><dt><span class="section"><a href="debug_mode_design.html#debug_mode.design.methods.precond">Precondition Checking</a></span></dt><dt><span class="section"><a href="debug_mode_design.html#debug_mode.design.methods.coexistence">Release- and debug-mode coexistence</a></span></dt><dd><dl><dt><span class="section"><a href="debug_mode_design.html#methods.coexistence.compile">Compile-time coexistence of release- and debug-mode components</a></span></dt><dt><span class="section"><a href="debug_mode_design.html#methods.coexistence.link">Link- and run-time coexistence of release- and
diff --git a/libstdc++-v3/doc/html/manual/debug_mode_design.html b/libstdc++-v3/doc/html/manual/debug_mode_design.html
index b8d391f185c..e24109f98bd 100644
--- a/libstdc++-v3/doc/html/manual/debug_mode_design.html
+++ b/libstdc++-v3/doc/html/manual/debug_mode_design.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="prev" href="debug_mode_using.html" title="Using" /><link rel="next" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode_using.html">Prev</a> </td><th width="60%" align="center">Chapter 17. Debug Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.debug_mode.design"></a>Design</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="prev" href="debug_mode_using.html" title="Using" /><link rel="next" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode_using.html">Prev</a> </td><th width="60%" align="center">Chapter 17. Debug Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.debug_mode.design"></a>Design</h2></div></div></div><p>
   </p><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="debug_mode.design.goals"></a>Goals</h3></div></div></div><p>
     </p><p> The libstdc++ debug mode replaces unsafe (but efficient) standard
   containers and iterators with semantically equivalent safe standard
diff --git a/libstdc++-v3/doc/html/manual/debug_mode_semantics.html b/libstdc++-v3/doc/html/manual/debug_mode_semantics.html
index e877ca6198a..fb84276be88 100644
--- a/libstdc++-v3/doc/html/manual/debug_mode_semantics.html
+++ b/libstdc++-v3/doc/html/manual/debug_mode_semantics.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Semantics</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="prev" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="next" href="debug_mode_using.html" title="Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Semantics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode.html">Prev</a> </td><th width="60%" align="center">Chapter 17. Debug Mode</th><td width="20%" align="right"> <a accesskey="n" href="debug_mode_using.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.debug_mode.semantics"></a>Semantics</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Semantics</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="prev" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="next" href="debug_mode_using.html" title="Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Semantics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode.html">Prev</a> </td><th width="60%" align="center">Chapter 17. Debug Mode</th><td width="20%" align="right"> <a accesskey="n" href="debug_mode_using.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.debug_mode.semantics"></a>Semantics</h2></div></div></div><p>
   </p><p>A program that uses the C++ standard library correctly
   will maintain the same semantics under debug mode as it had with
   the normal (release) library. All functional and exception-handling
diff --git a/libstdc++-v3/doc/html/manual/debug_mode_using.html b/libstdc++-v3/doc/html/manual/debug_mode_using.html
index 756d4793c3b..99142903dfb 100644
--- a/libstdc++-v3/doc/html/manual/debug_mode_using.html
+++ b/libstdc++-v3/doc/html/manual/debug_mode_using.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Using</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="prev" href="debug_mode_semantics.html" title="Semantics" /><link rel="next" href="debug_mode_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode_semantics.html">Prev</a> </td><th width="60%" align="center">Chapter 17. Debug Mode</th><td width="20%" align="right"> <a accesskey="n" href="debug_mode_design.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.debug_mode.using"></a>Using</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Using</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, debug" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="debug_mode.html" title="Chapter 17. Debug Mode" /><link rel="prev" href="debug_mode_semantics.html" title="Semantics" /><link rel="next" href="debug_mode_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode_semantics.html">Prev</a> </td><th width="60%" align="center">Chapter 17. Debug Mode</th><td width="20%" align="right"> <a accesskey="n" href="debug_mode_design.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.debug_mode.using"></a>Using</h2></div></div></div><p>
   </p><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="debug_mode.using.mode"></a>Using the Debug Mode</h3></div></div></div><p>To use the libstdc++ debug mode, compile your application with the
   compiler flag <code class="code">-D_GLIBCXX_DEBUG</code>. Note that this flag
   changes the sizes and behavior of standard class templates such
@@ -18,6 +18,6 @@
   mode or with debug mode. The
   following table provides the names and headers of the debugging
   containers:
-</p><div class="table"><a id="table.debug_mode_containers"></a><p class="title"><strong>Table 17.1. Debugging Containers</strong></p><div class="table-contents"><table summary="Debugging Containers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Container</th><th align="left">Header</th><th align="left">Debug container</th><th align="left">Debug header</th></tr></thead><tbody><tr><td align="left"><code class="classname">std::bitset</code></td><td align="left"><code class="filename">bitset</code></td><td align="left"><code class="classname">__gnu_debug::bitset</code></td><td align="left"><code class="filename">&lt;debug/bitset&gt;</code></td></tr><tr><td align="left"><code class="classname">std::deque</code></td><td align="left"><code class="filename">deque</code></td><td align="left"><code class="classname">__gnu_debug::deque</code></td><td align="left"><code class="filename">&lt;debug/deque&gt;</code></td></tr><tr><td align="left"><code class="classname">std::list</code></td><td align="left"><code class="filename">list</code></td><td align="left"><code class="classname">__gnu_debug::list</code></td><td align="left"><code class="filename">&lt;debug/list&gt;</code></td></tr><tr><td align="left"><code class="classname">std::map</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="classname">__gnu_debug::map</code></td><td align="left"><code class="filename">&lt;debug/map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::multimap</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="classname">__gnu_debug::multimap</code></td><td align="left"><code class="filename">&lt;debug/map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::multiset</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="classname">__gnu_debug::multiset</code></td><td align="left"><code class="filename">&lt;debug/set&gt;</code></td></tr><tr><td align="left"><code class="classname">std::set</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="classname">__gnu_debug::set</code></td><td align="left"><code class="filename">&lt;debug/set&gt;</code></td></tr><tr><td align="left"><code class="classname">std::string</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="classname">__gnu_debug::string</code></td><td align="left"><code class="filename">&lt;debug/string&gt;</code></td></tr><tr><td align="left"><code class="classname">std::wstring</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="classname">__gnu_debug::wstring</code></td><td align="left"><code class="filename">&lt;debug/string&gt;</code></td></tr><tr><td align="left"><code class="classname">std::basic_string</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="classname">__gnu_debug::basic_string</code></td><td align="left"><code class="filename">&lt;debug/string&gt;</code></td></tr><tr><td align="left"><code class="classname">std::vector</code></td><td align="left"><code class="filename">vector</code></td><td align="left"><code class="classname">__gnu_debug::vector</code></td><td align="left"><code class="filename">&lt;debug/vector&gt;</code></td></tr></tbody></table></div></div><br class="table-break" /><p>In addition, when compiling in C++11 mode, these additional
+</p><div class="table"><a id="table.debug_mode_containers"></a><p class="title"><strong>Table 17.1. Debugging Containers</strong></p><div class="table-contents"><table class="table" summary="Debugging Containers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Container</th><th align="left">Header</th><th align="left">Debug container</th><th align="left">Debug header</th></tr></thead><tbody><tr><td align="left"><code class="classname">std::bitset</code></td><td align="left"><code class="filename">bitset</code></td><td align="left"><code class="classname">__gnu_debug::bitset</code></td><td align="left"><code class="filename">&lt;debug/bitset&gt;</code></td></tr><tr><td align="left"><code class="classname">std::deque</code></td><td align="left"><code class="filename">deque</code></td><td align="left"><code class="classname">__gnu_debug::deque</code></td><td align="left"><code class="filename">&lt;debug/deque&gt;</code></td></tr><tr><td align="left"><code class="classname">std::list</code></td><td align="left"><code class="filename">list</code></td><td align="left"><code class="classname">__gnu_debug::list</code></td><td align="left"><code class="filename">&lt;debug/list&gt;</code></td></tr><tr><td align="left"><code class="classname">std::map</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="classname">__gnu_debug::map</code></td><td align="left"><code class="filename">&lt;debug/map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::multimap</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="classname">__gnu_debug::multimap</code></td><td align="left"><code class="filename">&lt;debug/map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::multiset</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="classname">__gnu_debug::multiset</code></td><td align="left"><code class="filename">&lt;debug/set&gt;</code></td></tr><tr><td align="left"><code class="classname">std::set</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="classname">__gnu_debug::set</code></td><td align="left"><code class="filename">&lt;debug/set&gt;</code></td></tr><tr><td align="left"><code class="classname">std::string</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="classname">__gnu_debug::string</code></td><td align="left"><code class="filename">&lt;debug/string&gt;</code></td></tr><tr><td align="left"><code class="classname">std::wstring</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="classname">__gnu_debug::wstring</code></td><td align="left"><code class="filename">&lt;debug/string&gt;</code></td></tr><tr><td align="left"><code class="classname">std::basic_string</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="classname">__gnu_debug::basic_string</code></td><td align="left"><code class="filename">&lt;debug/string&gt;</code></td></tr><tr><td align="left"><code class="classname">std::vector</code></td><td align="left"><code class="filename">vector</code></td><td align="left"><code class="classname">__gnu_debug::vector</code></td><td align="left"><code class="filename">&lt;debug/vector&gt;</code></td></tr></tbody></table></div></div><br class="table-break" /><p>In addition, when compiling in C++11 mode, these additional
 containers have additional debug capability.
-</p><div class="table"><a id="table.debug_mode_containers_cxx11"></a><p class="title"><strong>Table 17.2. Debugging Containers C++11</strong></p><div class="table-contents"><table summary="Debugging Containers C++11" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Container</th><th align="left">Header</th><th align="left">Debug container</th><th align="left">Debug header</th></tr></thead><tbody><tr><td align="left"><code class="classname">std::unordered_map</code></td><td align="left"><code class="filename">unordered_map</code></td><td align="left"><code class="classname">__gnu_debug::unordered_map</code></td><td align="left"><code class="filename">&lt;debug/unordered_map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::unordered_multimap</code></td><td align="left"><code class="filename">unordered_map</code></td><td align="left"><code class="classname">__gnu_debug::unordered_multimap</code></td><td align="left"><code class="filename">&lt;debug/unordered_map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::unordered_set</code></td><td align="left"><code class="filename">unordered_set</code></td><td align="left"><code class="classname">__gnu_debug::unordered_set</code></td><td align="left"><code class="filename">&lt;debug/unordered_set&gt;</code></td></tr><tr><td align="left"><code class="classname">std::unordered_multiset</code></td><td align="left"><code class="filename">unordered_set</code></td><td align="left"><code class="classname">__gnu_debug::unordered_multiset</code></td><td align="left"><code class="filename">&lt;debug/unordered_set&gt;</code></td></tr></tbody></table></div></div><br class="table-break" /></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="debug_mode_semantics.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="debug_mode.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="debug_mode_design.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Semantics </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Design</td></tr></table></div></body></html>
\ No newline at end of file
+</p><div class="table"><a id="table.debug_mode_containers_cxx11"></a><p class="title"><strong>Table 17.2. Debugging Containers C++11</strong></p><div class="table-contents"><table class="table" summary="Debugging Containers C++11" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Container</th><th align="left">Header</th><th align="left">Debug container</th><th align="left">Debug header</th></tr></thead><tbody><tr><td align="left"><code class="classname">std::unordered_map</code></td><td align="left"><code class="filename">unordered_map</code></td><td align="left"><code class="classname">__gnu_debug::unordered_map</code></td><td align="left"><code class="filename">&lt;debug/unordered_map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::unordered_multimap</code></td><td align="left"><code class="filename">unordered_map</code></td><td align="left"><code class="classname">__gnu_debug::unordered_multimap</code></td><td align="left"><code class="filename">&lt;debug/unordered_map&gt;</code></td></tr><tr><td align="left"><code class="classname">std::unordered_set</code></td><td align="left"><code class="filename">unordered_set</code></td><td align="left"><code class="classname">__gnu_debug::unordered_set</code></td><td align="left"><code class="filename">&lt;debug/unordered_set&gt;</code></td></tr><tr><td align="left"><code class="classname">std::unordered_multiset</code></td><td align="left"><code class="filename">unordered_set</code></td><td align="left"><code class="classname">__gnu_debug::unordered_multiset</code></td><td align="left"><code class="filename">&lt;debug/unordered_set&gt;</code></td></tr></tbody></table></div></div><br class="table-break" /></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="debug_mode_semantics.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="debug_mode.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="debug_mode_design.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Semantics </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Design</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/manual/diagnostics.html b/libstdc++-v3/doc/html/manual/diagnostics.html
index a90d5dfcc1d..67d1c13b4ad 100644
--- a/libstdc++-v3/doc/html/manual/diagnostics.html
+++ b/libstdc++-v3/doc/html/manual/diagnostics.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 5.  Diagnostics</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="termination.html" title="Termination" /><link rel="next" href="errno.html" title="Use of errno by the library" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 5. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 5.  Diagnostics</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="termination.html" title="Termination" /><link rel="next" href="errno.html" title="Use of errno by the library" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 5. 
   Diagnostics
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="termination.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/documentation_hacking.html b/libstdc++-v3/doc/html/manual/documentation_hacking.html
index f3e372603a9..274cdaca908 100644
--- a/libstdc++-v3/doc/html/manual/documentation_hacking.html
+++ b/libstdc++-v3/doc/html/manual/documentation_hacking.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Writing and Generating Documentation</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, documentation, style, docbook, doxygen" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="next" href="internals.html" title="Porting to New Hardware or Operating Systems" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Writing and Generating Documentation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix_porting.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Writing and Generating Documentation</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, documentation, style, docbook, doxygen" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="next" href="internals.html" title="Porting to New Hardware or Operating Systems" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Writing and Generating Documentation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix_porting.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
   Porting and Maintenance
   
 </th><td width="20%" align="right"> <a accesskey="n" href="internals.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="appendix.porting.doc"></a>Writing and Generating Documentation</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="doc.intro"></a>Introduction</h3></div></div></div><p>
@@ -112,7 +112,7 @@
       supported, and are always aliased to dummy rules. These
       unsupported formats are: <span class="emphasis"><em>info</em></span>,
       <span class="emphasis"><em>ps</em></span>, and <span class="emphasis"><em>dvi</em></span>.
-    </p></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="doc.doxygen"></a>Doxygen</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="doxygen.prereq"></a>Prerequisites</h4></div></div></div><div class="table"><a id="table.doxygen_prereq"></a><p class="title"><strong>Table B.1. Doxygen Prerequisites</strong></p><div class="table-contents"><table summary="Doxygen Prerequisites" border="1"><colgroup><col align="center" class="c1" /><col align="center" class="c2" /><col align="center" class="c3" /></colgroup><thead><tr><th align="center">Tool</th><th align="center">Version</th><th align="center">Required By</th></tr></thead><tbody><tr><td align="center">coreutils</td><td align="center">8.5</td><td align="center">all</td></tr><tr><td align="center">bash</td><td align="center">4.1</td><td align="center">all</td></tr><tr><td align="center">doxygen</td><td align="center">1.7.6.1</td><td align="center">all</td></tr><tr><td align="center">graphviz</td><td align="center">2.26</td><td align="center">graphical hierarchies</td></tr><tr><td align="center">pdflatex</td><td align="center">2007-59</td><td align="center">pdf output</td></tr></tbody></table></div></div><br class="table-break" /><p>
+    </p></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="doc.doxygen"></a>Doxygen</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="doxygen.prereq"></a>Prerequisites</h4></div></div></div><div class="table"><a id="table.doxygen_prereq"></a><p class="title"><strong>Table B.1. Doxygen Prerequisites</strong></p><div class="table-contents"><table class="table" summary="Doxygen Prerequisites" border="1"><colgroup><col align="center" class="c1" /><col align="center" class="c2" /><col align="center" class="c3" /></colgroup><thead><tr><th align="center">Tool</th><th align="center">Version</th><th align="center">Required By</th></tr></thead><tbody><tr><td align="center">coreutils</td><td align="center">8.5</td><td align="center">all</td></tr><tr><td align="center">bash</td><td align="center">4.1</td><td align="center">all</td></tr><tr><td align="center">doxygen</td><td align="center">1.7.6.1</td><td align="center">all</td></tr><tr><td align="center">graphviz</td><td align="center">2.26</td><td align="center">graphical hierarchies</td></tr><tr><td align="center">pdflatex</td><td align="center">2007-59</td><td align="center">pdf output</td></tr></tbody></table></div></div><br class="table-break" /><p>
 	Prerequisite tools are Bash 2.0 or later,
 	<a class="link" href="http://www.doxygen.org" target="_top">Doxygen</a>, and
 	the <a class="link" href="http://www.gnu.org/software/coreutils/" target="_top">GNU
@@ -322,7 +322,7 @@
 	writing Doxygen comments. Single and double quotes, and
 	separators in filenames are two common trouble spots. When in
 	doubt, consult the following table.
-      </p><div class="table"><a id="table.doxygen_cmp"></a><p class="title"><strong>Table B.2. HTML to Doxygen Markup Comparison</strong></p><div class="table-contents"><table summary="HTML to Doxygen Markup Comparison" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">HTML</th><th align="left">Doxygen</th></tr></thead><tbody><tr><td align="left">\</td><td align="left">\\</td></tr><tr><td align="left">"</td><td align="left">\"</td></tr><tr><td align="left">'</td><td align="left">\'</td></tr><tr><td align="left">&lt;i&gt;</td><td align="left">@a word</td></tr><tr><td align="left">&lt;b&gt;</td><td align="left">@b word</td></tr><tr><td align="left">&lt;code&gt;</td><td align="left">@c word</td></tr><tr><td align="left">&lt;em&gt;</td><td align="left">@a word</td></tr><tr><td align="left">&lt;em&gt;</td><td align="left">&lt;em&gt;two words or more&lt;/em&gt;</td></tr></tbody></table></div></div><br class="table-break" /></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="doc.docbook"></a>Docbook</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="docbook.prereq"></a>Prerequisites</h4></div></div></div><div class="table"><a id="table.docbook_prereq"></a><p class="title"><strong>Table B.3. Docbook Prerequisites</strong></p><div class="table-contents"><table summary="Docbook Prerequisites" border="1"><colgroup><col align="center" class="c1" /><col align="center" class="c2" /><col align="center" class="c3" /></colgroup><thead><tr><th align="center">Tool</th><th align="center">Version</th><th align="center">Required By</th></tr></thead><tbody><tr><td align="center">docbook5-style-xsl</td><td align="center">1.76.1</td><td align="center">all</td></tr><tr><td align="center">xsltproc</td><td align="center">1.1.26</td><td align="center">all</td></tr><tr><td align="center">xmllint</td><td align="center">2.7.7</td><td align="center">validation</td></tr><tr><td align="center">dblatex</td><td align="center">0.3</td><td align="center">pdf output</td></tr><tr><td align="center">pdflatex</td><td align="center">2007-59</td><td align="center">pdf output</td></tr><tr><td align="center">docbook2X</td><td align="center">0.8.8</td><td align="center">info output</td></tr><tr><td align="center">epub3 stylesheets</td><td align="center">b3</td><td align="center">epub output</td></tr></tbody></table></div></div><br class="table-break" /><p>
+      </p><div class="table"><a id="table.doxygen_cmp"></a><p class="title"><strong>Table B.2. HTML to Doxygen Markup Comparison</strong></p><div class="table-contents"><table class="table" summary="HTML to Doxygen Markup Comparison" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">HTML</th><th align="left">Doxygen</th></tr></thead><tbody><tr><td align="left">\</td><td align="left">\\</td></tr><tr><td align="left">"</td><td align="left">\"</td></tr><tr><td align="left">'</td><td align="left">\'</td></tr><tr><td align="left">&lt;i&gt;</td><td align="left">@a word</td></tr><tr><td align="left">&lt;b&gt;</td><td align="left">@b word</td></tr><tr><td align="left">&lt;code&gt;</td><td align="left">@c word</td></tr><tr><td align="left">&lt;em&gt;</td><td align="left">@a word</td></tr><tr><td align="left">&lt;em&gt;</td><td align="left">&lt;em&gt;two words or more&lt;/em&gt;</td></tr></tbody></table></div></div><br class="table-break" /></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="doc.docbook"></a>Docbook</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="docbook.prereq"></a>Prerequisites</h4></div></div></div><div class="table"><a id="table.docbook_prereq"></a><p class="title"><strong>Table B.3. Docbook Prerequisites</strong></p><div class="table-contents"><table class="table" summary="Docbook Prerequisites" border="1"><colgroup><col align="center" class="c1" /><col align="center" class="c2" /><col align="center" class="c3" /></colgroup><thead><tr><th align="center">Tool</th><th align="center">Version</th><th align="center">Required By</th></tr></thead><tbody><tr><td align="center">docbook5-style-xsl</td><td align="center">1.76.1</td><td align="center">all</td></tr><tr><td align="center">xsltproc</td><td align="center">1.1.26</td><td align="center">all</td></tr><tr><td align="center">xmllint</td><td align="center">2.7.7</td><td align="center">validation</td></tr><tr><td align="center">dblatex</td><td align="center">0.3</td><td align="center">pdf output</td></tr><tr><td align="center">pdflatex</td><td align="center">2007-59</td><td align="center">pdf output</td></tr><tr><td align="center">docbook2X</td><td align="center">0.8.8</td><td align="center">info output</td></tr><tr><td align="center">epub3 stylesheets</td><td align="center">b3</td><td align="center">epub output</td></tr></tbody></table></div></div><br class="table-break" /><p>
 	Editing the DocBook sources requires an XML editor. Many
 	exist: some notable options
 	include <span class="command"><strong>emacs</strong></span>, <span class="application">Kate</span>,
@@ -530,11 +530,11 @@ make <code class="literal">XSL_STYLE_DIR="/usr/share/xml/docbook/stylesheet/nwal
 	<a class="link" href="http://tdg.docbook.org/tdg/5.0/ref-elements.html" target="_top">DocBook Element Reference</a>.
 	An incomplete reference for HTML to Docbook conversion is
 	detailed in the table below.
-      </p><div class="table"><a id="table.docbook_cmp"></a><p class="title"><strong>Table B.4. HTML to Docbook XML Markup Comparison</strong></p><div class="table-contents"><table summary="HTML to Docbook XML Markup Comparison" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">HTML</th><th align="left">Docbook</th></tr></thead><tbody><tr><td align="left">&lt;p&gt;</td><td align="left">&lt;para&gt;</td></tr><tr><td align="left">&lt;pre&gt;</td><td align="left">&lt;computeroutput&gt;, &lt;programlisting&gt;,
+      </p><div class="table"><a id="table.docbook_cmp"></a><p class="title"><strong>Table B.4. HTML to Docbook XML Markup Comparison</strong></p><div class="table-contents"><table class="table" summary="HTML to Docbook XML Markup Comparison" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">HTML</th><th align="left">Docbook</th></tr></thead><tbody><tr><td align="left">&lt;p&gt;</td><td align="left">&lt;para&gt;</td></tr><tr><td align="left">&lt;pre&gt;</td><td align="left">&lt;computeroutput&gt;, &lt;programlisting&gt;,
 	&lt;literallayout&gt;</td></tr><tr><td align="left">&lt;ul&gt;</td><td align="left">&lt;itemizedlist&gt;</td></tr><tr><td align="left">&lt;ol&gt;</td><td align="left">&lt;orderedlist&gt;</td></tr><tr><td align="left">&lt;il&gt;</td><td align="left">&lt;listitem&gt;</td></tr><tr><td align="left">&lt;dl&gt;</td><td align="left">&lt;variablelist&gt;</td></tr><tr><td align="left">&lt;dt&gt;</td><td align="left">&lt;term&gt;</td></tr><tr><td align="left">&lt;dd&gt;</td><td align="left">&lt;listitem&gt;</td></tr><tr><td align="left">&lt;a href=""&gt;</td><td align="left">&lt;ulink url=""&gt;</td></tr><tr><td align="left">&lt;code&gt;</td><td align="left">&lt;literal&gt;, &lt;programlisting&gt;</td></tr><tr><td align="left">&lt;strong&gt;</td><td align="left">&lt;emphasis&gt;</td></tr><tr><td align="left">&lt;em&gt;</td><td align="left">&lt;emphasis&gt;</td></tr><tr><td align="left">"</td><td align="left">&lt;quote&gt;</td></tr></tbody></table></div></div><br class="table-break" /><p>
   And examples of detailed markup for which there are no real HTML
   equivalents are listed in the table below.
-</p><div class="table"><a id="table.docbook_elem"></a><p class="title"><strong>Table B.5. Docbook XML Element Use</strong></p><div class="table-contents"><table summary="Docbook XML Element Use" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">Element</th><th align="left">Use</th></tr></thead><tbody><tr><td align="left">&lt;structname&gt;</td><td align="left">&lt;structname&gt;char_traits&lt;/structname&gt;</td></tr><tr><td align="left">&lt;classname&gt;</td><td align="left">&lt;classname&gt;string&lt;/classname&gt;</td></tr><tr><td align="left">&lt;function&gt;</td><td align="left">
+</p><div class="table"><a id="table.docbook_elem"></a><p class="title"><strong>Table B.5. Docbook XML Element Use</strong></p><div class="table-contents"><table class="table" summary="Docbook XML Element Use" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">Element</th><th align="left">Use</th></tr></thead><tbody><tr><td align="left">&lt;structname&gt;</td><td align="left">&lt;structname&gt;char_traits&lt;/structname&gt;</td></tr><tr><td align="left">&lt;classname&gt;</td><td align="left">&lt;classname&gt;string&lt;/classname&gt;</td></tr><tr><td align="left">&lt;function&gt;</td><td align="left">
 	<p>&lt;function&gt;clear()&lt;/function&gt;</p>
 	<p>&lt;function&gt;fs.clear()&lt;/function&gt;</p>
       </td></tr><tr><td align="left">&lt;type&gt;</td><td align="left">&lt;type&gt;long long&lt;/type&gt;</td></tr><tr><td align="left">&lt;varname&gt;</td><td align="left">&lt;varname&gt;fs&lt;/varname&gt;</td></tr><tr><td align="left">&lt;literal&gt;</td><td align="left">
diff --git a/libstdc++-v3/doc/html/manual/dynamic_memory.html b/libstdc++-v3/doc/html/manual/dynamic_memory.html
index 3bd6ca10e23..57599117cb8 100644
--- a/libstdc++-v3/doc/html/manual/dynamic_memory.html
+++ b/libstdc++-v3/doc/html/manual/dynamic_memory.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Dynamic Memory</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="support.html" title="Chapter 4.  Support" /><link rel="prev" href="support.html" title="Chapter 4.  Support" /><link rel="next" href="termination.html" title="Termination" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Dynamic Memory</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="support.html">Prev</a> </td><th width="60%" align="center">Chapter 4. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Dynamic Memory</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="support.html" title="Chapter 4.  Support" /><link rel="prev" href="support.html" title="Chapter 4.  Support" /><link rel="next" href="termination.html" title="Termination" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Dynamic Memory</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="support.html">Prev</a> </td><th width="60%" align="center">Chapter 4. 
   Support
   
 </th><td width="20%" align="right"> <a accesskey="n" href="termination.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.support.memory"></a>Dynamic Memory</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/errno.html b/libstdc++-v3/doc/html/manual/errno.html
index 5daeda73ac9..a6d6128efbc 100644
--- a/libstdc++-v3/doc/html/manual/errno.html
+++ b/libstdc++-v3/doc/html/manual/errno.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Use of errno by the library</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="diagnostics.html" title="Chapter 5.  Diagnostics" /><link rel="prev" href="diagnostics.html" title="Chapter 5.  Diagnostics" /><link rel="next" href="concept_checking.html" title="Concept Checking" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Use of errno by the library</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="diagnostics.html">Prev</a> </td><th width="60%" align="center">Chapter 5. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Use of errno by the library</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="diagnostics.html" title="Chapter 5.  Diagnostics" /><link rel="prev" href="diagnostics.html" title="Chapter 5.  Diagnostics" /><link rel="next" href="concept_checking.html" title="Concept Checking" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Use of errno by the library</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="diagnostics.html">Prev</a> </td><th width="60%" align="center">Chapter 5. 
   Diagnostics
   
 </th><td width="20%" align="right"> <a accesskey="n" href="concept_checking.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.diagnostics.errno"></a>Use of errno by the library</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/ext_algorithms.html b/libstdc++-v3/doc/html/manual/ext_algorithms.html
index 2ff989e2f72..805734fefbe 100644
--- a/libstdc++-v3/doc/html/manual/ext_algorithms.html
+++ b/libstdc++-v3/doc/html/manual/ext_algorithms.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 25. Algorithms</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_utilities.html" title="Chapter 24. Utilities" /><link rel="next" href="ext_numerics.html" title="Chapter 26. Numerics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 25. Algorithms</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_utilities.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 25. Algorithms</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_utilities.html" title="Chapter 24. Utilities" /><link rel="next" href="ext_numerics.html" title="Chapter 26. Numerics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 25. Algorithms</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_utilities.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_numerics.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.algorithms"></a>Chapter 25. Algorithms</h2></div></div></div><p>25.1.6 (count, count_if) is extended with two more versions of count
diff --git a/libstdc++-v3/doc/html/manual/ext_compile_checks.html b/libstdc++-v3/doc/html/manual/ext_compile_checks.html
index 227bd25d541..c60b03381ba 100644
--- a/libstdc++-v3/doc/html/manual/ext_compile_checks.html
+++ b/libstdc++-v3/doc/html/manual/ext_compile_checks.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 16. Compile Time Checks</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_preface.html" title="" /><link rel="next" href="debug_mode.html" title="Chapter 17. Debug Mode" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 16. Compile Time Checks</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_preface.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 16. Compile Time Checks</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_preface.html" title="" /><link rel="next" href="debug_mode.html" title="Chapter 17. Debug Mode" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 16. Compile Time Checks</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_preface.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="debug_mode.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.compile_checks"></a>Chapter 16. Compile Time Checks</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/ext_concurrency.html b/libstdc++-v3/doc/html/manual/ext_concurrency.html
index 688bdd06a97..5d9b33204c2 100644
--- a/libstdc++-v3/doc/html/manual/ext_concurrency.html
+++ b/libstdc++-v3/doc/html/manual/ext_concurrency.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 30. Concurrency</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_demangling.html" title="Chapter 29. Demangling" /><link rel="next" href="ext_concurrency_impl.html" title="Implementation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 30. Concurrency</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_demangling.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 30. Concurrency</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_demangling.html" title="Chapter 29. Demangling" /><link rel="next" href="ext_concurrency_impl.html" title="Implementation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 30. Concurrency</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_demangling.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_concurrency_impl.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.concurrency"></a>Chapter 30. Concurrency</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="ext_concurrency.html#manual.ext.concurrency.design">Design</a></span></dt><dd><dl><dt><span class="section"><a href="ext_concurrency.html#manual.ext.concurrency.design.threads">Interface to Locks and Mutexes</a></span></dt><dt><span class="section"><a href="ext_concurrency.html#manual.ext.concurrency.design.atomics">Interface to Atomic Functions</a></span></dt></dl></dd><dt><span class="section"><a href="ext_concurrency_impl.html">Implementation</a></span></dt><dd><dl><dt><span class="section"><a href="ext_concurrency_impl.html#manual.ext.concurrency.impl.atomic_fallbacks">Using Built-in Atomic Functions</a></span></dt><dt><span class="section"><a href="ext_concurrency_impl.html#manual.ext.concurrency.impl.thread">Thread Abstraction</a></span></dt></dl></dd><dt><span class="section"><a href="ext_concurrency_use.html">Use</a></span></dt></dl></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.concurrency.design"></a>Design</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.concurrency.design.threads"></a>Interface to Locks and Mutexes</h3></div></div></div><p>The file <code class="filename">&lt;ext/concurrence.h&gt;</code>
diff --git a/libstdc++-v3/doc/html/manual/ext_concurrency_impl.html b/libstdc++-v3/doc/html/manual/ext_concurrency_impl.html
index bb5d144e056..5fecd4afec0 100644
--- a/libstdc++-v3/doc/html/manual/ext_concurrency_impl.html
+++ b/libstdc++-v3/doc/html/manual/ext_concurrency_impl.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="ext_concurrency.html" title="Chapter 30. Concurrency" /><link rel="prev" href="ext_concurrency.html" title="Chapter 30. Concurrency" /><link rel="next" href="ext_concurrency_use.html" title="Use" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_concurrency.html">Prev</a> </td><th width="60%" align="center">Chapter 30. Concurrency</th><td width="20%" align="right"> <a accesskey="n" href="ext_concurrency_use.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.concurrency.impl"></a>Implementation</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.concurrency.impl.atomic_fallbacks"></a>Using Built-in Atomic Functions</h3></div></div></div><p>The functions for atomic operations described above are either
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="ext_concurrency.html" title="Chapter 30. Concurrency" /><link rel="prev" href="ext_concurrency.html" title="Chapter 30. Concurrency" /><link rel="next" href="ext_concurrency_use.html" title="Use" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_concurrency.html">Prev</a> </td><th width="60%" align="center">Chapter 30. Concurrency</th><td width="20%" align="right"> <a accesskey="n" href="ext_concurrency_use.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.concurrency.impl"></a>Implementation</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.concurrency.impl.atomic_fallbacks"></a>Using Built-in Atomic Functions</h3></div></div></div><p>The functions for atomic operations described above are either
 implemented via compiler intrinsics (if the underlying host is
 capable) or by library fallbacks.</p><p>Compiler intrinsics (builtins) are always preferred.  However, as
 the compiler builtins for atomics are not universally implemented,
diff --git a/libstdc++-v3/doc/html/manual/ext_concurrency_use.html b/libstdc++-v3/doc/html/manual/ext_concurrency_use.html
index 646ef624d17..85c85b4a7b2 100644
--- a/libstdc++-v3/doc/html/manual/ext_concurrency_use.html
+++ b/libstdc++-v3/doc/html/manual/ext_concurrency_use.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Use</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="ext_concurrency.html" title="Chapter 30. Concurrency" /><link rel="prev" href="ext_concurrency_impl.html" title="Implementation" /><link rel="next" href="appendix.html" title="Part IV.  Appendices" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Use</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_concurrency_impl.html">Prev</a> </td><th width="60%" align="center">Chapter 30. Concurrency</th><td width="20%" align="right"> <a accesskey="n" href="appendix.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.concurrency.use"></a>Use</h2></div></div></div><p>Typical usage of the last two constructs is demonstrated as follows:
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Use</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="ext_concurrency.html" title="Chapter 30. Concurrency" /><link rel="prev" href="ext_concurrency_impl.html" title="Implementation" /><link rel="next" href="appendix.html" title="Part IV.  Appendices" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Use</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_concurrency_impl.html">Prev</a> </td><th width="60%" align="center">Chapter 30. Concurrency</th><td width="20%" align="right"> <a accesskey="n" href="appendix.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.concurrency.use"></a>Use</h2></div></div></div><p>Typical usage of the last two constructs is demonstrated as follows:
 </p><pre class="programlisting">
 #include &lt;ext/concurrence.h&gt;
 
diff --git a/libstdc++-v3/doc/html/manual/ext_containers.html b/libstdc++-v3/doc/html/manual/ext_containers.html
index 224da127ffc..a75b7939116 100644
--- a/libstdc++-v3/doc/html/manual/ext_containers.html
+++ b/libstdc++-v3/doc/html/manual/ext_containers.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 23. HP/SGI Extensions</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="policy_data_structures_ack.html" title="Acknowledgments" /><link rel="next" href="ext_sgi.html" title="Deprecated" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 23. HP/SGI Extensions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures_ack.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 23. HP/SGI Extensions</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="policy_data_structures_ack.html" title="Acknowledgments" /><link rel="next" href="ext_sgi.html" title="Deprecated" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 23. HP/SGI Extensions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures_ack.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_sgi.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.containers"></a>Chapter 23. HP/SGI Extensions</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="ext_containers.html#manual.ext.containers.sgi">Backwards Compatibility</a></span></dt><dt><span class="section"><a href="ext_sgi.html">Deprecated</a></span></dt></dl></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.containers.sgi"></a>Backwards Compatibility</h2></div></div></div><p>A few extensions and nods to backwards-compatibility have
diff --git a/libstdc++-v3/doc/html/manual/ext_demangling.html b/libstdc++-v3/doc/html/manual/ext_demangling.html
index 7d6df2d8c41..cbca6c0ee64 100644
--- a/libstdc++-v3/doc/html/manual/ext_demangling.html
+++ b/libstdc++-v3/doc/html/manual/ext_demangling.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 29. Demangling</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_io.html" title="Chapter 28. Input and Output" /><link rel="next" href="ext_concurrency.html" title="Chapter 30. Concurrency" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 29. Demangling</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_io.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 29. Demangling</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_io.html" title="Chapter 28. Input and Output" /><link rel="next" href="ext_concurrency.html" title="Chapter 30. Concurrency" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 29. Demangling</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_io.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_concurrency.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.demangle"></a>Chapter 29. Demangling</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/ext_io.html b/libstdc++-v3/doc/html/manual/ext_io.html
index c99a49f5b12..7e24bedef7f 100644
--- a/libstdc++-v3/doc/html/manual/ext_io.html
+++ b/libstdc++-v3/doc/html/manual/ext_io.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 28. Input and Output</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_iterators.html" title="Chapter 27. Iterators" /><link rel="next" href="ext_demangling.html" title="Chapter 29. Demangling" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 28. Input and Output</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_iterators.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 28. Input and Output</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_iterators.html" title="Chapter 27. Iterators" /><link rel="next" href="ext_demangling.html" title="Chapter 29. Demangling" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 28. Input and Output</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_iterators.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_demangling.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.io"></a>Chapter 28. Input and Output</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="ext_io.html#manual.ext.io.filebuf_derived">Derived filebufs</a></span></dt></dl></div><p>
diff --git a/libstdc++-v3/doc/html/manual/ext_iterators.html b/libstdc++-v3/doc/html/manual/ext_iterators.html
index 83c07801700..b349aab81ad 100644
--- a/libstdc++-v3/doc/html/manual/ext_iterators.html
+++ b/libstdc++-v3/doc/html/manual/ext_iterators.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 27. Iterators</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_numerics.html" title="Chapter 26. Numerics" /><link rel="next" href="ext_io.html" title="Chapter 28. Input and Output" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 27. Iterators</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_numerics.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 27. Iterators</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_numerics.html" title="Chapter 26. Numerics" /><link rel="next" href="ext_io.html" title="Chapter 28. Input and Output" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 27. Iterators</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_numerics.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_io.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.iterators"></a>Chapter 27. Iterators</h2></div></div></div><p>24.3.2 describes <code class="code">struct iterator</code>, which didn't exist in the
diff --git a/libstdc++-v3/doc/html/manual/ext_numerics.html b/libstdc++-v3/doc/html/manual/ext_numerics.html
index 720eccfcf2d..4689c95a61d 100644
--- a/libstdc++-v3/doc/html/manual/ext_numerics.html
+++ b/libstdc++-v3/doc/html/manual/ext_numerics.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 26. Numerics</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_algorithms.html" title="Chapter 25. Algorithms" /><link rel="next" href="ext_iterators.html" title="Chapter 27. Iterators" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 26. Numerics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_algorithms.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 26. Numerics</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_algorithms.html" title="Chapter 25. Algorithms" /><link rel="next" href="ext_iterators.html" title="Chapter 27. Iterators" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 26. Numerics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_algorithms.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_iterators.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.numerics"></a>Chapter 26. Numerics</h2></div></div></div><p>26.4, the generalized numeric operations such as <code class="code">accumulate</code>,
diff --git a/libstdc++-v3/doc/html/manual/ext_preface.html b/libstdc++-v3/doc/html/manual/ext_preface.html
index 7913fde2f7a..0de4378a6c1 100644
--- a/libstdc++-v3/doc/html/manual/ext_preface.html
+++ b/libstdc++-v3/doc/html/manual/ext_preface.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title></title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="extensions.html" title="Part III.  Extensions" /><link rel="next" href="ext_compile_checks.html" title="Chapter 16. Compile Time Checks" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center"></th></tr><tr><td width="20%" align="left"><a accesskey="p" href="extensions.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title></title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="extensions.html" title="Part III.  Extensions" /><link rel="next" href="ext_compile_checks.html" title="Chapter 16. Compile Time Checks" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center"></th></tr><tr><td width="20%" align="left"><a accesskey="p" href="extensions.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_compile_checks.html">Next</a></td></tr></table><hr /></div><div class="preface"><div class="titlepage"><div><div><h1 class="title"><a id="manual.ext.preface"></a></h1></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/ext_sgi.html b/libstdc++-v3/doc/html/manual/ext_sgi.html
index a39ff6caf28..d2013410045 100644
--- a/libstdc++-v3/doc/html/manual/ext_sgi.html
+++ b/libstdc++-v3/doc/html/manual/ext_sgi.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Deprecated</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="ext_containers.html" title="Chapter 23. HP/SGI Extensions" /><link rel="prev" href="ext_containers.html" title="Chapter 23. HP/SGI Extensions" /><link rel="next" href="ext_utilities.html" title="Chapter 24. Utilities" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Deprecated</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_containers.html">Prev</a> </td><th width="60%" align="center">Chapter 23. HP/SGI Extensions</th><td width="20%" align="right"> <a accesskey="n" href="ext_utilities.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.containers.deprecated_sgi"></a>Deprecated</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Deprecated</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="ext_containers.html" title="Chapter 23. HP/SGI Extensions" /><link rel="prev" href="ext_containers.html" title="Chapter 23. HP/SGI Extensions" /><link rel="next" href="ext_utilities.html" title="Chapter 24. Utilities" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Deprecated</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_containers.html">Prev</a> </td><th width="60%" align="center">Chapter 23. HP/SGI Extensions</th><td width="20%" align="right"> <a accesskey="n" href="ext_utilities.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.containers.deprecated_sgi"></a>Deprecated</h2></div></div></div><p>
      The SGI hashing classes <code class="classname">hash_set</code> and
      <code class="classname">hash_set</code> have been deprecated by the
      unordered_set, unordered_multiset, unordered_map,
diff --git a/libstdc++-v3/doc/html/manual/ext_utilities.html b/libstdc++-v3/doc/html/manual/ext_utilities.html
index 01c86714f45..6200b01e30c 100644
--- a/libstdc++-v3/doc/html/manual/ext_utilities.html
+++ b/libstdc++-v3/doc/html/manual/ext_utilities.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 24. Utilities</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_sgi.html" title="Deprecated" /><link rel="next" href="ext_algorithms.html" title="Chapter 25. Algorithms" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 24. Utilities</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_sgi.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 24. Utilities</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="ext_sgi.html" title="Deprecated" /><link rel="next" href="ext_algorithms.html" title="Chapter 25. Algorithms" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 24. Utilities</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="ext_sgi.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="ext_algorithms.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.util"></a>Chapter 24. Utilities</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/extensions.html b/libstdc++-v3/doc/html/manual/extensions.html
index ac3e1a96101..57dd1132c1d 100644
--- a/libstdc++-v3/doc/html/manual/extensions.html
+++ b/libstdc++-v3/doc/html/manual/extensions.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part III.  Extensions</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="io_and_c.html" title="Interacting with C" /><link rel="next" href="ext_preface.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part III.  Extensions</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="io_and_c.html" title="Interacting with C" /><link rel="next" href="ext_preface.html" title="" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part III. 
   Extensions
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="io_and_c.html">Prev</a> </td><th width="60%" align="center">The GNU C++ Library Manual</th><td width="20%" align="right"> <a accesskey="n" href="ext_preface.html">Next</a></td></tr></table><hr /></div><div class="part"><div class="titlepage"><div><div><h1 class="title"><a id="manual.ext"></a>Part III. 
diff --git a/libstdc++-v3/doc/html/manual/facets.html b/libstdc++-v3/doc/html/manual/facets.html
index 0a5b79e2be2..c02dea78c15 100644
--- a/libstdc++-v3/doc/html/manual/facets.html
+++ b/libstdc++-v3/doc/html/manual/facets.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Facets</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="localization.html" title="Chapter 8.  Localization" /><link rel="prev" href="localization.html" title="Chapter 8.  Localization" /><link rel="next" href="containers.html" title="Chapter 9.  Containers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Facets</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="localization.html">Prev</a> </td><th width="60%" align="center">Chapter 8. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Facets</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="localization.html" title="Chapter 8.  Localization" /><link rel="prev" href="localization.html" title="Chapter 8.  Localization" /><link rel="next" href="containers.html" title="Chapter 9.  Containers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Facets</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="localization.html">Prev</a> </td><th width="60%" align="center">Chapter 8. 
   Localization
   
 </th><td width="20%" align="right"> <a accesskey="n" href="containers.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.localization.facet"></a>Facets</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="std.localization.facet.ctype"></a>ctype</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="facet.ctype.impl"></a>Implementation</h4></div></div></div><div class="section"><div class="titlepage"><div><div><h5 class="title"><a id="facet.ctype.impl.spec"></a>Specializations</h5></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/fstreams.html b/libstdc++-v3/doc/html/manual/fstreams.html
index 897057600f9..f3d56e60ed7 100644
--- a/libstdc++-v3/doc/html/manual/fstreams.html
+++ b/libstdc++-v3/doc/html/manual/fstreams.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>File Based Streams</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="stringstreams.html" title="Memory Based Streams" /><link rel="next" href="io_and_c.html" title="Interacting with C" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">File Based Streams</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="stringstreams.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>File Based Streams</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="stringstreams.html" title="Memory Based Streams" /><link rel="next" href="io_and_c.html" title="Interacting with C" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">File Based Streams</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="stringstreams.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
   Input and Output
   
 </th><td width="20%" align="right"> <a accesskey="n" href="io_and_c.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.io.filestreams"></a>File Based Streams</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="std.io.filestreams.copying_a_file"></a>Copying a File</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/generalized_numeric_operations.html b/libstdc++-v3/doc/html/manual/generalized_numeric_operations.html
index 7d0e03994a1..8aa676040b5 100644
--- a/libstdc++-v3/doc/html/manual/generalized_numeric_operations.html
+++ b/libstdc++-v3/doc/html/manual/generalized_numeric_operations.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Generalized Operations</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="numerics.html" title="Chapter 12.  Numerics" /><link rel="prev" href="numerics.html" title="Chapter 12.  Numerics" /><link rel="next" href="numerics_and_c.html" title="Interacting with C" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Generalized Operations</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="numerics.html">Prev</a> </td><th width="60%" align="center">Chapter 12. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Generalized Operations</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="numerics.html" title="Chapter 12.  Numerics" /><link rel="prev" href="numerics.html" title="Chapter 12.  Numerics" /><link rel="next" href="numerics_and_c.html" title="Interacting with C" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Generalized Operations</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="numerics.html">Prev</a> </td><th width="60%" align="center">Chapter 12. 
   Numerics
   
 </th><td width="20%" align="right"> <a accesskey="n" href="numerics_and_c.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.numerics.generalized_ops"></a>Generalized Operations</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/index.html b/libstdc++-v3/doc/html/manual/index.html
index 2d6fbbafe48..ec39379387d 100644
--- a/libstdc++-v3/doc/html/manual/index.html
+++ b/libstdc++-v3/doc/html/manual/index.html
@@ -1,10 +1,10 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>The GNU C++ Library Manual</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="../index.html" title="The GNU C++ Library" /><link rel="prev" href="../index.html" title="The GNU C++ Library" /><link rel="next" href="intro.html" title="Part I.  Introduction" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">The GNU C++ Library Manual</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="../index.html">Prev</a> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="intro.html">Next</a></td></tr></table><hr /></div><div class="book"><div class="titlepage"><div><div><h1 class="title"><a id="manual"></a>The GNU C++ Library Manual</h1></div><div><div class="authorgroup"><div class="author"><h3 class="author"><span class="firstname"></span> <span class="surname"></span></h3></div><div class="author"><h3 class="author"><span class="firstname">Paolo</span> <span class="surname">Carlini</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Phil</span> <span class="surname">Edwards</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Doug</span> <span class="surname">Gregor</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Benjamin</span> <span class="surname">Kosnik</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Dhruv</span> <span class="surname">Matani</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Jason</span> <span class="surname">Merrill</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Mark</span> <span class="surname">Mitchell</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Nathan</span> <span class="surname">Myers</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Felix</span> <span class="surname">Natter</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Stefan</span> <span class="surname">Olsson</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Silvius</span> <span class="surname">Rus</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Johannes</span> <span class="surname">Singler</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Ami</span> <span class="surname">Tavory</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Jonathan</span> <span class="surname">Wakely</span></h3></div></div></div><div><p class="copyright">Copyright © 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>The GNU C++ Library Manual</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="../index.html" title="The GNU C++ Library" /><link rel="prev" href="../index.html" title="The GNU C++ Library" /><link rel="next" href="intro.html" title="Part I.  Introduction" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">The GNU C++ Library Manual</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="../index.html">Prev</a> </td><th width="60%" align="center"> </th><td width="20%" align="right"> <a accesskey="n" href="intro.html">Next</a></td></tr></table><hr /></div><div class="book"><div class="titlepage"><div><div><h1 class="title"><a id="manual"></a>The GNU C++ Library Manual</h1></div><div><div class="authorgroup"><div class="author"><h3 class="author"><span class="firstname"></span> <span class="surname"></span></h3></div><div class="author"><h3 class="author"><span class="firstname">Paolo</span> <span class="surname">Carlini</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Phil</span> <span class="surname">Edwards</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Doug</span> <span class="surname">Gregor</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Benjamin</span> <span class="surname">Kosnik</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Dhruv</span> <span class="surname">Matani</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Jason</span> <span class="surname">Merrill</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Mark</span> <span class="surname">Mitchell</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Nathan</span> <span class="surname">Myers</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Felix</span> <span class="surname">Natter</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Stefan</span> <span class="surname">Olsson</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Silvius</span> <span class="surname">Rus</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Johannes</span> <span class="surname">Singler</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Ami</span> <span class="surname">Tavory</span></h3></div><div class="author"><h3 class="author"><span class="firstname">Jonathan</span> <span class="surname">Wakely</span></h3></div></div></div><div><p class="copyright">Copyright © 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 
       <a class="link" href="http://www.fsf.org" target="_top">FSF</a>
     </p></div></div><hr /></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="part"><a href="intro.html">I. 
   Introduction
   
-</a></span></dt><dd><dl><dt><span class="chapter"><a href="status.html">1. Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="status.html#status.iso.201z">C++ 201z</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr24733">C++ TR 24733</a></span></dt></dl></dd><dt><span class="section"><a href="license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></dd><dt><span class="chapter"><a href="setup.html">2. Setup</a></span></dt><dd><dl><dt><span class="section"><a href="setup.html#manual.intro.setup.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="configure.html">Configure</a></span></dt><dt><span class="section"><a href="make.html">Make</a></span></dt></dl></dd><dt><span class="chapter"><a href="using.html">3. Using</a></span></dt><dd><dl><dt><span class="section"><a href="using.html#manual.intro.using.flags">Command Options</a></span></dt><dt><span class="section"><a href="using_headers.html">Headers</a></span></dt><dd><dl><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.all">Header Files</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.mixing">Mixing Headers</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.cheaders">The C Headers and <code class="code">namespace std</code></a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.pre">Precompiled Headers</a></span></dt></dl></dd><dt><span class="section"><a href="using_macros.html">Macros</a></span></dt><dt><span class="section"><a href="using_dual_abi.html">Dual ABI</a></span></dt><dd><dl><dt><span class="section"><a href="using_dual_abi.html#manual.intro.using.abi.trouble">Troubleshooting</a></span></dt></dl></dd><dt><span class="section"><a href="using_namespaces.html">Namespaces</a></span></dt><dd><dl><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.all">Available Namespaces</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.std">namespace std</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.comp">Using Namespace Composition</a></span></dt></dl></dd><dt><span class="section"><a href="using_dynamic_or_shared.html">Linking</a></span></dt><dd><dl><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.freestanding">Almost Nothing</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.dynamic">Finding Dynamic or Shared Libraries</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.experimental">Experimental Library Extensions</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html">Concurrency</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.thread_safety">Thread Safety</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.atomics">Atomics</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.io">IO</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#concurrency.io.structure">Structure</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.defaults">Defaults</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.future">Future</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.alt">Alternatives</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.containers">Containers</a></span></dt></dl></dd><dt><span class="section"><a href="using_exceptions.html">Exceptions</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.safety">Exception Safety</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.propagating">Exception Neutrality</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.no">Doing without</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.compat">Compatibility</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.c">With <code class="literal">C</code></a></span></dt><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.posix">With <code class="literal">POSIX</code> thread cancellation</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="debug.html">Debugging Support</a></span></dt><dd><dl><dt><span class="section"><a href="debug.html#debug.compiler">Using <span class="command"><strong>g++</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.req">Debug Versions of Library Binary Files</a></span></dt><dt><span class="section"><a href="debug.html#debug.memory">Memory Leak Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.races">Data Race Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.gdb">Using <span class="command"><strong>gdb</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.exceptions">Tracking uncaught exceptions</a></span></dt><dt><span class="section"><a href="debug.html#debug.debug_mode">Debug Mode</a></span></dt><dt><span class="section"><a href="debug.html#debug.compile_time_checks">Compile Time Checking</a></span></dt><dt><span class="section"><a href="debug.html#debug.profile_mode">Profile-based Performance Analysis</a></span></dt></dl></dd></dl></dd></dl></dd><dt><span class="part"><a href="std_contents.html">II. 
+</a></span></dt><dd><dl><dt><span class="chapter"><a href="status.html">1. Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="status.html#status.iso.2017">C++ 2017</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr24733">C++ TR 24733</a></span></dt><dt><span class="section"><a href="status.html#status.iso.specfun">C++ IS 29124</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.specfun.specific">Implementation Specific Behavior</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></dd><dt><span class="chapter"><a href="setup.html">2. Setup</a></span></dt><dd><dl><dt><span class="section"><a href="setup.html#manual.intro.setup.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="configure.html">Configure</a></span></dt><dt><span class="section"><a href="make.html">Make</a></span></dt></dl></dd><dt><span class="chapter"><a href="using.html">3. Using</a></span></dt><dd><dl><dt><span class="section"><a href="using.html#manual.intro.using.flags">Command Options</a></span></dt><dt><span class="section"><a href="using_headers.html">Headers</a></span></dt><dd><dl><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.all">Header Files</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.mixing">Mixing Headers</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.cheaders">The C Headers and <code class="code">namespace std</code></a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.pre">Precompiled Headers</a></span></dt></dl></dd><dt><span class="section"><a href="using_macros.html">Macros</a></span></dt><dt><span class="section"><a href="using_dual_abi.html">Dual ABI</a></span></dt><dd><dl><dt><span class="section"><a href="using_dual_abi.html#manual.intro.using.abi.trouble">Troubleshooting</a></span></dt></dl></dd><dt><span class="section"><a href="using_namespaces.html">Namespaces</a></span></dt><dd><dl><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.all">Available Namespaces</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.std">namespace std</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.comp">Using Namespace Composition</a></span></dt></dl></dd><dt><span class="section"><a href="using_dynamic_or_shared.html">Linking</a></span></dt><dd><dl><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.freestanding">Almost Nothing</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.dynamic">Finding Dynamic or Shared Libraries</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.experimental">Experimental Library Extensions</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html">Concurrency</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.thread_safety">Thread Safety</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.atomics">Atomics</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.io">IO</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#concurrency.io.structure">Structure</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.defaults">Defaults</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.future">Future</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.alt">Alternatives</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.containers">Containers</a></span></dt></dl></dd><dt><span class="section"><a href="using_exceptions.html">Exceptions</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.safety">Exception Safety</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.propagating">Exception Neutrality</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.no">Doing without</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.compat">Compatibility</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.c">With <code class="literal">C</code></a></span></dt><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.posix">With <code class="literal">POSIX</code> thread cancellation</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="debug.html">Debugging Support</a></span></dt><dd><dl><dt><span class="section"><a href="debug.html#debug.compiler">Using <span class="command"><strong>g++</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.req">Debug Versions of Library Binary Files</a></span></dt><dt><span class="section"><a href="debug.html#debug.memory">Memory Leak Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.races">Data Race Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.gdb">Using <span class="command"><strong>gdb</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.exceptions">Tracking uncaught exceptions</a></span></dt><dt><span class="section"><a href="debug.html#debug.debug_mode">Debug Mode</a></span></dt><dt><span class="section"><a href="debug.html#debug.compile_time_checks">Compile Time Checking</a></span></dt><dt><span class="section"><a href="debug.html#debug.profile_mode">Profile-based Performance Analysis</a></span></dt></dl></dd></dl></dd></dl></dd><dt><span class="part"><a href="std_contents.html">II. 
     Standard Contents
   </a></span></dt><dd><dl><dt><span class="chapter"><a href="support.html">4. 
   Support
@@ -149,7 +149,7 @@ Support for C++11 dialect.
 	  </a></dt><dt>22.10. <a href="policy_data_structures_design.html#id-1.3.5.9.4.3.3.3.23">Non-unique Mapping Containers</a></dt><dt>22.11. <a href="policy_data_structures_design.html#id-1.3.5.9.4.3.4.3.5">Point Iterator Hierarchy</a></dt><dt>22.12. <a href="policy_data_structures_design.html#id-1.3.5.9.4.3.4.4.5">Invalidation Guarantee Tags Hierarchy</a></dt><dt>22.13. <a href="policy_data_structures_design.html#id-1.3.5.9.4.3.5.7.4">Container Tag Hierarchy</a></dt><dt>22.14. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.2.3">Hash functions, ranged-hash functions, and
 	      range-hashing functions</a></dt><dt>22.15. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.5.3.4">Insert hash sequence diagram</a></dt><dt>22.16. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.5.3.6">Insert hash sequence diagram with a null policy</a></dt><dt>22.17. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.5.5.5">Hash policy class diagram</a></dt><dt>22.18. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.3.4.7">Balls and bins</a></dt><dt>22.19. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.3.5.3.6">Insert resize sequence diagram</a></dt><dt>22.20. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.3.5.3.9">Standard resize policy trigger sequence
 		diagram</a></dt><dt>22.21. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.3.5.3.10">Standard resize policy size sequence
-		diagram</a></dt><dt>22.22. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.5">Tree node invariants</a></dt><dt>22.23. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.8">Tree node invalidation</a></dt><dt>22.24. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.3">A tree and its update policy</a></dt><dt>22.25. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.8">Restoring node invariants</a></dt><dt>22.26. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.10">Insert update sequence</a></dt><dt>22.27. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.20">Useless update path</a></dt><dt>22.28. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.4.3.2.10">A PATRICIA trie</a></dt><dt>22.29. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.4.3.3.5">A trie and its update policy</a></dt><dt>22.30. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.5.3.3.3">A simple list</a></dt><dt>22.31. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.5.3.3.6">The counter algorithm</a></dt><dt>22.32. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.6.3.3.3">Underlying Priority-Queue Data-Structures.</a></dt><dt>22.33. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.6.3.4.4">Priority-Queue Data-Structure Tags.</a></dt><dt>B.1. <a href="appendix_porting.html#fig.build_hacking.deps">Configure and Build File Dependencies</a></dt></dl></div><div class="list-of-tables"><p><strong>List of Tables</strong></p><dl><dt>1.1. <a href="status.html#table.cxx98_status">C++ 1998/2003 Implementation Status</a></dt><dt>1.2. <a href="status.html#table.cxx11_status">C++ 2011 Implementation Status</a></dt><dt>1.3. <a href="status.html#table.cxx14_status">C++ 2014 Implementation Status</a></dt><dt>1.4. <a href="status.html#table.ts_status">C++ Technical Specifications Implementation Status</a></dt><dt>1.5. <a href="status.html#table.cxx1z_status">C++ 201z Implementation Status</a></dt><dt>1.6. <a href="status.html#table.cxx1z_ts_status">C++ Technical Specifications Implementation Status</a></dt><dt>1.7. <a href="status.html#table.tr1_status">C++ TR1 Implementation Status</a></dt><dt>1.8. <a href="status.html#table.decfp_status">C++ TR 24733 Implementation Status</a></dt><dt>3.1. <a href="using.html#table.cmd_options">C++ Command Options</a></dt><dt>3.2. <a href="using_headers.html#table.cxx98_headers">C++ 1998 Library Headers</a></dt><dt>3.3. <a href="using_headers.html#table.cxx98_cheaders">C++ 1998 Library Headers for C Library Facilities</a></dt><dt>3.4. <a href="using_headers.html#table.cxx11_headers">C++ 2011 Library Headers</a></dt><dt>3.5. <a href="using_headers.html#table.cxx11_cheaders">C++ 2011 Library Headers for C Library Facilities</a></dt><dt>3.6. <a href="using_headers.html#table.tr1_headers">C++ TR 1 Library Headers</a></dt><dt>3.7. <a href="using_headers.html#table.tr1_cheaders">C++ TR 1 Library Headers for C Library Facilities</a></dt><dt>3.8. <a href="using_headers.html#table.decfp_headers">C++ TR 24733 Decimal Floating-Point Header</a></dt><dt>3.9. <a href="using_headers.html#table.abi_headers">C++ ABI Headers</a></dt><dt>3.10. <a href="using_headers.html#table.ext_headers">Extension Headers</a></dt><dt>3.11. <a href="using_headers.html#table.debug_headers">Extension Debug Headers</a></dt><dt>3.12. <a href="using_headers.html#table.profile_headers">Extension Profile Headers</a></dt><dt>3.13. <a href="using_headers.html#table.parallel_headers">Extension Parallel Headers</a></dt><dt>17.1. <a href="debug_mode_using.html#table.debug_mode_containers">Debugging Containers</a></dt><dt>17.2. <a href="debug_mode_using.html#table.debug_mode_containers_cxx11">Debugging Containers C++11</a></dt><dt>18.1. <a href="parallel_mode_using.html#table.parallel_algos">Parallel Algorithms</a></dt><dt>19.1. <a href="profile_mode_design.html#table.profile_code_loc">Profile Code Location</a></dt><dt>19.2. <a href="profile_mode_diagnostics.html#table.profile_diagnostics">Profile Diagnostics</a></dt><dt>21.1. <a href="bitmap_allocator_impl.html#table.bitmap_alloc">Bitmap Allocator Memory Map</a></dt><dt>B.1. <a href="documentation_hacking.html#table.doxygen_prereq">Doxygen Prerequisites</a></dt><dt>B.2. <a href="documentation_hacking.html#table.doxygen_cmp">HTML to Doxygen Markup Comparison</a></dt><dt>B.3. <a href="documentation_hacking.html#table.docbook_prereq">Docbook Prerequisites</a></dt><dt>B.4. <a href="documentation_hacking.html#table.docbook_cmp">HTML to Docbook XML Markup Comparison</a></dt><dt>B.5. <a href="documentation_hacking.html#table.docbook_elem">Docbook XML Element Use</a></dt><dt>B.6. <a href="api.html#table.extension_allocators">Extension Allocators</a></dt><dt>B.7. <a href="api.html#table.extension_allocators2">Extension Allocators Continued</a></dt></dl></div><div class="list-of-equations"><p><strong>List of Equations</strong></p><dl><dt>22.1. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.2.15">Ranged Hash Function</a></dt><dt>22.2. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.3.3">Range-Hashing, Division Method</a></dt><dt>22.3. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.3.9">Division via Prime Modulo</a></dt><dt>22.4. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.3.11">Division via Bit Mask</a></dt><dt>22.5. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.4.7">
+		diagram</a></dt><dt>22.22. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.5">Tree node invariants</a></dt><dt>22.23. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.8">Tree node invalidation</a></dt><dt>22.24. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.3">A tree and its update policy</a></dt><dt>22.25. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.8">Restoring node invariants</a></dt><dt>22.26. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.10">Insert update sequence</a></dt><dt>22.27. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.3.3.2.11.20">Useless update path</a></dt><dt>22.28. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.4.3.2.10">A PATRICIA trie</a></dt><dt>22.29. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.4.3.3.5">A trie and its update policy</a></dt><dt>22.30. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.5.3.3.3">A simple list</a></dt><dt>22.31. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.5.3.3.6">The counter algorithm</a></dt><dt>22.32. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.6.3.3.3">Underlying Priority-Queue Data-Structures.</a></dt><dt>22.33. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.6.3.4.4">Priority-Queue Data-Structure Tags.</a></dt><dt>B.1. <a href="appendix_porting.html#fig.build_hacking.deps">Configure and Build File Dependencies</a></dt></dl></div><div class="list-of-tables"><p><strong>List of Tables</strong></p><dl><dt>1.1. <a href="status.html#table.cxx98_status">C++ 1998/2003 Implementation Status</a></dt><dt>1.2. <a href="status.html#table.cxx11_status">C++ 2011 Implementation Status</a></dt><dt>1.3. <a href="status.html#table.cxx14_status">C++ 2014 Implementation Status</a></dt><dt>1.4. <a href="status.html#table.ts_status">C++ Technical Specifications Implementation Status</a></dt><dt>1.5. <a href="status.html#table.cxx17_status">C++ 2017 Implementation Status</a></dt><dt>1.6. <a href="status.html#table.cxx17_ts_status">C++ Technical Specifications Implementation Status</a></dt><dt>1.7. <a href="status.html#table.tr1_status">C++ TR1 Implementation Status</a></dt><dt>1.8. <a href="status.html#table.decfp_status">C++ TR 24733 Implementation Status</a></dt><dt>1.9. <a href="status.html#table.specfun_status">C++ Special Functions Implementation Status</a></dt><dt>3.1. <a href="using.html#table.cmd_options">C++ Command Options</a></dt><dt>3.2. <a href="using_headers.html#table.cxx98_headers">C++ 1998 Library Headers</a></dt><dt>3.3. <a href="using_headers.html#table.cxx98_cheaders">C++ 1998 Library Headers for C Library Facilities</a></dt><dt>3.4. <a href="using_headers.html#table.cxx11_headers">C++ 2011 Library Headers</a></dt><dt>3.5. <a href="using_headers.html#table.cxx11_cheaders">C++ 2011 Library Headers for C Library Facilities</a></dt><dt>3.6. <a href="using_headers.html#table.tr1_headers">C++ TR 1 Library Headers</a></dt><dt>3.7. <a href="using_headers.html#table.tr1_cheaders">C++ TR 1 Library Headers for C Library Facilities</a></dt><dt>3.8. <a href="using_headers.html#table.decfp_headers">C++ TR 24733 Decimal Floating-Point Header</a></dt><dt>3.9. <a href="using_headers.html#table.abi_headers">C++ ABI Headers</a></dt><dt>3.10. <a href="using_headers.html#table.ext_headers">Extension Headers</a></dt><dt>3.11. <a href="using_headers.html#table.debug_headers">Extension Debug Headers</a></dt><dt>3.12. <a href="using_headers.html#table.profile_headers">Extension Profile Headers</a></dt><dt>3.13. <a href="using_headers.html#table.parallel_headers">Extension Parallel Headers</a></dt><dt>17.1. <a href="debug_mode_using.html#table.debug_mode_containers">Debugging Containers</a></dt><dt>17.2. <a href="debug_mode_using.html#table.debug_mode_containers_cxx11">Debugging Containers C++11</a></dt><dt>18.1. <a href="parallel_mode_using.html#table.parallel_algos">Parallel Algorithms</a></dt><dt>19.1. <a href="profile_mode_design.html#table.profile_code_loc">Profile Code Location</a></dt><dt>19.2. <a href="profile_mode_diagnostics.html#table.profile_diagnostics">Profile Diagnostics</a></dt><dt>21.1. <a href="bitmap_allocator_impl.html#table.bitmap_alloc">Bitmap Allocator Memory Map</a></dt><dt>B.1. <a href="documentation_hacking.html#table.doxygen_prereq">Doxygen Prerequisites</a></dt><dt>B.2. <a href="documentation_hacking.html#table.doxygen_cmp">HTML to Doxygen Markup Comparison</a></dt><dt>B.3. <a href="documentation_hacking.html#table.docbook_prereq">Docbook Prerequisites</a></dt><dt>B.4. <a href="documentation_hacking.html#table.docbook_cmp">HTML to Docbook XML Markup Comparison</a></dt><dt>B.5. <a href="documentation_hacking.html#table.docbook_elem">Docbook XML Element Use</a></dt><dt>B.6. <a href="api.html#table.extension_allocators">Extension Allocators</a></dt><dt>B.7. <a href="api.html#table.extension_allocators2">Extension Allocators Continued</a></dt></dl></div><div class="list-of-equations"><p><strong>List of Equations</strong></p><dl><dt>22.1. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.2.15">Ranged Hash Function</a></dt><dt>22.2. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.3.3">Range-Hashing, Division Method</a></dt><dt>22.3. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.3.9">Division via Prime Modulo</a></dt><dt>22.4. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.3.11">Division via Bit Mask</a></dt><dt>22.5. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.4.7">
 		A Standard String Hash Function
 	      </a></dt><dt>22.6. <a href="policy_data_structures_design.html#id-1.3.5.9.4.4.2.3.2.4.12">
 		Only k String DNA Hash
diff --git a/libstdc++-v3/doc/html/manual/internals.html b/libstdc++-v3/doc/html/manual/internals.html
index c28e668f8ba..f61b0b12b80 100644
--- a/libstdc++-v3/doc/html/manual/internals.html
+++ b/libstdc++-v3/doc/html/manual/internals.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Porting to New Hardware or Operating Systems</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, internals" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="documentation_hacking.html" title="Writing and Generating Documentation" /><link rel="next" href="test.html" title="Testing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Porting to New Hardware or Operating Systems</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="documentation_hacking.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Porting to New Hardware or Operating Systems</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, internals" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="documentation_hacking.html" title="Writing and Generating Documentation" /><link rel="next" href="test.html" title="Testing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Porting to New Hardware or Operating Systems</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="documentation_hacking.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
   Porting and Maintenance
   
 </th><td width="20%" align="right"> <a accesskey="n" href="test.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="appendix.porting.internals"></a>Porting to New Hardware or Operating Systems</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/intro.html b/libstdc++-v3/doc/html/manual/intro.html
index e79897bd8d1..8c24f521e1e 100644
--- a/libstdc++-v3/doc/html/manual/intro.html
+++ b/libstdc++-v3/doc/html/manual/intro.html
@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part I.  Introduction</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="index.html" title="The GNU C++ Library Manual" /><link rel="next" href="status.html" title="Chapter 1. Status" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part I. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part I.  Introduction</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="index.html" title="The GNU C++ Library Manual" /><link rel="next" href="status.html" title="Chapter 1. Status" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part I. 
   Introduction
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="index.html">Prev</a> </td><th width="60%" align="center">The GNU C++ Library Manual</th><td width="20%" align="right"> <a accesskey="n" href="status.html">Next</a></td></tr></table><hr /></div><div class="part"><div class="titlepage"><div><div><h1 class="title"><a id="manual.intro"></a>Part I. 
   Introduction
   <a id="id-1.3.3.1.1.1" class="indexterm"></a>
-</h1></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="chapter"><a href="status.html">1. Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="status.html#status.iso.201z">C++ 201z</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr24733">C++ TR 24733</a></span></dt></dl></dd><dt><span class="section"><a href="license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></dd><dt><span class="chapter"><a href="setup.html">2. Setup</a></span></dt><dd><dl><dt><span class="section"><a href="setup.html#manual.intro.setup.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="configure.html">Configure</a></span></dt><dt><span class="section"><a href="make.html">Make</a></span></dt></dl></dd><dt><span class="chapter"><a href="using.html">3. Using</a></span></dt><dd><dl><dt><span class="section"><a href="using.html#manual.intro.using.flags">Command Options</a></span></dt><dt><span class="section"><a href="using_headers.html">Headers</a></span></dt><dd><dl><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.all">Header Files</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.mixing">Mixing Headers</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.cheaders">The C Headers and <code class="code">namespace std</code></a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.pre">Precompiled Headers</a></span></dt></dl></dd><dt><span class="section"><a href="using_macros.html">Macros</a></span></dt><dt><span class="section"><a href="using_dual_abi.html">Dual ABI</a></span></dt><dd><dl><dt><span class="section"><a href="using_dual_abi.html#manual.intro.using.abi.trouble">Troubleshooting</a></span></dt></dl></dd><dt><span class="section"><a href="using_namespaces.html">Namespaces</a></span></dt><dd><dl><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.all">Available Namespaces</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.std">namespace std</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.comp">Using Namespace Composition</a></span></dt></dl></dd><dt><span class="section"><a href="using_dynamic_or_shared.html">Linking</a></span></dt><dd><dl><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.freestanding">Almost Nothing</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.dynamic">Finding Dynamic or Shared Libraries</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.experimental">Experimental Library Extensions</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html">Concurrency</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.thread_safety">Thread Safety</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.atomics">Atomics</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.io">IO</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#concurrency.io.structure">Structure</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.defaults">Defaults</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.future">Future</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.alt">Alternatives</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.containers">Containers</a></span></dt></dl></dd><dt><span class="section"><a href="using_exceptions.html">Exceptions</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.safety">Exception Safety</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.propagating">Exception Neutrality</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.no">Doing without</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.compat">Compatibility</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.c">With <code class="literal">C</code></a></span></dt><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.posix">With <code class="literal">POSIX</code> thread cancellation</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="debug.html">Debugging Support</a></span></dt><dd><dl><dt><span class="section"><a href="debug.html#debug.compiler">Using <span class="command"><strong>g++</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.req">Debug Versions of Library Binary Files</a></span></dt><dt><span class="section"><a href="debug.html#debug.memory">Memory Leak Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.races">Data Race Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.gdb">Using <span class="command"><strong>gdb</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.exceptions">Tracking uncaught exceptions</a></span></dt><dt><span class="section"><a href="debug.html#debug.debug_mode">Debug Mode</a></span></dt><dt><span class="section"><a href="debug.html#debug.compile_time_checks">Compile Time Checking</a></span></dt><dt><span class="section"><a href="debug.html#debug.profile_mode">Profile-based Performance Analysis</a></span></dt></dl></dd></dl></dd></dl></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="index.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="index.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="status.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">The GNU C++ Library Manual </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Chapter 1. Status</td></tr></table></div></body></html>
\ No newline at end of file
+</h1></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="chapter"><a href="status.html">1. Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="status.html#status.iso.2017">C++ 2017</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr24733">C++ TR 24733</a></span></dt><dt><span class="section"><a href="status.html#status.iso.specfun">C++ IS 29124</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.specfun.specific">Implementation Specific Behavior</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></dd><dt><span class="chapter"><a href="setup.html">2. Setup</a></span></dt><dd><dl><dt><span class="section"><a href="setup.html#manual.intro.setup.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="configure.html">Configure</a></span></dt><dt><span class="section"><a href="make.html">Make</a></span></dt></dl></dd><dt><span class="chapter"><a href="using.html">3. Using</a></span></dt><dd><dl><dt><span class="section"><a href="using.html#manual.intro.using.flags">Command Options</a></span></dt><dt><span class="section"><a href="using_headers.html">Headers</a></span></dt><dd><dl><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.all">Header Files</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.mixing">Mixing Headers</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.cheaders">The C Headers and <code class="code">namespace std</code></a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.pre">Precompiled Headers</a></span></dt></dl></dd><dt><span class="section"><a href="using_macros.html">Macros</a></span></dt><dt><span class="section"><a href="using_dual_abi.html">Dual ABI</a></span></dt><dd><dl><dt><span class="section"><a href="using_dual_abi.html#manual.intro.using.abi.trouble">Troubleshooting</a></span></dt></dl></dd><dt><span class="section"><a href="using_namespaces.html">Namespaces</a></span></dt><dd><dl><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.all">Available Namespaces</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.std">namespace std</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.comp">Using Namespace Composition</a></span></dt></dl></dd><dt><span class="section"><a href="using_dynamic_or_shared.html">Linking</a></span></dt><dd><dl><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.freestanding">Almost Nothing</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.dynamic">Finding Dynamic or Shared Libraries</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.experimental">Experimental Library Extensions</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html">Concurrency</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.thread_safety">Thread Safety</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.atomics">Atomics</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.io">IO</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#concurrency.io.structure">Structure</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.defaults">Defaults</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.future">Future</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.alt">Alternatives</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.containers">Containers</a></span></dt></dl></dd><dt><span class="section"><a href="using_exceptions.html">Exceptions</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.safety">Exception Safety</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.propagating">Exception Neutrality</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.no">Doing without</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.compat">Compatibility</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.c">With <code class="literal">C</code></a></span></dt><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.posix">With <code class="literal">POSIX</code> thread cancellation</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="debug.html">Debugging Support</a></span></dt><dd><dl><dt><span class="section"><a href="debug.html#debug.compiler">Using <span class="command"><strong>g++</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.req">Debug Versions of Library Binary Files</a></span></dt><dt><span class="section"><a href="debug.html#debug.memory">Memory Leak Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.races">Data Race Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.gdb">Using <span class="command"><strong>gdb</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.exceptions">Tracking uncaught exceptions</a></span></dt><dt><span class="section"><a href="debug.html#debug.debug_mode">Debug Mode</a></span></dt><dt><span class="section"><a href="debug.html#debug.compile_time_checks">Compile Time Checking</a></span></dt><dt><span class="section"><a href="debug.html#debug.profile_mode">Profile-based Performance Analysis</a></span></dt></dl></dd></dl></dd></dl></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="index.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="index.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="status.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">The GNU C++ Library Manual </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Chapter 1. Status</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/manual/io.html b/libstdc++-v3/doc/html/manual/io.html
index b67a08fb3fa..75a63f739c8 100644
--- a/libstdc++-v3/doc/html/manual/io.html
+++ b/libstdc++-v3/doc/html/manual/io.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 13.  Input and Output</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="numerics_and_c.html" title="Interacting with C" /><link rel="next" href="streambufs.html" title="Stream Buffers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 13. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 13.  Input and Output</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="numerics_and_c.html" title="Interacting with C" /><link rel="next" href="streambufs.html" title="Stream Buffers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 13. 
   Input and Output
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="numerics_and_c.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/io_and_c.html b/libstdc++-v3/doc/html/manual/io_and_c.html
index a7ee907e405..71aafa2f71e 100644
--- a/libstdc++-v3/doc/html/manual/io_and_c.html
+++ b/libstdc++-v3/doc/html/manual/io_and_c.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Interacting with C</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="fstreams.html" title="File Based Streams" /><link rel="next" href="atomics.html" title="Chapter 14.  Atomics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Interacting with C</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="fstreams.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Interacting with C</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="fstreams.html" title="File Based Streams" /><link rel="next" href="atomics.html" title="Chapter 14.  Atomics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Interacting with C</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="fstreams.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
   Input and Output
   
 </th><td width="20%" align="right"> <a accesskey="n" href="atomics.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.io.c"></a>Interacting with C</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="std.io.c.FILE"></a>Using FILE* and file descriptors</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/iterators.html b/libstdc++-v3/doc/html/manual/iterators.html
index 70dbe491be5..874de4ad9ac 100644
--- a/libstdc++-v3/doc/html/manual/iterators.html
+++ b/libstdc++-v3/doc/html/manual/iterators.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 10.  Iterators</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="containers_and_c.html" title="Interacting with C" /><link rel="next" href="algorithms.html" title="Chapter 11.  Algorithms" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 10. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 10.  Iterators</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="containers_and_c.html" title="Interacting with C" /><link rel="next" href="algorithms.html" title="Chapter 11.  Algorithms" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 10. 
   Iterators
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="containers_and_c.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/license.html b/libstdc++-v3/doc/html/manual/license.html
index 1b361f699d2..c1d708acc51 100644
--- a/libstdc++-v3/doc/html/manual/license.html
+++ b/libstdc++-v3/doc/html/manual/license.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>License</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="status.html" title="Chapter 1. Status" /><link rel="prev" href="status.html" title="Chapter 1. Status" /><link rel="next" href="bugs.html" title="Bugs" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">License</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="status.html">Prev</a> </td><th width="60%" align="center">Chapter 1. Status</th><td width="20%" align="right"> <a accesskey="n" href="bugs.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.status.license"></a>License</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>License</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="status.html" title="Chapter 1. Status" /><link rel="prev" href="status.html" title="Chapter 1. Status" /><link rel="next" href="bugs.html" title="Bugs" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">License</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="status.html">Prev</a> </td><th width="60%" align="center">Chapter 1. Status</th><td width="20%" align="right"> <a accesskey="n" href="bugs.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.status.license"></a>License</h2></div></div></div><p>
     There are two licenses affecting GNU libstdc++: one for the code,
     and one for the documentation.
     </p><p>
diff --git a/libstdc++-v3/doc/html/manual/localization.html b/libstdc++-v3/doc/html/manual/localization.html
index fe283a7b68e..ead068a16bb 100644
--- a/libstdc++-v3/doc/html/manual/localization.html
+++ b/libstdc++-v3/doc/html/manual/localization.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 8.  Localization</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="strings.html" title="Chapter 7.  Strings" /><link rel="next" href="facets.html" title="Facets" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 8. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 8.  Localization</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="strings.html" title="Chapter 7.  Strings" /><link rel="next" href="facets.html" title="Facets" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 8. 
   Localization
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="strings.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/make.html b/libstdc++-v3/doc/html/manual/make.html
index 7648f14fc49..f273e4b204f 100644
--- a/libstdc++-v3/doc/html/manual/make.html
+++ b/libstdc++-v3/doc/html/manual/make.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Make</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="setup.html" title="Chapter 2. Setup" /><link rel="prev" href="configure.html" title="Configure" /><link rel="next" href="using.html" title="Chapter 3. Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Make</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="configure.html">Prev</a> </td><th width="60%" align="center">Chapter 2. Setup</th><td width="20%" align="right"> <a accesskey="n" href="using.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.setup.make"></a>Make</h2></div></div></div><p>If you have never done this before, you should read the basic
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Make</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="setup.html" title="Chapter 2. Setup" /><link rel="prev" href="configure.html" title="Configure" /><link rel="next" href="using.html" title="Chapter 3. Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Make</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="configure.html">Prev</a> </td><th width="60%" align="center">Chapter 2. Setup</th><td width="20%" align="right"> <a accesskey="n" href="using.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.setup.make"></a>Make</h2></div></div></div><p>If you have never done this before, you should read the basic
       <a class="link" href="http://gcc.gnu.org/install/" target="_top">GCC Installation
       Instructions</a> first.  Read <span class="emphasis"><em>all of them</em></span>.
       <span class="emphasis"><em>Twice.</em></span>
diff --git a/libstdc++-v3/doc/html/manual/memory.html b/libstdc++-v3/doc/html/manual/memory.html
index 4f6b0b16a96..d95202cd100 100644
--- a/libstdc++-v3/doc/html/manual/memory.html
+++ b/libstdc++-v3/doc/html/manual/memory.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Memory</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="prev" href="pairs.html" title="Pairs" /><link rel="next" href="traits.html" title="Traits" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Memory</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="pairs.html">Prev</a> </td><th width="60%" align="center">Chapter 6. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Memory</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="prev" href="pairs.html" title="Pairs" /><link rel="next" href="traits.html" title="Traits" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Memory</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="pairs.html">Prev</a> </td><th width="60%" align="center">Chapter 6. 
   Utilities
   
 </th><td width="20%" align="right"> <a accesskey="n" href="traits.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.util.memory"></a>Memory</h2></div></div></div><p>
@@ -676,4 +676,4 @@ be private.
 	</a>
       </em>. </span><span class="subtitle">
       N2461
-    . </span></p></div></div></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="pairs.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="utilities.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="traits.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Pairs </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Traits</td></tr></table></div></body></html>
+    . </span></p></div></div></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="pairs.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="utilities.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="traits.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Pairs </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Traits</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/manual/mt_allocator.html b/libstdc++-v3/doc/html/manual/mt_allocator.html
index a47d0e1375d..42a0125692e 100644
--- a/libstdc++-v3/doc/html/manual/mt_allocator.html
+++ b/libstdc++-v3/doc/html/manual/mt_allocator.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 20. The mt_allocator</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="profile_mode_diagnostics.html" title="Diagnostics" /><link rel="next" href="mt_allocator_design.html" title="Design Issues" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 20. The mt_allocator</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_diagnostics.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 20. The mt_allocator</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="profile_mode_diagnostics.html" title="Diagnostics" /><link rel="next" href="mt_allocator_design.html" title="Design Issues" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 20. The mt_allocator</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_diagnostics.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator_design.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.allocator.mt"></a>Chapter 20. The mt_allocator</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="mt_allocator.html#allocator.mt.intro">Intro</a></span></dt><dt><span class="section"><a href="mt_allocator_design.html">Design Issues</a></span></dt><dd><dl><dt><span class="section"><a href="mt_allocator_design.html#allocator.mt.overview">Overview</a></span></dt></dl></dd><dt><span class="section"><a href="mt_allocator_impl.html">Implementation</a></span></dt><dd><dl><dt><span class="section"><a href="mt_allocator_impl.html#allocator.mt.tune">Tunable Parameters</a></span></dt><dt><span class="section"><a href="mt_allocator_impl.html#allocator.mt.init">Initialization</a></span></dt><dt><span class="section"><a href="mt_allocator_impl.html#allocator.mt.deallocation">Deallocation Notes</a></span></dt></dl></dd><dt><span class="section"><a href="mt_allocator_ex_single.html">Single Thread Example</a></span></dt><dt><span class="section"><a href="mt_allocator_ex_multi.html">Multiple Thread Example</a></span></dt></dl></div><p>
diff --git a/libstdc++-v3/doc/html/manual/mt_allocator_design.html b/libstdc++-v3/doc/html/manual/mt_allocator_design.html
index 3f82c635019..b08c0b7641a 100644
--- a/libstdc++-v3/doc/html/manual/mt_allocator_design.html
+++ b/libstdc++-v3/doc/html/manual/mt_allocator_design.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design Issues</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="next" href="mt_allocator_impl.html" title="Implementation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design Issues</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator_impl.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.design_issues"></a>Design Issues</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="allocator.mt.overview"></a>Overview</h3></div></div></div><p> There are three general components to the allocator: a datum
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design Issues</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="next" href="mt_allocator_impl.html" title="Implementation" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design Issues</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator_impl.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.design_issues"></a>Design Issues</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="allocator.mt.overview"></a>Overview</h3></div></div></div><p> There are three general components to the allocator: a datum
 describing the characteristics of the memory pool, a policy class
 containing this pool that links instantiation types to common or
 individual pools, and a class inheriting from the policy class that is
diff --git a/libstdc++-v3/doc/html/manual/mt_allocator_ex_multi.html b/libstdc++-v3/doc/html/manual/mt_allocator_ex_multi.html
index 32e76ea9b3c..1cd1749d911 100644
--- a/libstdc++-v3/doc/html/manual/mt_allocator_ex_multi.html
+++ b/libstdc++-v3/doc/html/manual/mt_allocator_ex_multi.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Multiple Thread Example</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator_ex_single.html" title="Single Thread Example" /><link rel="next" href="bitmap_allocator.html" title="Chapter 21. The bitmap_allocator" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Multiple Thread Example</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_ex_single.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="bitmap_allocator.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.example_multi"></a>Multiple Thread Example</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Multiple Thread Example</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator_ex_single.html" title="Single Thread Example" /><link rel="next" href="bitmap_allocator.html" title="Chapter 21. The bitmap_allocator" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Multiple Thread Example</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_ex_single.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="bitmap_allocator.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.example_multi"></a>Multiple Thread Example</h2></div></div></div><p>
 In the ST example we never used the thread_id variable present in each block.
 Let's start by explaining the purpose of this in a MT application.
 </p><p>
diff --git a/libstdc++-v3/doc/html/manual/mt_allocator_ex_single.html b/libstdc++-v3/doc/html/manual/mt_allocator_ex_single.html
index 22e603a2e6f..7e30f72b953 100644
--- a/libstdc++-v3/doc/html/manual/mt_allocator_ex_single.html
+++ b/libstdc++-v3/doc/html/manual/mt_allocator_ex_single.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Single Thread Example</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator_impl.html" title="Implementation" /><link rel="next" href="mt_allocator_ex_multi.html" title="Multiple Thread Example" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Single Thread Example</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_impl.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator_ex_multi.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.example_single"></a>Single Thread Example</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Single Thread Example</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator_impl.html" title="Implementation" /><link rel="next" href="mt_allocator_ex_multi.html" title="Multiple Thread Example" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Single Thread Example</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_impl.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator_ex_multi.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.example_single"></a>Single Thread Example</h2></div></div></div><p>
 Let's start by describing how the data on a freelist is laid out in memory.
 This is the first two blocks in freelist for thread id 3 in bin 3 (8 bytes):
 </p><pre class="programlisting">
diff --git a/libstdc++-v3/doc/html/manual/mt_allocator_impl.html b/libstdc++-v3/doc/html/manual/mt_allocator_impl.html
index 7c1d020c201..b5539e2da9b 100644
--- a/libstdc++-v3/doc/html/manual/mt_allocator_impl.html
+++ b/libstdc++-v3/doc/html/manual/mt_allocator_impl.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator_design.html" title="Design Issues" /><link rel="next" href="mt_allocator_ex_single.html" title="Single Thread Example" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_design.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator_ex_single.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.impl"></a>Implementation</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="allocator.mt.tune"></a>Tunable Parameters</h3></div></div></div><p>Certain allocation parameters can be modified, or tuned. There
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, allocator" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /><link rel="prev" href="mt_allocator_design.html" title="Design Issues" /><link rel="next" href="mt_allocator_ex_single.html" title="Single Thread Example" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="mt_allocator_design.html">Prev</a> </td><th width="60%" align="center">Chapter 20. The mt_allocator</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator_ex_single.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="allocator.mt.impl"></a>Implementation</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="allocator.mt.tune"></a>Tunable Parameters</h3></div></div></div><p>Certain allocation parameters can be modified, or tuned. There
 exists a nested <code class="code">struct __pool_base::_Tune</code> that contains all
 these parameters, which include settings for
 </p><div class="itemizedlist"><ul class="itemizedlist" style="list-style-type: disc; "><li class="listitem"><p>Alignment</p></li><li class="listitem"><p>Maximum bytes before calling <code class="code">::operator new</code> directly</p></li><li class="listitem"><p>Minimum bytes</p></li><li class="listitem"><p>Size of underlying global allocations</p></li><li class="listitem"><p>Maximum number of supported threads</p></li><li class="listitem"><p>Migration of deallocations to the global free list</p></li><li class="listitem"><p>Shunt for global <code class="code">new</code> and <code class="code">delete</code></p></li></ul></div><p>Adjusting parameters for a given instance of an allocator can only
diff --git a/libstdc++-v3/doc/html/manual/numerics.html b/libstdc++-v3/doc/html/manual/numerics.html
index 17e6fc3210d..e56cbdca0ea 100644
--- a/libstdc++-v3/doc/html/manual/numerics.html
+++ b/libstdc++-v3/doc/html/manual/numerics.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 12.  Numerics</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="algorithms.html" title="Chapter 11.  Algorithms" /><link rel="next" href="generalized_numeric_operations.html" title="Generalized Operations" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 12. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 12.  Numerics</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="algorithms.html" title="Chapter 11.  Algorithms" /><link rel="next" href="generalized_numeric_operations.html" title="Generalized Operations" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 12. 
   Numerics
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="algorithms.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/numerics_and_c.html b/libstdc++-v3/doc/html/manual/numerics_and_c.html
index b3a78cbb5d8..8179dbf0f48 100644
--- a/libstdc++-v3/doc/html/manual/numerics_and_c.html
+++ b/libstdc++-v3/doc/html/manual/numerics_and_c.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Interacting with C</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="numerics.html" title="Chapter 12.  Numerics" /><link rel="prev" href="generalized_numeric_operations.html" title="Generalized Operations" /><link rel="next" href="io.html" title="Chapter 13.  Input and Output" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Interacting with C</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="generalized_numeric_operations.html">Prev</a> </td><th width="60%" align="center">Chapter 12. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Interacting with C</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="numerics.html" title="Chapter 12.  Numerics" /><link rel="prev" href="generalized_numeric_operations.html" title="Generalized Operations" /><link rel="next" href="io.html" title="Chapter 13.  Input and Output" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Interacting with C</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="generalized_numeric_operations.html">Prev</a> </td><th width="60%" align="center">Chapter 12. 
   Numerics
   
 </th><td width="20%" align="right"> <a accesskey="n" href="io.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.numerics.c"></a>Interacting with C</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="numerics.c.array"></a>Numerics vs. Arrays</h3></div></div></div><p>One of the major reasons why FORTRAN can chew through numbers so well
diff --git a/libstdc++-v3/doc/html/manual/pairs.html b/libstdc++-v3/doc/html/manual/pairs.html
index e6a299d5027..06f2ab678c5 100644
--- a/libstdc++-v3/doc/html/manual/pairs.html
+++ b/libstdc++-v3/doc/html/manual/pairs.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Pairs</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="prev" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="next" href="memory.html" title="Memory" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Pairs</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="utilities.html">Prev</a> </td><th width="60%" align="center">Chapter 6. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Pairs</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="prev" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="next" href="memory.html" title="Memory" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Pairs</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="utilities.html">Prev</a> </td><th width="60%" align="center">Chapter 6. 
   Utilities
   
 </th><td width="20%" align="right"> <a accesskey="n" href="memory.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.util.pairs"></a>Pairs</h2></div></div></div><p>The <code class="code">pair&lt;T1,T2&gt;</code> is a simple and handy way to
diff --git a/libstdc++-v3/doc/html/manual/parallel_mode.html b/libstdc++-v3/doc/html/manual/parallel_mode.html
index 09510fddf89..db64389b5b1 100644
--- a/libstdc++-v3/doc/html/manual/parallel_mode.html
+++ b/libstdc++-v3/doc/html/manual/parallel_mode.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 18. Parallel Mode</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="debug_mode_design.html" title="Design" /><link rel="next" href="parallel_mode_semantics.html" title="Semantics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 18. Parallel Mode</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode_design.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 18. Parallel Mode</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="debug_mode_design.html" title="Design" /><link rel="next" href="parallel_mode_semantics.html" title="Semantics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 18. Parallel Mode</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug_mode_design.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode_semantics.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.parallel_mode"></a>Chapter 18. Parallel Mode</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="parallel_mode.html#manual.ext.parallel_mode.intro">Intro</a></span></dt><dt><span class="section"><a href="parallel_mode_semantics.html">Semantics</a></span></dt><dt><span class="section"><a href="parallel_mode_using.html">Using</a></span></dt><dd><dl><dt><span class="section"><a href="parallel_mode_using.html#parallel_mode.using.prereq_flags">Prerequisite Compiler Flags</a></span></dt><dt><span class="section"><a href="parallel_mode_using.html#parallel_mode.using.parallel_mode">Using Parallel Mode</a></span></dt><dt><span class="section"><a href="parallel_mode_using.html#parallel_mode.using.specific">Using Specific Parallel Components</a></span></dt></dl></dd><dt><span class="section"><a href="parallel_mode_design.html">Design</a></span></dt><dd><dl><dt><span class="section"><a href="parallel_mode_design.html#parallel_mode.design.intro">Interface Basics</a></span></dt><dt><span class="section"><a href="parallel_mode_design.html#parallel_mode.design.tuning">Configuration and Tuning</a></span></dt><dd><dl><dt><span class="section"><a href="parallel_mode_design.html#parallel_mode.design.tuning.omp">Setting up the OpenMP Environment</a></span></dt><dt><span class="section"><a href="parallel_mode_design.html#parallel_mode.design.tuning.compile">Compile Time Switches</a></span></dt><dt><span class="section"><a href="parallel_mode_design.html#parallel_mode.design.tuning.settings">Run Time Settings and Defaults</a></span></dt></dl></dd><dt><span class="section"><a href="parallel_mode_design.html#parallel_mode.design.impl">Implementation Namespaces</a></span></dt></dl></dd><dt><span class="section"><a href="parallel_mode_test.html">Testing</a></span></dt><dt><span class="bibliography"><a href="parallel_mode.html#parallel_mode.biblio">Bibliography</a></span></dt></dl></div><p> The libstdc++ parallel mode is an experimental parallel
diff --git a/libstdc++-v3/doc/html/manual/parallel_mode_design.html b/libstdc++-v3/doc/html/manual/parallel_mode_design.html
index e1dc09cd07c..2e5a0d3bcaa 100644
--- a/libstdc++-v3/doc/html/manual/parallel_mode_design.html
+++ b/libstdc++-v3/doc/html/manual/parallel_mode_design.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode_using.html" title="Using" /><link rel="next" href="parallel_mode_test.html" title="Testing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_using.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode_test.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.design"></a>Design</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode_using.html" title="Using" /><link rel="next" href="parallel_mode_test.html" title="Testing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_using.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode_test.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.design"></a>Design</h2></div></div></div><p>
   </p><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="parallel_mode.design.intro"></a>Interface Basics</h3></div></div></div><p>
 All parallel algorithms are intended to have signatures that are
 equivalent to the ISO C++ algorithms replaced. For instance, the
diff --git a/libstdc++-v3/doc/html/manual/parallel_mode_semantics.html b/libstdc++-v3/doc/html/manual/parallel_mode_semantics.html
index 3d049c551c4..dcc29736de4 100644
--- a/libstdc++-v3/doc/html/manual/parallel_mode_semantics.html
+++ b/libstdc++-v3/doc/html/manual/parallel_mode_semantics.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Semantics</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="next" href="parallel_mode_using.html" title="Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Semantics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode_using.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.semantics"></a>Semantics</h2></div></div></div><p> The parallel mode STL algorithms are currently not exception-safe,
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Semantics</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="next" href="parallel_mode_using.html" title="Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Semantics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode_using.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.semantics"></a>Semantics</h2></div></div></div><p> The parallel mode STL algorithms are currently not exception-safe,
 i.e. user-defined functors must not throw exceptions.
 Also, the order of execution is not guaranteed for some functions, of course.
 Therefore, user-defined functors should not have any concurrent side effects.
diff --git a/libstdc++-v3/doc/html/manual/parallel_mode_test.html b/libstdc++-v3/doc/html/manual/parallel_mode_test.html
index d68e17a3ad9..a542c196554 100644
--- a/libstdc++-v3/doc/html/manual/parallel_mode_test.html
+++ b/libstdc++-v3/doc/html/manual/parallel_mode_test.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Testing</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode_design.html" title="Design" /><link rel="next" href="profile_mode.html" title="Chapter 19. Profile Mode" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Testing</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_design.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.test"></a>Testing</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Testing</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode_design.html" title="Design" /><link rel="next" href="profile_mode.html" title="Chapter 19. Profile Mode" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Testing</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_design.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.test"></a>Testing</h2></div></div></div><p>
     Both the normal conformance and regression tests and the
     supplemental performance tests work.
   </p><p>
diff --git a/libstdc++-v3/doc/html/manual/parallel_mode_using.html b/libstdc++-v3/doc/html/manual/parallel_mode_using.html
index cebabad4220..1f151f4ed7b 100644
--- a/libstdc++-v3/doc/html/manual/parallel_mode_using.html
+++ b/libstdc++-v3/doc/html/manual/parallel_mode_using.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Using</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode_semantics.html" title="Semantics" /><link rel="next" href="parallel_mode_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_semantics.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode_design.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.using"></a>Using</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="parallel_mode.using.prereq_flags"></a>Prerequisite Compiler Flags</h3></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Using</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, parallel" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="parallel_mode.html" title="Chapter 18. Parallel Mode" /><link rel="prev" href="parallel_mode_semantics.html" title="Semantics" /><link rel="next" href="parallel_mode_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_semantics.html">Prev</a> </td><th width="60%" align="center">Chapter 18. Parallel Mode</th><td width="20%" align="right"> <a accesskey="n" href="parallel_mode_design.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.parallel_mode.using"></a>Using</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="parallel_mode.using.prereq_flags"></a>Prerequisite Compiler Flags</h3></div></div></div><p>
   Any use of parallel functionality requires additional compiler
   and runtime support, in particular support for OpenMP. Adding this support is
   not difficult: just compile your application with the compiler
@@ -63,4 +63,4 @@ Then compile this code with the prerequisite compiler flags
 flags for atomic operations.)
 </p><p> The following table provides the names and headers of all the
   parallel algorithms that can be used in a similar manner:
-</p><div class="table"><a id="table.parallel_algos"></a><p class="title"><strong>Table 18.1. Parallel Algorithms</strong></p><div class="table-contents"><table summary="Parallel Algorithms" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Algorithm</th><th align="left">Header</th><th align="left">Parallel algorithm</th><th align="left">Parallel header</th></tr></thead><tbody><tr><td align="left"><code class="function">std::accumulate</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::accumulate</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::adjacent_difference</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::adjacent_difference</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::inner_product</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::inner_product</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::partial_sum</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::partial_sum</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::adjacent_find</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::adjacent_find</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::count</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::count</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::count_if</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::count_if</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::equal</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::equal</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::find</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::find</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::find_if</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::find_if</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::find_first_of</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::find_first_of</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::for_each</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::for_each</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::generate</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::generate</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::generate_n</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::generate_n</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::lexicographical_compare</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::lexicographical_compare</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::mismatch</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::mismatch</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::search</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::search</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::search_n</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::search_n</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::transform</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::transform</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::replace</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::replace</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::replace_if</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::replace_if</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::max_element</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::max_element</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::merge</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::merge</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::min_element</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::min_element</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::nth_element</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::nth_element</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::partial_sort</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::partial_sort</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::partition</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::partition</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::random_shuffle</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::random_shuffle</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_union</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_union</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_intersection</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_intersection</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_symmetric_difference</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_symmetric_difference</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_difference</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_difference</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::sort</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::sort</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::stable_sort</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::stable_sort</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::unique_copy</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::unique_copy</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr></tbody></table></div></div><br class="table-break" /></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="parallel_mode_semantics.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="parallel_mode.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="parallel_mode_design.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Semantics </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Design</td></tr></table></div></body></html>
\ No newline at end of file
+</p><div class="table"><a id="table.parallel_algos"></a><p class="title"><strong>Table 18.1. Parallel Algorithms</strong></p><div class="table-contents"><table class="table" summary="Parallel Algorithms" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Algorithm</th><th align="left">Header</th><th align="left">Parallel algorithm</th><th align="left">Parallel header</th></tr></thead><tbody><tr><td align="left"><code class="function">std::accumulate</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::accumulate</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::adjacent_difference</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::adjacent_difference</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::inner_product</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::inner_product</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::partial_sum</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="function">__gnu_parallel::partial_sum</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr><tr><td align="left"><code class="function">std::adjacent_find</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::adjacent_find</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::count</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::count</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::count_if</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::count_if</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::equal</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::equal</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::find</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::find</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::find_if</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::find_if</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::find_first_of</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::find_first_of</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::for_each</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::for_each</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::generate</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::generate</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::generate_n</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::generate_n</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::lexicographical_compare</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::lexicographical_compare</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::mismatch</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::mismatch</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::search</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::search</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::search_n</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::search_n</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::transform</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::transform</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::replace</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::replace</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::replace_if</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::replace_if</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::max_element</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::max_element</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::merge</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::merge</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::min_element</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::min_element</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::nth_element</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::nth_element</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::partial_sort</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::partial_sort</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::partition</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::partition</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::random_shuffle</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::random_shuffle</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_union</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_union</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_intersection</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_intersection</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_symmetric_difference</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_symmetric_difference</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::set_difference</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::set_difference</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::sort</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::sort</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::stable_sort</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::stable_sort</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr><tr><td align="left"><code class="function">std::unique_copy</code></td><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="function">__gnu_parallel::unique_copy</code></td><td align="left"><code class="filename">parallel/algorithm</code></td></tr></tbody></table></div></div><br class="table-break" /></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="parallel_mode_semantics.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="parallel_mode.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="parallel_mode_design.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Semantics </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Design</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/manual/policy_based_data_structures_test.html b/libstdc++-v3/doc/html/manual/policy_based_data_structures_test.html
index b59d59cb245..1be7c2056a2 100644
--- a/libstdc++-v3/doc/html/manual/policy_based_data_structures_test.html
+++ b/libstdc++-v3/doc/html/manual/policy_based_data_structures_test.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Testing</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_data_structures_design.html" title="Design" /><link rel="next" href="policy_data_structures_ack.html" title="Acknowledgments" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Testing</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures_design.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="policy_data_structures_ack.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="pbds.test"></a>Testing</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="pbds.test.regression"></a>Regression</h3></div></div></div><p>The library contains a single comprehensive regression test.
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Testing</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_data_structures_design.html" title="Design" /><link rel="next" href="policy_data_structures_ack.html" title="Acknowledgments" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Testing</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures_design.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="policy_data_structures_ack.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="pbds.test"></a>Testing</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="pbds.test.regression"></a>Regression</h3></div></div></div><p>The library contains a single comprehensive regression test.
     For a given container type in this library, the test creates
     an object of the container type and an object of the
     corresponding standard type (e.g., <code class="classname">std::set</code>). It
@@ -49,7 +49,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_hash_text_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -182,7 +182,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_cc_hash_int_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -284,7 +284,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_gp_hash_int_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -384,7 +384,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_cc_hash_int_subscript_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -484,7 +484,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_gp_hash_int_subscript_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -571,7 +571,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_cc_hash_int_subscript_insert.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -671,7 +671,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_gp_hash_int_subscript_insert.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -779,7 +779,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_hash_zlob_int_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -899,7 +899,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_hash_int_erase_mem.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="5" align="left">
 		    n_hash_map_ncah
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_map</code>
@@ -1007,7 +1007,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_tree_text_insert_node.png" align="middle" /></div><p>
 	      The abbreviated names in the legend of the graphic above are
 	      instantiated with the types in the following table.
-	    </p></div><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	    </p></div><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_map
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::map</code>
@@ -1040,7 +1040,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_tree_text_insert_vector.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_map
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::map</code>
@@ -1061,7 +1061,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_tree_text_insert_trie.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_map
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::map</code>
@@ -1117,7 +1117,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_tree_text_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_map
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::map</code>
@@ -1225,7 +1225,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_tree_text_lor_find.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_map
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::map</code>
@@ -1313,7 +1313,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_tree_split_join.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_set
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::set</code>
@@ -1416,7 +1416,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_tree_order_statistics.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_set
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::set</code>
@@ -1489,7 +1489,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_small_s2p_tree.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::multimap</code>
@@ -1551,7 +1551,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_small_s2p_hash.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_hash_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_multimap</code>
@@ -1660,7 +1660,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_large_s2p_tree.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::multimap</code>
@@ -1722,7 +1722,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_large_s2p_hash.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_hash_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_multimap</code>
@@ -1834,7 +1834,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_insert_small_s2p_tree.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::multimap</code>
@@ -1896,7 +1896,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_small_s2p_hash.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_hash_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_multimap</code>
@@ -2008,7 +2008,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_insert_large_s2p_tree.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::multimap</code>
@@ -2070,7 +2070,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_large_s2p_hash.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_hash_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_multimap</code>
@@ -2177,7 +2177,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_insert_mem_small_s2p_tree.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::multimap</code>
@@ -2239,7 +2239,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_large_s2p_hash.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_hash_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_multimap</code>
@@ -2346,7 +2346,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_insert_mem_large_s2p_tree.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::multimap</code>
@@ -2408,7 +2408,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_multimap_text_find_large_s2p_hash.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="7" align="left">
 		    n_hash_mmap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::tr1::unordered_multimap</code>
@@ -2514,7 +2514,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_text_push.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -2576,7 +2576,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_pairing_priority_queue_text_push.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -2650,7 +2650,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_text_push_pop.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -2712,7 +2712,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_pairing_priority_queue_text_push_pop.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code> adapting <code class="classname">std::vector</code>
@@ -2779,7 +2779,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_int_push.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -2841,7 +2841,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_binary_priority_queue_int_push.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code> adapting <code class="classname">std::vector</code>
@@ -2896,7 +2896,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_int_push_pop.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -2996,7 +2996,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_text_pop_mem.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -3089,7 +3089,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_text_join.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -3188,7 +3188,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_text_modify_up.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -3250,7 +3250,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_pairing_priority_queue_text_modify_up_thin.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    thin_heap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">priority_queue</code>
@@ -3321,7 +3321,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_priority_queue_text_modify_down.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    n_pq_vector
 		  </td></tr><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
@@ -3383,7 +3383,7 @@
 	  </p><div class="informalfigure"><div class="mediaobject" align="center"><img src="../images/pbds_pairing_priority_queue_text_modify_down_thin.png" align="middle" /></div></div><p>
 	    The abbreviated names in the legend of the graphic above are
 	    instantiated with the types in the following table.
-	  </p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
+	  </p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /></colgroup><thead><tr><th align="left"><span class="emphasis"><em>Name/Instantiating Type</em></span></th><th align="left"><span class="emphasis"><em>Parameter</em></span></th><th align="left"><span class="emphasis"><em>Details</em></span></th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td colspan="3" align="left">
 		    thin_heap
 		  </td></tr><tr><td align="left">
 		    <code class="classname">priority_queue</code>
@@ -3588,7 +3588,7 @@
 	  underlying data structures in terms of orders of growth. It is
 	  interesting to note that this table implies something about the
 	  constants of the operations as well (see Amortized <code class="function">push</code>
-	  and <code class="function">pop</code> operations).</p><div class="informaltable"><table border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /></colgroup><thead><tr><th align="left"> </th><th align="left"><span class="emphasis"><em><code class="function">push</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">pop</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">modify</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">erase</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">join</code></em></span></th></tr></thead><tbody><tr><td align="left">
+	  and <code class="function">pop</code> operations).</p><div class="informaltable"><table class="informaltable" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /></colgroup><thead><tr><th align="left"> </th><th align="left"><span class="emphasis"><em><code class="function">push</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">pop</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">modify</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">erase</code></em></span></th><th align="left"><span class="emphasis"><em><code class="function">join</code></em></span></th></tr></thead><tbody><tr><td align="left">
 		    <code class="classname">std::priority_queue</code>
 		  </td><td align="left">
 		    Θ(n) worst
diff --git a/libstdc++-v3/doc/html/manual/policy_data_structures.html b/libstdc++-v3/doc/html/manual/policy_data_structures.html
index f3d0fb59bf7..8b2b87d320e 100644
--- a/libstdc++-v3/doc/html/manual/policy_data_structures.html
+++ b/libstdc++-v3/doc/html/manual/policy_data_structures.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 22. Policy-Based Data Structures</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="bitmap_allocator_impl.html" title="Implementation" /><link rel="next" href="policy_data_structures_using.html" title="Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 22. Policy-Based Data Structures</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bitmap_allocator_impl.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 22. Policy-Based Data Structures</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="bitmap_allocator_impl.html" title="Implementation" /><link rel="next" href="policy_data_structures_using.html" title="Using" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 22. Policy-Based Data Structures</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bitmap_allocator_impl.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="policy_data_structures_using.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.containers.pbds"></a>Chapter 22. Policy-Based Data Structures</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="policy_data_structures.html#pbds.intro">Intro</a></span></dt><dd><dl><dt><span class="section"><a href="policy_data_structures.html#pbds.intro.issues">Performance Issues</a></span></dt><dd><dl><dt><span class="section"><a href="policy_data_structures.html#pbds.intro.issues.associative">Associative</a></span></dt><dt><span class="section"><a href="policy_data_structures.html#pbds.intro.issues.priority_queue">Priority Que</a></span></dt></dl></dd><dt><span class="section"><a href="policy_data_structures.html#pbds.intro.motivation">Goals</a></span></dt><dd><dl><dt><span class="section"><a href="policy_data_structures.html#pbds.intro.motivation.associative">Associative</a></span></dt><dd><dl><dt><span class="section"><a href="policy_data_structures.html#motivation.associative.policy">Policy Choices</a></span></dt><dt><span class="section"><a href="policy_data_structures.html#motivation.associative.underlying">Underlying Data Structures</a></span></dt><dt><span class="section"><a href="policy_data_structures.html#motivation.associative.iterators">Iterators</a></span></dt><dt><span class="section"><a href="policy_data_structures.html#motivation.associative.functions">Functional</a></span></dt></dl></dd><dt><span class="section"><a href="policy_data_structures.html#pbds.intro.motivation.priority_queue">Priority Queues</a></span></dt><dd><dl><dt><span class="section"><a href="policy_data_structures.html#motivation.priority_queue.policy">Policy Choices</a></span></dt><dt><span class="section"><a href="policy_data_structures.html#motivation.priority_queue.underlying">Underlying Data Structures</a></span></dt><dt><span class="section"><a href="policy_data_structures.html#motivation.priority_queue.binary_heap">Binary Heaps</a></span></dt></dl></dd></dl></dd></dl></dd><dt><span class="section"><a href="policy_data_structures_using.html">Using</a></span></dt><dd><dl><dt><span class="section"><a href="policy_data_structures_using.html#pbds.using.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="policy_data_structures_using.html#pbds.using.organization">Organization</a></span></dt><dt><span class="section"><a href="policy_data_structures_using.html#pbds.using.tutorial">Tutorial</a></span></dt><dd><dl><dt><span class="section"><a href="policy_data_structures_using.html#pbds.using.tutorial.basic">Basic Use</a></span></dt><dt><span class="section"><a href="policy_data_structures_using.html#pbds.using.tutorial.configuring">
diff --git a/libstdc++-v3/doc/html/manual/policy_data_structures_ack.html b/libstdc++-v3/doc/html/manual/policy_data_structures_ack.html
index e450da28cb6..55fbb4d2c92 100644
--- a/libstdc++-v3/doc/html/manual/policy_data_structures_ack.html
+++ b/libstdc++-v3/doc/html/manual/policy_data_structures_ack.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Acknowledgments</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_based_data_structures_test.html" title="Testing" /><link rel="next" href="ext_containers.html" title="Chapter 23. HP/SGI Extensions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Acknowledgments</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_based_data_structures_test.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="ext_containers.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="pbds.ack"></a>Acknowledgments</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Acknowledgments</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_based_data_structures_test.html" title="Testing" /><link rel="next" href="ext_containers.html" title="Chapter 23. HP/SGI Extensions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Acknowledgments</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_based_data_structures_test.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="ext_containers.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="pbds.ack"></a>Acknowledgments</h2></div></div></div><p>
       Written by Ami Tavory and Vladimir Dreizin (IBM Haifa Research
       Laboratories), and Benjamin Kosnik (Red Hat).
     </p><p>
diff --git a/libstdc++-v3/doc/html/manual/policy_data_structures_design.html b/libstdc++-v3/doc/html/manual/policy_data_structures_design.html
index b0b983899e4..c6d6a3a4dfc 100644
--- a/libstdc++-v3/doc/html/manual/policy_data_structures_design.html
+++ b/libstdc++-v3/doc/html/manual/policy_data_structures_design.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_data_structures_using.html" title="Using" /><link rel="next" href="policy_based_data_structures_test.html" title="Testing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures_using.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="policy_based_data_structures_test.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="containers.pbds.design"></a>Design</h2></div></div></div><p></p><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="pbds.design.concepts"></a>Concepts</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="pbds.design.concepts.null_type"></a>Null Policy Classes</h4></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_data_structures_using.html" title="Using" /><link rel="next" href="policy_based_data_structures_test.html" title="Testing" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures_using.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="policy_based_data_structures_test.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="containers.pbds.design"></a>Design</h2></div></div></div><p></p><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="pbds.design.concepts"></a>Concepts</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="pbds.design.concepts.null_type"></a>Null Policy Classes</h4></div></div></div><p>
 	Associative containers are typically parametrized by various
 	policies. For example, a hash-based associative container is
 	parametrized by a hash-functor, transforming each key into an
diff --git a/libstdc++-v3/doc/html/manual/policy_data_structures_using.html b/libstdc++-v3/doc/html/manual/policy_data_structures_using.html
index a698152ef84..a45c64544b0 100644
--- a/libstdc++-v3/doc/html/manual/policy_data_structures_using.html
+++ b/libstdc++-v3/doc/html/manual/policy_data_structures_using.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Using</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="next" href="policy_data_structures_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="policy_data_structures_design.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="containers.pbds.using"></a>Using</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="pbds.using.prereq"></a>Prerequisites</h3></div></div></div><p>The library contains only header files, and does not require any
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Using</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, policy, container, data, structure, associated, tree, trie, hash, metaprogramming" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="prev" href="policy_data_structures.html" title="Chapter 22. Policy-Based Data Structures" /><link rel="next" href="policy_data_structures_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="policy_data_structures.html">Prev</a> </td><th width="60%" align="center">Chapter 22. Policy-Based Data Structures</th><td width="20%" align="right"> <a accesskey="n" href="policy_data_structures_design.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="containers.pbds.using"></a>Using</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="pbds.using.prereq"></a>Prerequisites</h3></div></div></div><p>The library contains only header files, and does not require any
       other libraries except the standard C++ library . All classes are
       defined in namespace <code class="code">__gnu_pbds</code>. The library internally
       uses macros beginning with <code class="code">PB_DS</code>, but
diff --git a/libstdc++-v3/doc/html/manual/profile_mode.html b/libstdc++-v3/doc/html/manual/profile_mode.html
index 3f32b30b9f8..5c4bdec172a 100644
--- a/libstdc++-v3/doc/html/manual/profile_mode.html
+++ b/libstdc++-v3/doc/html/manual/profile_mode.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 19. Profile Mode</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="parallel_mode_test.html" title="Testing" /><link rel="next" href="profile_mode_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 19. Profile Mode</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_test.html">Prev</a> </td><th width="60%" align="center">Part III. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 19. Profile Mode</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="extensions.html" title="Part III.  Extensions" /><link rel="prev" href="parallel_mode_test.html" title="Testing" /><link rel="next" href="profile_mode_design.html" title="Design" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 19. Profile Mode</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="parallel_mode_test.html">Prev</a> </td><th width="60%" align="center">Part III. 
   Extensions
   
 </th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_design.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.ext.profile_mode"></a>Chapter 19. Profile Mode</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="profile_mode.html#manual.ext.profile_mode.intro">Intro</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode.html#manual.ext.profile_mode.using">Using the Profile Mode</a></span></dt><dt><span class="section"><a href="profile_mode.html#manual.ext.profile_mode.tuning">Tuning the Profile Mode</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_design.html">Design</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_design.html#manual.ext.profile_mode.design.wrapper">Wrapper Model</a></span></dt><dt><span class="section"><a href="profile_mode_design.html#manual.ext.profile_mode.design.instrumentation">Instrumentation</a></span></dt><dt><span class="section"><a href="profile_mode_design.html#manual.ext.profile_mode.design.rtlib">Run Time Behavior</a></span></dt><dt><span class="section"><a href="profile_mode_design.html#manual.ext.profile_mode.design.analysis">Analysis and Diagnostics</a></span></dt><dt><span class="section"><a href="profile_mode_design.html#manual.ext.profile_mode.design.cost-model">Cost Model</a></span></dt><dt><span class="section"><a href="profile_mode_design.html#manual.ext.profile_mode.design.reports">Reports</a></span></dt><dt><span class="section"><a href="profile_mode_design.html#manual.ext.profile_mode.design.testing">Testing</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_api.html">Extensions for Custom Containers</a></span></dt><dt><span class="section"><a href="profile_mode_cost_model.html">Empirical Cost Model</a></span></dt><dt><span class="section"><a href="profile_mode_impl.html">Implementation Issues</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_impl.html#manual.ext.profile_mode.implementation.stack">Stack Traces</a></span></dt><dt><span class="section"><a href="profile_mode_impl.html#manual.ext.profile_mode.implementation.symbols">Symbolization of Instruction Addresses</a></span></dt><dt><span class="section"><a href="profile_mode_impl.html#manual.ext.profile_mode.implementation.concurrency">Concurrency</a></span></dt><dt><span class="section"><a href="profile_mode_impl.html#manual.ext.profile_mode.implementation.stdlib-in-proflib">Using the Standard Library in the Instrumentation Implementation</a></span></dt><dt><span class="section"><a href="profile_mode_impl.html#manual.ext.profile_mode.implementation.malloc-hooks">Malloc Hooks</a></span></dt><dt><span class="section"><a href="profile_mode_impl.html#manual.ext.profile_mode.implementation.construction-destruction">Construction and Destruction of Global Objects</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_devel.html">Developer Information</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_devel.html#manual.ext.profile_mode.developer.bigpic">Big Picture</a></span></dt><dt><span class="section"><a href="profile_mode_devel.html#manual.ext.profile_mode.developer.howto">How To Add A Diagnostic</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_diagnostics.html">Diagnostics</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.template">Diagnostic Template</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.containers">Containers</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.hashtable_too_small">Hashtable Too Small</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.hashtable_too_large">Hashtable Too Large</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.inefficient_hash">Inefficient Hash</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.vector_too_small">Vector Too Small</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.vector_too_large">Vector Too Large</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.vector_to_hashtable">Vector to Hashtable</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.hashtable_to_vector">Hashtable to Vector</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.vector_to_list">Vector to List</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.list_to_vector">List to Vector</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.list_to_slist">List to Forward List (Slist)</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.assoc_ord_to_unord">Ordered to Unordered Associative Container</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.algorithms">Algorithms</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.algorithms.sort">Sort Algorithm Performance</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.locality">Data Locality</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.locality.sw_prefetch">Need Software Prefetch</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.locality.linked">Linked Structure Locality</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.mthread">Multithreaded Data Access</a></span></dt><dd><dl><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.mthread.ddtest">Data Dependence Violations at Container Level</a></span></dt><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.mthread.false_share">False Sharing</a></span></dt></dl></dd><dt><span class="section"><a href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.statistics">Statistics</a></span></dt></dl></dd><dt><span class="bibliography"><a href="profile_mode.html#profile_mode.biblio">Bibliography</a></span></dt></dl></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.intro"></a>Intro</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/profile_mode_api.html b/libstdc++-v3/doc/html/manual/profile_mode_api.html
index ea0c4acd51a..e63bd5701c6 100644
--- a/libstdc++-v3/doc/html/manual/profile_mode_api.html
+++ b/libstdc++-v3/doc/html/manual/profile_mode_api.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Extensions for Custom Containers</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_design.html" title="Design" /><link rel="next" href="profile_mode_cost_model.html" title="Empirical Cost Model" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Extensions for Custom Containers</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_design.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_cost_model.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.api"></a>Extensions for Custom Containers</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Extensions for Custom Containers</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_design.html" title="Design" /><link rel="next" href="profile_mode_cost_model.html" title="Empirical Cost Model" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Extensions for Custom Containers</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_design.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_cost_model.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.api"></a>Extensions for Custom Containers</h2></div></div></div><p>
   Many large projects use their own data structures instead of the ones in the
   standard library.  If these data structures are similar in functionality
   to the standard library, they can be instrumented with the same hooks
diff --git a/libstdc++-v3/doc/html/manual/profile_mode_cost_model.html b/libstdc++-v3/doc/html/manual/profile_mode_cost_model.html
index f42e879366c..bc87048b4df 100644
--- a/libstdc++-v3/doc/html/manual/profile_mode_cost_model.html
+++ b/libstdc++-v3/doc/html/manual/profile_mode_cost_model.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Empirical Cost Model</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_api.html" title="Extensions for Custom Containers" /><link rel="next" href="profile_mode_impl.html" title="Implementation Issues" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Empirical Cost Model</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_api.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_impl.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.cost_model"></a>Empirical Cost Model</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Empirical Cost Model</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_api.html" title="Extensions for Custom Containers" /><link rel="next" href="profile_mode_impl.html" title="Implementation Issues" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Empirical Cost Model</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_api.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_impl.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.cost_model"></a>Empirical Cost Model</h2></div></div></div><p>
   Currently, the cost model uses formulas with predefined relative weights
   for alternative containers or container implementations.  For instance,
   iterating through a vector is X times faster than iterating through a list.
diff --git a/libstdc++-v3/doc/html/manual/profile_mode_design.html b/libstdc++-v3/doc/html/manual/profile_mode_design.html
index 8fe9eddba2a..03b7a79712a 100644
--- a/libstdc++-v3/doc/html/manual/profile_mode_design.html
+++ b/libstdc++-v3/doc/html/manual/profile_mode_design.html
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="next" href="profile_mode_api.html" title="Extensions for Custom Containers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_api.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.design"></a>Design</h2></div></div></div><p>
-</p><div class="table"><a id="table.profile_code_loc"></a><p class="title"><strong>Table 19.1. Profile Code Location</strong></p><div class="table-contents"><table summary="Profile Code Location" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">Code Location</th><th align="left">Use</th></tr></thead><tbody><tr><td align="left"><code class="code">libstdc++-v3/include/std/*</code></td><td align="left">Preprocessor code to redirect to profile extension headers.</td></tr><tr><td align="left"><code class="code">libstdc++-v3/include/profile/*</code></td><td align="left">Profile extension public headers (map, vector, ...).</td></tr><tr><td align="left"><code class="code">libstdc++-v3/include/profile/impl/*</code></td><td align="left">Profile extension internals.  Implementation files are
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="next" href="profile_mode_api.html" title="Extensions for Custom Containers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_api.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.design"></a>Design</h2></div></div></div><p>
+</p><div class="table"><a id="table.profile_code_loc"></a><p class="title"><strong>Table 19.1. Profile Code Location</strong></p><div class="table-contents"><table class="table" summary="Profile Code Location" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">Code Location</th><th align="left">Use</th></tr></thead><tbody><tr><td align="left"><code class="code">libstdc++-v3/include/std/*</code></td><td align="left">Preprocessor code to redirect to profile extension headers.</td></tr><tr><td align="left"><code class="code">libstdc++-v3/include/profile/*</code></td><td align="left">Profile extension public headers (map, vector, ...).</td></tr><tr><td align="left"><code class="code">libstdc++-v3/include/profile/impl/*</code></td><td align="left">Profile extension internals.  Implementation files are
      only included from <code class="code">impl/profiler.h</code>, which is the only
      file included from the public headers.</td></tr></tbody></table></div></div><br class="table-break" /><p>
 </p><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.profile_mode.design.wrapper"></a>Wrapper Model</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/profile_mode_devel.html b/libstdc++-v3/doc/html/manual/profile_mode_devel.html
index db96f989b24..768c610ba80 100644
--- a/libstdc++-v3/doc/html/manual/profile_mode_devel.html
+++ b/libstdc++-v3/doc/html/manual/profile_mode_devel.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Developer Information</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_impl.html" title="Implementation Issues" /><link rel="next" href="profile_mode_diagnostics.html" title="Diagnostics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Developer Information</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_impl.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_diagnostics.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.developer"></a>Developer Information</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.profile_mode.developer.bigpic"></a>Big Picture</h3></div></div></div><p>The profile mode headers are included with
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Developer Information</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_impl.html" title="Implementation Issues" /><link rel="next" href="profile_mode_diagnostics.html" title="Diagnostics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Developer Information</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_impl.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_diagnostics.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.developer"></a>Developer Information</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.profile_mode.developer.bigpic"></a>Big Picture</h3></div></div></div><p>The profile mode headers are included with
    <code class="code">-D_GLIBCXX_PROFILE</code> through preprocessor directives in
    <code class="code">include/std/*</code>.
   </p><p>Instrumented implementations are provided in
diff --git a/libstdc++-v3/doc/html/manual/profile_mode_diagnostics.html b/libstdc++-v3/doc/html/manual/profile_mode_diagnostics.html
index f32edfbe461..9a0fa51dec6 100644
--- a/libstdc++-v3/doc/html/manual/profile_mode_diagnostics.html
+++ b/libstdc++-v3/doc/html/manual/profile_mode_diagnostics.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Diagnostics</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_devel.html" title="Developer Information" /><link rel="next" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Diagnostics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_devel.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.diagnostics"></a>Diagnostics</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Diagnostics</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_devel.html" title="Developer Information" /><link rel="next" href="mt_allocator.html" title="Chapter 20. The mt_allocator" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Diagnostics</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_devel.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="mt_allocator.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.diagnostics"></a>Diagnostics</h2></div></div></div><p>
   The table below presents all the diagnostics we intend to implement.
   Each diagnostic has a corresponding compile time switch
   <code class="code">-D_GLIBCXX_PROFILE_&lt;diagnostic&gt;</code>.
@@ -17,7 +17,7 @@
   A high accuracy means that the diagnostic is unlikely to be wrong.
   These grades are not perfect.  They are just meant to guide users with
   specific needs or time budgets.
-  </p><div class="table"><a id="table.profile_diagnostics"></a><p class="title"><strong>Table 19.2. Profile Diagnostics</strong></p><div class="table-contents"><table summary="Profile Diagnostics" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left">Group</th><th align="left">Flag</th><th align="left">Benefit</th><th align="left">Cost</th><th align="left">Freq.</th><th align="left">Implemented</th><td class="auto-generated"> </td></tr></thead><tbody><tr><td align="left"><a class="link" href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.containers" title="Containers">
+  </p><div class="table"><a id="table.profile_diagnostics"></a><p class="title"><strong>Table 19.2. Profile Diagnostics</strong></p><div class="table-contents"><table class="table" summary="Profile Diagnostics" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /><col align="left" class="c6" /><col align="left" class="c7" /></colgroup><thead><tr><th align="left">Group</th><th align="left">Flag</th><th align="left">Benefit</th><th align="left">Cost</th><th align="left">Freq.</th><th align="left">Implemented</th><td class="auto-generated"> </td></tr></thead><tbody><tr><td align="left"><a class="link" href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.containers" title="Containers">
     CONTAINERS</a></td><td align="left"><a class="link" href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.hashtable_too_small" title="Hashtable Too Small">
     HASHTABLE_TOO_SMALL</a></td><td align="left">10</td><td align="left">1</td><td align="left"> </td><td align="left">10</td><td align="left">yes</td></tr><tr><td align="left"> </td><td align="left"><a class="link" href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.hashtable_too_large" title="Hashtable Too Large">
     HASHTABLE_TOO_LARGE</a></td><td align="left">5</td><td align="left">1</td><td align="left"> </td><td align="left">10</td><td align="left">yes</td></tr><tr><td align="left"> </td><td align="left"><a class="link" href="profile_mode_diagnostics.html#manual.ext.profile_mode.analysis.inefficient_hash" title="Inefficient Hash">
diff --git a/libstdc++-v3/doc/html/manual/profile_mode_impl.html b/libstdc++-v3/doc/html/manual/profile_mode_impl.html
index 1644f3e57a7..e9495273d52 100644
--- a/libstdc++-v3/doc/html/manual/profile_mode_impl.html
+++ b/libstdc++-v3/doc/html/manual/profile_mode_impl.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation Issues</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_cost_model.html" title="Empirical Cost Model" /><link rel="next" href="profile_mode_devel.html" title="Developer Information" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation Issues</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_cost_model.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_devel.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.implementation"></a>Implementation Issues</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.profile_mode.implementation.stack"></a>Stack Traces</h3></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Implementation Issues</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, library, profile" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="profile_mode.html" title="Chapter 19. Profile Mode" /><link rel="prev" href="profile_mode_cost_model.html" title="Empirical Cost Model" /><link rel="next" href="profile_mode_devel.html" title="Developer Information" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Implementation Issues</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="profile_mode_cost_model.html">Prev</a> </td><th width="60%" align="center">Chapter 19. Profile Mode</th><td width="20%" align="right"> <a accesskey="n" href="profile_mode_devel.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.ext.profile_mode.implementation"></a>Implementation Issues</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.ext.profile_mode.implementation.stack"></a>Stack Traces</h3></div></div></div><p>
   Accurate stack traces are needed during profiling since we group events by
   call context and dynamic instance.  Without accurate traces, diagnostics
   may be hard to interpret.  For instance, when giving advice to the user
diff --git a/libstdc++-v3/doc/html/manual/setup.html b/libstdc++-v3/doc/html/manual/setup.html
index 61e8bc644d0..636a4401602 100644
--- a/libstdc++-v3/doc/html/manual/setup.html
+++ b/libstdc++-v3/doc/html/manual/setup.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 2. Setup</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="intro.html" title="Part I.  Introduction" /><link rel="prev" href="bugs.html" title="Bugs" /><link rel="next" href="configure.html" title="Configure" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 2. Setup</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bugs.html">Prev</a> </td><th width="60%" align="center">Part I. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 2. Setup</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="intro.html" title="Part I.  Introduction" /><link rel="prev" href="bugs.html" title="Bugs" /><link rel="next" href="configure.html" title="Configure" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 2. Setup</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="bugs.html">Prev</a> </td><th width="60%" align="center">Part I. 
   Introduction
   
 </th><td width="20%" align="right"> <a accesskey="n" href="configure.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.intro.setup"></a>Chapter 2. Setup</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="setup.html#manual.intro.setup.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="configure.html">Configure</a></span></dt><dt><span class="section"><a href="make.html">Make</a></span></dt></dl></div><p>To transform libstdc++ sources into installed include files
diff --git a/libstdc++-v3/doc/html/manual/source_code_style.html b/libstdc++-v3/doc/html/manual/source_code_style.html
index e17530c1763..19224b7ec13 100644
--- a/libstdc++-v3/doc/html/manual/source_code_style.html
+++ b/libstdc++-v3/doc/html/manual/source_code_style.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Coding Style</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="prev" href="source_organization.html" title="Directory Layout and Source Conventions" /><link rel="next" href="source_design_notes.html" title="Design Notes" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Coding Style</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="source_organization.html">Prev</a> </td><th width="60%" align="center">Appendix A. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Coding Style</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="prev" href="source_organization.html" title="Directory Layout and Source Conventions" /><link rel="next" href="source_design_notes.html" title="Design Notes" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Coding Style</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="source_organization.html">Prev</a> </td><th width="60%" align="center">Appendix A. 
   Contributing
   
 </th><td width="20%" align="right"> <a accesskey="n" href="source_design_notes.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="contrib.coding_style"></a>Coding Style</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/source_design_notes.html b/libstdc++-v3/doc/html/manual/source_design_notes.html
index dcc3ac33ba6..341da0d3ed9 100644
--- a/libstdc++-v3/doc/html/manual/source_design_notes.html
+++ b/libstdc++-v3/doc/html/manual/source_design_notes.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design Notes</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="prev" href="source_code_style.html" title="Coding Style" /><link rel="next" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design Notes</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="source_code_style.html">Prev</a> </td><th width="60%" align="center">Appendix A. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Design Notes</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="prev" href="source_code_style.html" title="Coding Style" /><link rel="next" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Design Notes</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="source_code_style.html">Prev</a> </td><th width="60%" align="center">Appendix A. 
   Contributing
   
 </th><td width="20%" align="right"> <a accesskey="n" href="appendix_porting.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="contrib.design_notes"></a>Design Notes</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/source_organization.html b/libstdc++-v3/doc/html/manual/source_organization.html
index 6b8cc450dde..92044484f69 100644
--- a/libstdc++-v3/doc/html/manual/source_organization.html
+++ b/libstdc++-v3/doc/html/manual/source_organization.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Directory Layout and Source Conventions</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="prev" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="next" href="source_code_style.html" title="Coding Style" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Directory Layout and Source Conventions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix_contributing.html">Prev</a> </td><th width="60%" align="center">Appendix A. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Directory Layout and Source Conventions</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="prev" href="appendix_contributing.html" title="Appendix A.  Contributing" /><link rel="next" href="source_code_style.html" title="Coding Style" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Directory Layout and Source Conventions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="appendix_contributing.html">Prev</a> </td><th width="60%" align="center">Appendix A. 
   Contributing
   
 </th><td width="20%" align="right"> <a accesskey="n" href="source_code_style.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="contrib.organization"></a>Directory Layout and Source Conventions</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/status.html b/libstdc++-v3/doc/html/manual/status.html
index 0bc72aea674..b0127f82135 100644
--- a/libstdc++-v3/doc/html/manual/status.html
+++ b/libstdc++-v3/doc/html/manual/status.html
@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 1. Status</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="intro.html" title="Part I.  Introduction" /><link rel="prev" href="intro.html" title="Part I.  Introduction" /><link rel="next" href="license.html" title="License" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 1. Status</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="intro.html">Prev</a> </td><th width="60%" align="center">Part I. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 1. Status</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="intro.html" title="Part I.  Introduction" /><link rel="prev" href="intro.html" title="Part I.  Introduction" /><link rel="next" href="license.html" title="License" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 1. Status</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="intro.html">Prev</a> </td><th width="60%" align="center">Part I. 
   Introduction
   
-</th><td width="20%" align="right"> <a accesskey="n" href="license.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.intro.status"></a>Chapter 1. Status</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="status.html#status.iso.201z">C++ 201z</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr24733">C++ TR 24733</a></span></dt></dl></dd><dt><span class="section"><a href="license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.status.iso"></a>Implementation Status</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="status.iso.1998"></a>C++ 1998/2003</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="iso.1998.status"></a>Implementation Status</h4></div></div></div><p>
+</th><td width="20%" align="right"> <a accesskey="n" href="license.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.intro.status"></a>Chapter 1. Status</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="status.html#manual.intro.status.iso">Implementation Status</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#status.iso.1998">C++ 1998/2003</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.1998.status">Implementation Status</a></span></dt><dt><span class="section"><a href="status.html#iso.1998.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2011">C++ 2011</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2011.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.2014">C++ 2014</a></span></dt><dt><span class="section"><a href="status.html#status.iso.2017">C++ 2017</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.2017.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr1">C++ TR1</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.tr1.specific">Implementation Specific Behavior</a></span></dt></dl></dd><dt><span class="section"><a href="status.html#status.iso.tr24733">C++ TR 24733</a></span></dt><dt><span class="section"><a href="status.html#status.iso.specfun">C++ IS 29124</a></span></dt><dd><dl><dt><span class="section"><a href="status.html#iso.specfun.specific">Implementation Specific Behavior</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="license.html">License</a></span></dt><dd><dl><dt><span class="section"><a href="license.html#manual.intro.status.license.gpl">The Code: GPL</a></span></dt><dt><span class="section"><a href="license.html#manual.intro.status.license.fdl">The Documentation: GPL, FDL</a></span></dt></dl></dd><dt><span class="section"><a href="bugs.html">Bugs</a></span></dt><dd><dl><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.impl">Implementation Bugs</a></span></dt><dt><span class="section"><a href="bugs.html#manual.intro.status.bugs.iso">Standard Bugs</a></span></dt></dl></dd></dl></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.status.iso"></a>Implementation Status</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="status.iso.1998"></a>C++ 1998/2003</h3></div></div></div><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="iso.1998.status"></a>Implementation Status</h4></div></div></div><p>
 This status table is based on the table of contents of ISO/IEC 14882:2003.
 </p><p>
 This page describes the C++ support in mainline GCC SVN, not in any
 particular release.
-</p><div class="table"><a id="table.cxx98_status"></a><p class="title"><strong>Table 1.1. C++ 1998/2003 Implementation Status</strong></p><div class="table-contents"><table summary="C++ 1998/2003 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
+</p><div class="table"><a id="table.cxx98_status"></a><p class="title"><strong>Table 1.1. C++ 1998/2003 Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ 1998/2003 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
 	<span class="emphasis"><em>18</em></span>
       </td><td colspan="3" align="left">
 	<span class="emphasis"><em>Language support</em></span>
@@ -154,7 +154,7 @@ presence of the required flag.
 </p><p>
 This page describes the C++11 support in mainline GCC SVN, not in any
 particular release.
-</p><div class="table"><a id="table.cxx11_status"></a><p class="title"><strong>Table 1.2. C++ 2011 Implementation Status</strong></p><div class="table-contents"><table summary="C++ 2011 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
+</p><div class="table"><a id="table.cxx11_status"></a><p class="title"><strong>Table 1.2. C++ 2011 Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ 2011 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
 	<span class="emphasis"><em>18</em></span>
       </td><td colspan="3" align="left">
 	<span class="emphasis"><em>Language support</em></span>
@@ -387,7 +387,7 @@ presence of the required flag.
 </p><p>
 This page describes the C++14 and library TS support in mainline GCC SVN,
 not in any particular release.
-</p><div class="table"><a id="table.cxx14_status"></a><p class="title"><strong>Table 1.3. C++ 2014 Implementation Status</strong></p><div class="table-contents"><table summary="C++ 2014 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Paper</th><th align="left">Title</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
+</p><div class="table"><a id="table.cxx14_status"></a><p class="title"><strong>Table 1.3. C++ 2014 Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ 2014 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Paper</th><th align="left">Title</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
 	<a class="link" href="http://www.open-std.org/JTC1/sc22/WG21/docs/papers/2013/n3669.pdf" target="_top">
 	  N3669
 	</a>
@@ -459,7 +459,7 @@ not in any particular release.
 	<a class="link" href="http://www.open-std.org/JTC1/sc22/WG21/docs/papers/2013/n3644.pdf" target="_top">
 	  N3644
 	</a>
-      </td><td align="left">Null Forward Iterators</td><td align="left">Partial</td><td align="left">Only affects Debug Mode</td></tr></tbody></table></div></div><br class="table-break" /><div class="table"><a id="table.ts_status"></a><p class="title"><strong>Table 1.4. C++ Technical Specifications Implementation Status</strong></p><div class="table-contents"><table summary="C++ Technical Specifications Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Paper</th><th align="left">Title</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr bgcolor="#C8B0B0"><td align="left">
+      </td><td align="left">Null Forward Iterators</td><td align="left">Partial</td><td align="left">Only affects Debug Mode</td></tr></tbody></table></div></div><br class="table-break" /><div class="table"><a id="table.ts_status"></a><p class="title"><strong>Table 1.4. C++ Technical Specifications Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ Technical Specifications Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Paper</th><th align="left">Title</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr bgcolor="#C8B0B0"><td align="left">
 	<a class="link" href="http://www.open-std.org/JTC1/sc22/WG21/docs/papers/2013/n3662.html" target="_top">
 	  N3662
 	</a>
@@ -511,27 +511,27 @@ not in any particular release.
 	Link with
 	<a class="link" href="using_dynamic_or_shared.html#manual.intro.using.linkage.experimental" title="Experimental Library Extensions">
 	<code class="option">-lstdc++fs</code></a>
-      </td></tr></tbody></table></div></div><br class="table-break" /></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="status.iso.201z"></a>C++ 201z</h3></div></div></div><p>
-In this implementation the <code class="literal">-std=gnu++1z</code> or
-<code class="literal">-std=c++1z</code> flag must be used to enable language
+      </td></tr></tbody></table></div></div><br class="table-break" /></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="status.iso.2017"></a>C++ 2017</h3></div></div></div><p>
+In this implementation the <code class="literal">-std=gnu++17</code> or
+<code class="literal">-std=c++17</code> flag must be used to enable language
 and library
 features. See <a class="link" href="using.html#manual.intro.using.flags" title="Command Options">dialect</a>
 options. The pre-defined symbol
 <code class="constant">__cplusplus</code> is used to check for the
 presence of the required flag.
 </p><p>
-This section describes the C++1z and library TS support in mainline GCC SVN,
+This section describes the C++17 and library TS support in mainline GCC SVN,
 not in any particular release.
 </p><p>
 The following table lists new library features that have been accepted into
-the C++1z working draft. The "Proposal" column provides a link to the
+the C++17 working draft. The "Proposal" column provides a link to the
 ISO C++ committee proposal that describes the feature, while the "Status"
 column indicates the first version of GCC that contains an implementation of
 this feature (if it has been implemented).
 The "SD-6 Feature Test" column shows the corresponding macro or header from
 <a class="link" href="https://isocpp.org/std/standing-documents/sd-6-sg10-feature-test-recommendations" target="_top">SD-6:
 Feature-testing recommendations for C++</a>.
-</p><div class="table"><a id="table.cxx1z_status"></a><p class="title"><strong>Table 1.5. C++ 201z Implementation Status</strong></p><div class="table-contents"><table summary="C++ 201z Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Library Feature</th><th align="left">Proposal</th><th align="left">Status</th><th align="left">SD-6 Feature Test</th></tr></thead><tbody><tr bgcolor="#C8B0B0"><td align="left">
+</p><div class="table"><a id="table.cxx17_status"></a><p class="title"><strong>Table 1.5. C++ 2017 Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ 2017 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Library Feature</th><th align="left">Proposal</th><th align="left">Status</th><th align="left">SD-6 Feature Test</th></tr></thead><tbody><tr bgcolor="#C8B0B0"><td align="left">
 	<code class="code">constexpr std::hardware_{constructive,destructive}_interference_size</code>
       </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html" target="_top">
@@ -541,7 +541,7 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0137r1.html" target="_top">
 	P0137R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__cpp_lib_launder &gt;= 201606</code> </td></tr><tr><td align="left">Wording for <code class="code">std::uncaught_exceptions</code></td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__cpp_lib_launder &gt;= 201606</code> </td></tr><tr><td align="left">Wording for <code class="code">std::uncaught_exceptions</code></td><td align="left">
 	<a class="link" href="http://www.open-std.org/JTC1/sc22/WG21/docs/papers/2014/n4259.pdf" target="_top">
 	  N4259
 	</a>
@@ -549,43 +549,57 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0088r3.html" target="_top">
 	P0088R3
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__has_include(&lt;variant&gt;)</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">optional</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__has_include(&lt;variant&gt;)</code>,
+              <code class="code">__cpp_lib_variant &gt;= 201603</code>
+              (since 7.3, see Note 1)
+      </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">optional</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__has_include(&lt;optional&gt;)</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">any</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__has_include(&lt;optional&gt;)</code>,
+              <code class="code">__cpp_lib_optional &gt;= 201603</code>
+              (since 7.3, see Note 1)
+      </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">any</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__has_include(&lt;any&gt;)</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">string_view</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__has_include(&lt;any&gt;)</code>,
+              <code class="code">__cpp_lib_any &gt;= 201603</code>
+              (since 7.3, see Note 1)
+      </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">string_view</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__has_include(&lt;string_view&gt;)</code> </td></tr><tr bgcolor="#C8B0B0"><td align="left"> Library Fundamentals V1 TS Components: <code class="code">memory_resource</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__has_include(&lt;string_view&gt;)</code>,
+              <code class="code">__cpp_lib_string_view &gt;= 201603</code>
+              (since 7.3, see Note 1)
+      </td></tr><tr bgcolor="#C8B0B0"><td align="left"> Library Fundamentals V1 TS Components: <code class="code">memory_resource</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> No </td><td align="left"> <code class="code">__has_include(&lt;memory_resource&gt;)</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">apply</code> </td><td align="left">
+      </td><td align="center"> No </td><td align="left"> <code class="code">__has_include(&lt;memory_resource&gt;)</code>,
+              <code class="code">__cpp_lib_memory_resource &gt;= 201603</code>
+      </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">apply</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__cpp_lib_apply &gt;= 201603</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">shared_ptr&lt;T[]&gt;</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__cpp_lib_apply &gt;= 201603</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: <code class="code">shared_ptr&lt;T[]&gt;</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__cpp_lib_shared_ptr_arrays &gt;= 201603</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: Searchers </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__cpp_lib_shared_ptr_arrays &gt;= 201603</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: Searchers </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__cpp_lib_boyer_moore_searcher &gt;= 201603</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: Sampling </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__cpp_lib_boyer_moore_searcher &gt;= 201603</code> </td></tr><tr><td align="left"> Library Fundamentals V1 TS Components: Sampling </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0220r1.html" target="_top">
 	P0220R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> <code class="code">__cpp_lib_sample &gt;= 201603</code> </td></tr><tr><td align="left"> Constant View: A proposal for a <code class="code">std::as_const</code> helper function template	</td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> <code class="code">__cpp_lib_sample &gt;= 201603</code> </td></tr><tr><td align="left"> Constant View: A proposal for a <code class="code">std::as_const</code> helper function template	</td><td align="left">
 	<a class="link" href="" target="_top">
 	P0007R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_as_const &gt;= 201510 </code></td></tr><tr><td align="left"> Improving pair and tuple </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_as_const &gt;= 201510 </code></td></tr><tr><td align="left"> Improving pair and tuple </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4387" target="_top">
 	N4387
 	</a>
@@ -593,7 +607,7 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0209r2.pdf" target="_top">
 	P0209R2
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_make_from_tuple &gt;= 201606 </code></td></tr><tr bgcolor="#C8B0B0"><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_make_from_tuple &gt;= 201606 </code></td></tr><tr bgcolor="#C8B0B0"><td align="left">
         Removing <code class="code">auto_ptr</code>, <code class="code">random_shuffle()</code>,
         And Old <code class="code">&lt;functional&gt;</code> Stuff
       </td><td align="left">
@@ -604,15 +618,15 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0174r2.html" target="_top">
 	P0174R2
 	</a>
-      </td><td align="center"> No </td><td align="left"> </td></tr><tr><td align="left"> Making <code class="code">std::owner_less</code> more flexible </td><td align="left">
+      </td><td align="center"> No  (kept for backwards compatibility)</td><td align="left"> </td></tr><tr><td align="left"> Making <code class="code">std::owner_less</code> more flexible </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0074r0.html" target="_top">
 	P0074R0
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_transparent_operators &gt;= 201510 </code></td></tr><tr><td align="left"> <code class="code">std::addressof</code> should be constexpr </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_transparent_operators &gt;= 201510 </code></td></tr><tr><td align="left"> <code class="code">std::addressof</code> should be constexpr </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0304r0.html#2296" target="_top">
 	LWG2296
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_addressof_constexpr &gt;= 201603 </code></td></tr><tr><td align="left"> Safe conversions in <code class="code">unique_ptr&lt;T[]&gt;</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_addressof_constexpr &gt;= 201603 </code></td></tr><tr><td align="left"> Safe conversions in <code class="code">unique_ptr&lt;T[]&gt;</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4089.pdf" target="_top">
 	N4089
 	</a>
@@ -624,7 +638,7 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0033r1.html" target="_top">
 	P0033R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code">__cpp_lib_enable_shared_from_this &gt;= 201603</code></td></tr><tr><td align="left"> A proposal to add <code class="code">invoke</code> function template </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code">__cpp_lib_enable_shared_from_this &gt;= 201603</code></td></tr><tr><td align="left"> A proposal to add <code class="code">invoke</code> function template </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4169.html" target="_top">
 	N4169
 	</a>
@@ -636,23 +650,23 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0005r4.html" target="_top">
 	P0005R4
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code">__cpp_lib_not_fn &gt;= 201603</code></td></tr><tr><td align="left"> Fixes for <code class="code">not_fn</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code">__cpp_lib_not_fn &gt;= 201603</code></td></tr><tr><td align="left"> Fixes for <code class="code">not_fn</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0358r1.html" target="_top">
 	P0358R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> </td></tr><tr><td align="left"> Fixing a design mistake in the searchers interface in Library Fundamentals </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> </td></tr><tr><td align="left"> Fixing a design mistake in the searchers interface in Library Fundamentals </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0253r1.pdf" target="_top">
 	P0253R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"> </td></tr><tr><td align="left"> Extending memory management tools </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> </td></tr><tr><td align="left"> Extending memory management tools </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0040r3.html" target="_top">
 	P0040R3
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> </code></td></tr><tr><td align="left"> <code class="code">shared_ptr::weak_type</code></td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> </code></td></tr><tr><td align="left"> <code class="code">shared_ptr::weak_type</code></td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0163r0.html" target="_top">
 	P0163R0
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_shared_ptr_weak_type &gt;= 201606</code></td></tr><tr><td align="left">Transformation Trait Alias <code class="code">void_t</code></td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_shared_ptr_weak_type &gt;= 201606</code></td></tr><tr><td align="left">Transformation Trait Alias <code class="code">void_t</code></td><td align="left">
 	<a class="link" href="http://www.open-std.org/JTC1/sc22/WG21/docs/papers/2014/n3911.pdf" target="_top">
 	  N3911
 	</a>
@@ -664,7 +678,7 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0006r0.html" target="_top">
 	P0006R0
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_type_trait_variable_templates &gt;= 201510 </code></td></tr><tr><td align="left"> Logical Operator Type Traits</td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_type_trait_variable_templates &gt;= 201510 </code></td></tr><tr><td align="left"> Logical Operator Type Traits</td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0013r1.html" target="_top">
 	P0013R1
 	</a>
@@ -676,19 +690,21 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0077r2.html" target="_top">
 	P0077R2
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_is_callable &gt;= 201603 </code></td></tr><tr><td align="left"> has_unique_object_representations </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_is_callable &gt;= 201603 </code></td></tr><tr><td align="left"> has_unique_object_representations </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0258r2.html" target="_top">
 	P0258R2
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_has_unique_object_representations &gt;= 201606 </code></td></tr><tr><td align="left"> Polishing <code class="code">&lt;chrono&gt;</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_has_unique_object_representations &gt;= 201606 </code></td></tr><tr><td align="left"> Polishing <code class="code">&lt;chrono&gt;</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0092r1.html" target="_top">
 	P0092R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_chrono &gt;= 201510 </code></td></tr><tr><td align="left"> Adding more constexpr to <code class="code">&lt;chrono&gt;</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_chrono &gt;= 201510 </code></td></tr><tr><td align="left"> Adding more constexpr to <code class="code">&lt;chrono&gt;</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0505r0.html" target="_top">
 	P0505R0
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> ??? </code></td></tr><tr><td align="left"> Constexpr for <code class="code">std::char_traits</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_chrono &gt;= 201611 </code>
+             (since 7.3, see Note 2)
+      </td></tr><tr><td align="left"> Constexpr for <code class="code">std::char_traits</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0426r1.html" target="_top">
 	P0426R1
 	</a>
@@ -696,11 +712,11 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0254r2.pdf" target="_top">
 	P0254R2
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> </code></td></tr><tr><td align="left"> Give 'std::string' a non-const '.data()' member function </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> </code></td></tr><tr><td align="left"> Give 'std::string' a non-const '.data()' member function </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0272r1.html" target="_top">
 	P0272R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> </code></td></tr><tr><td align="left">Cleaning-up noexcept in the Library</td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> </code></td></tr><tr><td align="left">Cleaning-up noexcept in the Library</td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4258.pdf" target="_top">
 	  N4258
 	</a>
@@ -712,13 +728,13 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4510.html" target="_top">
 	N4510
 	</a>
-      </td><td align="center"> 6.2 (3.0)</td><td align="left"><code class="code"> __cpp_lib_incomplete_container_elements &gt;= 201505 </code>
-	     (the feature was always supported, but the macro was not defined until GCC 6.2)
+      </td><td align="center"> 3.0 </td><td align="left"><code class="code"> __cpp_lib_incomplete_container_elements &gt;= 201505 </code>
+	     (since 6.2, see Note 2)
       </td></tr><tr><td align="left"> Emplace return type </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0084r2.pdf" target="_top">
 	P0084R2
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> ??? </code></td></tr><tr><td align="left">Improved insertion interface for unique-key maps</td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"> </td></tr><tr><td align="left">Improved insertion interface for unique-key maps</td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4279.html" target="_top">
 	  N4279
 	</a>
@@ -728,7 +744,7 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0083r3.pdf" target="_top">
 	P0083R3
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_node_extract &gt;= 201606 </code></td></tr><tr><td align="left">Non-member <code class="code">size()</code> and more</td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_node_extract &gt;= 201606 </code></td></tr><tr><td align="left">Non-member <code class="code">size()</code> and more</td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4280.pdf" target="_top">
 	  N4280
 	</a>
@@ -736,67 +752,86 @@ Feature-testing recommendations for C++</a>.
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0031r0.html" target="_top">
 	P0031R0
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_array_constexpr &gt;= 201603 </code></td></tr><tr bgcolor="#C8B0B0"><td align="left"> The Parallelism TS Should be Standardized	 </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_array_constexpr &gt;= 201603 </code></td></tr><tr bgcolor="#C8B0B0"><td align="left"> The Parallelism TS Should be Standardized	 </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0024r2.html" target="_top">
 	P0024R2
 	</a>
       </td><td align="center"> No </td><td align="left"><code class="code"> __has_include(&lt;execution&gt;) </code>,
-	     <code class="code"> __cpp_lib_parallel_algorithm &gt;= 201603 </code></td></tr><tr><td align="left"> An algorithm to "clamp" a value between a pair of boundary values </td><td align="left">
+	     <code class="code"> __cpp_lib_execution &gt;= 201603 </code>,
+	     <code class="code"> __cpp_lib_parallel_algorithm &gt;= 201603 </code>
+      </td></tr><tr><td align="left"> An algorithm to "clamp" a value between a pair of boundary values </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0025r0.html" target="_top">
 	P0025R0
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_clamp &gt;= 201603 </code></td></tr><tr><td align="left"> Adopt Selected Library Fundamentals V2 Components for C++17 </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_clamp &gt;= 201603 </code></td></tr><tr><td align="left"> Adopt Selected Library Fundamentals V2 Components for C++17 </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0295r0.pdf" target="_top">
 	P0295R0
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_gcd &gt;= 201606 </code>,
-	     <code class="code"> __cpp_lib_lcm &gt;= 201606 </code>
-      </td></tr><tr><td align="left"> Proposal to Introduce a 3-Argument Overload to <code class="code">std::hypot</code> </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_gcd_lcm &gt;= 201606 </code></td></tr><tr><td align="left"> Proposal to Introduce a 3-Argument Overload to <code class="code">std::hypot</code> </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0030r1.pdf" target="_top">
 	P0030R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_hypot &gt;= 201603 </code></td></tr><tr><td align="left"> Mathematical Special Functions for C++17 </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_hypot &gt;= 201603 </code></td></tr><tr><td align="left"> Mathematical Special Functions for C++17 </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0226r1.pdf" target="_top">
 	P0226R1
 	</a>
-      </td><td align="center"> 7 (6.1) </td><td align="left"><code class="code"> __cpp_lib_math_special_functions &gt;= 201603 </code>
-	     (for GCC 6 or pre-C++17 define
-	     <code class="code">__STDCPP_WANT_MATH_SPEC_FUNCS__</code> to a non-zero value
-	     and test for <code class="code">__STDCPP_MATH_SPEC_FUNCS__ &gt;= 201003L</code>)
-      </td></tr><tr bgcolor="#C8B0B0"><td align="left">Adopt the File System TS for C++17	 </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_math_special_functions &gt;= 201603 </code>
+	     (see Note 3)
+      </td></tr><tr><td align="left">Adopt the File System TS for C++17	 </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0218r1.html" target="_top">
 	P0218R1
 	</a>
-      </td><td align="center"> No </td><td align="left"><code class="code"> __has_include(&lt;filesystem&gt;) </code>,
-	     <code class="code"> __cpp_lib_filesystem &gt;= 201603 </code></td></tr><tr bgcolor="#C8B0B0"><td align="left"> Relative Paths for Filesystem</td><td align="left">
+      </td><td align="center"> 8 </td><td align="left"><code class="code"> __has_include(&lt;filesystem&gt;) </code>,
+	     <code class="code"> __cpp_lib_filesystem &gt;= 201603 </code>
+	     (requires linking with <code class="code">-lstdc++fs</code>)
+      </td></tr><tr><td align="left"> Relative Paths for Filesystem</td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0219r1.html" target="_top">
 	P0219R1
 	</a>
-      </td><td align="center"> No </td><td align="left"> </td></tr><tr bgcolor="#C8B0B0"><td align="left"> Adapting string_view by filesystem paths </td><td align="left">
+      </td><td align="center"> 8 </td><td align="left"><code class="code"> __cpp_lib_filesystem &gt;= 201606 </code></td></tr><tr><td align="left"> Adapting string_view by filesystem paths </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0392r0.pdf" target="_top">
 	P0392R0
 	</a>
-      </td><td align="center"> No </td><td align="left"> </td></tr><tr><td align="left"> constexpr <code class="code">atomic&lt;T&gt;::is_always_lock_free</code>	 </td><td align="left">
+      </td><td align="center"> 8 </td><td align="left"><code class="code"> __cpp_lib_filesystem &gt;= 201606 </code></td></tr><tr><td align="left"> Directory Entry Caching for Filesystem </td><td align="left">
+       <a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0317r1.html" target="_top">
+       P0317R1
+       </a>
+      </td><td align="center"> 8 </td><td align="left"><code class="code"> __cpp_lib_filesystem &gt;= 201703 </code></td></tr><tr><td align="left"> constexpr <code class="code">atomic&lt;T&gt;::is_always_lock_free</code>	 </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0152r1.html" target="_top">
 	P0152R1
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_atomic_is_always_lock_free &gt;= 201603 </code></td></tr><tr><td align="left">A proposal to add <code class="code">shared_mutex</code> (untimed) (Revision 4)</td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_atomic_is_always_lock_free &gt;= 201603 </code></td></tr><tr><td align="left">A proposal to add <code class="code">shared_mutex</code> (untimed) (Revision 4)</td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4508.html" target="_top">
 	  N4508
 	</a>
-      </td><td align="center"> 6.1 </td><td align="left"><code class="code"> __cpp_lib_shared_mutex &gt;= 201505 </code></td></tr><tr><td align="left"> Variadic <code class="code">lock_guard</code> </td><td align="left">
+      </td><td align="center"> 6.1 </td><td align="left"><code class="code"> __cpp_lib_shared_mutex &gt;= 201505 </code></td></tr><tr><td align="left"> Variadic <code class="code">lock_guard</code> (Rev. 5) </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0156r2.html" target="_top">
 	P0156R2
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> __cpp_lib_scoped_lock &gt;= 201703 </code></td></tr><tr><td align="left"> byte type definition </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_scoped_lock &gt;= 201703 </code></td></tr><tr><td align="left"> A byte type definition </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0298r3.pdf" target="_top">
 	P0298R3
 	</a>
-      </td><td align="center"> 7 </td><td align="left"><code class="code"> ??? </code></td></tr><tr bgcolor="#C8B0B0"><td align="left"> Elementary string conversions </td><td align="left">
+      </td><td align="center"> 7.1 </td><td align="left"><code class="code"> __cpp_lib_byte &gt;= 201603 </code> (since 7.3, see Note 2)
+      </td></tr><tr bgcolor="#B0B0B0"><td align="left"> Elementary string conversions </td><td align="left">
 	<a class="link" href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0067r5.html" target="_top">
 	P0067R5
 	</a>
-      </td><td align="center"> No </td><td align="left"><code class="code"> ??? </code></td></tr></tbody></table></div></div><br class="table-break" /><div class="table"><a id="table.cxx1z_ts_status"></a><p class="title"><strong>Table 1.6. C++ Technical Specifications Implementation Status</strong></p><div class="table-contents"><table summary="C++ Technical Specifications Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Paper</th><th align="left">Title</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
+      </td><td align="center"> 8 (only integral types supported) </td><td align="left"><code class="code"> __has_include(&lt;charconv&gt;) </code>
+             <code class="code"> __cpp_lib_to_chars &gt;= 201611 </code></td></tr></tbody></table></div></div><br class="table-break" /><p>
+Note 1: This feature is supported in GCC 7.1 and 7.2 but before GCC 7.3 the
+<code class="code">__cpp_lib</code> macro is not defined, and compilation will fail if the
+header is included without using <code class="option">-std</code> to enable C++17 support.
+</p><p>
+Note 2: This feature is supported in older releases but the
+<code class="code">__cpp_lib</code> macro is not defined to the right value
+(or not defined at all) until the version shown in parentheses.
+</p><p>
+Note 3: The mathematical special functions are enabled in C++17 mode from
+GCC 7.1 onwards. For GCC 6.x or for C++11/C++14 define
+<code class="code">__STDCPP_WANT_MATH_SPEC_FUNCS__</code> to a non-zero value
+and test for <code class="code">__STDCPP_MATH_SPEC_FUNCS__ &gt;= 201003L</code>.
+</p><div class="table"><a id="table.cxx17_ts_status"></a><p class="title"><strong>Table 1.6. C++ Technical Specifications Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ Technical Specifications Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Paper</th><th align="left">Title</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
 	<a class="link" href="http://www.open-std.org/JTC1/SC22/WG21/docs/papers/2014/n4076.html" target="_top">
 	  N4076
 	</a>
@@ -846,69 +881,69 @@ Feature-testing recommendations for C++</a>.
       2011 Implementation Specific Behavior</a>. This section only
       documents behaviour which is new in the 2017 standard.
    </p><p>
-      <span class="emphasis"><em>20.6.5 [optional.bad_optional_access]</em></span>
+      <span class="emphasis"><em>23.6.5 [optional.bad_optional_access]</em></span>
       <code class="code">what()</code> returns <code class="literal">"bad optional access"</code>.
    </p><p>
-      <span class="emphasis"><em>20.7.2 [variant.variant]</em></span>
+      <span class="emphasis"><em>23.7.3 [variant.variant]</em></span>
       Whether <code class="classname">variant</code> supports over-aligned types
       should be documented here.
    </p><p>
-      <span class="emphasis"><em>20.7.10 [variant.bad.access]</em></span>
+      <span class="emphasis"><em>23.7.10 [variant.bad.access]</em></span>
       <code class="code">what()</code> returns <code class="literal">"Unexpected index"</code>.
    </p><p>
-      <span class="emphasis"><em>20.12.5.2 [memory.resource.pool.options]</em></span>
+      <span class="emphasis"><em>23.12.5.2 [memory.resource.pool.options]</em></span>
       The limits for maximum number of blocks and largest allocation size
       supported by <code class="classname">pool_options</code> should be documented
       here.
    </p><p>
-      <span class="emphasis"><em>20.12.6.1 [memory.resource.monotonic.buffer.ctor]</em></span>
+      <span class="emphasis"><em>23.12.6.1 [memory.resource.monotonic.buffer.ctor]</em></span>
       The default <code class="code">next_buffer_size</code> and growth factor should
       be documented here.
    </p><p>
-      <span class="emphasis"><em>20.15.4.3 [meta.unary.prop]</em></span>
+      <span class="emphasis"><em>23.15.4.3 [meta.unary.prop]</em></span>
       The predicate condition for
       <code class="code">has_unique_object_representations</code> is true for all scalar
       types except floating point types.
    </p><p>
-      <span class="emphasis"><em>20.19.3 [execpol.type],
-      25.2.3 [algorithms.parallel.exec]</em></span>
+      <span class="emphasis"><em>23.19.3 [execpol.type],
+      28.4.3 [algorithms.parallel.exec]</em></span>
       There are no implementation-defined execution policies.
    </p><p>
-      <span class="emphasis"><em>22.4.2 [string.view.template]</em></span>
+      <span class="emphasis"><em>24.4.2 [string.view.template]</em></span>
       <code class="classname">basic_string_view&lt;C, T&gt;::iterator</code> is
       <code class="code">C*</code> and
       <code class="classname">basic_string_view&lt;C, T&gt;::const_iterator</code> is
       <code class="code">const C*</code>.
    </p><p>
-      <span class="emphasis"><em>25.2.3 [algorithms.parallel.exec]</em></span>
+      <span class="emphasis"><em>28.4.3 [algorithms.parallel.exec]</em></span>
       Threads of execution created by <code class="classname">std::thread</code>
       provide concurrent forward progress guarantees, so threads of execution
       implicitly created by the library will provide parallel forward
       progress guarantees.
    </p><p>
-      <span class="emphasis"><em>26.4.1 [cfenv.syn]</em></span>
+      <span class="emphasis"><em>29.4.1 [cfenv.syn]</em></span>
       The effects of the <code class="filename">&lt;cfenv&gt;</code> functions
       depends on whether the <code class="code">FENV_ACCESS</code> pragma is supported,
       and on the C library that provides the header.
    </p><p>
-      <span class="emphasis"><em>26.6.9 [c.math.rand]</em></span>
+      <span class="emphasis"><em>29.6.9 [c.math.rand]</em></span>
       Whether the <code class="function">rand</code> function may introduce data
       races depends on the target C library that provides the function.
    </p><p>
-      <span class="emphasis"><em>26.9.5 [sf.cmath]</em></span>
+      <span class="emphasis"><em>29.9.5 [sf.cmath]</em></span>
       The effect of calling the mathematical special functions with large
       inputs should be documented here.
    </p><p>
-      <span class="emphasis"><em>27.10.2.1 [fs.conform.9945]</em></span>
+      <span class="emphasis"><em>30.10.2.1 [fs.conform.9945]</em></span>
       The behavior of the filesystem library implementation will depend on
       the target operating system. Some features will not be not supported
       on some targets.
    </p><p>
-      <span class="emphasis"><em>27.10.6 [fs.filesystem.syn]</em></span>
+      <span class="emphasis"><em>30.10.5 [fs.filesystem.syn]</em></span>
       The clock used for file times is
       <code class="classname">std::chrono::system_clock</code>.
    </p><p>
-      <span class="emphasis"><em>27.10.8 [path.generic]</em></span>
+      <span class="emphasis"><em>30.10.7.1 [fs.path.generic]</em></span>
       dot-dot in the root-directory refers to the root-directory itself.
    </p></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="status.iso.tr1"></a>C++ TR1</h3></div></div></div><p>
 This table is based on the table of contents of ISO/IEC DTR 19768
@@ -921,7 +956,7 @@ In this implementation the header names are prefixed by
 </p><p>
 This page describes the TR1 support in mainline GCC SVN, not in any particular
 release.
-</p><div class="table"><a id="table.tr1_status"></a><p class="title"><strong>Table 1.7. C++ TR1 Implementation Status</strong></p><div class="table-contents"><table summary="C++ TR1 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left"><span class="emphasis"><em>2</em></span></td><td colspan="3" align="left"><span class="emphasis"><em>General Utilities</em></span></td></tr><tr><td align="left">2.1</td><td align="left">Reference wrappers</td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left">2.1.1</td><td align="left">Additions to header <code class="code">&lt;functional&gt;</code> synopsis</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2</td><td align="left">Class template <code class="code">reference_wrapper</code></td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left">2.1.2.1</td><td align="left"><code class="code">reference_wrapper</code> construct/copy/destroy</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.2</td><td align="left"><code class="code">reference_wrapper</code> assignment</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.3</td><td align="left"><code class="code">reference_wrapper</code> access</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.4</td><td align="left"><code class="code">reference_wrapper</code> invocation</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.5</td><td align="left"><code class="code">reference_wrapper</code> helper functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.2</td><td align="left">Smart pointers</td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left">2.2.1</td><td align="left">Additions to header <code class="code">&lt;memory&gt;</code> synopsis</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.2.2</td><td align="left">Class <code class="code">bad_weak_ptr</code></td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.2.3</td><td align="left">Class template <code class="code">shared_ptr</code></td><td align="left"> </td><td align="left">
+</p><div class="table"><a id="table.tr1_status"></a><p class="title"><strong>Table 1.7. C++ TR1 Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ TR1 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left"><span class="emphasis"><em>2</em></span></td><td colspan="3" align="left"><span class="emphasis"><em>General Utilities</em></span></td></tr><tr><td align="left">2.1</td><td align="left">Reference wrappers</td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left">2.1.1</td><td align="left">Additions to header <code class="code">&lt;functional&gt;</code> synopsis</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2</td><td align="left">Class template <code class="code">reference_wrapper</code></td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left">2.1.2.1</td><td align="left"><code class="code">reference_wrapper</code> construct/copy/destroy</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.2</td><td align="left"><code class="code">reference_wrapper</code> assignment</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.3</td><td align="left"><code class="code">reference_wrapper</code> access</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.4</td><td align="left"><code class="code">reference_wrapper</code> invocation</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.1.2.5</td><td align="left"><code class="code">reference_wrapper</code> helper functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.2</td><td align="left">Smart pointers</td><td align="left"> </td><td align="left"> </td></tr><tr><td align="left">2.2.1</td><td align="left">Additions to header <code class="code">&lt;memory&gt;</code> synopsis</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.2.2</td><td align="left">Class <code class="code">bad_weak_ptr</code></td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">2.2.3</td><td align="left">Class template <code class="code">shared_ptr</code></td><td align="left"> </td><td align="left">
 	<p>
 	  Uses code from
 	  <a class="link" href="http://www.boost.org/libs/smart_ptr/shared_ptr.htm" target="_top">boost::shared_ptr</a>.
@@ -942,7 +977,7 @@ decimal floating-point arithmetic
 </p><p>
 This page describes the TR 24733 support in mainline GCC SVN, not in any
 particular release.
-</p><div class="table"><a id="table.decfp_status"></a><p class="title"><strong>Table 1.8. C++ TR 24733 Implementation Status</strong></p><div class="table-contents"><table summary="C++ TR 24733 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
+</p><div class="table"><a id="table.decfp_status"></a><p class="title"><strong>Table 1.8. C++ TR 24733 Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ TR 24733 Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr><td align="left">
 	<span class="emphasis"><em>0</em></span>
       </td><td colspan="3" align="left">
 	<span class="emphasis"><em>Introduction</em></span>
@@ -962,7 +997,74 @@ particular release.
 	<span class="emphasis"><em>4</em></span>
       </td><td colspan="3" align="left">
 	<span class="emphasis"><em>Notes on C compatibility</em></span>
-      </td></tr></tbody></table></div></div><br class="table-break" /></div></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="intro.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="intro.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="license.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Part I. 
+      </td></tr></tbody></table></div></div><br class="table-break" /></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="status.iso.specfun"></a>C++ IS 29124</h3></div></div></div><p>
+This table is based on the table of contents of ISO/IEC FDIS 29124
+Doc No: N3060 Date: 2010-03-06
+Extensions to the C++ Library to support mathematical special functions
+</p><p>
+Complete support for IS 29124 is in GCC 6.1 and later releases, when using
+at least C++11 (for older releases or C++98/C++03 use TR1 instead).
+For C++11 and C++14 the additions to the library are not declared by their
+respective headers unless <code class="code">__STDCPP_WANT_MATH_SPEC_FUNCS__</code>
+is defined as a macro that expands to a non-zero integer constant.
+For C++17 the special functions are always declared (since GCC 7.1).
+</p><p>
+When the special functions are declared the macro
+<code class="code">__STDCPP_MATH_SPEC_FUNCS__</code> is defined to <code class="code">201003L</code>.
+</p><p>
+In addition to the special functions defined in IS 29124, for
+non-strict modes (i.e. <code class="code">-std=gnu++NN</code> modes) the
+hypergeometric functions and confluent hypergeometric functions
+from TR1 are also provided, defined in namespace
+<code class="code">__gnu_cxx</code>.
+</p><div class="table"><a id="table.specfun_status"></a><p class="title"><strong>Table 1.9. C++ Special Functions Implementation Status</strong></p><div class="table-contents"><table class="table" summary="C++ Special Functions Implementation Status" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><thead><tr><th align="left">Section</th><th align="left">Description</th><th align="left">Status</th><th align="left">Comments</th></tr></thead><tbody><tr bgcolor="#B0B0B0"><td align="left">7</td><td align="left">Macro names</td><td align="left">Partial</td><td align="left">No diagnostic for inconsistent definitions of
+             <code class="code">__STDCPP_WANT_MATH_SPEC_FUNCS__</code></td></tr><tr><td align="left">8</td><td align="left">Mathematical special functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1</td><td align="left">Additions to header <code class="code">&lt;cmath&gt;</code> synopsis</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.1</td><td align="left">associated Laguerre polynomials</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.2</td><td align="left">associated Legendre functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.3</td><td align="left">beta function</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.4</td><td align="left">(complete) elliptic integral of the first kind</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.5</td><td align="left">(complete) elliptic integral of the second kind</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.6</td><td align="left">(complete) elliptic integral of the third kind</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.7</td><td align="left">regular modified cylindrical Bessel functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.8</td><td align="left">cylindrical Bessel functions (of the first kind)</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.9</td><td align="left">irregular modified cylindrical Bessel functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.10</td><td align="left">cylindrical Neumann functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.11</td><td align="left">(incomplete) elliptic integral of the first kind</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.12</td><td align="left">(incomplete) elliptic integral of the second kind</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.13</td><td align="left">(incomplete) elliptic integral of the third kind</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.14</td><td align="left">exponential integral</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.15</td><td align="left">Hermite polynomials</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.16</td><td align="left">Laguerre polynomials</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.17</td><td align="left">Legendre polynomials</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.18</td><td align="left">Riemann zeta function</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.19</td><td align="left">spherical Bessel functions (of the first kind)</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.20</td><td align="left">spherical associated Legendre functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.1.21</td><td align="left">spherical Neumann functions</td><td align="left">Y</td><td align="left"> </td></tr><tr><td align="left">8.2</td><td align="left">Additions to header <code class="code">&lt;math.h&gt;</code></td><td align="left">Y</td><td align="left"> </td></tr><tr bgcolor="#B0B0B0"><td align="left">8.3</td><td align="left">The header <code class="code">&lt;ctgmath&gt;</code></td><td align="left">Partial</td><td align="left">Conflicts with C++ 2011 requirements.</td></tr><tr bgcolor="#C8B0B0"><td align="left">8.4</td><td align="left">The header <code class="code">&lt;tgmath.h&gt;</code></td><td align="left">N</td><td align="left">Conflicts with C++ 2011 requirements.</td></tr></tbody></table></div></div><br class="table-break" /><div class="section"><div class="titlepage"><div><div><h4 class="title"><a id="iso.specfun.specific"></a>Implementation Specific Behavior</h4></div></div></div><p>For behaviour which is specified by the 2011 standard,
+      see <a class="link" href="status.html#iso.2011.specific" title="Implementation Specific Behavior">C++ 2011 Implementation
+      Specific Behavior</a>. This section documents behaviour which
+      is required by IS 29124.
+   </p><p>
+     <span class="emphasis"><em>7.2 [macro.user]/3 /4</em></span> The functions declared in
+     Clause 8 are only declared when
+     <code class="code">__STDCPP_WANT_MATH_SPEC_FUNCS__ == 1</code>
+     (or in C++17 mode, for GCC 7.1 and later).
+   </p><p>
+     <span class="emphasis"><em>8.1.1 [sf.cmath.Lnm]/1</em></span> The effect of calling
+     these functions with <code class="code">n &gt;= 128</code> or <code class="code">m &gt;= 128</code>
+     should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.2 [sf.cmath.Plm]/3</em></span> The effect of calling
+     these functions with <code class="code">l &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.3 [sf.cmath.I]/3</em></span> The effect of calling
+     these functions with <code class="code">nu &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.8 [sf.cmath.J]/3</em></span> The effect of calling
+     these functions with <code class="code">nu &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.9 [sf.cmath.K]/3</em></span> The effect of calling
+     these functions with <code class="code">nu &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.10 [sf.cmath.N]/3</em></span> The effect of calling
+     these functions with <code class="code">nu &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.15 [sf.cmath.Hn]/3</em></span> The effect of calling
+     these functions with <code class="code">n &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.16 [sf.cmath.Ln]/3</em></span> The effect of calling
+     these functions with <code class="code">n &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.17 [sf.cmath.Pl]/3</em></span> The effect of calling
+     these functions with <code class="code">l &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.19 [sf.cmath.j]/3</em></span> The effect of calling
+     these functions with <code class="code">n &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.20 [sf.cmath.Ylm]/3</em></span> The effect of calling
+     these functions with <code class="code">l &gt;= 128</code> should be described here.
+   </p><p>
+     <span class="emphasis"><em>8.1.21 [sf.cmath.n]/3</em></span> The effect of calling
+     these functions with <code class="code">n &gt;= 128</code> should be described here.
+   </p></div></div></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="intro.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="intro.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="license.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Part I. 
   Introduction
   
  </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> License</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/manual/std_contents.html b/libstdc++-v3/doc/html/manual/std_contents.html
index c7c6114aa63..cfb5db68a22 100644
--- a/libstdc++-v3/doc/html/manual/std_contents.html
+++ b/libstdc++-v3/doc/html/manual/std_contents.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part II.  Standard Contents</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="debug.html" title="Debugging Support" /><link rel="next" href="support.html" title="Chapter 4.  Support" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part II. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Part II.  Standard Contents</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="index.html" title="The GNU C++ Library Manual" /><link rel="prev" href="debug.html" title="Debugging Support" /><link rel="next" href="support.html" title="Chapter 4.  Support" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Part II. 
     Standard Contents
   </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="debug.html">Prev</a> </td><th width="60%" align="center">The GNU C++ Library Manual</th><td width="20%" align="right"> <a accesskey="n" href="support.html">Next</a></td></tr></table><hr /></div><div class="part"><div class="titlepage"><div><div><h1 class="title"><a id="manual.std"></a>Part II. 
     Standard Contents
diff --git a/libstdc++-v3/doc/html/manual/streambufs.html b/libstdc++-v3/doc/html/manual/streambufs.html
index 0dddd929e37..d3c592177f0 100644
--- a/libstdc++-v3/doc/html/manual/streambufs.html
+++ b/libstdc++-v3/doc/html/manual/streambufs.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Stream Buffers</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="io.html" title="Chapter 13.  Input and Output" /><link rel="next" href="stringstreams.html" title="Memory Based Streams" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Stream Buffers</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="io.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Stream Buffers</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="io.html" title="Chapter 13.  Input and Output" /><link rel="next" href="stringstreams.html" title="Memory Based Streams" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Stream Buffers</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="io.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
   Input and Output
   
 </th><td width="20%" align="right"> <a accesskey="n" href="stringstreams.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.io.streambufs"></a>Stream Buffers</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="io.streambuf.derived"></a>Derived streambuf Classes</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/strings.html b/libstdc++-v3/doc/html/manual/strings.html
index 075ee1b188a..492142659f9 100644
--- a/libstdc++-v3/doc/html/manual/strings.html
+++ b/libstdc++-v3/doc/html/manual/strings.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 7.  Strings</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="traits.html" title="Traits" /><link rel="next" href="localization.html" title="Chapter 8.  Localization" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 7. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 7.  Strings</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="traits.html" title="Traits" /><link rel="next" href="localization.html" title="Chapter 8.  Localization" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 7. 
   Strings
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="traits.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/stringstreams.html b/libstdc++-v3/doc/html/manual/stringstreams.html
index 2ade2bebbdf..e12e6543e74 100644
--- a/libstdc++-v3/doc/html/manual/stringstreams.html
+++ b/libstdc++-v3/doc/html/manual/stringstreams.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Memory Based Streams</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="streambufs.html" title="Stream Buffers" /><link rel="next" href="fstreams.html" title="File Based Streams" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Memory Based Streams</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="streambufs.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Memory Based Streams</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="io.html" title="Chapter 13.  Input and Output" /><link rel="prev" href="streambufs.html" title="Stream Buffers" /><link rel="next" href="fstreams.html" title="File Based Streams" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Memory Based Streams</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="streambufs.html">Prev</a> </td><th width="60%" align="center">Chapter 13. 
   Input and Output
   
 </th><td width="20%" align="right"> <a accesskey="n" href="fstreams.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.io.memstreams"></a>Memory Based Streams</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="std.io.memstreams.compat"></a>Compatibility With strstream</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/support.html b/libstdc++-v3/doc/html/manual/support.html
index 365d2964439..9ed3e5f8cbf 100644
--- a/libstdc++-v3/doc/html/manual/support.html
+++ b/libstdc++-v3/doc/html/manual/support.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 4.  Support</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="next" href="dynamic_memory.html" title="Dynamic Memory" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 4. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 4.  Support</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="next" href="dynamic_memory.html" title="Dynamic Memory" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 4. 
   Support
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="std_contents.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/html/manual/termination.html b/libstdc++-v3/doc/html/manual/termination.html
index f5a01b3f2d2..3e0d455f1c7 100644
--- a/libstdc++-v3/doc/html/manual/termination.html
+++ b/libstdc++-v3/doc/html/manual/termination.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Termination</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="support.html" title="Chapter 4.  Support" /><link rel="prev" href="dynamic_memory.html" title="Dynamic Memory" /><link rel="next" href="diagnostics.html" title="Chapter 5.  Diagnostics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Termination</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="dynamic_memory.html">Prev</a> </td><th width="60%" align="center">Chapter 4. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Termination</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="support.html" title="Chapter 4.  Support" /><link rel="prev" href="dynamic_memory.html" title="Dynamic Memory" /><link rel="next" href="diagnostics.html" title="Chapter 5.  Diagnostics" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Termination</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="dynamic_memory.html">Prev</a> </td><th width="60%" align="center">Chapter 4. 
   Support
   
 </th><td width="20%" align="right"> <a accesskey="n" href="diagnostics.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.support.termination"></a>Termination</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="support.termination.handlers"></a>Termination Handlers</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/test.html b/libstdc++-v3/doc/html/manual/test.html
index f19a8e82a2c..aed5a320b56 100644
--- a/libstdc++-v3/doc/html/manual/test.html
+++ b/libstdc++-v3/doc/html/manual/test.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Testing</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, test, testsuite, performance, conformance, ABI, exception safety" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="internals.html" title="Porting to New Hardware or Operating Systems" /><link rel="next" href="abi.html" title="ABI Policy and Guidelines" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Testing</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="internals.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Testing</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, test, testsuite, performance, conformance, ABI, exception safety" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="appendix_porting.html" title="Appendix B.  Porting and Maintenance" /><link rel="prev" href="internals.html" title="Porting to New Hardware or Operating Systems" /><link rel="next" href="abi.html" title="ABI Policy and Guidelines" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Testing</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="internals.html">Prev</a> </td><th width="60%" align="center">Appendix B. 
   Porting and Maintenance
   
 </th><td width="20%" align="right"> <a accesskey="n" href="abi.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.setup.test"></a>Testing</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/traits.html b/libstdc++-v3/doc/html/manual/traits.html
index d5cac08beb5..d2dffad05ad 100644
--- a/libstdc++-v3/doc/html/manual/traits.html
+++ b/libstdc++-v3/doc/html/manual/traits.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Traits</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="prev" href="memory.html" title="Memory" /><link rel="next" href="strings.html" title="Chapter 7.  Strings" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Traits</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="memory.html">Prev</a> </td><th width="60%" align="center">Chapter 6. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Traits</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="utilities.html" title="Chapter 6.  Utilities" /><link rel="prev" href="memory.html" title="Memory" /><link rel="next" href="strings.html" title="Chapter 7.  Strings" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Traits</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="memory.html">Prev</a> </td><th width="60%" align="center">Chapter 6. 
   Utilities
   
 </th><td width="20%" align="right"> <a accesskey="n" href="strings.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.util.traits"></a>Traits</h2></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/unordered_associative.html b/libstdc++-v3/doc/html/manual/unordered_associative.html
index c5159cb549f..8811178f814 100644
--- a/libstdc++-v3/doc/html/manual/unordered_associative.html
+++ b/libstdc++-v3/doc/html/manual/unordered_associative.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Unordered Associative</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="containers.html" title="Chapter 9.  Containers" /><link rel="prev" href="associative.html" title="Associative" /><link rel="next" href="containers_and_c.html" title="Interacting with C" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Unordered Associative</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="associative.html">Prev</a> </td><th width="60%" align="center">Chapter 9. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Unordered Associative</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="containers.html" title="Chapter 9.  Containers" /><link rel="prev" href="associative.html" title="Associative" /><link rel="next" href="containers_and_c.html" title="Interacting with C" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Unordered Associative</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="associative.html">Prev</a> </td><th width="60%" align="center">Chapter 9. 
   Containers
   
 </th><td width="20%" align="right"> <a accesskey="n" href="containers_and_c.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="std.containers.unordered"></a>Unordered Associative</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="containers.unordered.insert_hints"></a>Insertion Hints</h3></div></div></div><p>
diff --git a/libstdc++-v3/doc/html/manual/using.html b/libstdc++-v3/doc/html/manual/using.html
index c7f091d5d48..332725b2aaa 100644
--- a/libstdc++-v3/doc/html/manual/using.html
+++ b/libstdc++-v3/doc/html/manual/using.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 3. Using</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="intro.html" title="Part I.  Introduction" /><link rel="prev" href="make.html" title="Make" /><link rel="next" href="using_headers.html" title="Headers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 3. Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="make.html">Prev</a> </td><th width="60%" align="center">Part I. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 3. Using</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="intro.html" title="Part I.  Introduction" /><link rel="prev" href="make.html" title="Make" /><link rel="next" href="using_headers.html" title="Headers" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 3. Using</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="make.html">Prev</a> </td><th width="60%" align="center">Part I. 
   Introduction
   
 </th><td width="20%" align="right"> <a accesskey="n" href="using_headers.html">Next</a></td></tr></table><hr /></div><div class="chapter"><div class="titlepage"><div><div><h2 class="title"><a id="manual.intro.using"></a>Chapter 3. Using</h2></div></div></div><div class="toc"><p><strong>Table of Contents</strong></p><dl class="toc"><dt><span class="section"><a href="using.html#manual.intro.using.flags">Command Options</a></span></dt><dt><span class="section"><a href="using_headers.html">Headers</a></span></dt><dd><dl><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.all">Header Files</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.mixing">Mixing Headers</a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.cheaders">The C Headers and <code class="code">namespace std</code></a></span></dt><dt><span class="section"><a href="using_headers.html#manual.intro.using.headers.pre">Precompiled Headers</a></span></dt></dl></dd><dt><span class="section"><a href="using_macros.html">Macros</a></span></dt><dt><span class="section"><a href="using_dual_abi.html">Dual ABI</a></span></dt><dd><dl><dt><span class="section"><a href="using_dual_abi.html#manual.intro.using.abi.trouble">Troubleshooting</a></span></dt></dl></dd><dt><span class="section"><a href="using_namespaces.html">Namespaces</a></span></dt><dd><dl><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.all">Available Namespaces</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.std">namespace std</a></span></dt><dt><span class="section"><a href="using_namespaces.html#manual.intro.using.namespaces.comp">Using Namespace Composition</a></span></dt></dl></dd><dt><span class="section"><a href="using_dynamic_or_shared.html">Linking</a></span></dt><dd><dl><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.freestanding">Almost Nothing</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.dynamic">Finding Dynamic or Shared Libraries</a></span></dt><dt><span class="section"><a href="using_dynamic_or_shared.html#manual.intro.using.linkage.experimental">Experimental Library Extensions</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html">Concurrency</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.prereq">Prerequisites</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.thread_safety">Thread Safety</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.atomics">Atomics</a></span></dt><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.io">IO</a></span></dt><dd><dl><dt><span class="section"><a href="using_concurrency.html#concurrency.io.structure">Structure</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.defaults">Defaults</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.future">Future</a></span></dt><dt><span class="section"><a href="using_concurrency.html#concurrency.io.alt">Alternatives</a></span></dt></dl></dd><dt><span class="section"><a href="using_concurrency.html#manual.intro.using.concurrency.containers">Containers</a></span></dt></dl></dd><dt><span class="section"><a href="using_exceptions.html">Exceptions</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.safety">Exception Safety</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.propagating">Exception Neutrality</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.no">Doing without</a></span></dt><dt><span class="section"><a href="using_exceptions.html#intro.using.exception.compat">Compatibility</a></span></dt><dd><dl><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.c">With <code class="literal">C</code></a></span></dt><dt><span class="section"><a href="using_exceptions.html#using.exception.compat.posix">With <code class="literal">POSIX</code> thread cancellation</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="debug.html">Debugging Support</a></span></dt><dd><dl><dt><span class="section"><a href="debug.html#debug.compiler">Using <span class="command"><strong>g++</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.req">Debug Versions of Library Binary Files</a></span></dt><dt><span class="section"><a href="debug.html#debug.memory">Memory Leak Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.races">Data Race Hunting</a></span></dt><dt><span class="section"><a href="debug.html#debug.gdb">Using <span class="command"><strong>gdb</strong></span></a></span></dt><dt><span class="section"><a href="debug.html#debug.exceptions">Tracking uncaught exceptions</a></span></dt><dt><span class="section"><a href="debug.html#debug.debug_mode">Debug Mode</a></span></dt><dt><span class="section"><a href="debug.html#debug.compile_time_checks">Compile Time Checking</a></span></dt><dt><span class="section"><a href="debug.html#debug.profile_mode">Profile-based Performance Analysis</a></span></dt></dl></dd></dl></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.flags"></a>Command Options</h2></div></div></div><p>
@@ -13,7 +13,7 @@
       By default, <span class="command"><strong>g++</strong></span> is equivalent to
       <span class="command"><strong>g++ -std=gnu++14</strong></span> since GCC 6, and
       <span class="command"><strong>g++ -std=gnu++98</strong></span> for older releases.
-    </p><div class="table"><a id="table.cmd_options"></a><p class="title"><strong>Table 3.1. C++ Command Options</strong></p><div class="table-contents"><table summary="C++ Command Options" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">Option Flags</th><th align="left">Description</th></tr></thead><tbody><tr><td align="left"><code class="literal">-std=c++98</code> or <code class="literal">-std=c++03</code>
+    </p><div class="table"><a id="table.cmd_options"></a><p class="title"><strong>Table 3.1. C++ Command Options</strong></p><div class="table-contents"><table class="table" summary="C++ Command Options" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><thead><tr><th align="left">Option Flags</th><th align="left">Description</th></tr></thead><tbody><tr><td align="left"><code class="literal">-std=c++98</code> or <code class="literal">-std=c++03</code>
       </td><td align="left">Use the 1998 ISO C++ standard plus amendments.</td></tr><tr><td align="left"><code class="literal">-std=gnu++98</code> or <code class="literal">-std=gnu++03</code>
       </td><td align="left">As directly above, with GNU extensions.</td></tr><tr><td align="left"><code class="literal">-std=c++11</code></td><td align="left">Use the 2011 ISO C++ standard.</td></tr><tr><td align="left"><code class="literal">-std=gnu++11</code></td><td align="left">As directly above, with GNU extensions.</td></tr><tr><td align="left"><code class="literal">-std=c++14</code></td><td align="left">Use the 2014 ISO C++ standard.</td></tr><tr><td align="left"><code class="literal">-std=gnu++14</code></td><td align="left">As directly above, with GNU extensions.</td></tr><tr><td align="left"><code class="literal">-fexceptions</code></td><td align="left">See <a class="link" href="using_exceptions.html#intro.using.exception.no" title="Doing without">exception-free dialect</a></td></tr><tr><td align="left"><code class="literal">-frtti</code></td><td align="left">As above, but RTTI-free dialect.</td></tr><tr><td align="left"><code class="literal">-pthread</code></td><td align="left">For ISO C++11
         <code class="filename">&lt;thread&gt;</code>,
diff --git a/libstdc++-v3/doc/html/manual/using_concurrency.html b/libstdc++-v3/doc/html/manual/using_concurrency.html
index 2b8d951ce4d..62d0a0b5ba8 100644
--- a/libstdc++-v3/doc/html/manual/using_concurrency.html
+++ b/libstdc++-v3/doc/html/manual/using_concurrency.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Concurrency</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_dynamic_or_shared.html" title="Linking" /><link rel="next" href="using_exceptions.html" title="Exceptions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Concurrency</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_dynamic_or_shared.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_exceptions.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.concurrency"></a>Concurrency</h2></div></div></div><p>This section discusses issues surrounding the proper compilation
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Concurrency</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_dynamic_or_shared.html" title="Linking" /><link rel="next" href="using_exceptions.html" title="Exceptions" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Concurrency</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_dynamic_or_shared.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_exceptions.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.concurrency"></a>Concurrency</h2></div></div></div><p>This section discusses issues surrounding the proper compilation
       of multithreaded applications which use the Standard C++
       library.  This information is GCC-specific since the C++
       standard does not address matters of multithreaded applications.
diff --git a/libstdc++-v3/doc/html/manual/using_dual_abi.html b/libstdc++-v3/doc/html/manual/using_dual_abi.html
index 24f4f7a7778..4a62c0267be 100644
--- a/libstdc++-v3/doc/html/manual/using_dual_abi.html
+++ b/libstdc++-v3/doc/html/manual/using_dual_abi.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Dual ABI</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_macros.html" title="Macros" /><link rel="next" href="using_namespaces.html" title="Namespaces" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Dual ABI</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_macros.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_namespaces.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.abi"></a>Dual ABI</h2></div></div></div><p> In the GCC 5.1 release libstdc++ introduced a new library ABI that
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Dual ABI</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_macros.html" title="Macros" /><link rel="next" href="using_namespaces.html" title="Namespaces" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Dual ABI</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_macros.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_namespaces.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.abi"></a>Dual ABI</h2></div></div></div><p> In the GCC 5.1 release libstdc++ introduced a new library ABI that
   includes new implementations of <code class="classname">std::string</code> and
   <code class="classname">std::list</code>. These changes were necessary to conform
   to the 2011 C++ standard which forbids Copy-On-Write strings and requires
diff --git a/libstdc++-v3/doc/html/manual/using_dynamic_or_shared.html b/libstdc++-v3/doc/html/manual/using_dynamic_or_shared.html
index 8defacd1e3b..3c9e5fa54b0 100644
--- a/libstdc++-v3/doc/html/manual/using_dynamic_or_shared.html
+++ b/libstdc++-v3/doc/html/manual/using_dynamic_or_shared.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Linking</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_namespaces.html" title="Namespaces" /><link rel="next" href="using_concurrency.html" title="Concurrency" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Linking</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_namespaces.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_concurrency.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.linkage"></a>Linking</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.linkage.freestanding"></a>Almost Nothing</h3></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Linking</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_namespaces.html" title="Namespaces" /><link rel="next" href="using_concurrency.html" title="Concurrency" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Linking</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_namespaces.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_concurrency.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.linkage"></a>Linking</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.linkage.freestanding"></a>Almost Nothing</h3></div></div></div><p>
 	Or as close as it gets: freestanding. This is a minimal
 	configuration, with only partial support for the standard
 	library. Assume only the following header files can be used:
diff --git a/libstdc++-v3/doc/html/manual/using_exceptions.html b/libstdc++-v3/doc/html/manual/using_exceptions.html
index 2d9c6af36d8..7ca18493239 100644
--- a/libstdc++-v3/doc/html/manual/using_exceptions.html
+++ b/libstdc++-v3/doc/html/manual/using_exceptions.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Exceptions</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="C++, exception, error, exception neutrality, exception safety, exception propagation, -fno-exceptions" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_concurrency.html" title="Concurrency" /><link rel="next" href="debug.html" title="Debugging Support" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Exceptions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_concurrency.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="debug.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.exceptions"></a>Exceptions</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Exceptions</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="C++, exception, error, exception neutrality, exception safety, exception propagation, -fno-exceptions" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_concurrency.html" title="Concurrency" /><link rel="next" href="debug.html" title="Debugging Support" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Exceptions</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_concurrency.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="debug.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.exceptions"></a>Exceptions</h2></div></div></div><p>
 The C++ language provides language support for stack unwinding
 with <code class="literal">try</code> and <code class="literal">catch</code> blocks and
 the <code class="literal">throw</code> keyword.
diff --git a/libstdc++-v3/doc/html/manual/using_headers.html b/libstdc++-v3/doc/html/manual/using_headers.html
index bb147944ad9..a25323d40ec 100644
--- a/libstdc++-v3/doc/html/manual/using_headers.html
+++ b/libstdc++-v3/doc/html/manual/using_headers.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Headers</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using.html" title="Chapter 3. Using" /><link rel="next" href="using_macros.html" title="Macros" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Headers</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_macros.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.headers"></a>Headers</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.headers.all"></a>Header Files</h3></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Headers</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using.html" title="Chapter 3. Using" /><link rel="next" href="using_macros.html" title="Macros" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Headers</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_macros.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.headers"></a>Headers</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.headers.all"></a>Header Files</h3></div></div></div><p>
      The C++ standard specifies the entire set of header files that
      must be available to all hosted implementations.  Actually, the
      word "files" is a misnomer, since the contents of the
@@ -18,19 +18,19 @@
      the 1998 standard as updated for 2003, and the current 2011 standard.
    </p><p>
      C++98/03 include files. These are available in the default compilation mode, i.e. <code class="code">-std=c++98</code> or <code class="code">-std=gnu++98</code>.
-   </p><div class="table"><a id="table.cxx98_headers"></a><p class="title"><strong>Table 3.2. C++ 1998 Library Headers</strong></p><div class="table-contents"><table summary="C++ 1998 Library Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="filename">bitset</code></td><td align="left"><code class="filename">complex</code></td><td align="left"><code class="filename">deque</code></td><td align="left"><code class="filename">exception</code></td></tr><tr><td align="left"><code class="filename">fstream</code></td><td align="left"><code class="filename">functional</code></td><td align="left"><code class="filename">iomanip</code></td><td align="left"><code class="filename">ios</code></td><td align="left"><code class="filename">iosfwd</code></td></tr><tr><td align="left"><code class="filename">iostream</code></td><td align="left"><code class="filename">istream</code></td><td align="left"><code class="filename">iterator</code></td><td align="left"><code class="filename">limits</code></td><td align="left"><code class="filename">list</code></td></tr><tr><td align="left"><code class="filename">locale</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="filename">memory</code></td><td align="left"><code class="filename">new</code></td><td align="left"><code class="filename">numeric</code></td></tr><tr><td align="left"><code class="filename">ostream</code></td><td align="left"><code class="filename">queue</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="filename">sstream</code></td><td align="left"><code class="filename">stack</code></td></tr><tr><td align="left"><code class="filename">stdexcept</code></td><td align="left"><code class="filename">streambuf</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="filename">utility</code></td><td align="left"><code class="filename">typeinfo</code></td></tr><tr><td align="left"><code class="filename">valarray</code></td><td align="left"><code class="filename">vector</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.cxx98_cheaders"></a><p class="title"><strong>Table 3.3. C++ 1998 Library Headers for C Library Facilities</strong></p><div class="table-contents"><table summary="C++ 1998 Library Headers for C Library Facilities" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">cassert</code></td><td align="left"><code class="filename">cerrno</code></td><td align="left"><code class="filename">cctype</code></td><td align="left"><code class="filename">cfloat</code></td><td align="left"><code class="filename">ciso646</code></td></tr><tr><td align="left"><code class="filename">climits</code></td><td align="left"><code class="filename">clocale</code></td><td align="left"><code class="filename">cmath</code></td><td align="left"><code class="filename">csetjmp</code></td><td align="left"><code class="filename">csignal</code></td></tr><tr><td align="left"><code class="filename">cstdarg</code></td><td align="left"><code class="filename">cstddef</code></td><td align="left"><code class="filename">cstdio</code></td><td align="left"><code class="filename">cstdlib</code></td><td align="left"><code class="filename">cstring</code></td></tr><tr><td align="left"><code class="filename">ctime</code></td><td align="left"><code class="filename">cwchar</code></td><td align="left"><code class="filename">cwctype</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p>
+   </p><div class="table"><a id="table.cxx98_headers"></a><p class="title"><strong>Table 3.2. C++ 1998 Library Headers</strong></p><div class="table-contents"><table class="table" summary="C++ 1998 Library Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="filename">bitset</code></td><td align="left"><code class="filename">complex</code></td><td align="left"><code class="filename">deque</code></td><td align="left"><code class="filename">exception</code></td></tr><tr><td align="left"><code class="filename">fstream</code></td><td align="left"><code class="filename">functional</code></td><td align="left"><code class="filename">iomanip</code></td><td align="left"><code class="filename">ios</code></td><td align="left"><code class="filename">iosfwd</code></td></tr><tr><td align="left"><code class="filename">iostream</code></td><td align="left"><code class="filename">istream</code></td><td align="left"><code class="filename">iterator</code></td><td align="left"><code class="filename">limits</code></td><td align="left"><code class="filename">list</code></td></tr><tr><td align="left"><code class="filename">locale</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="filename">memory</code></td><td align="left"><code class="filename">new</code></td><td align="left"><code class="filename">numeric</code></td></tr><tr><td align="left"><code class="filename">ostream</code></td><td align="left"><code class="filename">queue</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="filename">sstream</code></td><td align="left"><code class="filename">stack</code></td></tr><tr><td align="left"><code class="filename">stdexcept</code></td><td align="left"><code class="filename">streambuf</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="filename">utility</code></td><td align="left"><code class="filename">typeinfo</code></td></tr><tr><td align="left"><code class="filename">valarray</code></td><td align="left"><code class="filename">vector</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.cxx98_cheaders"></a><p class="title"><strong>Table 3.3. C++ 1998 Library Headers for C Library Facilities</strong></p><div class="table-contents"><table class="table" summary="C++ 1998 Library Headers for C Library Facilities" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">cassert</code></td><td align="left"><code class="filename">cerrno</code></td><td align="left"><code class="filename">cctype</code></td><td align="left"><code class="filename">cfloat</code></td><td align="left"><code class="filename">ciso646</code></td></tr><tr><td align="left"><code class="filename">climits</code></td><td align="left"><code class="filename">clocale</code></td><td align="left"><code class="filename">cmath</code></td><td align="left"><code class="filename">csetjmp</code></td><td align="left"><code class="filename">csignal</code></td></tr><tr><td align="left"><code class="filename">cstdarg</code></td><td align="left"><code class="filename">cstddef</code></td><td align="left"><code class="filename">cstdio</code></td><td align="left"><code class="filename">cstdlib</code></td><td align="left"><code class="filename">cstring</code></td></tr><tr><td align="left"><code class="filename">ctime</code></td><td align="left"><code class="filename">cwchar</code></td><td align="left"><code class="filename">cwctype</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p>
 C++11 include files. These are only available in C++11 compilation
 mode, i.e. <code class="literal">-std=c++11</code> or <code class="literal">-std=gnu++11</code>.
-</p><p></p><div class="table"><a id="table.cxx11_headers"></a><p class="title"><strong>Table 3.4. C++ 2011 Library Headers</strong></p><div class="table-contents"><table summary="C++ 2011 Library Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="filename">array</code></td><td align="left"><code class="filename">bitset</code></td><td align="left"><code class="filename">chrono</code></td><td align="left"><code class="filename">complex</code></td></tr><tr><td align="left"><code class="filename">condition_variable</code></td><td align="left"><code class="filename">deque</code></td><td align="left"><code class="filename">exception</code></td><td align="left"><code class="filename">forward_list</code></td><td align="left"><code class="filename">fstream</code></td></tr><tr><td align="left"><code class="filename">functional</code></td><td align="left"><code class="filename">future</code></td><td align="left"><code class="filename">initalizer_list</code></td><td align="left"><code class="filename">iomanip</code></td><td align="left"><code class="filename">ios</code></td></tr><tr><td align="left"><code class="filename">iosfwd</code></td><td align="left"><code class="filename">iostream</code></td><td align="left"><code class="filename">istream</code></td><td align="left"><code class="filename">iterator</code></td><td align="left"><code class="filename">limits</code></td></tr><tr><td align="left"><code class="filename">list</code></td><td align="left"><code class="filename">locale</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="filename">memory</code></td><td align="left"><code class="filename">mutex</code></td></tr><tr><td align="left"><code class="filename">new</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="filename">ostream</code></td><td align="left"><code class="filename">queue</code></td><td align="left"><code class="filename">random</code></td></tr><tr><td align="left"><code class="filename">ratio</code></td><td align="left"><code class="filename">regex</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="filename">sstream</code></td><td align="left"><code class="filename">stack</code></td></tr><tr><td align="left"><code class="filename">stdexcept</code></td><td align="left"><code class="filename">streambuf</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="filename">system_error</code></td><td align="left"><code class="filename">thread</code></td></tr><tr><td align="left"><code class="filename">tuple</code></td><td align="left"><code class="filename">type_traits</code></td><td align="left"><code class="filename">typeinfo</code></td><td align="left"><code class="filename">unordered_map</code></td><td align="left"><code class="filename">unordered_set</code></td></tr><tr><td align="left"><code class="filename">utility</code></td><td align="left"><code class="filename">valarray</code></td><td align="left"><code class="filename">vector</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.cxx11_cheaders"></a><p class="title"><strong>Table 3.5. C++ 2011 Library Headers for C Library Facilities</strong></p><div class="table-contents"><table summary="C++ 2011 Library Headers for C Library Facilities" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">cassert</code></td><td align="left"><code class="filename">ccomplex</code></td><td align="left"><code class="filename">cctype</code></td><td align="left"><code class="filename">cerrno</code></td><td align="left"><code class="filename">cfenv</code></td></tr><tr><td align="left"><code class="filename">cfloat</code></td><td align="left"><code class="filename">cinttypes</code></td><td align="left"><code class="filename">ciso646</code></td><td align="left"><code class="filename">climits</code></td><td align="left"><code class="filename">clocale</code></td></tr><tr><td align="left"><code class="filename">cmath</code></td><td align="left"><code class="filename">csetjmp</code></td><td align="left"><code class="filename">csignal</code></td><td align="left"><code class="filename">cstdarg</code></td><td align="left"><code class="filename">cstdbool</code></td></tr><tr><td align="left"><code class="filename">cstddef</code></td><td align="left"><code class="filename">cstdint</code></td><td align="left"><code class="filename">cstdlib</code></td><td align="left"><code class="filename">cstdio</code></td><td align="left"><code class="filename">cstring</code></td></tr><tr><td align="left"><code class="filename">ctgmath</code></td><td align="left"><code class="filename">ctime</code></td><td align="left"><code class="filename">cuchar</code></td><td align="left"><code class="filename">cwchar</code></td><td align="left"><code class="filename">cwctype</code></td></tr></tbody></table></div></div><br class="table-break" /><p>
+</p><p></p><div class="table"><a id="table.cxx11_headers"></a><p class="title"><strong>Table 3.4. C++ 2011 Library Headers</strong></p><div class="table-contents"><table class="table" summary="C++ 2011 Library Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">algorithm</code></td><td align="left"><code class="filename">array</code></td><td align="left"><code class="filename">bitset</code></td><td align="left"><code class="filename">chrono</code></td><td align="left"><code class="filename">complex</code></td></tr><tr><td align="left"><code class="filename">condition_variable</code></td><td align="left"><code class="filename">deque</code></td><td align="left"><code class="filename">exception</code></td><td align="left"><code class="filename">forward_list</code></td><td align="left"><code class="filename">fstream</code></td></tr><tr><td align="left"><code class="filename">functional</code></td><td align="left"><code class="filename">future</code></td><td align="left"><code class="filename">initalizer_list</code></td><td align="left"><code class="filename">iomanip</code></td><td align="left"><code class="filename">ios</code></td></tr><tr><td align="left"><code class="filename">iosfwd</code></td><td align="left"><code class="filename">iostream</code></td><td align="left"><code class="filename">istream</code></td><td align="left"><code class="filename">iterator</code></td><td align="left"><code class="filename">limits</code></td></tr><tr><td align="left"><code class="filename">list</code></td><td align="left"><code class="filename">locale</code></td><td align="left"><code class="filename">map</code></td><td align="left"><code class="filename">memory</code></td><td align="left"><code class="filename">mutex</code></td></tr><tr><td align="left"><code class="filename">new</code></td><td align="left"><code class="filename">numeric</code></td><td align="left"><code class="filename">ostream</code></td><td align="left"><code class="filename">queue</code></td><td align="left"><code class="filename">random</code></td></tr><tr><td align="left"><code class="filename">ratio</code></td><td align="left"><code class="filename">regex</code></td><td align="left"><code class="filename">set</code></td><td align="left"><code class="filename">sstream</code></td><td align="left"><code class="filename">stack</code></td></tr><tr><td align="left"><code class="filename">stdexcept</code></td><td align="left"><code class="filename">streambuf</code></td><td align="left"><code class="filename">string</code></td><td align="left"><code class="filename">system_error</code></td><td align="left"><code class="filename">thread</code></td></tr><tr><td align="left"><code class="filename">tuple</code></td><td align="left"><code class="filename">type_traits</code></td><td align="left"><code class="filename">typeinfo</code></td><td align="left"><code class="filename">unordered_map</code></td><td align="left"><code class="filename">unordered_set</code></td></tr><tr><td align="left"><code class="filename">utility</code></td><td align="left"><code class="filename">valarray</code></td><td align="left"><code class="filename">vector</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.cxx11_cheaders"></a><p class="title"><strong>Table 3.5. C++ 2011 Library Headers for C Library Facilities</strong></p><div class="table-contents"><table class="table" summary="C++ 2011 Library Headers for C Library Facilities" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">cassert</code></td><td align="left"><code class="filename">ccomplex</code></td><td align="left"><code class="filename">cctype</code></td><td align="left"><code class="filename">cerrno</code></td><td align="left"><code class="filename">cfenv</code></td></tr><tr><td align="left"><code class="filename">cfloat</code></td><td align="left"><code class="filename">cinttypes</code></td><td align="left"><code class="filename">ciso646</code></td><td align="left"><code class="filename">climits</code></td><td align="left"><code class="filename">clocale</code></td></tr><tr><td align="left"><code class="filename">cmath</code></td><td align="left"><code class="filename">csetjmp</code></td><td align="left"><code class="filename">csignal</code></td><td align="left"><code class="filename">cstdarg</code></td><td align="left"><code class="filename">cstdbool</code></td></tr><tr><td align="left"><code class="filename">cstddef</code></td><td align="left"><code class="filename">cstdint</code></td><td align="left"><code class="filename">cstdlib</code></td><td align="left"><code class="filename">cstdio</code></td><td align="left"><code class="filename">cstring</code></td></tr><tr><td align="left"><code class="filename">ctgmath</code></td><td align="left"><code class="filename">ctime</code></td><td align="left"><code class="filename">cuchar</code></td><td align="left"><code class="filename">cwchar</code></td><td align="left"><code class="filename">cwctype</code></td></tr></tbody></table></div></div><br class="table-break" /><p>
   In addition, TR1 includes as:
-</p><div class="table"><a id="table.tr1_headers"></a><p class="title"><strong>Table 3.6. C++ TR 1 Library Headers</strong></p><div class="table-contents"><table summary="C++ TR 1 Library Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">tr1/array</code></td><td align="left"><code class="filename">tr1/complex</code></td><td align="left"><code class="filename">tr1/memory</code></td><td align="left"><code class="filename">tr1/functional</code></td><td align="left"><code class="filename">tr1/random</code></td></tr><tr><td align="left"><code class="filename">tr1/regex</code></td><td align="left"><code class="filename">tr1/tuple</code></td><td align="left"><code class="filename">tr1/type_traits</code></td><td align="left"><code class="filename">tr1/unordered_map</code></td><td align="left"><code class="filename">tr1/unordered_set</code></td></tr><tr><td align="left"><code class="filename">tr1/utility</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.tr1_cheaders"></a><p class="title"><strong>Table 3.7. C++ TR 1 Library Headers for C Library Facilities</strong></p><div class="table-contents"><table summary="C++ TR 1 Library Headers for C Library Facilities" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">tr1/ccomplex</code></td><td align="left"><code class="filename">tr1/cfenv</code></td><td align="left"><code class="filename">tr1/cfloat</code></td><td align="left"><code class="filename">tr1/cmath</code></td><td align="left"><code class="filename">tr1/cinttypes</code></td></tr><tr><td align="left"><code class="filename">tr1/climits</code></td><td align="left"><code class="filename">tr1/cstdarg</code></td><td align="left"><code class="filename">tr1/cstdbool</code></td><td align="left"><code class="filename">tr1/cstdint</code></td><td align="left"><code class="filename">tr1/cstdio</code></td></tr><tr><td align="left"><code class="filename">tr1/cstdlib</code></td><td align="left"><code class="filename">tr1/ctgmath</code></td><td align="left"><code class="filename">tr1/ctime</code></td><td align="left"><code class="filename">tr1/cwchar</code></td><td align="left"><code class="filename">tr1/cwctype</code></td></tr></tbody></table></div></div><br class="table-break" /><p>Decimal floating-point arithmetic is available if the C++
+</p><div class="table"><a id="table.tr1_headers"></a><p class="title"><strong>Table 3.6. C++ TR 1 Library Headers</strong></p><div class="table-contents"><table class="table" summary="C++ TR 1 Library Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">tr1/array</code></td><td align="left"><code class="filename">tr1/complex</code></td><td align="left"><code class="filename">tr1/memory</code></td><td align="left"><code class="filename">tr1/functional</code></td><td align="left"><code class="filename">tr1/random</code></td></tr><tr><td align="left"><code class="filename">tr1/regex</code></td><td align="left"><code class="filename">tr1/tuple</code></td><td align="left"><code class="filename">tr1/type_traits</code></td><td align="left"><code class="filename">tr1/unordered_map</code></td><td align="left"><code class="filename">tr1/unordered_set</code></td></tr><tr><td align="left"><code class="filename">tr1/utility</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.tr1_cheaders"></a><p class="title"><strong>Table 3.7. C++ TR 1 Library Headers for C Library Facilities</strong></p><div class="table-contents"><table class="table" summary="C++ TR 1 Library Headers for C Library Facilities" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">tr1/ccomplex</code></td><td align="left"><code class="filename">tr1/cfenv</code></td><td align="left"><code class="filename">tr1/cfloat</code></td><td align="left"><code class="filename">tr1/cmath</code></td><td align="left"><code class="filename">tr1/cinttypes</code></td></tr><tr><td align="left"><code class="filename">tr1/climits</code></td><td align="left"><code class="filename">tr1/cstdarg</code></td><td align="left"><code class="filename">tr1/cstdbool</code></td><td align="left"><code class="filename">tr1/cstdint</code></td><td align="left"><code class="filename">tr1/cstdio</code></td></tr><tr><td align="left"><code class="filename">tr1/cstdlib</code></td><td align="left"><code class="filename">tr1/ctgmath</code></td><td align="left"><code class="filename">tr1/ctime</code></td><td align="left"><code class="filename">tr1/cwchar</code></td><td align="left"><code class="filename">tr1/cwctype</code></td></tr></tbody></table></div></div><br class="table-break" /><p>Decimal floating-point arithmetic is available if the C++
 compiler supports scalar decimal floating-point types defined via
 <code class="code">__attribute__((mode(SD|DD|LD)))</code>.
-</p><div class="table"><a id="table.decfp_headers"></a><p class="title"><strong>Table 3.8. C++ TR 24733 Decimal Floating-Point Header</strong></p><div class="table-contents"><table summary="C++ TR 24733 Decimal Floating-Point Header" border="1"><colgroup><col align="left" class="c1" /></colgroup><tbody><tr><td align="left"><code class="filename">decimal/decimal</code></td></tr></tbody></table></div></div><br class="table-break" /><p>
+</p><div class="table"><a id="table.decfp_headers"></a><p class="title"><strong>Table 3.8. C++ TR 24733 Decimal Floating-Point Header</strong></p><div class="table-contents"><table class="table" summary="C++ TR 24733 Decimal Floating-Point Header" border="1"><colgroup><col align="left" class="c1" /></colgroup><tbody><tr><td align="left"><code class="filename">decimal/decimal</code></td></tr></tbody></table></div></div><br class="table-break" /><p>
   Also included are files for the C++ ABI interface:
-</p><div class="table"><a id="table.abi_headers"></a><p class="title"><strong>Table 3.9. C++ ABI Headers</strong></p><div class="table-contents"><table summary="C++ ABI Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><tbody><tr><td align="left"><code class="filename">cxxabi.h</code></td><td align="left"><code class="filename">cxxabi_forced.h</code></td></tr></tbody></table></div></div><br class="table-break" /><p>
+</p><div class="table"><a id="table.abi_headers"></a><p class="title"><strong>Table 3.9. C++ ABI Headers</strong></p><div class="table-contents"><table class="table" summary="C++ ABI Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><tbody><tr><td align="left"><code class="filename">cxxabi.h</code></td><td align="left"><code class="filename">cxxabi_forced.h</code></td></tr></tbody></table></div></div><br class="table-break" /><p>
   And a large variety of extensions.
-</p><div class="table"><a id="table.ext_headers"></a><p class="title"><strong>Table 3.10. Extension Headers</strong></p><div class="table-contents"><table summary="Extension Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">ext/algorithm</code></td><td align="left"><code class="filename">ext/atomicity.h</code></td><td align="left"><code class="filename">ext/array_allocator.h</code></td><td align="left"><code class="filename">ext/bitmap_allocator.h</code></td><td align="left"><code class="filename">ext/cast.h</code></td></tr><tr><td align="left"><code class="filename">ext/codecvt_specializations.h</code></td><td align="left"><code class="filename">ext/concurrence.h</code></td><td align="left"><code class="filename">ext/debug_allocator.h</code></td><td align="left"><code class="filename">ext/enc_filebuf.h</code></td><td align="left"><code class="filename">ext/extptr_allocator.h</code></td></tr><tr><td align="left"><code class="filename">ext/functional</code></td><td align="left"><code class="filename">ext/iterator</code></td><td align="left"><code class="filename">ext/malloc_allocator.h</code></td><td align="left"><code class="filename">ext/memory</code></td><td align="left"><code class="filename">ext/mt_allocator.h</code></td></tr><tr><td align="left"><code class="filename">ext/new_allocator.h</code></td><td align="left"><code class="filename">ext/numeric</code></td><td align="left"><code class="filename">ext/numeric_traits.h</code></td><td align="left"><code class="filename">ext/pb_ds/assoc_container.h</code></td><td align="left"><code class="filename">ext/pb_ds/priority_queue.h</code></td></tr><tr><td align="left"><code class="filename">ext/pod_char_traits.h</code></td><td align="left"><code class="filename">ext/pool_allocator.h</code></td><td align="left"><code class="filename">ext/rb_tree</code></td><td align="left"><code class="filename">ext/rope</code></td><td align="left"><code class="filename">ext/slist</code></td></tr><tr><td align="left"><code class="filename">ext/stdio_filebuf.h</code></td><td align="left"><code class="filename">ext/stdio_sync_filebuf.h</code></td><td align="left"><code class="filename">ext/throw_allocator.h</code></td><td align="left"><code class="filename">ext/typelist.h</code></td><td align="left"><code class="filename">ext/type_traits.h</code></td></tr><tr><td align="left"><code class="filename">ext/vstring.h</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.debug_headers"></a><p class="title"><strong>Table 3.11. Extension Debug Headers</strong></p><div class="table-contents"><table summary="Extension Debug Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">debug/bitset</code></td><td align="left"><code class="filename">debug/deque</code></td><td align="left"><code class="filename">debug/list</code></td><td align="left"><code class="filename">debug/map</code></td><td align="left"><code class="filename">debug/set</code></td></tr><tr><td align="left"><code class="filename">debug/string</code></td><td align="left"><code class="filename">debug/unordered_map</code></td><td align="left"><code class="filename">debug/unordered_set</code></td><td align="left"><code class="filename">debug/vector</code></td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.profile_headers"></a><p class="title"><strong>Table 3.12. Extension Profile Headers</strong></p><div class="table-contents"><table summary="Extension Profile Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><tbody><tr><td align="left"><code class="filename">profile/bitset</code></td><td align="left"><code class="filename">profile/deque</code></td><td align="left"><code class="filename">profile/list</code></td><td align="left"><code class="filename">profile/map</code></td></tr><tr><td align="left"><code class="filename">profile/set</code></td><td align="left"><code class="filename">profile/unordered_map</code></td><td align="left"><code class="filename">profile/unordered_set</code></td><td align="left"><code class="filename">profile/vector</code></td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.parallel_headers"></a><p class="title"><strong>Table 3.13. Extension Parallel Headers</strong></p><div class="table-contents"><table summary="Extension Parallel Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><tbody><tr><td align="left"><code class="filename">parallel/algorithm</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr></tbody></table></div></div><br class="table-break" /></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.headers.mixing"></a>Mixing Headers</h3></div></div></div><p> A few simple rules.
+</p><div class="table"><a id="table.ext_headers"></a><p class="title"><strong>Table 3.10. Extension Headers</strong></p><div class="table-contents"><table class="table" summary="Extension Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">ext/algorithm</code></td><td align="left"><code class="filename">ext/atomicity.h</code></td><td align="left"><code class="filename">ext/array_allocator.h</code></td><td align="left"><code class="filename">ext/bitmap_allocator.h</code></td><td align="left"><code class="filename">ext/cast.h</code></td></tr><tr><td align="left"><code class="filename">ext/codecvt_specializations.h</code></td><td align="left"><code class="filename">ext/concurrence.h</code></td><td align="left"><code class="filename">ext/debug_allocator.h</code></td><td align="left"><code class="filename">ext/enc_filebuf.h</code></td><td align="left"><code class="filename">ext/extptr_allocator.h</code></td></tr><tr><td align="left"><code class="filename">ext/functional</code></td><td align="left"><code class="filename">ext/iterator</code></td><td align="left"><code class="filename">ext/malloc_allocator.h</code></td><td align="left"><code class="filename">ext/memory</code></td><td align="left"><code class="filename">ext/mt_allocator.h</code></td></tr><tr><td align="left"><code class="filename">ext/new_allocator.h</code></td><td align="left"><code class="filename">ext/numeric</code></td><td align="left"><code class="filename">ext/numeric_traits.h</code></td><td align="left"><code class="filename">ext/pb_ds/assoc_container.h</code></td><td align="left"><code class="filename">ext/pb_ds/priority_queue.h</code></td></tr><tr><td align="left"><code class="filename">ext/pod_char_traits.h</code></td><td align="left"><code class="filename">ext/pool_allocator.h</code></td><td align="left"><code class="filename">ext/rb_tree</code></td><td align="left"><code class="filename">ext/rope</code></td><td align="left"><code class="filename">ext/slist</code></td></tr><tr><td align="left"><code class="filename">ext/stdio_filebuf.h</code></td><td align="left"><code class="filename">ext/stdio_sync_filebuf.h</code></td><td align="left"><code class="filename">ext/throw_allocator.h</code></td><td align="left"><code class="filename">ext/typelist.h</code></td><td align="left"><code class="filename">ext/type_traits.h</code></td></tr><tr><td align="left"><code class="filename">ext/vstring.h</code></td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.debug_headers"></a><p class="title"><strong>Table 3.11. Extension Debug Headers</strong></p><div class="table-contents"><table class="table" summary="Extension Debug Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /><col align="left" class="c5" /></colgroup><tbody><tr><td align="left"><code class="filename">debug/bitset</code></td><td align="left"><code class="filename">debug/deque</code></td><td align="left"><code class="filename">debug/list</code></td><td align="left"><code class="filename">debug/map</code></td><td align="left"><code class="filename">debug/set</code></td></tr><tr><td align="left"><code class="filename">debug/string</code></td><td align="left"><code class="filename">debug/unordered_map</code></td><td align="left"><code class="filename">debug/unordered_set</code></td><td align="left"><code class="filename">debug/vector</code></td><td class="auto-generated"> </td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.profile_headers"></a><p class="title"><strong>Table 3.12. Extension Profile Headers</strong></p><div class="table-contents"><table class="table" summary="Extension Profile Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /><col align="left" class="c3" /><col align="left" class="c4" /></colgroup><tbody><tr><td align="left"><code class="filename">profile/bitset</code></td><td align="left"><code class="filename">profile/deque</code></td><td align="left"><code class="filename">profile/list</code></td><td align="left"><code class="filename">profile/map</code></td></tr><tr><td align="left"><code class="filename">profile/set</code></td><td align="left"><code class="filename">profile/unordered_map</code></td><td align="left"><code class="filename">profile/unordered_set</code></td><td align="left"><code class="filename">profile/vector</code></td></tr></tbody></table></div></div><br class="table-break" /><p></p><div class="table"><a id="table.parallel_headers"></a><p class="title"><strong>Table 3.13. Extension Parallel Headers</strong></p><div class="table-contents"><table class="table" summary="Extension Parallel Headers" border="1"><colgroup><col align="left" class="c1" /><col align="left" class="c2" /></colgroup><tbody><tr><td align="left"><code class="filename">parallel/algorithm</code></td><td align="left"><code class="filename">parallel/numeric</code></td></tr></tbody></table></div></div><br class="table-break" /></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.headers.mixing"></a>Mixing Headers</h3></div></div></div><p> A few simple rules.
 </p><p>First, mixing different dialects of the standard headers is not
 possible. It's an all-or-nothing affair. Thus, code like
 </p><pre class="programlisting">
diff --git a/libstdc++-v3/doc/html/manual/using_macros.html b/libstdc++-v3/doc/html/manual/using_macros.html
index f5a8a223a8b..e001d6fde79 100644
--- a/libstdc++-v3/doc/html/manual/using_macros.html
+++ b/libstdc++-v3/doc/html/manual/using_macros.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Macros</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_headers.html" title="Headers" /><link rel="next" href="using_dual_abi.html" title="Dual ABI" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Macros</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_headers.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_dual_abi.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.macros"></a>Macros</h2></div></div></div><p>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Macros</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_headers.html" title="Headers" /><link rel="next" href="using_dual_abi.html" title="Dual ABI" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Macros</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_headers.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_dual_abi.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.macros"></a>Macros</h2></div></div></div><p>
      All library macros begin with <code class="code">_GLIBCXX_</code>.
    </p><p>
      Furthermore, all pre-processor macros, switches, and
@@ -108,4 +108,16 @@
 	mode</a>.
       </p></dd><dt><span class="term"><code class="code">__STDCPP_WANT_MATH_SPEC_FUNCS__</code></span></dt><dd><p>Undefined by default. When defined to a non-zero integer constant,
 	enables support for ISO/IEC 29124 Special Math Functions.
-      </p></dd></dl></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="using_headers.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="using.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="using_dual_abi.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Headers </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Dual ABI</td></tr></table></div></body></html>
\ No newline at end of file
+      </p></dd></dl></div><dt><span class="term"><code class="code">_GLIBCXX_SANITIZE_VECTOR</code></span></dt><dd><p>
+	Undefined by default. When defined, <code class="classname">std::vector</code>
+        operations will be annotated so that AddressSanitizer can detect
+        invalid accesses to the unused capacity of a
+        <code class="classname">std::vector</code>. These annotations are only
+        enabled for
+        <code class="classname">std::vector&lt;T, std::allocator&lt;T&gt;&gt;</code>
+        and only when <code class="classname">std::allocator</code> is derived from
+        <a class="xref" href="memory.html#allocator.impl" title="Implementation">the section called “Implementation�</a>. The annotations
+        must be present on all vector operations or none, so this macro must
+        be defined to the same value for all translation units that create,
+        destroy or modify vectors.
+      </p></dd></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="using_headers.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="using.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="using_dual_abi.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Headers </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Dual ABI</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/manual/using_namespaces.html b/libstdc++-v3/doc/html/manual/using_namespaces.html
index 0fe9ce33399..b036d6fa6ac 100644
--- a/libstdc++-v3/doc/html/manual/using_namespaces.html
+++ b/libstdc++-v3/doc/html/manual/using_namespaces.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Namespaces</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_dual_abi.html" title="Dual ABI" /><link rel="next" href="using_dynamic_or_shared.html" title="Linking" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Namespaces</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_dual_abi.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_dynamic_or_shared.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.namespaces"></a>Namespaces</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.namespaces.all"></a>Available Namespaces</h3></div></div></div><p> There are three main namespaces.
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Namespaces</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="using.html" title="Chapter 3. Using" /><link rel="prev" href="using_dual_abi.html" title="Dual ABI" /><link rel="next" href="using_dynamic_or_shared.html" title="Linking" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Namespaces</th></tr><tr><td width="20%" align="left"><a accesskey="p" href="using_dual_abi.html">Prev</a> </td><th width="60%" align="center">Chapter 3. Using</th><td width="20%" align="right"> <a accesskey="n" href="using_dynamic_or_shared.html">Next</a></td></tr></table><hr /></div><div class="section"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="manual.intro.using.namespaces"></a>Namespaces</h2></div></div></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="manual.intro.using.namespaces.all"></a>Available Namespaces</h3></div></div></div><p> There are three main namespaces.
 </p><div class="itemizedlist"><ul class="itemizedlist" style="list-style-type: disc; "><li class="listitem"><p>std</p><p>The ISO C++ standards specify that "all library entities are defined
 within namespace std." This includes namespaces nested
 within namespace <code class="code">std</code>, such as namespace
diff --git a/libstdc++-v3/doc/html/manual/utilities.html b/libstdc++-v3/doc/html/manual/utilities.html
index 6cf4f750927..3f8fadb3415 100644
--- a/libstdc++-v3/doc/html/manual/utilities.html
+++ b/libstdc++-v3/doc/html/manual/utilities.html
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 6.  Utilities</title><meta name="generator" content="DocBook XSL-NS Stylesheets V1.78.1" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="concept_checking.html" title="Concept Checking" /><link rel="next" href="pairs.html" title="Pairs" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 6. 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Chapter 6.  Utilities</title><meta name="generator" content="DocBook XSL Stylesheets Vsnapshot" /><meta name="keywords" content="ISO C++, library" /><meta name="keywords" content="ISO C++, runtime, library" /><link rel="home" href="../index.html" title="The GNU C++ Library" /><link rel="up" href="std_contents.html" title="Part II.  Standard Contents" /><link rel="prev" href="concept_checking.html" title="Concept Checking" /><link rel="next" href="pairs.html" title="Pairs" /></head><body><div class="navheader"><table width="100%" summary="Navigation header"><tr><th colspan="3" align="center">Chapter 6. 
   Utilities
   
 </th></tr><tr><td width="20%" align="left"><a accesskey="p" href="concept_checking.html">Prev</a> </td><th width="60%" align="center">Part II. 
diff --git a/libstdc++-v3/doc/xml/faq.xml b/libstdc++-v3/doc/xml/faq.xml
index 703ade5b20d..1a21e48007a 100644
--- a/libstdc++-v3/doc/xml/faq.xml
+++ b/libstdc++-v3/doc/xml/faq.xml
@@ -30,13 +30,18 @@
   <answer xml:id="faq.what.a">
     <para>
      The GNU Standard C++ Library v3 is an ongoing project to
-     implement the ISO 14882 Standard C++ library as described in
-     clauses 17 through 30 and annex D.  For those who want to see
+     implement the ISO 14882 C++ Standard Library as described in
+     clauses 20 through 33 and annex D (prior to the 2017 standard
+     the library clauses started with 17).  For those who want to see
      exactly how far the project has come, or just want the latest
      bleeding-edge code, the up-to-date source is available over
-     anonymous SVN, and can be browsed over
-     the <link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="https://gcc.gnu.org/svn.html">web</link>.
+     anonymous SVN, and can be browsed over the
+     <link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="https://gcc.gnu.org/svn.html">web</link>.
     </para> 
+
+    <para>
+    N.B. The library is called libstdc++ <emphasis>not</emphasis> stdlibc++.
+    </para>
   </answer>
 </qandaentry>
 
diff --git a/libstdc++-v3/doc/xml/manual/intro.xml b/libstdc++-v3/doc/xml/manual/intro.xml
index 782817e0698..2df9c5fa6a7 100644
--- a/libstdc++-v3/doc/xml/manual/intro.xml
+++ b/libstdc++-v3/doc/xml/manual/intro.xml
@@ -50,6 +50,10 @@
     <!-- Section 01.6 : Status C++ TR24733 -->
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" parse="xml" href="status_cxxtr24733.xml">
     </xi:include>
+
+    <!-- Section 01.7 : Status C++ IS 24733 -->
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" parse="xml" href="status_cxxis29124.xml">
+    </xi:include>
   </section>
 
   <!-- Section 02 : License -->
@@ -988,6 +992,12 @@ requirements of the license of GCC.
     <listitem><para>Add deleted constructors.
     </para></listitem></varlistentry>
 
+    <varlistentry xml:id="manual.bugs.dr2354"><term><link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="&DR;#2354">2332</link>:
+       <emphasis>Unnecessary copying when inserting into maps with braced-init syntax</emphasis>
+    </term>
+    <listitem><para>Add overloads of <code>insert</code> taking <code>value_type&amp;&amp;</code> rvalues.
+    </para></listitem></varlistentry>
+
     <varlistentry xml:id="manual.bugs.dr2399"><term><link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="&DR;#2399">2399</link>:
 	<emphasis><code>shared_ptr</code>'s constructor from <code>unique_ptr</code> should be constrained</emphasis>
     </term>
diff --git a/libstdc++-v3/doc/xml/manual/status_cxx2017.xml b/libstdc++-v3/doc/xml/manual/status_cxx2017.xml
index fd66ac503a8..aa0914cff72 100644
--- a/libstdc++-v3/doc/xml/manual/status_cxx2017.xml
+++ b/libstdc++-v3/doc/xml/manual/status_cxx2017.xml
@@ -1,17 +1,17 @@
 <section xmlns="http://docbook.org/ns/docbook" version="5.0" 
-	 xml:id="status.iso.201z" xreflabel="Status C++ 201z">
-<?dbhtml filename="status_iso_cxx201z.html"?>
+	 xml:id="status.iso.2017" xreflabel="Status C++ 2017">
+<?dbhtml filename="status_iso_cxx2017.html"?>
 
-<info><title>C++ 201z</title>
+<info><title>C++ 2017</title>
   <keywordset>
     <keyword>ISO C++</keyword>
-    <keyword>201z</keyword>
+    <keyword>2017</keyword>
   </keywordset>
 </info>
 
 <para>
-In this implementation the <literal>-std=gnu++1z</literal> or
-<literal>-std=c++1z</literal> flag must be used to enable language
+In this implementation the <literal>-std=gnu++17</literal> or
+<literal>-std=c++17</literal> flag must be used to enable language
 and library
 features. See <link linkend="manual.intro.using.flags">dialect</link>
 options. The pre-defined symbol
@@ -20,13 +20,13 @@ presence of the required flag.
 </para>
 
 <para>
-This section describes the C++1z and library TS support in mainline GCC SVN,
+This section describes the C++17 and library TS support in mainline GCC SVN,
 not in any particular release.
 </para>
 
 <para>
 The following table lists new library features that have been accepted into
-the C++1z working draft. The "Proposal" column provides a link to the
+the C++17 working draft. The "Proposal" column provides a link to the
 ISO C++ committee proposal that describes the feature, while the "Status"
 column indicates the first version of GCC that contains an implementation of
 this feature (if it has been implemented).
@@ -35,8 +35,8 @@ The "SD-6 Feature Test" column shows the corresponding macro or header from
 Feature-testing recommendations for C++</link>.
 </para>
 
-<table frame="all" xml:id="table.cxx1z_status">
-<title>C++ 201z Implementation Status</title>
+<table frame="all" xml:id="table.cxx17_status">
+<title>C++ 2017 Implementation Status</title>
 
 <tgroup cols="4" align="left" colsep="0" rowsep="1">
 <colspec colname="c1"/>
@@ -75,7 +75,7 @@ Feature-testing recommendations for C++</link>.
 	P0137R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__cpp_lib_launder >= 201606</code> </entry>
     </row>
 
@@ -97,9 +97,10 @@ Feature-testing recommendations for C++</link>.
 	P0088R3
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__has_include(&lt;variant&gt;)</code>,
               <code>__cpp_lib_variant >= 201603</code>
+              (since 7.3, see Note 1)
       </entry>
     </row>
 
@@ -110,9 +111,10 @@ Feature-testing recommendations for C++</link>.
 	P0220R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__has_include(&lt;optional&gt;)</code>,
               <code>__cpp_lib_optional >= 201603</code>
+              (since 7.3, see Note 1)
       </entry>
     </row>
 
@@ -123,9 +125,10 @@ Feature-testing recommendations for C++</link>.
 	P0220R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__has_include(&lt;any&gt;)</code>,
               <code>__cpp_lib_any >= 201603</code>
+              (since 7.3, see Note 1)
       </entry>
     </row>
 
@@ -136,9 +139,10 @@ Feature-testing recommendations for C++</link>.
 	P0220R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__has_include(&lt;string_view&gt;)</code>,
               <code>__cpp_lib_string_view >= 201603</code>
+              (since 7.3, see Note 1)
       </entry>
     </row>
 
@@ -163,7 +167,7 @@ Feature-testing recommendations for C++</link>.
 	P0220R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__cpp_lib_apply >= 201603</code> </entry>
     </row>
 
@@ -174,7 +178,7 @@ Feature-testing recommendations for C++</link>.
 	P0220R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__cpp_lib_shared_ptr_arrays >= 201603</code> </entry>
     </row>
 
@@ -185,7 +189,7 @@ Feature-testing recommendations for C++</link>.
 	P0220R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__cpp_lib_boyer_moore_searcher >= 201603</code> </entry>
     </row>
 
@@ -196,7 +200,7 @@ Feature-testing recommendations for C++</link>.
 	P0220R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry> <code>__cpp_lib_sample >= 201603</code> </entry>
     </row>
 
@@ -207,7 +211,7 @@ Feature-testing recommendations for C++</link>.
 	P0007R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_as_const >= 201510 </code></entry>
     </row>
 
@@ -229,7 +233,7 @@ Feature-testing recommendations for C++</link>.
 	P0209R2
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_make_from_tuple >= 201606 </code></entry>
     </row>
 
@@ -256,7 +260,7 @@ Feature-testing recommendations for C++</link>.
 	P0174R2
 	</link>
       </entry>
-      <entry align="center"> No </entry>
+      <entry align="center"> No  (kept for backwards compatibility)</entry>
       <entry/>
     </row>
 
@@ -267,7 +271,7 @@ Feature-testing recommendations for C++</link>.
 	P0074R0
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_transparent_operators >= 201510 </code></entry>
     </row>
 
@@ -278,7 +282,7 @@ Feature-testing recommendations for C++</link>.
 	LWG2296
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_addressof_constexpr >= 201603 </code></entry>
     </row>
 
@@ -311,7 +315,7 @@ Feature-testing recommendations for C++</link>.
 	P0033R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code>__cpp_lib_enable_shared_from_this >= 201603</code></entry>
     </row>
 
@@ -344,7 +348,7 @@ Feature-testing recommendations for C++</link>.
 	P0005R4
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code>__cpp_lib_not_fn >= 201603</code></entry>
     </row>
 
@@ -355,7 +359,7 @@ Feature-testing recommendations for C++</link>.
 	P0358R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry/>
     </row>
 
@@ -366,7 +370,7 @@ Feature-testing recommendations for C++</link>.
 	P0253R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry/>
     </row>
 
@@ -377,7 +381,7 @@ Feature-testing recommendations for C++</link>.
 	P0040R3
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> </code></entry>
     </row>
 
@@ -388,7 +392,7 @@ Feature-testing recommendations for C++</link>.
 	P0163R0
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_shared_ptr_weak_type >= 201606</code></entry>
     </row>
 
@@ -421,7 +425,7 @@ Feature-testing recommendations for C++</link>.
 	P0006R0
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_type_trait_variable_templates >= 201510 </code></entry>
     </row>
 
@@ -454,7 +458,7 @@ Feature-testing recommendations for C++</link>.
 	P0077R2
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_is_callable >= 201603 </code></entry>
     </row>
 
@@ -465,7 +469,7 @@ Feature-testing recommendations for C++</link>.
 	P0258R2
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_has_unique_object_representations >= 201606 </code></entry>
     </row>
 
@@ -476,7 +480,7 @@ Feature-testing recommendations for C++</link>.
 	P0092R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_chrono >= 201510 </code></entry>
     </row>
 
@@ -487,8 +491,10 @@ Feature-testing recommendations for C++</link>.
 	P0505R0
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
-      <entry><code> ??? </code></entry>
+      <entry align="center"> 7.1 </entry>
+      <entry><code> __cpp_lib_chrono >= 201611 </code>
+             (since 7.3, see Note 2)
+      </entry>
     </row>
 
     <row>
@@ -509,7 +515,7 @@ Feature-testing recommendations for C++</link>.
 	P0254R2
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> </code></entry>
     </row>
 
@@ -520,7 +526,7 @@ Feature-testing recommendations for C++</link>.
 	P0272R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> </code></entry>
     </row>
 
@@ -553,9 +559,9 @@ Feature-testing recommendations for C++</link>.
 	N4510
 	</link>
       </entry>
-      <entry align="center"> 6.2 (3.0)</entry>
+      <entry align="center"> 3.0 </entry>
       <entry><code> __cpp_lib_incomplete_container_elements >= 201505 </code>
-	     (the feature was always supported, but the macro was not defined until GCC 6.2)
+	     (since 6.2, see Note 2)
       </entry>
     </row>
 
@@ -566,8 +572,8 @@ Feature-testing recommendations for C++</link>.
 	P0084R2
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
-      <entry><code> ??? </code></entry>
+      <entry align="center"> 7.1 </entry>
+      <entry/>
     </row>
 
     <row>
@@ -590,7 +596,7 @@ Feature-testing recommendations for C++</link>.
 	P0083R3
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_node_extract >= 201606 </code></entry>
     </row>
 
@@ -612,7 +618,7 @@ Feature-testing recommendations for C++</link>.
 	P0031R0
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_array_constexpr >= 201603 </code></entry>
     </row>
 
@@ -638,7 +644,7 @@ Feature-testing recommendations for C++</link>.
 	P0025R0
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_clamp >= 201603 </code></entry>
     </row>
 
@@ -649,7 +655,7 @@ Feature-testing recommendations for C++</link>.
 	P0295R0
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_gcd_lcm >= 201606 </code></entry>
     </row>
 
@@ -660,7 +666,7 @@ Feature-testing recommendations for C++</link>.
 	P0030R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_hypot >= 201603 </code></entry>
     </row>
 
@@ -671,59 +677,67 @@ Feature-testing recommendations for C++</link>.
 	P0226R1
 	</link>
       </entry>
-      <entry align="center"> 7 (6.1) </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_math_special_functions >= 201603 </code>
-	     (for GCC 6 or pre-C++17 define
-	     <code>__STDCPP_WANT_MATH_SPEC_FUNCS__</code> to a non-zero value
-	     and test for <code>__STDCPP_MATH_SPEC_FUNCS__ >= 201003L</code>)
+	     (see Note 3)
       </entry>
     </row>
 
     <row>
-      <?dbhtml bgcolor="#C8B0B0" ?>
       <entry>Adopt the File System TS for C++17	 </entry>
       <entry>
 	<link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0218r1.html">
 	P0218R1
 	</link>
       </entry>
-      <entry align="center"> No </entry>
+      <entry align="center"> 8 </entry>
       <entry><code> __has_include(&lt;filesystem&gt;) </code>,
-	     <code> __cpp_lib_filesystem >= 201603 </code></entry>
+	     <code> __cpp_lib_filesystem >= 201603 </code>
+	     (requires linking with <code>-lstdc++fs</code>)
+      </entry>
     </row>
 
     <row>
-      <?dbhtml bgcolor="#C8B0B0" ?>
       <entry> Relative Paths for Filesystem</entry>
       <entry>
 	<link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0219r1.html">
 	P0219R1
 	</link>
       </entry>
-      <entry align="center"> No </entry>
-      <entry/>
+      <entry align="center"> 8 </entry>
+      <entry><code> __cpp_lib_filesystem >= 201606 </code></entry>
     </row>
 
     <row>
-      <?dbhtml bgcolor="#C8B0B0" ?>
       <entry> Adapting string_view by filesystem paths </entry>
       <entry>
 	<link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0392r0.pdf">
 	P0392R0
 	</link>
       </entry>
-      <entry align="center"> No </entry>
-      <entry/>
+      <entry align="center"> 8 </entry>
+      <entry><code> __cpp_lib_filesystem >= 201606 </code></entry>
     </row>
 
     <row>
+      <entry> Directory Entry Caching for Filesystem </entry>
+      <entry>
+       <link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0317r1.html">
+       P0317R1
+       </link>
+      </entry>
+      <entry align="center"> 8 </entry>
+      <entry><code> __cpp_lib_filesystem >= 201703 </code></entry>
+     </row>
+
+    <row>
       <entry> constexpr <code>atomic&lt;T&gt;::is_always_lock_free</code>	 </entry>
       <entry>
 	<link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0152r1.html">
 	P0152R1
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_atomic_is_always_lock_free >= 201603 </code></entry>
     </row>
 
@@ -739,37 +753,39 @@ Feature-testing recommendations for C++</link>.
     </row>
 
     <row>
-      <entry> Variadic <code>lock_guard</code> </entry>
+      <entry> Variadic <code>lock_guard</code> (Rev. 5) </entry>
       <entry>
 	<link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0156r2.html">
 	P0156R2
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
+      <entry align="center"> 7.1 </entry>
       <entry><code> __cpp_lib_scoped_lock >= 201703 </code></entry>
     </row>
 
     <row>
-      <entry> byte type definition </entry>
+      <entry> A byte type definition </entry>
       <entry>
 	<link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0298r3.pdf">
 	P0298R3
 	</link>
       </entry>
-      <entry align="center"> 7 </entry>
-      <entry><code> ??? </code></entry>
+      <entry align="center"> 7.1 </entry>
+      <entry><code> __cpp_lib_byte >= 201603 </code> (since 7.3, see Note 2)
+      </entry>
     </row>
 
     <row>
-      <?dbhtml bgcolor="#C8B0B0" ?>
+      <?dbhtml bgcolor="#B0B0B0" ?>
       <entry> Elementary string conversions </entry>
       <entry>
 	<link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0067r5.html">
 	P0067R5
 	</link>
       </entry>
-      <entry align="center"> No </entry>
-      <entry><code> ??? </code></entry>
+      <entry align="center"> 8 (only integral types supported) </entry>
+      <entry><code> __has_include(&lt;charconv&gt;) </code>
+             <code> __cpp_lib_to_chars >= 201611 </code></entry>
     </row>
 
 
@@ -777,8 +793,26 @@ Feature-testing recommendations for C++</link>.
 </tgroup>
 </table>
 
+<para>
+Note 1: This feature is supported in GCC 7.1 and 7.2 but before GCC 7.3 the
+<code>__cpp_lib</code> macro is not defined, and compilation will fail if the
+header is included without using <option>-std</option> to enable C++17 support.
+</para>
+
+<para>
+Note 2: This feature is supported in older releases but the
+<code>__cpp_lib</code> macro is not defined to the right value
+(or not defined at all) until the version shown in parentheses.
+</para>
+
+<para>
+Note 3: The mathematical special functions are enabled in C++17 mode from
+GCC 7.1 onwards. For GCC 6.x or for C++11/C++14 define
+<code>__STDCPP_WANT_MATH_SPEC_FUNCS__</code> to a non-zero value
+and test for <code>__STDCPP_MATH_SPEC_FUNCS__ >= 201003L</code>.
+</para>
 
-<table frame="all" xml:id="table.cxx1z_ts_status">
+<table frame="all" xml:id="table.cxx17_ts_status">
 <title>C++ Technical Specifications Implementation Status</title>
 
 <tgroup cols="4" align="left" colsep="0" rowsep="1">
@@ -935,49 +969,49 @@ Feature-testing recommendations for C++</link>.
    </para>
 
    <para>
-      <emphasis>20.6.5 [optional.bad_optional_access]</emphasis>
+      <emphasis>23.6.5 [optional.bad_optional_access]</emphasis>
       <code>what()</code> returns <literal>"bad optional access"</literal>.
    </para>
 
    <para>
-      <emphasis>20.7.2 [variant.variant]</emphasis>
+      <emphasis>23.7.3 [variant.variant]</emphasis>
       Whether <classname>variant</classname> supports over-aligned types
       should be documented here.
    </para>
 
    <para>
-      <emphasis>20.7.10 [variant.bad.access]</emphasis>
+      <emphasis>23.7.10 [variant.bad.access]</emphasis>
       <code>what()</code> returns <literal>"Unexpected index"</literal>.
    </para>
 
    <para>
-      <emphasis>20.12.5.2 [memory.resource.pool.options]</emphasis>
+      <emphasis>23.12.5.2 [memory.resource.pool.options]</emphasis>
       The limits for maximum number of blocks and largest allocation size
       supported by <classname>pool_options</classname> should be documented
       here.
    </para>
 
    <para>
-      <emphasis>20.12.6.1 [memory.resource.monotonic.buffer.ctor]</emphasis>
+      <emphasis>23.12.6.1 [memory.resource.monotonic.buffer.ctor]</emphasis>
       The default <code>next_buffer_size</code> and growth factor should
       be documented here.
    </para>
 
    <para>
-      <emphasis>20.15.4.3 [meta.unary.prop]</emphasis>
+      <emphasis>23.15.4.3 [meta.unary.prop]</emphasis>
       The predicate condition for
       <code>has_unique_object_representations</code> is true for all scalar
       types except floating point types.
    </para>
 
    <para>
-      <emphasis>20.19.3 [execpol.type],
-      25.2.3 [algorithms.parallel.exec]</emphasis>
+      <emphasis>23.19.3 [execpol.type],
+      28.4.3 [algorithms.parallel.exec]</emphasis>
       There are no implementation-defined execution policies.
    </para>
 
    <para>
-      <emphasis>22.4.2 [string.view.template]</emphasis>
+      <emphasis>24.4.2 [string.view.template]</emphasis>
       <classname>basic_string_view&lt;C, T&gt;::iterator</classname> is
       <code>C*</code> and
       <classname>basic_string_view&lt;C, T&gt;::const_iterator</classname> is
@@ -986,7 +1020,7 @@ Feature-testing recommendations for C++</link>.
 
 
    <para>
-      <emphasis>25.2.3 [algorithms.parallel.exec]</emphasis>
+      <emphasis>28.4.3 [algorithms.parallel.exec]</emphasis>
       Threads of execution created by <classname>std::thread</classname>
       provide concurrent forward progress guarantees, so threads of execution
       implicitly created by the library will provide parallel forward
@@ -994,39 +1028,39 @@ Feature-testing recommendations for C++</link>.
    </para>
 
    <para>
-      <emphasis>26.4.1 [cfenv.syn]</emphasis>
+      <emphasis>29.4.1 [cfenv.syn]</emphasis>
       The effects of the <filename>&lt;cfenv&gt;</filename> functions
       depends on whether the <code>FENV_ACCESS</code> pragma is supported,
       and on the C library that provides the header.
    </para>
 
    <para>
-      <emphasis>26.6.9 [c.math.rand]</emphasis>
+      <emphasis>29.6.9 [c.math.rand]</emphasis>
       Whether the <function>rand</function> function may introduce data
       races depends on the target C library that provides the function.
    </para>
 
    <para>
-      <emphasis>26.9.5 [sf.cmath]</emphasis>
+      <emphasis>29.9.5 [sf.cmath]</emphasis>
       The effect of calling the mathematical special functions with large
       inputs should be documented here.
    </para>
 
    <para>
-      <emphasis>27.10.2.1 [fs.conform.9945]</emphasis>
+      <emphasis>30.10.2.1 [fs.conform.9945]</emphasis>
       The behavior of the filesystem library implementation will depend on
       the target operating system. Some features will not be not supported
       on some targets.
    </para>
 
    <para>
-      <emphasis>27.10.6 [fs.filesystem.syn]</emphasis>
+      <emphasis>30.10.5 [fs.filesystem.syn]</emphasis>
       The clock used for file times is
       <classname>std::chrono::system_clock</classname>.
    </para>
 
    <para>
-      <emphasis>27.10.8 [path.generic]</emphasis>
+      <emphasis>30.10.7.1 [fs.path.generic]</emphasis>
       dot-dot in the root-directory refers to the root-directory itself.
    </para>
 
diff --git a/libstdc++-v3/doc/xml/manual/status_cxxis29124.xml b/libstdc++-v3/doc/xml/manual/status_cxxis29124.xml
new file mode 100644
index 00000000000..40a90fc9944
--- /dev/null
+++ b/libstdc++-v3/doc/xml/manual/status_cxxis29124.xml
@@ -0,0 +1,315 @@
+<section xmlns="http://docbook.org/ns/docbook" version="5.0"
+	 xml:id="status.iso.specfun" xreflabel="Status C++ 29124">
+<?dbhtml filename="status_iso_cxxis29124.html"?>
+
+<info><title>C++ IS 29124</title>
+  <keywordset>
+    <keyword>ISO C++</keyword>
+    <keyword>Special Functions</keyword>
+  </keywordset>
+</info>
+
+<para>
+This table is based on the table of contents of ISO/IEC FDIS 29124
+Doc No: N3060 Date: 2010-03-06
+Extensions to the C++ Library to support mathematical special functions
+</para>
+
+<para>
+Complete support for IS 29124 is in GCC 6.1 and later releases, when using
+at least C++11 (for older releases or C++98/C++03 use TR1 instead).
+For C++11 and C++14 the additions to the library are not declared by their
+respective headers unless <code>__STDCPP_WANT_MATH_SPEC_FUNCS__</code>
+is defined as a macro that expands to a non-zero integer constant.
+For C++17 the special functions are always declared (since GCC 7.1).
+</para>
+
+<para>
+When the special functions are declared the macro
+<code>__STDCPP_MATH_SPEC_FUNCS__</code> is defined to <code>201003L</code>.
+</para>
+
+<para>
+In addition to the special functions defined in IS 29124, for
+non-strict modes (i.e. <code>-std=gnu++NN</code> modes) the
+hypergeometric functions and confluent hypergeometric functions
+from TR1 are also provided, defined in namespace
+<code>__gnu_cxx</code>.
+</para>
+
+<!-- Status is Yes or No, Broken/Partial-->
+<!--
+   Yes
+
+   No
+      <?dbhtml bgcolor="#C8B0B0" ?>
+   Broken/Partial
+      <?dbhtml bgcolor="#B0B0B0" ?>
+-->
+<table frame="all" xml:id="table.specfun_status">
+<title>C++ Special Functions Implementation Status</title>
+
+<tgroup cols="4" align="left" colsep="0" rowsep="1">
+<colspec colname="c1"/>
+<colspec colname="c2"/>
+<colspec colname="c3"/>
+<colspec colname="c4"/>
+
+  <thead>
+    <row>
+      <entry>Section</entry>
+      <entry>Description</entry>
+      <entry>Status</entry>
+      <entry>Comments</entry>
+    </row>
+  </thead>
+  <tbody>
+    <row>
+      <?dbhtml bgcolor="#B0B0B0" ?>
+      <entry>7</entry>
+      <entry>Macro names</entry>
+      <entry>Partial</entry>
+      <entry>No diagnostic for inconsistent definitions of
+             <code>__STDCPP_WANT_MATH_SPEC_FUNCS__</code></entry>
+    </row>
+    <row>
+      <entry>8</entry>
+      <entry>Mathematical special functions</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1</entry>
+      <entry>Additions to header <code>&lt;cmath&gt;</code> synopsis</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.1</entry>
+      <entry>associated Laguerre polynomials</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.2</entry>
+      <entry>associated Legendre functions</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.3</entry>
+      <entry>beta function</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.4</entry>
+      <entry>(complete) elliptic integral of the first kind</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.5</entry>
+      <entry>(complete) elliptic integral of the second kind</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.6</entry>
+      <entry>(complete) elliptic integral of the third kind</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.7</entry>
+      <entry>regular modified cylindrical Bessel functions</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.8</entry>
+      <entry>cylindrical Bessel functions (of the first kind)</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.9</entry>
+      <entry>irregular modified cylindrical Bessel functions</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.10</entry>
+      <entry>cylindrical Neumann functions</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.11</entry>
+      <entry>(incomplete) elliptic integral of the first kind</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.12</entry>
+      <entry>(incomplete) elliptic integral of the second kind</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.13</entry>
+      <entry>(incomplete) elliptic integral of the third kind</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.14</entry>
+      <entry>exponential integral</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.15</entry>
+      <entry>Hermite polynomials</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.16</entry>
+      <entry>Laguerre polynomials</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.17</entry>
+      <entry>Legendre polynomials</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.18</entry>
+      <entry>Riemann zeta function</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.19</entry>
+      <entry>spherical Bessel functions (of the first kind)</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.20</entry>
+      <entry>spherical associated Legendre functions</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.1.21</entry>
+      <entry>spherical Neumann functions</entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <entry>8.2</entry>
+      <entry>Additions to header <code>&lt;math.h&gt;</code></entry>
+      <entry>Y</entry>
+      <entry/>
+    </row>
+    <row>
+      <?dbhtml bgcolor="#B0B0B0" ?>
+      <entry>8.3</entry>
+      <entry>The header <code>&lt;ctgmath&gt;</code></entry>
+      <entry>Partial</entry>
+      <entry>Conflicts with C++ 2011 requirements.</entry>
+    </row>
+    <row>
+      <?dbhtml bgcolor="#C8B0B0" ?>
+      <entry>8.4</entry>
+      <entry>The header <code>&lt;tgmath.h&gt;</code></entry>
+      <entry>N</entry>
+      <entry>Conflicts with C++ 2011 requirements.</entry>
+    </row>
+  </tbody>
+</tgroup>
+</table>
+
+<section xml:id="iso.specfun.specific" xreflabel="Implementation Specific"><info><title>Implementation Specific Behavior</title></info>
+
+   <para>For behaviour which is specified by the 2011 standard,
+      see <link linkend="iso.2011.specific">C++ 2011 Implementation
+      Specific Behavior</link>. This section documents behaviour which
+      is required by IS 29124.
+   </para>
+
+   <para>
+     <emphasis>7.2 [macro.user]/3 /4</emphasis> The functions declared in
+     Clause 8 are only declared when
+     <code>__STDCPP_WANT_MATH_SPEC_FUNCS__ == 1</code>
+     (or in C++17 mode, for GCC 7.1 and later).
+   </para>
+
+   <para>
+     <emphasis>8.1.1 [sf.cmath.Lnm]/1</emphasis> The effect of calling
+     these functions with <code>n >= 128</code> or <code>m >= 128</code>
+     should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.2 [sf.cmath.Plm]/3</emphasis> The effect of calling
+     these functions with <code>l >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.3 [sf.cmath.I]/3</emphasis> The effect of calling
+     these functions with <code>nu >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.8 [sf.cmath.J]/3</emphasis> The effect of calling
+     these functions with <code>nu >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.9 [sf.cmath.K]/3</emphasis> The effect of calling
+     these functions with <code>nu >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.10 [sf.cmath.N]/3</emphasis> The effect of calling
+     these functions with <code>nu >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.15 [sf.cmath.Hn]/3</emphasis> The effect of calling
+     these functions with <code>n >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.16 [sf.cmath.Ln]/3</emphasis> The effect of calling
+     these functions with <code>n >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.17 [sf.cmath.Pl]/3</emphasis> The effect of calling
+     these functions with <code>l >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.19 [sf.cmath.j]/3</emphasis> The effect of calling
+     these functions with <code>n >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.20 [sf.cmath.Ylm]/3</emphasis> The effect of calling
+     these functions with <code>l >= 128</code> should be described here.
+   </para>
+
+   <para>
+     <emphasis>8.1.21 [sf.cmath.n]/3</emphasis> The effect of calling
+     these functions with <code>n >= 128</code> should be described here.
+   </para>
+
+</section>
+
+</section>
diff --git a/libstdc++-v3/include/Makefile.am b/libstdc++-v3/include/Makefile.am
index 236c2d6059f..add2277df80 100644
--- a/libstdc++-v3/include/Makefile.am
+++ b/libstdc++-v3/include/Makefile.am
@@ -37,6 +37,7 @@ std_headers = \
 	${std_srcdir}/complex \
 	${std_srcdir}/condition_variable \
 	${std_srcdir}/deque \
+	${std_srcdir}/filesystem \
 	${std_srcdir}/forward_list \
 	${std_srcdir}/fstream \
 	${std_srcdir}/functional \
@@ -104,6 +105,10 @@ bits_headers = \
 	${bits_srcdir}/enable_special_members.h \
 	${bits_srcdir}/forward_list.h \
 	${bits_srcdir}/forward_list.tcc \
+	${bits_srcdir}/fs_dir.h \
+	${bits_srcdir}/fs_fwd.h \
+	${bits_srcdir}/fs_ops.h \
+	${bits_srcdir}/fs_path.h \
 	${bits_srcdir}/fstream.tcc \
 	${bits_srcdir}/functexcept.h \
 	${bits_srcdir}/functional_hash.h \
@@ -125,7 +130,6 @@ bits_headers = \
 	${bits_srcdir}/locale_facets_nonio.tcc \
 	${bits_srcdir}/localefwd.h \
 	${bits_srcdir}/mask_array.h \
-	${bits_srcdir}/specfun.h \
 	${bits_srcdir}/memoryfwd.h \
 	${bits_srcdir}/move.h \
 	${bits_srcdir}/node_handle.h \
@@ -156,6 +160,7 @@ bits_headers = \
 	${bits_srcdir}/shared_ptr_atomic.h \
 	${bits_srcdir}/shared_ptr_base.h \
 	${bits_srcdir}/slice_array.h \
+	${bits_srcdir}/specfun.h \
 	${bits_srcdir}/sstream.tcc \
 	${bits_srcdir}/std_abs.h \
 	${bits_srcdir}/std_function.h \
@@ -1026,7 +1031,7 @@ stamp-bits: ${bits_headers}
 	@$(STAMP) stamp-bits
 
 stamp-bits-sup: stamp-bits ${bits_sup_headers}
-	@-cd ${bits_builddir} && $(LN_S) $? . 2>/dev/null
+	@-cd ${bits_builddir} && $(LN_S) ${bits_sup_headers} . 2>/dev/null
 	@$(STAMP) stamp-bits-sup
 
 stamp-c_base: ${c_base_headers}
diff --git a/libstdc++-v3/include/Makefile.in b/libstdc++-v3/include/Makefile.in
index 39dfede549b..0e3fe49142e 100644
--- a/libstdc++-v3/include/Makefile.in
+++ b/libstdc++-v3/include/Makefile.in
@@ -330,6 +330,7 @@ std_headers = \
 	${std_srcdir}/complex \
 	${std_srcdir}/condition_variable \
 	${std_srcdir}/deque \
+	${std_srcdir}/filesystem \
 	${std_srcdir}/forward_list \
 	${std_srcdir}/fstream \
 	${std_srcdir}/functional \
@@ -397,6 +398,10 @@ bits_headers = \
 	${bits_srcdir}/enable_special_members.h \
 	${bits_srcdir}/forward_list.h \
 	${bits_srcdir}/forward_list.tcc \
+	${bits_srcdir}/fs_dir.h \
+	${bits_srcdir}/fs_fwd.h \
+	${bits_srcdir}/fs_ops.h \
+	${bits_srcdir}/fs_path.h \
 	${bits_srcdir}/fstream.tcc \
 	${bits_srcdir}/functexcept.h \
 	${bits_srcdir}/functional_hash.h \
@@ -418,7 +423,6 @@ bits_headers = \
 	${bits_srcdir}/locale_facets_nonio.tcc \
 	${bits_srcdir}/localefwd.h \
 	${bits_srcdir}/mask_array.h \
-	${bits_srcdir}/specfun.h \
 	${bits_srcdir}/memoryfwd.h \
 	${bits_srcdir}/move.h \
 	${bits_srcdir}/node_handle.h \
@@ -449,6 +453,7 @@ bits_headers = \
 	${bits_srcdir}/shared_ptr_atomic.h \
 	${bits_srcdir}/shared_ptr_base.h \
 	${bits_srcdir}/slice_array.h \
+	${bits_srcdir}/specfun.h \
 	${bits_srcdir}/sstream.tcc \
 	${bits_srcdir}/std_abs.h \
 	${bits_srcdir}/std_function.h \
@@ -1475,7 +1480,7 @@ stamp-bits: ${bits_headers}
 	@$(STAMP) stamp-bits
 
 stamp-bits-sup: stamp-bits ${bits_sup_headers}
-	@-cd ${bits_builddir} && $(LN_S) $? . 2>/dev/null
+	@-cd ${bits_builddir} && $(LN_S) ${bits_sup_headers} . 2>/dev/null
 	@$(STAMP) stamp-bits-sup
 
 stamp-c_base: ${c_base_headers}
diff --git a/libstdc++-v3/include/bits/fs_dir.h b/libstdc++-v3/include/bits/fs_dir.h
new file mode 100644
index 00000000000..579a269711e
--- /dev/null
+++ b/libstdc++-v3/include/bits/fs_dir.h
@@ -0,0 +1,525 @@
+// Filesystem directory utilities -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+/** @file include/bits/fs_dir.h
+ *  This is an internal header file, included by other library headers.
+ *  Do not attempt to use it directly. @headername{filesystem}
+ */
+
+#ifndef _GLIBCXX_FS_DIR_H
+#define _GLIBCXX_FS_DIR_H 1
+
+#if __cplusplus >= 201703L
+# include <typeinfo>
+# include <ext/concurrence.h>
+# include <bits/unique_ptr.h>
+# include <bits/shared_ptr.h>
+
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+
+namespace filesystem
+{
+  /**
+   * @ingroup filesystem
+   * @{
+   */
+
+  class file_status
+  {
+  public:
+    // constructors and destructor
+    file_status() noexcept : file_status(file_type::none) {}
+
+    explicit
+    file_status(file_type __ft, perms __prms = perms::unknown) noexcept
+    : _M_type(__ft), _M_perms(__prms) { }
+
+    file_status(const file_status&) noexcept = default;
+    file_status(file_status&&) noexcept = default;
+    ~file_status() = default;
+
+    file_status& operator=(const file_status&) noexcept = default;
+    file_status& operator=(file_status&&) noexcept = default;
+
+    // observers
+    file_type  type() const noexcept { return _M_type; }
+    perms      permissions() const noexcept { return _M_perms; }
+
+    // modifiers
+    void       type(file_type __ft) noexcept { _M_type = __ft; }
+    void       permissions(perms __prms) noexcept { _M_perms = __prms; }
+
+  private:
+    file_type	_M_type;
+    perms	_M_perms;
+  };
+
+_GLIBCXX_BEGIN_NAMESPACE_CXX11
+
+  struct _Dir;
+  class directory_iterator;
+  class recursive_directory_iterator;
+
+  class directory_entry
+  {
+  public:
+    // constructors and destructor
+    directory_entry() noexcept = default;
+    directory_entry(const directory_entry&) = default;
+    directory_entry(directory_entry&&) noexcept = default;
+
+    explicit
+    directory_entry(const filesystem::path& __p)
+    : _M_path(__p)
+    { refresh(); }
+
+    directory_entry(const filesystem::path& __p, error_code& __ec)
+    : _M_path(__p)
+    {
+      refresh(__ec);
+      if (__ec)
+	_M_path.clear();
+    }
+
+    ~directory_entry() = default;
+
+    // modifiers
+    directory_entry& operator=(const directory_entry&) = default;
+    directory_entry& operator=(directory_entry&&) noexcept = default;
+
+    void
+    assign(const filesystem::path& __p)
+    {
+      _M_path = __p;
+      refresh();
+    }
+
+    void
+    assign(const filesystem::path& __p, error_code& __ec)
+    {
+      _M_path = __p;
+      refresh(__ec);
+    }
+
+    void
+    replace_filename(const filesystem::path& __p)
+    {
+      _M_path.replace_filename(__p);
+      refresh();
+    }
+
+    void
+    replace_filename(const filesystem::path& __p, error_code& __ec)
+    {
+      _M_path.replace_filename(__p);
+      refresh(__ec);
+    }
+
+    void refresh() { _M_type = symlink_status().type(); }
+    void refresh(error_code& __ec) { _M_type = symlink_status(__ec).type(); }
+
+    // observers
+    const filesystem::path& path() const noexcept { return _M_path; }
+    operator const filesystem::path& () const noexcept { return _M_path; }
+
+    bool
+    exists() const
+    { return filesystem::exists(file_status{_M_file_type()}); }
+
+    bool
+    exists(error_code& __ec) const noexcept
+    { return filesystem::exists(file_status{_M_file_type(__ec)}); }
+
+    bool
+    is_block_file() const
+    { return _M_file_type() == file_type::block; }
+
+    bool
+    is_block_file(error_code& __ec) const noexcept
+    { return _M_file_type(__ec) == file_type::block; }
+
+    bool
+    is_character_file() const
+    { return _M_file_type() == file_type::character; }
+
+    bool
+    is_character_file(error_code& __ec) const noexcept
+    { return _M_file_type(__ec) == file_type::character; }
+
+    bool
+    is_directory() const
+    { return _M_file_type() == file_type::directory; }
+
+    bool
+    is_directory(error_code& __ec) const noexcept
+    { return _M_file_type(__ec) == file_type::directory; }
+
+    bool
+    is_fifo() const
+    { return _M_file_type() == file_type::fifo; }
+
+    bool
+    is_fifo(error_code& __ec) const noexcept
+    { return _M_file_type(__ec) == file_type::fifo; }
+
+    bool
+    is_other() const
+    { return filesystem::is_other(file_status{_M_file_type()}); }
+
+    bool
+    is_other(error_code& __ec) const noexcept
+    { return filesystem::is_other(file_status{_M_file_type(__ec)}); }
+
+    bool
+    is_regular_file() const
+    { return _M_file_type() == file_type::regular; }
+
+    bool
+    is_regular_file(error_code& __ec) const noexcept
+    { return _M_file_type(__ec) == file_type::regular; }
+
+    bool
+    is_socket() const
+    { return _M_file_type() == file_type::socket; }
+
+    bool
+    is_socket(error_code& __ec) const noexcept
+    { return _M_file_type(__ec) == file_type::socket; }
+
+    bool
+    is_symlink() const
+    {
+      if (_M_type != file_type::none)
+	return _M_type == file_type::symlink;
+      return symlink_status().type() == file_type::symlink;
+    }
+
+    bool
+    is_symlink(error_code& __ec) const noexcept
+    {
+      if (_M_type != file_type::none)
+	return _M_type == file_type::symlink;
+      return symlink_status(__ec).type() == file_type::symlink;
+    }
+
+    uintmax_t
+    file_size() const
+    { return filesystem::file_size(_M_path); }
+
+    uintmax_t
+    file_size(error_code& __ec) const noexcept
+    { return filesystem::file_size(_M_path, __ec); }
+
+    uintmax_t
+    hard_link_count() const
+    { return filesystem::hard_link_count(_M_path); }
+
+    uintmax_t
+    hard_link_count(error_code& __ec) const noexcept
+    { return filesystem::hard_link_count(_M_path, __ec); }
+
+    file_time_type
+    last_write_time() const
+    { return filesystem::last_write_time(_M_path); }
+
+
+    file_time_type
+    last_write_time(error_code& __ec) const noexcept
+    { return filesystem::last_write_time(_M_path, __ec); }
+
+    file_status
+    status() const
+    { return filesystem::status(_M_path); }
+
+    file_status
+    status(error_code& __ec) const noexcept
+    { return filesystem::status(_M_path, __ec); }
+
+    file_status
+    symlink_status() const
+    { return filesystem::symlink_status(_M_path); }
+
+    file_status
+    symlink_status(error_code& __ec) const noexcept
+    { return filesystem::symlink_status(_M_path, __ec); }
+
+    bool
+    operator< (const directory_entry& __rhs) const noexcept
+    { return _M_path < __rhs._M_path; }
+
+    bool
+    operator==(const directory_entry& __rhs) const noexcept
+    { return _M_path == __rhs._M_path; }
+
+    bool
+    operator!=(const directory_entry& __rhs) const noexcept
+    { return _M_path != __rhs._M_path; }
+
+    bool
+    operator<=(const directory_entry& __rhs) const noexcept
+    { return _M_path <= __rhs._M_path; }
+
+    bool
+    operator> (const directory_entry& __rhs) const noexcept
+    { return _M_path > __rhs._M_path; }
+
+    bool
+    operator>=(const directory_entry& __rhs) const noexcept
+    { return _M_path >= __rhs._M_path; }
+
+  private:
+    friend class _Dir;
+    friend class directory_iterator;
+    friend class recursive_directory_iterator;
+
+    directory_entry(const filesystem::path& __p, file_type __t)
+    : _M_path(__p), _M_type(__t)
+    { }
+
+    // Equivalent to status().type() but uses cached value, if any.
+    file_type
+    _M_file_type() const
+    {
+      if (_M_type != file_type::none && _M_type != file_type::symlink)
+	return _M_type;
+      return status().type();
+    }
+
+    // Equivalent to status(__ec).type() but uses cached value, if any.
+    file_type
+    _M_file_type(error_code& __ec) const noexcept
+    {
+      if (_M_type != file_type::none && _M_type != file_type::symlink)
+	return _M_type;
+      return status(__ec).type();
+    }
+
+    filesystem::path	_M_path;
+    file_type		_M_type = file_type::none;
+  };
+
+  struct __directory_iterator_proxy
+  {
+    const directory_entry& operator*() const& noexcept { return _M_entry; }
+
+    directory_entry operator*() && noexcept { return std::move(_M_entry); }
+
+  private:
+    friend class directory_iterator;
+    friend class recursive_directory_iterator;
+
+    explicit
+    __directory_iterator_proxy(const directory_entry& __e) : _M_entry(__e) { }
+
+    directory_entry _M_entry;
+  };
+
+  class directory_iterator
+  {
+  public:
+    typedef directory_entry        value_type;
+    typedef ptrdiff_t              difference_type;
+    typedef const directory_entry* pointer;
+    typedef const directory_entry& reference;
+    typedef input_iterator_tag     iterator_category;
+
+    directory_iterator() = default;
+
+    explicit
+    directory_iterator(const path& __p)
+    : directory_iterator(__p, directory_options::none, nullptr) { }
+
+    directory_iterator(const path& __p, directory_options __options)
+    : directory_iterator(__p, __options, nullptr) { }
+
+    directory_iterator(const path& __p, error_code& __ec)
+    : directory_iterator(__p, directory_options::none, __ec) { }
+
+    directory_iterator(const path& __p, directory_options __options,
+		       error_code& __ec)
+    : directory_iterator(__p, __options, &__ec) { }
+
+    directory_iterator(const directory_iterator& __rhs) = default;
+
+    directory_iterator(directory_iterator&& __rhs) noexcept = default;
+
+    ~directory_iterator() = default;
+
+    directory_iterator&
+    operator=(const directory_iterator& __rhs) = default;
+
+    directory_iterator&
+    operator=(directory_iterator&& __rhs) noexcept = default;
+
+    const directory_entry& operator*() const;
+    const directory_entry* operator->() const { return &**this; }
+    directory_iterator&    operator++();
+    directory_iterator&    increment(error_code& __ec);
+
+    __directory_iterator_proxy operator++(int)
+    {
+      __directory_iterator_proxy __pr{**this};
+      ++*this;
+      return __pr;
+    }
+
+  private:
+    directory_iterator(const path&, directory_options, error_code*);
+
+    friend bool
+    operator==(const directory_iterator& __lhs,
+               const directory_iterator& __rhs);
+
+    friend class recursive_directory_iterator;
+
+    std::shared_ptr<_Dir> _M_dir;
+  };
+
+  inline directory_iterator
+  begin(directory_iterator __iter) noexcept
+  { return __iter; }
+
+  inline directory_iterator
+  end(directory_iterator) noexcept
+  { return directory_iterator(); }
+
+  inline bool
+  operator==(const directory_iterator& __lhs, const directory_iterator& __rhs)
+  {
+    return !__rhs._M_dir.owner_before(__lhs._M_dir)
+      && !__lhs._M_dir.owner_before(__rhs._M_dir);
+  }
+
+  inline bool
+  operator!=(const directory_iterator& __lhs, const directory_iterator& __rhs)
+  { return !(__lhs == __rhs); }
+
+  class recursive_directory_iterator
+  {
+  public:
+    typedef directory_entry        value_type;
+    typedef ptrdiff_t              difference_type;
+    typedef const directory_entry* pointer;
+    typedef const directory_entry& reference;
+    typedef input_iterator_tag     iterator_category;
+
+    recursive_directory_iterator() = default;
+
+    explicit
+    recursive_directory_iterator(const path& __p)
+    : recursive_directory_iterator(__p, directory_options::none, nullptr) { }
+
+    recursive_directory_iterator(const path& __p, directory_options __options)
+    : recursive_directory_iterator(__p, __options, nullptr) { }
+
+    recursive_directory_iterator(const path& __p, directory_options __options,
+                                 error_code& __ec)
+    : recursive_directory_iterator(__p, __options, &__ec) { }
+
+    recursive_directory_iterator(const path& __p, error_code& __ec)
+    : recursive_directory_iterator(__p, directory_options::none, &__ec) { }
+
+    recursive_directory_iterator(
+        const recursive_directory_iterator&) = default;
+
+    recursive_directory_iterator(recursive_directory_iterator&&) = default;
+
+    ~recursive_directory_iterator();
+
+    // observers
+    directory_options  options() const { return _M_options; }
+    int                depth() const;
+    bool               recursion_pending() const { return _M_pending; }
+
+    const directory_entry& operator*() const;
+    const directory_entry* operator->() const { return &**this; }
+
+    // modifiers
+    recursive_directory_iterator&
+    operator=(const recursive_directory_iterator& __rhs) noexcept;
+    recursive_directory_iterator&
+    operator=(recursive_directory_iterator&& __rhs) noexcept;
+
+    recursive_directory_iterator& operator++();
+    recursive_directory_iterator& increment(error_code& __ec);
+
+    __directory_iterator_proxy operator++(int)
+    {
+      __directory_iterator_proxy __pr{**this};
+      ++*this;
+      return __pr;
+    }
+
+    void pop();
+    void pop(error_code&);
+
+    void disable_recursion_pending() { _M_pending = false; }
+
+  private:
+    recursive_directory_iterator(const path&, directory_options, error_code*);
+
+    friend bool
+    operator==(const recursive_directory_iterator& __lhs,
+               const recursive_directory_iterator& __rhs);
+
+    struct _Dir_stack;
+    std::shared_ptr<_Dir_stack> _M_dirs;
+    directory_options _M_options = {};
+    bool _M_pending = false;
+  };
+
+  inline recursive_directory_iterator
+  begin(recursive_directory_iterator __iter) noexcept
+  { return __iter; }
+
+  inline recursive_directory_iterator
+  end(recursive_directory_iterator) noexcept
+  { return recursive_directory_iterator(); }
+
+  inline bool
+  operator==(const recursive_directory_iterator& __lhs,
+             const recursive_directory_iterator& __rhs)
+  {
+    return !__rhs._M_dirs.owner_before(__lhs._M_dirs)
+      && !__lhs._M_dirs.owner_before(__rhs._M_dirs);
+  }
+
+  inline bool
+  operator!=(const recursive_directory_iterator& __lhs,
+             const recursive_directory_iterator& __rhs)
+  { return !(__lhs == __rhs); }
+
+_GLIBCXX_END_NAMESPACE_CXX11
+
+  // @} group filesystem
+} // namespace filesystem
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
+
+#endif // C++17
+
+#endif // _GLIBCXX_FS_DIR_H
diff --git a/libstdc++-v3/include/bits/fs_fwd.h b/libstdc++-v3/include/bits/fs_fwd.h
new file mode 100644
index 00000000000..f408a39b974
--- /dev/null
+++ b/libstdc++-v3/include/bits/fs_fwd.h
@@ -0,0 +1,348 @@
+// Filesystem declarations -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+/** @file include/bits/fs_fwd.h
+ *  This is an internal header file, included by other library headers.
+ *  Do not attempt to use it directly. @headername{filesystem}
+ */
+
+#ifndef _GLIBCXX_FS_FWD_H
+#define _GLIBCXX_FS_FWD_H 1
+
+#if __cplusplus >= 201703L
+
+#include <system_error>
+#include <cstdint>
+#include <chrono>
+
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+
+namespace filesystem
+{
+#if _GLIBCXX_USE_CXX11_ABI
+inline namespace __cxx11 __attribute__((__abi_tag__ ("cxx11"))) { }
+#endif
+
+  /**
+   * @defgroup filesystem Filesystem
+   *
+   * Utilities for performing operations on file systems and their components,
+   * such as paths, regular files, and directories.
+   *
+   * @{
+   */
+
+  class file_status;
+_GLIBCXX_BEGIN_NAMESPACE_CXX11
+  class path;
+  class filesystem_error;
+  class directory_entry;
+  class directory_iterator;
+  class recursive_directory_iterator;
+_GLIBCXX_END_NAMESPACE_CXX11
+
+  struct space_info
+  {
+    uintmax_t capacity;
+    uintmax_t free;
+    uintmax_t available;
+  };
+
+  enum class file_type : signed char {
+      none = 0, not_found = -1, regular = 1, directory = 2, symlink = 3,
+      block = 4, character = 5, fifo = 6, socket = 7, unknown = 8
+  };
+
+  /// Bitmask type
+  enum class copy_options : unsigned short {
+      none = 0,
+      skip_existing = 1, overwrite_existing = 2, update_existing = 4,
+      recursive = 8,
+      copy_symlinks = 16, skip_symlinks = 32,
+      directories_only = 64, create_symlinks = 128, create_hard_links = 256
+  };
+
+  constexpr copy_options
+  operator&(copy_options __x, copy_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<copy_options>::type;
+    return static_cast<copy_options>(
+	static_cast<__utype>(__x) & static_cast<__utype>(__y));
+  }
+
+  constexpr copy_options
+  operator|(copy_options __x, copy_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<copy_options>::type;
+    return static_cast<copy_options>(
+	static_cast<__utype>(__x) | static_cast<__utype>(__y));
+  }
+
+  constexpr copy_options
+  operator^(copy_options __x, copy_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<copy_options>::type;
+    return static_cast<copy_options>(
+	static_cast<__utype>(__x) ^ static_cast<__utype>(__y));
+  }
+
+  constexpr copy_options
+  operator~(copy_options __x) noexcept
+  {
+    using __utype = typename std::underlying_type<copy_options>::type;
+    return static_cast<copy_options>(~static_cast<__utype>(__x));
+  }
+
+  inline copy_options&
+  operator&=(copy_options& __x, copy_options __y) noexcept
+  { return __x = __x & __y; }
+
+  inline copy_options&
+  operator|=(copy_options& __x, copy_options __y) noexcept
+  { return __x = __x | __y; }
+
+  inline copy_options&
+  operator^=(copy_options& __x, copy_options __y) noexcept
+  { return __x = __x ^ __y; }
+
+
+  /// Bitmask type
+  enum class perms : unsigned {
+      none		=  0,
+      owner_read	=  0400,
+      owner_write	=  0200,
+      owner_exec	=  0100,
+      owner_all		=  0700,
+      group_read	=   040,
+      group_write	=   020,
+      group_exec	=   010,
+      group_all		=   070,
+      others_read	=    04,
+      others_write	=    02,
+      others_exec	=    01,
+      others_all	=    07,
+      all		=  0777,
+      set_uid		= 04000,
+      set_gid		= 02000,
+      sticky_bit	= 01000,
+      mask		= 07777,
+      unknown		=  0xFFFF,
+  };
+
+  constexpr perms
+  operator&(perms __x, perms __y) noexcept
+  {
+    using __utype = typename std::underlying_type<perms>::type;
+    return static_cast<perms>(
+	static_cast<__utype>(__x) & static_cast<__utype>(__y));
+  }
+
+  constexpr perms
+  operator|(perms __x, perms __y) noexcept
+  {
+    using __utype = typename std::underlying_type<perms>::type;
+    return static_cast<perms>(
+	static_cast<__utype>(__x) | static_cast<__utype>(__y));
+  }
+
+  constexpr perms
+  operator^(perms __x, perms __y) noexcept
+  {
+    using __utype = typename std::underlying_type<perms>::type;
+    return static_cast<perms>(
+	static_cast<__utype>(__x) ^ static_cast<__utype>(__y));
+  }
+
+  constexpr perms
+  operator~(perms __x) noexcept
+  {
+    using __utype = typename std::underlying_type<perms>::type;
+    return static_cast<perms>(~static_cast<__utype>(__x));
+  }
+
+  inline perms&
+  operator&=(perms& __x, perms __y) noexcept
+  { return __x = __x & __y; }
+
+  inline perms&
+  operator|=(perms& __x, perms __y) noexcept
+  { return __x = __x | __y; }
+
+  inline perms&
+  operator^=(perms& __x, perms __y) noexcept
+  { return __x = __x ^ __y; }
+
+  /// Bitmask type
+  enum class perm_options : unsigned {
+      replace	= 0x1,
+      add	= 0x2,
+      remove	= 0x4,
+      nofollow	= 0x8
+  };
+
+  constexpr perm_options
+  operator&(perm_options __x, perm_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<perm_options>::type;
+    return static_cast<perm_options>(
+	static_cast<__utype>(__x) & static_cast<__utype>(__y));
+  }
+
+  constexpr perm_options
+  operator|(perm_options __x, perm_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<perm_options>::type;
+    return static_cast<perm_options>(
+	static_cast<__utype>(__x) | static_cast<__utype>(__y));
+  }
+
+  constexpr perm_options
+  operator^(perm_options __x, perm_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<perm_options>::type;
+    return static_cast<perm_options>(
+	static_cast<__utype>(__x) ^ static_cast<__utype>(__y));
+  }
+
+  constexpr perm_options
+  operator~(perm_options __x) noexcept
+  {
+    using __utype = typename std::underlying_type<perm_options>::type;
+    return static_cast<perm_options>(~static_cast<__utype>(__x));
+  }
+
+  inline perm_options&
+  operator&=(perm_options& __x, perm_options __y) noexcept
+  { return __x = __x & __y; }
+
+  inline perm_options&
+  operator|=(perm_options& __x, perm_options __y) noexcept
+  { return __x = __x | __y; }
+
+  inline perm_options&
+  operator^=(perm_options& __x, perm_options __y) noexcept
+  { return __x = __x ^ __y; }
+
+  // Bitmask type
+  enum class directory_options : unsigned char {
+      none = 0, follow_directory_symlink = 1, skip_permission_denied = 2
+  };
+
+  constexpr directory_options
+  operator&(directory_options __x, directory_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<directory_options>::type;
+    return static_cast<directory_options>(
+	static_cast<__utype>(__x) & static_cast<__utype>(__y));
+  }
+
+  constexpr directory_options
+  operator|(directory_options __x, directory_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<directory_options>::type;
+    return static_cast<directory_options>(
+	static_cast<__utype>(__x) | static_cast<__utype>(__y));
+  }
+
+  constexpr directory_options
+  operator^(directory_options __x, directory_options __y) noexcept
+  {
+    using __utype = typename std::underlying_type<directory_options>::type;
+    return static_cast<directory_options>(
+	static_cast<__utype>(__x) ^ static_cast<__utype>(__y));
+  }
+
+  constexpr directory_options
+  operator~(directory_options __x) noexcept
+  {
+    using __utype = typename std::underlying_type<directory_options>::type;
+    return static_cast<directory_options>(~static_cast<__utype>(__x));
+  }
+
+  inline directory_options&
+  operator&=(directory_options& __x, directory_options __y) noexcept
+  { return __x = __x & __y; }
+
+  inline directory_options&
+  operator|=(directory_options& __x, directory_options __y) noexcept
+  { return __x = __x | __y; }
+
+  inline directory_options&
+  operator^=(directory_options& __x, directory_options __y) noexcept
+  { return __x = __x ^ __y; }
+
+  using file_time_type = std::chrono::system_clock::time_point;
+
+  // operational functions
+
+  void copy(const path& __from, const path& __to, copy_options __options);
+  void copy(const path& __from, const path& __to, copy_options __options,
+	    error_code&) noexcept;
+
+  bool copy_file(const path& __from, const path& __to, copy_options __option);
+  bool copy_file(const path& __from, const path& __to, copy_options __option,
+		 error_code&) noexcept;
+
+  path current_path();
+
+  bool exists(file_status) noexcept;
+
+  bool is_other(file_status) noexcept;
+
+  uintmax_t file_size(const path&);
+  uintmax_t file_size(const path&, error_code&) noexcept;
+  uintmax_t hard_link_count(const path&);
+  uintmax_t hard_link_count(const path&, error_code&) noexcept;
+  file_time_type last_write_time(const path&);
+  file_time_type last_write_time(const path&, error_code&) noexcept;
+
+  void permissions(const path&, perms, perm_options, error_code&);
+
+  path proximate(const path& __p, const path& __base, error_code& __ec);
+  path proximate(const path& __p, const path& __base, error_code& __ec);
+
+  path relative(const path& __p, const path& __base, error_code& __ec);
+
+  file_status status(const path&);
+  file_status status(const path&, error_code&) noexcept;
+
+  bool status_known(file_status) noexcept;
+
+  file_status symlink_status(const path&);
+  file_status symlink_status(const path&, error_code&) noexcept;
+
+  bool is_regular_file(file_status) noexcept;
+  bool is_symlink(file_status) noexcept;
+
+  // @} group filesystem
+} // namespace filesystem
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
+
+#endif // C++17
+
+#endif // _GLIBCXX_FS_FWD_H
diff --git a/libstdc++-v3/include/bits/fs_ops.h b/libstdc++-v3/include/bits/fs_ops.h
new file mode 100644
index 00000000000..075d61e2a63
--- /dev/null
+++ b/libstdc++-v3/include/bits/fs_ops.h
@@ -0,0 +1,311 @@
+// Filesystem operational functions -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your __option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+/** @file include/bits/fs_fwd.h
+ *  This is an internal header file, included by other library headers.
+ *  Do not attempt to use it directly. @headername{filesystem}
+ */
+
+#ifndef _GLIBCXX_FS_OPS_H
+#define _GLIBCXX_FS_OPS_H 1
+
+#if __cplusplus >= 201703L
+
+#include <cstdint>
+
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+
+namespace filesystem
+{
+  /**
+   * @ingroup filesystem
+   * @{
+   */
+
+  path absolute(const path& __p);
+  path absolute(const path& __p, error_code& __ec);
+
+  path canonical(const path& __p);
+  path canonical(const path& __p, error_code& __ec);
+
+  inline void
+  copy(const path& __from, const path& __to)
+  { copy(__from, __to, copy_options::none); }
+
+  inline void
+  copy(const path& __from, const path& __to, error_code& __ec)
+  { copy(__from, __to, copy_options::none, __ec); }
+
+  void copy(const path& __from, const path& __to, copy_options __options);
+  void copy(const path& __from, const path& __to, copy_options __options,
+	    error_code& __ec);
+
+  inline bool
+  copy_file(const path& __from, const path& __to)
+  { return copy_file(__from, __to, copy_options::none); }
+
+  inline bool
+  copy_file(const path& __from, const path& __to, error_code& __ec)
+  { return copy_file(__from, __to, copy_options::none, __ec); }
+
+  bool copy_file(const path& __from, const path& __to, copy_options __option);
+  bool copy_file(const path& __from, const path& __to, copy_options __option,
+		 error_code& __ec);
+
+  void copy_symlink(const path& __existing_symlink, const path& __new_symlink);
+  void copy_symlink(const path& __existing_symlink, const path& __new_symlink,
+		    error_code& __ec) noexcept;
+
+  bool create_directories(const path& __p);
+  bool create_directories(const path& __p, error_code& __ec);
+
+  bool create_directory(const path& __p);
+  bool create_directory(const path& __p, error_code& __ec) noexcept;
+
+  bool create_directory(const path& __p, const path& attributes);
+  bool create_directory(const path& __p, const path& attributes,
+			error_code& __ec) noexcept;
+
+  void create_directory_symlink(const path& __to, const path& __new_symlink);
+  void create_directory_symlink(const path& __to, const path& __new_symlink,
+				error_code& __ec) noexcept;
+
+  void create_hard_link(const path& __to, const path& __new_hard_link);
+  void create_hard_link(const path& __to, const path& __new_hard_link,
+			error_code& __ec) noexcept;
+
+  void create_symlink(const path& __to, const path& __new_symlink);
+  void create_symlink(const path& __to, const path& __new_symlink,
+		      error_code& __ec) noexcept;
+
+  path current_path();
+  path current_path(error_code& __ec);
+  void current_path(const path& __p);
+  void current_path(const path& __p, error_code& __ec) noexcept;
+
+  bool
+  equivalent(const path& __p1, const path& __p2);
+
+  bool
+  equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept;
+
+  inline bool
+  exists(file_status __s) noexcept
+  { return status_known(__s) && __s.type() != file_type::not_found; }
+
+  inline bool
+  exists(const path& __p)
+  { return exists(status(__p)); }
+
+  inline bool
+  exists(const path& __p, error_code& __ec) noexcept
+  {
+    auto __s = status(__p, __ec);
+    if (status_known(__s))
+      __ec.clear();
+    return exists(__s);
+  }
+
+  uintmax_t file_size(const path& __p);
+  uintmax_t file_size(const path& __p, error_code& __ec) noexcept;
+
+  uintmax_t hard_link_count(const path& __p);
+  uintmax_t hard_link_count(const path& __p, error_code& __ec) noexcept;
+
+  inline bool
+  is_block_file(file_status __s) noexcept
+  { return __s.type() == file_type::block; }
+
+  inline bool
+  is_block_file(const path& __p)
+  { return is_block_file(status(__p)); }
+
+  inline bool
+  is_block_file(const path& __p, error_code& __ec) noexcept
+  { return is_block_file(status(__p, __ec)); }
+
+  inline bool
+  is_character_file(file_status __s) noexcept
+  { return __s.type() == file_type::character; }
+
+  inline bool
+  is_character_file(const path& __p)
+  { return is_character_file(status(__p)); }
+
+  inline bool
+  is_character_file(const path& __p, error_code& __ec) noexcept
+  { return is_character_file(status(__p, __ec)); }
+
+  inline bool
+  is_directory(file_status __s) noexcept
+  { return __s.type() == file_type::directory; }
+
+  inline bool
+  is_directory(const path& __p)
+  { return is_directory(status(__p)); }
+
+  inline bool
+  is_directory(const path& __p, error_code& __ec) noexcept
+  { return is_directory(status(__p, __ec)); }
+
+  bool is_empty(const path& __p);
+  bool is_empty(const path& __p, error_code& __ec);
+
+  inline bool
+  is_fifo(file_status __s) noexcept
+  { return __s.type() == file_type::fifo; }
+
+  inline bool
+  is_fifo(const path& __p)
+  { return is_fifo(status(__p)); }
+
+  inline bool
+  is_fifo(const path& __p, error_code& __ec) noexcept
+  { return is_fifo(status(__p, __ec)); }
+
+  inline bool
+  is_other(file_status __s) noexcept
+  {
+    return exists(__s) && !is_regular_file(__s) && !is_directory(__s)
+      && !is_symlink(__s);
+  }
+
+  inline bool
+  is_other(const path& __p)
+  { return is_other(status(__p)); }
+
+  inline bool
+  is_other(const path& __p, error_code& __ec) noexcept
+  { return is_other(status(__p, __ec)); }
+
+  inline bool
+  is_regular_file(file_status __s) noexcept
+  { return __s.type() == file_type::regular; }
+
+  inline bool
+  is_regular_file(const path& __p)
+  { return is_regular_file(status(__p)); }
+
+  inline bool
+  is_regular_file(const path& __p, error_code& __ec) noexcept
+  { return is_regular_file(status(__p, __ec)); }
+
+  inline bool
+  is_socket(file_status __s) noexcept
+  { return __s.type() == file_type::socket; }
+
+  inline bool
+  is_socket(const path& __p)
+  { return is_socket(status(__p)); }
+
+  inline bool
+  is_socket(const path& __p, error_code& __ec) noexcept
+  { return is_socket(status(__p, __ec)); }
+
+  inline bool
+  is_symlink(file_status __s) noexcept
+  { return __s.type() == file_type::symlink; }
+
+  inline bool
+  is_symlink(const path& __p)
+  { return is_symlink(symlink_status(__p)); }
+
+  inline bool
+  is_symlink(const path& __p, error_code& __ec) noexcept
+  { return is_symlink(symlink_status(__p, __ec)); }
+
+  file_time_type  last_write_time(const path& __p);
+  file_time_type  last_write_time(const path& __p, error_code& __ec) noexcept;
+  void last_write_time(const path& __p, file_time_type __new_time);
+  void last_write_time(const path& __p, file_time_type __new_time,
+		       error_code& __ec) noexcept;
+
+  void
+  permissions(const path& __p, perms __prms,
+	      perm_options __opts = perm_options::replace);
+
+  inline void
+  permissions(const path& __p, perms __prms, error_code& __ec) noexcept
+  { permissions(__p, __prms, perm_options::replace, __ec); }
+
+  void
+  permissions(const path& __p, perms __prms, perm_options __opts,
+	      error_code& __ec);
+
+  inline path proximate(const path& __p, error_code& __ec)
+  { return proximate(__p, current_path(), __ec); }
+
+  path proximate(const path& __p, const path& __base = current_path());
+  path proximate(const path& __p, const path& __base, error_code& __ec);
+
+  path read_symlink(const path& __p);
+  path read_symlink(const path& __p, error_code& __ec);
+
+  inline path relative(const path& __p, error_code& __ec)
+  { return relative(__p, current_path(), __ec); }
+
+  path relative(const path& __p, const path& __base = current_path());
+  path relative(const path& __p, const path& __base, error_code& __ec);
+
+  bool remove(const path& __p);
+  bool remove(const path& __p, error_code& __ec) noexcept;
+
+  uintmax_t remove_all(const path& __p);
+  uintmax_t remove_all(const path& __p, error_code& __ec);
+
+  void rename(const path& __from, const path& __to);
+  void rename(const path& __from, const path& __to, error_code& __ec) noexcept;
+
+  void resize_file(const path& __p, uintmax_t __size);
+  void resize_file(const path& __p, uintmax_t __size, error_code& __ec) noexcept;
+
+  space_info space(const path& __p);
+  space_info space(const path& __p, error_code& __ec) noexcept;
+
+  file_status status(const path& __p);
+  file_status status(const path& __p, error_code& __ec) noexcept;
+
+  inline bool status_known(file_status __s) noexcept
+  { return __s.type() != file_type::none; }
+
+  file_status symlink_status(const path& __p);
+  file_status symlink_status(const path& __p, error_code& __ec) noexcept;
+
+  path temp_directory_path();
+  path temp_directory_path(error_code& __ec);
+
+  path weakly_canonical(const path& __p);
+  path weakly_canonical(const path& __p, error_code& __ec);
+
+  // @} group filesystem
+} // namespace filesystem
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
+
+#endif // C++17
+
+#endif // _GLIBCXX_FS_OPS_H
diff --git a/libstdc++-v3/include/bits/fs_path.h b/libstdc++-v3/include/bits/fs_path.h
new file mode 100644
index 00000000000..7d97cdfbb81
--- /dev/null
+++ b/libstdc++-v3/include/bits/fs_path.h
@@ -0,0 +1,1166 @@
+// Class filesystem::path -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+/** @file include/bits/fs_path.h
+ *  This is an internal header file, included by other library headers.
+ *  Do not attempt to use it directly. @headername{filesystem}
+ */
+
+#ifndef _GLIBCXX_FS_PATH_H
+#define _GLIBCXX_FS_PATH_H 1
+
+#if __cplusplus >= 201703L
+
+#include <utility>
+#include <type_traits>
+#include <vector>
+#include <locale>
+#include <iosfwd>
+#include <codecvt>
+#include <string_view>
+#include <system_error>
+#include <bits/stl_algobase.h>
+#include <bits/quoted_string.h>
+#include <bits/locale_conv.h>
+
+#if defined(_WIN32) && !defined(__CYGWIN__)
+# define _GLIBCXX_FILESYSTEM_IS_WINDOWS 1
+# include <algorithm>
+#endif
+
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+
+namespace filesystem
+{
+_GLIBCXX_BEGIN_NAMESPACE_CXX11
+
+  /**
+   * @ingroup filesystem
+   * @{
+   */
+
+  /// A filesystem path.
+  class path
+  {
+    template<typename _CharT>
+      struct __is_encoded_char : std::false_type { };
+
+    template<typename _Iter,
+	     typename _Iter_traits = std::iterator_traits<_Iter>>
+      using __is_path_iter_src
+	= __and_<__is_encoded_char<typename _Iter_traits::value_type>,
+		 std::is_base_of<std::input_iterator_tag,
+				 typename _Iter_traits::iterator_category>>;
+
+    template<typename _Iter>
+      static __is_path_iter_src<_Iter>
+      __is_path_src(_Iter, int);
+
+    template<typename _CharT, typename _Traits, typename _Alloc>
+      static __is_encoded_char<_CharT>
+      __is_path_src(const basic_string<_CharT, _Traits, _Alloc>&, int);
+
+    template<typename _CharT, typename _Traits>
+      static __is_encoded_char<_CharT>
+      __is_path_src(const basic_string_view<_CharT, _Traits>&, int);
+
+    template<typename _Unknown>
+      static std::false_type
+      __is_path_src(const _Unknown&, ...);
+
+    template<typename _Tp1, typename _Tp2>
+      struct __constructible_from;
+
+    template<typename _Iter>
+      struct __constructible_from<_Iter, _Iter>
+      : __is_path_iter_src<_Iter>
+      { };
+
+    template<typename _Source>
+      struct __constructible_from<_Source, void>
+      : decltype(__is_path_src(std::declval<_Source>(), 0))
+      { };
+
+    template<typename _Tp1, typename _Tp2 = void>
+      using _Path = typename
+	std::enable_if<__and_<__not_<is_same<_Tp1, path>>,
+			      __constructible_from<_Tp1, _Tp2>>::value,
+		       path>::type;
+
+    template<typename _Source>
+      static _Source
+      _S_range_begin(_Source __begin) { return __begin; }
+
+    struct __null_terminated { };
+
+    template<typename _Source>
+      static __null_terminated
+      _S_range_end(_Source) { return {}; }
+
+    template<typename _CharT, typename _Traits, typename _Alloc>
+      static const _CharT*
+      _S_range_begin(const basic_string<_CharT, _Traits, _Alloc>& __str)
+      { return __str.data(); }
+
+    template<typename _CharT, typename _Traits, typename _Alloc>
+      static const _CharT*
+      _S_range_end(const basic_string<_CharT, _Traits, _Alloc>& __str)
+      { return __str.data() + __str.size(); }
+
+    template<typename _CharT, typename _Traits>
+      static const _CharT*
+      _S_range_begin(const basic_string_view<_CharT, _Traits>& __str)
+      { return __str.data(); }
+
+    template<typename _CharT, typename _Traits>
+      static const _CharT*
+      _S_range_end(const basic_string_view<_CharT, _Traits>& __str)
+      { return __str.data() + __str.size(); }
+
+    template<typename _Tp,
+	     typename _Iter = decltype(_S_range_begin(std::declval<_Tp>())),
+	     typename _Val = typename std::iterator_traits<_Iter>::value_type>
+      using __value_type_is_char
+	= typename std::enable_if<std::is_same<_Val, char>::value>::type;
+
+  public:
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+    typedef wchar_t				value_type;
+    static constexpr value_type			preferred_separator = L'\\';
+#else
+    typedef char				value_type;
+    static constexpr value_type			preferred_separator = '/';
+#endif
+    typedef std::basic_string<value_type>	string_type;
+
+    enum format { native_format, generic_format, auto_format };
+
+    // constructors and destructor
+
+    path() noexcept { }
+
+    path(const path& __p) = default;
+
+    path(path&& __p) noexcept
+    : _M_pathname(std::move(__p._M_pathname)), _M_type(__p._M_type)
+    {
+      _M_split_cmpts();
+      __p.clear();
+    }
+
+    path(string_type&& __source, format = auto_format)
+    : _M_pathname(std::move(__source))
+    { _M_split_cmpts(); }
+
+    template<typename _Source,
+	     typename _Require = _Path<_Source>>
+      path(_Source const& __source, format = auto_format)
+      : _M_pathname(_S_convert(_S_range_begin(__source),
+			       _S_range_end(__source)))
+      { _M_split_cmpts(); }
+
+    template<typename _InputIterator,
+	     typename _Require = _Path<_InputIterator, _InputIterator>>
+      path(_InputIterator __first, _InputIterator __last, format = auto_format)
+      : _M_pathname(_S_convert(__first, __last))
+      { _M_split_cmpts(); }
+
+    template<typename _Source,
+	     typename _Require = _Path<_Source>,
+	     typename _Require2 = __value_type_is_char<_Source>>
+      path(_Source const& __source, const locale& __loc, format = auto_format)
+      : _M_pathname(_S_convert_loc(_S_range_begin(__source),
+				   _S_range_end(__source), __loc))
+      { _M_split_cmpts(); }
+
+    template<typename _InputIterator,
+	     typename _Require = _Path<_InputIterator, _InputIterator>,
+	     typename _Require2 = __value_type_is_char<_InputIterator>>
+      path(_InputIterator __first, _InputIterator __last, const locale& __loc,
+	   format = auto_format)
+      : _M_pathname(_S_convert_loc(__first, __last, __loc))
+      { _M_split_cmpts(); }
+
+    ~path() = default;
+
+    // assignments
+
+    path& operator=(const path& __p) = default;
+    path& operator=(path&& __p) noexcept;
+    path& operator=(string_type&& __source);
+    path& assign(string_type&& __source);
+
+    template<typename _Source>
+      _Path<_Source>&
+      operator=(_Source const& __source)
+      { return *this = path(__source); }
+
+    template<typename _Source>
+      _Path<_Source>&
+      assign(_Source const& __source)
+      { return *this = path(__source); }
+
+    template<typename _InputIterator>
+      _Path<_InputIterator, _InputIterator>&
+      assign(_InputIterator __first, _InputIterator __last)
+      { return *this = path(__first, __last); }
+
+    // appends
+
+    path& operator/=(const path& __p)
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      if (__p.is_absolute()
+	  || (__p.has_root_name() && __p.root_name() != root_name()))
+	operator=(__p);
+      else
+	{
+	  string_type __pathname;
+	  if (__p.has_root_directory())
+	    __pathname = root_name().native();
+	  else if (has_filename() || (!has_root_directory() && is_absolute()))
+	    __pathname = _M_pathname + preferred_separator;
+	  __pathname += __p.relative_path().native(); // XXX is this right?
+	  _M_pathname.swap(__pathname);
+	  _M_split_cmpts();
+	}
+#else
+      // Much simpler, as any path with root-name or root-dir is absolute.
+      if (__p.is_absolute())
+	operator=(__p);
+      else
+	{
+	  if (has_filename() || (_M_type == _Type::_Root_name))
+	    _M_pathname += preferred_separator;
+	  _M_pathname += __p.native();
+	  _M_split_cmpts();
+	}
+#endif
+      return *this;
+    }
+
+    template <class _Source>
+      _Path<_Source>&
+      operator/=(_Source const& __source)
+      { return append(__source); }
+
+    template<typename _Source>
+      _Path<_Source>&
+      append(_Source const& __source)
+      {
+	return _M_append(_S_convert(_S_range_begin(__source),
+				    _S_range_end(__source)));
+      }
+
+    template<typename _InputIterator>
+      _Path<_InputIterator, _InputIterator>&
+      append(_InputIterator __first, _InputIterator __last)
+      { return _M_append(_S_convert(__first, __last)); }
+
+    // concatenation
+
+    path& operator+=(const path& __x);
+    path& operator+=(const string_type& __x);
+    path& operator+=(const value_type* __x);
+    path& operator+=(value_type __x);
+    path& operator+=(basic_string_view<value_type> __x);
+
+    template<typename _Source>
+      _Path<_Source>&
+      operator+=(_Source const& __x) { return concat(__x); }
+
+    template<typename _CharT>
+      _Path<_CharT*, _CharT*>&
+      operator+=(_CharT __x);
+
+    template<typename _Source>
+      _Path<_Source>&
+      concat(_Source const& __x)
+      { return *this += _S_convert(_S_range_begin(__x), _S_range_end(__x)); }
+
+    template<typename _InputIterator>
+      _Path<_InputIterator, _InputIterator>&
+      concat(_InputIterator __first, _InputIterator __last)
+      { return *this += _S_convert(__first, __last); }
+
+    // modifiers
+
+    void clear() noexcept { _M_pathname.clear(); _M_split_cmpts(); }
+
+    path& make_preferred();
+    path& remove_filename();
+    path& replace_filename(const path& __replacement);
+    path& replace_extension(const path& __replacement = path());
+
+    void swap(path& __rhs) noexcept;
+
+    // native format observers
+
+    const string_type&  native() const noexcept { return _M_pathname; }
+    const value_type*   c_str() const noexcept { return _M_pathname.c_str(); }
+    operator string_type() const { return _M_pathname; }
+
+    template<typename _CharT, typename _Traits = std::char_traits<_CharT>,
+	     typename _Allocator = std::allocator<_CharT>>
+      std::basic_string<_CharT, _Traits, _Allocator>
+      string(const _Allocator& __a = _Allocator()) const;
+
+    std::string    string() const;
+#if _GLIBCXX_USE_WCHAR_T
+    std::wstring   wstring() const;
+#endif
+    std::string    u8string() const;
+    std::u16string u16string() const;
+    std::u32string u32string() const;
+
+    // generic format observers
+    template<typename _CharT, typename _Traits = std::char_traits<_CharT>,
+	     typename _Allocator = std::allocator<_CharT>>
+      std::basic_string<_CharT, _Traits, _Allocator>
+      generic_string(const _Allocator& __a = _Allocator()) const;
+
+    std::string    generic_string() const;
+#if _GLIBCXX_USE_WCHAR_T
+    std::wstring   generic_wstring() const;
+#endif
+    std::string    generic_u8string() const;
+    std::u16string generic_u16string() const;
+    std::u32string generic_u32string() const;
+
+    // compare
+
+    int compare(const path& __p) const noexcept;
+    int compare(const string_type& __s) const;
+    int compare(const value_type* __s) const;
+    int compare(const basic_string_view<value_type> __s) const;
+
+    // decomposition
+
+    path root_name() const;
+    path root_directory() const;
+    path root_path() const;
+    path relative_path() const;
+    path parent_path() const;
+    path filename() const;
+    path stem() const;
+    path extension() const;
+
+    // query
+
+    bool empty() const noexcept { return _M_pathname.empty(); }
+    bool has_root_name() const;
+    bool has_root_directory() const;
+    bool has_root_path() const;
+    bool has_relative_path() const;
+    bool has_parent_path() const;
+    bool has_filename() const;
+    bool has_stem() const;
+    bool has_extension() const;
+    bool is_absolute() const;
+    bool is_relative() const { return !is_absolute(); }
+
+    // generation
+    path lexically_normal() const;
+    path lexically_relative(const path& base) const;
+    path lexically_proximate(const path& base) const;
+
+    // iterators
+    class iterator;
+    typedef iterator const_iterator;
+
+    iterator begin() const;
+    iterator end() const;
+
+  private:
+    enum class _Type : unsigned char {
+	_Multi, _Root_name, _Root_dir, _Filename
+    };
+
+    path(string_type __str, _Type __type) : _M_pathname(__str), _M_type(__type)
+    {
+      __glibcxx_assert(_M_type != _Type::_Multi);
+    }
+
+    enum class _Split { _Stem, _Extension };
+
+    path& _M_append(string_type&& __str)
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      operator/=(path(std::move(__str)));
+#else
+      if (!_M_pathname.empty() && !_S_is_dir_sep(_M_pathname.back())
+	  && (__str.empty() || !_S_is_dir_sep(__str.front())))
+	_M_pathname += preferred_separator;
+      _M_pathname += __str;
+      _M_split_cmpts();
+#endif
+      return *this;
+    }
+
+    pair<const string_type*, size_t> _M_find_extension() const;
+
+    template<typename _CharT>
+      struct _Cvt;
+
+    static string_type
+    _S_convert(value_type* __src, __null_terminated)
+    { return string_type(__src); }
+
+    static string_type
+    _S_convert(const value_type* __src, __null_terminated)
+    { return string_type(__src); }
+
+    template<typename _Iter>
+      static string_type
+      _S_convert(_Iter __first, _Iter __last)
+      {
+	using __value_type = typename std::iterator_traits<_Iter>::value_type;
+	return _Cvt<typename remove_cv<__value_type>::type>::
+	  _S_convert(__first, __last);
+      }
+
+    template<typename _InputIterator>
+      static string_type
+      _S_convert(_InputIterator __src, __null_terminated)
+      {
+	using _Tp = typename std::iterator_traits<_InputIterator>::value_type;
+	std::basic_string<typename remove_cv<_Tp>::type> __tmp;
+	for (; *__src != _Tp{}; ++__src)
+	  __tmp.push_back(*__src);
+	return _S_convert(__tmp.c_str(), __tmp.c_str() + __tmp.size());
+      }
+
+    static string_type
+    _S_convert_loc(const char* __first, const char* __last,
+		   const std::locale& __loc);
+
+    template<typename _Iter>
+      static string_type
+      _S_convert_loc(_Iter __first, _Iter __last, const std::locale& __loc)
+      {
+	const std::string __str(__first, __last);
+	return _S_convert_loc(__str.data(), __str.data()+__str.size(), __loc);
+      }
+
+    template<typename _InputIterator>
+      static string_type
+      _S_convert_loc(_InputIterator __src, __null_terminated,
+		     const std::locale& __loc)
+      {
+	std::string __tmp;
+	while (*__src != '\0')
+	  __tmp.push_back(*__src++);
+	return _S_convert_loc(__tmp.data(), __tmp.data()+__tmp.size(), __loc);
+      }
+
+    template<typename _CharT, typename _Traits, typename _Allocator>
+      static basic_string<_CharT, _Traits, _Allocator>
+      _S_str_convert(const string_type&, const _Allocator& __a);
+
+    bool _S_is_dir_sep(value_type __ch)
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      return __ch == L'/' || __ch == preferred_separator;
+#else
+      return __ch == '/';
+#endif
+    }
+
+    void _M_split_cmpts();
+    void _M_trim();
+    void _M_add_root_name(size_t __n);
+    void _M_add_root_dir(size_t __pos);
+    void _M_add_filename(size_t __pos, size_t __n);
+
+    string_type _M_pathname;
+
+    struct _Cmpt;
+    using _List = _GLIBCXX_STD_C::vector<_Cmpt>;
+    _List _M_cmpts; // empty unless _M_type == _Type::_Multi
+    _Type _M_type = _Type::_Multi;
+  };
+
+  template<>
+    struct path::__is_encoded_char<char> : std::true_type
+    { using value_type = char; };
+
+  template<>
+    struct path::__is_encoded_char<wchar_t> : std::true_type
+    { using value_type = wchar_t; };
+
+  template<>
+    struct path::__is_encoded_char<char16_t> : std::true_type
+    { using value_type = char16_t; };
+
+  template<>
+    struct path::__is_encoded_char<char32_t> : std::true_type
+    { using value_type = char32_t; };
+
+  template<typename _Tp>
+    struct path::__is_encoded_char<const _Tp> : __is_encoded_char<_Tp> { };
+
+  inline void swap(path& __lhs, path& __rhs) noexcept { __lhs.swap(__rhs); }
+
+  size_t hash_value(const path& __p) noexcept;
+
+  /// Compare paths
+  inline bool operator<(const path& __lhs, const path& __rhs) noexcept
+  { return __lhs.compare(__rhs) < 0; }
+
+  /// Compare paths
+  inline bool operator<=(const path& __lhs, const path& __rhs) noexcept
+  { return !(__rhs < __lhs); }
+
+  /// Compare paths
+  inline bool operator>(const path& __lhs, const path& __rhs) noexcept
+  { return __rhs < __lhs; }
+
+  /// Compare paths
+  inline bool operator>=(const path& __lhs, const path& __rhs) noexcept
+  { return !(__lhs < __rhs); }
+
+  /// Compare paths
+  inline bool operator==(const path& __lhs, const path& __rhs) noexcept
+  { return __lhs.compare(__rhs) == 0; }
+
+  /// Compare paths
+  inline bool operator!=(const path& __lhs, const path& __rhs) noexcept
+  { return !(__lhs == __rhs); }
+
+  /// Append one path to another
+  inline path operator/(const path& __lhs, const path& __rhs)
+  { return path(__lhs) /= __rhs; }
+
+  /// Write a path to a stream
+  template<typename _CharT, typename _Traits>
+    basic_ostream<_CharT, _Traits>&
+    operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p)
+    {
+      auto __tmp = __p.string<_CharT, _Traits>();
+      using __quoted_string
+	= std::__detail::_Quoted_string<decltype(__tmp)&, _CharT>;
+      __os << __quoted_string{__tmp, '"', '\\'};
+      return __os;
+    }
+
+  /// Read a path from a stream
+  template<typename _CharT, typename _Traits>
+    basic_istream<_CharT, _Traits>&
+    operator>>(basic_istream<_CharT, _Traits>& __is, path& __p)
+    {
+      basic_string<_CharT, _Traits> __tmp;
+      using __quoted_string
+	= std::__detail::_Quoted_string<decltype(__tmp)&, _CharT>;
+      if (__is >> __quoted_string{ __tmp, '"', '\\' })
+	__p = std::move(__tmp);
+      return __is;
+    }
+
+  template<typename _Source>
+    inline auto
+    u8path(const _Source& __source)
+    -> decltype(filesystem::path(__source, std::locale::classic()))
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      const std::string __u8str{__source};
+      return std::filesystem::u8path(__u8str.begin(), __u8str.end());
+#else
+      return path{ __source };
+#endif
+    }
+
+  template<typename _InputIterator>
+    inline auto
+    u8path(_InputIterator __first, _InputIterator __last)
+    -> decltype(filesystem::path(__first, __last, std::locale::classic()))
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      codecvt_utf8<value_type> __cvt;
+      string_type __tmp;
+      if (__str_codecvt_in(__first, __last, __tmp, __cvt))
+	return path{ __tmp };
+      else
+	return {};
+#else
+      return path{ __first, __last };
+#endif
+    }
+
+  class filesystem_error : public std::system_error
+  {
+  public:
+    filesystem_error(const string& __what_arg, error_code __ec)
+    : system_error(__ec, __what_arg) { }
+
+    filesystem_error(const string& __what_arg, const path& __p1,
+		     error_code __ec)
+    : system_error(__ec, __what_arg), _M_path1(__p1) { }
+
+    filesystem_error(const string& __what_arg, const path& __p1,
+		     const path& __p2, error_code __ec)
+    : system_error(__ec, __what_arg), _M_path1(__p1), _M_path2(__p2)
+    { }
+
+    ~filesystem_error();
+
+    const path& path1() const noexcept { return _M_path1; }
+    const path& path2() const noexcept { return _M_path2; }
+    const char* what() const noexcept { return _M_what.c_str(); }
+
+  private:
+    std::string _M_gen_what();
+
+    path _M_path1;
+    path _M_path2;
+    std::string _M_what = _M_gen_what();
+  };
+
+  struct path::_Cmpt : path
+  {
+    _Cmpt(string_type __s, _Type __t, size_t __pos)
+      : path(std::move(__s), __t), _M_pos(__pos) { }
+
+    _Cmpt() : _M_pos(-1) { }
+
+    size_t _M_pos;
+  };
+
+  // specialize _Cvt for degenerate 'noconv' case
+  template<>
+    struct path::_Cvt<path::value_type>
+    {
+      template<typename _Iter>
+	static string_type
+	_S_convert(_Iter __first, _Iter __last)
+	{ return string_type{__first, __last}; }
+    };
+
+  template<typename _CharT>
+    struct path::_Cvt
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      static string_type
+      _S_wconvert(const char* __f, const char* __l, true_type)
+      {
+	using _Cvt = std::codecvt<wchar_t, char, mbstate_t>;
+	const auto& __cvt = std::use_facet<_Cvt>(std::locale{});
+	std::wstring __wstr;
+	if (__str_codecvt_in(__f, __l, __wstr, __cvt))
+	    return __wstr;
+	_GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	      "Cannot convert character sequence",
+	      std::make_error_code(errc::illegal_byte_sequence)));
+      }
+
+      static string_type
+      _S_wconvert(const _CharT* __f, const _CharT* __l, false_type)
+      {
+	std::codecvt_utf8<_CharT> __cvt;
+	std::string __str;
+	if (__str_codecvt_out(__f, __l, __str, __cvt))
+	  {
+	    const char* __f2 = __str.data();
+	    const char* __l2 = __f2 + __str.size();
+	    std::codecvt_utf8<wchar_t> __wcvt;
+	    std::wstring __wstr;
+	    if (__str_codecvt_in(__f2, __l2, __wstr, __wcvt))
+	      return __wstr;
+	  }
+	_GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	      "Cannot convert character sequence",
+	      std::make_error_code(errc::illegal_byte_sequence)));
+      }
+
+      static string_type
+      _S_convert(const _CharT* __f, const _CharT* __l)
+      {
+	return _S_wconvert(__f, __l, is_same<_CharT, char>{});
+      }
+#else
+      static string_type
+      _S_convert(const _CharT* __f, const _CharT* __l)
+      {
+	std::codecvt_utf8<_CharT> __cvt;
+	std::string __str;
+	if (__str_codecvt_out(__f, __l, __str, __cvt))
+	  return __str;
+	_GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	      "Cannot convert character sequence",
+	      std::make_error_code(errc::illegal_byte_sequence)));
+      }
+#endif
+
+      static string_type
+      _S_convert(_CharT* __f, _CharT* __l)
+      {
+	return _S_convert(const_cast<const _CharT*>(__f),
+			  const_cast<const _CharT*>(__l));
+      }
+
+      template<typename _Iter>
+	static string_type
+	_S_convert(_Iter __first, _Iter __last)
+	{
+	  const std::basic_string<_CharT> __str(__first, __last);
+	  return _S_convert(__str.data(), __str.data() + __str.size());
+	}
+
+      template<typename _Iter, typename _Cont>
+	static string_type
+	_S_convert(__gnu_cxx::__normal_iterator<_Iter, _Cont> __first,
+		  __gnu_cxx::__normal_iterator<_Iter, _Cont> __last)
+	{ return _S_convert(__first.base(), __last.base()); }
+    };
+
+  /// An iterator for the components of a path
+  class path::iterator
+  {
+  public:
+    using difference_type	= std::ptrdiff_t;
+    using value_type		= path;
+    using reference		= const path&;
+    using pointer		= const path*;
+    using iterator_category	= std::bidirectional_iterator_tag;
+
+    iterator() : _M_path(nullptr), _M_cur(), _M_at_end() { }
+
+    iterator(const iterator&) = default;
+    iterator& operator=(const iterator&) = default;
+
+    reference operator*() const;
+    pointer   operator->() const { return std::__addressof(**this); }
+
+    iterator& operator++();
+    iterator  operator++(int) { auto __tmp = *this; ++*this; return __tmp; }
+
+    iterator& operator--();
+    iterator  operator--(int) { auto __tmp = *this; --*this; return __tmp; }
+
+    friend bool operator==(const iterator& __lhs, const iterator& __rhs)
+    { return __lhs._M_equals(__rhs); }
+
+    friend bool operator!=(const iterator& __lhs, const iterator& __rhs)
+    { return !__lhs._M_equals(__rhs); }
+
+  private:
+    friend class path;
+
+    iterator(const path* __path, path::_List::const_iterator __iter)
+    : _M_path(__path), _M_cur(__iter), _M_at_end()
+    { }
+
+    iterator(const path* __path, bool __at_end)
+    : _M_path(__path), _M_cur(), _M_at_end(__at_end)
+    { }
+
+    bool _M_equals(iterator) const;
+
+    const path* 		_M_path;
+    path::_List::const_iterator _M_cur;
+    bool			_M_at_end;  // only used when type != _Multi
+  };
+
+
+  inline path&
+  path::operator=(path&& __p) noexcept
+  {
+    _M_pathname = std::move(__p._M_pathname);
+    _M_cmpts = std::move(__p._M_cmpts);
+    _M_type = __p._M_type;
+    __p.clear();
+    return *this;
+  }
+
+  inline path&
+  path::operator=(string_type&& __source)
+  { return *this = path(std::move(__source)); }
+
+  inline path&
+  path::assign(string_type&& __source)
+  { return *this = path(std::move(__source)); }
+
+  inline path&
+  path::operator+=(const path& __p)
+  {
+    return operator+=(__p.native());
+  }
+
+  inline path&
+  path::operator+=(const string_type& __x)
+  {
+    _M_pathname += __x;
+    _M_split_cmpts();
+    return *this;
+  }
+
+  inline path&
+  path::operator+=(const value_type* __x)
+  {
+    _M_pathname += __x;
+    _M_split_cmpts();
+    return *this;
+  }
+
+  inline path&
+  path::operator+=(value_type __x)
+  {
+    _M_pathname += __x;
+    _M_split_cmpts();
+    return *this;
+  }
+
+  inline path&
+  path::operator+=(basic_string_view<value_type> __x)
+  {
+    _M_pathname.append(__x.data(), __x.size());
+    _M_split_cmpts();
+    return *this;
+  }
+
+  template<typename _CharT>
+    inline path::_Path<_CharT*, _CharT*>&
+    path::operator+=(_CharT __x)
+    {
+      auto* __addr = std::__addressof(__x);
+      return concat(__addr, __addr + 1);
+    }
+
+  inline path&
+  path::make_preferred()
+  {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+    std::replace(_M_pathname.begin(), _M_pathname.end(), L'/',
+		 preferred_separator);
+#endif
+    return *this;
+  }
+
+  inline void path::swap(path& __rhs) noexcept
+  {
+    _M_pathname.swap(__rhs._M_pathname);
+    _M_cmpts.swap(__rhs._M_cmpts);
+    std::swap(_M_type, __rhs._M_type);
+  }
+
+  template<typename _CharT, typename _Traits, typename _Allocator>
+    std::basic_string<_CharT, _Traits, _Allocator>
+    path::_S_str_convert(const string_type& __str, const _Allocator& __a)
+    {
+      if (__str.size() == 0)
+	return std::basic_string<_CharT, _Traits, _Allocator>(__a);
+
+      const value_type* __first = __str.data();
+      const value_type* __last = __first + __str.size();
+
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      using _CharAlloc = __alloc_rebind<_Allocator, char>;
+      using _String = basic_string<char, char_traits<char>, _CharAlloc>;
+      using _WString = basic_string<_CharT, _Traits, _Allocator>;
+
+      // use codecvt_utf8<wchar_t> to convert native string to UTF-8
+      codecvt_utf8<value_type> __cvt;
+      _String __u8str{_CharAlloc{__a}};
+      if (__str_codecvt_out(__first, __last, __u8str, __cvt))
+	{
+	  if constexpr (is_same_v<_CharT, char>)
+	    return __u8str;
+	  else
+	    {
+	      _WString __wstr;
+	      // use codecvt_utf8<_CharT> to convert UTF-8 to wide string
+	      codecvt_utf8<_CharT> __cvt;
+	      const char* __f = __u8str.data();
+	      const char* __l = __f + __u8str.size();
+	      if (__str_codecvt_in(__f, __l, __wstr, __cvt))
+		return __wstr;
+	    }
+	}
+#else
+      codecvt_utf8<_CharT> __cvt;
+      basic_string<_CharT, _Traits, _Allocator> __wstr{__a};
+      if (__str_codecvt_in(__first, __last, __wstr, __cvt))
+	return __wstr;
+#endif
+      _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	    "Cannot convert character sequence",
+	    std::make_error_code(errc::illegal_byte_sequence)));
+    }
+
+  template<typename _CharT, typename _Traits, typename _Allocator>
+    inline basic_string<_CharT, _Traits, _Allocator>
+    path::string(const _Allocator& __a) const
+    {
+      if constexpr (is_same_v<_CharT, value_type>)
+#if _GLIBCXX_USE_CXX11_ABI
+	return { _M_pathname, __a };
+#else
+	return { _M_pathname, string_type::size_type(0), __a };
+#endif
+      else
+	return _S_str_convert<_CharT, _Traits>(_M_pathname, __a);
+    }
+
+  inline std::string
+  path::string() const { return string<char>(); }
+
+#if _GLIBCXX_USE_WCHAR_T
+  inline std::wstring
+  path::wstring() const { return string<wchar_t>(); }
+#endif
+
+  inline std::string
+  path::u8string() const
+  {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+    std::string __str;
+    // convert from native encoding to UTF-8
+    codecvt_utf8<value_type> __cvt;
+    const value_type* __first = _M_pathname.data();
+    const value_type* __last = __first + _M_pathname.size();
+    if (__str_codecvt_out(__first, __last, __str, __cvt))
+      return __str;
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	  "Cannot convert character sequence",
+	  std::make_error_code(errc::illegal_byte_sequence)));
+#else
+    return _M_pathname;
+#endif
+  }
+
+  inline std::u16string
+  path::u16string() const { return string<char16_t>(); }
+
+  inline std::u32string
+  path::u32string() const { return string<char32_t>(); }
+
+  template<typename _CharT, typename _Traits, typename _Allocator>
+    inline std::basic_string<_CharT, _Traits, _Allocator>
+    path::generic_string(const _Allocator& __a) const
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      const value_type __slash = L'/';
+#else
+      const value_type __slash = '/';
+#endif
+      string_type __str(__a);
+
+      if (_M_type == _Type::_Root_dir)
+	__str.assign(1, __slash);
+      else
+	{
+	  __str.reserve(_M_pathname.size());
+	  bool __add_slash = false;
+	  for (auto& __elem : *this)
+	    {
+	      if (__add_slash)
+		__str += __slash;
+	      __str += __elem._M_pathname;
+	      __add_slash = __elem._M_type == _Type::_Filename;
+	    }
+	}
+
+      if constexpr (is_same_v<_CharT, value_type>)
+	return __str;
+      else
+	return _S_str_convert<_CharT, _Traits>(__str, __a);
+    }
+
+  inline std::string
+  path::generic_string() const
+  { return generic_string<char>(); }
+
+#if _GLIBCXX_USE_WCHAR_T
+  inline std::wstring
+  path::generic_wstring() const
+  { return generic_string<wchar_t>(); }
+#endif
+
+  inline std::string
+  path::generic_u8string() const
+  { return generic_string(); }
+
+  inline std::u16string
+  path::generic_u16string() const
+  { return generic_string<char16_t>(); }
+
+  inline std::u32string
+  path::generic_u32string() const
+  { return generic_string<char32_t>(); }
+
+  inline int
+  path::compare(const string_type& __s) const { return compare(path(__s)); }
+
+  inline int
+  path::compare(const value_type* __s) const { return compare(path(__s)); }
+
+  inline int
+  path::compare(basic_string_view<value_type> __s) const
+  { return compare(path(__s)); }
+
+  inline path
+  path::filename() const
+  {
+    if (empty())
+      return {};
+    else if (_M_type == _Type::_Filename)
+      return *this;
+    else if (_M_type == _Type::_Multi)
+      {
+	if (_M_pathname.back() == preferred_separator)
+	  return {};
+	auto& __last = *--end();
+	if (__last._M_type == _Type::_Filename)
+	  return __last;
+      }
+    return {};
+  }
+
+  inline path
+  path::stem() const
+  {
+    auto ext = _M_find_extension();
+    if (ext.first && ext.second != 0)
+      return path{ext.first->substr(0, ext.second)};
+    return {};
+  }
+
+  inline path
+  path::extension() const
+  {
+    auto ext = _M_find_extension();
+    if (ext.first && ext.second != string_type::npos)
+      return path{ext.first->substr(ext.second)};
+    return {};
+  }
+
+  inline bool
+  path::has_stem() const
+  {
+    auto ext = _M_find_extension();
+    return ext.first && ext.second != 0;
+  }
+
+  inline bool
+  path::has_extension() const
+  {
+    auto ext = _M_find_extension();
+    return ext.first && ext.second != string_type::npos;
+  }
+
+  inline bool
+  path::is_absolute() const
+  {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+    return has_root_name();
+#else
+    return has_root_directory();
+#endif
+  }
+
+  inline path::iterator
+  path::begin() const
+  {
+    if (_M_type == _Type::_Multi)
+      return iterator(this, _M_cmpts.begin());
+    return iterator(this, false);
+  }
+
+  inline path::iterator
+  path::end() const
+  {
+    if (_M_type == _Type::_Multi)
+      return iterator(this, _M_cmpts.end());
+    return iterator(this, true);
+  }
+
+  inline path::iterator&
+  path::iterator::operator++()
+  {
+    __glibcxx_assert(_M_path != nullptr);
+    if (_M_path->_M_type == _Type::_Multi)
+      {
+	__glibcxx_assert(_M_cur != _M_path->_M_cmpts.end());
+	++_M_cur;
+      }
+    else
+      {
+	__glibcxx_assert(!_M_at_end);
+	_M_at_end = true;
+      }
+    return *this;
+  }
+
+  inline path::iterator&
+  path::iterator::operator--()
+  {
+    __glibcxx_assert(_M_path != nullptr);
+    if (_M_path->_M_type == _Type::_Multi)
+      {
+	__glibcxx_assert(_M_cur != _M_path->_M_cmpts.begin());
+	--_M_cur;
+      }
+    else
+      {
+	__glibcxx_assert(_M_at_end);
+	_M_at_end = false;
+      }
+    return *this;
+  }
+
+  inline path::iterator::reference
+  path::iterator::operator*() const
+  {
+    __glibcxx_assert(_M_path != nullptr);
+    if (_M_path->_M_type == _Type::_Multi)
+      {
+	__glibcxx_assert(_M_cur != _M_path->_M_cmpts.end());
+	return *_M_cur;
+      }
+    return *_M_path;
+  }
+
+  inline bool
+  path::iterator::_M_equals(iterator __rhs) const
+  {
+    if (_M_path != __rhs._M_path)
+      return false;
+    if (_M_path == nullptr)
+      return true;
+    if (_M_path->_M_type == path::_Type::_Multi)
+      return _M_cur == __rhs._M_cur;
+    return _M_at_end == __rhs._M_at_end;
+  }
+
+  // @} group filesystem
+_GLIBCXX_END_NAMESPACE_CXX11
+} // namespace filesystem
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
+
+#endif // C++17
+
+#endif // _GLIBCXX_FS_PATH_H
diff --git a/libstdc++-v3/include/bits/hashtable_policy.h b/libstdc++-v3/include/bits/hashtable_policy.h
index da6d49c96c4..e19f92abd76 100644
--- a/libstdc++-v3/include/bits/hashtable_policy.h
+++ b/libstdc++-v3/include/bits/hashtable_policy.h
@@ -31,7 +31,9 @@
 #ifndef _HASHTABLE_POLICY_H
 #define _HASHTABLE_POLICY_H 1
 
-#include <bits/stl_algobase.h> // for std::min.
+#include <tuple>		// for std::tuple, std::forward_as_tuple
+#include <cstdint>		// for std::uint_fast64_t
+#include <bits/stl_algobase.h>	// for std::min.
 
 namespace std _GLIBCXX_VISIBILITY(default)
 {
diff --git a/libstdc++-v3/include/bits/node_handle.h b/libstdc++-v3/include/bits/node_handle.h
index c7694a1e0ef..4a830630c89 100644
--- a/libstdc++-v3/include/bits/node_handle.h
+++ b/libstdc++-v3/include/bits/node_handle.h
@@ -37,7 +37,6 @@
 # define __cpp_lib_node_extract 201606
 
 #include <optional>
-#include <tuple>
 #include <bits/alloc_traits.h>
 #include <bits/ptr_traits.h>
 
@@ -283,26 +282,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       _Iterator		position = _Iterator();
       bool		inserted = false;
       _NodeHandle	node;
-
-      template<size_t _Idx>
-	decltype(auto) get() &
-	{ return std::get<_Idx>(std::tie(inserted, position, node)); }
-
-      template<size_t _Idx>
-	decltype(auto) get() const &
-	{ return std::get<_Idx>(std::tie(inserted, position, node)); }
-
-      template<size_t _Idx>
-	decltype(auto) get() &&
-	{
-	  return std::move(std::get<_Idx>(std::tie(inserted, position, node)));
-	}
-
-      template<size_t _Idx>
-	decltype(auto) get() const &&
-	{
-	  return std::move(std::get<_Idx>(std::tie(inserted, position, node)));
-	}
     };
 
 _GLIBCXX_END_NAMESPACE_VERSION
diff --git a/libstdc++-v3/include/bits/specfun.h b/libstdc++-v3/include/bits/specfun.h
index 0aaebeab7ac..6bb3ec09681 100644
--- a/libstdc++-v3/include/bits/specfun.h
+++ b/libstdc++-v3/include/bits/specfun.h
@@ -1201,6 +1201,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
+#ifndef __STRICT_ANSI__
 namespace __gnu_cxx _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
@@ -1305,6 +1306,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace __gnu_cxx
+#endif // __STRICT_ANSI__
 
 #pragma GCC visibility pop
 
diff --git a/libstdc++-v3/include/bits/stl_algo.h b/libstdc++-v3/include/bits/stl_algo.h
index ea413b1b155..b5facef55dd 100644
--- a/libstdc++-v3/include/bits/stl_algo.h
+++ b/libstdc++-v3/include/bits/stl_algo.h
@@ -180,7 +180,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     _InputIterator
     __find_if_not_n(_InputIterator __first, _Distance& __len, _Predicate __pred)
     {
-      for (; __len; --__len, ++__first)
+      for (; __len; --__len,  (void) ++__first)
 	if (!__pred(__first))
 	  break;
       return __first;
@@ -4462,7 +4462,7 @@ _GLIBCXX_BEGIN_NAMESPACE_ALGO
 	    __typeof__(__gen())>)
 
       for (__decltype(__n + 0) __niter = __n;
-	   __niter > 0; --__niter, ++__first)
+	   __niter > 0; --__niter, (void) ++__first)
 	*__first = __gen();
       return __first;
     }
diff --git a/libstdc++-v3/include/bits/stl_algobase.h b/libstdc++-v3/include/bits/stl_algobase.h
index a80934c4faa..bfdfc7ded5e 100644
--- a/libstdc++-v3/include/bits/stl_algobase.h
+++ b/libstdc++-v3/include/bits/stl_algobase.h
@@ -738,7 +738,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     __fill_n_a(_OutputIterator __first, _Size __n, const _Tp& __value)
     {
       for (__decltype(__n + 0) __niter = __n;
-	   __niter > 0; --__niter, ++__first)
+	   __niter > 0; --__niter, (void) ++__first)
 	*__first = __value;
       return __first;
     }
@@ -750,7 +750,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     {
       const _Tp __tmp = __value;
       for (__decltype(__n + 0) __niter = __n;
-	   __niter > 0; --__niter, ++__first)
+	   __niter > 0; --__niter, (void) ++__first)
 	*__first = __tmp;
       return __first;
     }
@@ -796,7 +796,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	static bool
 	equal(_II1 __first1, _II1 __last1, _II2 __first2)
 	{
-	  for (; __first1 != __last1; ++__first1, (void)++__first2)
+	  for (; __first1 != __last1; ++__first1, (void) ++__first2)
 	    if (!(*__first1 == *__first2))
 	      return false;
 	  return true;
diff --git a/libstdc++-v3/include/bits/stl_bvector.h b/libstdc++-v3/include/bits/stl_bvector.h
index d24e760d01b..ac548846b0e 100644
--- a/libstdc++-v3/include/bits/stl_bvector.h
+++ b/libstdc++-v3/include/bits/stl_bvector.h
@@ -417,7 +417,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	if (__last._M_offset != 0)
 	  __fill_bvector(__last._M_p, 0, __last._M_offset, __x);
       }
-    else
+    else if (__first._M_offset != __last._M_offset)
       __fill_bvector(__first._M_p, __first._M_offset, __last._M_offset, __x);
   }
 
diff --git a/libstdc++-v3/include/bits/stl_iterator.h b/libstdc++-v3/include/bits/stl_iterator.h
index 88667a568c3..74ef0d9bf88 100644
--- a/libstdc++-v3/include/bits/stl_iterator.h
+++ b/libstdc++-v3/include/bits/stl_iterator.h
@@ -1244,6 +1244,27 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #define _GLIBCXX_MAKE_MOVE_IF_NOEXCEPT_ITERATOR(_Iter) (_Iter)
 #endif // C++11
 
+#if __cpp_deduction_guides >= 201606
+  // These helper traits are used for deduction guides
+  // of associative containers.
+  template<typename _InputIterator>
+    using __iter_key_t = remove_const_t<
+    typename iterator_traits<_InputIterator>::value_type::first_type>;
+
+  template<typename _InputIterator>
+    using __iter_val_t =
+    typename iterator_traits<_InputIterator>::value_type::second_type;
+
+  template<typename _T1, typename _T2>
+    struct pair;
+
+  template<typename _InputIterator>
+    using __iter_to_alloc_t =
+    pair<add_const_t<__iter_key_t<_InputIterator>>,
+	 __iter_val_t<_InputIterator>>;
+
+#endif
+
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace
 
diff --git a/libstdc++-v3/include/bits/stl_map.h b/libstdc++-v3/include/bits/stl_map.h
index 0e8a98a96c1..8a9e6c9c329 100644
--- a/libstdc++-v3/include/bits/stl_map.h
+++ b/libstdc++-v3/include/bits/stl_map.h
@@ -778,7 +778,6 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       /**
        *  @brief Attempts to insert a std::pair into the %map.
-
        *  @param __x Pair to be inserted (see std::make_pair for easy
        *	     creation of pairs).
        *
@@ -791,12 +790,19 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  first element (the key) is not already present in the %map.
        *
        *  Insertion requires logarithmic time.
+       *  @{
        */
       std::pair<iterator, bool>
       insert(const value_type& __x)
       { return _M_t._M_insert_unique(__x); }
 
 #if __cplusplus >= 201103L
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 2354. Unnecessary copying when inserting into maps with braced-init
+      std::pair<iterator, bool>
+      insert(value_type&& __x)
+      { return _M_t._M_insert_unique(std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -804,6 +810,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	insert(_Pair&& __x)
 	{ return _M_t._M_insert_unique(std::forward<_Pair>(__x)); }
 #endif
+      // @}
 
 #if __cplusplus >= 201103L
       /**
@@ -840,6 +847,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  for more on @a hinting.
        *
        *  Insertion requires logarithmic time (if the hint is not taken).
+       *  @{
        */
       iterator
 #if __cplusplus >= 201103L
@@ -850,6 +858,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       { return _M_t._M_insert_unique_(__position, __x); }
 
 #if __cplusplus >= 201103L
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 2354. Unnecessary copying when inserting into maps with braced-init
+      iterator
+      insert(const_iterator __position, value_type&& __x)
+      { return _M_t._M_insert_unique_(__position, std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -858,6 +872,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	{ return _M_t._M_insert_unique_(__position,
 					std::forward<_Pair>(__x)); }
 #endif
+      // @}
 
       /**
        *  @brief Template function that attempts to insert a range of elements.
@@ -1366,6 +1381,40 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 		  const map<_K1, _T1, _C1, _A1>&);
     };
 
+
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Compare = less<__iter_key_t<_InputIterator>>,
+	   typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    map(_InputIterator, _InputIterator,
+	_Compare = _Compare(), _Allocator = _Allocator())
+    -> map<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+	   _Compare, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Compare = less<_Key>,
+	   typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	   typename = _RequireAllocator<_Allocator>>
+    map(initializer_list<pair<_Key, _Tp>>,
+	_Compare = _Compare(), _Allocator = _Allocator())
+    -> map<_Key, _Tp, _Compare, _Allocator>;
+
+  template <typename _InputIterator, typename _Allocator,
+	    typename = _RequireInputIter<_InputIterator>,
+	    typename = _RequireAllocator<_Allocator>>
+    map(_InputIterator, _InputIterator, _Allocator)
+    -> map<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+	   less<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    map(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> map<_Key, _Tp, less<_Key>, _Allocator>;
+
+#endif
+
   /**
    *  @brief  Map equality comparison.
    *  @param  __x  A %map.
diff --git a/libstdc++-v3/include/bits/stl_multimap.h b/libstdc++-v3/include/bits/stl_multimap.h
index 7e3cea48a47..1a16bf982e3 100644
--- a/libstdc++-v3/include/bits/stl_multimap.h
+++ b/libstdc++-v3/include/bits/stl_multimap.h
@@ -526,12 +526,19 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  thus multiple pairs with the same key can be inserted.
        *
        *  Insertion requires logarithmic time.
+       *  @{
        */
       iterator
       insert(const value_type& __x)
       { return _M_t._M_insert_equal(__x); }
 
 #if __cplusplus >= 201103L
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 2354. Unnecessary copying when inserting into maps with braced-init
+      iterator
+      insert(value_type&& __x)
+      { return _M_t._M_insert_equal(std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -539,6 +546,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	insert(_Pair&& __x)
 	{ return _M_t._M_insert_equal(std::forward<_Pair>(__x)); }
 #endif
+      // @}
 
       /**
        *  @brief Inserts a std::pair into the %multimap.
@@ -559,6 +567,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  https://gcc.gnu.org/onlinedocs/libstdc++/manual/associative.html#containers.associative.insert_hints
        *
        *  Insertion requires logarithmic time (if the hint is not taken).
+       * @{
        */
       iterator
 #if __cplusplus >= 201103L
@@ -569,6 +578,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       { return _M_t._M_insert_equal_(__position, __x); }
 
 #if __cplusplus >= 201103L
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 2354. Unnecessary copying when inserting into maps with braced-init
+      iterator
+      insert(const_iterator __position, value_type&& __x)
+      { return _M_t._M_insert_equal_(__position, std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -577,6 +592,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	{ return _M_t._M_insert_equal_(__position,
 				       std::forward<_Pair>(__x)); }
 #endif
+      // @}
 
       /**
        *  @brief A template function that attempts to insert a range
@@ -1031,6 +1047,39 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 		  const multimap<_K1, _T1, _C1, _A1>&);
   };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Compare = less<__iter_key_t<_InputIterator>>,
+	   typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(_InputIterator, _InputIterator,
+	     _Compare = _Compare(), _Allocator = _Allocator())
+    -> multimap<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+		_Compare, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Compare = less<_Key>,
+	   typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(initializer_list<pair<_Key, _Tp>>,
+	     _Compare = _Compare(), _Allocator = _Allocator())
+    -> multimap<_Key, _Tp, _Compare, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(_InputIterator, _InputIterator, _Allocator)
+    -> multimap<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+		less<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> multimap<_Key, _Tp, less<_Key>, _Allocator>;
+
+#endif
+
   /**
    *  @brief  Multimap equality comparison.
    *  @param  __x  A %multimap.
diff --git a/libstdc++-v3/include/bits/stl_multiset.h b/libstdc++-v3/include/bits/stl_multiset.h
index 517e77e9372..d34b6758e18 100644
--- a/libstdc++-v3/include/bits/stl_multiset.h
+++ b/libstdc++-v3/include/bits/stl_multiset.h
@@ -881,6 +881,43 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 		   const multiset<_K1, _C1, _A1>&);
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Compare =
+	     less<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	     allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+   multiset(_InputIterator, _InputIterator,
+	    _Compare = _Compare(), _Allocator = _Allocator())
+   -> multiset<typename iterator_traits<_InputIterator>::value_type,
+	       _Compare, _Allocator>;
+
+ template<typename _Key,
+	  typename _Compare = less<_Key>,
+	  typename _Allocator = allocator<_Key>,
+	  typename = _RequireAllocator<_Allocator>>
+   multiset(initializer_list<_Key>,
+	    _Compare = _Compare(), _Allocator = _Allocator())
+   -> multiset<_Key, _Compare, _Allocator>;
+
+ template<typename _InputIterator, typename _Allocator,
+	  typename = _RequireInputIter<_InputIterator>,
+	  typename = _RequireAllocator<_Allocator>>
+   multiset(_InputIterator, _InputIterator, _Allocator)
+   -> multiset<typename iterator_traits<_InputIterator>::value_type,
+	       less<typename iterator_traits<_InputIterator>::value_type>,
+	       _Allocator>;
+
+ template<typename _Key, typename _Allocator,
+	  typename = _RequireAllocator<_Allocator>>
+   multiset(initializer_list<_Key>, _Allocator)
+   -> multiset<_Key, less<_Key>, _Allocator>;
+
+#endif
+
   /**
    *  @brief  Multiset equality comparison.
    *  @param  __x  A %multiset.
@@ -971,6 +1008,7 @@ _GLIBCXX_END_NAMESPACE_CONTAINER
       _S_get_tree(_GLIBCXX_STD_C::multiset<_Val, _Cmp2, _Alloc>& __set)
       { return __set._M_t; }
     };
+
 #endif // C++17
 
 _GLIBCXX_END_NAMESPACE_VERSION
diff --git a/libstdc++-v3/include/bits/stl_set.h b/libstdc++-v3/include/bits/stl_set.h
index e804a7cae70..3a7992c4dab 100644
--- a/libstdc++-v3/include/bits/stl_set.h
+++ b/libstdc++-v3/include/bits/stl_set.h
@@ -898,6 +898,41 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	operator<(const set<_K1, _C1, _A1>&, const set<_K1, _C1, _A1>&);
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Compare =
+	     less<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	     allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    set(_InputIterator, _InputIterator,
+	_Compare = _Compare(), _Allocator = _Allocator())
+    -> set<typename iterator_traits<_InputIterator>::value_type,
+	  _Compare, _Allocator>;
+
+  template<typename _Key, typename _Compare = less<_Key>,
+	   typename _Allocator = allocator<_Key>,
+	   typename = _RequireAllocator<_Allocator>>
+    set(initializer_list<_Key>,
+	_Compare = _Compare(), _Allocator = _Allocator())
+    -> set<_Key, _Compare, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    set(_InputIterator, _InputIterator, _Allocator)
+    -> set<typename iterator_traits<_InputIterator>::value_type,
+	   less<typename iterator_traits<_InputIterator>::value_type>,
+	   _Allocator>;
+
+  template<typename _Key, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    set(initializer_list<_Key>, _Allocator)
+    -> set<_Key, less<_Key>, _Allocator>;
+
+#endif
 
   /**
    *  @brief  Set equality comparison.
diff --git a/libstdc++-v3/include/bits/stl_uninitialized.h b/libstdc++-v3/include/bits/stl_uninitialized.h
index c2ba863ed98..bac3fd01f7a 100644
--- a/libstdc++-v3/include/bits/stl_uninitialized.h
+++ b/libstdc++-v3/include/bits/stl_uninitialized.h
@@ -206,7 +206,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  _ForwardIterator __cur = __first;
 	  __try
 	    {
-	      for (; __n > 0; --__n, ++__cur)
+	      for (; __n > 0; --__n, (void) ++__cur)
 		std::_Construct(std::__addressof(*__cur), __x);
 	      return __cur;
 	    }
@@ -347,7 +347,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       __try
 	{
 	  typedef __gnu_cxx::__alloc_traits<_Allocator> __traits;
-	  for (; __n > 0; --__n, ++__cur)
+	  for (; __n > 0; --__n, (void) ++__cur)
 	    __traits::construct(__alloc, std::__addressof(*__cur), __x);
 	  return __cur;
 	}
@@ -523,7 +523,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  _ForwardIterator __cur = __first;
 	  __try
 	    {
-	      for (; __n > 0; --__n, ++__cur)
+	      for (; __n > 0; --__n, (void) ++__cur)
 		std::_Construct(std::__addressof(*__cur));
 	      return __cur;
 	    }
@@ -627,7 +627,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       __try
 	{
 	  typedef __gnu_cxx::__alloc_traits<_Allocator> __traits;
-	  for (; __n > 0; --__n, ++__cur)
+	  for (; __n > 0; --__n, (void) ++__cur)
 	    __traits::construct(__alloc, std::__addressof(*__cur));
 	  return __cur;
 	}
@@ -687,7 +687,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  _ForwardIterator __cur = __first;
 	  __try
 	    {
-	      for (; __n > 0; --__n, ++__cur)
+	      for (; __n > 0; --__n, (void) ++__cur)
 		std::_Construct_novalue(std::__addressof(*__cur));
 	      return __cur;
 	    }
@@ -747,7 +747,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       _ForwardIterator __cur = __result;
       __try
 	{
-	  for (; __n > 0; --__n, ++__first, ++__cur)
+	  for (; __n > 0; --__n, (void) ++__first, ++__cur)
 	    std::_Construct(std::__addressof(*__cur), *__first);
 	  return __cur;
 	}
@@ -775,7 +775,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       _ForwardIterator __cur = __result;
       __try
 	{
-	  for (; __n > 0; --__n, ++__first, ++__cur)
+	  for (; __n > 0; --__n, (void) ++__first, ++__cur)
 	    std::_Construct(std::__addressof(*__cur), *__first);
 	  return {__first, __cur};
 	}
diff --git a/libstdc++-v3/include/bits/string_view.tcc b/libstdc++-v3/include/bits/string_view.tcc
index 4d98f8668a0..5c53c584381 100644
--- a/libstdc++-v3/include/bits/string_view.tcc
+++ b/libstdc++-v3/include/bits/string_view.tcc
@@ -119,7 +119,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename _CharT, typename _Traits>
     constexpr typename basic_string_view<_CharT, _Traits>::size_type
     basic_string_view<_CharT, _Traits>::
-    find_first_of(const _CharT* __str, size_type __pos, size_type __n) const
+    find_first_of(const _CharT* __str, size_type __pos,
+		  size_type __n) const noexcept
     {
       __glibcxx_requires_string_len(__str, __n);
       for (; __n && __pos < this->_M_len; ++__pos)
@@ -135,7 +136,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename _CharT, typename _Traits>
     constexpr typename basic_string_view<_CharT, _Traits>::size_type
     basic_string_view<_CharT, _Traits>::
-    find_last_of(const _CharT* __str, size_type __pos, size_type __n) const
+    find_last_of(const _CharT* __str, size_type __pos,
+		 size_type __n) const noexcept
     {
       __glibcxx_requires_string_len(__str, __n);
       size_type __size = this->size();
@@ -156,7 +158,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename _CharT, typename _Traits>
     constexpr typename basic_string_view<_CharT, _Traits>::size_type
     basic_string_view<_CharT, _Traits>::
-    find_first_not_of(const _CharT* __str, size_type __pos, size_type __n) const
+    find_first_not_of(const _CharT* __str, size_type __pos,
+		      size_type __n) const noexcept
     {
       __glibcxx_requires_string_len(__str, __n);
       for (; __pos < this->_M_len; ++__pos)
@@ -179,7 +182,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename _CharT, typename _Traits>
     constexpr typename basic_string_view<_CharT, _Traits>::size_type
     basic_string_view<_CharT, _Traits>::
-    find_last_not_of(const _CharT* __str, size_type __pos, size_type __n) const
+    find_last_not_of(const _CharT* __str, size_type __pos,
+		     size_type __n) const noexcept
     {
       __glibcxx_requires_string_len(__str, __n);
       size_type __size = this->_M_len;
diff --git a/libstdc++-v3/include/bits/unordered_map.h b/libstdc++-v3/include/bits/unordered_map.h
index df1302c80c0..385e4bd4df4 100644
--- a/libstdc++-v3/include/bits/unordered_map.h
+++ b/libstdc++-v3/include/bits/unordered_map.h
@@ -579,6 +579,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       insert(const value_type& __x)
       { return _M_h.insert(__x); }
 
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 2354. Unnecessary copying when inserting into maps with braced-init
+      std::pair<iterator, bool>
+      insert(value_type&& __x)
+      { return _M_h.insert(std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -613,6 +619,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       insert(const_iterator __hint, const value_type& __x)
       { return _M_h.insert(__hint, __x); }
 
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 2354. Unnecessary copying when inserting into maps with braced-init
+      iterator
+      insert(const_iterator __hint, value_type&& __x)
+      { return _M_h.insert(__hint, std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -1118,6 +1130,82 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 		   const unordered_map<_Key1, _Tp1, _Hash1, _Pred1, _Alloc1>&);
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash = hash<__iter_key_t<_InputIterator>>,
+	   typename _Pred = equal_to<__iter_key_t<_InputIterator>>,
+	   typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator,
+		  typename unordered_map<int, int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>,
+		     _Hash, _Pred, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash = hash<_Key>,
+	   typename _Pred = equal_to<_Key>,
+	   typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>,
+		  typename unordered_map<int, int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_map<_Key, _Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator,
+		  typename unordered_map<int, int>::size_type, _Allocator)
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>,
+		     hash<__iter_key_t<_InputIterator>>,
+		     equal_to<__iter_key_t<_InputIterator>>,
+		     _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator, _Allocator)
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>,
+		     hash<__iter_key_t<_InputIterator>>,
+		     equal_to<__iter_key_t<_InputIterator>>,
+		     _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator,
+		  typename unordered_map<int, int>::size_type,
+		  _Hash, _Allocator)
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>, _Hash,
+		     equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>,
+		  typename unordered_map<int, int>::size_type,
+		  _Allocator)
+    -> unordered_map<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> unordered_map<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>,
+		  typename unordered_map<int, int>::size_type,
+		  _Hash, _Allocator)
+    -> unordered_map<_Key, _Tp, _Hash, equal_to<_Key>, _Allocator>;
+
+#endif
+
   /**
    *  @brief A standard container composed of equivalent keys
    *  (possibly containing multiple of each key value) that associates
@@ -1468,6 +1556,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       insert(const value_type& __x)
       { return _M_h.insert(__x); }
 
+      iterator
+      insert(value_type&& __x)
+      { return _M_h.insert(std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -1500,6 +1592,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       insert(const_iterator __hint, const value_type& __x)
       { return _M_h.insert(__hint, __x); }
 
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 2354. Unnecessary copying when inserting into maps with braced-init
+      iterator
+      insert(const_iterator __hint, value_type&& __x)
+      { return _M_h.insert(__hint, std::move(__x)); }
+
       template<typename _Pair, typename = typename
 	       std::enable_if<std::is_constructible<value_type,
 						    _Pair&&>::value>::type>
@@ -1871,6 +1969,82 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 					    _Hash1, _Pred1, _Alloc1>&);
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash = hash<__iter_key_t<_InputIterator>>,
+	   typename _Pred = equal_to<__iter_key_t<_InputIterator>>,
+	   typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator,
+		       unordered_multimap<int, int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>, _Hash, _Pred,
+			  _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash = hash<_Key>,
+	   typename _Pred = equal_to<_Key>,
+	   typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>,
+		       unordered_multimap<int, int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multimap<_Key, _Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator,
+		       unordered_multimap<int, int>::size_type, _Allocator)
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>,
+			  hash<__iter_key_t<_InputIterator>>,
+			  equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator, _Allocator)
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>,
+			  hash<__iter_key_t<_InputIterator>>,
+			  equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator,
+		       unordered_multimap<int, int>::size_type, _Hash,
+		       _Allocator)
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>, _Hash,
+			  equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>,
+		       unordered_multimap<int, int>::size_type,
+		       _Allocator)
+    -> unordered_multimap<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> unordered_multimap<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>,
+		       unordered_multimap<int, int>::size_type,
+		       _Hash, _Allocator)
+    -> unordered_multimap<_Key, _Tp, _Hash, equal_to<_Key>, _Allocator>;
+
+#endif
+
   template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
     inline void
     swap(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
diff --git a/libstdc++-v3/include/bits/unordered_set.h b/libstdc++-v3/include/bits/unordered_set.h
index df57915f31a..416dbbcfb96 100644
--- a/libstdc++-v3/include/bits/unordered_set.h
+++ b/libstdc++-v3/include/bits/unordered_set.h
@@ -805,6 +805,70 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 		   const unordered_set<_Value1, _Hash1, _Pred1, _Alloc1>&);
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash =
+	   hash<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Pred =
+	   equal_to<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	   allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(_InputIterator, _InputIterator,
+		  unordered_set<int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_set<typename iterator_traits<_InputIterator>::value_type,
+		     _Hash, _Pred, _Allocator>;
+
+  template<typename _Tp, typename _Hash = hash<_Tp>,
+	   typename _Pred = equal_to<_Tp>,
+	   typename _Allocator = allocator<_Tp>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(initializer_list<_Tp>,
+		  unordered_set<int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_set<_Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(_InputIterator, _InputIterator,
+		  unordered_set<int>::size_type, _Allocator)
+    -> unordered_set<typename iterator_traits<_InputIterator>::value_type,
+		     hash<
+		       typename iterator_traits<_InputIterator>::value_type>,
+		     equal_to<
+		       typename iterator_traits<_InputIterator>::value_type>,
+		     _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(_InputIterator, _InputIterator,
+		  unordered_set<int>::size_type,
+		  _Hash, _Allocator)
+    -> unordered_set<typename iterator_traits<_InputIterator>::value_type,
+		     _Hash,
+		     equal_to<
+		       typename iterator_traits<_InputIterator>::value_type>,
+		     _Allocator>;
+
+  template<typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(initializer_list<_Tp>,
+		  unordered_set<int>::size_type, _Allocator)
+    -> unordered_set<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
+
+  template<typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(initializer_list<_Tp>,
+		  unordered_set<int>::size_type, _Hash, _Allocator)
+    -> unordered_set<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
+
+#endif
+
   /**
    *  @brief A standard container composed of equivalent keys
    *  (possibly containing multiple of each key value) in which the
@@ -1517,6 +1581,75 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 		 const unordered_multiset<_Value1, _Hash1, _Pred1, _Alloc1>&);
     };
 
+
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash =
+	   hash<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Pred =
+	   equal_to<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	   allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(_InputIterator, _InputIterator,
+		       unordered_multiset<int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multiset<typename iterator_traits<_InputIterator>::value_type,
+                          _Hash, _Pred, _Allocator>;
+
+  template<typename _Tp, typename _Hash = hash<_Tp>,
+	   typename _Pred = equal_to<_Tp>,
+	   typename _Allocator = allocator<_Tp>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(initializer_list<_Tp>,
+		       unordered_multiset<int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multiset<_Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(_InputIterator, _InputIterator,
+		       unordered_multiset<int>::size_type, _Allocator)
+    -> unordered_multiset<typename iterator_traits<_InputIterator>::value_type,
+			  hash<typename
+			       iterator_traits<_InputIterator>::value_type>,
+			  equal_to<typename
+				   iterator_traits<_InputIterator>::value_type>,
+			  _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(_InputIterator, _InputIterator,
+		       unordered_multiset<int>::size_type,
+		       _Hash, _Allocator)
+    -> unordered_multiset<typename
+			  iterator_traits<_InputIterator>::value_type,
+			  _Hash,
+			  equal_to<
+			    typename
+			    iterator_traits<_InputIterator>::value_type>,
+			  _Allocator>;
+
+  template<typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(initializer_list<_Tp>,
+		       unordered_multiset<int>::size_type, _Allocator)
+    -> unordered_multiset<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
+
+  template<typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(initializer_list<_Tp>,
+		       unordered_multiset<int>::size_type, _Hash, _Allocator)
+    -> unordered_multiset<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
+
+#endif
+
   template<class _Value, class _Hash, class _Pred, class _Alloc>
     inline void
     swap(unordered_set<_Value, _Hash, _Pred, _Alloc>& __x,
diff --git a/libstdc++-v3/include/c_compatibility/math.h b/libstdc++-v3/include/c_compatibility/math.h
index 84755c8c35c..28c7c8368f0 100644
--- a/libstdc++-v3/include/c_compatibility/math.h
+++ b/libstdc++-v3/include/c_compatibility/math.h
@@ -111,7 +111,7 @@ using std::tgamma;
 using std::trunc;
 #endif // C++11 && _GLIBCXX_USE_C99_MATH_TR1
 
-#if __STDCPP_WANT_MATH_SPEC_FUNCS__ == 1
+#if _GLIBCXX_USE_STD_SPEC_FUNCS
 using std::assoc_laguerref;
 using std::assoc_laguerrel;
 using std::assoc_laguerre;
@@ -175,7 +175,7 @@ using std::sph_legendre;
 using std::sph_neumannf;
 using std::sph_neumannl;
 using std::sph_neumann;
-#endif // __STDCPP_WANT_MATH_SPEC_FUNCS__
+#endif // _GLIBCXX_USE_STD_SPEC_FUNCS
 
 #endif // _GLIBCXX_MATH_H
 #endif // __cplusplus
diff --git a/libstdc++-v3/include/c_global/cstddef b/libstdc++-v3/include/c_global/cstddef
index 09754ee45da..11d268b7f81 100644
--- a/libstdc++-v3/include/c_global/cstddef
+++ b/libstdc++-v3/include/c_global/cstddef
@@ -57,9 +57,11 @@ namespace std
 }
 #endif
 
-#if __cplusplus > 201402L
+#if __cplusplus >= 201703L
 namespace std
 {
+#define __cpp_lib_byte 201603
+
   /// std::byte
   enum class byte : unsigned char {};
 
diff --git a/libstdc++-v3/include/debug/array b/libstdc++-v3/include/debug/array
index 9c279221040..95edc84f38b 100644
--- a/libstdc++-v3/include/debug/array
+++ b/libstdc++-v3/include/debug/array
@@ -306,6 +306,14 @@ namespace __debug
       return _GLIBCXX_STD_C::__array_traits<_Tp, _Nm>::
 	_S_ref(__arr._M_elems, _Int);
     }
+
+  template<std::size_t _Int, typename _Tp, std::size_t _Nm>
+    constexpr const _Tp&&
+    get(const array<_Tp, _Nm>&& __arr) noexcept
+    {
+      static_assert(_Int < _Nm, "index is out of bounds");
+      return std::move(__debug::get<_Int>(__arr));
+    }
 } // namespace __debug
 
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
diff --git a/libstdc++-v3/include/debug/deque b/libstdc++-v3/include/debug/deque
index 7e3c193c103..5816b4b1624 100644
--- a/libstdc++-v3/include/debug/deque
+++ b/libstdc++-v3/include/debug/deque
@@ -624,6 +624,16 @@ namespace __debug
       _M_base() const _GLIBCXX_NOEXCEPT	{ return *this; }
     };
 
+#if __cpp_deduction_guides >= 201606
+  template<typename _InputIterator, typename _ValT
+	     = typename iterator_traits<_InputIterator>::value_type,
+	   typename _Allocator = allocator<_ValT>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    deque(_InputIterator, _InputIterator, _Allocator = _Allocator())
+      -> deque<_ValT, _Allocator>;
+#endif
+
   template<typename _Tp, typename _Alloc>
     inline bool
     operator==(const deque<_Tp, _Alloc>& __lhs,
diff --git a/libstdc++-v3/include/debug/forward_list b/libstdc++-v3/include/debug/forward_list
index cd343302b00..20c7ebe4a8f 100644
--- a/libstdc++-v3/include/debug/forward_list
+++ b/libstdc++-v3/include/debug/forward_list
@@ -772,6 +772,16 @@ namespace __debug
       _M_base() const noexcept { return *this; }
     };
 
+#if __cpp_deduction_guides >= 201606
+  template<typename _InputIterator, typename _ValT
+	     = typename iterator_traits<_InputIterator>::value_type,
+	   typename _Allocator = allocator<_ValT>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    forward_list(_InputIterator, _InputIterator, _Allocator = _Allocator())
+      -> forward_list<_ValT, _Allocator>;
+#endif
+
   template<typename _Tp, typename _Alloc>
     bool
     operator==(const forward_list<_Tp, _Alloc>& __lx,
diff --git a/libstdc++-v3/include/debug/list b/libstdc++-v3/include/debug/list
index a45ece244bd..94d325c6e6a 100644
--- a/libstdc++-v3/include/debug/list
+++ b/libstdc++-v3/include/debug/list
@@ -769,6 +769,16 @@ namespace __debug
       _M_base() const _GLIBCXX_NOEXCEPT	{ return *this; }
     };
 
+#if __cpp_deduction_guides >= 201606
+  template<typename _InputIterator, typename _ValT
+	     = typename iterator_traits<_InputIterator>::value_type,
+	   typename _Allocator = allocator<_ValT>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    list(_InputIterator, _InputIterator, _Allocator = _Allocator())
+      -> list<_ValT, _Allocator>;
+#endif
+
   template<typename _Tp, typename _Alloc>
     inline bool
     operator==(const list<_Tp, _Alloc>& __lhs,
diff --git a/libstdc++-v3/include/debug/map.h b/libstdc++-v3/include/debug/map.h
index c275979052c..d5a2c9f6e6f 100644
--- a/libstdc++-v3/include/debug/map.h
+++ b/libstdc++-v3/include/debug/map.h
@@ -667,6 +667,39 @@ namespace __debug
       _M_base() const _GLIBCXX_NOEXCEPT	{ return *this; }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+ template<typename _InputIterator,
+	  typename _Compare = less<__iter_key_t<_InputIterator>>,
+	  typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	  typename = _RequireInputIter<_InputIterator>,
+	  typename = _RequireAllocator<_Allocator>>
+  map(_InputIterator, _InputIterator,
+      _Compare = _Compare(), _Allocator = _Allocator())
+  -> map<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+	 _Compare, _Allocator>;
+
+ template<typename _Key, typename _Tp, typename _Compare = less<_Key>,
+	  typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	  typename = _RequireAllocator<_Allocator>>
+   map(initializer_list<pair<_Key, _Tp>>,
+       _Compare = _Compare(), _Allocator = _Allocator())
+   -> map<_Key, _Tp, _Compare, _Allocator>;
+
+ template <typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+   map(_InputIterator, _InputIterator, _Allocator)
+   -> map<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+	  less<__iter_key_t<_InputIterator>>, _Allocator>;
+
+ template<typename _Key, typename _Tp, typename _Allocator,
+	  typename = _RequireAllocator<_Allocator>>
+   map(initializer_list<pair<_Key, _Tp>>, _Allocator)
+   -> map<_Key, _Tp, less<_Key>, _Allocator>;
+
+#endif
+
   template<typename _Key, typename _Tp,
 	   typename _Compare, typename _Allocator>
     inline bool
diff --git a/libstdc++-v3/include/debug/multimap.h b/libstdc++-v3/include/debug/multimap.h
index f971cfa41f0..6c7b2ab4f8d 100644
--- a/libstdc++-v3/include/debug/multimap.h
+++ b/libstdc++-v3/include/debug/multimap.h
@@ -555,6 +555,39 @@ namespace __debug
       _M_base() const _GLIBCXX_NOEXCEPT { return *this; }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Compare = less<__iter_key_t<_InputIterator>>,
+	   typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(_InputIterator, _InputIterator,
+	     _Compare = _Compare(), _Allocator = _Allocator())
+    -> multimap<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+		_Compare, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Compare = less<_Key>,
+	   typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(initializer_list<pair<_Key, _Tp>>,
+	     _Compare = _Compare(), _Allocator = _Allocator())
+    -> multimap<_Key, _Tp, _Compare, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(_InputIterator, _InputIterator, _Allocator)
+    -> multimap<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
+    less<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> multimap<_Key, _Tp, less<_Key>, _Allocator>;
+
+#endif
+
   template<typename _Key, typename _Tp,
 	   typename _Compare, typename _Allocator>
     inline bool
diff --git a/libstdc++-v3/include/debug/multiset.h b/libstdc++-v3/include/debug/multiset.h
index 97c7b843f27..59331b4bc6b 100644
--- a/libstdc++-v3/include/debug/multiset.h
+++ b/libstdc++-v3/include/debug/multiset.h
@@ -542,6 +542,43 @@ namespace __debug
       _M_base() const _GLIBCXX_NOEXCEPT { return *this; }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Compare =
+	     less<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	     allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+   multiset(_InputIterator, _InputIterator,
+	    _Compare = _Compare(), _Allocator = _Allocator())
+   -> multiset<typename iterator_traits<_InputIterator>::value_type,
+	       _Compare, _Allocator>;
+
+ template<typename _Key,
+	  typename _Compare = less<_Key>,
+	  typename _Allocator = allocator<_Key>,
+	  typename = _RequireAllocator<_Allocator>>
+   multiset(initializer_list<_Key>,
+	    _Compare = _Compare(), _Allocator = _Allocator())
+   -> multiset<_Key, _Compare, _Allocator>;
+
+ template<typename _InputIterator, typename _Allocator,
+	  typename = _RequireInputIter<_InputIterator>,
+	  typename = _RequireAllocator<_Allocator>>
+   multiset(_InputIterator, _InputIterator, _Allocator)
+   -> multiset<typename iterator_traits<_InputIterator>::value_type,
+	       less<typename iterator_traits<_InputIterator>::value_type>,
+	       _Allocator>;
+
+ template<typename _Key, typename _Allocator,
+	  typename = _RequireAllocator<_Allocator>>
+   multiset(initializer_list<_Key>, _Allocator)
+   -> multiset<_Key, less<_Key>, _Allocator>;
+
+#endif
+
   template<typename _Key, typename _Compare, typename _Allocator>
     inline bool
     operator==(const multiset<_Key, _Compare, _Allocator>& __lhs,
diff --git a/libstdc++-v3/include/debug/set.h b/libstdc++-v3/include/debug/set.h
index 07032865bc9..5353cfe87b2 100644
--- a/libstdc++-v3/include/debug/set.h
+++ b/libstdc++-v3/include/debug/set.h
@@ -560,6 +560,42 @@ namespace __debug
       _M_base() const _GLIBCXX_NOEXCEPT	{ return *this; }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Compare =
+	     less<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	     allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    set(_InputIterator, _InputIterator,
+       _Compare = _Compare(), _Allocator = _Allocator())
+   -> set<typename iterator_traits<_InputIterator>::value_type,
+	  _Compare, _Allocator>;
+
+ template<typename _Key, typename _Compare = less<_Key>,
+	  typename _Allocator = allocator<_Key>,
+	  typename = _RequireAllocator<_Allocator>>
+   set(initializer_list<_Key>,
+       _Compare = _Compare(), _Allocator = _Allocator())
+   -> set<_Key, _Compare, _Allocator>;
+
+ template<typename _InputIterator, typename _Allocator,
+	  typename = _RequireInputIter<_InputIterator>,
+	  typename = _RequireAllocator<_Allocator>>
+   set(_InputIterator, _InputIterator, _Allocator)
+   -> set<typename iterator_traits<_InputIterator>::value_type,
+	  less<typename iterator_traits<_InputIterator>::value_type>,
+	  _Allocator>;
+
+ template<typename _Key, typename _Allocator,
+	  typename = _RequireAllocator<_Allocator>>
+   set(initializer_list<_Key>, _Allocator)
+   -> set<_Key, less<_Key>, _Allocator>;
+
+#endif
+
   template<typename _Key, typename _Compare, typename _Allocator>
     inline bool
     operator==(const set<_Key, _Compare, _Allocator>& __lhs,
diff --git a/libstdc++-v3/include/debug/unordered_map b/libstdc++-v3/include/debug/unordered_map
index 51a55ae21f3..c5734304846 100644
--- a/libstdc++-v3/include/debug/unordered_map
+++ b/libstdc++-v3/include/debug/unordered_map
@@ -616,6 +616,82 @@ namespace __debug
       }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash = hash<__iter_key_t<_InputIterator>>,
+	   typename _Pred = equal_to<__iter_key_t<_InputIterator>>,
+	   typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator,
+		  typename unordered_map<int, int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>,
+		     _Hash, _Pred, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash = hash<_Key>,
+	   typename _Pred = equal_to<_Key>,
+	   typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>,
+		  typename unordered_map<int, int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_map<_Key, _Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator,
+		  typename unordered_map<int, int>::size_type, _Allocator)
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>,
+		     hash<__iter_key_t<_InputIterator>>,
+		     equal_to<__iter_key_t<_InputIterator>>,
+		     _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator, _Allocator)
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>,
+		     hash<__iter_key_t<_InputIterator>>,
+		     equal_to<__iter_key_t<_InputIterator>>,
+		     _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(_InputIterator, _InputIterator,
+		  typename unordered_map<int, int>::size_type,
+		  _Hash, _Allocator)
+    -> unordered_map<__iter_key_t<_InputIterator>,
+		     __iter_val_t<_InputIterator>, _Hash,
+		     equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>,
+		  typename unordered_map<int, int>::size_type,
+		  _Allocator)
+    -> unordered_map<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> unordered_map<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_map(initializer_list<pair<_Key, _Tp>>,
+		  typename unordered_map<int, int>::size_type,
+		  _Hash, _Allocator)
+    -> unordered_map<_Key, _Tp, _Hash, equal_to<_Key>, _Allocator>;
+
+#endif
+
   template<typename _Key, typename _Tp, typename _Hash,
 	   typename _Pred, typename _Alloc>
     inline void
@@ -1110,6 +1186,82 @@ namespace __debug
       }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash = hash<__iter_key_t<_InputIterator>>,
+	   typename _Pred = equal_to<__iter_key_t<_InputIterator>>,
+	   typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator,
+		       unordered_multimap<int, int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>, _Hash, _Pred,
+			  _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash = hash<_Key>,
+	   typename _Pred = equal_to<_Key>,
+	   typename _Allocator = allocator<pair<const _Key, _Tp>>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>,
+		       unordered_multimap<int, int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multimap<_Key, _Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator,
+		       unordered_multimap<int, int>::size_type, _Allocator)
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>,
+			  hash<__iter_key_t<_InputIterator>>,
+			  equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator, _Allocator)
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>,
+			  hash<__iter_key_t<_InputIterator>>,
+			  equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(_InputIterator, _InputIterator,
+		       unordered_multimap<int, int>::size_type, _Hash,
+		       _Allocator)
+    -> unordered_multimap<__iter_key_t<_InputIterator>,
+			  __iter_val_t<_InputIterator>, _Hash,
+			  equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>,
+		       unordered_multimap<int, int>::size_type,
+		       _Allocator)
+    -> unordered_multimap<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
+    -> unordered_multimap<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
+
+  template<typename _Key, typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multimap(initializer_list<pair<_Key, _Tp>>,
+		       unordered_multimap<int, int>::size_type,
+		       _Hash, _Allocator)
+    -> unordered_multimap<_Key, _Tp, _Hash, equal_to<_Key>, _Allocator>;
+
+#endif
+
   template<typename _Key, typename _Tp, typename _Hash,
 	   typename _Pred, typename _Alloc>
     inline void
diff --git a/libstdc++-v3/include/debug/unordered_set b/libstdc++-v3/include/debug/unordered_set
index 923ff2bc4d8..1fe493fe52f 100644
--- a/libstdc++-v3/include/debug/unordered_set
+++ b/libstdc++-v3/include/debug/unordered_set
@@ -530,6 +530,70 @@ namespace __debug
       }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash =
+	   hash<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Pred =
+	   equal_to<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	   allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(_InputIterator, _InputIterator,
+		  unordered_set<int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_set<typename iterator_traits<_InputIterator>::value_type,
+		     _Hash, _Pred, _Allocator>;
+
+  template<typename _Tp, typename _Hash = hash<_Tp>,
+	   typename _Pred = equal_to<_Tp>,
+	   typename _Allocator = allocator<_Tp>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(initializer_list<_Tp>,
+		  unordered_set<int>::size_type = {},
+		  _Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
+    -> unordered_set<_Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(_InputIterator, _InputIterator,
+		  unordered_set<int>::size_type, _Allocator)
+    -> unordered_set<typename iterator_traits<_InputIterator>::value_type,
+		     hash<
+		       typename iterator_traits<_InputIterator>::value_type>,
+		     equal_to<
+		       typename iterator_traits<_InputIterator>::value_type>,
+		     _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(_InputIterator, _InputIterator,
+		  unordered_set<int>::size_type,
+		  _Hash, _Allocator)
+    -> unordered_set<typename iterator_traits<_InputIterator>::value_type,
+		     _Hash,
+		     equal_to<
+		       typename iterator_traits<_InputIterator>::value_type>,
+		     _Allocator>;
+
+  template<typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(initializer_list<_Tp>,
+		  unordered_set<int>::size_type, _Allocator)
+    -> unordered_set<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
+
+  template<typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_set(initializer_list<_Tp>,
+		  unordered_set<int>::size_type, _Hash, _Allocator)
+    -> unordered_set<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
+
+#endif
+
   template<typename _Value, typename _Hash, typename _Pred, typename _Alloc>
     inline void
     swap(unordered_set<_Value, _Hash, _Pred, _Alloc>& __x,
@@ -1012,6 +1076,74 @@ namespace __debug
       }
     };
 
+#if __cpp_deduction_guides >= 201606
+
+  template<typename _InputIterator,
+	   typename _Hash =
+	   hash<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Pred =
+	   equal_to<typename iterator_traits<_InputIterator>::value_type>,
+	   typename _Allocator =
+	   allocator<typename iterator_traits<_InputIterator>::value_type>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(_InputIterator, _InputIterator,
+		       unordered_multiset<int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multiset<typename iterator_traits<_InputIterator>::value_type,
+                          _Hash, _Pred, _Allocator>;
+
+  template<typename _Tp, typename _Hash = hash<_Tp>,
+	   typename _Pred = equal_to<_Tp>,
+	   typename _Allocator = allocator<_Tp>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(initializer_list<_Tp>,
+		       unordered_multiset<int>::size_type = {},
+		       _Hash = _Hash(), _Pred = _Pred(),
+		       _Allocator = _Allocator())
+    -> unordered_multiset<_Tp, _Hash, _Pred, _Allocator>;
+
+  template<typename _InputIterator, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(_InputIterator, _InputIterator,
+		       unordered_multiset<int>::size_type, _Allocator)
+    -> unordered_multiset<typename iterator_traits<_InputIterator>::value_type,
+			  hash<typename
+			       iterator_traits<_InputIterator>::value_type>,
+			  equal_to<typename
+				   iterator_traits<_InputIterator>::value_type>,
+			  _Allocator>;
+
+  template<typename _InputIterator, typename _Hash, typename _Allocator,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(_InputIterator, _InputIterator,
+		       unordered_multiset<int>::size_type,
+		       _Hash, _Allocator)
+    -> unordered_multiset<typename
+			  iterator_traits<_InputIterator>::value_type,
+			  _Hash,
+			  equal_to<
+			    typename
+			    iterator_traits<_InputIterator>::value_type>,
+			  _Allocator>;
+
+  template<typename _Tp, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(initializer_list<_Tp>,
+		       unordered_multiset<int>::size_type, _Allocator)
+    -> unordered_multiset<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
+
+  template<typename _Tp, typename _Hash, typename _Allocator,
+	   typename = _RequireAllocator<_Allocator>>
+    unordered_multiset(initializer_list<_Tp>,
+		       unordered_multiset<int>::size_type, _Hash, _Allocator)
+    -> unordered_multiset<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
+
+#endif
+
   template<typename _Value, typename _Hash, typename _Pred, typename _Alloc>
     inline void
     swap(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
diff --git a/libstdc++-v3/include/debug/vector b/libstdc++-v3/include/debug/vector
index a6d18d94508..c24c2f3d45f 100644
--- a/libstdc++-v3/include/debug/vector
+++ b/libstdc++-v3/include/debug/vector
@@ -757,6 +757,16 @@ namespace __debug
     _GLIBCXX_NOEXCEPT_IF(noexcept(__lhs.swap(__rhs)))
     { __lhs.swap(__rhs); }
 
+#if __cpp_deduction_guides >= 201606
+  template<typename _InputIterator, typename _ValT
+	     = typename iterator_traits<_InputIterator>::value_type,
+	   typename _Allocator = allocator<_ValT>,
+	   typename = _RequireInputIter<_InputIterator>,
+	   typename = _RequireAllocator<_Allocator>>
+    vector(_InputIterator, _InputIterator, _Allocator = _Allocator())
+      -> vector<_ValT, _Allocator>;
+#endif
+
 } // namespace __debug
 
 #if __cplusplus >= 201103L
diff --git a/libstdc++-v3/include/experimental/bits/fs_dir.h b/libstdc++-v3/include/experimental/bits/fs_dir.h
index 1ff0d9b6def..ecadf37a9cd 100644
--- a/libstdc++-v3/include/experimental/bits/fs_dir.h
+++ b/libstdc++-v3/include/experimental/bits/fs_dir.h
@@ -49,7 +49,7 @@ namespace filesystem
 inline namespace v1
 {
   /**
-   * @ingroup filesystem
+   * @ingroup filesystem-ts
    * @{
    */
 
@@ -351,7 +351,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11
 
 _GLIBCXX_END_NAMESPACE_CXX11
 
-  // @} group filesystem
+  // @} group filesystem-ts
 } // namespace v1
 } // namespace filesystem
 } // namespace experimental
diff --git a/libstdc++-v3/include/experimental/bits/fs_fwd.h b/libstdc++-v3/include/experimental/bits/fs_fwd.h
index 7b851a3d4a8..ac43c5f44f5 100644
--- a/libstdc++-v3/include/experimental/bits/fs_fwd.h
+++ b/libstdc++-v3/include/experimental/bits/fs_fwd.h
@@ -53,7 +53,7 @@ inline namespace __cxx11 __attribute__((__abi_tag__ ("cxx11"))) { }
 #endif
 
   /**
-   * @defgroup filesystem Filesystem
+   * @defgroup filesystem-ts Filesystem TS
    * @ingroup experimental
    *
    * Utilities for performing operations on file systems and their components,
@@ -278,7 +278,7 @@ _GLIBCXX_END_NAMESPACE_CXX11
   bool is_regular_file(file_status) noexcept;
   bool is_symlink(file_status) noexcept;
 
-  // @} group filesystem
+  // @} group filesystem-ts
 } // namespace v1
 } // namespace filesystem
 } // namespace experimental
diff --git a/libstdc++-v3/include/experimental/bits/fs_ops.h b/libstdc++-v3/include/experimental/bits/fs_ops.h
index 387537260e0..fa7f1de6bc4 100644
--- a/libstdc++-v3/include/experimental/bits/fs_ops.h
+++ b/libstdc++-v3/include/experimental/bits/fs_ops.h
@@ -47,7 +47,7 @@ namespace filesystem
 inline namespace v1
 {
   /**
-   * @ingroup filesystem
+   * @ingroup filesystem-ts
    * @{
    */
 
@@ -285,7 +285,7 @@ inline namespace v1
   path temp_directory_path();
   path temp_directory_path(error_code& __ec);
 
-  // @} group filesystem
+  // @} group filesystem-ts
 } // namespace v1
 } // namespace filesystem
 } // namespace experimental
diff --git a/libstdc++-v3/include/experimental/bits/fs_path.h b/libstdc++-v3/include/experimental/bits/fs_path.h
index cde3897b8e5..3e9bc6357af 100644
--- a/libstdc++-v3/include/experimental/bits/fs_path.h
+++ b/libstdc++-v3/include/experimental/bits/fs_path.h
@@ -72,7 +72,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11
 #endif
 
   /**
-   * @ingroup filesystem
+   * @ingroup filesystem-ts
    * @{
    */
 
@@ -725,10 +725,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11
     pointer   operator->() const { return std::__addressof(**this); }
 
     iterator& operator++();
-    iterator  operator++(int) { auto __tmp = *this; ++_M_cur; return __tmp; }
+    iterator  operator++(int) { auto __tmp = *this; ++*this; return __tmp; }
 
     iterator& operator--();
-    iterator  operator--(int) { auto __tmp = *this; --_M_cur; return __tmp; }
+    iterator  operator--(int) { auto __tmp = *this; --*this; return __tmp; }
 
     friend bool operator==(const iterator& __lhs, const iterator& __rhs)
     { return __lhs._M_equals(__rhs); }
@@ -1079,7 +1079,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11
     return _M_at_end == __rhs._M_at_end;
   }
 
-  // @} group filesystem
+  // @} group filesystem-ts
 _GLIBCXX_END_NAMESPACE_CXX11
 } // namespace v1
 } // namespace filesystem
diff --git a/libstdc++-v3/include/experimental/filesystem b/libstdc++-v3/include/experimental/filesystem
index f0b19dd2910..90f6f9eabfe 100644
--- a/libstdc++-v3/include/experimental/filesystem
+++ b/libstdc++-v3/include/experimental/filesystem
@@ -40,36 +40,6 @@
 
 #define __cpp_lib_experimental_filesystem 201406
 
-namespace std _GLIBCXX_VISIBILITY(default)
-{
-_GLIBCXX_BEGIN_NAMESPACE_VERSION
-
-namespace experimental
-{
-namespace filesystem
-{
-inline namespace v1
-{
-  /**
-   * @ingroup filesystem
-   */
-    inline std::string filesystem_error::_M_gen_what()
-    {
-      std::string __what = "filesystem error: ";
-      __what += system_error::what();
-      if (!_M_path1.empty())
-	  __what += " [" + _M_path1.string() + ']';
-      if (!_M_path2.empty())
-	  __what += " [" + _M_path2.string() + ']';
-      return __what;
-    }
-} // namespace v1
-} // namespace filesystem
-} // namespace experimental
-
-_GLIBCXX_END_NAMESPACE_VERSION
-} // namespace std
-
 #endif // C++11
 
 #endif // _GLIBCXX_EXPERIMENTAL_FILESYSTEM
diff --git a/libstdc++-v3/include/experimental/string_view b/libstdc++-v3/include/experimental/string_view
index f05f152302a..8eaf9ec3d96 100644
--- a/libstdc++-v3/include/experimental/string_view
+++ b/libstdc++-v3/include/experimental/string_view
@@ -645,22 +645,22 @@ namespace experimental
   inline namespace string_view_literals
   {
     inline constexpr basic_string_view<char>
-    operator""sv(const char* __str, size_t __len)
+    operator""sv(const char* __str, size_t __len) noexcept
     { return basic_string_view<char>{__str, __len}; }
 
 #ifdef _GLIBCXX_USE_WCHAR_T
     inline constexpr basic_string_view<wchar_t>
-    operator""sv(const wchar_t* __str, size_t __len)
+    operator""sv(const wchar_t* __str, size_t __len) noexcept
     { return basic_string_view<wchar_t>{__str, __len}; }
 #endif
 
 #ifdef _GLIBCXX_USE_C99_STDINT_TR1
     inline constexpr basic_string_view<char16_t>
-    operator""sv(const char16_t* __str, size_t __len)
+    operator""sv(const char16_t* __str, size_t __len) noexcept
     { return basic_string_view<char16_t>{__str, __len}; }
 
     inline constexpr basic_string_view<char32_t>
-    operator""sv(const char32_t* __str, size_t __len)
+    operator""sv(const char32_t* __str, size_t __len) noexcept
     { return basic_string_view<char32_t>{__str, __len}; }
 #endif
   } // namespace string_literals
diff --git a/libstdc++-v3/include/precompiled/stdc++.h b/libstdc++-v3/include/precompiled/stdc++.h
index b2993cd379f..4e1a71afeb0 100644
--- a/libstdc++-v3/include/precompiled/stdc++.h
+++ b/libstdc++-v3/include/precompiled/stdc++.h
@@ -122,6 +122,7 @@
 #include <shared_mutex>
 #endif
 
-#if __cplusplus > 201402L
+#if __cplusplus >= 201703L
 #include <charconv>
+#include <filesystem>
 #endif
diff --git a/libstdc++-v3/include/std/array b/libstdc++-v3/include/std/array
index 1c7d6dc4ab1..01f7100bae0 100644
--- a/libstdc++-v3/include/std/array
+++ b/libstdc++-v3/include/std/array
@@ -328,6 +328,14 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	_S_ref(__arr._M_elems, _Int);
     }
 
+  template<std::size_t _Int, typename _Tp, std::size_t _Nm>
+    constexpr const _Tp&&
+    get(const array<_Tp, _Nm>&& __arr) noexcept
+    {
+      static_assert(_Int < _Nm, "array index is within bounds");
+      return std::move(_GLIBCXX_STD_C::get<_Int>(__arr));
+    }
+
 _GLIBCXX_END_NAMESPACE_CONTAINER
 } // namespace std
 
@@ -339,7 +347,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   /// tuple_size
   template<typename _Tp>
-    class tuple_size;
+    struct tuple_size;
 
   /// Partial specialization for std::array
   template<typename _Tp, std::size_t _Nm>
@@ -348,7 +356,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   /// tuple_element
   template<std::size_t _Int, typename _Tp>
-    class tuple_element;
+    struct tuple_element;
 
   /// Partial specialization for std::array
   template<std::size_t _Int, typename _Tp, std::size_t _Nm>
diff --git a/libstdc++-v3/include/std/chrono b/libstdc++-v3/include/std/chrono
index fc058fcd8d8..9491508e637 100644
--- a/libstdc++-v3/include/std/chrono
+++ b/libstdc++-v3/include/std/chrono
@@ -214,8 +214,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
         treat_as_floating_point<_Rep>::value;
 #endif // C++17
 
-#if __cplusplus > 201402L
-# define __cpp_lib_chrono 201510
+#if __cplusplus >= 201703L
+# define __cpp_lib_chrono 201611
 
     template<typename _ToDur, typename _Rep, typename _Period>
       constexpr __enable_if_is_duration<_ToDur>
diff --git a/libstdc++-v3/include/std/filesystem b/libstdc++-v3/include/std/filesystem
new file mode 100644
index 00000000000..b09997704c1
--- /dev/null
+++ b/libstdc++-v3/include/std/filesystem
@@ -0,0 +1,45 @@
+// <filesystem> -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+/** @file filesystem
+ *  This is a Standard C++ Library header.
+ */
+
+#ifndef _GLIBCXX_FILESYSTEM
+#define _GLIBCXX_FILESYSTEM 1
+
+#pragma GCC system_header
+
+#if __cplusplus >= 201703L
+
+#include <bits/fs_fwd.h>
+#include <bits/fs_path.h>
+#include <bits/fs_dir.h>
+#include <bits/fs_ops.h>
+
+#define __cpp_lib_filesystem 201703
+
+#endif // C++17
+
+#endif // _GLIBCXX_FILESYSTEM
diff --git a/libstdc++-v3/include/std/fstream b/libstdc++-v3/include/std/fstream
index 52830945fe2..a3324c004d7 100644
--- a/libstdc++-v3/include/std/fstream
+++ b/libstdc++-v3/include/std/fstream
@@ -216,6 +216,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  }
       }
 
+#if __cplusplus >= 201703L
+      template<typename _Path, typename _Result = _Path, typename _Path2
+	       = decltype(std::declval<_Path&>().make_preferred().native())>
+	using _If_path = enable_if_t<is_same_v<_Path, _Path2>, _Result>;
+#endif // C++17
+
     public:
       // Constructors/destructor:
       /**
@@ -306,7 +312,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       __filebuf_type*
       open(const std::string& __s, ios_base::openmode __mode)
       { return open(__s.c_str(), __mode); }
-#endif
+
+#if __cplusplus >= 201703L
+      /**
+       *  @brief  Opens an external file.
+       *  @param  __s  The name of the file, as a filesystem::path.
+       *  @param  __mode  The open mode flags.
+       *  @return  @c this on success, NULL on failure
+       */
+      template<typename _Path>
+	_If_path<_Path, __filebuf_type*>
+	open(const _Path& __s, ios_base::openmode __mode)
+	{ return open(__s.c_str(), __mode); }
+#endif // C++17
+#endif // C++11
 
       /**
        *  @brief  Closes the currently associated file.
@@ -487,9 +506,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __mode  Open file in specified mode (see std::ios_base).
        *
        *  @c ios_base::in is automatically included in @a __mode.
-       *
-       *  Tip:  When using std::string to hold the filename, you must use
-       *  .c_str() before passing it to this constructor.
        */
       explicit
       basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in)
@@ -516,13 +532,29 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	this->open(__s, __mode);
       }
 
+#if __cplusplus >= 201703L
+      /**
+       *  @param  Create an input file stream.
+       *  @param  __s  filesystem::path specifying the filename.
+       *  @param  __mode  Open file in specified mode (see std::ios_base).
+       *
+       *  @c ios_base::in is automatically included in @a __mode.
+       */
+      template<typename _Path, typename = _Require<
+	  is_constructible<__filebuf_type, const _Path&, ios_base::openmode>>>
+	basic_ifstream(const _Path& __s,
+		       ios_base::openmode __mode = ios_base::in)
+	: basic_ifstream(__s.c_str(), __mode)
+	{ }
+#endif // C++17
+
       basic_ifstream(const basic_ifstream&) = delete;
 
       basic_ifstream(basic_ifstream&& __rhs)
       : __istream_type(std::move(__rhs)),
       _M_filebuf(std::move(__rhs._M_filebuf))
       { __istream_type::set_rdbuf(&_M_filebuf); }
-#endif
+#endif // C++11
 
       /**
        *  @brief  The destructor does nothing.
@@ -587,9 +619,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *
        *  Calls @c std::basic_filebuf::open(s,__mode|in).  If that function
        *  fails, @c failbit is set in the stream's error state.
-       *
-       *  Tip:  When using std::string to hold the filename, you must use
-       *  .c_str() before passing it to this constructor.
        */
       void
       open(const char* __s, ios_base::openmode __mode = ios_base::in)
@@ -621,7 +650,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  // 409. Closing an fstream should clear error state
 	  this->clear();
       }
-#endif
+
+#if __cplusplus >= 201703L
+      /**
+       *  @brief  Opens an external file.
+       *  @param  __s  The name of the file, as a filesystem::path.
+       *  @param  __mode  The open mode flags.
+       *
+       *  Calls @c std::basic_filebuf::open(__s,__mode|in).  If that function
+       *  fails, @c failbit is set in the stream's error state.
+       */
+      template<typename _Path>
+	auto
+	open(const _Path& __s, ios_base::openmode __mode = ios_base::in)
+	-> decltype(_M_filebuf.open(__s, __mode))
+	{ open(__s.c_str(), __mode); }
+#endif // C++17
+#endif // C++11
 
       /**
        *  @brief  Close the file.
@@ -687,15 +732,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __s  Null terminated string specifying the filename.
        *  @param  __mode  Open file in specified mode (see std::ios_base).
        *
-       *  @c ios_base::out | @c ios_base::trunc is automatically included in
-       *  @a __mode.
-       *
-       *  Tip:  When using std::string to hold the filename, you must use
-       *  .c_str() before passing it to this constructor.
+       *  @c ios_base::out is automatically included in @a __mode.
        */
       explicit
       basic_ofstream(const char* __s,
-		     ios_base::openmode __mode = ios_base::out|ios_base::trunc)
+		     ios_base::openmode __mode = ios_base::out)
       : __ostream_type(), _M_filebuf()
       {
 	this->init(&_M_filebuf);
@@ -708,18 +749,33 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __s  std::string specifying the filename.
        *  @param  __mode  Open file in specified mode (see std::ios_base).
        *
-       *  @c ios_base::out | @c ios_base::trunc is automatically included in
-       *  @a __mode.
+       *  @c ios_base::out is automatically included in @a __mode.
        */
       explicit
       basic_ofstream(const std::string& __s,
-		     ios_base::openmode __mode = ios_base::out|ios_base::trunc)
+		     ios_base::openmode __mode = ios_base::out)
       : __ostream_type(), _M_filebuf()
       {
 	this->init(&_M_filebuf);
 	this->open(__s, __mode);
       }
 
+#if __cplusplus >= 201703L
+      /**
+       *  @param  Create an output file stream.
+       *  @param  __s  filesystem::path specifying the filename.
+       *  @param  __mode  Open file in specified mode (see std::ios_base).
+       *
+       *  @c ios_base::out is automatically included in @a __mode.
+       */
+      template<typename _Path, typename = _Require<
+	  is_constructible<__filebuf_type, const _Path&, ios_base::openmode>>>
+	basic_ofstream(const _Path& __s,
+		       ios_base::openmode __mode = ios_base::out)
+	: basic_ofstream(__s.c_str(), __mode)
+	{ }
+#endif // C++17
+
       basic_ofstream(const basic_ofstream&) = delete;
 
       basic_ofstream(basic_ofstream&& __rhs)
@@ -789,15 +845,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __s  The name of the file.
        *  @param  __mode  The open mode flags.
        *
-       *  Calls @c std::basic_filebuf::open(__s,__mode|out|trunc).  If that
+       *  Calls @c std::basic_filebuf::open(__s,__mode|out).  If that
        *  function fails, @c failbit is set in the stream's error state.
-       *
-       *  Tip:  When using std::string to hold the filename, you must use
-       *  .c_str() before passing it to this constructor.
        */
       void
-      open(const char* __s,
-	   ios_base::openmode __mode = ios_base::out | ios_base::trunc)
+      open(const char* __s, ios_base::openmode __mode = ios_base::out)
       {
 	if (!_M_filebuf.open(__s, __mode | ios_base::out))
 	  this->setstate(ios_base::failbit);
@@ -813,12 +865,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __s  The name of the file.
        *  @param  __mode  The open mode flags.
        *
-       *  Calls @c std::basic_filebuf::open(s,mode|out|trunc).  If that
+       *  Calls @c std::basic_filebuf::open(s,mode|out).  If that
        *  function fails, @c failbit is set in the stream's error state.
        */
       void
-      open(const std::string& __s,
-	   ios_base::openmode __mode = ios_base::out | ios_base::trunc)
+      open(const std::string& __s, ios_base::openmode __mode = ios_base::out)
       {
 	if (!_M_filebuf.open(__s, __mode | ios_base::out))
 	  this->setstate(ios_base::failbit);
@@ -827,7 +878,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  // 409. Closing an fstream should clear error state
 	  this->clear();
       }
-#endif
+
+#if __cplusplus >= 201703L
+      /**
+       *  @brief  Opens an external file.
+       *  @param  __s  The name of the file, as a filesystem::path.
+       *  @param  __mode  The open mode flags.
+       *
+       *  Calls @c std::basic_filebuf::open(__s,__mode|out).  If that
+       *  function fails, @c failbit is set in the stream's error state.
+       */
+      template<typename _Path>
+	auto
+	open(const _Path& __s, ios_base::openmode __mode = ios_base::out)
+	-> decltype(_M_filebuf.open(__s, __mode))
+	{ open(__s.c_str(), __mode); }
+#endif // C++17
+#endif // C++11
 
       /**
        *  @brief  Close the file.
@@ -894,9 +961,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @brief  Create an input/output file stream.
        *  @param  __s  Null terminated string specifying the filename.
        *  @param  __mode  Open file in specified mode (see std::ios_base).
-       *
-       *  Tip:  When using std::string to hold the filename, you must use
-       *  .c_str() before passing it to this constructor.
        */
       explicit
       basic_fstream(const char* __s,
@@ -922,6 +986,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	this->open(__s, __mode);
       }
 
+#if __cplusplus >= 201703L
+      /**
+       *  @param  Create an input/output file stream.
+       *  @param  __s  filesystem::path specifying the filename.
+       *  @param  __mode  Open file in specified mode (see std::ios_base).
+       */
+      template<typename _Path, typename = _Require<
+	  is_constructible<__filebuf_type, const _Path&, ios_base::openmode>>>
+	basic_fstream(const _Path& __s,
+		      ios_base::openmode __mode = ios_base::in | ios_base::out)
+	: basic_fstream(__s.c_str(), __mode)
+	{ }
+#endif // C++17
+
       basic_fstream(const basic_fstream&) = delete;
 
       basic_fstream(basic_fstream&& __rhs)
@@ -993,9 +1071,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *
        *  Calls @c std::basic_filebuf::open(__s,__mode).  If that
        *  function fails, @c failbit is set in the stream's error state.
-       *
-       *  Tip:  When using std::string to hold the filename, you must use
-       *  .c_str() before passing it to this constructor.
        */
       void
       open(const char* __s,
@@ -1029,7 +1104,24 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  // 409. Closing an fstream should clear error state
 	  this->clear();
       }
-#endif
+
+#if __cplusplus >= 201703L
+      /**
+       *  @brief  Opens an external file.
+       *  @param  __s  The name of the file, as a filesystem::path.
+       *  @param  __mode  The open mode flags.
+       *
+       *  Calls @c std::basic_filebuf::open(__s,__mode).  If that
+       *  function fails, @c failbit is set in the stream's error state.
+       */
+      template<typename _Path>
+	auto
+	open(const _Path& __s,
+	     ios_base::openmode __mode = ios_base::in | ios_base::out)
+	-> decltype(_M_filebuf.open(__s, __mode))
+	{ open(__s.c_str(), __mode); }
+#endif // C++17
+#endif // C++11
 
       /**
        *  @brief  Close the file.
diff --git a/libstdc++-v3/include/std/mutex b/libstdc++-v3/include/std/mutex
index 8c692a88ffd..50420ee22d4 100644
--- a/libstdc++-v3/include/std/mutex
+++ b/libstdc++-v3/include/std/mutex
@@ -688,6 +688,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
         __set_once_functor_lock_ptr(0);
 #endif
 
+#ifdef __clang_analyzer__
+      // PR libstdc++/82481
+      __once_callable = nullptr;
+      __once_call = nullptr;
+#endif
+
       if (__e)
 	__throw_system_error(__e);
     }
diff --git a/libstdc++-v3/include/std/string_view b/libstdc++-v3/include/std/string_view
index 97316ef49ab..1900b867841 100644
--- a/libstdc++-v3/include/std/string_view
+++ b/libstdc++-v3/include/std/string_view
@@ -96,14 +96,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       constexpr basic_string_view(const basic_string_view&) noexcept = default;
 
-      constexpr basic_string_view(const _CharT* __str)
+      constexpr basic_string_view(const _CharT* __str) noexcept
       : _M_len{__str == nullptr ? 0 : traits_type::length(__str)},
 	_M_str{__str}
       { }
 
-      constexpr basic_string_view(const _CharT* __str, size_type __len)
-      : _M_len{__len},
-        _M_str{__str}
+      constexpr
+      basic_string_view(const _CharT* __str, size_type __len) noexcept
+      : _M_len{__len}, _M_str{__str}
       { }
 
       constexpr basic_string_view&
@@ -177,17 +177,15 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       constexpr const _CharT&
       at(size_type __pos) const
       {
-	return __pos < this->_M_len
-	     ? *(this->_M_str + __pos)
-	     : (__throw_out_of_range_fmt(__N("basic_string_view::at: __pos "
-					     "(which is %zu) >= this->size() "
-					     "(which is %zu)"),
-					 __pos, this->size()),
-		*this->_M_str);
+	if (__pos >= _M_len)
+	  __throw_out_of_range_fmt(__N("basic_string_view::at: __pos "
+				       "(which is %zu) >= this->size() "
+				       "(which is %zu)"), __pos, this->size());
+	return *(this->_M_str + __pos);
       }
 
       constexpr const _CharT&
-      front() const
+      front() const noexcept
       {
 	// TODO: Assert to restore in a way compatible with the constexpr.
 	// __glibcxx_assert(this->_M_len > 0);
@@ -195,7 +193,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       }
 
       constexpr const _CharT&
-      back() const
+      back() const noexcept
       {
 	// TODO: Assert to restore in a way compatible with the constexpr.
 	// __glibcxx_assert(this->_M_len > 0);
@@ -209,7 +207,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       // [string.view.modifiers], modifiers:
 
       constexpr void
-      remove_prefix(size_type __n)
+      remove_prefix(size_type __n) noexcept
       {
 	__glibcxx_assert(this->_M_len >= __n);
 	this->_M_str += __n;
@@ -217,7 +215,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       }
 
       constexpr void
-      remove_suffix(size_type __n)
+      remove_suffix(size_type __n) noexcept
       { this->_M_len -= __n; }
 
       constexpr void
@@ -235,38 +233,27 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       copy(_CharT* __str, size_type __n, size_type __pos = 0) const
       {
 	__glibcxx_requires_string_len(__str, __n);
-	if (__pos > this->_M_len)
-	  __throw_out_of_range_fmt(__N("basic_string_view::copy: __pos "
-				       "(which is %zu) > this->size() "
-				       "(which is %zu)"),
-				   __pos, this->size());
-	size_type __rlen{std::min(__n, size_type{this->_M_len  - __pos})};
+	__pos = _M_check(__pos, "basic_string_view::copy");
+	const size_type __rlen = std::min(__n, _M_len - __pos);
 	for (auto __begin = this->_M_str + __pos,
 	     __end = __begin + __rlen; __begin != __end;)
 	  *__str++ = *__begin++;
 	return __rlen;
       }
 
-
-      // [string.view.ops], string operations:
-
       constexpr basic_string_view
-      substr(size_type __pos, size_type __n=npos) const
+      substr(size_type __pos, size_type __n = npos) const noexcept(false)
       {
-	return __pos <= this->_M_len
-	     ? basic_string_view{this->_M_str + __pos,
-				std::min(__n, size_type{this->_M_len  - __pos})}
-	     : (__throw_out_of_range_fmt(__N("basic_string_view::substr: __pos "
-					     "(which is %zu) > this->size() "
-					     "(which is %zu)"),
-				     __pos, this->size()), basic_string_view{});
+	__pos = _M_check(__pos, "basic_string_view::substr");
+	const size_type __rlen = std::min(__n, _M_len - __pos);
+	return basic_string_view{_M_str + __pos, __rlen};
       }
 
       constexpr int
       compare(basic_string_view __str) const noexcept
       {
-	int __ret = traits_type::compare(this->_M_str, __str._M_str,
-					 std::min(this->_M_len, __str._M_len));
+	const size_type __rlen = std::min(this->_M_len, __str._M_len);
+	int __ret = traits_type::compare(this->_M_str, __str._M_str, __rlen);
 	if (__ret == 0)
 	  __ret = _S_compare(this->_M_len, __str._M_len);
 	return __ret;
@@ -279,7 +266,9 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       constexpr int
       compare(size_type __pos1, size_type __n1,
 	      basic_string_view __str, size_type __pos2, size_type __n2) const
-      { return this->substr(__pos1, __n1).compare(__str.substr(__pos2, __n2)); }
+      {
+	return this->substr(__pos1, __n1).compare(__str.substr(__pos2, __n2));
+      }
 
       constexpr int
       compare(const _CharT* __str) const noexcept
@@ -291,7 +280,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       constexpr int
       compare(size_type __pos1, size_type __n1,
-	      const _CharT* __str, size_type __n2) const
+	      const _CharT* __str, size_type __n2) const noexcept(false)
       {
 	return this->substr(__pos1, __n1)
 		   .compare(basic_string_view(__str, __n2));
@@ -302,13 +291,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       { return this->find(__str._M_str, __pos, __str._M_len); }
 
       constexpr size_type
-      find(_CharT __c, size_type __pos=0) const noexcept;
+      find(_CharT __c, size_type __pos = 0) const noexcept;
 
       constexpr size_type
       find(const _CharT* __str, size_type __pos, size_type __n) const noexcept;
 
       constexpr size_type
-      find(const _CharT* __str, size_type __pos=0) const noexcept
+      find(const _CharT* __str, size_type __pos = 0) const noexcept
       { return this->find(__str, __pos, traits_type::length(__str)); }
 
       constexpr size_type
@@ -350,7 +339,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       { return this->rfind(__c, __pos); }
 
       constexpr size_type
-      find_last_of(const _CharT* __str, size_type __pos, size_type __n) const;
+      find_last_of(const _CharT* __str, size_type __pos,
+		   size_type __n) const noexcept;
 
       constexpr size_type
       find_last_of(const _CharT* __str, size_type __pos = npos) const noexcept
@@ -366,7 +356,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       constexpr size_type
       find_first_not_of(const _CharT* __str,
-			size_type __pos, size_type __n) const;
+			size_type __pos, size_type __n) const noexcept;
 
       constexpr size_type
       find_first_not_of(const _CharT* __str, size_type __pos = 0) const noexcept
@@ -385,7 +375,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       constexpr size_type
       find_last_not_of(const _CharT* __str,
-		       size_type __pos, size_type __n) const;
+		       size_type __pos, size_type __n) const noexcept;
 
       constexpr size_type
       find_last_not_of(const _CharT* __str,
@@ -396,7 +386,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       }
 
       constexpr size_type
-      _M_check(size_type __pos, const char* __s) const
+      _M_check(size_type __pos, const char* __s) const noexcept(false)
       {
 	if (__pos > this->size())
 	  __throw_out_of_range_fmt(__N("%s: __pos (which is %zu) > "
@@ -407,7 +397,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       // NB: _M_limit doesn't check for a bad __pos value.
       constexpr size_type
-      _M_limit(size_type __pos, size_type __off) const _GLIBCXX_NOEXCEPT
+      _M_limit(size_type __pos, size_type __off) const noexcept
       {
 	const bool __testoff =  __off < this->size() - __pos;
 	return __testoff ? __off : this->size() - __pos;
@@ -418,11 +408,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       static constexpr int
       _S_compare(size_type __n1, size_type __n2) noexcept
       {
-	return difference_type{__n1 - __n2} > std::numeric_limits<int>::max()
-	     ? std::numeric_limits<int>::max()
-	     : difference_type{__n1 - __n2} < std::numeric_limits<int>::min()
-	     ? std::numeric_limits<int>::min()
-	     : static_cast<int>(difference_type{__n1 - __n2});
+	const difference_type __diff{__n1 - __n2};
+	if (__diff > std::numeric_limits<int>::max())
+	  return std::numeric_limits<int>::max();
+	if (__diff < std::numeric_limits<int>::min())
+	  return std::numeric_limits<int>::min();
+	return static_cast<int>(__diff);
       }
 
       size_t	    _M_len;
@@ -636,22 +627,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   inline namespace string_view_literals
   {
     inline constexpr basic_string_view<char>
-    operator""sv(const char* __str, size_t __len)
+    operator""sv(const char* __str, size_t __len) noexcept
     { return basic_string_view<char>{__str, __len}; }
 
 #ifdef _GLIBCXX_USE_WCHAR_T
     inline constexpr basic_string_view<wchar_t>
-    operator""sv(const wchar_t* __str, size_t __len)
+    operator""sv(const wchar_t* __str, size_t __len) noexcept
     { return basic_string_view<wchar_t>{__str, __len}; }
 #endif
 
 #ifdef _GLIBCXX_USE_C99_STDINT_TR1
     inline constexpr basic_string_view<char16_t>
-    operator""sv(const char16_t* __str, size_t __len)
+    operator""sv(const char16_t* __str, size_t __len) noexcept
     { return basic_string_view<char16_t>{__str, __len}; }
 
     inline constexpr basic_string_view<char32_t>
-    operator""sv(const char32_t* __str, size_t __len)
+    operator""sv(const char32_t* __str, size_t __len) noexcept
     { return basic_string_view<char32_t>{__str, __len}; }
 #endif
   } // namespace string_literals
diff --git a/libstdc++-v3/include/std/tuple b/libstdc++-v3/include/std/tuple
index 1f5365ad026..ac03c9ec402 100644
--- a/libstdc++-v3/include/std/tuple
+++ b/libstdc++-v3/include/std/tuple
@@ -1329,6 +1329,15 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       return std::forward<__element_type&&>(std::get<__i>(__t));
     }
 
+  /// Return a const rvalue reference to the ith element of a const tuple rvalue.
+  template<std::size_t __i, typename... _Elements>
+    constexpr const __tuple_element_t<__i, tuple<_Elements...>>&&
+    get(const tuple<_Elements...>&& __t) noexcept
+    {
+      typedef __tuple_element_t<__i, tuple<_Elements...>> __element_type;
+      return std::forward<const __element_type&&>(std::get<__i>(__t));
+    }
+
 #if __cplusplus > 201103L
 
 #define __cpp_lib_tuples_by_type 201304
@@ -1360,6 +1369,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     constexpr const _Tp&
     get(const tuple<_Types...>& __t) noexcept
     { return std::__get_helper2<_Tp>(__t); }
+
+  /// Return a const reference to the unique element of type _Tp of
+  /// a const tuple rvalue.
+  template <typename _Tp, typename... _Types>
+    constexpr const _Tp&&
+    get(const tuple<_Types...>&& __t) noexcept
+    { return std::forward<const _Tp&&>(std::__get_helper2<_Tp>(__t)); }
 #endif
 
   // This class performs the comparison operations on tuples
diff --git a/libstdc++-v3/include/std/type_traits b/libstdc++-v3/include/std/type_traits
index 036f7667bd8..7eca08c1a50 100644
--- a/libstdc++-v3/include/std/type_traits
+++ b/libstdc++-v3/include/std/type_traits
@@ -2664,7 +2664,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     void operator=(__nonesuch const&) = delete;
   };
 
-#if __cplusplus > 201402L
+#if __cplusplus >= 201703L
 # define __cpp_lib_is_invocable 201703
 
   /// std::invoke_result
@@ -2739,7 +2739,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       = is_nothrow_invocable_r<_Fn, _Args...>::value;
 #endif // C++17
 
-#if __cplusplus > 201402L
+#if __cplusplus >= 201703L
 # define __cpp_lib_type_trait_variable_templates 201510L
 template <typename _Tp>
   inline constexpr bool is_void_v = is_void<_Tp>::value;
@@ -2943,6 +2943,16 @@ template <typename _From, typename _To>
 
 #endif // C++17
 
+#if __cplusplus > 201703L
+  /// Byte order
+  enum class endian
+  {
+    little = __ORDER_LITTLE_ENDIAN__,
+    big    = __ORDER_BIG_ENDIAN__,
+    native = __BYTE_ORDER__
+  };
+#endif // C++2a
+
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
diff --git a/libstdc++-v3/include/std/unordered_map b/libstdc++-v3/include/std/unordered_map
index 2cdcd377936..80773235b65 100644
--- a/libstdc++-v3/include/std/unordered_map
+++ b/libstdc++-v3/include/std/unordered_map
@@ -35,13 +35,12 @@
 # include <bits/c++0x_warning.h>
 #else
 
-#include <utility>
 #include <type_traits>
 #include <initializer_list>
-#include <tuple>
 #include <bits/allocator.h>
 #include <ext/alloc_traits.h>
 #include <ext/aligned_buffer.h>
+#include <bits/stl_pair.h>
 #include <bits/stl_function.h> // equal_to, _Identity, _Select1st
 #include <bits/functional_hash.h>
 #include <bits/hashtable.h>
diff --git a/libstdc++-v3/include/std/unordered_set b/libstdc++-v3/include/std/unordered_set
index 2646c0f2f00..faf7ebe23ae 100644
--- a/libstdc++-v3/include/std/unordered_set
+++ b/libstdc++-v3/include/std/unordered_set
@@ -35,13 +35,12 @@
 # include <bits/c++0x_warning.h>
 #else
 
-#include <utility>
 #include <type_traits>
 #include <initializer_list>
-#include <tuple>
 #include <bits/allocator.h>
 #include <ext/alloc_traits.h>
 #include <ext/aligned_buffer.h>
+#include <bits/stl_pair.h>
 #include <bits/stl_function.h> // equal_to, _Identity, _Select1st
 #include <bits/functional_hash.h>
 #include <bits/hashtable.h>
diff --git a/libstdc++-v3/include/std/utility b/libstdc++-v3/include/std/utility
index 29a626004f9..e7386320e2a 100644
--- a/libstdc++-v3/include/std/utility
+++ b/libstdc++-v3/include/std/utility
@@ -184,6 +184,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
         static constexpr const _Tp1&
         __const_get(const std::pair<_Tp1, _Tp2>& __pair) noexcept
         { return __pair.first; }
+
+      template<typename _Tp1, typename _Tp2>
+        static constexpr const _Tp1&&
+        __const_move_get(const std::pair<_Tp1, _Tp2>&& __pair) noexcept
+        { return std::forward<const _Tp1>(__pair.first); }
     };
 
   template<>
@@ -203,6 +208,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
         static constexpr const _Tp2&
         __const_get(const std::pair<_Tp1, _Tp2>& __pair) noexcept
         { return __pair.second; }
+
+      template<typename _Tp1, typename _Tp2>
+        static constexpr const _Tp2&&
+        __const_move_get(const std::pair<_Tp1, _Tp2>&& __pair) noexcept
+        { return std::forward<const _Tp2>(__pair.second); }
     };
 
   template<std::size_t _Int, class _Tp1, class _Tp2>
@@ -220,6 +230,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     get(const std::pair<_Tp1, _Tp2>& __in) noexcept
     { return __pair_get<_Int>::__const_get(__in); }
 
+  template<std::size_t _Int, class _Tp1, class _Tp2>
+    constexpr const typename tuple_element<_Int, std::pair<_Tp1, _Tp2>>::type&&
+    get(const std::pair<_Tp1, _Tp2>&& __in) noexcept
+    { return __pair_get<_Int>::__const_move_get(std::move(__in)); }
+
 #if __cplusplus > 201103L
 
 #define __cpp_lib_tuples_by_type 201304
@@ -240,6 +255,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     { return std::move(__p.first); }
 
   template <typename _Tp, typename _Up>
+    constexpr const _Tp&&
+    get(const pair<_Tp, _Up>&& __p) noexcept
+    { return std::move(__p.first); }
+
+  template <typename _Tp, typename _Up>
     constexpr _Tp&
     get(pair<_Up, _Tp>& __p) noexcept
     { return __p.second; }
@@ -254,6 +274,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     get(pair<_Up, _Tp>&& __p) noexcept
     { return std::move(__p.second); }
 
+  template <typename _Tp, typename _Up>
+    constexpr const _Tp&&
+    get(const pair<_Up, _Tp>&& __p) noexcept
+    { return std::move(__p.second); }
+
 #define __cpp_lib_exchange_function 201304
 
   /// Assign @p __new_val to @p __obj and return its previous value.
diff --git a/libstdc++-v3/python/libstdcxx/v6/printers.py b/libstdc++-v3/python/libstdcxx/v6/printers.py
index 04f413adf04..a0f5ede682c 100644
--- a/libstdc++-v3/python/libstdcxx/v6/printers.py
+++ b/libstdc++-v3/python/libstdcxx/v6/printers.py
@@ -973,8 +973,8 @@ class StdExpAnyPrinter(SingleObjContainerPrinter):
     "Print a std::any or std::experimental::any"
 
     def __init__ (self, typename, val):
-        self.typename = re.sub('^std::experimental::fundamentals_v\d::', 'std::experimental::', typename, 1)
-        self.typename = strip_versioned_namespace(self.typename)
+        self.typename = strip_versioned_namespace(typename)
+        self.typename = re.sub('^std::experimental::fundamentals_v\d::', 'std::experimental::', self.typename, 1)
         self.val = val
         self.contained_type = None
         contained_value = None
@@ -1021,8 +1021,8 @@ class StdExpOptionalPrinter(SingleObjContainerPrinter):
 
     def __init__ (self, typename, val):
         valtype = self._recognize (val.type.template_argument(0))
-        self.typename = re.sub('^std::(experimental::|)(fundamentals_v\d::|)(.*)', r'std::\1\3<%s>' % valtype, typename, 1)
-        self.typename = strip_versioned_namespace(self.typename)
+        self.typename = strip_versioned_namespace(typename)
+        self.typename = re.sub('^std::(experimental::|)(fundamentals_v\d::|)(.*)', r'std::\1\3<%s>' % valtype, self.typename, 1)
         if not self.typename.startswith('std::experimental'):
             val = val['_M_payload']
         self.val = val
@@ -1043,8 +1043,8 @@ class StdVariantPrinter(SingleObjContainerPrinter):
 
     def __init__(self, typename, val):
         alternatives = self._template_args(val)
-        self.typename = "%s<%s>" % (typename, ', '.join([self._recognize(alt) for alt in alternatives]))
-        self.typename = strip_versioned_namespace(self.typename)
+        self.typename = strip_versioned_namespace(typename)
+        self.typename = "%s<%s>" % (self.typename, ', '.join([self._recognize(alt) for alt in alternatives]))
         self.index = val['_M_index']
         if self.index >= len(alternatives):
             self.contained_type = None
@@ -1227,7 +1227,8 @@ class Printer(object):
     def add_version(self, base, name, function):
         self.add(base + name, function)
         if _versioned_namespace:
-            self.add(base + _versioned_namespace + name, function)
+            vbase = re.sub('^(std|__gnu_cxx)::', r'\g<0>%s' % _versioned_namespace, base)
+            self.add(vbase + name, function)
 
     # Add a name using _GLIBCXX_BEGIN_NAMESPACE_CONTAINER.
     def add_container(self, base, name, function):
@@ -1507,7 +1508,7 @@ def build_libstdcxx_dictionary ():
     # In order from:
     # http://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen/a01847.html
     libstdcxx_printer.add_version('std::', 'basic_string', StdStringPrinter)
-    libstdcxx_printer.add_version('std::', '__cxx11::basic_string', StdStringPrinter)
+    libstdcxx_printer.add_version('std::__cxx11::', 'basic_string', StdStringPrinter)
     libstdcxx_printer.add_container('std::', 'bitset', StdBitsetPrinter)
     libstdcxx_printer.add_container('std::', 'deque', StdDequePrinter)
     libstdcxx_printer.add_container('std::', 'list', StdListPrinter)
@@ -1555,15 +1556,15 @@ def build_libstdcxx_dictionary ():
     libstdcxx_printer.add_container('std::', 'forward_list',
                                     StdForwardListPrinter)
 
-    libstdcxx_printer.add_version('std::', 'tr1::shared_ptr', SharedPointerPrinter)
-    libstdcxx_printer.add_version('std::', 'tr1::weak_ptr', SharedPointerPrinter)
-    libstdcxx_printer.add_version('std::', 'tr1::unordered_map',
+    libstdcxx_printer.add_version('std::tr1::', 'shared_ptr', SharedPointerPrinter)
+    libstdcxx_printer.add_version('std::tr1::', 'weak_ptr', SharedPointerPrinter)
+    libstdcxx_printer.add_version('std::tr1::', 'unordered_map',
                                   Tr1UnorderedMapPrinter)
-    libstdcxx_printer.add_version('std::', 'tr1::unordered_set',
+    libstdcxx_printer.add_version('std::tr1::', 'unordered_set',
                                   Tr1UnorderedSetPrinter)
-    libstdcxx_printer.add_version('std::', 'tr1::unordered_multimap',
+    libstdcxx_printer.add_version('std::tr1::', 'unordered_multimap',
                                   Tr1UnorderedMapPrinter)
-    libstdcxx_printer.add_version('std::', 'tr1::unordered_multiset',
+    libstdcxx_printer.add_version('std::tr1::', 'unordered_multiset',
                                   Tr1UnorderedSetPrinter)
 
     # These are the C++11 printer registrations for -D_GLIBCXX_DEBUG cases.
@@ -1592,6 +1593,10 @@ def build_libstdcxx_dictionary ():
                                   'path', StdExpPathPrinter)
     libstdcxx_printer.add_version('std::experimental::filesystem::v1::__cxx11::',
                                   'path', StdExpPathPrinter)
+    libstdcxx_printer.add_version('std::filesystem::',
+                                  'path', StdExpPathPrinter)
+    libstdcxx_printer.add_version('std::filesystem::__cxx11::',
+                                  'path', StdExpPathPrinter)
 
     # C++17 components
     libstdcxx_printer.add_version('std::',
diff --git a/libstdc++-v3/src/c++11/istream-inst.cc b/libstdc++-v3/src/c++11/istream-inst.cc
index 96053044910..f5a9666b18c 100644
--- a/libstdc++-v3/src/c++11/istream-inst.cc
+++ b/libstdc++-v3/src/c++11/istream-inst.cc
@@ -97,6 +97,8 @@ _GLIBCXX_END_NAMESPACE_VERSION
 // XXX GLIBCXX_ABI Deprecated
 #ifdef _GLIBCXX_LONG_DOUBLE_COMPAT
 
+#pragma GCC diagnostic ignored "-Wattribute-alias"
+
 #define _GLIBCXX_LDBL_COMPAT(dbl, ldbl) \
   extern "C" void ldbl (void) __attribute__ ((alias (#dbl), weak))
 _GLIBCXX_LDBL_COMPAT (_ZNSirsERd, _ZNSirsERe);
diff --git a/libstdc++-v3/src/c++11/locale-inst.cc b/libstdc++-v3/src/c++11/locale-inst.cc
index b0f79ffd619..c1b8e8d2b6b 100644
--- a/libstdc++-v3/src/c++11/locale-inst.cc
+++ b/libstdc++-v3/src/c++11/locale-inst.cc
@@ -383,6 +383,8 @@ _GLIBCXX_END_NAMESPACE_VERSION
 #if defined _GLIBCXX_LONG_DOUBLE_COMPAT && defined C_is_char \
       && _GLIBCXX_USE_CXX11_ABI == 0
 
+#pragma GCC diagnostic ignored "-Wattribute-alias"
+
 #define _GLIBCXX_LDBL_COMPAT(dbl, ldbl) \
   extern "C" void ldbl (void) __attribute__ ((alias (#dbl), weak))
 
diff --git a/libstdc++-v3/src/c++11/ostream-inst.cc b/libstdc++-v3/src/c++11/ostream-inst.cc
index ef0038b51f2..12ddc8de02a 100644
--- a/libstdc++-v3/src/c++11/ostream-inst.cc
+++ b/libstdc++-v3/src/c++11/ostream-inst.cc
@@ -100,6 +100,8 @@ _GLIBCXX_END_NAMESPACE_VERSION
 // XXX GLIBCXX_ABI Deprecated
 #ifdef _GLIBCXX_LONG_DOUBLE_COMPAT
 
+#pragma GCC diagnostic ignored "-Wattribute-alias"
+
 #define _GLIBCXX_LDBL_COMPAT(dbl, ldbl) \
   extern "C" void ldbl (void) __attribute__ ((alias (#dbl), weak))
 _GLIBCXX_LDBL_COMPAT (_ZNSolsEd, _ZNSolsEe);
diff --git a/libstdc++-v3/src/c++11/wlocale-inst.cc b/libstdc++-v3/src/c++11/wlocale-inst.cc
index 26fdf348c13..5a565cb190b 100644
--- a/libstdc++-v3/src/c++11/wlocale-inst.cc
+++ b/libstdc++-v3/src/c++11/wlocale-inst.cc
@@ -37,6 +37,8 @@
 // XXX GLIBCXX_ABI Deprecated
 #if defined _GLIBCXX_LONG_DOUBLE_COMPAT
 
+#pragma GCC diagnostic ignored "-Wattribute-alias"
+
 #define _GLIBCXX_LDBL_COMPAT(dbl, ldbl) \
   extern "C" void ldbl (void) __attribute__ ((alias (#dbl), weak))
 
diff --git a/libstdc++-v3/src/c++98/hash-long-double-tr1-aux.cc b/libstdc++-v3/src/c++98/hash-long-double-tr1-aux.cc
index 28ee175ab0a..4612a44f04c 100644
--- a/libstdc++-v3/src/c++98/hash-long-double-tr1-aux.cc
+++ b/libstdc++-v3/src/c++98/hash-long-double-tr1-aux.cc
@@ -22,6 +22,8 @@
 // see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 // <http://www.gnu.org/licenses/>.
 
+#pragma GCC diagnostic ignored "-Wattribute-alias"
+
 namespace std _GLIBCXX_VISIBILITY(default)
 {
   namespace tr1
diff --git a/libstdc++-v3/src/filesystem/Makefile.am b/libstdc++-v3/src/filesystem/Makefile.am
index 836bdd390a7..19de2a324fa 100644
--- a/libstdc++-v3/src/filesystem/Makefile.am
+++ b/libstdc++-v3/src/filesystem/Makefile.am
@@ -30,7 +30,10 @@ if ENABLE_DUAL_ABI
 cxx11_abi_sources = \
 	cow-dir.cc \
 	cow-ops.cc \
-	cow-path.cc
+	cow-path.cc \
+	cow-std-dir.cc \
+	cow-std-ops.cc \
+	cow-std-path.cc
 else
 cxx11_abi_sources =
 endif
@@ -39,6 +42,9 @@ sources = \
 	dir.cc \
 	ops.cc \
 	path.cc \
+	std-dir.cc \
+	std-ops.cc \
+	std-path.cc \
 	${cxx11_abi_sources}
 
 # vpath % $(top_srcdir)/src/filesystem
@@ -52,7 +58,7 @@ libstdc__fs_la_SOURCES = $(sources)
 # as the occasion call for it.
 AM_CXXFLAGS = \
 	$(glibcxx_lt_pic_flag) $(glibcxx_compiler_shared_flag) \
-	-std=gnu++14 \
+	-std=gnu++17 \
 	$(WARN_CXXFLAGS) $(OPTIMIZE_CXXFLAGS)  $(CONFIG_CXXFLAGS)
 
 AM_MAKEFLAGS = \
diff --git a/libstdc++-v3/src/filesystem/Makefile.in b/libstdc++-v3/src/filesystem/Makefile.in
index a4fdf790ae2..847b19b6982 100644
--- a/libstdc++-v3/src/filesystem/Makefile.in
+++ b/libstdc++-v3/src/filesystem/Makefile.in
@@ -114,8 +114,10 @@ am__installdirs = "$(DESTDIR)$(toolexeclibdir)"
 LTLIBRARIES = $(toolexeclib_LTLIBRARIES)
 libstdc__fs_la_LIBADD =
 @ENABLE_DUAL_ABI_TRUE@am__objects_1 = cow-dir.lo cow-ops.lo \
-@ENABLE_DUAL_ABI_TRUE@	cow-path.lo
-am__objects_2 = dir.lo ops.lo path.lo $(am__objects_1)
+@ENABLE_DUAL_ABI_TRUE@	cow-path.lo cow-std-dir.lo \
+@ENABLE_DUAL_ABI_TRUE@	cow-std-ops.lo cow-std-path.lo
+am__objects_2 = dir.lo ops.lo path.lo std-dir.lo std-ops.lo \
+	std-path.lo $(am__objects_1)
 am_libstdc__fs_la_OBJECTS = $(am__objects_2)
 libstdc__fs_la_OBJECTS = $(am_libstdc__fs_la_OBJECTS)
 DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
@@ -362,12 +364,18 @@ headers =
 @ENABLE_DUAL_ABI_TRUE@cxx11_abi_sources = \
 @ENABLE_DUAL_ABI_TRUE@	cow-dir.cc \
 @ENABLE_DUAL_ABI_TRUE@	cow-ops.cc \
-@ENABLE_DUAL_ABI_TRUE@	cow-path.cc
+@ENABLE_DUAL_ABI_TRUE@	cow-path.cc \
+@ENABLE_DUAL_ABI_TRUE@	cow-std-dir.cc \
+@ENABLE_DUAL_ABI_TRUE@	cow-std-ops.cc \
+@ENABLE_DUAL_ABI_TRUE@	cow-std-path.cc
 
 sources = \
 	dir.cc \
 	ops.cc \
 	path.cc \
+	std-dir.cc \
+	std-ops.cc \
+	std-path.cc \
 	${cxx11_abi_sources}
 
 
@@ -381,7 +389,7 @@ libstdc__fs_la_SOURCES = $(sources)
 # as the occasion call for it.
 AM_CXXFLAGS = \
 	$(glibcxx_lt_pic_flag) $(glibcxx_compiler_shared_flag) \
-	-std=gnu++14 \
+	-std=gnu++17 \
 	$(WARN_CXXFLAGS) $(OPTIMIZE_CXXFLAGS)  $(CONFIG_CXXFLAGS)
 
 AM_MAKEFLAGS = \
diff --git a/libstdc++-v3/src/filesystem/cow-dir.cc b/libstdc++-v3/src/filesystem/cow-dir.cc
index 4e47bbdd7c8..96907abd712 100644
--- a/libstdc++-v3/src/filesystem/cow-dir.cc
+++ b/libstdc++-v3/src/filesystem/cow-dir.cc
@@ -1,4 +1,4 @@
-// Class filesystem::directory_entry etc. -*- C++ -*-
+// Class experimental::filesystem::directory_entry etc. -*- C++ -*-
 
 // Copyright (C) 2015-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/src/filesystem/cow-ops.cc b/libstdc++-v3/src/filesystem/cow-ops.cc
index 9ad91f0f11a..3b41d7c98ac 100644
--- a/libstdc++-v3/src/filesystem/cow-ops.cc
+++ b/libstdc++-v3/src/filesystem/cow-ops.cc
@@ -1,4 +1,4 @@
-// Filesystem operations -*- C++ -*-
+// Filesystem TS operations -*- C++ -*-
 
 // Copyright (C) 2015-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/src/filesystem/cow-path.cc b/libstdc++-v3/src/filesystem/cow-path.cc
index b216088ca32..0817b0ae491 100644
--- a/libstdc++-v3/src/filesystem/cow-path.cc
+++ b/libstdc++-v3/src/filesystem/cow-path.cc
@@ -1,4 +1,4 @@
-// Class filesystem::path -*- C++ -*-
+// Class experimental::filesystem::path -*- C++ -*-
 
 // Copyright (C) 2015-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/src/filesystem/cow-std-dir.cc b/libstdc++-v3/src/filesystem/cow-std-dir.cc
new file mode 100644
index 00000000000..edcf24506bb
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/cow-std-dir.cc
@@ -0,0 +1,26 @@
+// Class filesystem::directory_entry etc. -*- C++ -*-
+
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#define _GLIBCXX_USE_CXX11_ABI 0
+#include "std-dir.cc"
diff --git a/libstdc++-v3/src/filesystem/cow-std-ops.cc b/libstdc++-v3/src/filesystem/cow-std-ops.cc
new file mode 100644
index 00000000000..bea5470a890
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/cow-std-ops.cc
@@ -0,0 +1,26 @@
+// Filesystem operations -*- C++ -*-
+
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#define _GLIBCXX_USE_CXX11_ABI 0
+#include "std-ops.cc"
diff --git a/libstdc++-v3/src/filesystem/cow-std-path.cc b/libstdc++-v3/src/filesystem/cow-std-path.cc
new file mode 100644
index 00000000000..718ff952864
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/cow-std-path.cc
@@ -0,0 +1,26 @@
+// Class filesystem::path -*- C++ -*-
+
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#define _GLIBCXX_USE_CXX11_ABI 0
+#include "std-path.cc"
diff --git a/libstdc++-v3/src/filesystem/dir-common.h b/libstdc++-v3/src/filesystem/dir-common.h
new file mode 100644
index 00000000000..e8036321b20
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/dir-common.h
@@ -0,0 +1,149 @@
+// Filesystem directory iterator utilities -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef _GLIBCXX_DIR_COMMON_H
+#define _GLIBCXX_DIR_COMMON_H 1
+
+#include <string.h>  // strcmp
+#ifdef _GLIBCXX_HAVE_DIRENT_H
+# ifdef _GLIBCXX_HAVE_SYS_TYPES_H
+#  include <sys/types.h>
+# endif
+# include <dirent.h>
+#else
+# error "the <dirent.h> header is needed to build the Filesystem TS"
+#endif
+
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+# undef opendir
+# define opendir _wopendir
+#endif
+
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+namespace filesystem
+{
+
+struct _Dir_base
+{
+  _Dir_base(DIR* dirp = nullptr) : dirp(dirp) { }
+
+  // If no error occurs then dirp is non-null,
+  // otherwise null (whether error ignored or not).
+  _Dir_base(const char* p, bool skip_permission_denied,
+	    error_code& ec) noexcept
+  : dirp(::opendir(p))
+  {
+    if (dirp)
+      ec.clear();
+    else
+    {
+      const int err = errno;
+      if (err == EACCES && skip_permission_denied)
+	ec.clear();
+      else
+	ec.assign(err, std::generic_category());
+    }
+  }
+
+  _Dir_base(_Dir_base&& d) : dirp(std::exchange(d.dirp, nullptr)) { }
+
+  _Dir_base& operator=(_Dir_base&&) = delete;
+
+  ~_Dir_base() { if (dirp) ::closedir(dirp); }
+
+  const struct ::dirent*
+  advance(bool skip_permission_denied, error_code& ec) noexcept
+  {
+    ec.clear();
+
+    int err = std::exchange(errno, 0);
+    const struct ::dirent* entp = readdir(dirp);
+    // std::swap cannot be used with Bionic's errno
+    err = std::exchange(errno, err);
+
+    if (entp)
+      {
+	// skip past dot and dot-dot
+	if (!strcmp(entp->d_name, ".") || !strcmp(entp->d_name, ".."))
+	  return advance(skip_permission_denied, ec);
+	return entp;
+      }
+    else if (err)
+      {
+	if (err == EACCES && skip_permission_denied)
+	  return nullptr;
+	ec.assign(err, std::generic_category());
+	return nullptr;
+      }
+    else
+      {
+	// reached the end
+	return nullptr;
+      }
+  }
+
+  DIR*	dirp;
+};
+
+} // namespace filesystem
+
+// BEGIN/END macros must be defined before including this file.
+_GLIBCXX_BEGIN_NAMESPACE_FILESYSTEM
+inline file_type
+get_file_type(const ::dirent& d __attribute__((__unused__)))
+{
+#ifdef _GLIBCXX_HAVE_STRUCT_DIRENT_D_TYPE
+  switch (d.d_type)
+  {
+  case DT_BLK:
+    return file_type::block;
+  case DT_CHR:
+    return file_type::character;
+  case DT_DIR:
+    return file_type::directory;
+  case DT_FIFO:
+    return file_type::fifo;
+  case DT_LNK:
+    return file_type::symlink;
+  case DT_REG:
+    return file_type::regular;
+  case DT_SOCK:
+    return file_type::socket;
+  case DT_UNKNOWN:
+    return file_type::unknown;
+  default:
+    return file_type::none;
+  }
+#else
+  return file_type::none;
+#endif
+}
+_GLIBCXX_END_NAMESPACE_FILESYSTEM
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
+
+#endif // _GLIBCXX_DIR_COMMON_H
diff --git a/libstdc++-v3/src/filesystem/dir.cc b/libstdc++-v3/src/filesystem/dir.cc
index 9aecd8042a5..42f63e73984 100644
--- a/libstdc++-v3/src/filesystem/dir.cc
+++ b/libstdc++-v3/src/filesystem/dir.cc
@@ -31,40 +31,76 @@
 #include <stack>
 #include <string.h>
 #include <errno.h>
-#ifdef _GLIBCXX_HAVE_DIRENT_H
-# ifdef _GLIBCXX_HAVE_SYS_TYPES_H
-#  include <sys/types.h>
-# endif
-# include <dirent.h>
-#else
-# error "the <dirent.h> header is needed to build the Filesystem TS"
-#endif
-
-#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
-# undef opendir
-# define opendir _wopendir
-#endif
+#define _GLIBCXX_BEGIN_NAMESPACE_FILESYSTEM \
+  namespace experimental { namespace filesystem {
+#define _GLIBCXX_END_NAMESPACE_FILESYSTEM } }
+#include "dir-common.h"
 
 namespace fs = std::experimental::filesystem;
 
-struct fs::_Dir
+struct fs::_Dir : std::filesystem::_Dir_base
 {
-  _Dir() : dirp(nullptr) { }
+  _Dir(const fs::path& p, bool skip_permission_denied, error_code& ec)
+  : _Dir_base(p.c_str(), skip_permission_denied, ec)
+  {
+    if (!ec)
+      path = p;
+  }
 
-  _Dir(DIR* dirp, const fs::path& path) : dirp(dirp), path(path) { }
+  _Dir(DIR* dirp, const path& p) : _Dir_base(dirp), path(p) { }
 
-  _Dir(_Dir&& d)
-  : dirp(std::exchange(d.dirp, nullptr)), path(std::move(d.path)),
-    entry(std::move(d.entry)), type(d.type)
-  { }
+  _Dir(_Dir&&) = default;
 
-  _Dir& operator=(_Dir&&) = delete;
+  // Returns false when the end of the directory entries is reached.
+  // Reports errors by setting ec.
+  bool advance(bool skip_permission_denied, error_code& ec) noexcept
+  {
+    if (const auto entp = _Dir_base::advance(skip_permission_denied, ec))
+      {
+	entry = fs::directory_entry{path / entp->d_name};
+	type = get_file_type(*entp);
+	return true;
+      }
+    else if (!ec)
+      {
+	// reached the end
+	entry = {};
+	type = file_type::none;
+      }
+    return false;
+  }
+
+  bool advance(error_code& ec) noexcept { return advance(false, ec); }
 
-  ~_Dir() { if (dirp) ::closedir(dirp); }
+  // Returns false when the end of the directory entries is reached.
+  // Reports errors by throwing.
+  bool advance(bool skip_permission_denied = false)
+  {
+    error_code ec;
+    const bool ok = advance(skip_permission_denied, ec);
+    if (ec)
+      _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	      "directory iterator cannot advance", ec));
+    return ok;
+  }
 
-  bool advance(std::error_code*, directory_options = directory_options::none);
+  bool should_recurse(bool follow_symlink, error_code& ec) const
+  {
+    file_type type = this->type;
+    if (type == file_type::none || type == file_type::unknown)
+    {
+      type = entry.symlink_status(ec).type();
+      if (ec)
+	return false;
+    }
+
+    if (type == file_type::directory)
+      return true;
+    if (type == file_type::symlink)
+      return follow_symlink && is_directory(entry.status(ec));
+    return false;
+  }
 
-  DIR*			dirp;
   fs::path		path;
   directory_entry	entry;
   file_type		type = file_type::none;
@@ -78,119 +114,28 @@ namespace
     {
       return (obj & bits) != Bitmask::none;
     }
-
-  // Returns {dirp, p} on success, {} on error (whether ignored or not).
-  inline fs::_Dir
-  open_dir(const fs::path& p, fs::directory_options options,
-	   std::error_code* ec)
-  {
-    if (ec)
-      ec->clear();
-
-    if (DIR* dirp = ::opendir(p.c_str()))
-      return {dirp, p};
-
-    const int err = errno;
-    if (err == EACCES
-        && is_set(options, fs::directory_options::skip_permission_denied))
-      return {};
-
-    if (!ec)
-      _GLIBCXX_THROW_OR_ABORT(fs::filesystem_error(
-            "directory iterator cannot open directory", p,
-            std::error_code(err, std::generic_category())));
-
-    ec->assign(err, std::generic_category());
-    return {};
-  }
-
-  inline fs::file_type
-  get_file_type(const ::dirent& d __attribute__((__unused__)))
-  {
-#ifdef _GLIBCXX_HAVE_STRUCT_DIRENT_D_TYPE
-    switch (d.d_type)
-    {
-    case DT_BLK:
-      return fs::file_type::block;
-    case DT_CHR:
-      return fs::file_type::character;
-    case DT_DIR:
-      return fs::file_type::directory;
-    case DT_FIFO:
-      return fs::file_type::fifo;
-    case DT_LNK:
-      return fs::file_type::symlink;
-    case DT_REG:
-      return fs::file_type::regular;
-    case DT_SOCK:
-      return fs::file_type::socket;
-    case DT_UNKNOWN:
-      return fs::file_type::unknown;
-    default:
-      return fs::file_type::none;
-    }
-#else
-    return fs::file_type::none;
-#endif
-  }
-}
-
-
-// Returns false when the end of the directory entries is reached.
-// Reports errors by setting ec or throwing.
-bool
-fs::_Dir::advance(error_code* ec, directory_options options)
-{
-  if (ec)
-    ec->clear();
-
-  int err = std::exchange(errno, 0);
-  const auto entp = readdir(dirp);
-  // std::swap cannot be used with Bionic's errno
-  err = std::exchange(errno, err);
-
-  if (entp)
-    {
-      // skip past dot and dot-dot
-      if (!strcmp(entp->d_name, ".") || !strcmp(entp->d_name, ".."))
-	return advance(ec, options);
-      entry = fs::directory_entry{path / entp->d_name};
-      type = get_file_type(*entp);
-      return true;
-    }
-  else if (err)
-    {
-      if (err == EACCES
-        && is_set(options, directory_options::skip_permission_denied))
-	return false;
-
-      if (!ec)
-	_GLIBCXX_THROW_OR_ABORT(filesystem_error(
-	      "directory iterator cannot advance",
-	      std::error_code(err, std::generic_category())));
-      ec->assign(err, std::generic_category());
-      return false;
-    }
-  else
-    {
-      // reached the end
-      entry = {};
-      type = fs::file_type::none;
-      return false;
-    }
 }
 
 fs::directory_iterator::
-directory_iterator(const path& p, directory_options options, error_code* ec)
+directory_iterator(const path& p, directory_options options, error_code* ecptr)
 {
-  _Dir dir = open_dir(p, options, ec);
+  const bool skip_permission_denied
+    = is_set(options, directory_options::skip_permission_denied);
+
+  error_code ec;
+  _Dir dir(p, skip_permission_denied, ec);
 
   if (dir.dirp)
     {
       auto sp = std::make_shared<fs::_Dir>(std::move(dir));
-      if (sp->advance(ec, options))
+      if (sp->advance(skip_permission_denied, ec))
 	_M_dir.swap(sp);
     }
+  if (ecptr)
+    *ecptr = ec;
+  else if (ec)
+    _GLIBCXX_THROW_OR_ABORT(fs::filesystem_error(
+	  "directory iterator cannot open directory", p, ec));
 }
 
 const fs::directory_entry&
@@ -210,7 +155,7 @@ fs::directory_iterator::operator++()
     _GLIBCXX_THROW_OR_ABORT(filesystem_error(
 	  "cannot advance non-dereferenceable directory iterator",
 	  std::make_error_code(errc::invalid_argument)));
-  if (!_M_dir->advance(nullptr))
+  if (!_M_dir->advance())
     _M_dir.reset();
   return *this;
 }
@@ -223,13 +168,11 @@ fs::directory_iterator::increment(error_code& ec) noexcept
       ec = std::make_error_code(errc::invalid_argument);
       return *this;
     }
-  if (!_M_dir->advance(&ec))
+  if (!_M_dir->advance(ec))
     _M_dir.reset();
   return *this;
 }
 
-using Dir_iter_pair = std::pair<fs::_Dir, fs::directory_iterator>;
-
 struct fs::recursive_directory_iterator::_Dir_stack : std::stack<_Dir>
 {
   void clear() { c.clear(); }
@@ -240,6 +183,8 @@ recursive_directory_iterator(const path& p, directory_options options,
                              error_code* ec)
 : _M_options(options), _M_pending(true)
 {
+  if (ec)
+    ec->clear();
   if (DIR* dirp = ::opendir(p.c_str()))
     {
       auto sp = std::make_shared<_Dir_stack>();
@@ -252,11 +197,7 @@ recursive_directory_iterator(const path& p, directory_options options,
       const int err = errno;
       if (err == EACCES
 	  && is_set(options, fs::directory_options::skip_permission_denied))
-	{
-	  if (ec)
-	    ec->clear();
-	  return;
-	}
+	return;
 
       if (!ec)
 	_GLIBCXX_THROW_OR_ABORT(filesystem_error(
@@ -300,35 +241,6 @@ fs::recursive_directory_iterator::operator++()
   return *this;
 }
 
-namespace
-{
-  bool
-  recurse(const fs::_Dir& d, fs::directory_options options, std::error_code& ec)
-  {
-    bool follow_symlink
-      = is_set(options, fs::directory_options::follow_directory_symlink);
-#ifdef _GLIBCXX_HAVE_STRUCT_DIRENT_D_TYPE
-    if (d.type == fs::file_type::directory)
-      return true;
-    if (d.type == fs::file_type::symlink && follow_symlink)
-      return d.entry.status().type() == fs::file_type::directory;
-    if (d.type != fs::file_type::none && d.type != fs::file_type::unknown)
-      return false;
-#endif
-    const fs::path& path = d.entry.path();
-    auto type = fs::symlink_status(path, ec).type();
-    if (ec.value())
-      return false;
-    if (type == fs::file_type::symlink)
-      {
-	if (!follow_symlink)
-	  return false;
-	type = fs::status(path, ec).type();
-      }
-    return type == fs::file_type::directory;
-  }
-}
-
 fs::recursive_directory_iterator&
 fs::recursive_directory_iterator::increment(error_code& ec) noexcept
 {
@@ -338,11 +250,16 @@ fs::recursive_directory_iterator::increment(error_code& ec) noexcept
       return *this;
     }
 
+  const bool follow
+    = is_set(_M_options, directory_options::follow_directory_symlink);
+  const bool skip_permission_denied
+    = is_set(_M_options, directory_options::skip_permission_denied);
+
   auto& top = _M_dirs->top();
 
-  if (std::exchange(_M_pending, true) && recurse(top, _M_options, ec))
+  if (std::exchange(_M_pending, true) && top.should_recurse(follow, ec))
     {
-      _Dir dir = open_dir(top.entry.path(), _M_options, &ec);
+      _Dir dir(top.entry.path(), skip_permission_denied, ec);
       if (ec)
 	{
 	  _M_dirs.reset();
@@ -352,7 +269,7 @@ fs::recursive_directory_iterator::increment(error_code& ec) noexcept
 	  _M_dirs->push(std::move(dir));
     }
 
-  while (!_M_dirs->top().advance(&ec, _M_options) && !ec)
+  while (!_M_dirs->top().advance(skip_permission_denied, ec) && !ec)
     {
       _M_dirs->pop();
       if (_M_dirs->empty())
@@ -373,6 +290,9 @@ fs::recursive_directory_iterator::pop(error_code& ec)
       return;
     }
 
+  const bool skip_permission_denied
+    = is_set(_M_options, directory_options::skip_permission_denied);
+
   do {
     _M_dirs->pop();
     if (_M_dirs->empty())
@@ -381,7 +301,7 @@ fs::recursive_directory_iterator::pop(error_code& ec)
 	ec.clear();
 	return;
       }
-  } while (!_M_dirs->top().advance(&ec, _M_options));
+  } while (!_M_dirs->top().advance(skip_permission_denied, ec));
 }
 
 void
diff --git a/libstdc++-v3/src/filesystem/ops-common.h b/libstdc++-v3/src/filesystem/ops-common.h
new file mode 100644
index 00000000000..12c12b08f8c
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/ops-common.h
@@ -0,0 +1,148 @@
+// Filesystem operation utilities -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef _GLIBCXX_OPS_COMMON_H
+#define _GLIBCXX_OPS_COMMON_H 1
+
+#include <chrono>
+
+#ifdef _GLIBCXX_HAVE_UNISTD_H
+# include <unistd.h>
+# if defined(_GLIBCXX_HAVE_SYS_STAT_H) && defined(_GLIBCXX_HAVE_SYS_TYPES_H)
+#  include <sys/types.h>
+#  include <sys/stat.h>
+# endif
+#endif
+
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+namespace filesystem
+{
+  template<typename Bitmask>
+    inline bool is_set(Bitmask obj, Bitmask bits)
+    {
+      return (obj & bits) != Bitmask::none;
+    }
+
+  inline bool
+  is_not_found_errno(int err) noexcept
+  {
+    return err == ENOENT || err == ENOTDIR;
+  }
+
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+  typedef struct ::stat stat_type;
+
+  inline std::chrono::system_clock::time_point
+  file_time(const stat_type& st, std::error_code& ec) noexcept
+  {
+    using namespace std::chrono;
+#ifdef _GLIBCXX_USE_ST_MTIM
+    time_t s = st.st_mtim.tv_sec;
+    nanoseconds ns{st.st_mtim.tv_nsec};
+#else
+    time_t s = st.st_mtime;
+    nanoseconds ns{};
+#endif
+
+    if (s >= (nanoseconds::max().count() / 1e9))
+      {
+	ec = std::make_error_code(std::errc::value_too_large); // EOVERFLOW
+	return system_clock::time_point::min();
+      }
+    ec.clear();
+    return system_clock::time_point{seconds{s} + ns};
+  }
+
+  struct copy_options_existing_file
+  {
+    bool skip, update, overwrite;
+  };
+
+  bool
+  do_copy_file(const char* from, const char* to,
+	       copy_options_existing_file options,
+	       stat_type* from_st, stat_type* to_st,
+	       std::error_code& ec) noexcept;
+
+#endif // _GLIBCXX_HAVE_SYS_STAT_H
+
+} // namespace filesystem
+
+// BEGIN/END macros must be defined before including this file.
+_GLIBCXX_BEGIN_NAMESPACE_FILESYSTEM
+
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+  typedef struct ::stat stat_type;
+
+  inline file_type
+  make_file_type(const stat_type& st) noexcept
+  {
+#ifdef _GLIBCXX_HAVE_S_ISREG
+    if (S_ISREG(st.st_mode))
+      return file_type::regular;
+    else if (S_ISDIR(st.st_mode))
+      return file_type::directory;
+    else if (S_ISCHR(st.st_mode))
+      return file_type::character;
+    else if (S_ISBLK(st.st_mode))
+      return file_type::block;
+    else if (S_ISFIFO(st.st_mode))
+      return file_type::fifo;
+    else if (S_ISLNK(st.st_mode))
+      return file_type::symlink;
+    else if (S_ISSOCK(st.st_mode))
+      return file_type::socket;
+#endif
+    return file_type::unknown;
+  }
+
+  inline file_status
+  make_file_status(const stat_type& st) noexcept
+  {
+    return file_status{
+	make_file_type(st),
+	static_cast<perms>(st.st_mode) & perms::mask
+    };
+  }
+
+  inline std::filesystem::copy_options_existing_file
+  copy_file_options(copy_options opt)
+  {
+    using std::filesystem::is_set;
+    return {
+	is_set(opt, copy_options::skip_existing),
+	is_set(opt, copy_options::update_existing),
+	is_set(opt, copy_options::overwrite_existing)
+    };
+  }
+#endif // _GLIBCXX_HAVE_SYS_STAT_H
+
+_GLIBCXX_END_NAMESPACE_FILESYSTEM
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
+
+#endif // _GLIBCXX_OPS_COMMON_H
diff --git a/libstdc++-v3/src/filesystem/ops.cc b/libstdc++-v3/src/filesystem/ops.cc
index c711de16f06..1ec8883fde9 100644
--- a/libstdc++-v3/src/filesystem/ops.cc
+++ b/libstdc++-v3/src/filesystem/ops.cc
@@ -1,4 +1,4 @@
-// Filesystem operations -*- C++ -*-
+// Filesystem TS operations -*- C++ -*-
 
 // Copyright (C) 2014-2017 Free Software Foundation, Inc.
 //
@@ -35,26 +35,24 @@
 #include <stdio.h>
 #include <errno.h>
 #include <limits.h>  // PATH_MAX
-#ifdef _GLIBCXX_HAVE_UNISTD_H
-# include <unistd.h>
-# if defined(_GLIBCXX_HAVE_SYS_STAT_H) && defined(_GLIBCXX_HAVE_SYS_TYPES_H)
-#  include <sys/types.h>
-#  include <sys/stat.h>
-# endif
-#endif
 #ifdef _GLIBCXX_HAVE_FCNTL_H
-# include <fcntl.h>
+# include <fcntl.h>  // AT_FDCWD, AT_SYMLINK_NOFOLLOW
 #endif
-#ifdef _GLIBCXX_HAVE_SYS_STATVFS_H
-# include <sys/statvfs.h>
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+#  include <sys/stat.h>   // stat, utimensat, fchmodat
 #endif
-#ifdef _GLIBCXX_USE_SENDFILE
-# include <sys/sendfile.h>
+#ifdef _GLIBCXX_HAVE_SYS_STATVFS_H
+# include <sys/statvfs.h> // statvfs
 #endif
-#if _GLIBCXX_HAVE_UTIME_H
-# include <utime.h>
+#if !_GLIBCXX_USE_UTIMENSAT && _GLIBCXX_HAVE_UTIME_H
+# include <utime.h> // utime
 #endif
 
+#define _GLIBCXX_BEGIN_NAMESPACE_FILESYSTEM \
+  namespace experimental { namespace filesystem {
+#define _GLIBCXX_END_NAMESPACE_FILESYSTEM } }
+#include "ops-common.h"
+
 #ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
 # undef utime
 # define utime _wutime
@@ -240,254 +238,16 @@ fs::copy(const path& from, const path& to, copy_options options)
 
 namespace
 {
-  template<typename Bitmask>
-    inline bool is_set(Bitmask obj, Bitmask bits)
-    {
-      return (obj & bits) != Bitmask::none;
-    }
-}
+  using std::filesystem::is_set;
 
 #ifdef _GLIBCXX_HAVE_SYS_STAT_H
-namespace
-{
   typedef struct ::stat stat_type;
 
-  inline fs::file_type
-  make_file_type(const stat_type& st) noexcept
-  {
-    using fs::file_type;
-#ifdef _GLIBCXX_HAVE_S_ISREG
-    if (S_ISREG(st.st_mode))
-      return file_type::regular;
-    else if (S_ISDIR(st.st_mode))
-      return file_type::directory;
-    else if (S_ISCHR(st.st_mode))
-      return file_type::character;
-    else if (S_ISBLK(st.st_mode))
-      return file_type::block;
-    else if (S_ISFIFO(st.st_mode))
-      return file_type::fifo;
-    else if (S_ISLNK(st.st_mode))
-      return file_type::symlink;
-    else if (S_ISSOCK(st.st_mode))
-      return file_type::socket;
-#endif
-    return file_type::unknown;
-
-  }
-
-  inline fs::file_status
-  make_file_status(const stat_type& st) noexcept
-  {
-    return fs::file_status{
-	make_file_type(st),
-	static_cast<fs::perms>(st.st_mode) & fs::perms::mask
-    };
-  }
-
-  inline bool
-  is_not_found_errno(int err) noexcept
-  {
-    return err == ENOENT || err == ENOTDIR;
-  }
-
-  inline fs::file_time_type
-  file_time(const stat_type& st, std::error_code& ec) noexcept
-  {
-    using namespace std::chrono;
-#ifdef _GLIBCXX_USE_ST_MTIM
-    time_t s = st.st_mtim.tv_sec;
-    nanoseconds ns{st.st_mtim.tv_nsec};
-#else
-    time_t s = st.st_mtime;
-    nanoseconds ns{};
-#endif
-
-    if (s >= (nanoseconds::max().count() / 1e9))
-      {
-	ec = std::make_error_code(std::errc::value_too_large); // EOVERFLOW
-	return fs::file_time_type::min();
-      }
-    ec.clear();
-    return fs::file_time_type{seconds{s} + ns};
-  }
-
-  bool
-  do_copy_file(const fs::path& from, const fs::path& to,
-	       fs::copy_options option,
-	       stat_type* from_st, stat_type* to_st,
-	       std::error_code& ec) noexcept
-  {
-    stat_type st1, st2;
-    fs::file_status t, f;
-
-    if (to_st == nullptr)
-      {
-	if (::stat(to.c_str(), &st1))
-	  {
-	    int err = errno;
-	    if (!is_not_found_errno(err))
-	      {
-		ec.assign(err, std::generic_category());
-		return false;
-	      }
-	  }
-	else
-	  to_st = &st1;
-      }
-    else if (to_st == from_st)
-      to_st = nullptr;
-
-    if (to_st == nullptr)
-      t = fs::file_status{fs::file_type::not_found};
-    else
-      t = make_file_status(*to_st);
-
-    if (from_st == nullptr)
-      {
-	if (::stat(from.c_str(), &st2))
-	  {
-	    ec.assign(errno, std::generic_category());
-	    return false;
-	  }
-	else
-	  from_st = &st2;
-      }
-    f = make_file_status(*from_st);
-    // _GLIBCXX_RESOLVE_LIB_DEFECTS
-    // 2712. copy_file() has a number of unspecified error conditions
-    if (!is_regular_file(f))
-      {
-	ec = std::make_error_code(std::errc::not_supported);
-	return false;
-      }
-
-    using opts = fs::copy_options;
-
-    if (exists(t))
-      {
-	if (!is_regular_file(t))
-	  {
-	    ec = std::make_error_code(std::errc::not_supported);
-	    return false;
-	  }
-
-	if (to_st->st_dev == from_st->st_dev
-	    && to_st->st_ino == from_st->st_ino)
-	  {
-	    ec = std::make_error_code(std::errc::file_exists);
-	    return false;
-	  }
-
-	if (is_set(option, opts::skip_existing))
-	  {
-	    ec.clear();
-	    return false;
-	  }
-	else if (is_set(option, opts::update_existing))
-	  {
-	    const auto from_mtime = file_time(*from_st, ec);
-	    if (ec)
-	      return false;
-	    if ((from_mtime <= file_time(*to_st, ec)) || ec)
-	      return false;
-	  }
-	else if (!is_set(option, opts::overwrite_existing))
-	  {
-	    ec = std::make_error_code(std::errc::file_exists);
-	    return false;
-	  }
-	else if (!is_regular_file(t))
-	  {
-	    ec = std::make_error_code(std::errc::not_supported);
-	    return false;
-	  }
-      }
-
-    struct CloseFD {
-      ~CloseFD() { if (fd != -1) ::close(fd); }
-      bool close() { return ::close(std::exchange(fd, -1)) == 0; }
-      int fd;
-    };
-
-    CloseFD in = { ::open(from.c_str(), O_RDONLY) };
-    if (in.fd == -1)
-      {
-	ec.assign(errno, std::generic_category());
-	return false;
-      }
-    int oflag = O_WRONLY|O_CREAT;
-    if (is_set(option, opts::overwrite_existing|opts::update_existing))
-      oflag |= O_TRUNC;
-    else
-      oflag |= O_EXCL;
-    CloseFD out = { ::open(to.c_str(), oflag, S_IWUSR) };
-    if (out.fd == -1)
-      {
-	if (errno == EEXIST && is_set(option, opts::skip_existing))
-	  ec.clear();
-	else
-	  ec.assign(errno, std::generic_category());
-	return false;
-      }
-
-#ifdef _GLIBCXX_USE_FCHMOD
-    if (::fchmod(out.fd, from_st->st_mode))
-#elif defined _GLIBCXX_USE_FCHMODAT
-    if (::fchmodat(AT_FDCWD, to.c_str(), from_st->st_mode, 0))
-#else
-    if (::chmod(to.c_str(), from_st->st_mode))
-#endif
-      {
-	ec.assign(errno, std::generic_category());
-	return false;
-      }
-
-#ifdef _GLIBCXX_USE_SENDFILE
-    off_t offset = 0;
-    const auto n = ::sendfile(out.fd, in.fd, &offset, from_st->st_size);
-    if (n < 0 && (errno == ENOSYS || errno == EINVAL))
-      {
-#endif
-	__gnu_cxx::stdio_filebuf<char> sbin(in.fd, std::ios::in);
-	__gnu_cxx::stdio_filebuf<char> sbout(out.fd, std::ios::out);
-	if (sbin.is_open())
-	  in.fd = -1;
-	if (sbout.is_open())
-	  out.fd = -1;
-	if (from_st->st_size && !(std::ostream(&sbout) << &sbin))
-	  {
-	    ec = std::make_error_code(std::errc::io_error);
-	    return false;
-	  }
-	if (!sbout.close() || !sbin.close())
-	  {
-	    ec.assign(errno, std::generic_category());
-	    return false;
-	  }
-
-	ec.clear();
-	return true;
-
-#ifdef _GLIBCXX_USE_SENDFILE
-      }
-    if (n != from_st->st_size)
-      {
-	ec.assign(errno, std::generic_category());
-	return false;
-      }
-    if (!out.close() || !in.close())
-      {
-	ec.assign(errno, std::generic_category());
-	return false;
-      }
-
-    ec.clear();
-    return true;
-#endif
-  }
-}
-#endif
+  using std::filesystem::is_not_found_errno;
+  using std::filesystem::file_time;
+  using std::filesystem::do_copy_file;
+#endif // _GLIBCXX_HAVE_SYS_STAT_H
+} // namespace
 
 void
 fs::copy(const path& from, const path& to, copy_options options,
@@ -561,11 +321,13 @@ fs::copy(const path& from, const path& to, copy_options options,
       else if (is_set(options, copy_options::create_hard_links))
 	create_hard_link(from, to, ec);
       else if (is_directory(t))
-	do_copy_file(from, to / from.filename(), options, &from_st, 0, ec);
+	do_copy_file(from.c_str(), (to / from.filename()).c_str(),
+		     copy_file_options(options), &from_st, nullptr, ec);
       else
 	{
 	  auto ptr = exists(t) ? &to_st : &from_st;
-	  do_copy_file(from, to, options, &from_st, ptr,  ec);
+	  do_copy_file(from.c_str(), to.c_str(), copy_file_options(options),
+		       &from_st, ptr,  ec);
 	}
     }
   // _GLIBCXX_RESOLVE_LIB_DEFECTS
@@ -602,11 +364,12 @@ fs::copy_file(const path& from, const path& to, copy_options option)
 }
 
 bool
-fs::copy_file(const path& from, const path& to, copy_options option,
+fs::copy_file(const path& from, const path& to, copy_options options,
 	      error_code& ec) noexcept
 {
 #ifdef _GLIBCXX_HAVE_SYS_STAT_H
-  return do_copy_file(from, to, option, nullptr, nullptr, ec);
+  return do_copy_file(from.c_str(), to.c_str(), copy_file_options(options),
+		      nullptr, nullptr, ec);
 #else
   ec = std::make_error_code(std::errc::not_supported);
   return false;
@@ -1199,26 +962,45 @@ fs::read_symlink(const path& p)
 
 fs::path fs::read_symlink(const path& p, error_code& ec)
 {
+  path result;
 #ifdef _GLIBCXX_HAVE_SYS_STAT_H
   stat_type st;
   if (::lstat(p.c_str(), &st))
     {
       ec.assign(errno, std::generic_category());
-      return {};
+      return result;
     }
-  std::string buf(st.st_size, '\0');
-  ssize_t len = ::readlink(p.c_str(), &buf.front(), buf.size());
-  if (len == -1)
+  std::string buf(st.st_size ? st.st_size + 1 : 128, '\0');
+  do
     {
-      ec.assign(errno, std::generic_category());
-      return {};
+      ssize_t len = ::readlink(p.c_str(), buf.data(), buf.size());
+      if (len == -1)
+	{
+	  ec.assign(errno, std::generic_category());
+	  return result;
+	}
+      else if (len == (ssize_t)buf.size())
+	{
+	  if (buf.size() > 4096)
+	    {
+	      ec.assign(ENAMETOOLONG, std::generic_category());
+	      return result;
+	    }
+	  buf.resize(buf.size() * 2);
+	}
+      else
+	{
+	  buf.resize(len);
+	  result.assign(buf);
+	  ec.clear();
+	  break;
+	}
     }
-  ec.clear();
-  return path{buf.data(), buf.data()+len};
+  while (true);
 #else
   ec = std::make_error_code(std::errc::not_supported);
-  return {};
 #endif
+  return result;
 }
 
 
diff --git a/libstdc++-v3/src/filesystem/path.cc b/libstdc++-v3/src/filesystem/path.cc
index c66d52bf4b0..a5196674883 100644
--- a/libstdc++-v3/src/filesystem/path.cc
+++ b/libstdc++-v3/src/filesystem/path.cc
@@ -1,4 +1,4 @@
-// Class filesystem::path -*- C++ -*-
+// Class experimental::filesystem::path -*- C++ -*-
 
 // Copyright (C) 2014-2017 Free Software Foundation, Inc.
 //
@@ -28,9 +28,10 @@
 
 #include <experimental/filesystem>
 
-using std::experimental::filesystem::path;
+namespace fs = std::experimental::filesystem;
+using fs::path;
 
-std::experimental::filesystem::filesystem_error::~filesystem_error() = default;
+fs::filesystem_error::~filesystem_error() = default;
 
 constexpr path::value_type path::preferred_separator;
 
@@ -461,7 +462,7 @@ path::_S_convert_loc(const char* __first, const char* __last,
 }
 
 std::size_t
-std::experimental::filesystem::hash_value(const path& p) noexcept
+fs::hash_value(const path& p) noexcept
 {
   // [path.non-member]
   // "If for two paths, p1 == p2 then hash_value(p1) == hash_value(p2)."
@@ -477,3 +478,29 @@ std::experimental::filesystem::hash_value(const path& p) noexcept
     }
   return seed;
 }
+
+namespace std
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+namespace filesystem
+{
+  extern string
+  fs_err_concat(const string& __what, const string& __path1,
+		const string& __path2);
+} // namespace filesystem
+
+namespace experimental::filesystem::v1 {
+_GLIBCXX_BEGIN_NAMESPACE_CXX11
+
+  std::string filesystem_error::_M_gen_what()
+  {
+    using std::filesystem::fs_err_concat;
+    return fs_err_concat(system_error::what(), _M_path1.native(),
+			 _M_path2.native());
+  }
+
+_GLIBCXX_END_NAMESPACE_CXX11
+} // namespace experimental::filesystem::v1
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
diff --git a/libstdc++-v3/src/filesystem/std-dir.cc b/libstdc++-v3/src/filesystem/std-dir.cc
new file mode 100644
index 00000000000..553128e9096
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/std-dir.cc
@@ -0,0 +1,318 @@
+// Class filesystem::directory_entry etc. -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef _GLIBCXX_USE_CXX11_ABI
+# define _GLIBCXX_USE_CXX11_ABI 1
+#endif
+
+#include <filesystem>
+#include <experimental/filesystem>
+#include <utility>
+#include <stack>
+#include <string.h>
+#include <errno.h>
+#define _GLIBCXX_BEGIN_NAMESPACE_FILESYSTEM namespace filesystem {
+#define _GLIBCXX_END_NAMESPACE_FILESYSTEM }
+#include "dir-common.h"
+
+namespace fs = std::filesystem;
+
+struct fs::_Dir : _Dir_base
+{
+  _Dir(const fs::path& p, bool skip_permission_denied, error_code& ec)
+  : _Dir_base(p.c_str(), skip_permission_denied, ec)
+  {
+    if (!ec)
+      path = p;
+  }
+
+  _Dir(DIR* dirp, const path& p) : _Dir_base(dirp), path(p) { }
+
+  _Dir(_Dir&&) = default;
+
+  // Returns false when the end of the directory entries is reached.
+  // Reports errors by setting ec.
+  bool advance(bool skip_permission_denied, error_code& ec) noexcept
+  {
+    if (const auto entp = _Dir_base::advance(skip_permission_denied, ec))
+      {
+	entry = fs::directory_entry{path / entp->d_name, get_file_type(*entp)};
+	return true;
+      }
+    else if (!ec)
+      {
+	// reached the end
+	entry = {};
+      }
+    return false;
+  }
+
+  bool advance(error_code& ec) noexcept { return advance(false, ec); }
+
+  // Returns false when the end of the directory entries is reached.
+  // Reports errors by throwing.
+  bool advance(bool skip_permission_denied = false)
+  {
+    error_code ec;
+    const bool ok = advance(skip_permission_denied, ec);
+    if (ec)
+      _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	      "directory iterator cannot advance", ec));
+    return ok;
+  }
+
+  bool should_recurse(bool follow_symlink, error_code& ec) const
+  {
+    file_type type = entry._M_type;
+    if (type == file_type::none || type == file_type::unknown)
+    {
+      type = entry.symlink_status(ec).type();
+      if (ec)
+	return false;
+    }
+
+    if (type == file_type::directory)
+      return true;
+    if (type == file_type::symlink)
+      return follow_symlink && is_directory(entry.status(ec));
+    return false;
+  }
+
+  fs::path		path;
+  directory_entry	entry;
+};
+
+namespace
+{
+  template<typename Bitmask>
+    inline bool
+    is_set(Bitmask obj, Bitmask bits)
+    {
+      return (obj & bits) != Bitmask::none;
+    }
+}
+
+fs::directory_iterator::
+directory_iterator(const path& p, directory_options options, error_code* ecptr)
+{
+  const bool skip_permission_denied
+    = is_set(options, directory_options::skip_permission_denied);
+
+  error_code ec;
+  _Dir dir(p, skip_permission_denied, ec);
+
+  if (dir.dirp)
+    {
+      auto sp = std::make_shared<fs::_Dir>(std::move(dir));
+      if (sp->advance(skip_permission_denied, ec))
+	_M_dir.swap(sp);
+    }
+  if (ecptr)
+    *ecptr = ec;
+  else if (ec)
+    _GLIBCXX_THROW_OR_ABORT(fs::filesystem_error(
+	  "directory iterator cannot open directory", p, ec));
+}
+
+const fs::directory_entry&
+fs::directory_iterator::operator*() const
+{
+  if (!_M_dir)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	  "non-dereferenceable directory iterator",
+	  std::make_error_code(errc::invalid_argument)));
+  return _M_dir->entry;
+}
+
+fs::directory_iterator&
+fs::directory_iterator::operator++()
+{
+  if (!_M_dir)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	  "cannot advance non-dereferenceable directory iterator",
+	  std::make_error_code(errc::invalid_argument)));
+  if (!_M_dir->advance())
+    _M_dir.reset();
+  return *this;
+}
+
+fs::directory_iterator&
+fs::directory_iterator::increment(error_code& ec)
+{
+  if (!_M_dir)
+    {
+      ec = std::make_error_code(errc::invalid_argument);
+      return *this;
+    }
+  if (!_M_dir->advance(ec))
+    _M_dir.reset();
+  return *this;
+}
+
+struct fs::recursive_directory_iterator::_Dir_stack : std::stack<_Dir>
+{
+  void clear() { c.clear(); }
+};
+
+fs::recursive_directory_iterator::
+recursive_directory_iterator(const path& p, directory_options options,
+                             error_code* ecptr)
+: _M_options(options), _M_pending(true)
+{
+  if (DIR* dirp = ::opendir(p.c_str()))
+    {
+      if (ecptr)
+	ecptr->clear();
+      auto sp = std::make_shared<_Dir_stack>();
+      sp->push(_Dir{ dirp, p });
+      if (ecptr ? sp->top().advance(*ecptr) : sp->top().advance())
+	_M_dirs.swap(sp);
+    }
+  else
+    {
+      const int err = errno;
+      if (err == EACCES
+	  && is_set(options, fs::directory_options::skip_permission_denied))
+	{
+	  if (ecptr)
+	    ecptr->clear();
+	  return;
+	}
+
+      if (!ecptr)
+	_GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	      "recursive directory iterator cannot open directory", p,
+	      std::error_code(err, std::generic_category())));
+
+      ecptr->assign(err, std::generic_category());
+    }
+}
+
+fs::recursive_directory_iterator::~recursive_directory_iterator() = default;
+
+int
+fs::recursive_directory_iterator::depth() const
+{
+  return int(_M_dirs->size()) - 1;
+}
+
+const fs::directory_entry&
+fs::recursive_directory_iterator::operator*() const
+{
+  return _M_dirs->top().entry;
+}
+
+fs::recursive_directory_iterator&
+fs::recursive_directory_iterator::
+operator=(const recursive_directory_iterator& other) noexcept = default;
+
+fs::recursive_directory_iterator&
+fs::recursive_directory_iterator::
+operator=(recursive_directory_iterator&& other) noexcept = default;
+
+fs::recursive_directory_iterator&
+fs::recursive_directory_iterator::operator++()
+{
+  error_code ec;
+  increment(ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	  "cannot increment recursive directory iterator", ec));
+  return *this;
+}
+
+fs::recursive_directory_iterator&
+fs::recursive_directory_iterator::increment(error_code& ec)
+{
+  if (!_M_dirs)
+    {
+      ec = std::make_error_code(errc::invalid_argument);
+      return *this;
+    }
+
+  const bool follow
+    = is_set(_M_options, directory_options::follow_directory_symlink);
+  const bool skip_permission_denied
+    = is_set(_M_options, directory_options::skip_permission_denied);
+
+  auto& top = _M_dirs->top();
+
+  if (std::exchange(_M_pending, true) && top.should_recurse(follow, ec))
+    {
+      _Dir dir(top.entry.path(), skip_permission_denied, ec);
+      if (ec)
+	{
+	  _M_dirs.reset();
+	  return *this;
+	}
+      if (dir.dirp)
+	  _M_dirs->push(std::move(dir));
+    }
+
+  while (!_M_dirs->top().advance(skip_permission_denied, ec) && !ec)
+    {
+      _M_dirs->pop();
+      if (_M_dirs->empty())
+	{
+	  _M_dirs.reset();
+	  return *this;
+	}
+    }
+  return *this;
+}
+
+void
+fs::recursive_directory_iterator::pop(error_code& ec)
+{
+  if (!_M_dirs)
+    {
+      ec = std::make_error_code(errc::invalid_argument);
+      return;
+    }
+
+  const bool skip_permission_denied
+    = is_set(_M_options, directory_options::skip_permission_denied);
+
+  do {
+    _M_dirs->pop();
+    if (_M_dirs->empty())
+      {
+	_M_dirs.reset();
+	ec.clear();
+	return;
+      }
+  } while (!_M_dirs->top().advance(skip_permission_denied, ec));
+}
+
+void
+fs::recursive_directory_iterator::pop()
+{
+  error_code ec;
+  pop(ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error(_M_dirs
+	  ? "recursive directory iterator cannot pop"
+	  : "non-dereferenceable recursive directory iterator cannot pop",
+	  ec));
+}
diff --git a/libstdc++-v3/src/filesystem/std-ops.cc b/libstdc++-v3/src/filesystem/std-ops.cc
new file mode 100644
index 00000000000..fa5e19a36ba
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/std-ops.cc
@@ -0,0 +1,1535 @@
+// Filesystem operations -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef _GLIBCXX_USE_CXX11_ABI
+# define _GLIBCXX_USE_CXX11_ABI 1
+# define NEED_DO_COPY_FILE
+#endif
+
+#include <filesystem>
+#include <experimental/filesystem>
+#include <functional>
+#include <ostream>
+#include <stack>
+#include <ext/stdio_filebuf.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <limits.h>  // PATH_MAX
+#ifdef _GLIBCXX_HAVE_FCNTL_H
+# include <fcntl.h>  // AT_FDCWD, AT_SYMLINK_NOFOLLOW
+#endif
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+#  include <sys/stat.h>   // stat, utimensat, fchmodat
+#endif
+#ifdef _GLIBCXX_HAVE_SYS_STATVFS_H
+# include <sys/statvfs.h> // statvfs
+#endif
+#ifdef _GLIBCXX_USE_SENDFILE
+# include <sys/sendfile.h> // sendfile
+#endif
+#if !_GLIBCXX_USE_UTIMENSAT && _GLIBCXX_HAVE_UTIME_H
+# include <utime.h> // utime
+#endif
+
+#define _GLIBCXX_BEGIN_NAMESPACE_FILESYSTEM namespace filesystem {
+#define _GLIBCXX_END_NAMESPACE_FILESYSTEM }
+#include "ops-common.h"
+
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+# undef utime
+# define utime _wutime
+# undef chmod
+# define chmod _wchmod
+#endif
+
+namespace fs = std::filesystem;
+
+fs::path
+fs::absolute(const path& p)
+{
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  error_code ec;
+  path ret = absolute(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot make absolute path", p,
+	std::make_error_code(errc::not_supported)));
+  return ret;
+#else
+  return current_path() / p;
+#endif
+}
+
+fs::path
+fs::absolute(const path& p, error_code& ec)
+{
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  ec = std::make_error_code(errc::not_supported);
+  return {};
+#else
+  ec.clear();
+  return current_path() / p;
+#endif
+}
+
+namespace
+{
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  inline bool is_dot(wchar_t c) { return c == L'.'; }
+#else
+  inline bool is_dot(char c) { return c == '.'; }
+#endif
+
+  inline bool is_dot(const fs::path& path)
+  {
+    const auto& filename = path.native();
+    return filename.size() == 1 && is_dot(filename[0]);
+  }
+
+  inline bool is_dotdot(const fs::path& path)
+  {
+    const auto& filename = path.native();
+    return filename.size() == 2 && is_dot(filename[0]) && is_dot(filename[1]);
+  }
+
+  struct free_as_in_malloc
+  {
+    void operator()(void* p) const { ::free(p); }
+  };
+
+  using char_ptr = std::unique_ptr<char[], free_as_in_malloc>;
+}
+
+fs::path
+fs::canonical(const path& p, error_code& ec)
+{
+  path result;
+  const path pa = absolute(p, ec);
+  if (ec)
+    return result;
+
+#ifdef _GLIBCXX_USE_REALPATH
+  char_ptr buf{ nullptr };
+# if _XOPEN_VERSION < 700
+  // Not safe to call realpath(path, NULL)
+  buf.reset( (char*)::malloc(PATH_MAX) );
+# endif
+  if (char* rp = ::realpath(pa.c_str(), buf.get()))
+    {
+      if (buf == nullptr)
+	buf.reset(rp);
+      result.assign(rp);
+      ec.clear();
+      return result;
+    }
+  if (errno != ENAMETOOLONG)
+    {
+      ec.assign(errno, std::generic_category());
+      return result;
+    }
+#endif
+
+  if (!exists(pa, ec))
+    {
+      if (!ec)
+	ec = make_error_code(std::errc::no_such_file_or_directory);
+      return result;
+    }
+  // else: we know there are (currently) no unresolvable symlink loops
+
+  result = pa.root_path();
+
+  deque<path> cmpts;
+  for (auto& f : pa.relative_path())
+    cmpts.push_back(f);
+
+  int max_allowed_symlinks = 40;
+
+  while (!cmpts.empty() && !ec)
+    {
+      path f = std::move(cmpts.front());
+      cmpts.pop_front();
+
+      if (f.empty())
+	{
+	  // ignore empty element
+	}
+      else if (is_dot(f))
+	{
+	  if (!is_directory(result, ec) && !ec)
+	    ec.assign(ENOTDIR, std::generic_category());
+	}
+      else if (is_dotdot(f))
+	{
+	  auto parent = result.parent_path();
+	  if (parent.empty())
+	    result = pa.root_path();
+	  else
+	    result.swap(parent);
+	}
+      else
+	{
+	  result /= f;
+
+	  if (is_symlink(result, ec))
+	    {
+	      path link = read_symlink(result, ec);
+	      if (!ec)
+		{
+		  if (--max_allowed_symlinks == 0)
+		    ec.assign(ELOOP, std::generic_category());
+		  else
+		    {
+		      if (link.is_absolute())
+			{
+			  result = link.root_path();
+			  link = link.relative_path();
+			}
+		      else
+			result = result.parent_path();
+
+		      cmpts.insert(cmpts.begin(), link.begin(), link.end());
+		    }
+		}
+	    }
+	}
+    }
+
+  if (ec || !exists(result, ec))
+    result.clear();
+
+  return result;
+}
+
+fs::path
+fs::canonical(const path& p)
+{
+  error_code ec;
+  path res = canonical(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot make canonical path",
+					     p, ec));
+  return res;
+}
+
+void
+fs::copy(const path& from, const path& to, copy_options options)
+{
+  error_code ec;
+  copy(from, to, options, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot copy", from, to, ec));
+}
+
+namespace std::filesystem
+{
+  // Need this as there's no 'perm_options::none' enumerator.
+  inline bool is_set(fs::perm_options obj, fs::perm_options bits)
+  {
+    return (obj & bits) != fs::perm_options{};
+  }
+}
+
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+#ifdef NEED_DO_COPY_FILE
+bool
+fs::do_copy_file(const char* from, const char* to,
+		 copy_options_existing_file options,
+		 stat_type* from_st, stat_type* to_st,
+		 std::error_code& ec) noexcept
+{
+  stat_type st1, st2;
+  fs::file_status t, f;
+
+  if (to_st == nullptr)
+    {
+      if (::stat(to, &st1))
+	{
+	  const int err = errno;
+	  if (!is_not_found_errno(err))
+	    {
+	      ec.assign(err, std::generic_category());
+	      return false;
+	    }
+	}
+      else
+	to_st = &st1;
+    }
+  else if (to_st == from_st)
+    to_st = nullptr;
+
+  if (to_st == nullptr)
+    t = fs::file_status{fs::file_type::not_found};
+  else
+    t = make_file_status(*to_st);
+
+  if (from_st == nullptr)
+    {
+      if (::stat(from, &st2))
+	{
+	  ec.assign(errno, std::generic_category());
+	  return false;
+	}
+      else
+	from_st = &st2;
+    }
+  f = make_file_status(*from_st);
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2712. copy_file() has a number of unspecified error conditions
+  if (!is_regular_file(f))
+    {
+      ec = std::make_error_code(std::errc::not_supported);
+      return false;
+    }
+
+  if (exists(t))
+    {
+      if (!is_regular_file(t))
+	{
+	  ec = std::make_error_code(std::errc::not_supported);
+	  return false;
+	}
+
+      if (to_st->st_dev == from_st->st_dev
+	  && to_st->st_ino == from_st->st_ino)
+	{
+	  ec = std::make_error_code(std::errc::file_exists);
+	  return false;
+	}
+
+      if (options.skip)
+	{
+	  ec.clear();
+	  return false;
+	}
+      else if (options.update)
+	{
+	  const auto from_mtime = file_time(*from_st, ec);
+	  if (ec)
+	    return false;
+	  if ((from_mtime <= file_time(*to_st, ec)) || ec)
+	    return false;
+	}
+      else if (!options.overwrite)
+	{
+	  ec = std::make_error_code(std::errc::file_exists);
+	  return false;
+	}
+      else if (!is_regular_file(t))
+	{
+	  ec = std::make_error_code(std::errc::not_supported);
+	  return false;
+	}
+    }
+
+  struct CloseFD {
+    ~CloseFD() { if (fd != -1) ::close(fd); }
+    bool close() { return ::close(std::exchange(fd, -1)) == 0; }
+    int fd;
+  };
+
+  CloseFD in = { ::open(from, O_RDONLY) };
+  if (in.fd == -1)
+    {
+      ec.assign(errno, std::generic_category());
+      return false;
+    }
+  int oflag = O_WRONLY|O_CREAT;
+  if (options.overwrite || options.update)
+    oflag |= O_TRUNC;
+  else
+    oflag |= O_EXCL;
+  CloseFD out = { ::open(to, oflag, S_IWUSR) };
+  if (out.fd == -1)
+    {
+      if (errno == EEXIST && options.skip)
+	ec.clear();
+      else
+	ec.assign(errno, std::generic_category());
+      return false;
+    }
+
+#ifdef _GLIBCXX_USE_FCHMOD
+  if (::fchmod(out.fd, from_st->st_mode))
+#elif defined _GLIBCXX_USE_FCHMODAT
+  if (::fchmodat(AT_FDCWD, to, from_st->st_mode, 0))
+#else
+  if (::chmod(to, from_st->st_mode))
+#endif
+    {
+      ec.assign(errno, std::generic_category());
+      return false;
+    }
+
+#ifdef _GLIBCXX_USE_SENDFILE
+  off_t offset = 0;
+  const auto n = ::sendfile(out.fd, in.fd, &offset, from_st->st_size);
+  if (n < 0 && (errno == ENOSYS || errno == EINVAL))
+    {
+#endif // _GLIBCXX_USE_SENDFILE
+      __gnu_cxx::stdio_filebuf<char> sbin(in.fd, std::ios::in);
+      __gnu_cxx::stdio_filebuf<char> sbout(out.fd, std::ios::out);
+      if (sbin.is_open())
+	in.fd = -1;
+      if (sbout.is_open())
+	out.fd = -1;
+      if (from_st->st_size && !(std::ostream(&sbout) << &sbin))
+	{
+	  ec = std::make_error_code(std::errc::io_error);
+	  return false;
+	}
+      if (!sbout.close() || !sbin.close())
+	{
+	  ec.assign(errno, std::generic_category());
+	  return false;
+	}
+
+      ec.clear();
+      return true;
+
+#ifdef _GLIBCXX_USE_SENDFILE
+    }
+  if (n != from_st->st_size)
+    {
+      ec.assign(errno, std::generic_category());
+      return false;
+    }
+  if (!out.close() || !in.close())
+    {
+      ec.assign(errno, std::generic_category());
+      return false;
+    }
+
+  ec.clear();
+  return true;
+#endif // _GLIBCXX_USE_SENDFILE
+}
+#endif // NEED_DO_COPY_FILE
+#endif // _GLIBCXX_HAVE_SYS_STAT_H
+
+void
+fs::copy(const path& from, const path& to, copy_options options,
+	 error_code& ec)
+{
+  const bool skip_symlinks = is_set(options, copy_options::skip_symlinks);
+  const bool create_symlinks = is_set(options, copy_options::create_symlinks);
+  const bool copy_symlinks = is_set(options, copy_options::copy_symlinks);
+  const bool use_lstat = create_symlinks || skip_symlinks;
+
+  file_status f, t;
+  stat_type from_st, to_st;
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2681. filesystem::copy() cannot copy symlinks
+  if (use_lstat || copy_symlinks
+      ? ::lstat(from.c_str(), &from_st)
+      : ::stat(from.c_str(), &from_st))
+    {
+      ec.assign(errno, std::generic_category());
+      return;
+    }
+  if (use_lstat
+      ? ::lstat(to.c_str(), &to_st)
+      : ::stat(to.c_str(), &to_st))
+    {
+      if (!is_not_found_errno(errno))
+	{
+	  ec.assign(errno, std::generic_category());
+	  return;
+	}
+      t = file_status{file_type::not_found};
+    }
+  else
+    t = make_file_status(to_st);
+  f = make_file_status(from_st);
+
+  if (exists(t) && !is_other(t) && !is_other(f)
+      && to_st.st_dev == from_st.st_dev && to_st.st_ino == from_st.st_ino)
+    {
+      ec = std::make_error_code(std::errc::file_exists);
+      return;
+    }
+  if (is_other(f) || is_other(t))
+    {
+      ec = std::make_error_code(std::errc::not_supported);
+      return;
+    }
+  if (is_directory(f) && is_regular_file(t))
+    {
+      ec = std::make_error_code(std::errc::is_a_directory);
+      return;
+    }
+
+  if (is_symlink(f))
+    {
+      if (skip_symlinks)
+	ec.clear();
+      else if (!exists(t) && copy_symlinks)
+	copy_symlink(from, to, ec);
+      else
+	// Not clear what should be done here.
+	// "Otherwise report an error as specified in Error reporting (7)."
+	ec = std::make_error_code(std::errc::invalid_argument);
+    }
+  else if (is_regular_file(f))
+    {
+      if (is_set(options, copy_options::directories_only))
+	ec.clear();
+      else if (create_symlinks)
+	create_symlink(from, to, ec);
+      else if (is_set(options, copy_options::create_hard_links))
+	create_hard_link(from, to, ec);
+      else if (is_directory(t))
+	do_copy_file(from.c_str(), (to / from.filename()).c_str(),
+		     copy_file_options(options), &from_st, nullptr, ec);
+      else
+	{
+	  auto ptr = exists(t) ? &to_st : &from_st;
+	  do_copy_file(from.c_str(), to.c_str(), copy_file_options(options),
+		       &from_st, ptr, ec);
+	}
+    }
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2682. filesystem::copy() won't create a symlink to a directory
+  else if (is_directory(f) && create_symlinks)
+    ec = std::make_error_code(errc::is_a_directory);
+  else if (is_directory(f) && (is_set(options, copy_options::recursive)
+			       || options == copy_options::none))
+    {
+      if (!exists(t))
+	if (!create_directory(to, from, ec))
+	  return;
+      // set an unused bit in options to disable further recursion
+      if (!is_set(options, copy_options::recursive))
+	options |= static_cast<copy_options>(4096);
+      for (const directory_entry& x : directory_iterator(from))
+	copy(x.path(), to/x.path().filename(), options, ec);
+    }
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2683. filesystem::copy() says "no effects"
+  else
+    ec.clear();
+}
+
+bool
+fs::copy_file(const path& from, const path& to, copy_options option)
+{
+  error_code ec;
+  bool result = copy_file(from, to, option, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot copy file", from, to,
+					     ec));
+  return result;
+}
+
+bool
+fs::copy_file(const path& from, const path& to, copy_options options,
+	      error_code& ec)
+{
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+  return do_copy_file(from.c_str(), to.c_str(), copy_file_options(options),
+		      nullptr, nullptr, ec);
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+  return false;
+#endif
+}
+
+
+void
+fs::copy_symlink(const path& existing_symlink, const path& new_symlink)
+{
+  error_code ec;
+  copy_symlink(existing_symlink, new_symlink, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot copy symlink",
+	  existing_symlink, new_symlink, ec));
+}
+
+void
+fs::copy_symlink(const path& existing_symlink, const path& new_symlink,
+		 error_code& ec) noexcept
+{
+  auto p = read_symlink(existing_symlink, ec);
+  if (ec)
+    return;
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  if (is_directory(p))
+    {
+      create_directory_symlink(p, new_symlink, ec);
+      return;
+    }
+#endif
+  create_symlink(p, new_symlink, ec);
+}
+
+
+bool
+fs::create_directories(const path& p)
+{
+  error_code ec;
+  bool result = create_directories(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot create directories", p,
+					     ec));
+  return result;
+}
+
+bool
+fs::create_directories(const path& p, error_code& ec)
+{
+  if (p.empty())
+    {
+      ec = std::make_error_code(errc::invalid_argument);
+      return false;
+    }
+  std::stack<path> missing;
+  path pp = p;
+
+  while (pp.has_filename() && status(pp, ec).type() == file_type::not_found)
+    {
+      ec.clear();
+      const auto& filename = pp.filename();
+      if (!is_dot(filename) && !is_dotdot(filename))
+	missing.push(pp);
+      pp = pp.parent_path();
+
+      if (missing.size() > 1000) // sanity check
+	{
+	  ec = std::make_error_code(std::errc::filename_too_long);
+	  return false;
+	}
+    }
+
+  if (ec || missing.empty())
+    return false;
+
+  do
+    {
+      const path& top = missing.top();
+      create_directory(top, ec);
+      if (ec && is_directory(top))
+	ec.clear();
+      missing.pop();
+    }
+  while (!missing.empty() && !ec);
+
+  return missing.empty();
+}
+
+namespace
+{
+  bool
+  create_dir(const fs::path& p, fs::perms perm, std::error_code& ec)
+  {
+    bool created = false;
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+    ::mode_t mode = static_cast<std::underlying_type_t<fs::perms>>(perm);
+    if (::mkdir(p.c_str(), mode))
+      {
+	const int err = errno;
+	if (err != EEXIST || !is_directory(p))
+	  ec.assign(err, std::generic_category());
+	else
+	  ec.clear();
+      }
+    else
+      {
+	ec.clear();
+	created = true;
+      }
+#else
+    ec = std::make_error_code(std::errc::not_supported);
+#endif
+    return created;
+  }
+} // namespace
+
+bool
+fs::create_directory(const path& p)
+{
+  error_code ec;
+  bool result = create_directory(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot create directory", p,
+	  ec));
+  return result;
+}
+
+bool
+fs::create_directory(const path& p, error_code& ec) noexcept
+{
+  return create_dir(p, perms::all, ec);
+}
+
+
+bool
+fs::create_directory(const path& p, const path& attributes)
+{
+  error_code ec;
+  bool result = create_directory(p, attributes, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot create directory", p,
+	  ec));
+  return result;
+}
+
+bool
+fs::create_directory(const path& p, const path& attributes,
+		     error_code& ec) noexcept
+{
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+  stat_type st;
+  if (::stat(attributes.c_str(), &st))
+    {
+      ec.assign(errno, std::generic_category());
+      return false;
+    }
+  return create_dir(p, static_cast<perms>(st.st_mode), ec);
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+  return false;
+#endif
+}
+
+
+void
+fs::create_directory_symlink(const path& to, const path& new_symlink)
+{
+  error_code ec;
+  create_directory_symlink(to, new_symlink, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot create directory symlink",
+	  to, new_symlink, ec));
+}
+
+void
+fs::create_directory_symlink(const path& to, const path& new_symlink,
+			     error_code& ec) noexcept
+{
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  ec = std::make_error_code(std::errc::not_supported);
+#else
+  create_symlink(to, new_symlink, ec);
+#endif
+}
+
+
+void
+fs::create_hard_link(const path& to, const path& new_hard_link)
+{
+  error_code ec;
+  create_hard_link(to, new_hard_link, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot create hard link",
+	  to, new_hard_link, ec));
+}
+
+void
+fs::create_hard_link(const path& to, const path& new_hard_link,
+		     error_code& ec) noexcept
+{
+#ifdef _GLIBCXX_HAVE_UNISTD_H
+  if (::link(to.c_str(), new_hard_link.c_str()))
+    ec.assign(errno, std::generic_category());
+  else
+    ec.clear();
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+}
+
+void
+fs::create_symlink(const path& to, const path& new_symlink)
+{
+  error_code ec;
+  create_symlink(to, new_symlink, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot create symlink",
+	  to, new_symlink, ec));
+}
+
+void
+fs::create_symlink(const path& to, const path& new_symlink,
+		   error_code& ec) noexcept
+{
+#ifdef _GLIBCXX_HAVE_UNISTD_H
+  if (::symlink(to.c_str(), new_symlink.c_str()))
+    ec.assign(errno, std::generic_category());
+  else
+    ec.clear();
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+}
+
+
+fs::path
+fs::current_path()
+{
+  error_code ec;
+  path p = current_path(ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot get current path", ec));
+  return p;
+}
+
+fs::path
+fs::current_path(error_code& ec)
+{
+  path p;
+#ifdef _GLIBCXX_HAVE_UNISTD_H
+#ifdef __GLIBC__
+  if (char_ptr cwd = char_ptr{::getcwd(nullptr, 0)})
+    {
+      p.assign(cwd.get());
+      ec.clear();
+    }
+  else
+    ec.assign(errno, std::generic_category());
+#else
+  long path_max = pathconf(".", _PC_PATH_MAX);
+  size_t size;
+  if (path_max == -1)
+      size = 1024;
+  else if (path_max > 10240)
+      size = 10240;
+  else
+      size = path_max;
+  for (char_ptr buf; p.empty(); size *= 2)
+    {
+      buf.reset((char*)malloc(size));
+      if (buf)
+	{
+	  if (getcwd(buf.get(), size))
+	    {
+	      p.assign(buf.get());
+	      ec.clear();
+	    }
+	  else if (errno != ERANGE)
+	    {
+	      ec.assign(errno, std::generic_category());
+	      return {};
+	    }
+	}
+      else
+	{
+	  ec = std::make_error_code(std::errc::not_enough_memory);
+	  return {};
+	}
+    }
+#endif  // __GLIBC__
+#else   // _GLIBCXX_HAVE_UNISTD_H
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+  return p;
+}
+
+void
+fs::current_path(const path& p)
+{
+  error_code ec;
+  current_path(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot set current path", ec));
+}
+
+void
+fs::current_path(const path& p, error_code& ec) noexcept
+{
+#ifdef _GLIBCXX_HAVE_UNISTD_H
+  if (::chdir(p.c_str()))
+    ec.assign(errno, std::generic_category());
+  else
+    ec.clear();
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+}
+
+bool
+fs::equivalent(const path& p1, const path& p2)
+{
+  error_code ec;
+  auto result = equivalent(p1, p2, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot check file equivalence",
+	  p1, p2, ec));
+  return result;
+}
+
+bool
+fs::equivalent(const path& p1, const path& p2, error_code& ec) noexcept
+{
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+  int err = 0;
+  file_status s1, s2;
+  stat_type st1, st2;
+  if (::stat(p1.c_str(), &st1) == 0)
+    s1 = make_file_status(st1);
+  else if (is_not_found_errno(errno))
+    s1.type(file_type::not_found);
+  else
+    err = errno;
+
+  if (::stat(p2.c_str(), &st2) == 0)
+    s2 = make_file_status(st2);
+  else if (is_not_found_errno(errno))
+    s2.type(file_type::not_found);
+  else
+    err = errno;
+
+  if (exists(s1) && exists(s2))
+    {
+      if (is_other(s1) && is_other(s2))
+	{
+	  ec = std::make_error_code(std::errc::not_supported);
+	  return false;
+	}
+      ec.clear();
+      if (is_other(s1) || is_other(s2))
+	return false;
+      return st1.st_dev == st2.st_dev && st1.st_ino == st2.st_ino;
+    }
+  else if (!exists(s1) && !exists(s2))
+    ec = std::make_error_code(std::errc::no_such_file_or_directory);
+  else if (err)
+    ec.assign(err, std::generic_category());
+  else
+    ec.clear();
+  return false;
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+  return false;
+}
+
+std::uintmax_t
+fs::file_size(const path& p)
+{
+  error_code ec;
+  auto sz = file_size(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot get file size", p, ec));
+  return sz;
+}
+
+namespace
+{
+  template<typename Accessor, typename T>
+    inline T
+    do_stat(const fs::path& p, std::error_code& ec, Accessor f, T deflt)
+    {
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+      fs::stat_type st;
+      if (::stat(p.c_str(), &st))
+	{
+	  ec.assign(errno, std::generic_category());
+	  return deflt;
+	}
+      ec.clear();
+      return f(st);
+#else
+      ec = std::make_error_code(std::errc::not_supported);
+      return deflt;
+#endif
+    }
+}
+
+std::uintmax_t
+fs::file_size(const path& p, error_code& ec) noexcept
+{
+  struct S
+  {
+    S(const stat_type& st) : type(make_file_type(st)), size(st.st_size) { }
+    S() : type(file_type::not_found) { }
+    file_type type;
+    size_t size;
+  };
+  auto s = do_stat(p, ec, [](const auto& st) { return S{st}; }, S{});
+  if (s.type == file_type::regular)
+    return s.size;
+  if (!ec)
+    {
+      if (s.type == file_type::directory)
+	ec = std::make_error_code(std::errc::is_a_directory);
+      else
+	ec = std::make_error_code(std::errc::not_supported);
+    }
+  return -1;
+}
+
+std::uintmax_t
+fs::hard_link_count(const path& p)
+{
+  error_code ec;
+  auto count = hard_link_count(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot get link count", p, ec));
+  return count;
+}
+
+std::uintmax_t
+fs::hard_link_count(const path& p, error_code& ec) noexcept
+{
+  return do_stat(p, ec, std::mem_fn(&stat::st_nlink),
+		 static_cast<uintmax_t>(-1));
+}
+
+bool
+fs::is_empty(const path& p)
+{
+  error_code ec;
+  bool e = is_empty(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot check if file is empty",
+					     p, ec));
+  return e;
+}
+
+bool
+fs::is_empty(const path& p, error_code& ec)
+{
+  auto s = status(p, ec);
+  if (ec)
+    return false;
+  bool empty = fs::is_directory(s)
+    ? fs::directory_iterator(p, ec) == fs::directory_iterator()
+    : fs::file_size(p, ec) == 0;
+  return ec ? false : empty;
+}
+
+fs::file_time_type
+fs::last_write_time(const path& p)
+{
+  error_code ec;
+  auto t = last_write_time(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot get file time", p, ec));
+  return t;
+}
+
+fs::file_time_type
+fs::last_write_time(const path& p, error_code& ec) noexcept
+{
+  return do_stat(p, ec, [&ec](const auto& st) { return file_time(st, ec); },
+		 file_time_type::min());
+}
+
+void
+fs::last_write_time(const path& p, file_time_type new_time)
+{
+  error_code ec;
+  last_write_time(p, new_time, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot set file time", p, ec));
+}
+
+void
+fs::last_write_time(const path& p __attribute__((__unused__)),
+		    file_time_type new_time, error_code& ec) noexcept
+{
+  auto d = new_time.time_since_epoch();
+  auto s = chrono::duration_cast<chrono::seconds>(d);
+#if _GLIBCXX_USE_UTIMENSAT
+  auto ns = chrono::duration_cast<chrono::nanoseconds>(d - s);
+  if (ns < ns.zero()) // tv_nsec must be non-negative and less than 10e9.
+    {
+      --s;
+      ns += chrono::seconds(1);
+    }
+  struct ::timespec ts[2];
+  ts[0].tv_sec = 0;
+  ts[0].tv_nsec = UTIME_OMIT;
+  ts[1].tv_sec = static_cast<std::time_t>(s.count());
+  ts[1].tv_nsec = static_cast<long>(ns.count());
+  if (::utimensat(AT_FDCWD, p.c_str(), ts, 0))
+    ec.assign(errno, std::generic_category());
+  else
+    ec.clear();
+#elif _GLIBCXX_HAVE_UTIME_H
+  ::utimbuf times;
+  times.modtime = s.count();
+  times.actime = do_stat(p, ec, [](const auto& st) { return st.st_atime; },
+			 times.modtime);
+  if (::utime(p.c_str(), &times))
+    ec.assign(errno, std::generic_category());
+  else
+    ec.clear();
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+}
+
+void
+fs::permissions(const path& p, perms prms, perm_options opts)
+{
+  error_code ec;
+  permissions(p, prms, opts, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot set permissions", p, ec));
+}
+
+void
+fs::permissions(const path& p, perms prms, perm_options opts,
+		error_code& ec) noexcept
+{
+  const bool replace = is_set(opts, perm_options::replace);
+  const bool add = is_set(opts, perm_options::add);
+  const bool remove = is_set(opts, perm_options::remove);
+  const bool nofollow = is_set(opts, perm_options::nofollow);
+  if (((int)replace + (int)add + (int)remove) != 1)
+    {
+      ec = std::make_error_code(std::errc::invalid_argument);
+      return;
+    }
+
+  prms &= perms::mask;
+
+  file_status st;
+  if (add || remove || nofollow)
+    {
+      st = nofollow ? symlink_status(p, ec) : status(p, ec);
+      if (ec)
+	return;
+      auto curr = st.permissions();
+      if (add)
+	prms |= curr;
+      else if (remove)
+	prms = curr & ~prms;
+    }
+
+  int err = 0;
+#if _GLIBCXX_USE_FCHMODAT
+  const int flag = (nofollow && is_symlink(st)) ? AT_SYMLINK_NOFOLLOW : 0;
+  if (::fchmodat(AT_FDCWD, p.c_str(), static_cast<mode_t>(prms), flag))
+    err = errno;
+#else
+  if (nofollow && is_symlink(st))
+    ec = std::make_error_code(std::errc::operation_not_supported);
+  else if (::chmod(p.c_str(), static_cast<mode_t>(prms)))
+    err = errno;
+#endif
+
+  if (err)
+    ec.assign(err, std::generic_category());
+  else
+    ec.clear();
+}
+
+fs::path
+fs::proximate(const path& p, const path& base)
+{
+  return weakly_canonical(p).lexically_proximate(weakly_canonical(base));
+}
+
+fs::path
+fs::proximate(const path& p, const path& base, error_code& ec)
+{
+  path result;
+  const auto p2 = weakly_canonical(p, ec);
+  if (!ec)
+    {
+      const auto base2 = weakly_canonical(base, ec);
+      if (!ec)
+	result = p2.lexically_proximate(base2);
+    }
+  return result;
+}
+
+fs::path
+fs::read_symlink(const path& p)
+{
+  error_code ec;
+  path tgt = read_symlink(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("read_symlink", p, ec));
+  return tgt;
+}
+
+fs::path fs::read_symlink(const path& p, error_code& ec)
+{
+  path result;
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+  stat_type st;
+  if (::lstat(p.c_str(), &st))
+    {
+      ec.assign(errno, std::generic_category());
+      return result;
+    }
+  std::string buf(st.st_size ? st.st_size + 1 : 128, '\0');
+  do
+    {
+      ssize_t len = ::readlink(p.c_str(), buf.data(), buf.size());
+      if (len == -1)
+	{
+	  ec.assign(errno, std::generic_category());
+	  return result;
+	}
+      else if (len == (ssize_t)buf.size())
+	{
+	  if (buf.size() > 4096)
+	    {
+	      ec.assign(ENAMETOOLONG, std::generic_category());
+	      return result;
+	    }
+	  buf.resize(buf.size() * 2);
+	}
+      else
+	{
+	  buf.resize(len);
+	  result.assign(buf);
+	  ec.clear();
+	  break;
+	}
+    }
+  while (true);
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+  return result;
+}
+
+fs::path
+fs::relative(const path& p, const path& base)
+{
+  return weakly_canonical(p).lexically_relative(weakly_canonical(base));
+}
+
+fs::path
+fs::relative(const path& p, const path& base, error_code& ec)
+{
+  auto result = weakly_canonical(p, ec);
+  fs::path cbase;
+  if (!ec)
+    cbase = weakly_canonical(base, ec);
+  if (!ec)
+    result = result.lexically_relative(cbase);
+  if (ec)
+    result.clear();
+  return result;
+}
+
+bool
+fs::remove(const path& p)
+{
+  error_code ec;
+  bool result = fs::remove(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot remove", p, ec));
+  return result;
+}
+
+bool
+fs::remove(const path& p, error_code& ec) noexcept
+{
+  if (exists(symlink_status(p, ec)))
+    {
+      if (::remove(p.c_str()) == 0)
+	{
+	  ec.clear();
+	  return true;
+	}
+      else
+	ec.assign(errno, std::generic_category());
+    }
+  return false;
+}
+
+
+std::uintmax_t
+fs::remove_all(const path& p)
+{
+  error_code ec;
+  bool result = remove_all(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot remove all", p, ec));
+  return result;
+}
+
+std::uintmax_t
+fs::remove_all(const path& p, error_code& ec)
+{
+  auto fs = symlink_status(p, ec);
+  uintmax_t count = 0;
+  if (!ec && fs.type() == file_type::directory)
+    for (directory_iterator d(p, ec), end; !ec && d != end; ++d)
+      count += fs::remove_all(d->path(), ec);
+  if (ec)
+    return -1;
+  return fs::remove(p, ec) ? ++count : -1;  // fs:remove() calls ec.clear()
+}
+
+void
+fs::rename(const path& from, const path& to)
+{
+  error_code ec;
+  rename(from, to, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot rename", from, to, ec));
+}
+
+void
+fs::rename(const path& from, const path& to, error_code& ec) noexcept
+{
+  if (::rename(from.c_str(), to.c_str()))
+    ec.assign(errno, std::generic_category());
+  else
+    ec.clear();
+}
+
+void
+fs::resize_file(const path& p, uintmax_t size)
+{
+  error_code ec;
+  resize_file(p, size, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot resize file", p, ec));
+}
+
+void
+fs::resize_file(const path& p, uintmax_t size, error_code& ec) noexcept
+{
+#ifdef _GLIBCXX_HAVE_UNISTD_H
+  if (size > static_cast<uintmax_t>(std::numeric_limits<off_t>::max()))
+    ec.assign(EINVAL, std::generic_category());
+  else if (::truncate(p.c_str(), size))
+    ec.assign(errno, std::generic_category());
+  else
+    ec.clear();
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+}
+
+
+fs::space_info
+fs::space(const path& p)
+{
+  error_code ec;
+  space_info s = space(p, ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("cannot get free space", p, ec));
+  return s;
+}
+
+fs::space_info
+fs::space(const path& p, error_code& ec) noexcept
+{
+  space_info info = {
+    static_cast<uintmax_t>(-1),
+    static_cast<uintmax_t>(-1),
+    static_cast<uintmax_t>(-1)
+  };
+#ifdef _GLIBCXX_HAVE_SYS_STATVFS_H
+  struct ::statvfs f;
+  if (::statvfs(p.c_str(), &f))
+      ec.assign(errno, std::generic_category());
+  else
+    {
+      info = space_info{
+	f.f_blocks * f.f_frsize,
+	f.f_bfree * f.f_frsize,
+	f.f_bavail * f.f_frsize
+      };
+      ec.clear();
+    }
+#else
+  ec = std::make_error_code(std::errc::not_supported);
+#endif
+  return info;
+}
+
+#ifdef _GLIBCXX_HAVE_SYS_STAT_H
+fs::file_status
+fs::status(const fs::path& p, error_code& ec) noexcept
+{
+  file_status status;
+  stat_type st;
+  if (::stat(p.c_str(), &st))
+    {
+      int err = errno;
+      ec.assign(err, std::generic_category());
+      if (is_not_found_errno(err))
+	status.type(file_type::not_found);
+#ifdef EOVERFLOW
+      else if (err == EOVERFLOW)
+	status.type(file_type::unknown);
+#endif
+    }
+  else
+    {
+      status = make_file_status(st);
+      ec.clear();
+    }
+  return status;
+}
+
+fs::file_status
+fs::symlink_status(const fs::path& p, std::error_code& ec) noexcept
+{
+  file_status status;
+  stat_type st;
+  if (::lstat(p.c_str(), &st))
+    {
+      int err = errno;
+      ec.assign(err, std::generic_category());
+      if (is_not_found_errno(err))
+	status.type(file_type::not_found);
+    }
+  else
+    {
+      status = make_file_status(st);
+      ec.clear();
+    }
+  return status;
+}
+#endif
+
+fs::file_status
+fs::status(const fs::path& p)
+{
+  std::error_code ec;
+  auto result = status(p, ec);
+  if (result.type() == file_type::none)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("status", p, ec));
+  return result;
+}
+
+fs::file_status
+fs::symlink_status(const fs::path& p)
+{
+  std::error_code ec;
+  auto result = symlink_status(p, ec);
+  if (result.type() == file_type::none)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("symlink_status", p, ec));
+  return result;
+}
+
+fs::path fs::temp_directory_path()
+{
+  error_code ec;
+  path tmp = temp_directory_path(ec);
+  if (ec)
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error("temp_directory_path", ec));
+  return tmp;
+}
+
+fs::path fs::temp_directory_path(error_code& ec)
+{
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  ec = std::make_error_code(std::errc::not_supported);
+  return {}; // TODO
+#else
+  const char* tmpdir = nullptr;
+  const char* env[] = { "TMPDIR", "TMP", "TEMP", "TEMPDIR", nullptr };
+  for (auto e = env; tmpdir == nullptr && *e != nullptr; ++e)
+    tmpdir = ::getenv(*e);
+  path p = tmpdir ? tmpdir : "/tmp";
+  auto st = status(p, ec);
+  if (!ec)
+    {
+      if (is_directory(st))
+	{
+	  ec.clear();
+	  return p;
+	}
+      else
+	ec = std::make_error_code(std::errc::not_a_directory);
+    }
+  return {};
+#endif
+}
+
+fs::path
+fs::weakly_canonical(const path& p)
+{
+  path result;
+  if (exists(status(p)))
+    return canonical(p);
+
+  path tmp;
+  auto iter = p.begin(), end = p.end();
+  // find leading elements of p that exist:
+  while (iter != end)
+    {
+      tmp = result / *iter;
+      if (exists(status(tmp)))
+	swap(result, tmp);
+      else
+	break;
+      ++iter;
+    }
+  // canonicalize:
+  result = canonical(result);
+  // append the non-existing elements:
+  while (iter != end)
+    result /= *iter++;
+  // normalize:
+  return result.lexically_normal();
+}
+
+fs::path
+fs::weakly_canonical(const path& p, error_code& ec)
+{
+  path result;
+  file_status st = status(p, ec);
+  if (exists(st))
+    return canonical(p, ec);
+  else if (status_known(st))
+    ec.clear();
+  else
+    return result;
+
+  path tmp;
+  auto iter = p.begin(), end = p.end();
+  // find leading elements of p that exist:
+  while (iter != end)
+    {
+      tmp = result / *iter;
+      st = status(tmp, ec);
+      if (exists(st))
+	swap(result, tmp);
+      else
+	{
+	  if (status_known(st))
+	    ec.clear();
+	  break;
+	}
+      ++iter;
+    }
+  // canonicalize:
+  if (!ec)
+    result = canonical(result, ec);
+  if (ec)
+    result.clear();
+  else
+    {
+      // append the non-existing elements:
+      while (iter != end)
+	result /= *iter++;
+      // normalize:
+      result = result.lexically_normal();
+    }
+  return result;
+}
diff --git a/libstdc++-v3/src/filesystem/std-path.cc b/libstdc++-v3/src/filesystem/std-path.cc
new file mode 100644
index 00000000000..330aee72b13
--- /dev/null
+++ b/libstdc++-v3/src/filesystem/std-path.cc
@@ -0,0 +1,715 @@
+// Class filesystem::path -*- C++ -*-
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef _GLIBCXX_USE_CXX11_ABI
+# define _GLIBCXX_USE_CXX11_ABI 1
+#endif
+
+#include <filesystem>
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+# include <algorithm>
+#endif
+
+namespace fs = std::filesystem;
+using fs::path;
+
+fs::filesystem_error::~filesystem_error() = default;
+
+constexpr path::value_type path::preferred_separator;
+
+path&
+path::remove_filename()
+{
+  if (_M_type == _Type::_Multi)
+    {
+      if (!_M_cmpts.empty())
+	{
+	  auto cmpt = std::prev(_M_cmpts.end());
+	  if (cmpt->_M_type == _Type::_Filename && !cmpt->empty())
+	    {
+	      _M_pathname.erase(cmpt->_M_pos);
+	      auto prev = std::prev(cmpt);
+	      if (prev->_M_type == _Type::_Root_dir
+		  || prev->_M_type == _Type::_Root_name)
+		{
+		  _M_cmpts.erase(cmpt);
+		  _M_trim();
+		}
+	      else
+		cmpt->clear();
+	    }
+	}
+    }
+  else if (_M_type == _Type::_Filename)
+    clear();
+  if (!empty() && _M_pathname.back() != '/')
+    throw 1;
+  return *this;
+}
+
+path&
+path::replace_filename(const path& replacement)
+{
+  remove_filename();
+  operator/=(replacement);
+  return *this;
+}
+
+path&
+path::replace_extension(const path& replacement)
+{
+  auto ext = _M_find_extension();
+  // Any existing extension() is removed
+  if (ext.first && ext.second != string_type::npos)
+    {
+      if (ext.first == &_M_pathname)
+	_M_pathname.erase(ext.second);
+      else
+	{
+	  const auto& back = _M_cmpts.back();
+	  if (ext.first != &back._M_pathname)
+	    _GLIBCXX_THROW_OR_ABORT(
+		std::logic_error("path::replace_extension failed"));
+	  _M_pathname.erase(back._M_pos + ext.second);
+	}
+    }
+   // If replacement is not empty and does not begin with a dot character,
+   // a dot character is appended
+  if (!replacement.empty() && replacement.native()[0] != '.')
+    _M_pathname += '.';
+  operator+=(replacement);
+  return *this;
+}
+
+namespace
+{
+  template<typename Iter1, typename Iter2>
+    int do_compare(Iter1 begin1, Iter1 end1, Iter2 begin2, Iter2 end2)
+    {
+      int cmpt = 1;
+      while (begin1 != end1 && begin2 != end2)
+	{
+	  if (begin1->native() < begin2->native())
+	    return -cmpt;
+	  if (begin1->native() > begin2->native())
+	    return +cmpt;
+	  ++begin1;
+	  ++begin2;
+	  ++cmpt;
+	}
+      if (begin1 == end1)
+	{
+	  if (begin2 == end2)
+	    return 0;
+	  return -cmpt;
+	}
+      return +cmpt;
+    }
+}
+
+int
+path::compare(const path& p) const noexcept
+{
+  struct CmptRef
+  {
+    const path* ptr;
+    const string_type& native() const noexcept { return ptr->native(); }
+  };
+
+  if (empty() && p.empty())
+    return 0;
+  else if (_M_type == _Type::_Multi && p._M_type == _Type::_Multi)
+    return do_compare(_M_cmpts.begin(), _M_cmpts.end(),
+		      p._M_cmpts.begin(), p._M_cmpts.end());
+  else if (_M_type == _Type::_Multi)
+    {
+      CmptRef c[1] = { { &p } };
+      return do_compare(_M_cmpts.begin(), _M_cmpts.end(), c, c+1);
+    }
+  else if (p._M_type == _Type::_Multi)
+    {
+      CmptRef c[1] = { { this } };
+      return do_compare(c, c+1, p._M_cmpts.begin(), p._M_cmpts.end());
+    }
+  else
+    return _M_pathname.compare(p._M_pathname);
+}
+
+path
+path::root_name() const
+{
+  path __ret;
+  if (_M_type == _Type::_Root_name)
+    __ret = *this;
+  else if (_M_cmpts.size() && _M_cmpts.begin()->_M_type == _Type::_Root_name)
+    __ret = *_M_cmpts.begin();
+  return __ret;
+}
+
+path
+path::root_directory() const
+{
+  path __ret;
+  if (_M_type == _Type::_Root_dir)
+    {
+      __ret._M_type = _Type::_Root_dir;
+      __ret._M_pathname.assign(1, preferred_separator);
+    }
+  else if (!_M_cmpts.empty())
+    {
+      auto __it = _M_cmpts.begin();
+      if (__it->_M_type == _Type::_Root_name)
+        ++__it;
+      if (__it != _M_cmpts.end() && __it->_M_type == _Type::_Root_dir)
+        __ret = *__it;
+    }
+  return __ret;
+}
+
+path
+path::root_path() const
+{
+  path __ret;
+  if (_M_type == _Type::_Root_name)
+    __ret = *this;
+  else if (_M_type == _Type::_Root_dir)
+    {
+      __ret._M_pathname.assign(1, preferred_separator);
+      __ret._M_type = _Type::_Root_dir;
+    }
+  else if (!_M_cmpts.empty())
+    {
+      auto __it = _M_cmpts.begin();
+      if (__it->_M_type == _Type::_Root_name)
+        {
+          __ret = *__it++;
+          if (__it != _M_cmpts.end() && __it->_M_type == _Type::_Root_dir)
+	    __ret /= *__it;
+        }
+      else if (__it->_M_type == _Type::_Root_dir)
+        __ret = *__it;
+    }
+  return __ret;
+}
+
+path
+path::relative_path() const
+{
+  path __ret;
+  if (_M_type == _Type::_Filename)
+    __ret = *this;
+  else if (!_M_cmpts.empty())
+    {
+      auto __it = _M_cmpts.begin();
+      if (__it->_M_type == _Type::_Root_name)
+        ++__it;
+      if (__it != _M_cmpts.end() && __it->_M_type == _Type::_Root_dir)
+        ++__it;
+      if (__it != _M_cmpts.end())
+        __ret.assign(_M_pathname.substr(__it->_M_pos));
+    }
+  return __ret;
+}
+
+path
+path::parent_path() const
+{
+  path __ret;
+  if (!has_relative_path())
+    __ret = *this;
+  else if (_M_cmpts.size() >= 2)
+    {
+      for (auto __it = _M_cmpts.begin(), __end = std::prev(_M_cmpts.end());
+	   __it != __end; ++__it)
+	{
+	  __ret /= *__it;
+	}
+    }
+  return __ret;
+}
+
+bool
+path::has_root_name() const
+{
+  if (_M_type == _Type::_Root_name)
+    return true;
+  if (!_M_cmpts.empty() && _M_cmpts.begin()->_M_type == _Type::_Root_name)
+    return true;
+  return false;
+}
+
+bool
+path::has_root_directory() const
+{
+  if (_M_type == _Type::_Root_dir)
+    return true;
+  if (!_M_cmpts.empty())
+    {
+      auto __it = _M_cmpts.begin();
+      if (__it->_M_type == _Type::_Root_name)
+        ++__it;
+      if (__it != _M_cmpts.end() && __it->_M_type == _Type::_Root_dir)
+        return true;
+    }
+  return false;
+}
+
+bool
+path::has_root_path() const
+{
+  if (_M_type == _Type::_Root_name || _M_type == _Type::_Root_dir)
+    return true;
+  if (!_M_cmpts.empty())
+    {
+      auto __type = _M_cmpts.front()._M_type;
+      if (__type == _Type::_Root_name || __type == _Type::_Root_dir)
+        return true;
+    }
+  return false;
+}
+
+bool
+path::has_relative_path() const
+{
+  if (_M_type == _Type::_Filename)
+    return true;
+  if (!_M_cmpts.empty())
+    {
+      auto __it = _M_cmpts.begin();
+      if (__it->_M_type == _Type::_Root_name)
+        ++__it;
+      if (__it != _M_cmpts.end() && __it->_M_type == _Type::_Root_dir)
+        ++__it;
+      if (__it != _M_cmpts.end())
+        return true;
+    }
+  return false;
+}
+
+
+bool
+path::has_parent_path() const
+{
+  if (!has_relative_path())
+    return !empty();
+  return _M_cmpts.size() >= 2;
+}
+
+bool
+path::has_filename() const
+{
+  if (empty())
+    return false;
+  if (_M_type == _Type::_Filename)
+    return !_M_pathname.empty();
+  if (_M_type == _Type::_Multi)
+    {
+      if (_M_pathname.back() == preferred_separator)
+	return false;
+      return _M_cmpts.back().has_filename();
+    }
+  return false;
+}
+
+namespace
+{
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  inline bool is_dot(wchar_t c) { return c == L'.'; }
+#else
+  inline bool is_dot(char c) { return c == '.'; }
+#endif
+
+  inline bool is_dot(const fs::path& path)
+  {
+    const auto& filename = path.native();
+    return filename.size() == 1 && is_dot(filename[0]);
+  }
+
+  inline bool is_dotdot(const fs::path& path)
+  {
+    const auto& filename = path.native();
+    return filename.size() == 2 && is_dot(filename[0]) && is_dot(filename[1]);
+  }
+} // namespace
+
+path
+path::lexically_normal() const
+{
+  /*
+  C++17 [fs.path.generic] p6
+  - If the path is empty, stop.
+  - Replace each slash character in the root-name with a preferred-separator.
+  - Replace each directory-separator with a preferred-separator.
+  - Remove each dot filename and any immediately following directory-separator.
+  - As long as any appear, remove a non-dot-dot filename immediately followed
+    by a directory-separator and a dot-dot filename, along with any immediately
+    following directory-separator.
+  - If there is a root-directory, remove all dot-dot filenames and any
+    directory-separators immediately following them.
+  - If the last filename is dot-dot, remove any trailing directory-separator.
+  - If the path is empty, add a dot.
+  */
+  path ret;
+  // If the path is empty, stop.
+  if (empty())
+    return ret;
+  for (auto& p : *this)
+    {
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+      // Replace each slash character in the root-name
+      if (p.is_root_name())
+	{
+	  string_type s = p.native();
+	  std::replace(s.begin(), s.end(), L'/', L'\\');
+	  ret /= s;
+	  continue;
+	}
+#endif
+      if (is_dotdot(p))
+	{
+	  if (ret.has_filename())
+	    {
+	      // remove a non-dot-dot filename immediately followed by /..
+	      if (!is_dotdot(ret.filename()))
+		ret.remove_filename();
+	      else
+		ret /= p;
+	    }
+	  else if (!ret.has_relative_path())
+	    {
+	      if (!ret.is_absolute())
+		ret /= p;
+	    }
+	  else
+	    {
+	      // Got a path with a relative path (i.e. at least one non-root
+	      // element) and no filename at the end (i.e. empty last element),
+	      // so must have a trailing slash. See what is before it.
+	      auto elem = std::prev(ret.end(), 2);
+	      if (elem->has_filename() && !is_dotdot(*elem))
+		{
+		  // Remove the filename before the trailing slash
+		  // (equiv. to ret = ret.parent_path().remove_filename())
+		  ret._M_pathname.erase(elem._M_cur->_M_pos);
+		  ret._M_cmpts.erase(elem._M_cur, ret._M_cmpts.end());
+		}
+	      else // ???
+		ret /= p;
+	    }
+	}
+      else if (is_dot(p))
+	ret /= path();
+      else
+	ret /= p;
+    }
+
+  if (ret._M_cmpts.size() >= 2)
+    {
+      auto back = std::prev(ret.end());
+      // If the last filename is dot-dot, ...
+      if (back->empty() && is_dotdot(*std::prev(back)))
+	// ... remove any trailing directory-separator.
+	ret = ret.parent_path();
+    }
+  // If the path is empty, add a dot.
+  else if (ret.empty())
+    ret = ".";
+
+  return ret;
+}
+
+path
+path::lexically_relative(const path& base) const
+{
+  path ret;
+  if (root_name() != base.root_name())
+    return ret;
+  if (is_absolute() != base.is_absolute())
+    return ret;
+  if (!has_root_directory() && base.has_root_directory())
+    return ret;
+  auto [a, b] = std::mismatch(begin(), end(), base.begin(), base.end());
+  if (a == end() && b == base.end())
+    ret = ".";
+  else
+  {
+    int n = 0;
+    for (; b != base.end(); ++b)
+    {
+      const path& p = *b;
+      if (is_dotdot(p))
+	--n;
+      else if (!is_dot(p))
+	++n;
+    }
+    if (n >= 0)
+    {
+      const path dotdot("..");
+      while (n--)
+	ret /= dotdot;
+      for (; a != end(); ++a)
+	ret /= *a;
+    }
+  }
+  return ret;
+}
+
+path
+path::lexically_proximate(const path& base) const
+{
+  path rel = lexically_relative(base);
+  if (rel.empty())
+    rel = *this;
+  return rel;
+}
+
+std::pair<const path::string_type*, std::size_t>
+path::_M_find_extension() const
+{
+  const std::string* s = nullptr;
+
+  if (_M_type == _Type::_Filename)
+    s = &_M_pathname;
+  else if (_M_type == _Type::_Multi && !_M_cmpts.empty())
+    {
+      const auto& c = _M_cmpts.back();
+      if (c._M_type == _Type::_Filename)
+	s = &c._M_pathname;
+    }
+
+  if (s)
+    {
+      if (auto sz = s->size())
+	{
+	  if (sz <= 2 && (*s)[0] == '.')
+	    return { s, string_type::npos };
+	  const auto pos = s->rfind('.');
+	  return { s, pos ? pos : string_type::npos };
+	}
+    }
+  return {};
+}
+
+void
+path::_M_split_cmpts()
+{
+  _M_type = _Type::_Multi;
+  _M_cmpts.clear();
+
+  if (_M_pathname.empty())
+    return;
+
+  size_t pos = 0;
+  const size_t len = _M_pathname.size();
+
+  // look for root name or root directory
+  if (_S_is_dir_sep(_M_pathname[0]))
+    {
+#ifdef __CYGWIN__
+      // look for root name, such as "//foo"
+      if (len > 2 && _M_pathname[1] == _M_pathname[0])
+	{
+	  if (!_S_is_dir_sep(_M_pathname[2]))
+	    {
+	      // got root name, find its end
+	      pos = 3;
+	      while (pos < len && !_S_is_dir_sep(_M_pathname[pos]))
+		++pos;
+	      _M_add_root_name(pos);
+	      if (pos < len) // also got root directory
+		_M_add_root_dir(pos);
+	    }
+	  else
+	    {
+	      // got something like "///foo" which is just a root directory
+	      // composed of multiple redundant directory separators
+	      _M_add_root_dir(0);
+	    }
+	}
+      else
+#endif
+        {
+	  // got root directory
+	  if (_M_pathname.find_first_not_of('/') == string_type::npos)
+	    {
+	      // entire path is just slashes
+	      _M_type = _Type::_Root_dir;
+	      return;
+	    }
+	  _M_add_root_dir(0);
+	  ++pos;
+	}
+    }
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  else if (len > 1 && _M_pathname[1] == L':')
+    {
+      // got disk designator
+      _M_add_root_name(2);
+      if (len > 2 && _S_is_dir_sep(_M_pathname[2]))
+	_M_add_root_dir(2);
+      pos = 2;
+    }
+#endif
+
+  size_t back = pos;
+  while (pos < len)
+    {
+      if (_S_is_dir_sep(_M_pathname[pos]))
+	{
+	  if (back != pos)
+	    _M_add_filename(back, pos - back);
+	  back = ++pos;
+	}
+      else
+	++pos;
+    }
+
+  if (back != pos)
+    _M_add_filename(back, pos - back);
+  else if (_S_is_dir_sep(_M_pathname.back()))
+    {
+      // [fs.path.itr]/4
+      // An empty element, if trailing non-root directory-separator present.
+      if (_M_cmpts.back()._M_type == _Type::_Filename)
+	{
+	  const auto& last = _M_cmpts.back();
+	  pos = last._M_pos + last._M_pathname.size();
+	  _M_cmpts.emplace_back(string_type(), _Type::_Filename, pos);
+	}
+    }
+
+  _M_trim();
+}
+
+void
+path::_M_add_root_name(size_t n)
+{
+  _M_cmpts.emplace_back(_M_pathname.substr(0, n), _Type::_Root_name, 0);
+}
+
+void
+path::_M_add_root_dir(size_t pos)
+{
+  _M_cmpts.emplace_back(_M_pathname.substr(pos, 1), _Type::_Root_dir, pos);
+}
+
+void
+path::_M_add_filename(size_t pos, size_t n)
+{
+  _M_cmpts.emplace_back(_M_pathname.substr(pos, n), _Type::_Filename, pos);
+}
+
+void
+path::_M_trim()
+{
+  if (_M_cmpts.size() == 1)
+    {
+      _M_type = _M_cmpts.front()._M_type;
+      _M_cmpts.clear();
+    }
+}
+
+path::string_type
+path::_S_convert_loc(const char* __first, const char* __last,
+		     const std::locale& __loc)
+{
+#if _GLIBCXX_USE_WCHAR_T
+  auto& __cvt = std::use_facet<codecvt<wchar_t, char, mbstate_t>>(__loc);
+  basic_string<wchar_t> __ws;
+  if (!__str_codecvt_in(__first, __last, __ws, __cvt))
+    _GLIBCXX_THROW_OR_ABORT(filesystem_error(
+	  "Cannot convert character sequence",
+	  std::make_error_code(errc::illegal_byte_sequence)));
+#ifdef _GLIBCXX_FILESYSTEM_IS_WINDOWS
+  return __ws;
+#else
+  return _Cvt<wchar_t>::_S_convert(__ws.data(), __ws.data() + __ws.size());
+#endif
+#else
+  return {__first, __last};
+#endif
+}
+
+std::size_t
+fs::hash_value(const path& p) noexcept
+{
+  // [path.non-member]
+  // "If for two paths, p1 == p2 then hash_value(p1) == hash_value(p2)."
+  // Equality works as if by traversing the range [begin(), end()), meaning
+  // e.g. path("a//b") == path("a/b"), so we cannot simply hash _M_pathname
+  // but need to iterate over individual elements. Use the hash_combine from
+  // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n3876.pdf
+  size_t seed = 0;
+  for (const auto& x : p)
+    {
+      seed ^= std::hash<path::string_type>()(x.native()) + 0x9e3779b9
+	+ (seed<<6) + (seed>>2);
+    }
+  return seed;
+}
+
+namespace std
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+namespace filesystem
+{
+  string
+  fs_err_concat(const string& __what, const string& __path1,
+		  const string& __path2)
+  {
+    const size_t __len = 18 + __what.length()
+      + (__path1.length() ? __path1.length() + 3 : 0)
+      + (__path2.length() ? __path2.length() + 3 : 0);
+    string __ret;
+    __ret.reserve(__len);
+    __ret = "filesystem error: ";
+    __ret += __what;
+    if (!__path1.empty())
+      {
+	__ret += " [";
+	__ret += __path1;
+	__ret += ']';
+      }
+    if (!__path2.empty())
+      {
+	__ret += " [";
+	__ret += __path2;
+	__ret += ']';
+      }
+    return __ret;
+  }
+
+_GLIBCXX_BEGIN_NAMESPACE_CXX11
+
+  std::string filesystem_error::_M_gen_what()
+  {
+    return fs_err_concat(system_error::what(), _M_path1.native(),
+			 _M_path2.native());
+  }
+
+_GLIBCXX_END_NAMESPACE_CXX11
+
+} // filesystem
+_GLIBCXX_END_NAMESPACE_VERSION
+} // std
diff --git a/libstdc++-v3/testsuite/18_support/byte/requirements.cc b/libstdc++-v3/testsuite/18_support/byte/requirements.cc
index 4cb05df0405..74c8b64d6ce 100644
--- a/libstdc++-v3/testsuite/18_support/byte/requirements.cc
+++ b/libstdc++-v3/testsuite/18_support/byte/requirements.cc
@@ -20,6 +20,12 @@
 
 #include <cstddef>
 
+#ifndef __cpp_lib_byte
+# error "Feature-test macro for byte missing"
+#elif __cpp_lib_byte != 201603
+# error "Feature-test macro for byte has wrong value"
+#endif
+
 static_assert( sizeof(std::byte) == sizeof(unsigned char) );
 static_assert( alignof(std::byte) == alignof(unsigned char) );
 
diff --git a/libstdc++-v3/testsuite/18_support/headers/limits/synopsis.cc b/libstdc++-v3/testsuite/18_support/headers/limits/synopsis.cc
index e298374920b..91fdf37be53 100644
--- a/libstdc++-v3/testsuite/18_support/headers/limits/synopsis.cc
+++ b/libstdc++-v3/testsuite/18_support/headers/limits/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/20_util/duration/arithmetic/constexpr_c++17.cc b/libstdc++-v3/testsuite/20_util/duration/arithmetic/constexpr_c++17.cc
index 438d50afddf..0ba1b8cc706 100644
--- a/libstdc++-v3/testsuite/20_util/duration/arithmetic/constexpr_c++17.cc
+++ b/libstdc++-v3/testsuite/20_util/duration/arithmetic/constexpr_c++17.cc
@@ -20,6 +20,13 @@
 
 #include <chrono>
 #include <testsuite_common_types.h>
+
+#ifndef __cpp_lib_chrono
+# error "Feature-test macro for constexpr <chrono> missing"
+#elif __cpp_lib_chrono != 201611
+# error "Feature-test macro for constexpr <chrono> has wrong value"
+#endif
+
 constexpr auto test_operators()
 {
   std::chrono::nanoseconds d1 { 1 };
diff --git a/libstdc++-v3/testsuite/20_util/endian/1.cc b/libstdc++-v3/testsuite/20_util/endian/1.cc
new file mode 100644
index 00000000000..2720c5edcdb
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/endian/1.cc
@@ -0,0 +1,36 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++2a" }
+// { dg-do compile { target c++2a } }
+
+#include <type_traits>
+
+static_assert( std::is_enum_v<std::endian> );
+static_assert( std::endian::little != std::endian::big );
+static_assert( std::endian::native == std::endian::big
+		|| std::endian::native == std::endian::little );
+
+namespace gnu {
+  int little, big, native;
+}
+
+using namespace std;
+using namespace gnu;
+
+// std::endian is a scoped-enum so these should refer to gnu::native etc.
+int test = little + big + native;
diff --git a/libstdc++-v3/testsuite/20_util/from_chars/requirements.cc b/libstdc++-v3/testsuite/20_util/from_chars/requirements.cc
index 00b7d875664..6afc918909a 100644
--- a/libstdc++-v3/testsuite/20_util/from_chars/requirements.cc
+++ b/libstdc++-v3/testsuite/20_util/from_chars/requirements.cc
@@ -17,6 +17,7 @@
 
 // { dg-options "-std=gnu++17" }
 // { dg-do compile { target c++17 } }
+// { dg-require-normal-namespace "" }
 
 #include <charconv>
 
diff --git a/libstdc++-v3/testsuite/20_util/headers/functional/synopsis.cc b/libstdc++-v3/testsuite/20_util/headers/functional/synopsis.cc
index 466d3d47965..c001daa955a 100644
--- a/libstdc++-v3/testsuite/20_util/headers/functional/synopsis.cc
+++ b/libstdc++-v3/testsuite/20_util/headers/functional/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/20_util/headers/memory/synopsis.cc b/libstdc++-v3/testsuite/20_util/headers/memory/synopsis.cc
index adf5f4829b9..95f42acb7cf 100644
--- a/libstdc++-v3/testsuite/20_util/headers/memory/synopsis.cc
+++ b/libstdc++-v3/testsuite/20_util/headers/memory/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/20_util/headers/utility/synopsis.cc b/libstdc++-v3/testsuite/20_util/headers/utility/synopsis.cc
index 71f1903f2d5..95308139d5d 100644
--- a/libstdc++-v3/testsuite/20_util/headers/utility/synopsis.cc
+++ b/libstdc++-v3/testsuite/20_util/headers/utility/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/20_util/pair/astuple/get.cc b/libstdc++-v3/testsuite/20_util/pair/astuple/get.cc
index 80939741f6c..e81af3b2f33 100644
--- a/libstdc++-v3/testsuite/20_util/pair/astuple/get.cc
+++ b/libstdc++-v3/testsuite/20_util/pair/astuple/get.cc
@@ -27,4 +27,9 @@ void test01()
 
   float&& pfirst __attribute__((unused)) = std::get<0>(std::move(p));
   int&&  psecond __attribute__((unused)) = std::get<1>(std::move(p));
+
+  const std::pair<float, int> cp;
+
+  const float&& cpfirst __attribute__((unused)) = std::get<0>(std::move(cp));
+  const int&&  cpsecond __attribute__((unused)) = std::get<1>(std::move(cp));
 }
diff --git a/libstdc++-v3/testsuite/20_util/pair/astuple/get_by_type.cc b/libstdc++-v3/testsuite/20_util/pair/astuple/get_by_type.cc
index 8d8bf0efe94..1e70fbcf43f 100644
--- a/libstdc++-v3/testsuite/20_util/pair/astuple/get_by_type.cc
+++ b/libstdc++-v3/testsuite/20_util/pair/astuple/get_by_type.cc
@@ -25,4 +25,11 @@ void test01()
 
   float&& pfirst __attribute__((unused)) = std::get<float>(std::move(p));
   int&&  psecond __attribute__((unused)) = std::get<int>(std::move(p));
+
+  const std::pair<float, int> cp;
+
+  const float&& cpfirst __attribute__((unused)) =
+    std::get<float>(std::move(cp));
+  const int&&  cpsecond __attribute__((unused)) =
+    std::get<int>(std::move(cp));
 }
diff --git a/libstdc++-v3/testsuite/20_util/specialized_algorithms/memory_management_tools/1.cc b/libstdc++-v3/testsuite/20_util/specialized_algorithms/memory_management_tools/1.cc
index 39d3e76ee82..d118dd18ce9 100644
--- a/libstdc++-v3/testsuite/20_util/specialized_algorithms/memory_management_tools/1.cc
+++ b/libstdc++-v3/testsuite/20_util/specialized_algorithms/memory_management_tools/1.cc
@@ -19,8 +19,8 @@
 
 #include <memory>
 #include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
 #include <string>
-#include <array>
 #include <vector>
 #include <sstream>
 
@@ -56,36 +56,42 @@ struct ThrowAfterN
   DelCount dc;
 };
 
+template<typename T>
+  using FwdIteratorRange
+    = __gnu_test::test_container<T, __gnu_test::forward_iterator_wrapper>;
+
 void test01()
 {
   char test_data[] = "123456";
-  std::uninitialized_default_construct(std::begin(test_data),
-				       std::end(test_data));
+  FwdIteratorRange<char> r(test_data);
+  std::uninitialized_default_construct(std::begin(r), std::end(r));
   VERIFY(std::string(test_data) == "123456");
 }
 
 void test02()
 {
   char test_data[] = "123456";
-  std::uninitialized_value_construct(std::begin(test_data),
-				       std::end(test_data));
+  FwdIteratorRange<char> r(test_data);
+  std::uninitialized_value_construct(std::begin(r), std::end(r));
   VERIFY(std::string(test_data, 6) == std::string("\0\0\0\0\0\0", 6));
 }
 
 void test03()
 {
   char test_data[] = "123456";
-  auto end = std::uninitialized_default_construct_n(std::begin(test_data), 6);
+  FwdIteratorRange<char> r(test_data);
+  auto end = std::uninitialized_default_construct_n(std::begin(r), 6);
   VERIFY(std::string(test_data) == "123456");
-  VERIFY( end == test_data + 6 );
+  VERIFY( end == std::next(r.begin(), 6) );
 }
 
 void test04()
 {
   char test_data[] = "123456";
-  auto end = std::uninitialized_value_construct_n(std::begin(test_data), 5);
+  FwdIteratorRange<char> r(test_data);
+  auto end = std::uninitialized_value_construct_n(std::begin(r), 5);
   VERIFY(std::string(test_data, 6) == std::string("\0\0\0\0\0" "6", 6));
-  VERIFY( end == test_data + 5 );
+  VERIFY( end == std::next(r.begin(), 5) );
 }
 
 void test05()
@@ -122,33 +128,43 @@ void test07()
   free(x);
 }
 
+struct MoveOnly
+{
+  MoveOnly() : val(-1) { }
+  MoveOnly(MoveOnly&& m) : val(m.val) { m.val = -1; }
+  int val;
+};
+
 void test08()
 {
-  std::vector<std::unique_ptr<int>> source;
-  for (int i = 0; i < 10; ++i) source.push_back(std::make_unique<int>(i));
-  std::unique_ptr<int>* target =
-    (std::unique_ptr<int>*)malloc(sizeof(std::unique_ptr<int>)*10);
-  std::uninitialized_move(source.begin(), source.end(), target);
-  for (const auto& x : source) VERIFY(!x);
-  for (int i = 0; i < 10; ++i) VERIFY(bool(*(target+i)));
-  auto end = std::destroy_n(target, 10);
-  VERIFY( end == target + 10 );
+  MoveOnly source[10];
+  for (int i = 0; i < 10; ++i) source[i].val = i;
+  FwdIteratorRange<MoveOnly> src(source);
+  MoveOnly* target = (MoveOnly*)malloc(sizeof(MoveOnly)*10);
+  FwdIteratorRange<MoveOnly> tgt(target, target+10);
+  auto end = std::uninitialized_move(src.begin(), src.end(), tgt.begin());
+  VERIFY( end == std::next(tgt.begin(), 10) );
+  for (const auto& x : source) VERIFY( x.val == -1 );
+  for (int i = 0; i < 10; ++i) VERIFY( target[i].val == i );
+  auto end2 = std::destroy_n(tgt.begin(), 10);
+  VERIFY( end2 == std::next(tgt.begin(), 10) );
   free(target);
 }
 
 void test09()
 {
-  std::vector<std::unique_ptr<int>> source;
-  for (int i = 0; i < 10; ++i) source.push_back(std::make_unique<int>(i));
-  std::unique_ptr<int>* target =
-    (std::unique_ptr<int>*)malloc(sizeof(std::unique_ptr<int>)*10);
-  auto end = std::uninitialized_move_n(source.begin(), 10, target);
-  VERIFY( end.first == source.begin() + 10 );
-  VERIFY( end.second == target + 10 );
-  for (const auto& x : source) VERIFY(!x);
-  for (int i = 0; i < 10; ++i) VERIFY(bool(*(target+i)));
-  auto end2 = std::destroy_n(target, 10);
-  VERIFY( end2 == target + 10 );
+  MoveOnly source[10];
+  for (int i = 0; i < 10; ++i) source[i].val = i;
+  FwdIteratorRange<MoveOnly> src(source);
+  MoveOnly* target = (MoveOnly*)malloc(sizeof(MoveOnly)*10);
+  FwdIteratorRange<MoveOnly> tgt(target, target+10);
+  auto end = std::uninitialized_move_n(src.begin(), 10, tgt.begin());
+  VERIFY( end.first == std::next(src.begin(), 10) );
+  VERIFY( end.second == std::next(tgt.begin(), 10) );
+  for (const auto& x : source) VERIFY( x.val == -1 );
+  for (int i = 0; i < 10; ++i) VERIFY( target[i].val == i );
+  auto end2 = std::destroy_n(tgt.begin(), 10);
+  VERIFY( end2 == std::next(tgt.begin(), 10) );
   free(target);
 }
 
@@ -156,7 +172,8 @@ void test10()
 {
   char* x = (char*)malloc(sizeof(char)*10);
   for (int i = 0; i < 10; ++i) new (x+i) char;
-  std::destroy(x, x+10);
+  FwdIteratorRange<char> r(x, x+10);
+  std::destroy(r.begin(), r.end());
   free(x);
 }
 
@@ -164,8 +181,9 @@ void test11()
 {
   char* x = (char*)malloc(sizeof(char)*10);
   for (int i = 0; i < 10; ++i) new (x+i) char;
-  auto end = std::destroy_n(x, 10);
-  VERIFY( end == x + 10 );
+  FwdIteratorRange<char> r(x, x+10);
+  auto end = std::destroy_n(r.begin(), 10);
+  VERIFY( end == std::next(r.begin(), 10) );
   free(x);
 }
 
diff --git a/libstdc++-v3/testsuite/20_util/to_chars/requirements.cc b/libstdc++-v3/testsuite/20_util/to_chars/requirements.cc
index d50588bd902..4c13d8ae71a 100644
--- a/libstdc++-v3/testsuite/20_util/to_chars/requirements.cc
+++ b/libstdc++-v3/testsuite/20_util/to_chars/requirements.cc
@@ -17,6 +17,7 @@
 
 // { dg-options "-std=gnu++17" }
 // { dg-do compile { target c++17 } }
+// { dg-require-normal-namespace "" }
 
 #include <charconv>
 
diff --git a/libstdc++-v3/testsuite/20_util/tuple/element_access/get2.cc b/libstdc++-v3/testsuite/20_util/tuple/element_access/get2.cc
index 14bdb0c6b1f..bc0f1bc899b 100644
--- a/libstdc++-v3/testsuite/20_util/tuple/element_access/get2.cc
+++ b/libstdc++-v3/testsuite/20_util/tuple/element_access/get2.cc
@@ -37,4 +37,19 @@ void test01()
   short&&  t3one __attribute__((unused)) = std::get<0>(std::move(t3));
   int&&    t3two __attribute__((unused)) = std::get<1>(std::move(t3));
   double&& t3thr __attribute__((unused)) = std::get<2>(std::move(t3));
+
+  const std::tuple<int> ct1;
+
+  const int&&   ct1one __attribute__((unused)) = std::get<0>(std::move(ct1));
+
+  const std::tuple<float, int> ct2;
+
+  const float&& ct2one __attribute__((unused)) = std::get<0>(std::move(ct2));
+  const int&&   ct2two __attribute__((unused)) = std::get<1>(std::move(ct2));
+
+  const std::tuple<short, int, double> ct3;
+
+  const short&&  ct3one __attribute__((unused)) = std::get<0>(std::move(ct3));
+  const int&&    ct3two __attribute__((unused)) = std::get<1>(std::move(ct3));
+  const double&& ct3thr __attribute__((unused)) = std::get<2>(std::move(ct3));
 }
diff --git a/libstdc++-v3/testsuite/20_util/tuple/element_access/get2_by_type.cc b/libstdc++-v3/testsuite/20_util/tuple/element_access/get2_by_type.cc
index 337934d57e2..cdc1030059a 100644
--- a/libstdc++-v3/testsuite/20_util/tuple/element_access/get2_by_type.cc
+++ b/libstdc++-v3/testsuite/20_util/tuple/element_access/get2_by_type.cc
@@ -35,4 +35,22 @@ void test01()
   short&&  t3one __attribute__((unused)) = std::get<short>(std::move(t3));
   int&&    t3two __attribute__((unused)) = std::get<int>(std::move(t3));
   double&& t3thr __attribute__((unused)) = std::get<double>(std::move(t3));
+
+  const std::tuple<int> ct1;
+
+  const int&&   ct1one __attribute__((unused)) = std::get<int>(std::move(ct1));
+
+  const std::tuple<float, int> ct2;
+
+  const float&& ct2one __attribute__((unused)) = std::get<0>(std::move(ct2));
+  const int&&   ct2two __attribute__((unused)) = std::get<int>(std::move(ct2));
+
+  const std::tuple<short, int, double> ct3;
+
+  const short&&  ct3one __attribute__((unused)) =
+    std::get<short>(std::move(ct3));
+  const int&&    ct3two __attribute__((unused)) =
+    std::get<int>(std::move(ct3));
+  const double&& ct3thr __attribute__((unused)) =
+    std::get<double>(std::move(ct3));
 }
diff --git a/libstdc++-v3/testsuite/21_strings/headers/string/synopsis.cc b/libstdc++-v3/testsuite/21_strings/headers/string/synopsis.cc
index d27d22076a1..568d846cba7 100644
--- a/libstdc++-v3/testsuite/21_strings/headers/string/synopsis.cc
+++ b/libstdc++-v3/testsuite/21_strings/headers/string/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/22_locale/headers/locale/synopsis.cc b/libstdc++-v3/testsuite/22_locale/headers/locale/synopsis.cc
index 7204fd42a59..236d2e39348 100644
--- a/libstdc++-v3/testsuite/22_locale/headers/locale/synopsis.cc
+++ b/libstdc++-v3/testsuite/22_locale/headers/locale/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get.cc b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get.cc
index a8c403c6cc7..8bc0afb0e9c 100644
--- a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get.cc
+++ b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get.cc
@@ -27,4 +27,9 @@ void test01()
 
   int&& aone __attribute__((unused)) = std::get<0>(std::move(a));
   int&& atwo __attribute__((unused)) = std::get<1>(std::move(a));
+
+  const std::array<int, 2> ca{};
+
+  const int&& caone __attribute__((unused)) = std::get<0>(std::move(ca));
+  const int&& catwo __attribute__((unused)) = std::get<1>(std::move(ca));
 }
diff --git a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_debug_neg.cc b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_debug_neg.cc
index e433c6e0a63..8e8ef0d756e 100644
--- a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_debug_neg.cc
+++ b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_debug_neg.cc
@@ -22,4 +22,4 @@
 
 typedef std::tuple_element<1, std::array<int, 1>>::type type;
 
-// { dg-error "static assertion failed" "" { target *-*-* } 323 }
+// { dg-error "static assertion failed" "" { target *-*-* } 331 }
diff --git a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_neg.cc b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_neg.cc
index 493449a235b..4e8eb3291cb 100644
--- a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_neg.cc
+++ b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/tuple_element_neg.cc
@@ -22,4 +22,4 @@
 
 typedef std::tuple_element<1, std::array<int, 1>>::type type;
 
-// { dg-error "static assertion failed" "" { target *-*-* } 357 }
+// { dg-error "static assertion failed" "" { target *-*-* } 365 }
diff --git a/libstdc++-v3/testsuite/23_containers/headers/bitset/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/bitset/synopsis.cc
index 6ef085a5dc8..8f07ce22bbb 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/bitset/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/bitset/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-do compile }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/deque/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/deque/synopsis.cc
index aa2b7871145..76f10a5f5f9 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/deque/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/deque/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-do compile }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/forward_list/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/forward_list/synopsis.cc
index b1792ceb4d9..c9f1df572af 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/forward_list/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/forward_list/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-do compile { target c++11 } }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2008-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/list/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/list/synopsis.cc
index ab22b9fce7e..bcc03764b05 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/list/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/list/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-do compile }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/map/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/map/synopsis.cc
index f4a08265293..8d43335cbb1 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/map/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/map/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-do compile }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/queue/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/queue/synopsis.cc
index f03d5778161..8c549b3c4f5 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/queue/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/queue/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/set/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/set/synopsis.cc
index e50a044977c..9a8df57b411 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/set/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/set/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-do compile }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/stack/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/stack/synopsis.cc
index f1bac94f0f2..aff07dcea0d 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/stack/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/stack/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc
index c127b5dee17..12a7197032a 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-do compile }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/map/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/map/cons/deduction.cc
new file mode 100644
index 00000000000..3880cd5e79d
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/map/cons/deduction.cc
@@ -0,0 +1,68 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <map>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::map{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}}),
+	      std::map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::map{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}}}),
+	      std::map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::map{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}},
+		    std::less<int>{}, {}}),
+	      std::map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::map{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}},
+		    {}}),
+	      std::map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::map{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}},
+		    {}, SimpleAllocator<std::pair<const int, double>>{}}),
+	      std::map<int, double, std::less<int>,
+	      SimpleAllocator<std::pair<const int, double>>>>);
+
+void f()
+{
+  std::map<int, double> x;
+  static_assert(std::is_same_v<
+		decltype(std::map(x.begin(), x.end())),
+		std::map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::map{x.begin(), x.end(),
+		      std::less<int>{},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::map<int, double>>);
+  
+  static_assert(std::is_same_v<
+		decltype(std::map{x.begin(), x.end(),
+		      std::less<int>{}, {}}),
+		std::map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::map(x.begin(), x.end(),
+				  {})),
+		std::map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::map{x.begin(), x.end(),
+		      {},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::map{x.begin(), x.end(),
+		      {},
+		      SimpleAllocator<std::pair<const int, double>>{}}),
+		std::map<int, double, std::less<int>,
+		SimpleAllocator<std::pair<const int, double>>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/map/modifiers/erase/abi_tag.cc b/libstdc++-v3/testsuite/23_containers/map/modifiers/erase/abi_tag.cc
index 65e85893781..7b5beee5238 100644
--- a/libstdc++-v3/testsuite/23_containers/map/modifiers/erase/abi_tag.cc
+++ b/libstdc++-v3/testsuite/23_containers/map/modifiers/erase/abi_tag.cc
@@ -1,5 +1,6 @@
 // { dg-do compile { target c++11 } }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2013-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/map/modifiers/insert/dr2354.cc b/libstdc++-v3/testsuite/23_containers/map/modifiers/insert/dr2354.cc
new file mode 100644
index 00000000000..338d9fd3f1e
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/map/modifiers/insert/dr2354.cc
@@ -0,0 +1,32 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do compile { target c++11 } }
+
+#include <map>
+
+struct MoveOnly {
+  MoveOnly(int) { }
+  MoveOnly(MoveOnly&&) = default;
+};
+
+void
+test01()
+{
+  std::map<int, MoveOnly> m;
+  m.insert({1, 2});  // PR libstdc++/82522  - LWG 2354
+}
diff --git a/libstdc++-v3/testsuite/23_containers/multimap/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/multimap/cons/deduction.cc
new file mode 100644
index 00000000000..ee48bfda26b
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/multimap/cons/deduction.cc
@@ -0,0 +1,68 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <map>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::multimap{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}}),
+	      std::multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multimap{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}}}),
+	      std::multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multimap{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}},
+		    std::less<int>{}, {}}),
+	      std::multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multimap{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}},
+		    {}}),
+	      std::multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multimap{{std::pair{1, 2.0}, {2, 3.0}, {3, 4.0}},
+		    {}, SimpleAllocator<std::pair<const int, double>>{}}),
+	      std::multimap<int, double, std::less<int>,
+	      SimpleAllocator<std::pair<const int, double>>>>);
+
+void f()
+{
+  std::multimap<int, double> x;
+  static_assert(std::is_same_v<
+		decltype(std::multimap(x.begin(), x.end())),
+		std::multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multimap{x.begin(), x.end(),
+		      std::less<int>{},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::multimap<int, double>>);
+  
+  static_assert(std::is_same_v<
+		decltype(std::multimap{x.begin(), x.end(),
+		      std::less<int>{}, {}}),
+		std::multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multimap(x.begin(), x.end(),
+				  {})),
+		std::multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multimap{x.begin(), x.end(),
+		      {},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multimap{x.begin(), x.end(),
+		      {},
+		      SimpleAllocator<std::pair<const int, double>>{}}),
+		std::multimap<int, double, std::less<int>,
+		SimpleAllocator<std::pair<const int, double>>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/multimap/modifiers/erase/abi_tag.cc b/libstdc++-v3/testsuite/23_containers/multimap/modifiers/erase/abi_tag.cc
index a25dba74edb..10d1efef3e8 100644
--- a/libstdc++-v3/testsuite/23_containers/multimap/modifiers/erase/abi_tag.cc
+++ b/libstdc++-v3/testsuite/23_containers/multimap/modifiers/erase/abi_tag.cc
@@ -1,5 +1,6 @@
 // { dg-do compile { target c++11 } }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2013-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/multimap/modifiers/insert/dr2354.cc b/libstdc++-v3/testsuite/23_containers/multimap/modifiers/insert/dr2354.cc
new file mode 100644
index 00000000000..ca743ec4ce9
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/multimap/modifiers/insert/dr2354.cc
@@ -0,0 +1,32 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do compile { target c++11 } }
+
+#include <map>
+
+struct MoveOnly {
+  MoveOnly(int) { }
+  MoveOnly(MoveOnly&&) = default;
+};
+
+void
+test01()
+{
+  std::multimap<int, MoveOnly> m;
+  m.insert({1, 2});  // PR libstdc++/82522  - LWG 2354
+}
diff --git a/libstdc++-v3/testsuite/23_containers/multiset/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/multiset/cons/deduction.cc
new file mode 100644
index 00000000000..4ca3f98129a
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/multiset/cons/deduction.cc
@@ -0,0 +1,68 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <set>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::multiset{1, 2, 3}),
+	      std::multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multiset{1, 2, 3}),
+	      std::multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multiset{{1, 2, 3},
+		    std::less<int>{}, {}}),
+	      std::multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multiset{{1, 2, 3},
+		    {}}),
+	      std::multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::multiset{{1, 2, 3},
+		    {}, SimpleAllocator<int>{}}),
+	      std::multiset<int, std::less<int>,
+	      SimpleAllocator<int>>>);
+
+void f()
+{
+  std::multiset<int> x;
+
+  static_assert(std::is_same_v<
+		decltype(std::multiset(x.begin(), x.end())),
+		std::multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multiset{x.begin(), x.end(),
+		      std::less<int>{},
+		      std::allocator<int>{}}),
+		std::multiset<int>>);
+  
+  static_assert(std::is_same_v<
+		decltype(std::multiset{x.begin(), x.end(),
+		      std::less<int>{}, {}}),
+		std::multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multiset(x.begin(), x.end(),
+				  {})),
+		std::multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multiset{x.begin(), x.end(),
+		      {},
+		      std::allocator<int>{}}),
+		std::multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::multiset{x.begin(), x.end(),
+		      {},
+		      SimpleAllocator<int>{}}),
+		std::multiset<int, std::less<int>, SimpleAllocator<int>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/multiset/modifiers/erase/abi_tag.cc b/libstdc++-v3/testsuite/23_containers/multiset/modifiers/erase/abi_tag.cc
index fafe4b6609c..0ea0c7ea229 100644
--- a/libstdc++-v3/testsuite/23_containers/multiset/modifiers/erase/abi_tag.cc
+++ b/libstdc++-v3/testsuite/23_containers/multiset/modifiers/erase/abi_tag.cc
@@ -1,5 +1,6 @@
 // { dg-do compile { target c++11 } }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2013-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/set/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/set/cons/deduction.cc
new file mode 100644
index 00000000000..73d5cfdd227
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/set/cons/deduction.cc
@@ -0,0 +1,68 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <set>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::set{1, 2, 3}),
+	      std::set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::set{1, 2, 3}),
+	      std::set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::set{{1, 2, 3},
+		    std::less<int>{}, {}}),
+	      std::set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::set{{1, 2, 3},
+		    {}}),
+	      std::set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::set{{1, 2, 3},
+		    {}, SimpleAllocator<int>{}}),
+	      std::set<int, std::less<int>,
+	      SimpleAllocator<int>>>);
+
+void f()
+{
+  std::set<int> x;
+
+  static_assert(std::is_same_v<
+		decltype(std::set(x.begin(), x.end())),
+		std::set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::set{x.begin(), x.end(),
+		      std::less<int>{},
+		      std::allocator<int>{}}),
+		std::set<int>>);
+  
+  static_assert(std::is_same_v<
+		decltype(std::set{x.begin(), x.end(),
+		      std::less<int>{}, {}}),
+		std::set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::set(x.begin(), x.end(),
+				  {})),
+		std::set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::set{x.begin(), x.end(),
+		      {},
+		      std::allocator<int>{}}),
+		std::set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::set{x.begin(), x.end(),
+		      {},
+		      SimpleAllocator<int>{}}),
+		std::set<int, std::less<int>, SimpleAllocator<int>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/set/modifiers/erase/abi_tag.cc b/libstdc++-v3/testsuite/23_containers/set/modifiers/erase/abi_tag.cc
index 9734e6ab437..f89797a4780 100644
--- a/libstdc++-v3/testsuite/23_containers/set/modifiers/erase/abi_tag.cc
+++ b/libstdc++-v3/testsuite/23_containers/set/modifiers/erase/abi_tag.cc
@@ -1,5 +1,6 @@
 // { dg-do compile { target c++11 } }
 // { dg-require-normal-mode "" }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2013-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_map/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/unordered_map/cons/deduction.cc
new file mode 100644
index 00000000000..1905b20116b
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_map/cons/deduction.cc
@@ -0,0 +1,77 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <unordered_map>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_map{std::pair{1, 2.0},
+		    {2, 3.0}, {3, 4.0}}),
+	      std::unordered_map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_map{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}}}),
+	      std::unordered_map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_map{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}},
+		    {}, std::hash<int>{}, {}}),
+	      std::unordered_map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_map{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}},
+		    {}}),
+	      std::unordered_map<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_map{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}},
+		    {}, {}, {},
+		    SimpleAllocator<std::pair<const int, double>>{}}),
+	      std::unordered_map<int, double, std::hash<int>,
+	      std::equal_to<int>,
+	      SimpleAllocator<std::pair<const int, double>>>>);
+
+
+void f()
+{
+  std::unordered_map<int, double> x;
+  static_assert(std::is_same_v<
+		decltype(std::unordered_map(x.begin(), x.end())),
+		std::unordered_map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_map{x.begin(), x.end(),
+		      {}, std::hash<int>{}, {},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::unordered_map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_map{x.begin(), x.end(),
+		      {}, std::hash<int>{}, {}}),
+		std::unordered_map<int, double>>);
+  
+  static_assert(std::is_same_v<
+		decltype(std::unordered_map(x.begin(), x.end(),
+				  {})),
+		std::unordered_map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_map{x.begin(), x.end(),
+		      {}, {}, {},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::unordered_map<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_map{x.begin(), x.end(),
+		      {}, {}, {},
+		      SimpleAllocator<std::pair<const int, double>>{}}),
+		std::unordered_map<int, double, std::hash<int>,
+		std::equal_to<int>,
+		SimpleAllocator<std::pair<const int, double>>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_map/insert/dr2354.cc b/libstdc++-v3/testsuite/23_containers/unordered_map/insert/dr2354.cc
new file mode 100644
index 00000000000..fe5356594c3
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_map/insert/dr2354.cc
@@ -0,0 +1,32 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do compile { target c++11 } }
+
+#include <unordered_map>
+
+struct MoveOnly {
+  MoveOnly(int) { }
+  MoveOnly(MoveOnly&&) = default;
+};
+
+void
+test01()
+{
+  std::unordered_map<int, MoveOnly> m;
+  m.insert({1, 2});  // PR libstdc++/82522  - LWG 2354
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multimap/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/unordered_multimap/cons/deduction.cc
new file mode 100644
index 00000000000..db5e32b4ad2
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multimap/cons/deduction.cc
@@ -0,0 +1,77 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <unordered_map>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multimap{std::pair{1, 2.0},
+		    {2, 3.0}, {3, 4.0}}),
+	      std::unordered_multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multimap{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}}}),
+	      std::unordered_multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multimap{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}},
+		    {}, std::hash<int>{}, {}}),
+	      std::unordered_multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multimap{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}},
+		    {}}),
+	      std::unordered_multimap<int, double>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multimap{{std::pair{1, 2.0},
+		      {2, 3.0}, {3, 4.0}},
+		    {}, {}, {},
+		    SimpleAllocator<std::pair<const int, double>>{}}),
+	      std::unordered_multimap<int, double, std::hash<int>,
+	      std::equal_to<int>,
+	      SimpleAllocator<std::pair<const int, double>>>>);
+
+
+void f()
+{
+  std::unordered_multimap<int, double> x;
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multimap(x.begin(), x.end())),
+		std::unordered_multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multimap{x.begin(), x.end(),
+		      {}, std::hash<int>{}, {},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::unordered_multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multimap{x.begin(), x.end(),
+		      {}, std::hash<int>{}, {}}),
+		std::unordered_multimap<int, double>>);
+  
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multimap(x.begin(), x.end(),
+				  {})),
+		std::unordered_multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multimap{x.begin(), x.end(),
+		      {}, {}, {},
+		      std::allocator<std::pair<const int, double>>{}}),
+		std::unordered_multimap<int, double>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multimap{x.begin(), x.end(),
+		      {}, {}, {},
+		      SimpleAllocator<std::pair<const int, double>>{}}),
+		std::unordered_multimap<int, double, std::hash<int>,
+		std::equal_to<int>,
+		SimpleAllocator<std::pair<const int, double>>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multimap/insert/dr2354.cc b/libstdc++-v3/testsuite/23_containers/unordered_multimap/insert/dr2354.cc
new file mode 100644
index 00000000000..5a27242c4e0
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multimap/insert/dr2354.cc
@@ -0,0 +1,32 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do compile { target c++11 } }
+
+#include <unordered_map>
+
+struct MoveOnly {
+  MoveOnly(int) { }
+  MoveOnly(MoveOnly&&) = default;
+};
+
+void
+test01()
+{
+  std::unordered_multimap<int, MoveOnly> m;
+  m.insert({1, 2});  // PR libstdc++/82522  - LWG 2354
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multiset/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/unordered_multiset/cons/deduction.cc
new file mode 100644
index 00000000000..352176d1e7d
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multiset/cons/deduction.cc
@@ -0,0 +1,78 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <unordered_set>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multiset{1, 2, 3}),
+	      std::unordered_multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multiset{1, 2, 3}),
+	      std::unordered_multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multiset{{1, 2, 3},
+		    0, std::hash<int>{}, {}}),
+	      std::unordered_multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multiset{{1, 2, 3},
+		    {}}),
+	      std::unordered_multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multiset{{1, 2, 3},
+		    {}, {}, {}, std::allocator<int>{}}),
+	      std::unordered_multiset<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_multiset{{1, 2, 3},
+		    {}, {}, {}, SimpleAllocator<int>{}}),
+	      std::unordered_multiset<int, std::hash<int>,
+	      std::equal_to<int>,
+	      SimpleAllocator<int>>>);
+
+void f()
+{
+  std::unordered_multiset<int> x;
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multiset(x.begin(), x.end())),
+		std::unordered_multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multiset{x.begin(), x.end(),
+		      {},
+		      std::hash<int>{},
+		      std::equal_to<int>{},
+		      std::allocator<int>{}}),
+		std::unordered_multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multiset{x.begin(), x.end(),
+		      {}, std::hash<int>{}, {}}),
+		std::unordered_multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multiset(x.begin(), x.end(),
+				  {})),
+		std::unordered_multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multiset{x.begin(), x.end(),
+		      {}, {}, {},
+		      std::allocator<int>{}}),
+		std::unordered_multiset<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_multiset{x.begin(), x.end(),
+		      {}, {}, {},
+		      SimpleAllocator<int>{}}),
+		std::unordered_multiset<int, std::hash<int>,
+		std::equal_to<int>,
+		SimpleAllocator<int>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_set/cons/deduction.cc b/libstdc++-v3/testsuite/23_containers/unordered_set/cons/deduction.cc
new file mode 100644
index 00000000000..c7e1798ef7d
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_set/cons/deduction.cc
@@ -0,0 +1,78 @@
+// { dg-options "-std=gnu++17" }
+// { dg-do compile { target c++17 } }
+
+#include <unordered_set>
+#include <testsuite_allocator.h>
+
+using __gnu_test::SimpleAllocator;
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_set{1, 2, 3}),
+	      std::unordered_set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_set{1, 2, 3}),
+	      std::unordered_set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_set{{1, 2, 3},
+		    0, std::hash<int>{}, {}}),
+	      std::unordered_set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_set{{1, 2, 3},
+		    {}}),
+	      std::unordered_set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_set{{1, 2, 3},
+		    {}, {}, {}, std::allocator<int>{}}),
+	      std::unordered_set<int>>);
+
+static_assert(std::is_same_v<
+	      decltype(std::unordered_set{{1, 2, 3},
+		    {}, {}, {}, SimpleAllocator<int>{}}),
+	      std::unordered_set<int, std::hash<int>,
+	      std::equal_to<int>,
+	      SimpleAllocator<int>>>);
+
+void f()
+{
+  std::unordered_set<int> x;
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_set(x.begin(), x.end())),
+		std::unordered_set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_set{x.begin(), x.end(),
+		      {},
+		      std::hash<int>{},
+		      std::equal_to<int>{},
+		      std::allocator<int>{}}),
+		std::unordered_set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_set{x.begin(), x.end(),
+		      {}, std::hash<int>{}, {}}),
+		std::unordered_set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_set(x.begin(), x.end(),
+				  {})),
+		std::unordered_set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_set{x.begin(), x.end(),
+		      {}, {}, {},
+		      std::allocator<int>{}}),
+		std::unordered_set<int>>);
+
+  static_assert(std::is_same_v<
+		decltype(std::unordered_set{x.begin(), x.end(),
+		      {}, {}, {},
+		      SimpleAllocator<int>{}}),
+		std::unordered_set<int, std::hash<int>,
+		std::equal_to<int>,
+		SimpleAllocator<int>>>);
+}
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/82558.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/82558.cc
new file mode 100644
index 00000000000..6362688efb5
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/82558.cc
@@ -0,0 +1,32 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 23.3.8  class vector<bool>
+
+#include <vector>
+
+// libstdc++/82558
+void test01()
+{
+  std::vector<bool> v;
+  std::fill(v.begin(), v.begin(), false);
+}
+
+int main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis.cc b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis.cc
index 64659c59e4f..58b731a77d3 100644
--- a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis.cc
+++ b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis.cc
@@ -1,5 +1,6 @@
 // { dg-options "-std=gnu++98" }
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++11.cc b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++11.cc
index ae3c2652ee0..87f1488d016 100644
--- a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++11.cc
+++ b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++11.cc
@@ -1,5 +1,6 @@
 // { dg-options "-std=gnu++11" }
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2016-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++14.cc b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++14.cc
index de7ae1fd38a..7370a52d44b 100644
--- a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++14.cc
+++ b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++14.cc
@@ -1,5 +1,6 @@
 // { dg-options "-std=gnu++14" }
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2016-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++17.cc b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++17.cc
index 0cfab7d3bf4..8f4dd433fca 100644
--- a/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++17.cc
+++ b/libstdc++-v3/testsuite/24_iterators/headers/iterator/synopsis_c++17.cc
@@ -1,5 +1,6 @@
 // { dg-options "-std=gnu++17" }
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2016-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/25_algorithms/fill_n/1.cc b/libstdc++-v3/testsuite/25_algorithms/fill_n/1.cc
index d446e7a7b60..4d27f0bd22b 100644
--- a/libstdc++-v3/testsuite/25_algorithms/fill_n/1.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/fill_n/1.cc
@@ -22,21 +22,36 @@
 #include <algorithm>
 #include <vector>
 #include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+// Non-scalar type to exercise partial specialization of fill_n implementation.
+struct Value
+{
+  Value(int n) : n(n) { }
+
+  operator int() const { return n; }
+
+private:
+  int n;
+};
 
 void
 test01()
 {
   using namespace std;
+  using __gnu_test::test_container;
+  using __gnu_test::output_iterator_wrapper;
 
   const int A1[] = {3, 3, 3, 3, 3, 3, 3, 3, 3, 3};
   const int N1 = sizeof(A1) / sizeof(int);
   
   int i1[N1];
-  fill_n(i1, N1, 3);
+  test_container<int, output_iterator_wrapper> c1(i1, i1 + N1);
+  fill_n(c1.begin(), N1, 3);
   VERIFY( equal(i1, i1 + N1, A1) );
 
   vector<int> v1(N1);
-  fill_n(v1.begin(), N1, 3);
+  fill_n(v1.begin(), N1, Value(3));
   VERIFY( equal(v1.begin(), v1.end(), A1) );
 
   const char A2[] = {'\3', '\3', '\3', '\3', '\3',
diff --git a/libstdc++-v3/testsuite/25_algorithms/generate_n/1.cc b/libstdc++-v3/testsuite/25_algorithms/generate_n/1.cc
new file mode 100644
index 00000000000..dc3cb9f943b
--- /dev/null
+++ b/libstdc++-v3/testsuite/25_algorithms/generate_n/1.cc
@@ -0,0 +1,47 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include <algorithm>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+struct Inc
+{
+  int operator()() { return ++i; }
+
+  int i;
+};
+
+void
+test01()
+{
+  const int N = 3;
+  int array[N];
+
+  using __gnu_test::test_container;
+  using __gnu_test::output_iterator_wrapper;
+  test_container<int, output_iterator_wrapper> c(array, array + N);
+  std::generate_n(c.begin(), N, Inc());
+  VERIFY(array[0] == 1);
+  VERIFY(array[1] == 2);
+  VERIFY(array[2] == 3);
+}
+
+int main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/25_algorithms/stable_partition/1.cc b/libstdc++-v3/testsuite/25_algorithms/stable_partition/1.cc
index 75851578267..f29d8170a83 100644
--- a/libstdc++-v3/testsuite/25_algorithms/stable_partition/1.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/stable_partition/1.cc
@@ -21,6 +21,7 @@
 #include <functional>
 #include <testsuite_new_operators.h>
 #include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
 
 const int A[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17};
 const int B[] = {2, 4, 6, 8, 10, 12, 14, 16, 1, 3, 5, 7, 9, 11, 13, 15, 17};
@@ -41,11 +42,16 @@ void
 test01()
 {
   using std::stable_partition;
+  using __gnu_test::test_container;
+  using __gnu_test::forward_iterator_wrapper;
 
   int s1[N];
   std::copy(A, A + N, s1);
 
-  VERIFY( stable_partition(s1, s1 + N, Pred()) == s1 + M );
+  test_container<int, forward_iterator_wrapper> c(s1, s1+N);
+  forward_iterator_wrapper<int> expected = c.begin();
+  std::advance(expected, M);
+  VERIFY( stable_partition(c.begin(), c.end(), Pred()) == expected);
   VERIFY( std::equal(s1, s1 + N, B) );
 }
 
diff --git a/libstdc++-v3/testsuite/26_numerics/complex/abi_tag.cc b/libstdc++-v3/testsuite/26_numerics/complex/abi_tag.cc
index 09833add310..2f8569eb8b9 100644
--- a/libstdc++-v3/testsuite/26_numerics/complex/abi_tag.cc
+++ b/libstdc++-v3/testsuite/26_numerics/complex/abi_tag.cc
@@ -1,5 +1,6 @@
 // Test that the C++11 variants of real/imag have an ABI tag
 // { dg-do compile { target c++11 } }
+// { dg-require-normal-namespace "" }
 
 #include <complex>
 
diff --git a/libstdc++-v3/testsuite/26_numerics/headers/cmath/82644.cc b/libstdc++-v3/testsuite/26_numerics/headers/cmath/82644.cc
new file mode 100644
index 00000000000..956541a67d9
--- /dev/null
+++ b/libstdc++-v3/testsuite/26_numerics/headers/cmath/82644.cc
@@ -0,0 +1,27 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-D__STDCPP_WANT_MATH_SPEC_FUNCS__ -D__STRICT_ANSI__" }
+// { dg-do compile { target c++11 } }
+
+#define conf_hyperg 1
+#define conf_hypergf 2
+#define conf_hypergl 3
+#define hyperg 4
+#define hypergf 5
+#define hypergl 6
+#include <cmath>     // PR libstdc++/82644
diff --git a/libstdc++-v3/testsuite/26_numerics/headers/cmath/functions_global_c++17.cc b/libstdc++-v3/testsuite/26_numerics/headers/cmath/functions_global_c++17.cc
new file mode 100644
index 00000000000..ce3a0dd4ac2
--- /dev/null
+++ b/libstdc++-v3/testsuite/26_numerics/headers/cmath/functions_global_c++17.cc
@@ -0,0 +1,111 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17" }
+// { dg-do compile }
+
+#include <math.h>
+
+namespace gnu
+{
+  using ::acos;
+  using ::asin;
+  using ::atan;
+  using ::atan2;
+  using ::ceil;
+  using ::cos;
+  using ::cosh;
+  using ::exp;
+  using ::fabs;
+  using ::floor;
+  using ::fmod;
+  using ::frexp;
+  using ::ldexp;
+  using ::log;
+  using ::log10;
+  using ::modf;
+  using ::pow;
+  using ::sin;
+  using ::sinh;
+  using ::sqrt;
+  using ::tan;
+  using ::tanh;
+
+  using ::assoc_laguerre;
+  using ::assoc_laguerref;
+  using ::assoc_laguerrel;
+  using ::assoc_legendre;
+  using ::assoc_legendref;
+  using ::assoc_legendrel;
+  using ::beta;
+  using ::betaf;
+  using ::betal;
+  using ::comp_ellint_1;
+  using ::comp_ellint_1f;
+  using ::comp_ellint_1l;
+  using ::comp_ellint_2;
+  using ::comp_ellint_2f;
+  using ::comp_ellint_2l;
+  using ::comp_ellint_3;
+  using ::comp_ellint_3f;
+  using ::comp_ellint_3l;
+  using ::cyl_bessel_i;
+  using ::cyl_bessel_if;
+  using ::cyl_bessel_il;
+  using ::cyl_bessel_j;
+  using ::cyl_bessel_jf;
+  using ::cyl_bessel_jl;
+  using ::cyl_bessel_k;
+  using ::cyl_bessel_kf;
+  using ::cyl_bessel_kl;
+  using ::cyl_neumann;
+  using ::cyl_neumannf;
+  using ::cyl_neumannl;
+  using ::ellint_1;
+  using ::ellint_1f;
+  using ::ellint_1l;
+  using ::ellint_2;
+  using ::ellint_2f;
+  using ::ellint_2l;
+  using ::ellint_3;
+  using ::ellint_3f;
+  using ::ellint_3l;
+  using ::expint;
+  using ::expintf;
+  using ::expintl;
+  using ::hermite;
+  using ::hermitef;
+  using ::hermitel;
+  using ::laguerre;
+  using ::laguerref;
+  using ::laguerrel;
+  using ::legendre;
+  using ::legendref;
+  using ::legendrel;
+  using ::riemann_zeta;
+  using ::riemann_zetaf;
+  using ::riemann_zetal;
+  using ::sph_bessel;
+  using ::sph_besself;
+  using ::sph_bessell;
+  using ::sph_legendre;
+  using ::sph_legendref;
+  using ::sph_legendrel;
+  using ::sph_neumann;
+  using ::sph_neumannf;
+  using ::sph_neumannl;
+}
diff --git a/libstdc++-v3/testsuite/26_numerics/headers/complex/synopsis.cc b/libstdc++-v3/testsuite/26_numerics/headers/complex/synopsis.cc
index 11531862882..7ab5e7628e5 100644
--- a/libstdc++-v3/testsuite/26_numerics/headers/complex/synopsis.cc
+++ b/libstdc++-v3/testsuite/26_numerics/headers/complex/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/26_numerics/headers/valarray/synopsis.cc b/libstdc++-v3/testsuite/26_numerics/headers/valarray/synopsis.cc
index 2983aa73355..886d817b2d1 100644
--- a/libstdc++-v3/testsuite/26_numerics/headers/valarray/synopsis.cc
+++ b/libstdc++-v3/testsuite/26_numerics/headers/valarray/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/27_io/basic_filebuf/open/char/path.cc b/libstdc++-v3/testsuite/27_io/basic_filebuf/open/char/path.cc
new file mode 100644
index 00000000000..56fffde5f9b
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/basic_filebuf/open/char/path.cc
@@ -0,0 +1,41 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-fileio "" }
+// { dg-require-filesystem-ts "" }
+
+#include <fstream>
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+const std::filesystem::path filename = "filebuf_members-1.tst";
+
+void
+test01()
+{
+  std::filebuf fb;
+  fb.open(filename, std::ios::in);
+  VERIFY( fb.is_open() );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/basic_fstream/cons/char/path.cc b/libstdc++-v3/testsuite/27_io/basic_fstream/cons/char/path.cc
new file mode 100644
index 00000000000..4442c28c56b
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/basic_fstream/cons/char/path.cc
@@ -0,0 +1,48 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-fileio "" }
+// { dg-require-filesystem-ts "" }
+
+#include <fstream>
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+const std::filesystem::path filename = "ofstream_members-1.tst";
+
+void
+test01()
+{
+  std::fstream f(filename);
+  VERIFY( f.is_open() );
+}
+
+void
+test02()
+{
+  std::fstream f(filename, std::ios::out);
+  VERIFY( f.is_open() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/basic_fstream/open/char/path.cc b/libstdc++-v3/testsuite/27_io/basic_fstream/open/char/path.cc
new file mode 100644
index 00000000000..8d0127be2b8
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/basic_fstream/open/char/path.cc
@@ -0,0 +1,50 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-fileio "" }
+// { dg-require-filesystem-ts "" }
+
+#include <fstream>
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+const std::filesystem::path filename = "ofstream_members-1.tst";
+
+void
+test01()
+{
+  std::fstream f;
+  f.open(filename);
+  VERIFY( f.is_open() );
+}
+
+void
+test02()
+{
+  std::fstream f;
+  f.open(filename, std::ios::in|std::ios::out);
+  VERIFY( f.is_open() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/basic_ifstream/cons/char/path.cc b/libstdc++-v3/testsuite/27_io/basic_ifstream/cons/char/path.cc
new file mode 100644
index 00000000000..24286f5eeaf
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/basic_ifstream/cons/char/path.cc
@@ -0,0 +1,48 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-fileio "" }
+// { dg-require-filesystem-ts "" }
+
+#include <fstream>
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+const std::filesystem::path filename = "ifstream_members-1.tst";
+
+void
+test01()
+{
+  std::ifstream f(filename);
+  VERIFY( f.is_open() );
+}
+
+void
+test02()
+{
+  std::ifstream f(filename, std::ios::in);
+  VERIFY( f.is_open() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/basic_ifstream/open/char/path.cc b/libstdc++-v3/testsuite/27_io/basic_ifstream/open/char/path.cc
new file mode 100644
index 00000000000..192e0fe9e85
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/basic_ifstream/open/char/path.cc
@@ -0,0 +1,50 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-fileio "" }
+// { dg-require-filesystem-ts "" }
+
+#include <fstream>
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+const std::filesystem::path filename = "ifstream_members-1.tst";
+
+void
+test01()
+{
+  std::ifstream f;
+  f.open(filename);
+  VERIFY( f.is_open() );
+}
+
+void
+test02()
+{
+  std::ifstream f;
+  f.open(filename, std::ios::in);
+  VERIFY( f.is_open() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/basic_ofstream/cons/char/path.cc b/libstdc++-v3/testsuite/27_io/basic_ofstream/cons/char/path.cc
new file mode 100644
index 00000000000..c6b6b237dfd
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/basic_ofstream/cons/char/path.cc
@@ -0,0 +1,48 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-fileio "" }
+// { dg-require-filesystem-ts "" }
+
+#include <fstream>
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+const std::filesystem::path filename = "ofstream_members-1.tst";
+
+void
+test01()
+{
+  std::ofstream f(filename);
+  VERIFY( f.is_open() );
+}
+
+void
+test02()
+{
+  std::ofstream f(filename, std::ios::out);
+  VERIFY( f.is_open() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/basic_ofstream/open/char/path.cc b/libstdc++-v3/testsuite/27_io/basic_ofstream/open/char/path.cc
new file mode 100644
index 00000000000..a3fc0c7ff68
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/basic_ofstream/open/char/path.cc
@@ -0,0 +1,50 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-fileio "" }
+// { dg-require-filesystem-ts "" }
+
+#include <fstream>
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+const std::filesystem::path filename = "ofstream_members-1.tst";
+
+void
+test01()
+{
+  std::ofstream f;
+  f.open(filename);
+  VERIFY( f.is_open() );
+}
+
+void
+test02()
+{
+  std::ofstream f;
+  f.open(filename, std::ios::out);
+  VERIFY( f.is_open() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/file_status/1.cc b/libstdc++-v3/testsuite/27_io/filesystem/file_status/1.cc
new file mode 100644
index 00000000000..21613020f88
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/file_status/1.cc
@@ -0,0 +1,84 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+namespace fs = std::filesystem;
+
+template<typename... Args>
+constexpr bool nothrow_constructible() {
+  return std::is_nothrow_constructible<fs::file_status, Args...>::value;
+}
+
+void
+test01()
+{
+  fs::file_status st0;
+  VERIFY( st0.type() == fs::file_type::none );
+  VERIFY( st0.permissions() == fs::perms::unknown );
+  static_assert( nothrow_constructible<>(), "" );
+
+  fs::file_status st1(fs::file_type::regular);
+  VERIFY( st1.type() == fs::file_type::regular );
+  VERIFY( st1.permissions() == fs::perms::unknown );
+  static_assert( nothrow_constructible<fs::file_type>(), "" );
+
+  fs::file_status st2(fs::file_type::directory, fs::perms::owner_all);
+  VERIFY( st2.type() == fs::file_type::directory );
+  VERIFY( st2.permissions() == fs::perms::owner_all );
+  static_assert( nothrow_constructible<fs::file_type, fs::perms>(), "" );
+
+  static_assert( nothrow_constructible<const fs::file_status&>(), "" );
+  static_assert( nothrow_constructible<fs::file_status>(), "" );
+}
+
+void
+test02()
+{
+  fs::file_status st;
+  VERIFY( st.type() == fs::file_type::none );
+  VERIFY( st.permissions() == fs::perms::unknown );
+
+  st.type(fs::file_type::symlink);
+  VERIFY( st.type() == fs::file_type::symlink );
+  VERIFY( st.permissions() == fs::perms::unknown );
+
+  st.permissions(fs::perms::owner_all);
+  VERIFY( st.type() == fs::file_type::symlink );
+  VERIFY( st.permissions() == fs::perms::owner_all );
+}
+
+void check_non_explicit_constructor(fs::file_status) { }
+
+void
+test03()
+{
+  check_non_explicit_constructor( {} ); // LWG 2787
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/iterators/directory_iterator.cc b/libstdc++-v3/testsuite/27_io/filesystem/iterators/directory_iterator.cc
new file mode 100644
index 00000000000..c3e6f01670a
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/iterators/directory_iterator.cc
@@ -0,0 +1,150 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+
+  // Test non-existent path.
+  const auto p = __gnu_test::nonexistent_path();
+  fs::directory_iterator iter(p, ec);
+  VERIFY( ec );
+  VERIFY( iter == end(iter) );
+
+  // Test empty directory.
+  create_directory(p, fs::current_path(), ec);
+  VERIFY( !ec );
+  ec = bad_ec;
+  iter = fs::directory_iterator(p, ec);
+  VERIFY( !ec );
+  VERIFY( iter == end(iter) );
+
+  // Test non-empty directory.
+  ec = bad_ec;
+  create_directory_symlink(p, p / "l", ec);
+  VERIFY( !ec );
+  ec = bad_ec;
+  iter = fs::directory_iterator(p, ec);
+  VERIFY( !ec );
+  VERIFY( iter != fs::directory_iterator() );
+  VERIFY( iter->path() == p/"l" );
+  ++iter;
+  VERIFY( iter == end(iter) );
+
+  // Test inaccessible directory.
+  ec = bad_ec;
+  permissions(p, fs::perms::none, ec);
+  VERIFY( !ec );
+  ec = bad_ec;
+  iter = fs::directory_iterator(p, ec);
+  VERIFY( ec );
+  VERIFY( iter == end(iter) );
+
+  // Test inaccessible directory, skipping permission denied.
+  const auto opts = fs::directory_options::skip_permission_denied;
+  ec = bad_ec;
+  iter = fs::directory_iterator(p, opts, ec);
+  VERIFY( !ec );
+  VERIFY( iter == end(iter) );
+
+  permissions(p, fs::perms::owner_all, ec);
+  remove_all(p, ec);
+}
+
+void
+test02()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+  const auto p = __gnu_test::nonexistent_path();
+  ec = bad_ec;
+  create_directory(p, fs::current_path(), ec);
+  create_directory_symlink(p, p / "l", ec);
+  VERIFY( !ec );
+
+  // Test post-increment (libstdc++/71005)
+  ec = bad_ec;
+  auto iter = fs::directory_iterator(p, ec);
+  VERIFY( !ec );
+  VERIFY( iter != end(iter) );
+  const auto entry1 = *iter;
+  const auto entry2 = *iter++;
+  VERIFY( entry1 == entry2 );
+  VERIFY( entry1.path() == p/"l" );
+  VERIFY( iter == end(iter) );
+
+  remove_all(p, ec);
+}
+
+void
+test03()
+{
+  std::error_code ec = make_error_code(std::errc::invalid_argument);
+  const auto p = __gnu_test::nonexistent_path();
+  create_directories(p / "longer_than_small_string_buffer", ec);
+  VERIFY( !ec );
+
+  // Test for no reallocation on each dereference (this is a GNU extension)
+  auto iter = fs::directory_iterator(p, ec);
+  const auto* s1 = iter->path().c_str();
+  const auto* s2 = iter->path().c_str();
+  VERIFY( s1 == s2 );
+
+  remove_all(p, ec);
+}
+
+void
+test04()
+{
+  const fs::directory_iterator it;
+  VERIFY( it == fs::directory_iterator() );
+}
+
+void
+test05()
+{
+  auto p = __gnu_test::nonexistent_path();
+  create_directory(p);
+  create_directory_symlink(p, p / "l");
+  fs::directory_iterator it(p), endit;
+  VERIFY( begin(it) == it );
+  static_assert( noexcept(begin(it)), "begin is noexcept" );
+  VERIFY( end(it) == endit );
+  static_assert( noexcept(end(it)), "end is noexcept" );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+  test05();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/iterators/pop.cc b/libstdc++-v3/testsuite/27_io/filesystem/iterators/pop.cc
new file mode 100644
index 00000000000..02dc04f2ba9
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/iterators/pop.cc
@@ -0,0 +1,117 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  std::error_code ec;
+  fs::recursive_directory_iterator dir;
+  dir.pop(ec);  // This is undefined, but our implementation
+  VERIFY( ec ); // checks and returns an error.
+  VERIFY( dir == end(dir) );
+
+  std::error_code ec2;
+  try
+  {
+    dir.pop();
+  }
+  catch (const fs::filesystem_error& ex)
+  {
+    ec2 = ex.code();
+  }
+  VERIFY( ec2 == ec );
+}
+
+void
+test02()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+  const auto p = __gnu_test::nonexistent_path();
+  create_directories(p / "d1/d2/d3");
+  for (int i = 0; i < 3; ++i)
+  {
+    fs::recursive_directory_iterator dir(p);
+    VERIFY( dir != end(dir) );
+    std::advance(dir, i);
+    VERIFY( dir != end(dir) );
+    VERIFY( dir.depth() == i );
+    ec = bad_ec;
+    dir.pop(ec);
+    VERIFY( !ec );
+    VERIFY( dir == end(dir) );
+
+    dir = fs::recursive_directory_iterator(p);
+    std::advance(dir, i);
+    VERIFY( dir != end(dir) );
+    VERIFY( dir.depth() == i );
+    dir.pop();
+    VERIFY( dir == end(dir) );
+  }
+  remove_all(p, ec);
+}
+
+void
+test03()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+  const auto p = __gnu_test::nonexistent_path();
+  create_directories(p / "d1/d2/d3");
+  create_directories(p / "d1/d2/e3");
+  create_directories(p / "d1/e2/d3");
+  for (int i = 0; i < 3; ++i)
+  {
+    fs::recursive_directory_iterator dir(p);
+    std::advance(dir, i);
+    VERIFY( dir != end(dir) );
+    int expected_depth = i;
+    VERIFY( dir.depth() == expected_depth );
+    ec = bad_ec;
+    dir.pop(ec);
+    VERIFY( !ec );
+    if (dir != end(dir))
+      VERIFY( dir.depth() == (expected_depth - 1) );
+
+    dir = fs::recursive_directory_iterator(p);
+    std::advance(dir, i);
+    VERIFY( dir != end(dir) );
+    VERIFY( dir.depth() == i );
+    dir.pop();
+    if (dir != end(dir))
+      VERIFY( dir.depth() == (i -1) );
+  }
+  remove_all(p, ec);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/iterators/recursive_directory_iterator.cc b/libstdc++-v3/testsuite/27_io/filesystem/iterators/recursive_directory_iterator.cc
new file mode 100644
index 00000000000..1ef450fc907
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/iterators/recursive_directory_iterator.cc
@@ -0,0 +1,188 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+
+  // Test non-existent path.
+  const auto p = __gnu_test::nonexistent_path();
+  fs::recursive_directory_iterator iter(p, ec);
+  VERIFY( ec );
+  VERIFY( iter == end(iter) );
+
+  // Test empty directory.
+  ec = bad_ec;
+  create_directory(p, fs::current_path(), ec);
+  VERIFY( !ec );
+  ec = bad_ec;
+  iter = fs::recursive_directory_iterator(p, ec);
+  VERIFY( !ec );
+  VERIFY( iter == end(iter) );
+
+  // Test non-empty directory.
+  ec = bad_ec;
+  create_directories(p / "d1/d2", ec);
+  VERIFY( !ec );
+  ec = bad_ec;
+  iter = fs::recursive_directory_iterator(p, ec);
+  VERIFY( !ec );
+  VERIFY( iter != end(iter) );
+  VERIFY( iter->path() == p/"d1" );
+  ++iter;
+  VERIFY( iter->path() == p/"d1/d2" );
+  ++iter;
+  VERIFY( iter == end(iter) );
+
+  // Test inaccessible directory.
+  ec = bad_ec;
+  permissions(p, fs::perms::none, ec);
+  VERIFY( !ec );
+  iter = fs::recursive_directory_iterator(p, ec);
+  VERIFY( ec );
+  VERIFY( iter == end(iter) );
+
+  // Test inaccessible directory, skipping permission denied.
+  const auto opts = fs::directory_options::skip_permission_denied;
+  iter = fs::recursive_directory_iterator(p, opts, ec);
+  VERIFY( !ec );
+  VERIFY( iter == end(iter) );
+
+  // Test inaccessible sub-directory.
+  ec = bad_ec;
+  permissions(p, fs::perms::owner_all, ec);
+  VERIFY( !ec );
+  ec = bad_ec;
+  permissions(p/"d1/d2", fs::perms::none, ec);
+  VERIFY( !ec );
+  ec = bad_ec;
+  iter = fs::recursive_directory_iterator(p, ec);
+  VERIFY( !ec );
+  VERIFY( iter != end(iter) );
+  VERIFY( iter->path() == p/"d1" );
+  ++iter;              // should recurse into d1
+  VERIFY( iter->path() == p/"d1/d2" );
+  iter.increment(ec);  // should fail to recurse into p/d1/d2
+  VERIFY( ec );
+  VERIFY( iter == end(iter) );
+
+  // Test inaccessible sub-directory, skipping permission denied.
+  ec = bad_ec;
+  iter = fs::recursive_directory_iterator(p, opts, ec);
+  VERIFY( !ec );
+  VERIFY( iter != end(iter) );
+  VERIFY( iter->path() == p/"d1" );
+  ++iter;              // should recurse into d1
+  VERIFY( iter->path() == p/"d1/d2" );
+  ec = bad_ec;
+  iter.increment(ec);  // should fail to recurse into p/d1/d2, so skip it
+  VERIFY( !ec );
+  VERIFY( iter == end(iter) );
+
+  permissions(p/"d1/d2", fs::perms::owner_all, ec);
+  remove_all(p, ec);
+}
+
+void
+test02()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+  const auto p = __gnu_test::nonexistent_path();
+  ec = bad_ec;
+  create_directories(p / "d1/d2", ec);
+  VERIFY( !ec );
+
+  // Test post-increment (libstdc++/71005)
+  ec = bad_ec;
+  auto iter = fs::recursive_directory_iterator(p, ec);
+  VERIFY( !ec );
+  VERIFY( iter != end(iter) );
+  const auto entry1 = *iter;
+  const auto entry2 = *iter++;
+  VERIFY( entry1 == entry2 );
+  VERIFY( entry1.path() == p/"d1" );
+  const auto entry3 = *iter;
+  const auto entry4 = *iter++;
+  VERIFY( entry3 == entry4 );
+  VERIFY( entry3.path() == p/"d1/d2" );
+  VERIFY( iter == end(iter) );
+
+  remove_all(p, ec);
+}
+
+void
+test03()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+  const auto p = __gnu_test::nonexistent_path();
+  ec = bad_ec;
+  create_directories(p / "longer_than_small_string_buffer", ec);
+  VERIFY( !ec );
+
+  // Test for no reallocation on each dereference (this is a GNU extension)
+  auto iter = fs::recursive_directory_iterator(p, ec);
+  const auto* s1 = iter->path().c_str();
+  const auto* s2 = iter->path().c_str();
+  VERIFY( s1 == s2 );
+
+  remove_all(p, ec);
+}
+
+void
+test04()
+{
+  // libstdc++/71004
+  const fs::recursive_directory_iterator it;
+  VERIFY( it == end(it) );
+}
+
+void
+test05()
+{
+  auto p = __gnu_test::nonexistent_path();
+  create_directory(p);
+  create_directory_symlink(p, p / "l");
+  fs::recursive_directory_iterator it(p), endit;
+  VERIFY( begin(it) == it );
+  static_assert( noexcept(begin(it)), "begin is noexcept" );
+  VERIFY( end(it) == endit );
+  static_assert( noexcept(end(it)), "end is noexcept" );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+  test05();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/absolute.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/absolute.cc
new file mode 100644
index 00000000000..de1cd318f96
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/absolute.cc
@@ -0,0 +1,54 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// C++17 30.10.14.1 Absolute [fs.op.absolute]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+    VERIFY( absolute(p).is_absolute() );
+}
+
+void
+test02()
+{
+  path p1("/");
+  VERIFY( absolute(p1) == p1 );
+  path p2("/foo");
+  VERIFY( absolute(p2) == p2 );
+  path p3("foo");
+  VERIFY( absolute(p3) != p3 );
+  VERIFY( absolute(p3) == (std::filesystem::current_path()/p3) );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/canonical.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/canonical.cc
new file mode 100644
index 00000000000..47305a8f527
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/canonical.cc
@@ -0,0 +1,140 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+  auto p = __gnu_test::nonexistent_path();
+  canonical( p, ec );
+  VERIFY( ec );
+
+  create_directory(p);
+  auto p2 = canonical( p, ec );
+  compare_paths( p2, fs::current_path()/p );
+  VERIFY( !ec );
+
+  ec = bad_ec;
+  p2 = canonical( fs::current_path() / "." / (p.native() + "////././."), ec );
+  compare_paths( p2, fs::current_path()/p );
+  VERIFY( !ec );
+
+  ec = bad_ec;
+  p = fs::current_path();
+  p2 = canonical( p, ec );
+  compare_paths( p2, p );
+  VERIFY( !ec );
+
+  ec = bad_ec;
+  p = "/";
+  p = canonical( p, ec );
+  compare_paths( p, "/" );
+  VERIFY( !ec );
+
+  ec = bad_ec;
+  p = "/.";
+  p = canonical( p, ec );
+  compare_paths( p, "/" );
+  VERIFY( !ec );
+
+  ec = bad_ec;
+  p = "/..";
+  p = canonical( p, ec );
+  compare_paths( p, "/" );
+  VERIFY( !ec );
+
+  ec = bad_ec;
+  p = "/../.././.";
+  p = canonical( p, ec );
+  compare_paths( p, "/" );
+  VERIFY( !ec );
+}
+
+void
+test02()
+{
+  const fs::path p = __gnu_test::nonexistent_path();
+  std::error_code ec, ec2;
+  const fs::path res = canonical(p, ec);
+  VERIFY( ec );
+  VERIFY( res.empty() );
+
+#if __cpp_exceptions
+  fs::path e1, e2;
+  try {
+    canonical(p);
+  } catch (const fs::filesystem_error& e) {
+    e1 = e.path1();
+    e2 = e.path2();
+    ec2 = e.code();
+  }
+  VERIFY( e1 == p );
+  VERIFY( e2.empty() );
+  VERIFY( ec == ec2 );
+#endif
+}
+
+
+void
+test03()
+{
+  std::error_code ec;
+  auto dir = __gnu_test::nonexistent_path();
+  fs::create_directory(dir);
+  fs::path foo = dir/"foo", bar = dir/"bar";
+  fs::create_directory(foo);
+  fs::create_directory(bar);
+  fs::create_symlink("../bar", foo/"baz");
+
+  auto dirc = canonical(dir);
+  auto barc = canonical(bar);
+
+  auto p1 = fs::canonical(dir/"foo//.///..//./");
+  compare_paths( p1, dirc );
+  auto p2 = fs::canonical(dir/"foo//./baz///..//./");
+  compare_paths( p2, dirc );
+  auto p3 = fs::canonical(dir/"foo//./baz////./");
+  compare_paths( p3, barc );
+  auto p4 = fs::canonical(dir/"foo//./baz///..//./bar");
+  compare_paths( p4, barc );
+  auto p5 = fs::canonical(dir/"foo//./baz///..//./bar/");
+  compare_paths( p5, p4 );
+  auto p6 = fs::canonical(dir/"foo//./baz///..//./bar/.");
+  compare_paths( p6, p4 );
+
+  remove_all(dir);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/copy.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/copy.cc
new file mode 100644
index 00000000000..7825a4ef5dd
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/copy.cc
@@ -0,0 +1,200 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 15.3 Copy [fs.op.copy]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+namespace fs = std::filesystem;
+
+// Test error conditions.
+void
+test01()
+{
+  auto p = __gnu_test::nonexistent_path();
+  std::error_code ec;
+
+  VERIFY( !fs::exists(p) );
+  fs::copy(p, ".", fs::copy_options::none, ec);
+  VERIFY( ec );
+
+  ec.clear();
+  fs::copy(".", ".", fs::copy_options::none, ec);
+  VERIFY( ec );
+
+  __gnu_test::scoped_file f(p);
+  VERIFY( fs::is_directory(".") );
+  VERIFY( fs::is_regular_file(p) );
+  ec.clear();
+  fs::copy(".", p, fs::copy_options::none, ec);
+  VERIFY( ec );
+
+  auto to = __gnu_test::nonexistent_path();
+  ec.clear();
+  auto opts = fs::copy_options::create_symlinks;
+  fs::copy("/", to, opts, ec);
+  VERIFY( ec == std::make_error_code(std::errc::is_a_directory) );
+  VERIFY( !exists(to) );
+
+  ec.clear();
+  opts != fs::copy_options::recursive;
+  fs::copy("/", to, opts, ec);
+  VERIFY( ec == std::make_error_code(std::errc::is_a_directory) );
+  VERIFY( !exists(to) );
+}
+
+// Test is_symlink(f) case.
+void
+test02()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  auto from = __gnu_test::nonexistent_path();
+  auto to = __gnu_test::nonexistent_path();
+  std::error_code ec;
+
+  ec = bad_ec;
+  fs::create_symlink(".", from, ec);
+  VERIFY( !ec );
+  VERIFY( fs::exists(from) );
+
+  ec = bad_ec;
+  fs::copy(from, to, fs::copy_options::skip_symlinks, ec);
+  VERIFY( !ec );
+  VERIFY( !fs::exists(to) );
+
+  ec = bad_ec;
+  fs::copy(from, to, fs::copy_options::skip_symlinks, ec);
+  VERIFY( !ec );
+  VERIFY( !fs::exists(to) );
+
+  ec = bad_ec;
+  fs::copy(from, to,
+           fs::copy_options::skip_symlinks|fs::copy_options::copy_symlinks,
+           ec);
+  VERIFY( !ec );
+  VERIFY( !fs::exists(to) );
+
+  ec = bad_ec;
+  fs::copy(from, to, fs::copy_options::copy_symlinks, ec);
+  VERIFY( !ec );
+  VERIFY( fs::exists(to) );
+  VERIFY( is_symlink(to) );
+
+  ec.clear();
+  fs::copy(from, to, fs::copy_options::copy_symlinks, ec);
+  VERIFY( ec );
+
+  remove(from, ec);
+  remove(to, ec);
+}
+
+// Test is_regular_file(f) case.
+void
+test03()
+{
+  auto from = __gnu_test::nonexistent_path();
+  auto to = __gnu_test::nonexistent_path();
+
+  // test empty file
+  std::ofstream{from.native()};
+  VERIFY( fs::exists(from) );
+  VERIFY( fs::file_size(from) == 0 );
+  fs::copy(from, to);
+  VERIFY( fs::exists(to) );
+  VERIFY( fs::file_size(to) == 0 );
+
+  remove(to);
+  VERIFY( !fs::exists(to) );
+  std::ofstream{from.native()} << "Hello, filesystem!";
+  VERIFY( fs::file_size(from) != 0 );
+  fs::copy(from, to);
+  VERIFY( fs::exists(to) );
+  VERIFY( fs::file_size(to) == fs::file_size(from) );
+
+  remove(from);
+  remove(to);
+}
+
+// Test is_directory(f) case.
+void
+test04()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  auto from = __gnu_test::nonexistent_path();
+  auto to = __gnu_test::nonexistent_path();
+  std::error_code ec;
+
+  create_directories(from/"a/b/c");
+
+  {
+    __gnu_test::scoped_file f(to);
+    copy(from, to, ec);
+    VERIFY( ec );
+  }
+
+  __gnu_test::scoped_file f1(from/"a/f1");
+  std::ofstream{f1.path} << "file one";
+  __gnu_test::scoped_file f2(from/"a/b/f2");
+  std::ofstream{f2.path} << "file two";
+
+  copy(from, to, ec);
+  VERIFY( !ec );
+  VERIFY( exists(to) && is_empty(to) );
+  remove(to);
+
+  ec = bad_ec;
+  copy(from, to, fs::copy_options::recursive, ec);
+  VERIFY( !ec );
+  VERIFY( exists(to) && !is_empty(to) );
+  VERIFY( is_regular_file(to/"a/f1") && !is_empty(to/"a/f1") );
+  VERIFY( file_size(from/"a/f1") == file_size(to/"a/f1") );
+  VERIFY( is_regular_file(to/"a/b/f2") && !is_empty(to/"a/b/f2") );
+  VERIFY( file_size(from/"a/b/f2") == file_size(to/"a/b/f2") );
+  VERIFY( is_directory(to/"a/b/c") && is_empty(to/"a/b/c") );
+
+  f1.path.clear();
+  f2.path.clear();
+  remove_all(from, ec);
+  remove_all(to, ec);
+}
+
+// Test no-op cases.
+void
+test05()
+{
+  auto to = __gnu_test::nonexistent_path();
+  std::error_code ec = std::make_error_code(std::errc::invalid_argument);
+
+  fs::copy("/", to, fs::copy_options::copy_symlinks, ec);
+  VERIFY( !ec );  // Previous value should be cleared (LWG 2683)
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+  test05();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/copy_file.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/copy_file.cc
new file mode 100644
index 00000000000..69ab7fbab8d
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/copy_file.cc
@@ -0,0 +1,84 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// 15.4 Copy [fs.op.copy_file]
+
+#include <filesystem>
+#include <fstream>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  using std::filesystem::copy_options;
+  std::error_code ec;
+
+  auto from = __gnu_test::nonexistent_path();
+  auto to = __gnu_test::nonexistent_path();
+
+  // test non-existent file
+  bool b = copy_file(from, to, ec);
+  VERIFY( !b );
+  VERIFY( ec );
+  VERIFY( !exists(to) );
+
+  // test empty file
+  std::ofstream{from.native()};
+  VERIFY( exists(from) );
+  VERIFY( file_size(from) == 0 );
+
+  b = copy_file(from, to);
+  VERIFY( b );
+  VERIFY( exists(to) );
+  VERIFY( file_size(to) == 0 );
+  remove(to);
+  VERIFY( !exists(to) );
+  b = copy_file(from, to, copy_options::none, ec);
+  VERIFY( b );
+  VERIFY( !ec );
+  VERIFY( exists(to) );
+  VERIFY( file_size(to) == 0 );
+
+  std::ofstream{from.native()} << "Hello, filesystem!";
+  VERIFY( file_size(from) != 0 );
+  remove(to);
+  VERIFY( !exists(to) );
+  b = copy_file(from, to);
+  VERIFY( b );
+  VERIFY( exists(to) );
+  VERIFY( file_size(to) == file_size(from) );
+  remove(to);
+  VERIFY( !exists(to) );
+  b = copy_file(from, to);
+  VERIFY( b );
+  VERIFY( exists(to) );
+  VERIFY( file_size(to) == file_size(from) );
+
+  remove(from);
+  remove(to);
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/create_directories.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/create_directories.cc
new file mode 100644
index 00000000000..94596787196
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/create_directories.cc
@@ -0,0 +1,83 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+
+  // Test empty path.
+  bool b = fs::create_directories( "", ec );
+  VERIFY( ec );
+  VERIFY( !b );
+
+  // Test existing path.
+  ec = bad_ec;
+  b = fs::create_directories( fs::current_path(), ec );
+  VERIFY( !ec );
+  VERIFY( !b );
+
+  // Test non-existent path.
+  const auto p = __gnu_test::nonexistent_path();
+  ec = bad_ec;
+  b = fs::create_directories( p, ec );
+  VERIFY( !ec );
+  VERIFY( b );
+  VERIFY( is_directory(p) );
+
+  ec = bad_ec;
+  b = fs::create_directories( p/".", ec );
+  VERIFY( !ec );
+  VERIFY( !b );
+
+  ec = bad_ec;
+  b = fs::create_directories( p/"..", ec );
+  VERIFY( !ec );
+  VERIFY( !b );
+
+  ec = bad_ec;
+  b = fs::create_directories( p/"d1/d2/d3", ec );
+  VERIFY( !ec );
+  VERIFY( b );
+  VERIFY( is_directory(p/"d1/d2/d3") );
+
+  ec = bad_ec;
+  b = fs::create_directories( p/"./d4/../d5", ec );
+  VERIFY( !ec );
+  VERIFY( b );
+  VERIFY( is_directory(p/"./d4/../d5") );
+
+  std::uintmax_t count = remove_all(p, ec);
+  VERIFY( count == 6 );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/create_directory.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/create_directory.cc
new file mode 100644
index 00000000000..927a97c7a84
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/create_directory.cc
@@ -0,0 +1,65 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+
+  // Test empty path.
+  fs::path p;
+  bool b = create_directory( p, ec );
+  VERIFY( ec );
+  VERIFY( !b );
+
+  // Test non-existent path
+  p = __gnu_test::nonexistent_path();
+  VERIFY( !exists(p) );
+
+  ec = bad_ec;
+  b = create_directory(p, ec); // create the directory once
+  VERIFY( !ec );
+  VERIFY( b );
+  VERIFY( exists(p) );
+
+  // Test existing path (libstdc++/71036).
+  ec = bad_ec;
+  b = create_directory(p, ec);
+  VERIFY( !ec );
+  VERIFY( !b );
+  b = create_directory(p);
+  VERIFY( !b );
+
+  remove_all(p, ec);
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/create_symlink.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/create_symlink.cc
new file mode 100644
index 00000000000..7eff356b883
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/create_symlink.cc
@@ -0,0 +1,96 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  std::error_code ec, ec2;
+  __gnu_test::scoped_file f;
+  auto tgt = f.path;
+
+  // Test empty path.
+  fs::path p;
+  create_symlink(tgt, p, ec );
+  VERIFY( ec );
+  try
+  {
+    create_symlink(tgt, p);
+  }
+  catch (const std::filesystem::filesystem_error& ex)
+  {
+    ec2 = ex.code();
+    VERIFY( ex.path1() == tgt );
+    VERIFY( ex.path2() == p );
+  }
+  VERIFY( ec2 == ec );
+}
+
+void
+test02()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec, ec2;
+  __gnu_test::scoped_file f;
+  auto tgt = f.path;
+
+  // Test non-existent path
+  auto p = __gnu_test::nonexistent_path();
+  VERIFY( !exists(p) );
+
+  ec = bad_ec;
+  create_symlink(tgt, p, ec); // create the symlink once
+  VERIFY( !ec );
+  VERIFY( exists(p) );
+  VERIFY( is_symlink(p) );
+  remove(p);
+  create_symlink(tgt, p); // create the symlink again
+  VERIFY( exists(p) );
+  VERIFY( is_symlink(p) );
+
+  ec.clear();
+  create_symlink(tgt, p, ec); // Try to create existing symlink
+  VERIFY( ec );
+  try
+  {
+    create_symlink(tgt, p);
+  }
+  catch (const std::filesystem::filesystem_error& ex)
+  {
+    ec2 = ex.code();
+    VERIFY( ex.path1() == tgt );
+    VERIFY( ex.path2() == p );
+  }
+  VERIFY( ec2 == ec );
+
+  remove(p);
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/current_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/current_path.cc
new file mode 100644
index 00000000000..2ba3ff9fc53
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/current_path.cc
@@ -0,0 +1,58 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 15.11 Current path [fs.op.current_path]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  fs::path dot(".");
+  fs::path cwd = fs::current_path();
+  std::error_code ec;
+  fs::path cwd2 = fs::current_path(ec);
+  VERIFY( cwd == cwd2 );
+}
+
+void
+test02()
+{
+  auto oldwd = fs::current_path();
+  auto tmpdir = fs::temp_directory_path();
+  current_path(tmpdir);
+  VERIFY( canonical(fs::current_path()) == canonical(tmpdir) );
+  std::error_code ec;
+  current_path(oldwd, ec);
+  VERIFY( canonical(fs::current_path()) == canonical(oldwd) );
+  VERIFY( canonical(fs::current_path(ec)) == canonical(oldwd) );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/equivalent.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/equivalent.cc
new file mode 100644
index 00000000000..92d69c604ac
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/equivalent.cc
@@ -0,0 +1,74 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  auto p1 = __gnu_test::nonexistent_path();
+  auto p2 = __gnu_test::nonexistent_path();
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+  bool result;
+
+  result = equivalent(p1, p2, ec);
+  VERIFY( ec );
+  VERIFY( !result );
+
+  __gnu_test::scoped_file f1(p1);
+  ec = bad_ec;
+  result = equivalent(p1, p2, ec);
+  VERIFY( !ec );
+  VERIFY( !result );
+
+  __gnu_test::scoped_file f2(p2);
+  ec = bad_ec;
+  result = equivalent(p1, p2, ec);
+  VERIFY( !ec );
+  VERIFY( !result );
+
+  auto p3 = __gnu_test::nonexistent_path();
+  create_hard_link(p1, p3, ec);
+  if (ec)
+    return;  // hard links not supported
+  __gnu_test::scoped_file f3(p3, __gnu_test::scoped_file::adopt_file);
+
+  ec = bad_ec;
+  result = equivalent(p1, p3, ec);
+  VERIFY( !ec );
+  VERIFY( result );
+
+  ec = bad_ec;
+  result = equivalent(p2, p3, ec);
+  VERIFY( !ec );
+  VERIFY( !result );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/exists.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/exists.cc
new file mode 100644
index 00000000000..0775eff4d9b
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/exists.cc
@@ -0,0 +1,115 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+
+  VERIFY( exists(path{"/"}) );
+  VERIFY( exists(path{"/."}) );
+  VERIFY( exists(path{"."}) );
+  VERIFY( exists(path{".."}) );
+  VERIFY( exists(std::filesystem::current_path()) );
+
+  std::error_code ec;
+  ec = bad_ec;
+  VERIFY( exists(path{"/"}, ec) );
+  VERIFY( !ec );
+  ec = bad_ec;
+  VERIFY( exists(path{"/."}, ec) );
+  VERIFY( !ec );
+  ec = bad_ec;
+  VERIFY( exists(path{"."}, ec) );
+  VERIFY( !ec );
+  ec = bad_ec;
+  VERIFY( exists(path{".."}, ec) );
+  VERIFY( !ec );
+  ec = bad_ec;
+  VERIFY( exists(std::filesystem::current_path(), ec) );
+  VERIFY( !ec );
+}
+
+void
+test02()
+{
+  path rel = __gnu_test::nonexistent_path();
+  VERIFY( !exists(rel) );
+
+  std::error_code ec = std::make_error_code(std::errc::invalid_argument);
+  VERIFY( !exists(rel, ec) );
+  VERIFY( !ec ); // DR 2725
+}
+
+void
+test03()
+{
+  path abs = absolute(__gnu_test::nonexistent_path());
+  VERIFY( !exists(abs) );
+
+  std::error_code ec = std::make_error_code(std::errc::invalid_argument);
+  VERIFY( !exists(abs, ec) );
+  VERIFY( !ec ); // DR 2725
+}
+
+void
+test04()
+{
+  using std::filesystem::perms;
+  using std::filesystem::perm_options;
+  path p = __gnu_test::nonexistent_path();
+  create_directory(p);
+  permissions(p, perms::all, perm_options::remove);
+
+  auto unr = p / "unreachable";
+  std::error_code ec;
+  VERIFY( !exists(unr, ec) );
+  VERIFY( ec == std::errc::permission_denied );
+  ec.clear();
+  try
+  {
+    exists(unr);
+  }
+  catch(const std::filesystem::filesystem_error& ex)
+  {
+    ec = ex.code();
+    VERIFY( ex.path1() == unr );
+  }
+  VERIFY( ec == std::errc::permission_denied );
+
+  permissions(p, perms::owner_all);
+  remove(p);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/file_size.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/file_size.cc
new file mode 100644
index 00000000000..250d31891a0
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/file_size.cc
@@ -0,0 +1,71 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  std::error_code ec;
+  size_t size = fs::file_size(".", ec);
+  VERIFY( ec == std::errc::is_a_directory );
+  VERIFY( size == -1 );
+
+  try {
+    size = fs::file_size(".");
+    ec.clear();
+  } catch (const fs::filesystem_error& e) {
+    ec = e.code();
+  }
+  VERIFY( ec == std::errc::is_a_directory );
+  VERIFY( size == -1 );
+}
+
+void
+test02()
+{
+  fs::path p = __gnu_test::nonexistent_path();
+
+  std::error_code ec;
+  size_t size = fs::file_size(p, ec);
+  VERIFY( ec );
+  VERIFY( size == -1 );
+
+  try {
+    size = fs::file_size(p);
+    ec.clear();
+  } catch (const fs::filesystem_error& e) {
+    ec = e.code();
+  }
+  VERIFY( ec );
+  VERIFY( size == -1 );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/is_empty.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/is_empty.cc
new file mode 100644
index 00000000000..6d79a9139cf
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/is_empty.cc
@@ -0,0 +1,109 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  auto p = __gnu_test::nonexistent_path();
+  create_directory(p);
+  permissions(p, fs::perms::none);
+  std::error_code ec, ec2;
+
+  bool result = fs::is_empty(p, ec);
+  VERIFY( ec == std::make_error_code(std::errc::permission_denied) );
+  VERIFY( !result );
+
+  try {
+    fs::is_empty(p);
+  } catch (const fs::filesystem_error& e) {
+    ec2 = e.code();
+  }
+  VERIFY( ec2 == ec );
+
+  result = fs::is_empty(p/"f", ec);
+  VERIFY( ec == std::make_error_code(std::errc::permission_denied) );
+  VERIFY( !result );
+
+  try {
+    fs::is_empty(p/"f");
+  } catch (const fs::filesystem_error& e) {
+    ec2 = e.code();
+  }
+  VERIFY( ec2 == ec );
+
+  permissions(p, fs::perms::owner_all, ec);
+  remove_all(p, ec);
+}
+
+void
+test02()
+{
+  auto p = __gnu_test::nonexistent_path();
+  create_directory(p);
+  std::error_code ec, bad_ec = make_error_code(std::errc::invalid_argument);
+  bool empty;
+
+  ec = bad_ec;
+  empty = is_empty(p, ec);
+  VERIFY( !ec );
+  VERIFY( empty );
+  empty = is_empty(p);
+  VERIFY( empty );
+
+  __gnu_test::scoped_file f(p/"f");
+  ec = bad_ec;
+  empty = is_empty(f.path, ec);
+  VERIFY( !ec );
+  VERIFY( empty );
+  empty = is_empty(f.path);
+  VERIFY( empty );
+
+  std::ofstream{f.path.native()} << "data";
+  ec = bad_ec;
+  empty = is_empty(p, ec);
+  VERIFY( !ec );
+  VERIFY( !empty );
+  empty = is_empty(p);
+  VERIFY( !empty );
+
+  ec = bad_ec;
+  empty = is_empty(p, ec);
+  VERIFY( !ec );
+  VERIFY( !empty );
+  empty = is_empty(p);
+  VERIFY( !empty );
+
+  f.path.clear();
+  remove_all(p, ec);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/last_write_time.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/last_write_time.cc
new file mode 100644
index 00000000000..ecef3f85721
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/last_write_time.cc
@@ -0,0 +1,162 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// 15.25 Permissions [fs.op.last_write_time]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+#ifdef _GLIBCXX_HAVE_FCNTL_H
+# include <fcntl.h>
+#endif
+#if _GLIBCXX_HAVE_UTIME_H
+# include <utime.h>
+#endif
+
+using time_type = std::filesystem::file_time_type;
+
+void
+test01()
+{
+  // read times
+
+  auto p = __gnu_test::nonexistent_path();
+  std::error_code ec;
+  time_type mtime = last_write_time(p, ec);
+  VERIFY( ec );
+  VERIFY( ec == std::make_error_code(std::errc::no_such_file_or_directory) );
+#if __cpp_exceptions
+  bool caught = false;
+  try {
+    mtime = last_write_time(p);
+  } catch (std::system_error const& e) {
+    caught = true;
+    ec = e.code();
+  }
+  VERIFY( caught );
+  VERIFY( ec );
+  VERIFY( ec == std::make_error_code(std::errc::no_such_file_or_directory) );
+#endif
+
+  __gnu_test::scoped_file file(p);
+  VERIFY( exists(p) );
+  mtime = last_write_time(p, ec);
+  VERIFY( !ec );
+  VERIFY( mtime <= time_type::clock::now() );
+  VERIFY( mtime == last_write_time(p) );
+
+  auto end_of_time = time_type::duration::max();
+  auto last_second
+    = std::chrono::duration_cast<std::chrono::seconds>(end_of_time).count();
+  if (last_second > std::numeric_limits<std::time_t>::max())
+    return; // can't test overflow
+
+#if _GLIBCXX_USE_UTIMENSAT
+  struct ::timespec ts[2];
+  ts[0].tv_sec = 0;
+  ts[0].tv_nsec = UTIME_NOW;
+  ts[1].tv_sec = std::numeric_limits<std::time_t>::max() - 1;
+  ts[1].tv_nsec = 0;
+  VERIFY( !::utimensat(AT_FDCWD, p.c_str(), ts, 0) );
+#elif _GLIBCXX_HAVE_UTIME_H
+  ::utimbuf times;
+  times.modtime = std::numeric_limits<std::time_t>::max() - 1;
+  times.actime = std::numeric_limits<std::time_t>::max() - 1;
+  VERIFY( !::utime(p.c_str(), &times) );
+#else
+  return;
+#endif
+
+  mtime = last_write_time(p, ec);
+  VERIFY( ec );
+  VERIFY( ec == std::make_error_code(std::errc::value_too_large) );
+  VERIFY( mtime == time_type::min() );
+
+#if __cpp_exceptions
+  caught = false;
+  try {
+    mtime = last_write_time(p);
+  } catch (std::system_error const& e) {
+    caught = true;
+    ec = e.code();
+  }
+  VERIFY( caught );
+  VERIFY( ec );
+  VERIFY( ec == std::make_error_code(std::errc::value_too_large) );
+#endif
+}
+
+bool approx_equal(time_type file_time, time_type expected)
+{
+  auto delta = expected - file_time;
+  if (delta < delta.zero())
+    delta = -delta;
+  return delta < std::chrono::seconds(1);
+}
+
+void
+test02()
+{
+  // write times
+
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  __gnu_test::scoped_file f;
+  std::error_code ec;
+  time_type time;
+
+  time = last_write_time(f.path);
+  ec = bad_ec;
+  last_write_time(f.path, time, ec);
+  VERIFY( !ec );
+  VERIFY( approx_equal(last_write_time(f.path), time) );
+
+  ec = bad_ec;
+  time -= std::chrono::milliseconds(1000 * 60 * 10 + 15);
+  last_write_time(f.path, time, ec);
+  VERIFY( !ec );
+  VERIFY( approx_equal(last_write_time(f.path), time) );
+
+  ec = bad_ec;
+  time += std::chrono::milliseconds(1000 * 60 * 20 + 15);
+  last_write_time(f.path, time, ec);
+  VERIFY( !ec );
+  VERIFY( approx_equal(last_write_time(f.path), time) );
+
+  ec = bad_ec;
+  time = time_type();
+  last_write_time(f.path, time, ec);
+  VERIFY( !ec );
+  VERIFY( approx_equal(last_write_time(f.path), time) );
+
+  ec = bad_ec;
+  time -= std::chrono::milliseconds(1000 * 60 * 10 + 15);
+  last_write_time(f.path, time, ec);
+  VERIFY( !ec );
+  VERIFY( approx_equal(last_write_time(f.path), time) );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/permissions.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/permissions.cc
new file mode 100644
index 00000000000..e190e886230
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/permissions.cc
@@ -0,0 +1,175 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// C++17 30.10.14.26 Permissions [fs.op.permissions]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  using std::filesystem::perms;
+  using std::filesystem::perm_options;
+
+  auto p = __gnu_test::nonexistent_path();
+
+  __gnu_test::scoped_file f(p);
+  VERIFY( exists(p) );
+  permissions(p, perms::owner_all);
+  VERIFY( status(p).permissions() == perms::owner_all );
+  permissions(p, perms::group_read, perm_options::add);
+  VERIFY( status(p).permissions() == (perms::owner_all | perms::group_read) );
+  permissions(p, perms::group_read, perm_options::remove);
+  VERIFY( status(p).permissions() == perms::owner_all );
+}
+
+void
+test02()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  using std::filesystem::perms;
+  using std::filesystem::perm_options;
+
+  auto p = __gnu_test::nonexistent_path();
+
+  std::error_code ec;
+  permissions(p, perms::owner_all, ec);
+  VERIFY( ec );
+
+  __gnu_test::scoped_file f(p);
+  VERIFY( exists(p) );
+
+  ec = bad_ec;
+  permissions(p, perms::owner_all, ec);
+  VERIFY( !ec );
+  VERIFY( status(p).permissions() == perms::owner_all );
+  ec = bad_ec;
+  permissions(p, perms::group_read, perm_options::add, ec);
+  VERIFY( !ec );
+  VERIFY( status(p).permissions() == (perms::owner_all | perms::group_read) );
+  ec = bad_ec;
+  permissions(p, perms::group_read, perm_options::remove, ec);
+  VERIFY( !ec );
+  VERIFY( status(p).permissions() == perms::owner_all );
+}
+
+void
+test03()
+{
+  using std::filesystem::perms;
+  using std::filesystem::perm_options;
+
+  __gnu_test::scoped_file f;
+  VERIFY( exists(f.path) );
+
+  auto p = __gnu_test::nonexistent_path();
+  create_symlink(f.path, p);
+
+  std::error_code ec = make_error_code(std::errc::no_such_file_or_directory);
+  permissions(p, perms::owner_all,
+	      perm_options::replace|perm_options::nofollow, ec);
+  bool caught = false;
+  std::error_code ec2;
+  try
+  {
+    permissions(p, perms::owner_all,
+		perm_options::replace|perm_options::nofollow);
+  }
+  catch (const std::filesystem::filesystem_error& ex)
+  {
+    caught = true;
+    ec2 = ex.code();
+    VERIFY( ex.path1() == p );
+  }
+  // Both calls should succeed, or both should fail with same error:
+  if (ec)
+  {
+    VERIFY( caught );
+    VERIFY( ec == ec2 );
+  }
+  else
+    VERIFY( !caught );
+
+  remove(p);
+}
+
+void
+test04()
+{
+  using perms = std::filesystem::perms;
+
+  auto p = __gnu_test::nonexistent_path();
+  create_symlink(__gnu_test::nonexistent_path(), p);
+
+  std::error_code ec = make_error_code(std::errc::no_such_file_or_directory);
+  permissions(p, perms::owner_all, ec);
+  VERIFY( ec );
+  std::error_code ec2;
+  try
+  {
+    permissions(p, perms::owner_all);
+  }
+  catch (const std::filesystem::filesystem_error& ex)
+  {
+    ec2 = ex.code();
+    VERIFY( ex.path1() == p );
+  }
+  VERIFY( ec == ec2 );
+
+  remove(p);
+}
+
+void
+test05()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  using std::filesystem::perms;
+  using std::filesystem::perm_options;
+  std::error_code ec;
+
+  __gnu_test::scoped_file f;
+  auto p = perms::owner_write;
+
+  // symlink_nofollow should not give an error for non-symlinks
+  ec = bad_ec;
+  permissions(f.path, p, perm_options::replace|perm_options::nofollow, ec);
+  VERIFY( !ec );
+  auto st = status(f.path);
+  VERIFY( st.permissions() == p );
+  p |= perms::owner_read;
+  ec = bad_ec;
+  permissions(f.path, p, perm_options::replace|perm_options::nofollow, ec);
+  VERIFY( !ec );
+  st = status(f.path);
+  VERIFY( st.permissions() == p );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+  test05();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/proximate.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/proximate.cc
new file mode 100644
index 00000000000..99b6568abeb
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/proximate.cc
@@ -0,0 +1,69 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::proximate;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  compare_paths( proximate("/a/d", "/a/b/c"), "../../d" );
+  compare_paths( proximate("/a/b/c", "/a/d"), "../b/c" );
+  compare_paths( proximate("a/b/c", "a"), "b/c" );
+  compare_paths( proximate("a/b/c", "a/b/c/x/y"), "../.." );
+  compare_paths( proximate("a/b/c", "a/b/c"), "." );
+  compare_paths( proximate("a/b", "c/d"), "../../a/b" );
+}
+
+void
+test02()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec = bad_ec;
+  compare_paths( proximate("/a/d", "/a/b/c", ec), "../../d" );
+  VERIFY( !ec );
+  ec = bad_ec;
+  compare_paths( proximate("/a/b/c", "/a/d", ec), "../b/c" );
+  VERIFY( !ec );
+  ec = bad_ec;
+  compare_paths( proximate("a/b/c", "a", ec), "b/c" );
+  VERIFY( !ec );
+  ec = bad_ec;
+  compare_paths( proximate("a/b/c", "a/b/c/x/y", ec), "../.." );
+  VERIFY( !ec );
+  ec = bad_ec;
+  compare_paths( proximate("a/b/c", "a/b/c", ec), "." );
+  VERIFY( !ec );
+  ec = bad_ec;
+  compare_paths( proximate("a/b", "c/d", ec), "../../a/b" );
+  VERIFY( !ec );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/read_symlink.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/read_symlink.cc
new file mode 100644
index 00000000000..067a60950ea
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/read_symlink.cc
@@ -0,0 +1,51 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  auto p = __gnu_test::nonexistent_path();
+  std::error_code ec;
+
+  read_symlink(p, ec);
+  VERIFY( ec );
+
+  fs::path tgt = ".";
+  create_symlink(tgt, p);
+
+  auto result = read_symlink(p, ec);
+  VERIFY( !ec );
+  VERIFY( result == tgt );
+
+  remove(p);
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/relative.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/relative.cc
new file mode 100644
index 00000000000..c9765fa5680
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/relative.cc
@@ -0,0 +1,64 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  auto p = __gnu_test::nonexistent_path();
+  auto q = __gnu_test::nonexistent_path();
+
+  auto r = relative(p, q);
+  VERIFY( r == ".."/p );
+
+  r = relative(p, p/q);
+  VERIFY( r == ".." );
+
+  r = relative(p/q, p);
+  VERIFY( r == q );
+
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+
+  ec = bad_ec;
+  r = relative(p, q, ec);
+  VERIFY( !ec );
+  VERIFY( r == ".."/p );
+
+  ec = bad_ec;
+  r = relative(p, p/q, ec);
+  VERIFY( !ec );
+  VERIFY( r == ".." );
+
+  ec = bad_ec;
+  r = relative(p/q, p, ec);
+  VERIFY( !ec );
+  VERIFY( r == q );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/remove_all.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/remove_all.cc
new file mode 100644
index 00000000000..3f68f17f992
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/remove_all.cc
@@ -0,0 +1,92 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  std::error_code ec;
+  std::uintmax_t n;
+
+  n = fs::remove_all("", ec);
+  VERIFY( ec );
+  VERIFY( n == std::uintmax_t(-1) );
+
+  auto p = __gnu_test::nonexistent_path();
+  ec.clear();
+  n = remove_all(p, ec);
+  VERIFY( ec );
+  VERIFY( n == std::uintmax_t(-1) );
+
+  const auto bad_ec = ec;
+  auto link = __gnu_test::nonexistent_path();
+  create_symlink(p, link);  // dangling symlink
+  ec = bad_ec;
+  n = remove_all(link, ec);
+  VERIFY( !ec );
+  VERIFY( n == 1 );
+  VERIFY( !exists(symlink_status(link)) ); // DR 2721
+
+  __gnu_test::scoped_file f(p);
+  create_symlink(p, link);
+  ec = bad_ec;
+  n = remove_all(link, ec);
+  VERIFY( !ec );
+  VERIFY( n == 1 );
+  VERIFY( !exists(symlink_status(link)) );  // The symlink is removed, but
+  VERIFY( exists(p) );                      // its target is not.
+
+  auto dir = __gnu_test::nonexistent_path();
+  create_directories(dir/"a/b/c");
+  ec = bad_ec;
+  n = remove_all(dir/"a", ec);
+  VERIFY( !ec );
+  VERIFY( n == 3 );
+  VERIFY( exists(dir) );
+  VERIFY( !exists(dir/"a") );
+
+  create_directories(dir/"a/b/c");
+  __gnu_test::scoped_file a1(dir/"a/1");
+  __gnu_test::scoped_file a2(dir/"a/2");
+  __gnu_test::scoped_file b1(dir/"a/b/1");
+  __gnu_test::scoped_file b2(dir/"a/b/2");
+  ec = bad_ec;
+  n = remove_all(dir, ec);
+  VERIFY( !ec );
+  VERIFY( n == 8 );
+  VERIFY( !exists(dir) );
+
+  a1.path.clear();
+  a2.path.clear();
+  b1.path.clear();
+  b2.path.clear();
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/space.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/space.cc
new file mode 100644
index 00000000000..7b611352967
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/space.cc
@@ -0,0 +1,46 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// 30.10.14.3 Permissions [fs.op.space]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  std::filesystem::space_info s = std::filesystem::space("/");
+  std::error_code ec = make_error_code(std::errc::invalid_argument);
+  s = std::filesystem::space("/", ec);
+  VERIFY( !ec );
+  s = std::filesystem::space(__gnu_test::nonexistent_path(), ec);
+  VERIFY( ec );
+  VERIFY( s.capacity ==  static_cast<uintmax_t>(-1) );
+  VERIFY( s.free ==  static_cast<uintmax_t>(-1) );
+  VERIFY( s.available ==  static_cast<uintmax_t>(-1) );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/status.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/status.cc
new file mode 100644
index 00000000000..efe92b26a78
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/status.cc
@@ -0,0 +1,97 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  std::error_code ec = make_error_code(std::errc::invalid_argument);
+  fs::path dot = ".";
+
+  fs::file_status st1 = fs::status(dot, ec);
+  VERIFY( !ec );
+  VERIFY( st1.type() == fs::file_type::directory );
+
+  fs::file_status st2 = fs::status(dot);
+  VERIFY( st2.type() == fs::file_type::directory );
+}
+
+void
+test02()
+{
+  fs::path p = __gnu_test::nonexistent_path();
+
+  std::error_code ec;
+  fs::file_status st1 = fs::status(p, ec);
+  VERIFY( ec );
+  VERIFY( st1.type() == fs::file_type::not_found );
+
+  fs::file_status st2 = fs::status(p);
+  VERIFY( st2.type() == fs::file_type::not_found );
+}
+
+void
+test03()
+{
+  fs::path dir = __gnu_test::nonexistent_path();
+  fs::create_directory(dir);
+  __gnu_test::scoped_file d(dir, __gnu_test::scoped_file::adopt_file);
+  __gnu_test::scoped_file f(dir / "file");
+  fs::permissions(dir, fs::perms::none);
+
+  std::error_code ec;
+  fs::file_status st = fs::status(f.path, ec);
+  VERIFY( ec.value() == (int)std::errc::permission_denied );
+  VERIFY( st.type() == fs::file_type::none );
+
+#if __cpp_exceptions
+  bool caught = false;
+  std::error_code ec2;
+  fs::path p, p2;
+  try {
+    fs::symlink_status(f.path);
+  } catch (const fs::filesystem_error& e) {
+    caught = true;
+    p = e.path1();
+    p2 = e.path2();
+    ec2 = e.code();
+  }
+  VERIFY( caught );
+  VERIFY( ec2 == ec );
+  VERIFY( p == f.path );
+  VERIFY( p2.empty() );
+#endif
+
+  fs::permissions(dir, fs::perms::owner_all, ec);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/symlink_status.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/symlink_status.cc
new file mode 100644
index 00000000000..ae7a76be7c3
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/symlink_status.cc
@@ -0,0 +1,118 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec = bad_ec;
+  fs::path dot = ".";
+
+  fs::file_status st1 = fs::symlink_status(dot, ec);
+  VERIFY( !ec );
+  VERIFY( st1.type() == fs::file_type::directory );
+
+  fs::file_status st2 = fs::symlink_status(dot);
+  VERIFY( st2.type() == fs::file_type::directory );
+
+  fs::path link = __gnu_test::nonexistent_path();
+  create_directory_symlink(dot, link);
+
+  st1 = fs::symlink_status(link);
+  VERIFY( st1.type() == fs::file_type::symlink );
+  ec = bad_ec;
+  st2 = fs::symlink_status(link, ec);
+  VERIFY( !ec );
+  VERIFY( st2.type() == fs::file_type::symlink );
+}
+
+void
+test02()
+{
+  fs::path p = __gnu_test::nonexistent_path();
+
+  std::error_code ec;
+  fs::file_status st1 = fs::symlink_status(p, ec);
+  VERIFY( ec );
+  VERIFY( st1.type() == fs::file_type::not_found );
+
+  fs::file_status st2 = fs::symlink_status(p);
+  VERIFY( st2.type() == fs::file_type::not_found );
+}
+
+void
+test03()
+{
+  fs::path dir = __gnu_test::nonexistent_path();
+  fs::create_directory(dir);
+  __gnu_test::scoped_file d(dir, __gnu_test::scoped_file::adopt_file);
+  __gnu_test::scoped_file f(dir / "file");
+  fs::permissions(dir, fs::perms::none);
+  auto link = __gnu_test::nonexistent_path();
+  fs::create_symlink(f.path, link);
+  __gnu_test::scoped_file l(link, __gnu_test::scoped_file::adopt_file);
+
+  std::error_code ec;
+  fs::file_status st = fs::symlink_status(f.path, ec);
+  VERIFY( ec.value() == (int)std::errc::permission_denied );
+  VERIFY( st.type() == fs::file_type::none );
+
+  st = fs::symlink_status(link, ec);
+  VERIFY( !ec );
+  VERIFY( st.type() == fs::file_type::symlink );
+
+#if __cpp_exceptions
+  bool caught = false;
+  std::error_code ec2;
+  fs::path p, p2;
+  try {
+    fs::symlink_status(f.path);
+  } catch (const fs::filesystem_error& e) {
+    caught = true;
+    p = e.path1();
+    p2 = e.path2();
+    ec2 = e.code();
+  }
+  VERIFY( caught );
+  VERIFY( ec2.value() == (int)std::errc::permission_denied );
+  VERIFY( p == f.path );
+  VERIFY( p2.empty() );
+#endif
+
+  fs::file_status st2 = symlink_status(link);
+  VERIFY( st2.type() == fs::file_type::symlink );
+
+  fs::permissions(dir, fs::perms::owner_all, ec);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/temp_directory_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/temp_directory_path.cc
new file mode 100644
index 00000000000..ee065f0679f
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/temp_directory_path.cc
@@ -0,0 +1,127 @@
+// Copyright (C) 2015-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <stdlib.h>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+void
+clean_env()
+{
+  ::unsetenv("TMPDIR");
+  ::unsetenv("TMP");
+  ::unsetenv("TEMPDIR");
+  ::unsetenv("TEMP");
+}
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  clean_env();
+
+  if (!fs::exists("/tmp"))
+    return; // just give up
+
+  std::error_code ec = make_error_code(std::errc::invalid_argument);
+  fs::path p1 = fs::temp_directory_path(ec);
+  VERIFY( !ec );
+  VERIFY( exists(p1) );
+
+  fs::path p2 = fs::temp_directory_path();
+  VERIFY( p1 == p2 );
+}
+
+void
+test02()
+{
+  clean_env();
+
+  if (::setenv("TMPDIR", __gnu_test::nonexistent_path().string().c_str(), 1))
+    return; // just give up
+
+  std::error_code ec;
+  fs::path p = fs::temp_directory_path(ec);
+  VERIFY( ec );
+  VERIFY( p == fs::path() );
+
+  std::error_code ec2;
+  try {
+    p = fs::temp_directory_path();
+  } catch (const fs::filesystem_error& e) {
+    ec2 = e.code();
+  }
+  VERIFY( ec2 == ec );
+}
+
+void
+test03()
+{
+  auto p = __gnu_test::nonexistent_path();
+  create_directories(p/"tmp");
+  permissions(p, fs::perms::none);
+  setenv("TMPDIR", (p/"tmp").c_str(), 1);
+  std::error_code ec;
+  auto r = fs::temp_directory_path(ec); // libstdc++/PR71337
+  VERIFY( ec == std::make_error_code(std::errc::permission_denied) );
+  VERIFY( r == fs::path() );
+
+  std::error_code ec2;
+  try {
+    fs::temp_directory_path();
+  } catch (const fs::filesystem_error& e) {
+    ec2 = e.code();
+  }
+  VERIFY( ec2 == ec );
+
+  permissions(p, fs::perms::owner_all, ec);
+  remove_all(p, ec);
+}
+
+void
+test04()
+{
+  __gnu_test::scoped_file f;
+  setenv("TMPDIR", f.path.c_str(), 1);
+  std::error_code ec;
+  auto r = fs::temp_directory_path(ec);
+  VERIFY( ec == std::make_error_code(std::errc::not_a_directory) );
+  VERIFY( r == fs::path() );
+
+  std::error_code ec2;
+  try {
+    fs::temp_directory_path();
+  } catch (const fs::filesystem_error& e) {
+    ec2 = e.code();
+  }
+  VERIFY( ec2 == ec );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/operations/weakly_canonical.cc b/libstdc++-v3/testsuite/27_io/filesystem/operations/weakly_canonical.cc
new file mode 100644
index 00000000000..0f285e2bb59
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/operations/weakly_canonical.cc
@@ -0,0 +1,70 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+namespace fs = std::filesystem;
+
+void
+test01()
+{
+  auto dir = __gnu_test::nonexistent_path();
+  fs::create_directory(dir);
+  const auto dirc = canonical(dir);
+  fs::path foo = dir/"foo", bar = dir/"bar";
+  fs::create_directory(foo);
+  fs::create_directory(bar);
+  fs::create_directory(bar/"baz");
+  fs::create_symlink("../bar", foo/"bar");
+
+  auto p = fs::weakly_canonical(dir/"foo//./bar///../biz/.");
+  VERIFY( p == dirc/"biz/" );
+  p = fs::weakly_canonical(dir/"foo/.//bar/././baz/.");
+  VERIFY( p == dirc/"bar/baz" );
+  p = fs::weakly_canonical(fs::current_path()/dir/"bar//../foo/bar/baz");
+  VERIFY( p == dirc/"bar/baz" );
+
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
+  std::error_code ec;
+
+  ec = bad_ec;
+  p = fs::weakly_canonical(dir/"foo//./bar///../biz/.", ec);
+  VERIFY( !ec );
+  VERIFY( p == dirc/"biz/" );
+  ec = bad_ec;
+  p = fs::weakly_canonical(dir/"foo/.//bar/././baz/.", ec);
+  VERIFY( !ec );
+  VERIFY( p == dirc/"bar/baz" );
+  ec = bad_ec;
+  p = fs::weakly_canonical(fs::current_path()/dir/"bar//../foo/bar/baz", ec);
+  VERIFY( !ec );
+  VERIFY( p == dirc/"bar/baz" );
+
+  fs::remove_all(dir, ec);
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/append/path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/append/path.cc
new file mode 100644
index 00000000000..0942d6fdf7e
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/append/path.cc
@@ -0,0 +1,88 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 30.10.7.4.3 path appends [fs.path.append]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  const path p("/foo/bar");
+
+  path pp = p;
+  pp /= p;
+  compare_paths( pp, p );
+
+  path q("baz");
+
+  path qq = q;
+  qq /= q;
+  compare_paths( qq, "baz/baz" );
+
+  q /= p;
+  compare_paths( q, p );
+
+  path r = "";
+  r /= path();
+  VERIFY( r.empty() );
+
+  r /= path("rel");
+  VERIFY( !r.is_absolute() );
+
+  path s = "dir/";
+  s /= path("/file");
+  compare_paths( s, "/file" );
+
+  s = "dir/";
+  s /= path("file");
+  compare_paths( s, "dir/file" );
+}
+
+void
+test02()
+{
+  // C++17 [fs.path.append] p4
+
+  path p = path("//host") / "foo";
+  compare_paths( p, "//host/foo" );
+
+  path pp = path("//host/") / "foo";
+  compare_paths( pp, "//host/foo" );
+
+  path q = path("foo") / "";
+  compare_paths( q, "foo/" );
+
+  path qq = path("foo") / "/bar";
+  compare_paths( qq, "/bar" );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/assign/assign.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/assign/assign.cc
new file mode 100644
index 00000000000..d8797cae168
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/assign/assign.cc
@@ -0,0 +1,94 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  for (std::string s : __gnu_test::test_paths)
+  {
+    path p0 = s, p1, p2, p3, p4;
+
+    p1 = s;
+    compare_paths(p0, p1);
+
+    p2 = s.c_str();
+    compare_paths(p0, p2);
+
+#if _GLIBCXX_USE_WCHAR_T
+    std::wstring ws(s.begin(), s.end());
+
+    p3 = ws;
+    compare_paths(p0, p3);
+
+    p4 = ws.c_str();
+    compare_paths(p0, p4);
+#endif
+  }
+}
+
+void
+test02()
+{
+  for (std::string s : __gnu_test::test_paths)
+  {
+    path p0 = s, p1, p2, p3, p4, p5, p6, p7, p8;
+
+    p1.assign(s);
+    compare_paths(p0, p1);
+
+    p2.assign( s.begin(), s.end() );
+    compare_paths(p0, p2);
+
+    p3.assign( s.c_str() );
+    compare_paths(p0, p3);
+
+    p4.assign( s.c_str(), s.c_str() + s.size() );
+    compare_paths(p0, p4);
+
+#if _GLIBCXX_USE_WCHAR_T
+    std::wstring ws(s.begin(), s.end());
+
+    p5.assign(ws);
+    compare_paths(p0, p5);
+
+    p6.assign( ws.begin(), ws.end() );
+    compare_paths(p0, p6);
+
+    p7.assign( ws.c_str() );
+    compare_paths(p0, p7);
+
+    p8.assign( ws.c_str(), ws.c_str() + ws.size() );
+    compare_paths(p0, p8);
+#endif
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/assign/copy.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/assign/copy.cc
new file mode 100644
index 00000000000..27bf00ab126
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/assign/copy.cc
@@ -0,0 +1,56 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include <filesystem>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path copy;
+    copy = p;
+    __gnu_test::compare_paths(p, copy);
+  }
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path copy = p;
+    path move;
+    move = std::move(copy);
+    __gnu_test::compare_paths(p, move);
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/compare/compare.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/compare/compare.cc
new file mode 100644
index 00000000000..e444220ec25
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/compare/compare.cc
@@ -0,0 +1,51 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.8 path compare [path.compare]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p("/foo/bar");
+  VERIFY( p.compare(p) == 0 );
+  VERIFY( p.compare("/foo//bar") == 0 );
+
+  path q("/foo/baz");
+  VERIFY( p.compare(q) < 0 );
+  VERIFY( q.compare(p) > 0 );
+
+  path r("/foo/bar/.");
+  VERIFY( p.compare(r) < 0 );
+
+  VERIFY( path("a/b/").compare("a/b//") == 0 );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/compare/path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/compare/path.cc
new file mode 100644
index 00000000000..4bed5a23fd5
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/compare/path.cc
@@ -0,0 +1,51 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.8 path compare [path.compare]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  const path p0 = "/a/a/b/b";
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.compare(p) == 0 );
+    int cmp = p.compare(p0);
+    if (cmp == 0)
+      VERIFY( p0.compare(p) == 0 );
+    else if (cmp < 0)
+      VERIFY( p0.compare(p) > 0 );
+    else if (cmp > 0)
+      VERIFY( p0.compare(p) < 0 );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/compare/strings.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/compare/strings.cc
new file mode 100644
index 00000000000..be8f8b0d709
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/compare/strings.cc
@@ -0,0 +1,49 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.8 path compare [path.compare]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  const std::string s0 = "/a/a/b/b";
+  const path p0 = s0;
+  for (const std::string& s : __gnu_test::test_paths)
+  {
+    path p(s);
+    VERIFY( p.compare(s) == 0 );
+    VERIFY( p.compare(s.c_str()) == 0 );
+    VERIFY( p.compare(p0) == p.compare(s0) );
+    VERIFY( p.compare(p0) == p.compare(s0.c_str()) );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/concat/path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/concat/path.cc
new file mode 100644
index 00000000000..1a987c1966b
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/concat/path.cc
@@ -0,0 +1,67 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 30.10.7.4.4 path concatenation [fs.path.concat]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  const path p("/foo/bar");
+
+  path pp = p;
+  pp += p;
+  compare_paths( pp, "/foo/bar/foo/bar" );
+
+  path q("foo/bar");
+
+  path qq = q;
+  qq += q;
+  compare_paths( qq, "foo/barfoo/bar" );
+
+  q += p;
+  compare_paths( q, "foo/bar/foo/bar" );
+}
+
+void
+test02()
+{
+  for (path p : __gnu_test::test_paths)
+  {
+    auto prior_native = p.native();
+    path x("//blah/di/blah");
+    p += x;
+    VERIFY( p.native() == prior_native + x.native() );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/concat/strings.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/concat/strings.cc
new file mode 100644
index 00000000000..8d298f8a57d
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/concat/strings.cc
@@ -0,0 +1,56 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 30.10.7.4.4 path concatenation [fs.path.concat]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p("/");
+  p += path::string_type("foo");
+  VERIFY( p.filename() == "foo" );
+  p += "bar";
+  VERIFY( p.filename() == "foobar" );
+  p += '/';
+  VERIFY( p.parent_path() == "/foobar" && p.filename() == "" );
+#if _GLIBCXX_USE_WCHAR_T
+  p += L"baz.txt";
+#else
+  p += "baz.txt";
+#endif
+  VERIFY( p.filename() == "baz.txt" );
+  p.concat("/dir/");
+  VERIFY( p.parent_path() == "/foobar/baz.txt/dir" && p.filename() == "" );
+  std::string file = "file";
+  p.concat(file.begin(), file.end());
+  VERIFY( p.filename() == "file" );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/construct/copy.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/copy.cc
new file mode 100644
index 00000000000..cb084bdfeda
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/copy.cc
@@ -0,0 +1,55 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.1 path constructors [path.construct]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path copy = p;
+    __gnu_test::compare_paths(p, copy);
+  }
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path copy = p;
+    path move = std::move(copy);
+    __gnu_test::compare_paths(p, move);
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/construct/default.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/default.cc
new file mode 100644
index 00000000000..85ce129436d
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/default.cc
@@ -0,0 +1,51 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.1 path constructors [path.construct]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p;
+  VERIFY(  p.empty() );
+  VERIFY( !p.has_root_path() );
+  VERIFY( !p.has_root_name() );
+  VERIFY( !p.has_root_directory() );
+  VERIFY( !p.has_relative_path() );
+  VERIFY( !p.has_parent_path() );
+  VERIFY( !p.has_filename() );
+  VERIFY( !p.has_stem() );
+  VERIFY( !p.has_extension() );
+  VERIFY( !p.is_absolute() );
+  VERIFY(  p.is_relative() );
+  VERIFY( std::distance(p.begin(), p.end()) == 0 );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/construct/format.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/format.cc
new file mode 100644
index 00000000000..e7ed19cafe9
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/format.cc
@@ -0,0 +1,116 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  auto s = [&]() -> path::string_type { return "foo/bar"; };
+  path p0(s());
+  path p1(s(), path::auto_format);
+  VERIFY( p1 == p0 );
+  path p2(s(), path::native_format);
+  VERIFY( p2 == p0 );
+  path p3(s(), path::generic_format);
+  VERIFY( p3 == p0 );
+}
+
+void
+test02()
+{
+  path::string_type s = "foo/bar";
+  path p0(s);
+  path p1(s, path::auto_format);
+  VERIFY( p1 == p0 );
+  path p2(s, path::native_format);
+  VERIFY( p2 == p0 );
+  path p3(s, path::generic_format);
+  VERIFY( p3 == p0 );
+}
+
+void
+test03()
+{
+  const char* s = "foo/bar";
+  path p0(s);
+  path p1(s, path::auto_format);
+  VERIFY( p1 == p0 );
+  path p2(s, path::native_format);
+  VERIFY( p2 == p0 );
+  path p3(s, path::generic_format);
+  VERIFY( p3 == p0 );
+}
+
+void
+test04()
+{
+  const char s[] = "foo/bar";
+  path p0(std::begin(s), std::end(s));
+  path p1(std::begin(s), std::end(s), path::auto_format);
+  VERIFY( p1 == p0 );
+  path p2(std::begin(s), std::end(s), path::native_format);
+  VERIFY( p2 == p0 );
+  path p3(std::begin(s), std::end(s), path::generic_format);
+  VERIFY( p3 == p0 );
+}
+
+void
+test05()
+{
+  const char* s = "foo/bar";
+  std::locale loc;
+  path p0(s, loc);
+  path p1(s, loc, path::auto_format);
+  VERIFY( p1 == p0 );
+  path p2(s, loc, path::native_format);
+  VERIFY( p2 == p0 );
+  path p3(s, loc, path::generic_format);
+  VERIFY( p3 == p0 );
+}
+
+void
+test06()
+{
+  const char s[] = "foo/bar";
+  std::locale loc;
+  path p0(std::begin(s), std::end(s), loc);
+  path p1(std::begin(s), std::end(s), loc, path::auto_format);
+  VERIFY( p1 == p0 );
+  path p2(std::begin(s), std::end(s), loc, path::native_format);
+  VERIFY( p2 == p0 );
+  path p3(std::begin(s), std::end(s), loc, path::generic_format);
+  VERIFY( p3 == p0 );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+  test05();
+  test06();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/construct/locale.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/locale.cc
new file mode 100644
index 00000000000..8bbff810652
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/locale.cc
@@ -0,0 +1,40 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 30.10.7.4.1 path constructors [fs.path.construct]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p("/foo/bar", std::locale::classic());
+  VERIFY( p.native() == "/foo/bar" );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/construct/range.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/range.cc
new file mode 100644
index 00000000000..ad2cc18e512
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/range.cc
@@ -0,0 +1,112 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.1 path constructors [path.construct]
+
+#include <filesystem>
+#include <string>
+#include <testsuite_fs.h>
+#include <testsuite_iterators.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  for (std::string s : __gnu_test::test_paths)
+  {
+    path p1 = s;
+    path p2( s.begin(), s.end() );
+    path p3( s.c_str() );
+    path p4( s.c_str(), s.c_str() + s.size() );
+
+    compare_paths(p1, p2);
+    compare_paths(p1, p3);
+    compare_paths(p1, p4);
+
+#if _GLIBCXX_USE_WCHAR_T
+    std::wstring ws(s.begin(), s.end());
+    path p5 = ws;
+    path p6( ws.begin(), ws.end() );
+    path p7( ws.c_str() );
+    path p8( ws.c_str(), ws.c_str() + ws.size() );
+
+    compare_paths(p1, p5);
+    compare_paths(p1, p6);
+    compare_paths(p1, p7);
+    compare_paths(p1, p8);
+#endif
+
+    using __gnu_test::test_container;
+    using __gnu_test::input_iterator_wrapper;
+    // Test with input iterators and const value_types
+
+    test_container<char, input_iterator_wrapper>
+      r1((char*)s.c_str(), (char*)s.c_str() + s.size());
+    path p9(r1.begin(), r1.end());
+    compare_paths(p1, p9);
+
+    test_container<char, input_iterator_wrapper>
+      r2((char*)s.c_str(), (char*)s.c_str() + s.size() + 1); // includes null-terminator
+    path p10(r2.begin());
+    compare_paths(p1, p10);
+
+    test_container<const char, input_iterator_wrapper>
+      r3(s.c_str(), s.c_str() + s.size());
+    path p11(r3.begin(), r3.end());
+    compare_paths(p1, p11);
+
+    test_container<const char, input_iterator_wrapper>
+      r4(s.c_str(), s.c_str() + s.size() + 1); // includes null-terminator
+    path p12(r4.begin());
+    compare_paths(p1, p12);
+
+#if _GLIBCXX_USE_WCHAR_T
+    // Test with input iterators and const value_types
+    test_container<wchar_t, input_iterator_wrapper>
+      r5((wchar_t*)ws.c_str(), (wchar_t*)ws.c_str() + ws.size());
+    path p13(r5.begin(), r5.end());
+    compare_paths(p1, p13);
+
+    test_container<wchar_t, input_iterator_wrapper>
+      r6((wchar_t*)ws.c_str(), (wchar_t*)ws.c_str() + ws.size() + 1); // includes null-terminator
+    path p14(r6.begin());
+    compare_paths(p1, p14);
+
+    test_container<const wchar_t, input_iterator_wrapper>
+      r7(ws.c_str(), ws.c_str() + ws.size());
+    path p15(r7.begin(), r7.end());
+    compare_paths(p1, p15);
+
+    test_container<const wchar_t, input_iterator_wrapper>
+      r8(ws.c_str(), ws.c_str() + ws.size() + 1); // includes null-terminator
+    path p16(r8.begin());
+    compare_paths(p1, p16);
+#endif
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/construct/string_view.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/string_view.cc
new file mode 100644
index 00000000000..b9a52058791
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/construct/string_view.cc
@@ -0,0 +1,56 @@
+// { dg-options "-lstdc++fs -std=gnu++17" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.1 path constructors [path.construct]
+
+#include <filesystem>
+#include <string_view>
+#include <string>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  for (std::string s : __gnu_test::test_paths)
+  {
+    path p1 = s;
+    std::string_view sv(s);
+    path p2 = sv;
+    compare_paths(p1, p2);
+
+#if _GLIBCXX_USE_WCHAR_T
+    std::wstring ws(s.begin(), s.end());
+    path p3 = ws;
+    std::wstring_view wsv(ws);
+    path p4 = wsv;
+    compare_paths(p1, p4);
+#endif
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/extension.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/extension.cc
new file mode 100644
index 00000000000..13c9db275a0
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/extension.cc
@@ -0,0 +1,70 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  VERIFY( path("/foo/bar.txt").extension() == path(".txt") );
+  VERIFY( path("/foo/bar.baz.txt").extension() == path(".txt") );
+  VERIFY( path(".bar.baz.txt").extension() == path(".txt") );
+
+  VERIFY( path(".profile").extension() == path("") );
+  VERIFY( path(".profile.old").extension() == path(".old") );
+  VERIFY( path("..abc").extension() == path(".abc") );
+  VERIFY( path("...abc").extension() == path(".abc") );
+  VERIFY( path("abc..def").extension() == path(".def") );
+  VERIFY( path("abc...def").extension() == path(".def") );
+  VERIFY( path("abc.").extension() == path(".") );
+  VERIFY( path("abc..").extension() == path(".") );
+  VERIFY( path("abc.d.").extension() == path(".") );
+  VERIFY( path("..").extension() == path("") );
+  VERIFY( path(".").extension() == path("") );
+
+  VERIFY( path().extension() == path() );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    auto stem = p.stem();
+    auto ext = p.extension();
+    auto file = p.filename();
+    VERIFY( stem.native() + ext.native() == file.native() );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/filename.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/filename.cc
new file mode 100644
index 00000000000..8ba0fcc8cb9
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/filename.cc
@@ -0,0 +1,70 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// C++17 30.10.7.4.9 path decomposition [fs.path.decompose]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  VERIFY( path("/foo/bar.txt").filename() == "bar.txt" );
+  VERIFY( path("/foo/bar").filename()     == "bar"     );
+  VERIFY( path("/foo/bar/").filename()    == ""        );
+  VERIFY( path("/").filename()            == ""        );
+#ifdef __CYGWIN__
+  VERIFY( path("//host").filename()       == ""        );
+#else
+  VERIFY( path("//host").filename()       == "host"    );
+#endif
+  VERIFY( path(".").filename()            == "."       );
+  VERIFY( path("..").filename()           == ".."      );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path f = p.filename();
+    if (p.empty())
+      VERIFY( f.empty() );
+    else
+    {
+      const path back = *--p.end();
+      if (back.has_root_path())
+	VERIFY( f.empty() );
+      else
+	VERIFY( f == back );
+    }
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/parent_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/parent_path.cc
new file mode 100644
index 00000000000..c46566dad40
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/parent_path.cc
@@ -0,0 +1,73 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p0;
+  VERIFY( p0.parent_path() == p0 );
+  path p1 = "foo";
+  VERIFY( p1.parent_path() == p0 );
+  path p2 = "foo/bar";
+  VERIFY( p2.parent_path() == p1 );
+  path p3 = "/foo/bar";
+  VERIFY( p3.parent_path() == path("/foo") );
+  VERIFY( p3.parent_path().parent_path() == path("/") );
+  VERIFY( p3.parent_path().parent_path().parent_path() == path("/") );
+  path p4 = "/";
+  VERIFY( p4.parent_path() == p4 );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    if (p.begin() == p.end())
+      continue;
+    if (p.has_relative_path())
+    {
+      path pp;
+      for (auto i = p.begin(), end = --p.end(); i != end; ++i)
+      {
+	pp /= *i;
+      }
+      VERIFY( p.parent_path() == pp );
+    }
+    else
+      VERIFY( p.parent_path() == p );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/relative_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/relative_path.cc
new file mode 100644
index 00000000000..c8649724671
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/relative_path.cc
@@ -0,0 +1,70 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p1 = "foo";
+  VERIFY( p1.relative_path() == p1 );
+  path p2 = "foo/bar";
+  VERIFY( p2.relative_path() == p2 );
+  path p3 = "/foo/bar";
+  VERIFY( p3.relative_path() == p2 );
+}
+
+#include <iostream> // XXX
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    bool after_root = false;
+    const path prel = p.relative_path();
+    VERIFY( !prel.has_root_name() );
+    path rel;
+    for (const auto& cmpt : p)
+    {
+      if (!cmpt.has_root_path())
+        after_root = true;
+      if (after_root)
+        rel /= cmpt;
+    }
+    if (prel != rel)
+      std::cout << prel << ' ' << rel << '\n';
+    VERIFY( prel == rel );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_directory.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_directory.cc
new file mode 100644
index 00000000000..8cd1976a946
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_directory.cc
@@ -0,0 +1,59 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 30.10.7.4.9 path decomposition [fs.path.decompose]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p1 = "foo/bar";
+  VERIFY( p1.root_directory() == path() );
+  path p2 = "/foo/bar";
+  VERIFY( p2.root_directory() == path("/") );
+  path p3 = "//foo";
+  VERIFY( p3.root_directory() == path("/") );
+  path p4 = "///foo";
+  VERIFY( p4.root_directory() == path("/") );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path rootdir = p.root_directory();
+    VERIFY( !rootdir.has_relative_path() );
+    VERIFY( rootdir.empty() || rootdir.native() == "/");
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_name.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_name.cc
new file mode 100644
index 00000000000..5bb2ccd177e
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_name.cc
@@ -0,0 +1,43 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  VERIFY( path("/foo/bar.txt").extension() == ".txt" );
+  VERIFY( path("/foo/bar.baz.txt").extension() == ".txt" );
+  VERIFY( path(".").extension().empty() );
+  VERIFY( path("..").extension().empty() );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_path.cc
new file mode 100644
index 00000000000..bce0134a447
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/root_path.cc
@@ -0,0 +1,61 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// C++17 30.10.7.4.9 path decomposition [fs.path.decompose]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p1 = "foo/bar";
+  VERIFY( p1.root_path() == path() );
+  path p2 = "/foo/bar";
+  VERIFY( p2.root_path() == path("/") );
+}
+
+#undef VERIFY
+#define VERIFY(X) do { if (!(X)) { __builtin_puts("FAIL: " #X); } } while(false)
+#define DUMP(X, Y, Z) do { if (!(Y == Z)) { __builtin_printf("%s %s %s\n", X.c_str(), Y.c_str(), Z.c_str()); } } while(false)
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path rootp = p.root_path();
+    path rootn = p.root_name();
+    path rootd = p.root_directory();
+    VERIFY( rootp == (rootn / rootd) );
+    DUMP(p,  rootp , (rootn / rootd) );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/stem.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/stem.cc
new file mode 100644
index 00000000000..aec64e172ff
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/decompose/stem.cc
@@ -0,0 +1,62 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  VERIFY( path("/foo/bar.txt").stem() == path("bar") );
+  path p = "foo.bar.baz.tar";
+  std::vector<std::string> v;
+  for (; !p.extension().empty(); p = p.stem())
+    v.push_back(p.extension().native());
+  VERIFY( v.at(0) == ".tar" );
+  VERIFY( v.at(1) == ".baz" );
+  VERIFY( v.at(2) == ".bar" );
+
+  VERIFY( path(".profile").stem() == path(".profile") );
+  VERIFY( path(".profile.old").stem() == path(".profile") );
+  VERIFY( path("..abc").stem() == path(".") );
+  VERIFY( path("...abc").stem() == path("..") );
+  VERIFY( path("abc..def").stem() == path("abc.") );
+  VERIFY( path("abc...def").stem() == path("abc..") );
+  VERIFY( path("abc.").stem() == path("abc") );
+  VERIFY( path("abc..").stem() == path("abc.") );
+  VERIFY( path("abc.d.").stem() == path("abc.d") );
+  VERIFY( path("..").stem() == path("..") );
+  VERIFY( path(".").stem() == path(".") );
+
+  VERIFY( path().stem() == path() );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/generation/normal.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/generation/normal.cc
new file mode 100644
index 00000000000..df3b7154ab3
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/generation/normal.cc
@@ -0,0 +1,128 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  // C++17 [fs.path.gen] p2
+  compare_paths( path("foo/./bar/..").lexically_normal(), "foo/" );
+  compare_paths( path("foo/.///bar/../").lexically_normal(), "foo/" );
+}
+
+void
+test02()
+{
+  compare_paths( path("foo/../bar").lexically_normal(), "bar" );
+  compare_paths( path("../foo/../bar").lexically_normal(), "../bar" );
+  compare_paths( path("foo/../").lexically_normal(), "." );
+  compare_paths( path("../../").lexically_normal(), "../.." );
+  compare_paths( path("../").lexically_normal(), ".." );
+  compare_paths( path("./").lexically_normal(), "." );
+  compare_paths( path().lexically_normal(), "" );
+
+  compare_paths( path("/..").lexically_normal(), "/" );
+
+  // PR libstdc++/82777
+  compare_paths( path("./a/b/c/../.././b/c").lexically_normal(), "a/b/c" );
+  compare_paths( path("/a/b/c/../.././b/c").lexically_normal(), "/a/b/c" );
+}
+
+void
+test03()
+{
+  struct
+  {
+    const char* input;
+    const char* normalized;
+  } testcases[] = {
+    {""            , "" },
+    {"."           , "."  },
+    {".."          , ".." },
+    {"/"           , "/" },
+    {"//"          , "//" },
+
+    {"/foo"        , "/foo" },
+    {"/foo/"       , "/foo/" },
+    {"/foo/."      , "/foo/" },
+    {"/foo/bar/.." , "/foo/" },
+    {"/foo/.."     , "/" },
+
+    {"/."          , "/" },
+    {"/./"         , "/" },
+    {"/./."        , "/" },
+    {"/././"       , "/" },
+    {"/././."      , "/" },
+
+    {"./"          , "." },
+    {"./."         , "." },
+    {"././"        , "." },
+    {"././."       , "." },
+    {"./././"      , "." },
+    {"./././."     , "." },
+
+    {"foo/.."      , "." },
+    {"foo/../"     , "." },
+    {"foo/../.."   , ".." },
+
+    // with root name (OS-dependent):
+#if defined(_WIN32) && !defined(__CYGWIN__)
+    {"C:bar/.."    , "C:." },
+#else
+    {"C:bar/.."    , "." },
+#endif
+    {"C:/bar/.."   , "C:/" },
+    {"C:"          , "C:" },
+#ifdef __CYGWIN__
+    {"//host/bar/.." , "//host/" },
+    {"//host"        , "//host" },
+#else
+    {"//host/bar/.." , "/host/" },
+    {"//host"        , "/host" },
+#endif
+
+    // a few others:
+    {"foo/../foo/.."   , "." },
+    {"foo/../foo/../.."   , ".." },
+    {"../foo/../foo/.."   , ".." },
+    {"../.f/../f"   , "../f" },
+    {"../f/../.f"   , "../.f" },
+    {".././../."    , "../.." },
+    {".././.././"   , "../.." },
+    {"/.."          , "/" },
+  };
+  for (auto& test : testcases)
+    compare_paths( path(test.input).lexically_normal(), test.normalized );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/generation/proximate.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/generation/proximate.cc
new file mode 100644
index 00000000000..ee77e16d1f8
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/generation/proximate.cc
@@ -0,0 +1,55 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  // C++17 [fs.path.gen] p5
+  compare_paths( path("/a/d").lexically_proximate("/a/b/c"), "../../d" );
+  compare_paths( path("/a/b/c").lexically_proximate("/a/d"), "../b/c" );
+  compare_paths( path("a/b/c").lexically_proximate("a"), "b/c" );
+  compare_paths( path("a/b/c").lexically_proximate("a/b/c/x/y"), "../.." );
+  compare_paths( path("a/b/c").lexically_proximate("a/b/c"), "." );
+  compare_paths( path("a/b").lexically_proximate("c/d"), "../../a/b" );
+}
+
+void
+test02()
+{
+  path p = "a/b/c";
+  compare_paths( p.lexically_proximate(p), "." );
+  compare_paths( p.lexically_proximate("a/../a/b/../b/c/../c/."), "../../b/c" );
+  compare_paths( p.lexically_proximate("../../../"), p );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/generation/relative.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/generation/relative.cc
new file mode 100644
index 00000000000..3e78344a618
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/generation/relative.cc
@@ -0,0 +1,55 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  // C++17 [fs.path.gen] p5
+  compare_paths( path("/a/d").lexically_relative("/a/b/c"), "../../d" );
+  compare_paths( path("/a/b/c").lexically_relative("/a/d"), "../b/c" );
+  compare_paths( path("a/b/c").lexically_relative("a"), "b/c" );
+  compare_paths( path("a/b/c").lexically_relative("a/b/c/x/y"), "../.." );
+  compare_paths( path("a/b/c").lexically_relative("a/b/c"), "." );
+  compare_paths( path("a/b").lexically_relative("c/d"), "../../a/b" );
+}
+
+void
+test02()
+{
+  path p = "a/b/c";
+  compare_paths( p.lexically_relative(p), "." );
+  compare_paths( p.lexically_relative("a/../a/b/../b/c/../c/."), "../../b/c" );
+  compare_paths( p.lexically_relative("../../../"), "" );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/generic/generic_string.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/generic/generic_string.cc
new file mode 100644
index 00000000000..78224339b42
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/generic/generic_string.cc
@@ -0,0 +1,54 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// C++17 30.10.7.4.7 path generic format observers [fs.path.generic.obs]
+
+#include <filesystem>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  VERIFY( path().generic_string() == "" );
+  VERIFY( path("/").generic_string() == "/" );
+  VERIFY( path("////").generic_string() == "/" );
+#ifdef __CYGWIN__
+  VERIFY( path("//a").generic_string() == "//a" );
+  VERIFY( path("//a/").generic_string() == "//a/" );
+  VERIFY( path("//a/b").generic_string() == "//a/b" );
+#else
+  VERIFY( path("//a").generic_string() == "/a" );
+  VERIFY( path("//a/").generic_string() == "/a/" );
+  VERIFY( path("//a/b").generic_string() == "/a/b" );
+#endif
+  VERIFY( path("/a//b").generic_string() == "/a/b" );
+  VERIFY( path("/a//b/").generic_string() == "/a/b/" );
+  VERIFY( path("/a//b//").generic_string() == "/a/b/" );
+  VERIFY( path("/a//b//.").generic_string() == "/a/b/." );
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/itr/traversal.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/itr/traversal.cc
new file mode 100644
index 00000000000..7754140a714
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/itr/traversal.cc
@@ -0,0 +1,127 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// C++17 30.10.7.5 path iterators [fs.path.itr]
+
+#include <filesystem>
+#include <vector>
+#include <algorithm>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  path p;
+  VERIFY( p.begin() == p.end() );
+
+  std::vector<path> v, v2;
+
+  p = "/";
+  v.assign(p.begin(), p.end());
+  v2 = { "/" };
+  VERIFY( v == v2 );
+
+  p = "filename";
+  v.assign(p.begin(), p.end());
+  v2 = { "filename" };
+  VERIFY( v == v2 );
+
+  p = "dir/.";
+  v.assign(p.begin(), p.end());
+  v2 = { "dir", "." };
+  VERIFY( v == v2 );
+
+  p = "dir/";
+  v.assign(p.begin(), p.end());
+  v2 = { "dir", "" };
+  VERIFY( v == v2 );
+
+  p = "//rootname/dir/.";
+  v.assign(p.begin(), p.end());
+#ifdef __CYGWIN__
+  v2 = { "//rootname", "/", "dir", "." };
+#else
+  v2 = { "/", "rootname", "dir", "." };
+#endif
+  VERIFY( v == v2 );
+
+  p = "//rootname/dir/";
+  v.assign(p.begin(), p.end());
+#ifdef __CYGWIN__
+  v2 = { "//rootname", "/", "dir", "" };
+#else
+  v2 = { "/", "rootname", "dir", "" };
+#endif
+  VERIFY( v == v2 );
+
+  p = "//rootname/dir/filename";
+  v.assign(p.begin(), p.end());
+#ifdef __CYGWIN__
+  v2 = { "//rootname", "/", "dir", "filename" };
+#else
+  v2 = { "/", "rootname", "dir", "filename" };
+#endif
+  VERIFY( v == v2 );
+}
+
+void
+test02()
+{
+  using reverse_iterator = std::reverse_iterator<path::iterator>;
+  std::vector<path> fwd, rev;
+
+  for (const path& p : __gnu_test::test_paths)
+  {
+    const auto begin = p.begin(), end = p.end();
+    fwd.assign(begin, end);
+    rev.assign(reverse_iterator(end), reverse_iterator(begin));
+    VERIFY( fwd.size() == rev.size() );
+    VERIFY( std::equal(fwd.begin(), fwd.end(), rev.rbegin()) );
+  }
+}
+
+void
+test03()
+{
+  path paths[] = { "single", "multiple/elements" };
+  for (const path& p : paths)
+    for (auto iter = p.begin(); iter != p.end(); ++iter)
+    {
+      auto iter2 = iter;
+      ++iter;
+      iter2++;
+      VERIFY( iter2 == iter );
+      --iter;
+      iter2--;
+      VERIFY( iter2 == iter );
+    }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/clear.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/clear.cc
new file mode 100644
index 00000000000..31d984f1aa0
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/clear.cc
@@ -0,0 +1,46 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.5 path modifiers [path.modifiers]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (path p : __gnu_test::test_paths)
+  {
+    path empty;
+    p.clear();
+    VERIFY( p.empty() );
+    __gnu_test::compare_paths(p, empty);
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/make_preferred.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/make_preferred.cc
new file mode 100644
index 00000000000..6e306a41c2b
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/make_preferred.cc
@@ -0,0 +1,64 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 30.10.7.4.5 path modifiers [fs.path.modifiers]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+template<typename T, T sep>
+struct checker
+{
+  static void check(const char* s) { }
+};
+
+template<>
+struct checker<char, '/'>
+{
+  static void check()
+  {
+    VERIFY( path("foo/bar").make_preferred() == "foo/bar" );
+  }
+};
+
+template<>
+struct checker<wchar_t, L'\\'>
+{
+  static void check()
+  {
+    VERIFY( path("foo/bar").make_preferred() == L"foo\\bar" );
+  }
+};
+
+void
+test01()
+{
+  checker<path::value_type, path::preferred_separator>::check();
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/remove_filename.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/remove_filename.cc
new file mode 100644
index 00000000000..cb764cd1585
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/remove_filename.cc
@@ -0,0 +1,58 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// C++17 30.10.7.4.5 path modifiers [fs.path.modifiers]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  // C++17 [fs.path.modifiers] p8
+  compare_paths( path("foo/bar").remove_filename(), "foo/" );
+  compare_paths( path("foo/").remove_filename()   , "foo/" );
+  compare_paths( path("/foo").remove_filename()   , "/" );
+  compare_paths( path("/").remove_filename()      , "/" );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path p2(p);
+    p2.remove_filename();
+    p2 /= p.filename();
+    compare_paths( p2, p );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/replace_extension.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/replace_extension.cc
new file mode 100644
index 00000000000..fca189ce040
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/replace_extension.cc
@@ -0,0 +1,54 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 30.10.7.4.5 path modifiers [fs.path.modifiers]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  compare_paths( path("/foo.txt").replace_extension("cpp"), "/foo.cpp" );
+  compare_paths( path("/foo.txt").replace_extension(".cpp"), "/foo.cpp" );
+  compare_paths( path("/").replace_extension("bar"), "/.bar" );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path p2 = p;
+    compare_paths( p2.replace_extension(p2.extension()), p );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/replace_filename.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/replace_filename.cc
new file mode 100644
index 00000000000..d5c36565aca
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/replace_filename.cc
@@ -0,0 +1,55 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// C++17 30.10.7.4.5 path modifiers [fs.path.modifiers]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+using __gnu_test::compare_paths;
+
+void
+test01()
+{
+  // C++17 [fs.path.modifiers] p11
+  compare_paths( path("/foo").replace_filename("bar"), "/bar" );
+  compare_paths( path("/").replace_filename("bar")   , "/bar" );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path p2(p);
+    p2.replace_filename(p.filename());
+    compare_paths( p2, p );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/swap.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/swap.cc
new file mode 100644
index 00000000000..f1276042a95
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/modifiers/swap.cc
@@ -0,0 +1,45 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.5 path modifiers [path.modifiers]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  const path p("/foo/bar");
+  path p1;
+  path p2 = p;
+  p1.swap(p2);
+  VERIFY( p2.empty() );
+  __gnu_test::compare_paths(p1, p);
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/native/string.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/native/string.cc
new file mode 100644
index 00000000000..23d79700d97
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/native/string.cc
@@ -0,0 +1,70 @@
+// Copyright (C) 2016-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+#include <filesystem>
+#include <string>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  using namespace std::filesystem;
+  const std::string s = "abc";
+  path p(s);
+
+  VERIFY( p.native() == s );
+  VERIFY( p.c_str() == s );
+  VERIFY( static_cast<std::string>(p) == s );
+
+  std::string s2 = p; // implicit conversion
+  VERIFY( s2 == p.native() );
+}
+
+void
+test02()
+{
+  using namespace std::filesystem;
+  const char* s = "abc";
+  path p(s);
+
+  auto str = p.string<char>();
+  VERIFY( str == u"abc" );
+  VERIFY( str == p.string() );
+
+  auto strw = p.string<wchar_t>();
+  VERIFY( strw == L"abc" );
+  VERIFY( strw == p.wstring() );
+
+  auto str16 = p.string<char16_t>();
+  VERIFY( str16 == u"abc" );
+  VERIFY( str16 == p.u16string() );
+
+  auto str32 = p.string<char32_t>();
+  VERIFY( str32 == U"abc" );
+  VERIFY( str32 == p.u32string() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/nonmember/hash_value.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/nonmember/hash_value.cc
new file mode 100644
index 00000000000..20f42ac6ee5
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/nonmember/hash_value.cc
@@ -0,0 +1,52 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.6 path non-member functions [path.non-member]
+
+#include <filesystem>
+#include <testsuite_fs.h>
+#include <testsuite_hooks.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  VERIFY( hash_value(path("a//b")) == hash_value(path("a/b")) );
+  VERIFY( hash_value(path("a/")) == hash_value(path("a//")) );
+}
+
+void
+test02()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    path pp = p.native();
+    VERIFY( hash_value(p) == hash_value(pp) );
+  }
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/empty.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/empty.cc
new file mode 100644
index 00000000000..76277a0c9ff
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/empty.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const std::string& s : __gnu_test::test_paths)
+  {
+    VERIFY( s.empty() == path(s).empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_extension.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_extension.cc
new file mode 100644
index 00000000000..693bd389dcd
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_extension.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_extension() == !p.extension().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_filename.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_filename.cc
new file mode 100644
index 00000000000..ce99af3025f
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_filename.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_filename() == !p.filename().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_parent_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_parent_path.cc
new file mode 100644
index 00000000000..4e23b249607
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_parent_path.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_parent_path() == !p.parent_path().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_relative_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_relative_path.cc
new file mode 100644
index 00000000000..25e2dd5ea29
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_relative_path.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_relative_path() == !p.relative_path().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_directory.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_directory.cc
new file mode 100644
index 00000000000..66c8df20976
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_directory.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_root_directory() == !p.root_directory().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_name.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_name.cc
new file mode 100644
index 00000000000..f9d7157d646
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_name.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_root_name() == !p.root_name().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_path.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_path.cc
new file mode 100644
index 00000000000..f0bdb8156c6
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_root_path.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_root_path() == !p.root_path().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_stem.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_stem.cc
new file mode 100644
index 00000000000..18bf0159825
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/has_stem.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.has_stem() == !p.stem().empty() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/query/is_relative.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/query/is_relative.cc
new file mode 100644
index 00000000000..c6f6c7f98b1
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/query/is_relative.cc
@@ -0,0 +1,44 @@
+// { dg-options "-std=gnu++17 -lstdc++fs" }
+// { dg-do run { target c++17 } }
+// { dg-require-filesystem-ts "" }
+
+// Copyright (C) 2014-2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// 8.4.9 path decomposition [path.decompose]
+
+#include <filesystem>
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_fs.h>
+
+using std::filesystem::path;
+
+void
+test01()
+{
+  for (const path& p : __gnu_test::test_paths)
+  {
+    VERIFY( p.is_relative() == !p.is_absolute() );
+  }
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/27_io/headers/fstream/synopsis.cc b/libstdc++-v3/testsuite/27_io/headers/fstream/synopsis.cc
index f434f40ad2c..bc914b64698 100644
--- a/libstdc++-v3/testsuite/27_io/headers/fstream/synopsis.cc
+++ b/libstdc++-v3/testsuite/27_io/headers/fstream/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/27_io/headers/ios/synopsis.cc b/libstdc++-v3/testsuite/27_io/headers/ios/synopsis.cc
index c16d677fac2..1d3ba285166 100644
--- a/libstdc++-v3/testsuite/27_io/headers/ios/synopsis.cc
+++ b/libstdc++-v3/testsuite/27_io/headers/ios/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/27_io/headers/istream/synopsis.cc b/libstdc++-v3/testsuite/27_io/headers/istream/synopsis.cc
index 0173f1c30d1..894964a871a 100644
--- a/libstdc++-v3/testsuite/27_io/headers/istream/synopsis.cc
+++ b/libstdc++-v3/testsuite/27_io/headers/istream/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/27_io/headers/ostream/synopsis.cc b/libstdc++-v3/testsuite/27_io/headers/ostream/synopsis.cc
index 7529c7760ee..f3778cc1b2b 100644
--- a/libstdc++-v3/testsuite/27_io/headers/ostream/synopsis.cc
+++ b/libstdc++-v3/testsuite/27_io/headers/ostream/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/27_io/headers/sstream/synopsis.cc b/libstdc++-v3/testsuite/27_io/headers/sstream/synopsis.cc
index b925ee85a87..b7463685bf0 100644
--- a/libstdc++-v3/testsuite/27_io/headers/sstream/synopsis.cc
+++ b/libstdc++-v3/testsuite/27_io/headers/sstream/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/27_io/headers/streambuf/synopsis.cc b/libstdc++-v3/testsuite/27_io/headers/streambuf/synopsis.cc
index 2af15bbcfc4..93658ff7178 100644
--- a/libstdc++-v3/testsuite/27_io/headers/streambuf/synopsis.cc
+++ b/libstdc++-v3/testsuite/27_io/headers/streambuf/synopsis.cc
@@ -1,4 +1,5 @@
 // { dg-do compile }
+// { dg-require-normal-namespace "" }
 
 // Copyright (C) 2007-2017 Free Software Foundation, Inc.
 //
diff --git a/libstdc++-v3/testsuite/decimal/conversion-to-integral.cc b/libstdc++-v3/testsuite/decimal/conversion-to-integral.cc
index 6fd59a67b70..cc48fa8729d 100644
--- a/libstdc++-v3/testsuite/decimal/conversion-to-integral.cc
+++ b/libstdc++-v3/testsuite/decimal/conversion-to-integral.cc
@@ -64,7 +64,7 @@ void
 conversion_to_integral_128 (void)
 {
   #undef MAXVAL
-  #define MAXVAL LONG_LONG_MAX
+  #define MAXVAL __LONG_LONG_MAX__
   decimal128 a, b (1), c (-1), d (MAXVAL), e (-MAXVAL);
   long long ll;
 
diff --git a/libstdc++-v3/testsuite/experimental/filesystem/file_status/1.cc b/libstdc++-v3/testsuite/experimental/filesystem/file_status/1.cc
new file mode 100644
index 00000000000..970ad177b95
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/filesystem/file_status/1.cc
@@ -0,0 +1,75 @@
+// Copyright (C) 2017 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-options "-lstdc++fs" }
+// { dg-do run { target c++11 } }
+// { dg-require-filesystem-ts "" }
+
+#include <experimental/filesystem>
+#include <testsuite_hooks.h>
+
+namespace fs = std::experimental::filesystem;
+
+template<typename... Args>
+constexpr bool nothrow_constructible() {
+  return std::is_nothrow_constructible<fs::file_status, Args...>::value;
+}
+
+void
+test01()
+{
+  fs::file_status st0;
+  VERIFY( st0.type() == fs::file_type::none );
+  VERIFY( st0.permissions() == fs::perms::unknown );
+  static_assert( nothrow_constructible<>(), "" );
+
+  fs::file_status st1(fs::file_type::regular);
+  VERIFY( st1.type() == fs::file_type::regular );
+  VERIFY( st1.permissions() == fs::perms::unknown );
+  static_assert( nothrow_constructible<fs::file_type>(), "" );
+
+  fs::file_status st2(fs::file_type::directory, fs::perms::owner_all);
+  VERIFY( st2.type() == fs::file_type::directory );
+  VERIFY( st2.permissions() == fs::perms::owner_all );
+  static_assert( nothrow_constructible<fs::file_type, fs::perms>(), "" );
+
+  static_assert( nothrow_constructible<const fs::file_status&>(), "" );
+  static_assert( nothrow_constructible<fs::file_status>(), "" );
+}
+
+void
+test02()
+{
+  fs::file_status st;
+  VERIFY( st.type() == fs::file_type::none );
+  VERIFY( st.permissions() == fs::perms::unknown );
+
+  st.type(fs::file_type::symlink);
+  VERIFY( st.type() == fs::file_type::symlink );
+  VERIFY( st.permissions() == fs::perms::unknown );
+
+  st.permissions(fs::perms::owner_all);
+  VERIFY( st.type() == fs::file_type::symlink );
+  VERIFY( st.permissions() == fs::perms::owner_all );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/experimental/filesystem/iterators/recursive_directory_iterator.cc b/libstdc++-v3/testsuite/experimental/filesystem/iterators/recursive_directory_iterator.cc
index e7b5e53d43d..50cc7d45de8 100644
--- a/libstdc++-v3/testsuite/experimental/filesystem/iterators/recursive_directory_iterator.cc
+++ b/libstdc++-v3/testsuite/experimental/filesystem/iterators/recursive_directory_iterator.cc
@@ -28,6 +28,7 @@ namespace fs = std::experimental::filesystem;
 void
 test01()
 {
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
   std::error_code ec;
 
   // Test non-existent path.
@@ -37,15 +38,19 @@ test01()
   VERIFY( iter == end(iter) );
 
   // Test empty directory.
+  ec = bad_ec;
   create_directory(p, fs::current_path(), ec);
   VERIFY( !ec );
+  ec = bad_ec;
   iter = fs::recursive_directory_iterator(p, ec);
   VERIFY( !ec );
   VERIFY( iter == end(iter) );
 
   // Test non-empty directory.
-  create_directories(p / "d1/d2");
+  ec = bad_ec;
+  create_directories(p / "d1/d2", ec);
   VERIFY( !ec );
+  ec = bad_ec;
   iter = fs::recursive_directory_iterator(p, ec);
   VERIFY( !ec );
   VERIFY( iter != end(iter) );
@@ -56,6 +61,7 @@ test01()
   VERIFY( iter == end(iter) );
 
   // Test inaccessible directory.
+  ec = bad_ec;
   permissions(p, fs::perms::none, ec);
   VERIFY( !ec );
   iter = fs::recursive_directory_iterator(p, ec);
@@ -64,15 +70,19 @@ test01()
 
   // Test inaccessible directory, skipping permission denied.
   const auto opts = fs::directory_options::skip_permission_denied;
+  ec = bad_ec;
   iter = fs::recursive_directory_iterator(p, opts, ec);
   VERIFY( !ec );
   VERIFY( iter == end(iter) );
 
   // Test inaccessible sub-directory.
+  ec = bad_ec;
   permissions(p, fs::perms::owner_all, ec);
   VERIFY( !ec );
+  ec = bad_ec;
   permissions(p/"d1/d2", fs::perms::none, ec);
   VERIFY( !ec );
+  ec = bad_ec;
   iter = fs::recursive_directory_iterator(p, ec);
   VERIFY( !ec );
   VERIFY( iter != end(iter) );
@@ -84,12 +94,14 @@ test01()
   VERIFY( iter == end(iter) );
 
   // Test inaccessible sub-directory, skipping permission denied.
+  ec = bad_ec;
   iter = fs::recursive_directory_iterator(p, opts, ec);
   VERIFY( !ec );
   VERIFY( iter != end(iter) );
   VERIFY( iter->path() == p/"d1" );
   ++iter;              // should recurse into d1
   VERIFY( iter->path() == p/"d1/d2" );
+  ec = bad_ec;
   iter.increment(ec);  // should fail to recurse into p/d1/d2, so skip it
   VERIFY( !ec );
   VERIFY( iter == end(iter) );
@@ -101,12 +113,15 @@ test01()
 void
 test02()
 {
+  const std::error_code bad_ec = make_error_code(std::errc::invalid_argument);
   std::error_code ec;
   const auto p = __gnu_test::nonexistent_path();
+  ec = bad_ec;
   create_directories(p / "d1/d2", ec);
   VERIFY( !ec );
 
   // Test post-increment (libstdc++/71005)
+  ec = bad_ec;
   auto iter = fs::recursive_directory_iterator(p, ec);
   VERIFY( !ec );
   VERIFY( iter != end(iter) );
@@ -126,7 +141,7 @@ test02()
 void
 test03()
 {
-  std::error_code ec;
+  std::error_code ec = make_error_code(std::errc::invalid_argument);
   const auto p = __gnu_test::nonexistent_path();
   create_directories(p / "longer_than_small_string_buffer", ec);
   VERIFY( !ec );
diff --git a/libstdc++-v3/testsuite/experimental/filesystem/operations/create_directory.cc b/libstdc++-v3/testsuite/experimental/filesystem/operations/create_directory.cc
index f1c50a6e8e3..85e8281a6e1 100644
--- a/libstdc++-v3/testsuite/experimental/filesystem/operations/create_directory.cc
+++ b/libstdc++-v3/testsuite/experimental/filesystem/operations/create_directory.cc
@@ -50,7 +50,6 @@ test01()
   VERIFY( !ec );
   VERIFY( !b );
   b = create_directory(p);
-  VERIFY( !ec );
   VERIFY( !b );
 
   remove_all(p, ec);
diff --git a/libstdc++-v3/testsuite/experimental/filesystem/operations/temp_directory_path.cc b/libstdc++-v3/testsuite/experimental/filesystem/operations/temp_directory_path.cc
index e8a6dd3cd85..1689a2a0775 100644
--- a/libstdc++-v3/testsuite/experimental/filesystem/operations/temp_directory_path.cc
+++ b/libstdc++-v3/testsuite/experimental/filesystem/operations/temp_directory_path.cc
@@ -43,7 +43,7 @@ test01()
   if (!fs::exists("/tmp"))
     return; // just give up
 
-  std::error_code ec;
+  std::error_code ec = make_error_code(std::errc::invalid_argument);
   fs::path p1 = fs::temp_directory_path(ec);
   VERIFY( !ec );
   VERIFY( exists(p1) );
diff --git a/libstdc++-v3/testsuite/experimental/filesystem/path/construct/string_view.cc b/libstdc++-v3/testsuite/experimental/filesystem/path/construct/string_view.cc
index fbf8bd4d6d3..6879909c777 100644
--- a/libstdc++-v3/testsuite/experimental/filesystem/path/construct/string_view.cc
+++ b/libstdc++-v3/testsuite/experimental/filesystem/path/construct/string_view.cc
@@ -24,6 +24,7 @@
 #include <experimental/filesystem>
 #include <string_view>
 #include <string>
+#define USE_FILESYSTEM_TS
 #include <testsuite_fs.h>
 
 using std::experimental::filesystem::path;
diff --git a/libstdc++-v3/testsuite/experimental/filesystem/path/itr/traversal.cc b/libstdc++-v3/testsuite/experimental/filesystem/path/itr/traversal.cc
index bc1091476b5..41a292af4db 100644
--- a/libstdc++-v3/testsuite/experimental/filesystem/path/itr/traversal.cc
+++ b/libstdc++-v3/testsuite/experimental/filesystem/path/itr/traversal.cc
@@ -79,9 +79,27 @@ test02()
   }
 }
 
+void
+test03()
+{
+  path paths[] = { "single", "multiple/elements" };
+  for (const path& p : paths)
+    for (auto iter = p.begin(); iter != p.end(); ++iter)
+    {
+      auto iter2 = iter;
+      ++iter;
+      iter2++;
+      VERIFY( iter2 == iter );
+      --iter;
+      iter2--;
+      VERIFY( iter2 == iter );
+    }
+}
+
 int
 main()
 {
   test01();
   test02();
+  test03();
 }
diff --git a/libstdc++-v3/testsuite/lib/dg-options.exp b/libstdc++-v3/testsuite/lib/dg-options.exp
index 0dbf6592c14..57eff708eca 100644
--- a/libstdc++-v3/testsuite/lib/dg-options.exp
+++ b/libstdc++-v3/testsuite/lib/dg-options.exp
@@ -52,6 +52,15 @@ proc dg-require-normal-mode { args } {
     return
 }
 
+proc dg-require-normal-namespace { args } {
+    if { ![ check_v3_target_normal_namespace ] } {
+	upvar dg-do-what dg-do-what
+	set dg-do-what [list [lindex ${dg-do-what} 0] "N" "P"]
+	return
+    }
+    return
+}
+
 proc dg-require-parallel-mode { args } {
     if { ![ check_v3_target_parallel_mode ] } {
 	upvar dg-do-what dg-do-what
diff --git a/libstdc++-v3/testsuite/lib/libstdc++.exp b/libstdc++-v3/testsuite/lib/libstdc++.exp
index b081d8a5b90..02c88ede2ab 100644
--- a/libstdc++-v3/testsuite/lib/libstdc++.exp
+++ b/libstdc++-v3/testsuite/lib/libstdc++.exp
@@ -126,7 +126,7 @@ proc libstdc++_init { testfile } {
     # If a test doesn't have special options, use DEFAULT_CXXFLAGS.
     # Use this variable if the behavior
     #   1) only applies to libstdc++ testing
-    #   2) might need to be negated 
+    #   2) might need to be negated
     # In particular, some tests have to be run without precompiled
     # headers, or without assertions.
 
@@ -135,10 +135,10 @@ proc libstdc++_init { testfile } {
 	# Host specific goo here.
 	if { [string match "powerpc-*-darwin*" $target_triplet] } {
 	    append DEFAULT_CXXFLAGS " -multiply_defined suppress"
-	} 
+	}
 	if { [string match "powerpc-ibm-aix*" $target_triplet] } {
 	    append DEFAULT_CXXFLAGS " -Wl,-bmaxdata:0x20000000"
-	} 
+	}
     }
     v3track DEFAULT_CXXFLAGS 2
 
@@ -156,7 +156,7 @@ proc libstdc++_init { testfile } {
     # SHLIB_EXT on different platforms
     set gccdir [lookfor_file $tool_root_dir gcc/libgcc.a]
     if {$gccdir != ""} {
-        set gccdir [file dirname $gccdir]
+	set gccdir [file dirname $gccdir]
 	append ld_library_path_tmp ":${gccdir}"
     }
     v3track gccdir 3
@@ -166,7 +166,7 @@ proc libstdc++_init { testfile } {
     set libgompdir [lookfor_file $blddir/../libgomp .libs/libgomp.$shlib_ext]
     if {$libgompdir != ""} {
 	set v3-libgomp 1
-        set libgompdir [file dirname $libgompdir]
+	set libgompdir [file dirname $libgompdir]
 	append ld_library_path_tmp ":${libgompdir}"
 	verbose -log "libgomp support detected"
     }
@@ -177,7 +177,7 @@ proc libstdc++_init { testfile } {
     set libvtvdir [lookfor_file $blddir/../libvtv .libs/libvtv.$shlib_ext]
     if {$libvtvdir != ""} {
 	set v3-libvtv 1
-        set libvtvdir [file dirname $libvtvdir]
+	set libvtvdir [file dirname $libvtvdir]
 	append ld_library_path_tmp ":${libvtvdir}"
 	verbose -log "libvtv support detected"
     }
@@ -193,7 +193,7 @@ proc libstdc++_init { testfile } {
 	     && [isnative] } then {
 	    set v3-sharedlib 1
 	    verbose -log "shared library support detected"
-	} 
+	}
     }
     v3track v3-sharedlib 3
 
@@ -252,25 +252,25 @@ proc libstdc++_init { testfile } {
 	# $srcdir so we copy the testsuite headers into the current
 	# directory, and then add that to the search path.
 	foreach src [glob "${srcdir}/util/*.h" \
-		          "${srcdir}/util/*.cc" \
-		          "${srcdir}/util/*.tcc" \
-		          "${srcdir}/util/*.hpp" \
- 		          "${srcdir}/util/*/*.h" \
-		          "${srcdir}/util/*/*.cc" \
-		          "${srcdir}/util/*/*.tcc" \
+			  "${srcdir}/util/*.cc" \
+			  "${srcdir}/util/*.tcc" \
+			  "${srcdir}/util/*.hpp" \
+			  "${srcdir}/util/*/*.h" \
+			  "${srcdir}/util/*/*.cc" \
+			  "${srcdir}/util/*/*.tcc" \
 			  "${srcdir}/util/*/*.hpp" \
-		          "${srcdir}/util/*/*/*.h" \
-		          "${srcdir}/util/*/*/*.cc" \
-		          "${srcdir}/util/*/*/*.tcc" \
+			  "${srcdir}/util/*/*/*.h" \
+			  "${srcdir}/util/*/*/*.cc" \
+			  "${srcdir}/util/*/*/*.tcc" \
 			  "${srcdir}/util/*/*/*.hpp" \
-		          "${srcdir}/util/*/*/*/*.h" \
-		          "${srcdir}/util/*/*/*/*.cc" \
-		          "${srcdir}/util/*/*/*/*.tcc" \
-  			  "${srcdir}/util/*/*/*/*.hpp" \
-		          "${srcdir}/util/*/*/*/*/*.h" \
-		          "${srcdir}/util/*/*/*/*/*.cc" \
-		          "${srcdir}/util/*/*/*/*/*.tcc" \
-  			  "${srcdir}/util/*/*/*/*/*.hpp" ] {
+			  "${srcdir}/util/*/*/*/*.h" \
+			  "${srcdir}/util/*/*/*/*.cc" \
+			  "${srcdir}/util/*/*/*/*.tcc" \
+			  "${srcdir}/util/*/*/*/*.hpp" \
+			  "${srcdir}/util/*/*/*/*/*.h" \
+			  "${srcdir}/util/*/*/*/*/*.cc" \
+			  "${srcdir}/util/*/*/*/*/*.tcc" \
+			  "${srcdir}/util/*/*/*/*/*.hpp" ] {
 	    # Remove everything up to "util/..."
 	    set dst [string range $src [string length "${srcdir}/"] end]
 	    # Create the directory containing the file.
@@ -285,7 +285,7 @@ proc libstdc++_init { testfile } {
 	}
 	set includes "-Iutil"
     } elseif { [file exists $flags_file] } {
-        # If we find a testsuite_flags file, we're testing in the build dir.
+	# If we find a testsuite_flags file, we're testing in the build dir.
 	set cxx [exec sh $flags_file --build-cxx]
 	set cxxflags [exec sh $flags_file --cxxflags]
 	set cxxpchflags [exec sh $flags_file --cxxpchflags]
@@ -328,9 +328,9 @@ proc libstdc++_init { testfile } {
 	    verbose -log "Requested PCH file: $cxxpchflags"
 	    verbose -log "is not working, and will not be used."
 	    set cxxpchflags ""
-	} 
+	}
 	file delete $src
-     } 
+     }
     v3track cxxpchflags 2
 
     global PCH_CXXFLAGS
@@ -356,36 +356,36 @@ proc libstdc++_exit { } {
 proc libstdc++-dg-test { prog do_what extra_tool_flags } {
     # Set up the compiler flags, based on what we're going to do.
     switch $do_what {
-        "preprocess" {
-            set compile_type "preprocess"
-            set output_file "[file rootname [file tail $prog]].i"
-        }
-        "compile" {
-            set compile_type "assembly"
-            set output_file "[file rootname [file tail $prog]].s"
-        }
-        "assemble" {
-            set compile_type "object"
-            set output_file "[file rootname [file tail $prog]].o"
-        }
-        "link" {
-            set compile_type "executable"
-            set output_file "./[file rootname [file tail $prog]].exe"
-        }
-        "run" {
-            set compile_type "executable"
-            # FIXME: "./" is to cope with "." not being in $PATH.
-            # Should this be handled elsewhere?
-            # YES.
-            set output_file "./[file rootname [file tail $prog]].exe"
-            # This is the only place where we care if an executable was
-            # created or not.  If it was, dg.exp will try to run it.
-            catch { remote_file build delete $output_file }
-        }
+	"preprocess" {
+	    set compile_type "preprocess"
+	    set output_file "[file rootname [file tail $prog]].i"
+	}
+	"compile" {
+	    set compile_type "assembly"
+	    set output_file "[file rootname [file tail $prog]].s"
+	}
+	"assemble" {
+	    set compile_type "object"
+	    set output_file "[file rootname [file tail $prog]].o"
+	}
+	"link" {
+	    set compile_type "executable"
+	    set output_file "./[file rootname [file tail $prog]].exe"
+	}
+	"run" {
+	    set compile_type "executable"
+	    # FIXME: "./" is to cope with "." not being in $PATH.
+	    # Should this be handled elsewhere?
+	    # YES.
+	    set output_file "./[file rootname [file tail $prog]].exe"
+	    # This is the only place where we care if an executable was
+	    # created or not.  If it was, dg.exp will try to run it.
+	    catch { remote_file build delete $output_file }
+	}
 	default {
-            perror "$do_what: not a valid dg-do keyword"
-            return ""
-        }
+	    perror "$do_what: not a valid dg-do keyword"
+	    return ""
+	}
     }
 
     # Short-circut a bunch of complicated goo here for the special
@@ -404,7 +404,7 @@ proc libstdc++-dg-test { prog do_what extra_tool_flags } {
 	    regsub -all ".x c" $edit_tool_flags "" edit_tool_flags
 	    lappend options "additional_flags=$edit_tool_flags"
 	    set select_compile "v3_target_compile_as_c"
-        } else {
+	} else {
 	    lappend options "additional_flags=$extra_tool_flags"
 	}
     }
@@ -461,7 +461,7 @@ set v3-symver 0
 # Called from libstdc++-dg-test above.  Calls back into system's
 # target_compile to actually do the work.
 proc v3_target_compile { source dest type options } {
-    global gluefile 
+    global gluefile
     global wrap_flags
     global cxx
     global cxxflags
@@ -474,8 +474,8 @@ proc v3_target_compile { source dest type options } {
     lappend options "additional_flags=-fno-diagnostics-show-caret -fdiagnostics-color=never"
 
     if { [target_info needs_status_wrapper] != "" && [info exists gluefile] } {
-        lappend options "libs=${gluefile}"
-        lappend options "ldflags=${wrap_flags}"
+	lappend options "libs=${gluefile}"
+	lappend options "ldflags=${wrap_flags}"
     }
 
     set cxx_final $cxx
@@ -509,7 +509,7 @@ proc v3_target_compile { source dest type options } {
 # Called from libstdc++-dg-test above, but only for "C" compilation.
 # Calls back into system's target_compile to actually do the work.
 proc v3_target_compile_as_c { source dest type options } {
-    global gluefile 
+    global gluefile
     global wrap_flags
     global includes
     global flags_file
@@ -520,8 +520,8 @@ proc v3_target_compile_as_c { source dest type options } {
     global tool
 
     if { [target_info needs_status_wrapper] != "" && [info exists gluefile] } {
-        lappend options "libs=${gluefile}"
-        lappend options "ldflags=${wrap_flags}"
+	lappend options "libs=${gluefile}"
+	lappend options "ldflags=${wrap_flags}"
     }
 
     set tname [target_info name]
@@ -554,15 +554,15 @@ proc v3_target_compile_as_c { source dest type options } {
 	set crossbase "${comp_base_dir}/${machine}/include/c++/${version}"
 	set crosstarget "${crossbase}/${machine}"
 	set cc_final [concat $cc_final "-I$crossbase -I$crosstarget"]
-	# For a native compiler, the header files will be located at 
+	# For a native compiler, the header files will be located at
 	# the top level.
 	set includesbase "${comp_base_dir}/include/c++/${version}"
 	set includestarget "${includesbase}/${machine}"
 	set cc_final [concat $cc_final "-I$includesbase -I$includestarget"]
 
-        set libdir "-L${comp_base_dir}/lib"
+	set libdir "-L${comp_base_dir}/lib"
     } else {
-        set libdir "-L${blddir}/libsupc++/.libs"
+	set libdir "-L${blddir}/libsupc++/.libs"
 	set libdir [concat $libdir "-L${blddir}/src/.libs"]
     }
 
@@ -632,14 +632,14 @@ proc v3-build_support { } {
     # Build the support objects.
     set source_files [list testsuite_abi.cc testsuite_allocator.cc \
 			  testsuite_character.cc testsuite_hooks.cc \
-	       	          io/verified_cmd_line_input.cc \
+			  io/verified_cmd_line_input.cc \
 			  io/prog_bar.cc performance/time/elapsed_timer.cc ]
     foreach f $source_files {
 	set obj [file rootname $f].o
 	set object_file [file tail $obj]
 	# Compile with "-w" so that warnings issued by the compiler
 	# do not prevent compilation.
-        # Disable LTO so that ar/ranlib don't need the LTO plugin.
+	# Disable LTO so that ar/ranlib don't need the LTO plugin.
 	if { [v3_target_compile $srcdir/util/$f $object_file "object" \
 		  [list "incdir=$srcdir" "additional_flags=-w -fno-lto"]]
 	     != "" } {
@@ -691,7 +691,7 @@ proc v3-build_support { } {
 proc check_v3_target_fileio { } {
     global et_fileio_saved
     global et_fileio_target_name
-    global tool	
+    global tool
     global srcdir
 
     if { ![info exists et_fileio_target_name] } {
@@ -728,7 +728,7 @@ proc check_v3_target_fileio { } {
 	puts $f "#include <unistd.h>"
 	puts $f "#include <errno.h>"
 	puts $f "#include <string.h>"
-	puts $f "using namespace std;"	
+	puts $f "using namespace std;"
 	puts $f "int main ()"
 	puts $f "{"
 	puts $f "  int fd  = open (\"$testfile\", O_RDONLY);"
@@ -751,7 +751,7 @@ proc check_v3_target_fileio { } {
 	puts $f "    close (fd);"
 	puts $f "  }"
 	puts $f "  return ret;"
-	puts $f "}" 
+	puts $f "}"
 	close $f
 
 	set lines [v3_target_compile $src $exe executable ""]
@@ -779,7 +779,7 @@ proc check_v3_target_fileio { } {
 proc check_v3_target_c_std { } {
     global et_c_std_saved
     global et_c_std_target_name
-    global tool	
+    global tool
 
     if { ![info exists et_c_std_target_name] } {
 	set et_c_std_target_name ""
@@ -803,7 +803,7 @@ proc check_v3_target_c_std { } {
 
 	# Set up, compile, and execute a C++ test program that tries to use
 	# C99 functionality.
-        # For math bits, could use check_effective_target_c99_math.
+	# For math bits, could use check_effective_target_c99_math.
 	set src fileio[pid].cc
 	set exe fileio[pid].x
 
@@ -817,7 +817,7 @@ proc check_v3_target_c_std { } {
 	puts $f "  "
 	puts $f "  using std::wctomb;"
 	puts $f "  return i;"
-	puts $f "}" 
+	puts $f "}"
 	close $f
 
 	set lines [v3_target_compile $src $exe executable ""]
@@ -849,7 +849,7 @@ proc check_v3_target_sharedlib { } {
 proc check_v3_target_time { } {
     global et_time_saved
     global et_time_target_name
-    global tool	
+    global tool
 
     if { ![info exists et_time_target_name] } {
 	set et_time_target_name ""
@@ -877,11 +877,11 @@ proc check_v3_target_time { } {
 
 	set f [open $src "w"]
 	puts $f "#include <time.h>"
-	puts $f "using namespace std;"	
+	puts $f "using namespace std;"
 	puts $f "int main ()"
 	puts $f "{"
-	puts $f "  time (0);"	
-	puts $f "}" 
+	puts $f "  time (0);"
+	puts $f "}"
 	close $f
 
 	set lines [v3_target_compile $src /dev/null executable ""]
@@ -956,12 +956,12 @@ proc check_v3_target_namedlocale { args } {
       file delete $src
 
       if ![string match "" $lines] {
-        verbose "check_v3_target_namedlocale: compilation failed" 2
-        return $et_namedlocale
+	verbose "check_v3_target_namedlocale: compilation failed" 2
+	return $et_namedlocale
       }
       # else No error message, compilation succeeded.
     }
-	
+
     set result [${tool}_load "./$exe" "$args" ""]
     set status [lindex $result 0]
 
@@ -975,7 +975,7 @@ proc check_v3_target_namedlocale { args } {
 
 proc check_v3_target_debug_mode { } {
     global et_debug_mode
-    global tool	
+    global tool
 
     if { ![info exists et_debug_mode_target_name] } {
 	set et_debug_mode_target_name ""
@@ -1067,7 +1067,7 @@ proc check_v3_target_profile_mode { } {
 
 proc check_v3_target_normal_mode { } {
     global et_normal_mode
-    global tool	
+    global tool
 
     if { ![info exists et_normal_mode_target_name] } {
 	set et_normal_mode_target_name ""
@@ -1094,7 +1094,10 @@ proc check_v3_target_normal_mode { } {
 	set src normal_mode[pid].cc
 
 	set f [open $src "w"]
-	puts $f "#if defined(_GLIBCXX_DEBUG) || defined(_GLIBCXX_PROFILE) || defined(_GLIBCXX_PARALLEL)"
+	puts $f "#include <bits/c++config.h>"
+	puts $f "#if defined(_GLIBCXX_DEBUG) || \\"
+	puts $f "    defined(_GLIBCXX_PROFILE) || \\"
+	puts $f "    defined(_GLIBCXX_PARALLEL)"
 	puts $f "#  error No normal mode"
 	puts $f "#endif"
 	close $f
@@ -1111,12 +1114,59 @@ proc check_v3_target_normal_mode { } {
     return $et_normal_mode
 }
 
+proc check_v3_target_normal_namespace { } {
+    global et_normal_namespace
+    global tool
+
+    if { ![info exists et_normal_namespace_target_name] } {
+	set et_normal_namespace_target_name ""
+    }
+
+    # If the target has changed since we set the cached value, clear it.
+    set current_target [current_target_name]
+    if { $current_target != $et_normal_namespace_target_name } {
+	verbose "check_v3_target_normal_namespace: `$et_normal_namespace_target_name'" 2
+	set et_normal_namespace_target_name $current_target
+	if [info exists et_normal_namespace] {
+	    verbose "check_v3_target_normal_namespace: removing cached result" 2
+	    unset et_normal_namespace
+	}
+    }
+
+    if [info exists et_normal_namespace] {
+	verbose "check_v3_target_normal_namespace: using cached result" 2
+    } else {
+	set et_normal_namespace 0
+
+	# Set up and compile a C++ test program that depends
+	# on normal std namespace.
+	set src normal_namespace[pid].cc
+
+	set f [open $src "w"]
+	puts $f "#include <bits/c++config.h>"
+	puts $f "#if _GLIBCXX_INLINE_VERSION"
+	puts $f "#  error No normal namespace"
+	puts $f "#endif"
+	close $f
+
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	file delete $src
+
+	if [string match "" $lines] {
+	    # No error message, compilation succeeded.
+	    set et_normal_namespace 1
+	}
+    }
+    verbose "check_v3_target_normal_namespace: $et_normal_namespace" 2
+    return $et_normal_namespace
+}
+
 proc check_v3_target_parallel_mode { } {
     global cxxflags
     global v3-libgomp
     global et_parallel_mode
 
-    global tool	
+    global tool
 
     if { ![info exists et_parallel_mode_target_name] } {
 	set et_parallel_mode_target_name ""
@@ -1152,7 +1202,7 @@ proc check_v3_target_cstdint { } {
     global DEFAULT_CXXFLAGS
     global et_cstdint
 
-    global tool	
+    global tool
 
     if { ![info exists et_cstdint_target_name] } {
 	set et_cstdint_target_name ""
@@ -1381,49 +1431,49 @@ proc check_v3_target_gthreads { } {
     global tool
 
     if { ![info exists et_gthreads_target_name] } {
-        set et_gthreads_target_name ""
+	set et_gthreads_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_gthreads_target_name } {
-        verbose "check_v3_target_gthreads: `$et_gthreads_target_name'" 2
-        set et_gthreads_target_name $current_target
-        if [info exists et_gthreads] {
-            verbose "check_v3_target_gthreads: removing cached result" 2
-            unset et_gthreads
-        }
+	verbose "check_v3_target_gthreads: `$et_gthreads_target_name'" 2
+	set et_gthreads_target_name $current_target
+	if [info exists et_gthreads] {
+	    verbose "check_v3_target_gthreads: removing cached result" 2
+	    unset et_gthreads
+	}
     }
 
     if [info exists et_gthreads] {
-        verbose "check_v3_target_gthreads: using cached result" 2
+	verbose "check_v3_target_gthreads: using cached result" 2
     } else {
-        set et_gthreads 0
+	set et_gthreads 0
 
-        # Set up and preprocess a C++0x test program that depends
-        # on the gthreads facilities to be available.
-        set src gthreads[pid].cc
+	# Set up and preprocess a C++0x test program that depends
+	# on the gthreads facilities to be available.
+	set src gthreads[pid].cc
 
-        set f [open $src "w"]
+	set f [open $src "w"]
 	puts $f "#include <bits/c++config.h>"
-        puts $f "#ifndef _GLIBCXX_HAS_GTHREADS"
-        puts $f "#  error No gthread"
-        puts $f "#endif"
-        close $f
+	puts $f "#ifndef _GLIBCXX_HAS_GTHREADS"
+	puts $f "#  error No gthread"
+	puts $f "#endif"
+	close $f
 
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
 
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
 
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_gthreads 1
-        } else {
-            verbose "check_v3_target_gthreads: compilation failed" 2
-        }
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_gthreads 1
+	} else {
+	    verbose "check_v3_target_gthreads: compilation failed" 2
+	}
     }
     verbose "check_v3_target_gthreads: $et_gthreads" 2
     return $et_gthreads
@@ -1437,52 +1487,52 @@ proc check_v3_target_gthreads_timed { } {
     global tool
 
     if { ![info exists et_gthreads_timed_target_name] } {
-        set et_gthreads_timed_target_name ""
+	set et_gthreads_timed_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_gthreads_timed_target_name } {
-        verbose "check_v3_target_gthreads_timed: `$et_gthreads_timed_target_name'" 2
-        set et_gthreads_timed_target_name $current_target
-        if [info exists et_gthreads_timed] {
-            verbose "check_v3_target_gthreads_timed: removing cached result" 2
-            unset et_gthreads_timed
-        }
+	verbose "check_v3_target_gthreads_timed: `$et_gthreads_timed_target_name'" 2
+	set et_gthreads_timed_target_name $current_target
+	if [info exists et_gthreads_timed] {
+	    verbose "check_v3_target_gthreads_timed: removing cached result" 2
+	    unset et_gthreads_timed
+	}
     }
 
     if [info exists et_gthreads_timed] {
-        verbose "check_v3_target_gthreads_timed: using cached result" 2
+	verbose "check_v3_target_gthreads_timed: using cached result" 2
     } else {
-        set et_gthreads_timed 0
+	set et_gthreads_timed 0
 
-        # Set up and preprocess a C++0x test program that depends
-        # on the gthreads timed mutex facilities to be available.
-        set src gthreads_timed[pid].cc
+	# Set up and preprocess a C++0x test program that depends
+	# on the gthreads timed mutex facilities to be available.
+	set src gthreads_timed[pid].cc
 
-        set f [open $src "w"]
+	set f [open $src "w"]
 	puts $f "#include <bits/c++config.h>"
-        puts $f "#ifndef _GLIBCXX_HAS_GTHREADS"
-        puts $f "#  error No gthread"
-        puts $f "#endif"
-        puts $f "#if !_GTHREAD_USE_MUTEX_TIMEDLOCK"
-        puts $f "#  error No gthread timed mutexes"
-        puts $f "#endif"
-        close $f
-
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
-
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
-
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_gthreads_timed 1
-        } else {
-            verbose "check_v3_target_gthreads_timed: compilation failed" 2
-        }
+	puts $f "#ifndef _GLIBCXX_HAS_GTHREADS"
+	puts $f "#  error No gthread"
+	puts $f "#endif"
+	puts $f "#if !_GTHREAD_USE_MUTEX_TIMEDLOCK"
+	puts $f "#  error No gthread timed mutexes"
+	puts $f "#endif"
+	close $f
+
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
+
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_gthreads_timed 1
+	} else {
+	    verbose "check_v3_target_gthreads_timed: compilation failed" 2
+	}
     }
     verbose "check_v3_target_gthreads_timed: $et_gthreads_timed" 2
     return $et_gthreads_timed
@@ -1497,51 +1547,51 @@ proc check_v3_target_sleep { } {
     global tool
 
     if { ![info exists et_sleep_target_name] } {
-        set et_sleep_target_name ""
+	set et_sleep_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_sleep_target_name } {
-        verbose "check_v3_target_sleep: `$et_sleep_target_name'" 2
-        set et_sleep_target_name $current_target
-        if [info exists et_sleep] {
-            verbose "check_v3_target_sleep: removing cached result" 2
-            unset et_sleep
-        }
+	verbose "check_v3_target_sleep: `$et_sleep_target_name'" 2
+	set et_sleep_target_name $current_target
+	if [info exists et_sleep] {
+	    verbose "check_v3_target_sleep: removing cached result" 2
+	    unset et_sleep
+	}
     }
 
     if [info exists et_sleep] {
-        verbose "check_v3_target_sleep: using cached result" 2
+	verbose "check_v3_target_sleep: using cached result" 2
     } else {
-        set et_sleep 0
+	set et_sleep 0
 	
 	# Set up and preprocess a C++11 test program that depends
-        # on the sleep facilities to be available.
-        set src sleep[pid].cc
-
-        set f [open $src "w"]
-        puts $f "#include <bits/c++config.h>"
-        puts $f "#ifndef _GLIBCXX_USE_NANOSLEEP"
-        puts $f "# ifndef _GLIBCXX_HAVE_SLEEP"
-        puts $f "#  error No nanosleep or sleep"
-        puts $f "# endif"
-        puts $f "#endif"
-        close $f
-
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
-
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
-
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_sleep 1
-        } else {
-            verbose "check_v3_target_sleep: compilation failed" 2
-        }
+	# on the sleep facilities to be available.
+	set src sleep[pid].cc
+
+	set f [open $src "w"]
+	puts $f "#include <bits/c++config.h>"
+	puts $f "#ifndef _GLIBCXX_USE_NANOSLEEP"
+	puts $f "# ifndef _GLIBCXX_HAVE_SLEEP"
+	puts $f "#  error No nanosleep or sleep"
+	puts $f "# endif"
+	puts $f "#endif"
+	close $f
+
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
+
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_sleep 1
+	} else {
+	    verbose "check_v3_target_sleep: compilation failed" 2
+	}
     }
     verbose "check_v3_target_sleep: $et_sleep" 2
     return $et_sleep
@@ -1555,49 +1605,49 @@ proc check_v3_target_sched_yield { } {
     global tool
 
     if { ![info exists et_sched_yield_target_name] } {
-        set et_sched_yield_target_name ""
+	set et_sched_yield_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_sched_yield_target_name } {
-        verbose "check_v3_target_sched_yield: `$et_sched_yield_target_name'" 2
-        set et_sched_yield_target_name $current_target
-        if [info exists et_sched_yield] {
-            verbose "check_v3_target_sched_yield: removing cached result" 2
-            unset et_sched_yield
-        }
+	verbose "check_v3_target_sched_yield: `$et_sched_yield_target_name'" 2
+	set et_sched_yield_target_name $current_target
+	if [info exists et_sched_yield] {
+	    verbose "check_v3_target_sched_yield: removing cached result" 2
+	    unset et_sched_yield
+	}
     }
 
     if [info exists et_sched_yield] {
-        verbose "check_v3_target_sched_yield: using cached result" 2
+	verbose "check_v3_target_sched_yield: using cached result" 2
     } else {
-        set et_sched_yield 0
+	set et_sched_yield 0
 
-        # Set up and preprocess a C++0x test program that depends
-        # on the sched_yield facility to be available.
-        set src sched_yield[pid].cc
+	# Set up and preprocess a C++0x test program that depends
+	# on the sched_yield facility to be available.
+	set src sched_yield[pid].cc
 
-        set f [open $src "w"]
-        puts $f "#include <bits/c++config.h>"
-        puts $f "#ifndef _GLIBCXX_USE_SCHED_YIELD"
-        puts $f "#  error No sched yield"
-        puts $f "#endif"
-        close $f
+	set f [open $src "w"]
+	puts $f "#include <bits/c++config.h>"
+	puts $f "#ifndef _GLIBCXX_USE_SCHED_YIELD"
+	puts $f "#  error No sched yield"
+	puts $f "#endif"
+	close $f
 
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
 
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
 
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_sched_yield 1
-        } else {
-            verbose "check_v3_target_sched_yield: compilation failed" 2
-        }
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_sched_yield 1
+	} else {
+	    verbose "check_v3_target_sched_yield: compilation failed" 2
+	}
     }
     verbose "check_v3_target_sched_yield: $et_sched_yield" 2
     return $et_sched_yield
@@ -1611,49 +1661,49 @@ proc check_v3_target_string_conversions { } {
     global tool
 
     if { ![info exists et_string_conversions_target_name] } {
-        set et_string_conversions_target_name ""
+	set et_string_conversions_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_string_conversions_target_name } {
-        verbose "check_v3_target_string_conversions: `$et_string_conversions_target_name'" 2
-        set et_string_conversions_target_name $current_target
-        if [info exists et_string_conversions] {
-            verbose "check_v3_target_string_conversions: removing cached result" 2
-            unset et_string_conversions
-        }
+	verbose "check_v3_target_string_conversions: `$et_string_conversions_target_name'" 2
+	set et_string_conversions_target_name $current_target
+	if [info exists et_string_conversions] {
+	    verbose "check_v3_target_string_conversions: removing cached result" 2
+	    unset et_string_conversions
+	}
     }
 
     if [info exists et_string_conversions] {
-        verbose "check_v3_target_string_conversions: using cached result" 2
+	verbose "check_v3_target_string_conversions: using cached result" 2
     } else {
-        set et_string_conversions 0
+	set et_string_conversions 0
 	
 	# Set up and preprocess a C++0x test program that depends
-        # on the string_conversions facilities to be available.
-        set src string_conversions[pid].cc
-
-        set f [open $src "w"]
-        puts $f "#include <bits/c++config.h>"
-        puts $f "#if !(_GLIBCXX_USE_C99_STDIO && _GLIBCXX_USE_C99_STDLIB && _GLIBCXX_USE_C99_WCHAR) || defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)"
-        puts $f "#  error No string conversions"
-        puts $f "#endif"
-        close $f
-
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
-
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
-
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_string_conversions 1
-        } else {
-            verbose "check_v3_target_string_conversions: compilation failed" 2
-        }
+	# on the string_conversions facilities to be available.
+	set src string_conversions[pid].cc
+
+	set f [open $src "w"]
+	puts $f "#include <bits/c++config.h>"
+	puts $f "#if !(_GLIBCXX_USE_C99_STDIO && _GLIBCXX_USE_C99_STDLIB && _GLIBCXX_USE_C99_WCHAR) || defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)"
+	puts $f "#  error No string conversions"
+	puts $f "#endif"
+	close $f
+
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
+
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_string_conversions 1
+	} else {
+	    verbose "check_v3_target_string_conversions: compilation failed" 2
+	}
     }
     verbose "check_v3_target_string_conversions: $et_string_conversions" 2
     return $et_string_conversions
@@ -1667,49 +1717,49 @@ proc check_v3_target_swprintf { } {
     global tool
 
     if { ![info exists et_swprintf_target_name] } {
-        set et_swprintf_target_name ""
+	set et_swprintf_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_swprintf_target_name } {
-        verbose "check_v3_target_swprintf: `$et_swprintf_target_name'" 2
-        set et_swprintf_target_name $current_target
-        if [info exists et_swprintf] {
-            verbose "check_v3_target_swprintf: removing cached result" 2
-            unset et_swprintf
-        }
+	verbose "check_v3_target_swprintf: `$et_swprintf_target_name'" 2
+	set et_swprintf_target_name $current_target
+	if [info exists et_swprintf] {
+	    verbose "check_v3_target_swprintf: removing cached result" 2
+	    unset et_swprintf
+	}
     }
 
     if [info exists et_swprintf] {
-        verbose "check_v3_target_swprintf: using cached result" 2
+	verbose "check_v3_target_swprintf: using cached result" 2
     } else {
-        set et_swprintf 0
+	set et_swprintf 0
 	
 	# Set up and preprocess a C++0x test program that depends
-        # on a standard swprintf function to be available.
-        set src swprintf[pid].cc
-
-        set f [open $src "w"]
-        puts $f "#include <bits/c++config.h>"
-        puts $f "#if defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)"
-        puts $f "#  error No swprintf"
-        puts $f "#endif"
-        close $f
-
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
-
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
-
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_swprintf 1
-        } else {
-            verbose "check_v3_target_swprintf: compilation failed" 2
-        }
+	# on a standard swprintf function to be available.
+	set src swprintf[pid].cc
+
+	set f [open $src "w"]
+	puts $f "#include <bits/c++config.h>"
+	puts $f "#if defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)"
+	puts $f "#  error No swprintf"
+	puts $f "#endif"
+	close $f
+
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
+
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_swprintf 1
+	} else {
+	    verbose "check_v3_target_swprintf: compilation failed" 2
+	}
     }
     verbose "check_v3_target_swprintf: $et_swprintf" 2
     return $et_swprintf
@@ -1723,49 +1773,49 @@ proc check_v3_target_binary_io { } {
     global tool
 
     if { ![info exists et_binary_io_target_name] } {
-        set et_binary_io_target_name ""
+	set et_binary_io_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_binary_io_target_name } {
-        verbose "check_v3_target_binary_io: `$et_binary_io_target_name'" 2
-        set et_binary_io_target_name $current_target
-        if [info exists et_binary_io] {
-            verbose "check_v3_target_binary_io: removing cached result" 2
-            unset et_binary_io
-        }
+	verbose "check_v3_target_binary_io: `$et_binary_io_target_name'" 2
+	set et_binary_io_target_name $current_target
+	if [info exists et_binary_io] {
+	    verbose "check_v3_target_binary_io: removing cached result" 2
+	    unset et_binary_io
+	}
     }
 
     if [info exists et_binary_io] {
-        verbose "check_v3_target_binary_io: using cached result" 2
+	verbose "check_v3_target_binary_io: using cached result" 2
     } else {
-        set et_binary_io 0
+	set et_binary_io 0
 	
 	# Set up and preprocess a C++0x test program that depends
-        # on text and binary I/O being the same.
-        set src binary_io[pid].cc
-
-        set f [open $src "w"]
-        puts $f "#include <bits/c++config.h>"
-        puts $f "#if defined(_GLIBCXX_HAVE_DOS_BASED_FILESYSTEM)"
-        puts $f "#  error No binary io"
-        puts $f "#endif"
-        close $f
-
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
-
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
-
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_binary_io 1
-        } else {
-            verbose "check_v3_target_binary_io: compilation failed" 2
-        }
+	# on text and binary I/O being the same.
+	set src binary_io[pid].cc
+
+	set f [open $src "w"]
+	puts $f "#include <bits/c++config.h>"
+	puts $f "#if defined(_GLIBCXX_HAVE_DOS_BASED_FILESYSTEM)"
+	puts $f "#  error No binary io"
+	puts $f "#endif"
+	close $f
+
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
+
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_binary_io 1
+	} else {
+	    verbose "check_v3_target_binary_io: compilation failed" 2
+	}
     }
     verbose "check_v3_target_binary_io: $et_binary_io" 2
     return $et_binary_io
@@ -1803,7 +1853,7 @@ proc check_v3_target_nprocs { } {
 	set src nprocs[pid].cc
 
 	set f [open $src "w"]
-        puts $f "#include <bits/c++config.h>"
+	puts $f "#include <bits/c++config.h>"
 	puts $f "#if defined(_GLIBCXX_USE_GET_NPROCS)"
 	puts $f "#elif defined(_GLIBCXX_USE_PTHREADS_NUM_PROCESSORS_NP)"
 	puts $f "#elif defined(_GLIBCXX_USE_SYSCTL_HW_NCPU)"
@@ -1849,7 +1899,7 @@ proc check_v3_target_static_libstdcxx { } {
 	set src static-maybe[pid].cc
 
 	set f [open $src "w"]
-        puts $f "#include <iostream>"
+	puts $f "#include <iostream>"
 	puts $f "int main() {"
 	puts $f "int i(415);"
 	puts $f "std::cout<< i << std::endl;"
@@ -1883,46 +1933,46 @@ proc check_v3_target_little_endian { } {
     global tool
 
     if { ![info exists et_little_endian_target_name] } {
-        set et_little_endian_target_name ""
+	set et_little_endian_target_name ""
     }
 
     # If the target has changed since we set the cached value, clear it.
     set current_target [current_target_name]
     if { $current_target != $et_little_endian_target_name } {
-        verbose "check_v3_target_little_endian: `$et_little_endian_target_name'" 2
-        set et_little_endian_target_name $current_target
-        if [info exists et_little_endian] {
-            verbose "check_v3_target_little_endian: removing cached result" 2
-            unset et_little_endian
-        }
+	verbose "check_v3_target_little_endian: `$et_little_endian_target_name'" 2
+	set et_little_endian_target_name $current_target
+	if [info exists et_little_endian] {
+	    verbose "check_v3_target_little_endian: removing cached result" 2
+	    unset et_little_endian
+	}
     }
 
     if [info exists et_little_endian] {
-        verbose "check_v3_target_little_endian: using cached result" 2
+	verbose "check_v3_target_little_endian: using cached result" 2
     } else {
-        set et_little_endian 0
+	set et_little_endian 0
 	
-        set src little_endian[pid].cc
+	set src little_endian[pid].cc
 
-        set f [open $src "w"]
-        puts $f "#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__"
-        puts $f "#  error Not little endian"
-        puts $f "#endif"
-        close $f
+	set f [open $src "w"]
+	puts $f "#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__"
+	puts $f "#  error Not little endian"
+	puts $f "#endif"
+	close $f
 
-        set cxxflags_saved $cxxflags
-        set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
+	set cxxflags_saved $cxxflags
+	set cxxflags "$cxxflags $DEFAULT_CXXFLAGS -Werror"
 
-        set lines [v3_target_compile $src /dev/null preprocess ""]
-        set cxxflags $cxxflags_saved
-        file delete $src
+	set lines [v3_target_compile $src /dev/null preprocess ""]
+	set cxxflags $cxxflags_saved
+	file delete $src
 
-        if [string match "" $lines] {
-            # No error message, preprocessing succeeded.
-            set et_little_endian 1
-        } else {
-            verbose "check_v3_target_little_endian: compilation failed" 2
-        }
+	if [string match "" $lines] {
+	    # No error message, preprocessing succeeded.
+	    set et_little_endian 1
+	} else {
+	    verbose "check_v3_target_little_endian: compilation failed" 2
+	}
     }
     verbose "check_v3_target_little_endian: $et_little_endian" 2
     return $et_little_endian
@@ -2002,9 +2052,9 @@ proc check_effective_target_cxx11-abi { } {
     file delete $src
 
     if [string match "" $lines] {
-        # No error message, preprocessing succeeded.
-        verbose "check_v3_cxx11_abi: `1'" 2
-        return 1
+	# No error message, preprocessing succeeded.
+	verbose "check_v3_cxx11_abi: `1'" 2
+	return 1
     }
 
     verbose "check_v3_cxx11_abi: `0'" 2
diff --git a/libstdc++-v3/testsuite/util/testsuite_fs.h b/libstdc++-v3/testsuite/util/testsuite_fs.h
index 38ebd4fb079..c18dae28fcc 100644
--- a/libstdc++-v3/testsuite/util/testsuite_fs.h
+++ b/libstdc++-v3/testsuite/util/testsuite_fs.h
@@ -22,7 +22,14 @@
 #ifndef _TESTSUITE_FS_H
 #define _TESTSUITE_FS_H 1
 
+// Assume we want std::filesystem in C++17, unless USE_FILESYSTEM_TS defined:
+#if __cplusplus >= 201703L && ! defined USE_FILESYSTEM_TS
+#include <filesystem>
+namespace test_fs = std::filesystem;
+#else
 #include <experimental/filesystem>
+namespace test_fs = std::experimental::filesystem;
+#endif
 #include <fstream>
 #include <string>
 #include <cstdio>
@@ -33,13 +40,14 @@ namespace __gnu_test
 {
 #define PATH_CHK(p1, p2, fn) \
     if ( p1.fn() != p2.fn() ) \
-      throw std::experimental::filesystem::filesystem_error( #fn, p1, p2, \
+      throw test_fs::filesystem_error("comparing '" #fn "' failed", p1, p2, \
 	  std::make_error_code(std::errc::invalid_argument) )
 
   void
-  compare_paths(const std::experimental::filesystem::path& p1,
-		const std::experimental::filesystem::path& p2)
+  compare_paths(const test_fs::path& p1,
+		const test_fs::path& p2)
   {
+    PATH_CHK( p1, p2, native );
     PATH_CHK( p1, p2, string );
     PATH_CHK( p1, p2, empty );
     PATH_CHK( p1, p2, has_root_path );
@@ -55,7 +63,7 @@ namespace __gnu_test
     auto d1 = std::distance(p1.begin(), p1.end());
     auto d2 = std::distance(p2.begin(), p2.end());
     if( d1 != d2 )
-      throw std::experimental::filesystem::filesystem_error(
+      throw test_fs::filesystem_error(
 	  "distance(begin, end)", p1, p2,
 	  std::make_error_code(std::errc::invalid_argument) );
   }
@@ -67,15 +75,15 @@ namespace __gnu_test
 
   // This is NOT supposed to be a secure way to get a unique name!
   // We just need a path that doesn't exist for testing purposes.
-  std::experimental::filesystem::path
+  test_fs::path
   nonexistent_path()
   {
-    std::experimental::filesystem::path p;
+    test_fs::path p;
 #if defined(_GNU_SOURCE) || _XOPEN_SOURCE >= 500 || _POSIX_C_SOURCE >= 200112L
-    char tmp[] = "filesystem-ts-test.XXXXXX";
+    char tmp[] = "filesystem-test.XXXXXX";
     int fd = ::mkstemp(tmp);
     if (fd == -1)
-      throw std::experimental::filesystem::filesystem_error("mkstemp failed",
+      throw test_fs::filesystem_error("mkstemp failed",
 	  std::error_code(errno, std::generic_category()));
     ::unlink(tmp);
     ::close(fd);
@@ -88,7 +96,7 @@ namespace __gnu_test
 #else
     std::sprintf(buf,
 #endif
-      "filesystem-ts-test.%d.%lu", counter++, (unsigned long) ::getpid());
+      "filesystem-test.%d.%lu", counter++, (unsigned long) ::getpid());
     p = buf;
 #endif
     return p;
@@ -97,7 +105,7 @@ namespace __gnu_test
   // RAII helper to remove a file on scope exit.
   struct scoped_file
   {
-    using path_type = std::experimental::filesystem::path;
+    using path_type = test_fs::path;
 
     enum adopt_file_t { adopt_file };
 
diff --git a/libstdc++-v3/testsuite/util/testsuite_iterators.h b/libstdc++-v3/testsuite/util/testsuite_iterators.h
index d61b2162c14..d6b550dd416 100644
--- a/libstdc++-v3/testsuite/util/testsuite_iterators.h
+++ b/libstdc++-v3/testsuite/util/testsuite_iterators.h
@@ -180,6 +180,11 @@ namespace __gnu_test
 #endif
   };
 
+#if __cplusplus >= 201103L
+  template<typename T, typename U>
+    void operator,(const T&, const output_iterator_wrapper<U>&) = delete;
+#endif
+
   /**
    * @brief input_iterator wrapper for pointer
    *
@@ -270,6 +275,10 @@ namespace __gnu_test
 #endif
   };
 
+#if __cplusplus >= 201103L
+  template<typename T, typename U>
+    void operator,(const T&, const input_iterator_wrapper<U>&) = delete;
+#endif
 
   /**
    * @brief forward_iterator wrapper for pointer