CFG transparent RTL expansion:

	* Makefile.in (cfgexpand.o): New object file.
	(builtins.o): Add dependency on basic-block.h
	* builtins.c: Include basic-block.h
	(entry_of_function): New function.
	(expand_builtin_apply_args, expand_builtin_saveargs): Use it.
	* cfgexpand.c: New file.
	* expr.c (execute_expand, pass_expand): Kill.
	* pass.c (rest_of_compilation): Do not build CFG unless called from
	coverage code.
	* tree-cfg.c (delete_tree_cfg): Rename to..
	(delete_tree_cfg_annotations): ... this one; Do not remove the CFG itself.
	* tree-flow.h (delete_tree_cfg_annotations): Declare.
	(dleete_tree_cfg): Kill.
	* tree-optimize.c (execute_rebuild_bind, pass_rebuild_bind): Kill.
	(execute_del_cfg): Rename to...
	(execute_free_datastructures): This one...
	(pass_del_cfg): Rename to...
	(pass_free_datastructures): ... this one; Do not kill PROP_cfg.
	(init_tree_optimization_passes): Make cfg build and profiling to happen
	unconditionally.

	* tree-mudflap.c (mf_decl_cache_locals): Skip labels before
	inserting the cache variables.

	* tree-mudflap.c: Include headers to make basic_block available.
	Move functions around such that related functions are near each
	other.  Add prototypes for all static functions.  Add comments
	briefly explaining what IR the mudflap1 and mudflap2 work on and
	what they do.
	(mudflap_function_decls): Rename to execute_mudflap_function_decls.
	(mudflap_function_ops): Rename to execute_mudflap_function_ops.
	(pass_mudflap_1, pass_mudflap_2): Update.
	(mf_decl_cache_locals): Make it work on the CFG instead of the saved
	function tree.
	(mf_build_check_statement_for): Make it work on the CFG.
	(mf_xform_derefs_1): Likewise.  Cleanup code style.
	(mf_xform_derefs): Likewise.

	* tree-cfg.c (label_to_block): Invent the label destination for
	undefined labels.
	(cleanup_dead_labels): Update table in the case label_to_block added
	new label.


git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@83385 138bc75d-0d04-0410-961f-82ee72b054a4

1 files changed, 445 insertions, 320 deletions

diff --git a/gcc/tree-mudflap.c b/gcc/tree-mudflap.c
index 8a57efbe676..732ef5e4106 100644
--- a/gcc/tree-mudflap.c
+++ b/gcc/tree-mudflap.c
@@ -26,7 +26,11 @@ Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include "system.h"
 #include "coretypes.h"
 #include "tm.h"
+#include "hard-reg-set.h"
+#include "rtl.h"
 #include "tree.h"
+#include "tm_p.h"
+#include "basic-block.h"
 #include "flags.h"
 #include "function.h"
 #include "tree-inline.h"
@@ -43,50 +47,31 @@ Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 
 /* Internal function decls */
 
-static void mf_xform_derefs (tree);
-static void mf_xform_decls (tree, tree);
-static void mf_init_extern_trees (void);
-static void mf_decl_cache_locals (tree *);
-static void mf_decl_clear_locals (void);
+/* Helpers.  */
+static tree mf_build_string (const char *string);
 static tree mf_varname_tree (tree);
-static tree mx_xfn_xform_decls (tree *, int *, void *);
-
-static void mx_register_decls (tree, tree *);
-
-
-/* extern mudflap functions */
-
-static GTY ((param_is (union tree_node))) htab_t marked_trees = NULL;
-
-
-/* Mark and return the given tree node to prevent further mudflap
-   transforms.  */
-tree
-mf_mark (tree t)
-{
-  void **slot;
-
-  if (marked_trees == NULL)
-    marked_trees = htab_create_ggc (31, htab_hash_pointer, htab_eq_pointer, NULL);
+static tree mf_file_function_line_tree (location_t *);
 
-  slot = htab_find_slot (marked_trees, t, INSERT);
-  *slot = t;
-  return t;
-}
+/* Initialization of all the mf-runtime.h extern decls.  */
+static void mf_init_extern_trees (void);
 
+/* Indirection-related instrumentation.  */
+static void mf_decl_cache_locals (void);
+static void mf_decl_clear_locals (void);
+static void mf_xform_derefs (void);
+static void execute_mudflap_function_ops (void);
 
-int
-mf_marked_p (tree t)
-{
-  void *entry;
+/* Addressable variables instrumentation.  */
+static void mf_xform_decls (tree, tree);
+static tree mx_xfn_xform_decls (tree *, int *, void *);
+static void mx_register_decls (tree, tree *);
+static void execute_mudflap_function_decls (void);
 
-  if (marked_trees == NULL)
-    return 0;
 
-  entry = htab_find (marked_trees, t);
-  return (entry != NULL);
-}
+/* ------------------------------------------------------------------------ */
+/* Some generally helpful functions for mudflap instrumentation.  */
 
+/* Build a reference to a literal string.  */
 static tree
 mf_build_string (const char *string)
 {
@@ -106,192 +91,6 @@ mf_build_string (const char *string)
   return mf_mark (result);
 }
 
-/* Perform the declaration-related mudflap tree transforms on the
-   given function.  Update its DECL_SAVED_TREE.  */
-
-static void
-mudflap_function_decls (void)
-{
-  if (mf_marked_p (current_function_decl))
-    return;
-
-  push_gimplify_context ();
-
-  mf_init_extern_trees ();
-  mf_xform_decls (DECL_SAVED_TREE (current_function_decl),
-                  DECL_ARGUMENTS (current_function_decl));
-
-  pop_gimplify_context (NULL);
-}
-
-static bool
-gate_mudflap (void)
-{
-  return flag_mudflap != 0;
-}
-
-struct tree_opt_pass pass_mudflap_1 = 
-{
-  "mudflap1",                           /* name */
-  gate_mudflap,                         /* gate */
-  mudflap_function_decls,               /* execute */
-  NULL,                                 /* sub */
-  NULL,                                 /* next */
-  0,                                    /* static_pass_number */
-  0,                                    /* tv_id */
-  PROP_gimple_any,                      /* properties_required */
-  0,                                    /* properties_provided */
-  0,                                    /* properties_destroyed */
-  0,                                    /* todo_flags_start */
-  TODO_dump_func                        /* todo_flags_finish */
-};
-
-
-/* Same as above, for the indirection-related transforms.  */
-
-static void
-mudflap_function_ops (void)
-{
-  if (mf_marked_p (current_function_decl))
-    return;
-
-  push_gimplify_context ();
-
-  /* In multithreaded mode, don't cache the lookup cache parameters.  */
-  if (! flag_mudflap_threads)
-    mf_decl_cache_locals (&DECL_SAVED_TREE (current_function_decl));
-
-  mf_xform_derefs (DECL_SAVED_TREE (current_function_decl));
-
-  if (! flag_mudflap_threads)
-    mf_decl_clear_locals ();
-
-  pop_gimplify_context (NULL);
-}
-
-struct tree_opt_pass pass_mudflap_2 = 
-{
-  "mudflap2",                           /* name */
-  gate_mudflap,                         /* gate */
-  mudflap_function_ops,                 /* execute */
-  NULL,                                 /* sub */
-  NULL,                                 /* next */
-  0,                                    /* static_pass_number */
-  0,                                    /* tv_id */
-  PROP_gimple_leh,                      /* properties_required */
-  0,                                    /* properties_provided */
-  0,                                    /* properties_destroyed */
-  0,                                    /* todo_flags_start */
-  TODO_dump_func                        /* todo_flags_finish */
-};
-
-/* global tree nodes */
-
-/* Global tree objects for global variables and functions exported by
-   mudflap runtime library.  mf_init_extern_trees must be called
-   before using these.  */
-
-/* uintptr_t (usually "unsigned long") */
-static GTY (()) tree mf_uintptr_type;
-
-/* struct __mf_cache { uintptr_t low; uintptr_t high; }; */
-static GTY (()) tree mf_cache_struct_type;
-
-/* struct __mf_cache * const */
-static GTY (()) tree mf_cache_structptr_type;
-
-/* extern struct __mf_cache __mf_lookup_cache []; */
-static GTY (()) tree mf_cache_array_decl;
-
-/* extern const unsigned char __mf_lc_shift; */
-static GTY (()) tree mf_cache_shift_decl;
-
-/* extern const uintptr_t __mf_lc_mask; */
-static GTY (()) tree mf_cache_mask_decl;
-
-/* Their function-scope local shadows, used in single-threaded mode only.  */
-
-/* auto const unsigned char __mf_lc_shift_l; */
-static GTY (()) tree mf_cache_shift_decl_l;
-
-/* auto const uintptr_t __mf_lc_mask_l; */
-static GTY (()) tree mf_cache_mask_decl_l;
-
-/* extern void __mf_check (void *ptr, size_t sz, int type, const char *); */
-static GTY (()) tree mf_check_fndecl;
-
-/* extern void __mf_register (void *ptr, size_t sz, int type, const char *); */
-static GTY (()) tree mf_register_fndecl;
-
-/* extern void __mf_unregister (void *ptr, size_t sz); */
-static GTY (()) tree mf_unregister_fndecl;
-
-
-/* Initialize the global tree nodes that correspond to mf-runtime.h
-   declarations.  */
-static void
-mf_init_extern_trees (void)
-{
-  static bool done = false;
-
-  if (done)
-    return;
-  done = true;
-
-  mf_uintptr_type = TREE_TYPE (mflang_lookup_decl ("uintptr_t"));
-  mf_cache_array_decl = mf_mark (mflang_lookup_decl ("__mf_lookup_cache"));
-  mf_cache_struct_type = TREE_TYPE (TREE_TYPE (mf_cache_array_decl));
-  mf_cache_structptr_type = build_pointer_type (mf_cache_struct_type);
-  mf_cache_shift_decl = mf_mark (mflang_lookup_decl ("__mf_lc_shift"));
-  mf_cache_mask_decl = mf_mark (mflang_lookup_decl ("__mf_lc_mask"));
-  mf_check_fndecl = mflang_lookup_decl ("__mf_check");
-  mf_register_fndecl = mflang_lookup_decl ("__mf_register");
-  mf_unregister_fndecl = mflang_lookup_decl ("__mf_unregister");
-}
-
-
-
-/* Create and initialize local shadow variables for the lookup cache
-   globals.  Put their decls in the *_l globals for use by
-   mf_build_check_statement_for.  */
-
-static void
-mf_decl_cache_locals (tree* body)
-{
-  tree_stmt_iterator i = tsi_start (*body);
-  tree t;
-
-  mf_cache_shift_decl_l
-    = mf_mark (create_tmp_var (TREE_TYPE (mf_cache_shift_decl),
-                               "__mf_lookup_shift_l"));
-
-  mf_cache_mask_decl_l
-    = mf_mark (create_tmp_var (TREE_TYPE (mf_cache_mask_decl),
-                               "__mf_lookup_mask_l"));
-
-  /* Build initialization nodes for them.  */
-  t = build (MODIFY_EXPR, TREE_TYPE (mf_cache_shift_decl_l),
-             mf_cache_shift_decl_l, mf_cache_shift_decl);
-  annotate_with_locus (t, DECL_SOURCE_LOCATION (current_function_decl));
-  gimplify_stmt (&t);
-  tsi_link_before (&i, t, TSI_NEW_STMT);
-
-  t = build (MODIFY_EXPR, TREE_TYPE (mf_cache_mask_decl_l),
-             mf_cache_mask_decl_l, mf_cache_mask_decl);
-  annotate_with_locus (t, DECL_SOURCE_LOCATION (current_function_decl));
-  gimplify_stmt (&t);
-  tsi_link_before (&i, t, TSI_NEW_STMT);
-}
-
-
-static void
-mf_decl_clear_locals (void)
-{
-  /* Unset local shadows.  */
-  mf_cache_shift_decl_l = NULL_TREE;
-  mf_cache_mask_decl_l = NULL_TREE;
-}
-
 /* Create a properly typed STRING_CST node that describes the given
    declaration.  It will be used as an argument for __mf_register().
    Try to construct a helpful string, including file/function/variable
@@ -432,35 +231,229 @@ mf_file_function_line_tree (location_t *locus)
 }
 
 
+/* global tree nodes */
+
+/* Global tree objects for global variables and functions exported by
+   mudflap runtime library.  mf_init_extern_trees must be called
+   before using these.  */
+
+/* uintptr_t (usually "unsigned long") */
+static GTY (()) tree mf_uintptr_type;
+
+/* struct __mf_cache { uintptr_t low; uintptr_t high; }; */
+static GTY (()) tree mf_cache_struct_type;
+
+/* struct __mf_cache * const */
+static GTY (()) tree mf_cache_structptr_type;
+
+/* extern struct __mf_cache __mf_lookup_cache []; */
+static GTY (()) tree mf_cache_array_decl;
+
+/* extern const unsigned char __mf_lc_shift; */
+static GTY (()) tree mf_cache_shift_decl;
+
+/* extern const uintptr_t __mf_lc_mask; */
+static GTY (()) tree mf_cache_mask_decl;
+
+/* Their function-scope local shadows, used in single-threaded mode only. */
+
+/* auto const unsigned char __mf_lc_shift_l; */
+static GTY (()) tree mf_cache_shift_decl_l;
+
+/* auto const uintptr_t __mf_lc_mask_l; */
+static GTY (()) tree mf_cache_mask_decl_l;
+
+/* extern void __mf_check (void *ptr, size_t sz, int type, const char *); */
+static GTY (()) tree mf_check_fndecl;
+
+/* extern void __mf_register (void *ptr, size_t sz, int type, const char *); */
+static GTY (()) tree mf_register_fndecl;
+
+/* extern void __mf_unregister (void *ptr, size_t sz); */
+static GTY (()) tree mf_unregister_fndecl;
+
+
+/* Initialize the global tree nodes that correspond to mf-runtime.h
+   declarations.  */
+static void
+mf_init_extern_trees (void)
+{
+  static bool done = false;
+
+  if (done)
+    return;
+  done = true;
+
+  mf_uintptr_type = TREE_TYPE (mflang_lookup_decl ("uintptr_t"));
+  mf_cache_array_decl = mf_mark (mflang_lookup_decl ("__mf_lookup_cache"));
+  mf_cache_struct_type = TREE_TYPE (TREE_TYPE (mf_cache_array_decl));
+  mf_cache_structptr_type = build_pointer_type (mf_cache_struct_type);
+  mf_cache_shift_decl = mf_mark (mflang_lookup_decl ("__mf_lc_shift"));
+  mf_cache_mask_decl = mf_mark (mflang_lookup_decl ("__mf_lc_mask"));
+  mf_check_fndecl = mflang_lookup_decl ("__mf_check");
+  mf_register_fndecl = mflang_lookup_decl ("__mf_register");
+  mf_unregister_fndecl = mflang_lookup_decl ("__mf_unregister");
+}
+
+
+/* ------------------------------------------------------------------------ */
+/* Memory reference transforms. Perform the mudflap indirection-related
+   tree transforms on the current function.
+
+   This is the second part of the mudflap instrumentation.  It works on
+   low-level GIMPLE using the CFG, because we want to run this pass after
+   tree optimizations have been performed, but we have to preserve the CFG
+   for expansion from trees to RTL.  */
+
+static void
+execute_mudflap_function_ops (void)
+{
+  if (mf_marked_p (current_function_decl))
+    return;
+
+  push_gimplify_context ();
+
+  /* In multithreaded mode, don't cache the lookup cache parameters.  */
+  if (! flag_mudflap_threads)
+    mf_decl_cache_locals ();
+
+  mf_xform_derefs ();
+
+  if (! flag_mudflap_threads)
+    mf_decl_clear_locals ();
+
+  pop_gimplify_context (NULL);
+}
+
+/* Create and initialize local shadow variables for the lookup cache
+   globals.  Put their decls in the *_l globals for use by
+   mf_build_check_statement_for. */
+
+static void
+mf_decl_cache_locals (void)
+{
+  tree t, shift_init_stmts, mask_init_stmts;
+  tree_stmt_iterator tsi;
+
+  /* Build the cache vars.  */
+  mf_cache_shift_decl_l
+    = mf_mark (create_tmp_var (TREE_TYPE (mf_cache_shift_decl),
+                               "__mf_lookup_shift_l"));
+
+  mf_cache_mask_decl_l
+    = mf_mark (create_tmp_var (TREE_TYPE (mf_cache_mask_decl),
+                               "__mf_lookup_mask_l"));
+
+  /* Build initialization nodes for the cache vars.  We just load the
+     globals into the cache variables.  */
+  t = build (MODIFY_EXPR, TREE_TYPE (mf_cache_shift_decl_l),
+             mf_cache_shift_decl_l, mf_cache_shift_decl);
+  annotate_with_locus (t, DECL_SOURCE_LOCATION (current_function_decl));
+  gimplify_to_stmt_list (&t);
+  shift_init_stmts = t;
+
+  t = build (MODIFY_EXPR, TREE_TYPE (mf_cache_mask_decl_l),
+             mf_cache_mask_decl_l, mf_cache_mask_decl);
+  annotate_with_locus (t, DECL_SOURCE_LOCATION (current_function_decl));
+  gimplify_to_stmt_list (&t);
+  mask_init_stmts = t;
+
+  /* Anticipating multiple entry points, we insert the cache vars
+     initializers in each successor of the ENTRY_BLOCK_PTR.  */
+  for (tsi = tsi_start (shift_init_stmts);
+       ! tsi_end_p (tsi);
+       tsi_next (&tsi))
+    insert_edge_copies (tsi_stmt (tsi), ENTRY_BLOCK_PTR);
+
+  for (tsi = tsi_start (mask_init_stmts);
+       ! tsi_end_p (tsi);
+       tsi_next (&tsi))
+    insert_edge_copies (tsi_stmt (tsi), ENTRY_BLOCK_PTR);
+  bsi_commit_edge_inserts (NULL);
+}
+
+
 static void
-mf_build_check_statement_for (tree addr, tree size, tree_stmt_iterator *iter,
+mf_decl_clear_locals (void)
+{
+  /* Unset local shadows. */
+  mf_cache_shift_decl_l = NULL_TREE;
+  mf_cache_mask_decl_l = NULL_TREE;
+}
+
+static void
+mf_build_check_statement_for (tree addr, tree size,
+			      block_stmt_iterator *instr_bsi,
                               location_t *locus, tree dirflag)
 {
+  tree_stmt_iterator head, tsi;
   tree ptrtype = TREE_TYPE (addr);
-  tree stmt, cond, t, u, v;
+  block_stmt_iterator bsi;
+  basic_block cond_bb, then_bb, join_bb;
+  edge e;
+  tree cond, t, u, v, l1, l2;
   tree mf_value;
   tree mf_base;
   tree mf_elem;
 
+  /* We first need to split the current basic block, and start altering
+     the CFG.  This allows us to insert the statements we're about to
+     construct into the right basic blocks.  The label l1 is the label
+     of the block for the THEN clause of the conditional jump we're
+     about to construct, and l2 is the ELSE clause, which is just the
+     continuation of the old statement stream.  */
+  l1 = create_artificial_label ();
+  l2 = create_artificial_label ();
+  cond_bb = bb_for_stmt (bsi_stmt (*instr_bsi));
+  bsi = *instr_bsi;
+  bsi_prev (&bsi);
+  if (! bsi_end_p (bsi))
+    {
+      e = split_block (cond_bb, bsi_stmt (bsi));
+      cond_bb = e->src;
+      join_bb = e->dest;
+    }
+  else
+    {
+      join_bb = cond_bb;
+      cond_bb = create_empty_bb (join_bb->prev_bb);
+      e = make_edge (cond_bb, join_bb, 0);
+    }
+  e->flags = EDGE_FALSE_VALUE;
+  then_bb = create_empty_bb (cond_bb);
+  make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
+  make_edge (then_bb, join_bb, EDGE_FALLTHRU);
+
+  /* We expect that the conditional jump we will construct will not
+     be taken very often as it basically is an exception condition.  */
+  predict_edge_def (then_bb->pred, PRED_MUDFLAP, NOT_TAKEN);
+
+  /* Update dominance info.  Note that bb_join's data was
+     updated by split_block.  */
+  if (dom_computed[CDI_DOMINATORS] >= DOM_CONS_OK)
+    {
+      set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
+      set_immediate_dominator (CDI_DOMINATORS, join_bb, cond_bb);
+    }
+
   /* Build our local variables.  */
   mf_value = create_tmp_var (ptrtype, "__mf_value");
   mf_elem = create_tmp_var (mf_cache_structptr_type, "__mf_elem");
   mf_base = create_tmp_var (mf_uintptr_type, "__mf_base");
 
   /* Build: __mf_value = <address expression>.  */
-  stmt = build (MODIFY_EXPR, void_type_node, mf_value, addr);
-  if (locus != NULL) 
-    annotate_with_locus (stmt, *locus);
-  gimplify_stmt (&stmt);
-  tsi_link_before (iter, stmt, TSI_SAME_STMT);
+  t = build (MODIFY_EXPR, void_type_node, mf_value, unshare_expr (addr));
+  SET_EXPR_LOCUS (t, locus);
+  gimplify_to_stmt_list (&t);
+  head = tsi_start (t);
+  tsi = tsi_last (t);
 
   /* Build: __mf_base = (uintptr_t)__mf_value.  */
-  stmt = build (MODIFY_EXPR, void_type_node, mf_base,
-                build1 (NOP_EXPR, mf_uintptr_type, mf_value));
-  if (locus != NULL) 
-    annotate_with_locus (stmt, *locus);
-  gimplify_stmt (&stmt);
-  tsi_link_before (iter, stmt, TSI_SAME_STMT);
+  t = build (MODIFY_EXPR, void_type_node, mf_base,
+             build1 (NOP_EXPR, mf_uintptr_type, mf_value));
+  SET_EXPR_LOCUS (t, locus);
+  gimplify_to_stmt_list (&t);
+  tsi_link_after (&tsi, t, TSI_CONTINUE_LINKING);
 
   /* Build: __mf_elem = &__mf_lookup_cache [(__mf_base >> __mf_shift)
                                             & __mf_mask].  */
@@ -472,81 +465,120 @@ mf_build_check_statement_for (tree addr, tree size, tree_stmt_iterator *iter,
              TREE_TYPE (TREE_TYPE (mf_cache_array_decl)),
              mf_cache_array_decl, t);
   t = build1 (ADDR_EXPR, mf_cache_structptr_type, t);
-  stmt = build (MODIFY_EXPR, void_type_node, mf_elem, t);
-  if (locus != NULL) 
-    annotate_with_locus (stmt, *locus);
-  gimplify_stmt (&stmt);
-  tsi_link_before (iter, stmt, TSI_SAME_STMT);
+  t = build (MODIFY_EXPR, void_type_node, mf_elem, t);
+  SET_EXPR_LOCUS (t, locus);
+  gimplify_to_stmt_list (&t);
+  tsi_link_after (&tsi, t, TSI_CONTINUE_LINKING);
 
   /* Quick validity check.
-     if (__builtin_expect ((__mf_elem->low > __mf_base)
-                           | (__mf_elem_high < __mf_base + sizeof(T) - 1),
-                           0))
-        {
-          __mf_check ();
-          ... and only if single-threaded:
-          __mf_lookup_shift_1 = f...;
-          __mf_lookup_mask_l = ...;
-        }
-     */
 
-  /* __mf_elem->low  */
+     if (__mf_elem->low > __mf_base
+	 || (__mf_elem_high < __mf_base + sizeof(T) - 1))
+	{
+	  __mf_check ();
+	  ... and only if single-threaded:
+	  __mf_lookup_shift_1 = f...;
+	  __mf_lookup_mask_l = ...;
+	}
+
+     It is expected that this body of code is rarely executed so we mark
+     the edge to the THEN clause of the conditional jump as unlikely.  */
+
+  /* Construct t <-- '__mf_elem->low  > __mf_base'.  */
   t = build (COMPONENT_REF, mf_uintptr_type,
              build1 (INDIRECT_REF, mf_cache_struct_type, mf_elem),
              TYPE_FIELDS (mf_cache_struct_type));
+  t = build (GT_EXPR, boolean_type_node, t, mf_base);
+
+  /* Construct '__mf_elem->high < __mf_base + sizeof(T) - 1'.
+
+     First build:
+	1) u <--  '__mf_elem->high'
+	2) v <--  '__mf_base + sizeof (T) - 1'.
+
+     Then build 'u <-- (u < v).  */
+
 
-  /* __mf_elem->high  */
   u = build (COMPONENT_REF, mf_uintptr_type,
              build1 (INDIRECT_REF, mf_cache_struct_type, mf_elem),
              TREE_CHAIN (TYPE_FIELDS (mf_cache_struct_type)));
 
-  /* __mf_base + sizeof (T) - 1 */
-  v = size_binop (MINUS_EXPR, size, size_one_node);
-  v = convert (mf_uintptr_type, v);
+  v = convert (mf_uintptr_type,
+	       size_binop (MINUS_EXPR, size, size_one_node));
   v = fold (build (PLUS_EXPR, mf_uintptr_type, mf_base, v));
 
-  t = build (TRUTH_OR_EXPR, boolean_type_node,
-             build (GT_EXPR, boolean_type_node, t, mf_base),
-             build (LT_EXPR, boolean_type_node, u, v));
+  u = build (LT_EXPR, boolean_type_node, u, v);
+
+  /* Build the composed conditional: t <-- 't || u'.  Then store the
+     result of the evaluation of 't' in a temporary variable which we
+     can use as the condition for the conditional jump.  */
+  t = build (TRUTH_OR_EXPR, boolean_type_node, t, u);
+  cond = create_tmp_var (boolean_type_node, "__mf_unlikely_cond");
+  t = build (MODIFY_EXPR, boolean_type_node, cond, t);
+  gimplify_to_stmt_list (&t);
+  tsi_link_after (&tsi, t, TSI_CONTINUE_LINKING);
+
+  /* Build the conditional jump.  'cond' is just a temporary so we can
+     simply build a void COND_EXPR.  We do need labels in both arms though.  */
+  t = build (COND_EXPR, void_type_node, cond,
+	     build (GOTO_EXPR, void_type_node, tree_block_label (then_bb)),
+	     build (GOTO_EXPR, void_type_node, tree_block_label (join_bb)));
+  SET_EXPR_LOCUS (t, locus);
+  tsi_link_after (&tsi, t, TSI_CONTINUE_LINKING);
 
-  /* Mark condition as UNLIKELY using __builtin_expect.  */
-  u = tree_cons (NULL_TREE, integer_zero_node, NULL_TREE);
-  u = tree_cons (NULL_TREE, convert (long_integer_type_node, t), u);
-  cond = build_function_call_expr (built_in_decls[BUILT_IN_EXPECT], u);
+  /* At this point, after so much hard work, we have only constructed
+     the conditional jump,
 
-  /* Build up the body of the cache-miss handling:
-     __mf_check(); refresh *_l vars.  */
+     if (__mf_elem->low > __mf_base
+	 || (__mf_elem_high < __mf_base + sizeof(T) - 1))
 
-  stmt = NULL;
+     The lowered GIMPLE tree representing this code is in the statement
+     list starting at 'head'.
+
+     We can insert this now in the current basic block, ie. the one that
+     the statement we're instrumenting was originally in.  */
+  bsi = bsi_last (cond_bb);
+  for (tsi = head; ! tsi_end_p (tsi); tsi_next (&tsi))
+    bsi_insert_after (&bsi, tsi_stmt (tsi), BSI_CONTINUE_LINKING);
+
+  /*  Now build up the body of the cache-miss handling:
+
+     __mf_check();
+     refresh *_l vars.
+
+     This is the body of the conditional.  */
   
   u = tree_cons (NULL_TREE, mf_file_function_line_tree (locus), NULL_TREE);
   u = tree_cons (NULL_TREE, dirflag, u);
   u = tree_cons (NULL_TREE, size, u);
   u = tree_cons (NULL_TREE, mf_value, u);
   t = build_function_call_expr (mf_check_fndecl, u);
-  append_to_statement_list (t, &stmt);
+  gimplify_to_stmt_list (&t);
+  head = tsi_start (t);
+  tsi = tsi_last (t);
 
   if (! flag_mudflap_threads)
     {
       t = build (MODIFY_EXPR, void_type_node,
                  mf_cache_shift_decl_l, mf_cache_shift_decl);
-      append_to_statement_list (t, &stmt);
+      tsi_link_after (&tsi, t, TSI_CONTINUE_LINKING);
 
       t = build (MODIFY_EXPR, void_type_node,
                  mf_cache_mask_decl_l, mf_cache_mask_decl);
-      append_to_statement_list (t, &stmt);
+      tsi_link_after (&tsi, t, TSI_CONTINUE_LINKING);
     }
 
-  stmt = build (COND_EXPR, void_type_node, cond, stmt, build_empty_stmt ());
-  if (locus != NULL) 
-    annotate_with_locus (stmt, *locus);
-  gimplify_to_stmt_list (&stmt);
-  lower_stmt_body (stmt, NULL);
-  tsi_link_before (iter, stmt, TSI_SAME_STMT);
+  /* Insert the check code in the THEN block.  */
+  bsi = bsi_start (then_bb);
+  for (tsi = head; ! tsi_end_p (tsi); tsi_next (&tsi))
+    bsi_insert_after (&bsi, tsi_stmt (tsi), BSI_CONTINUE_LINKING);
+
+  *instr_bsi = bsi_start (join_bb);
+  bsi_next (instr_bsi);
 }
 
 static void
-mf_xform_derefs_1 (tree_stmt_iterator *iter, tree *tp,
+mf_xform_derefs_1 (block_stmt_iterator *iter, tree *tp,
                    location_t *locus, tree dirflag)
 {
   tree type, ptr_type, addr, size, t;
@@ -574,15 +606,14 @@ mf_xform_derefs_1 (tree_stmt_iterator *iter, tree *tp,
           {
             tree dom = TYPE_DOMAIN (TREE_TYPE (op0));
 
-            /* Test for index in range.  Break if not.  */
-            if (!dom)
-              break;
-            if (!TYPE_MIN_VALUE (dom) || !really_constant_p (TYPE_MIN_VALUE (dom)))
-              break;
-            if (!TYPE_MAX_VALUE (dom) || !really_constant_p (TYPE_MAX_VALUE (dom)))
-              break;
-            if (tree_int_cst_lt (op1, TYPE_MIN_VALUE (dom))
-                || tree_int_cst_lt (TYPE_MAX_VALUE (dom), op1))
+	    /* Test for index in range.  Break if not.  */
+	    if (!dom
+		|| (! TYPE_MIN_VALUE (dom)
+		    || ! really_constant_p (TYPE_MIN_VALUE (dom)))
+		|| (! TYPE_MAX_VALUE (dom)
+		    || ! really_constant_p (TYPE_MAX_VALUE (dom)))
+		|| (tree_int_cst_lt (op1, TYPE_MIN_VALUE (dom))
+		    || tree_int_cst_lt (TYPE_MAX_VALUE (dom), op1)))
               break;
 
             /* If we're looking at a non-external VAR_DECL, then the 
@@ -679,45 +710,74 @@ mf_xform_derefs_1 (tree_stmt_iterator *iter, tree *tp,
 }
 
 static void
-mf_xform_derefs (tree fnbody)
+mf_xform_derefs (void)
 {
-  tree_stmt_iterator i = tsi_start (fnbody);
+  basic_block bb, next;
+  block_stmt_iterator i;
+  int saved_last_basic_block = last_basic_block;
 
-  for (i = tsi_start (fnbody); !tsi_end_p (i); tsi_next (&i))
+  bb = ENTRY_BLOCK_PTR ->next_bb;
+  do
     {
-      tree s = tsi_stmt (i);
-
-      /* Only a few GIMPLE statements can reference memory.  */
-      switch (TREE_CODE (s))
-        {
-        case MODIFY_EXPR:  /*  This includes INIT_EXPR after gimplification.  */
-          mf_xform_derefs_1 (&i, &TREE_OPERAND (s, 0), EXPR_LOCUS (s),
-                             integer_one_node);
-          mf_xform_derefs_1 (&i, &TREE_OPERAND (s, 1), EXPR_LOCUS (s),
-                             integer_zero_node);
-          break;
-
-        case RETURN_EXPR:
-          if (TREE_OPERAND (s, 0) != NULL_TREE)
-            {
-              if (TREE_CODE (TREE_OPERAND (s, 0)) == MODIFY_EXPR)
-                mf_xform_derefs_1 (&i, &TREE_OPERAND (TREE_OPERAND (s, 0), 1), EXPR_LOCUS (s),
-                                   integer_zero_node);
-              else
-                mf_xform_derefs_1 (&i, &TREE_OPERAND (s, 0), EXPR_LOCUS (s),
-                                   integer_zero_node);
-            }
-          break;
-
-        default:
-          ;
-        }
+      next = bb->next_bb;
+      for (i = bsi_start (bb); !bsi_end_p (i); bsi_next (&i))
+	{
+	  tree s = bsi_stmt (i);
+
+	  /* Only a few GIMPLE statements can reference memory.  */
+	  switch (TREE_CODE (s))
+	    {
+	    case MODIFY_EXPR:
+	      mf_xform_derefs_1 (&i, &TREE_OPERAND (s, 0), EXPR_LOCUS (s),
+				 integer_one_node);
+	      mf_xform_derefs_1 (&i, &TREE_OPERAND (s, 1), EXPR_LOCUS (s),
+				 integer_zero_node);
+	      break;
+
+	    case RETURN_EXPR:
+	      if (TREE_OPERAND (s, 0) != NULL_TREE)
+		{
+		  if (TREE_CODE (TREE_OPERAND (s, 0)) == MODIFY_EXPR)
+		    mf_xform_derefs_1 (&i, &TREE_OPERAND (TREE_OPERAND (s, 0), 1),
+				       EXPR_LOCUS (s), integer_zero_node);
+		  else
+		    mf_xform_derefs_1 (&i, &TREE_OPERAND (s, 0), EXPR_LOCUS (s),
+				       integer_zero_node);
+		}
+	      break;
+
+	    default:
+	      ;
+	    }
+	}
+      bb = next;
     }
+  while (bb && bb->index <= saved_last_basic_block);
 }
 
 /* ------------------------------------------------------------------------ */
-/* ADDR_EXPR transform */
+/* ADDR_EXPR transforms.  Perform the declaration-related mudflap tree
+   transforms on the current function.
+
+   This is the first part of the mudflap instrumentation.  It works on
+   high-level GIMPLE because after lowering, all variables are moved out
+   of their BIND_EXPR binding context, and we lose liveness information
+   for the declarations we wish to instrument.  */
+
+static void
+execute_mudflap_function_decls (void)
+{
+  if (mf_marked_p (current_function_decl))
+    return;
 
+  push_gimplify_context ();
+
+  mf_init_extern_trees ();
+  mf_xform_decls (DECL_SAVED_TREE (current_function_decl),
+                  DECL_ARGUMENTS (current_function_decl));
+
+  pop_gimplify_context (NULL);
+}
 
 /* This struct is passed between mf_xform_decls to store state needed
    during the traversal searching for objects that have their
@@ -936,13 +996,41 @@ mf_xform_decls (tree fnbody, tree fnparams)
 
 
 /* ------------------------------------------------------------------------ */
+/* Externally visible mudflap functions.  */
+
+
+/* Mark and return the given tree node to prevent further mudflap
+   transforms.  */
+static GTY ((param_is (union tree_node))) htab_t marked_trees = NULL;
+
+tree
+mf_mark (tree t)
+{
+  void **slot;
+
+  if (marked_trees == NULL)
+    marked_trees = htab_create_ggc (31, htab_hash_pointer, htab_eq_pointer, NULL);
+
+  slot = htab_find_slot (marked_trees, t, INSERT);
+  *slot = t;
+  return t;
+}
+
+int
+mf_marked_p (tree t)
+{
+  void *entry;
 
+  if (marked_trees == NULL)
+    return 0;
+
+  entry = htab_find (marked_trees, t);
+  return (entry != NULL);
+}
 
 /* Remember given node as a static of some kind: global data,
    function-scope static, or an anonymous constant.  Its assembler
-   label is given.
-*/
-
+   label is given.  */
 
 /* A list of globals whose incomplete declarations we encountered.
    Instead of emitting the __mf_register call for them here, it's
@@ -1066,7 +1154,6 @@ mudflap_enqueue_constant (tree obj)
 }
 
 
-
 /* Emit any file-wide instrumentation.  */
 void
 mudflap_finish_file (void)
@@ -1093,5 +1180,43 @@ mudflap_finish_file (void)
 }
 
 
+static bool
+gate_mudflap (void)
+{
+  return flag_mudflap != 0;
+}
+
+struct tree_opt_pass pass_mudflap_1 = 
+{
+  "mudflap1",                           /* name */
+  gate_mudflap,                         /* gate */
+  execute_mudflap_function_decls,       /* execute */
+  NULL,                                 /* sub */
+  NULL,                                 /* next */
+  0,                                    /* static_pass_number */
+  0,                                    /* tv_id */
+  PROP_gimple_any,                      /* properties_required */
+  0,                                    /* properties_provided */
+  0,                                    /* properties_destroyed */
+  0,                                    /* todo_flags_start */
+  TODO_dump_func                        /* todo_flags_finish */
+};
+
+struct tree_opt_pass pass_mudflap_2 = 
+{
+  "mudflap2",                           /* name */
+  gate_mudflap,                         /* gate */
+  execute_mudflap_function_ops,         /* execute */
+  NULL,                                 /* sub */
+  NULL,                                 /* next */
+  0,                                    /* static_pass_number */
+  0,                                    /* tv_id */
+  PROP_gimple_leh,                      /* properties_required */
+  0,                                    /* properties_provided */
+  0,                                    /* properties_destroyed */
+  0,                                    /* todo_flags_start */
+  TODO_verify_flow | TODO_verify_stmts
+  | TODO_dump_func                      /* todo_flags_finish */
+};
 
 #include "gt-tree-mudflap.h"