2012-07-13 Bill Schmidt <wschmidt@linux.ibm.com>

	* doc/tm.texi: Regenerate.
	* doc/tm.texi.in (TARGET_VECTORIZE_INIT_COST): New hook.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	* targhooks.c (default_init_cost): New function.
	(default_add_stmt_cost): Likewise.
	(default_finish_cost): Likewise.
	(default_destroy_cost_data): Likewise.
	* targhooks.h (default_init_cost): New decl.
	(default_add_stmt_cost): Likewise.
	(default_finish_cost): Likewise.
	(default_destroy_cost_data): Likewise.
	* target.def (init_cost): New DEFHOOK.
	(add_stmt_cost): Likewise.
	(finish_cost): Likewise.
	(destroy_cost_data): Likewise.
	* target.h (struct _stmt_vec_info): New extern decl.
	(stmt_vectype): Likewise.
	(stmt_in_inner_loop_p): Likewise.
	* tree-vectorizer.h (stmt_info_for_cost): New struct/typedef.
	(stmt_vector_for_cost): New VEC/typedef.
	(add_stmt_info_to_vec): New function.
	(struct _slp_tree): Remove cost.inside_of_loop field.
	(struct _slp_instance): Remove cost.inside_of_loop field; add
	stmt_cost_vec field.
	(SLP_INSTANCE_INSIDE_OF_LOOP_COST): Remove macro.
	(SLP_INSTANCE_STMT_COST_VEC): New accessor macro.
	(SLP_TREE_INSIDE_OF_LOOP_COST): Remove macro.
	(struct _vect_peel_extended_info): Add stmt_cost_vec field.
	(struct _loop_vec_info): Add target_cost_data field.
	(LOOP_VINFO_TARGET_COST_DATA): New accessor macro.
	(struct _bb_vec_info): Add target_cost_data field.
	(BB_VINFO_TARGET_COST_DATA): New accessor macro.
	(struct _stmt_vec_info): Remove cost.inside_of_loop field.
	(STMT_VINFO_INSIDE_OF_LOOP_COST): Remove macro.
	(stmt_vinfo_set_inside_of_loop_cost): Remove function.
	(init_cost): New function.
	(add_stmt_cost): Likewise.
	(finish_cost): Likewise.
	(destroy_cost_data): Likewise.
	(vect_model_simple_cost): Change parameter list.
	(vect_model_store_cost): Likewise.
	(vect_model_load_cost): Likewise.
	(record_stmt_cost): New extern decl.
	(vect_get_load_cost): Change parameter list.
	(vect_get_store_cost): Likewise.
	* tree-vect-loop.c (new_loop_vec_info): Call init_cost.
	(destroy_loop_vec_info): Call destroy_cost_data.
	(vect_estimate_min_profitable_iters): Remove old calculation of
	inside costs; call finish_cost instead.
	(vect_model_reduction_cost): Call add_stmt_cost instead of old
	inside-costs calculation.
	(vect_model_induction_cost): Likewise.
	* tree-vect-data-refs.c (vect_get_data_access_cost): Change to
	return a stmt_vector_for_cost; modify calls to vect_get_load_cost
	and vect_get_store_cost to obtain the value to return.
	(vect_peeling_hash_get_lowest_cost): Obtain a stmt_cost_vec from
	vect_get_data_access_cost and store it in the minimum peeling
	structure.
	(vect_peeling_hash_choose_best_peeling): Change the parameter list
	to add a (stmt_vector_for_cost *) output parameter, and set its value.
	(vect_enhance_data_refs_alignment): Ignore the new return value from
	calls to vect_get_data_access_cost; obtain stmt_cost_vec from
	vect_peeling_hash_choose_best_peeling and pass its contents to the
	target cost model.
	* tree-vect-stmts.c (stmt_vectype): New function.
	(stmt_in_inner_loop_p): Likewise.
	(record_stmt_cost): Likewise.
	(vect_model_simple_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation; don't call
	stmt_vinfo_set_inside_of_loop_cost.
	(vect_model_promotion_demotion_cost): Call add_stmt_cost instead of
	old calculation; don't call stmt_vinfo_set_inside_of_loop_cost.
	(vect_model_store_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation; add stmt_cost_vec
	parameter to vect_get_store_cost call; don't call
	stmt_vinfo_set_inside_of_loop_cost.
	(vect_get_store_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation.
	(vect_model_load_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation; add stmt_cost_vec
	parameter to vect_get_load_cost call; don't call
	stmt_vinfo_set_inside_of_loop_cost.
	(vect_get_load_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation.
	(vectorizable_call): Add NULL parameter to vect_model_simple_cost call.
	(vectorizable_conversion): Likewise.
	(vectorizable_assignment): Likewise.
	(vectorizable_shift): Likewise.
	(vectorizable_operation): Likewise.
	(vectorizable_store): Add NULL parameter to vect_model_store_cost call.
	(vectorizable_load): Add NULL parameter to vect_model_load_cost call.
	(new_stmt_vec_info): Don't set STMT_VINFO_INSIDE_OF_LOOP_COST.
	* config/spu/spu.c (TARGET_VECTORIZE_INIT_COST): New macro def.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	(spu_init_cost): New function.
	(spu_add_stmt_cost): Likewise.
	(spu_finish_cost): Likewise.
	(spu_destroy_cost_data): Likewise.
	* config/i386/i386.c (ix86_init_cost): New function.
	(ix86_add_stmt_cost): Likewise.
	(ix86_finish_cost): Likewise.
	(ix86_destroy_cost_data): Likewise.
	(TARGET_VECTORIZE_INIT_COST): New macro def.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	* config/rs6000/rs6000.c (TARGET_VECTORIZE_INIT_COST): New macro def.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	(rs6000_init_cost): New function.
	(rs6000_add_stmt_cost): Likewise.
	(rs6000_finish_cost): Likewise.
	(rs6000_destroy_cost_data): Likewise.
	* tree-vect-slp.c (vect_free_slp_instance): Free stmt_cost_vec.
	(vect_create_new_slp_node): Don't set SLP_TREE_INSIDE_OF_LOOP_COST.
	(vect_get_and_check_slp_defs): Add stmt_cost_vec parameter; add
	stmt_cost_vec parameter to vect_model_store_cost and
	vect_model_simple_cost calls.
	(vect_build_slp_tree): Remove inside_cost parameter; add stmt_cost_vec
	parameter; add stmt_cost_vec parameter to vect_get_and_check_slp_defs,
	vect_model_load_cost, and recursive vect_build_slp_tree calls; prevent
	calculating cost more than once for loads; remove inside_cost
	parameter from recursive vect_build_slp_tree calls; call
	record_stmt_cost instead of old calculation.
	(vect_analyze_slp_instance): Allocate stmt_cost_vec and save it with
	the instance; free it on premature exit; remove inside_cost parameter
	from vect_build_slp_tree call; add stmt_cost_vec parameter to
	vect_build_slp_tree call; don't set SLP_INSTANCE_INSIDE_OF_LOOP_COST.
	(new_bb_vec_info): Call init_cost.
	(destroy_bb_vec_info): Call destroy_cost_data.
	(vect_bb_vectorization_profitable_p): Call add_stmt_cost for each
	statement recorded with an SLP instance; call finish_cost instead of
	the old calculation.
	(vect_update_slp_costs_according_to_vf): Record statement costs from
	SLP instances, multiplying by the appropriate number of copies; don't
	update SLP_INSTANCE_INSIDE_OF_LOOP_COST.


git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@189460 138bc75d-0d04-0410-961f-82ee72b054a4

1 files changed, 68 insertions, 32 deletions

diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 5d11552a1a0..478343e041a 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -94,6 +94,7 @@ vect_free_slp_instance (slp_instance instance)
   vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
   VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (instance));
   VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (instance));
+  VEC_free (stmt_info_for_cost, heap, SLP_INSTANCE_STMT_COST_VEC (instance));
 }
 
 
@@ -122,7 +123,6 @@ vect_create_new_slp_node (VEC (gimple, heap) *scalar_stmts)
   SLP_TREE_VEC_STMTS (node) = NULL;
   SLP_TREE_CHILDREN (node) = VEC_alloc (slp_void_p, heap, nops);
   SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0;
-  SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0;
 
   return node;
 }
@@ -179,7 +179,8 @@ static bool
 vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
                              slp_tree slp_node, gimple stmt,
 			     int ncopies_for_cost, bool first,
-                             VEC (slp_oprnd_info, heap) **oprnds_info)
+                             VEC (slp_oprnd_info, heap) **oprnds_info,
+			     stmt_vector_for_cost *stmt_cost_vec)
 {
   tree oprnd;
   unsigned int i, number_of_oprnds;
@@ -320,7 +321,7 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 	      if (REFERENCE_CLASS_P (lhs))
 		/* Store.  */
                 vect_model_store_cost (stmt_info, ncopies_for_cost, false,
-                                        dt, slp_node);
+				       dt, slp_node, stmt_cost_vec);
 	      else
 		{
 		  enum vect_def_type dts[2];
@@ -329,7 +330,7 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 		  /* Not memory operation (we don't call this function for
 		     loads).  */
 		  vect_model_simple_cost (stmt_info, ncopies_for_cost, dts,
-					  slp_node);
+					  slp_node, stmt_cost_vec);
 		}
 	    }
 	}
@@ -446,12 +447,12 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 
 static bool
 vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
-                     slp_tree *node, unsigned int group_size,
-                     int *inside_cost, int *outside_cost,
+                     slp_tree *node, unsigned int group_size, int *outside_cost,
                      int ncopies_for_cost, unsigned int *max_nunits,
                      VEC (int, heap) **load_permutation,
                      VEC (slp_tree, heap) **loads,
-                     unsigned int vectorization_factor, bool *loads_permuted)
+                     unsigned int vectorization_factor, bool *loads_permuted,
+		     stmt_vector_for_cost *stmt_cost_vec)
 {
   unsigned int i;
   VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (*node);
@@ -470,7 +471,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
   HOST_WIDE_INT dummy;
   bool permutation = false;
   unsigned int load_place;
-  gimple first_load, prev_first_load = NULL;
+  gimple first_load = NULL, prev_first_load = NULL, old_first_load = NULL;
   VEC (slp_oprnd_info, heap) *oprnds_info;
   unsigned int nops;
   slp_oprnd_info oprnd_info;
@@ -711,7 +712,8 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 	      /* Store.  */
 	      if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node,
 						stmt, ncopies_for_cost,
-						(i == 0), &oprnds_info))
+						(i == 0), &oprnds_info,
+						stmt_cost_vec))
 		{
 	  	  vect_free_oprnd_info (&oprnds_info);
  		  return false;
@@ -754,6 +756,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
                   return false;
                 }
 
+	      old_first_load = first_load;
               first_load = GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt));
               if (prev_first_load)
                 {
@@ -778,7 +781,9 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
               else
                 prev_first_load = first_load;
 
-              if (first_load == stmt)
+	      /* In some cases a group of loads is just the same load
+		 repeated N times.  Only analyze its cost once.  */
+              if (first_load == stmt && old_first_load != first_load)
                 {
                   first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
                   if (vect_supportable_dr_alignment (first_dr, false)
@@ -797,7 +802,8 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 
                   /* Analyze costs (for the first stmt in the group).  */
                   vect_model_load_cost (vinfo_for_stmt (stmt),
-                                        ncopies_for_cost, false, *node);
+                                        ncopies_for_cost, false, *node,
+					stmt_cost_vec);
                 }
 
               /* Store the place of this load in the interleaving chain.  In
@@ -871,7 +877,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 	  /* Find the def-stmts.  */
 	  if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt,
 					    ncopies_for_cost, (i == 0),
-					    &oprnds_info))
+					    &oprnds_info, stmt_cost_vec))
 	    {
 	      vect_free_oprnd_info (&oprnds_info);
 	      return false;
@@ -880,7 +886,6 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
     }
 
   /* Add the costs of the node to the overall instance costs.  */
-  *inside_cost += SLP_TREE_INSIDE_OF_LOOP_COST (*node);
   *outside_cost += SLP_TREE_OUTSIDE_OF_LOOP_COST (*node);
 
   /* Grouped loads were reached - stop the recursion.  */
@@ -889,11 +894,10 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
       VEC_safe_push (slp_tree, heap, *loads, *node);
       if (permutation)
         {
-
+	  gimple first_stmt = VEC_index (gimple, stmts, 0);
           *loads_permuted = true;
-          *inside_cost 
-            += targetm.vectorize.builtin_vectorization_cost (vec_perm, NULL, 0) 
-               * group_size;
+	  (void) record_stmt_cost (stmt_cost_vec, group_size, vec_perm, 
+				   vinfo_for_stmt (first_stmt), 0);
         }
       else
         {
@@ -919,9 +923,10 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
       child = vect_create_new_slp_node (oprnd_info->def_stmts);
       if (!child
           || !vect_build_slp_tree (loop_vinfo, bb_vinfo, &child, group_size,
-				inside_cost, outside_cost, ncopies_for_cost,
-				max_nunits, load_permutation, loads,
-				vectorization_factor, loads_permuted))
+				   outside_cost, ncopies_for_cost,
+				   max_nunits, load_permutation, loads,
+				   vectorization_factor, loads_permuted,
+				   stmt_cost_vec))
         {
 	  if (child)
 	    oprnd_info->def_stmts = NULL;
@@ -1459,13 +1464,14 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
   tree vectype, scalar_type = NULL_TREE;
   gimple next;
   unsigned int vectorization_factor = 0;
-  int inside_cost = 0, outside_cost = 0, ncopies_for_cost, i;
+  int outside_cost = 0, ncopies_for_cost, i;
   unsigned int max_nunits = 0;
   VEC (int, heap) *load_permutation;
   VEC (slp_tree, heap) *loads;
   struct data_reference *dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
   bool loads_permuted = false;
   VEC (gimple, heap) *scalar_stmts;
+  stmt_vector_for_cost stmt_cost_vec;
 
   if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)))
     {
@@ -1551,12 +1557,14 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 
   load_permutation = VEC_alloc (int, heap, group_size * group_size);
   loads = VEC_alloc (slp_tree, heap, group_size);
+  stmt_cost_vec = VEC_alloc (stmt_info_for_cost, heap, 10);
 
   /* Build the tree for the SLP instance.  */
   if (vect_build_slp_tree (loop_vinfo, bb_vinfo, &node, group_size,
-                           &inside_cost, &outside_cost, ncopies_for_cost,
+                           &outside_cost, ncopies_for_cost,
 			   &max_nunits, &load_permutation, &loads,
-			   vectorization_factor, &loads_permuted))
+			   vectorization_factor, &loads_permuted,
+			   &stmt_cost_vec))
     {
       /* Calculate the unrolling factor based on the smallest type.  */
       if (max_nunits > nunits)
@@ -1568,6 +1576,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
           if (vect_print_dump_info (REPORT_SLP))
             fprintf (vect_dump, "Build SLP failed: unrolling required in basic"
                                " block SLP");
+	  VEC_free (stmt_info_for_cost, heap, stmt_cost_vec);
           return false;
         }
 
@@ -1577,7 +1586,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
       SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size;
       SLP_INSTANCE_UNROLLING_FACTOR (new_instance) = unrolling_factor;
       SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (new_instance) = outside_cost;
-      SLP_INSTANCE_INSIDE_OF_LOOP_COST (new_instance) = inside_cost;
+      SLP_INSTANCE_STMT_COST_VEC (new_instance) = stmt_cost_vec;
       SLP_INSTANCE_LOADS (new_instance) = loads;
       SLP_INSTANCE_FIRST_LOAD_STMT (new_instance) = NULL;
       SLP_INSTANCE_LOAD_PERMUTATION (new_instance) = load_permutation;
@@ -1617,6 +1626,8 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 
       return true;
     }
+  else
+    VEC_free (stmt_info_for_cost, heap, stmt_cost_vec);
 
   /* Failed to SLP.  */
   /* Free the allocated memory.  */
@@ -1812,6 +1823,7 @@ new_bb_vec_info (basic_block bb)
 
   BB_VINFO_GROUPED_STORES (res) = VEC_alloc (gimple, heap, 10);
   BB_VINFO_SLP_INSTANCES (res) = VEC_alloc (slp_instance, heap, 2);
+  BB_VINFO_TARGET_COST_DATA (res) = init_cost (NULL);
 
   bb->aux = res;
   return res;
@@ -1846,6 +1858,7 @@ destroy_bb_vec_info (bb_vec_info bb_vinfo)
   free_dependence_relations (BB_VINFO_DDRS (bb_vinfo));
   VEC_free (gimple, heap, BB_VINFO_GROUPED_STORES (bb_vinfo));
   VEC_free (slp_instance, heap, BB_VINFO_SLP_INSTANCES (bb_vinfo));
+  destroy_cost_data (BB_VINFO_TARGET_COST_DATA (bb_vinfo));
   free (bb_vinfo);
   bb->aux = NULL;
 }
@@ -1918,8 +1931,8 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo)
 {
   VEC (slp_instance, heap) *slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
   slp_instance instance;
-  int i;
-  unsigned int vec_outside_cost = 0, vec_inside_cost = 0, scalar_cost = 0;
+  int i, j;
+  unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0;
   unsigned int stmt_cost;
   gimple stmt;
   gimple_stmt_iterator si;
@@ -1927,12 +1940,19 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo)
   stmt_vec_info stmt_info = NULL;
   tree dummy_type = NULL;
   int dummy = 0;
+  stmt_vector_for_cost stmt_cost_vec;
+  stmt_info_for_cost *ci;
 
   /* Calculate vector costs.  */
   FOR_EACH_VEC_ELT (slp_instance, slp_instances, i, instance)
     {
       vec_outside_cost += SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (instance);
-      vec_inside_cost += SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance);
+      stmt_cost_vec = SLP_INSTANCE_STMT_COST_VEC (instance);
+
+      FOR_EACH_VEC_ELT (stmt_info_for_cost, stmt_cost_vec, j, ci)
+	(void) add_stmt_cost (BB_VINFO_TARGET_COST_DATA (bb_vinfo),
+			      ci->count, ci->kind,
+			      vinfo_for_stmt (ci->stmt), ci->misalign);
     }
 
   /* Calculate scalar cost.  */
@@ -1961,6 +1981,9 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo)
       scalar_cost += stmt_cost;
     }
 
+  /* Complete the target-specific cost calculation.  */
+  vec_inside_cost = finish_cost (BB_VINFO_TARGET_COST_DATA (bb_vinfo));
+
   if (vect_print_dump_info (REPORT_COST))
     {
       fprintf (vect_dump, "Cost model analysis: \n");
@@ -2072,7 +2095,7 @@ vect_slp_analyze_bb_1 (basic_block bb)
       vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
     }
 
-   if (!vect_verify_datarefs_alignment (NULL, bb_vinfo))
+  if (!vect_verify_datarefs_alignment (NULL, bb_vinfo))
     {
       if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
         fprintf (vect_dump, "not vectorized: unsupported alignment in basic "
@@ -2175,17 +2198,30 @@ vect_slp_analyze_bb (basic_block bb)
 void
 vect_update_slp_costs_according_to_vf (loop_vec_info loop_vinfo)
 {
-  unsigned int i, vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+  unsigned int i, j, vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
   slp_instance instance;
+  stmt_vector_for_cost stmt_cost_vec;
+  stmt_info_for_cost *si;
 
   if (vect_print_dump_info (REPORT_SLP))
     fprintf (vect_dump, "=== vect_update_slp_costs_according_to_vf ===");
 
   FOR_EACH_VEC_ELT (slp_instance, slp_instances, i, instance)
-    /* We assume that costs are linear in ncopies.  */
-    SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance) *= vf
-      / SLP_INSTANCE_UNROLLING_FACTOR (instance);
+    {
+      /* We assume that costs are linear in ncopies.  */
+      int ncopies = vf / SLP_INSTANCE_UNROLLING_FACTOR (instance);
+
+      /* Record the instance's instructions in the target cost model.
+	 This was delayed until here because the count of instructions
+	 isn't known beforehand.  */
+      stmt_cost_vec = SLP_INSTANCE_STMT_COST_VEC (instance);
+
+      FOR_EACH_VEC_ELT (stmt_info_for_cost, stmt_cost_vec, j, si)
+	(void) add_stmt_cost (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo),
+			      si->count * ncopies, si->kind,
+			      vinfo_for_stmt (si->stmt), si->misalign);
+    }
 }