diff options
Diffstat (limited to 'gcc/tree-vect-data-refs.c')
| -rw-r--r-- | gcc/tree-vect-data-refs.c | 233 |
1 files changed, 118 insertions, 115 deletions
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 188581f6246..1fe5047c70b 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -129,31 +129,6 @@ vect_get_smallest_scalar_type (gimple stmt, HOST_WIDE_INT *lhs_size_unit, } -/* Find the place of the data-ref in STMT in the interleaving chain that starts - from FIRST_STMT. Return -1 if the data-ref is not a part of the chain. */ - -int -vect_get_place_in_interleaving_chain (gimple stmt, gimple first_stmt) -{ - gimple next_stmt = first_stmt; - int result = 0; - - if (first_stmt != GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt))) - return -1; - - while (next_stmt && next_stmt != stmt) - { - result++; - next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); - } - - if (next_stmt) - return result; - else - return -1; -} - - /* Check if data references pointed by DR_I and DR_J are same or belong to same interleaving group. Return FALSE if drs are different, otherwise return TRUE. */ @@ -280,6 +255,23 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, /* Unknown data dependence. */ if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know) { + if (STMT_VINFO_GATHER_P (stmtinfo_a) + || STMT_VINFO_GATHER_P (stmtinfo_b)) + { + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "versioning for alias not supported for: " + "can't determine dependence between "); + dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, + DR_REF (dra)); + dump_printf (MSG_MISSED_OPTIMIZATION, " and "); + dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, + DR_REF (drb)); + } + return false; + } + if (dump_enabled_p ()) { dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -299,6 +291,23 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, /* Known data dependence. */ if (DDR_NUM_DIST_VECTS (ddr) == 0) { + if (STMT_VINFO_GATHER_P (stmtinfo_a) + || STMT_VINFO_GATHER_P (stmtinfo_b)) + { + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "versioning for alias not supported for: " + "bad dist vector for "); + dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, + DR_REF (dra)); + dump_printf (MSG_MISSED_OPTIMIZATION, " and "); + dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, + DR_REF (drb)); + } + return false; + } + if (dump_enabled_p ()) { dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -332,14 +341,34 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (drb)); } - /* For interleaving, mark that there is a read-write dependency if - necessary. We check before that one of the data-refs is store. */ - if (DR_IS_READ (dra)) - GROUP_READ_WRITE_DEPENDENCE (stmtinfo_a) = true; - else - { - if (DR_IS_READ (drb)) - GROUP_READ_WRITE_DEPENDENCE (stmtinfo_b) = true; + /* When we perform grouped accesses and perform implicit CSE + by detecting equal accesses and doing disambiguation with + runtime alias tests like for + .. = a[i]; + .. = a[i+1]; + a[i] = ..; + a[i+1] = ..; + *p = ..; + .. = a[i]; + .. = a[i+1]; + where we will end up loading { a[i], a[i+1] } once, make + sure that inserting group loads before the first load and + stores after the last store will do the right thing. */ + if ((STMT_VINFO_GROUPED_ACCESS (stmtinfo_a) + && GROUP_SAME_DR_STMT (stmtinfo_a)) + || (STMT_VINFO_GROUPED_ACCESS (stmtinfo_b) + && GROUP_SAME_DR_STMT (stmtinfo_b))) + { + gimple earlier_stmt; + earlier_stmt = get_earlier_stmt (DR_STMT (dra), DR_STMT (drb)); + if (DR_IS_WRITE + (STMT_VINFO_DATA_REF (vinfo_for_stmt (earlier_stmt)))) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "READ_WRITE dependence in interleaving."); + return true; + } } continue; @@ -450,6 +479,13 @@ vect_slp_analyze_data_ref_dependence (struct data_dependence_relation *ddr) if (DR_IS_READ (dra) && DR_IS_READ (drb)) return false; + /* If dra and drb are part of the same interleaving chain consider + them independent. */ + if (STMT_VINFO_GROUPED_ACCESS (vinfo_for_stmt (DR_STMT (dra))) + && (GROUP_FIRST_ELEMENT (vinfo_for_stmt (DR_STMT (dra))) + == GROUP_FIRST_ELEMENT (vinfo_for_stmt (DR_STMT (drb))))) + return false; + /* Unknown data dependence. */ if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know) { @@ -1440,20 +1476,35 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) } else { - /* If we don't know all the misalignment values, we prefer - peeling for data-ref that has maximum number of data-refs + /* If we don't know any misalignment values, we prefer + peeling for data-ref that has the maximum number of data-refs with the same alignment, unless the target prefers to align stores over load. */ if (all_misalignments_unknown) { - if (same_align_drs_max - < STMT_VINFO_SAME_ALIGN_REFS (stmt_info).length () - || !dr0) + unsigned same_align_drs + = STMT_VINFO_SAME_ALIGN_REFS (stmt_info).length (); + if (!dr0 + || same_align_drs_max < same_align_drs) { - same_align_drs_max - = STMT_VINFO_SAME_ALIGN_REFS (stmt_info).length (); + same_align_drs_max = same_align_drs; dr0 = dr; } + /* For data-refs with the same number of related + accesses prefer the one where the misalign + computation will be invariant in the outermost loop. */ + else if (same_align_drs_max == same_align_drs) + { + struct loop *ivloop0, *ivloop; + ivloop0 = outermost_invariant_loop_for_expr + (loop, DR_BASE_ADDRESS (dr0)); + ivloop = outermost_invariant_loop_for_expr + (loop, DR_BASE_ADDRESS (dr)); + if ((ivloop && !ivloop0) + || (ivloop && ivloop0 + && flow_loop_nested_p (ivloop, ivloop0))) + dr0 = dr; + } if (!first_store && DR_IS_WRITE (dr)) first_store = dr; @@ -1462,8 +1513,6 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) /* If there are both known and unknown misaligned accesses in the loop, we choose peeling amount according to the known accesses. */ - - if (!supportable_dr_alignment) { dr0 = dr; @@ -2068,17 +2117,6 @@ vect_analyze_group_access (struct data_reference *dr) return false; } - /* Check that there is no load-store dependencies for this loads - to prevent a case of load-store-load to the same location. */ - if (GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (next)) - || GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (prev))) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "READ_WRITE dependence in interleaving."); - return false; - } - /* For load use the same data-ref load. */ GROUP_SAME_DR_STMT (vinfo_for_stmt (next)) = prev; @@ -2775,56 +2813,6 @@ vect_check_gather (gimple stmt, loop_vec_info loop_vinfo, tree *basep, return decl; } -/* Check wether a non-affine load in STMT (being in the loop referred to - in LOOP_VINFO) is suitable for handling as strided load. That is the case - if its address is a simple induction variable. If so return the base - of that induction variable in *BASEP and the (loop-invariant) step - in *STEPP, both only when that pointer is non-zero. - - This handles ARRAY_REFs (with variant index) and MEM_REFs (with variant - base pointer) only. */ - -static bool -vect_check_strided_load (gimple stmt, loop_vec_info loop_vinfo) -{ - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); - tree base, off; - affine_iv iv; - - if (!DR_IS_READ (dr)) - return false; - - base = DR_REF (dr); - - if (TREE_CODE (base) == REALPART_EXPR - || TREE_CODE (base) == IMAGPART_EXPR) - base = TREE_OPERAND (base, 0); - - if (TREE_CODE (base) == ARRAY_REF) - { - off = TREE_OPERAND (base, 1); - base = TREE_OPERAND (base, 0); - } - else if (TREE_CODE (base) == MEM_REF) - { - off = TREE_OPERAND (base, 0); - base = TREE_OPERAND (base, 1); - } - else - return false; - - if (TREE_CODE (off) != SSA_NAME) - return false; - - if (!expr_invariant_in_loop_p (loop, base) - || !simple_iv (loop, loop_containing_stmt (stmt), off, &iv, true)) - return false; - - return true; -} - /* Function vect_analyze_data_refs. Find all the data references in the loop or basic block. @@ -3215,6 +3203,17 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, } return false; } + else + { + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_NOTE, vect_location, + "got vectype for stmt: "); + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); + dump_generic_expr (MSG_NOTE, TDF_SLIM, + STMT_VINFO_VECTYPE (stmt_info)); + } + } /* Adjust the minimal vectorization factor according to the vector type. */ @@ -3250,10 +3249,8 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, else if (loop_vinfo && TREE_CODE (DR_STEP (dr)) != INTEGER_CST) { - bool strided_load = false; - if (!nested_in_vect_loop_p (loop, stmt)) - strided_load = vect_check_strided_load (stmt, loop_vinfo); - if (!strided_load) + if (nested_in_vect_loop_p (loop, stmt) + || !DR_IS_READ (dr)) { if (dump_enabled_p ()) { @@ -3304,13 +3301,13 @@ vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name) switch (var_kind) { case vect_simple_var: - prefix = "vect_"; + prefix = "vect"; break; case vect_scalar_var: - prefix = "stmp_"; + prefix = "stmp"; break; case vect_pointer_var: - prefix = "vect_p"; + prefix = "vectp"; break; default: gcc_unreachable (); @@ -3318,7 +3315,7 @@ vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name) if (name) { - char* tmp = concat (prefix, name, NULL); + char* tmp = concat (prefix, "_", name, NULL); new_vect_var = create_tmp_reg (type, tmp); free (tmp); } @@ -3572,6 +3569,8 @@ vect_create_data_ref_ptr (gimple stmt, tree aggr_type, struct loop *at_loop, dump_generic_expr (MSG_NOTE, TDF_SLIM, aggr_type); if (TREE_CODE (dr_base_type) == ARRAY_TYPE) dump_printf (MSG_NOTE, " vectorizing an array ref: "); + else if (TREE_CODE (dr_base_type) == VECTOR_TYPE) + dump_printf (MSG_NOTE, " vectorizing a vector ref: "); else if (TREE_CODE (dr_base_type) == RECORD_TYPE) dump_printf (MSG_NOTE, " vectorizing a record based array ref: "); else @@ -3845,7 +3844,8 @@ tree vect_create_destination_var (tree scalar_dest, tree vectype) { tree vec_dest; - const char *new_name; + const char *name; + char *new_name; tree type; enum vect_var_kind kind; @@ -3854,10 +3854,13 @@ vect_create_destination_var (tree scalar_dest, tree vectype) gcc_assert (TREE_CODE (scalar_dest) == SSA_NAME); - new_name = get_name (scalar_dest); - if (!new_name) - new_name = "var_"; + name = get_name (scalar_dest); + if (name) + asprintf (&new_name, "%s_%u", name, SSA_NAME_VERSION (scalar_dest)); + else + asprintf (&new_name, "_%u", SSA_NAME_VERSION (scalar_dest)); vec_dest = vect_get_new_vect_var (type, kind, new_name); + free (new_name); return vec_dest; } |