diff options
Diffstat (limited to 'gcc/gimple-ssa-store-merging.c')
| -rw-r--r-- | gcc/gimple-ssa-store-merging.c | 369 |
1 files changed, 270 insertions, 99 deletions
diff --git a/gcc/gimple-ssa-store-merging.c b/gcc/gimple-ssa-store-merging.c index 5a293d7f307..bd9ba28ee51 100644 --- a/gcc/gimple-ssa-store-merging.c +++ b/gcc/gimple-ssa-store-merging.c @@ -125,6 +125,8 @@ #include "tree-cfg.h" #include "tree-eh.h" #include "target.h" +#include "gimplify-me.h" +#include "selftest.h" /* The maximum size (in bits) of the stores this pass should generate. */ #define MAX_STORE_BITSIZE (BITS_PER_WORD) @@ -140,19 +142,17 @@ struct store_immediate_info { unsigned HOST_WIDE_INT bitsize; unsigned HOST_WIDE_INT bitpos; - tree val; - tree dest; gimple *stmt; unsigned int order; - store_immediate_info (unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT, tree, - tree, gimple *, unsigned int); + store_immediate_info (unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT, + gimple *, unsigned int); }; store_immediate_info::store_immediate_info (unsigned HOST_WIDE_INT bs, - unsigned HOST_WIDE_INT bp, tree v, - tree d, gimple *st, + unsigned HOST_WIDE_INT bp, + gimple *st, unsigned int ord) - : bitsize (bs), bitpos (bp), val (v), dest (d), stmt (st), order (ord) + : bitsize (bs), bitpos (bp), stmt (st), order (ord) { } @@ -338,7 +338,7 @@ clear_bit_region (unsigned char *ptr, unsigned int start, else if (start != 0) { clear_bit_region (ptr, start, BITS_PER_UNIT - start); - clear_bit_region (ptr + 1, 0, len - (BITS_PER_UNIT - start) + 1); + clear_bit_region (ptr + 1, 0, len - (BITS_PER_UNIT - start)); } /* Whole bytes need to be cleared. */ else if (start == 0 && len > BITS_PER_UNIT) @@ -432,13 +432,24 @@ encode_tree_to_bitpos (tree expr, unsigned char *ptr, int bitlen, int bitpos, contain a sign bit due to sign-extension). */ unsigned int padding = byte_size - ROUND_UP (bitlen, BITS_PER_UNIT) / BITS_PER_UNIT - 1; - if (BYTES_BIG_ENDIAN) + if (padding != 0 + || bitlen % BITS_PER_UNIT != 0) { - tmpbuf += padding; + /* On big-endian the padding is at the 'front' so just skip the initial + bytes. */ + if (BYTES_BIG_ENDIAN) + tmpbuf += padding; + byte_size -= padding; if (bitlen % BITS_PER_UNIT != 0) - clear_bit_region_be (tmpbuf, BITS_PER_UNIT - 1, - BITS_PER_UNIT - (bitlen % BITS_PER_UNIT)); + { + if (BYTES_BIG_ENDIAN) + clear_bit_region_be (tmpbuf, BITS_PER_UNIT - 1, + BITS_PER_UNIT - (bitlen % BITS_PER_UNIT)); + else + clear_bit_region (tmpbuf, bitlen, + byte_size * BITS_PER_UNIT - bitlen); + } } /* Clear the bit region in PTR where the bits from TMPBUF will be @@ -547,7 +558,7 @@ merged_store_group::merged_store_group (store_immediate_info *info) /* VAL has memory allocated for it in apply_stores once the group width has been finalized. */ val = NULL; - align = get_object_alignment (info->dest); + align = get_object_alignment (gimple_assign_lhs (info->stmt)); stores.create (1); stores.safe_push (info); last_stmt = info->stmt; @@ -644,14 +655,16 @@ merged_store_group::apply_stores () FOR_EACH_VEC_ELT (stores, i, info) { unsigned int pos_in_buffer = info->bitpos - start; - bool ret = encode_tree_to_bitpos (info->val, val, info->bitsize, - pos_in_buffer, buf_size); + bool ret = encode_tree_to_bitpos (gimple_assign_rhs1 (info->stmt), + val, info->bitsize, + pos_in_buffer, buf_size); if (dump_file && (dump_flags & TDF_DETAILS)) { if (ret) { fprintf (dump_file, "After writing "); - print_generic_expr (dump_file, info->val, 0); + print_generic_expr (dump_file, + gimple_assign_rhs1 (info->stmt), 0); fprintf (dump_file, " of size " HOST_WIDE_INT_PRINT_DEC " at position %d the merged region contains:\n", info->bitsize, pos_in_buffer); @@ -670,13 +683,15 @@ merged_store_group::apply_stores () struct imm_store_chain_info { + tree base_addr; auto_vec<struct store_immediate_info *> m_store_info; auto_vec<merged_store_group *> m_merged_store_groups; - bool terminate_and_process_chain (tree); + imm_store_chain_info (tree b_a) : base_addr (b_a) {} + bool terminate_and_process_chain (); bool coalesce_immediate_stores (); - bool output_merged_store (tree, merged_store_group *); - bool output_merged_stores (tree); + bool output_merged_store (merged_store_group *); + bool output_merged_stores (); }; const pass_data pass_data_tree_store_merging = { @@ -712,8 +727,9 @@ private: hash_map<tree_operand_hash, struct imm_store_chain_info *> m_stores; bool terminate_and_process_all_chains (); - bool terminate_all_aliasing_chains (tree, tree, bool, gimple *); - bool terminate_and_release_chain (tree); + bool terminate_all_aliasing_chains (imm_store_chain_info **, + bool, gimple *); + bool terminate_and_release_chain (imm_store_chain_info *); }; // class pass_store_merging /* Terminate and process all recorded chains. Return true if any changes @@ -726,7 +742,7 @@ pass_store_merging::terminate_and_process_all_chains () = m_stores.begin (); bool ret = false; for (; iter != m_stores.end (); ++iter) - ret |= terminate_and_release_chain ((*iter).first); + ret |= terminate_and_release_chain ((*iter).second); return ret; } @@ -740,7 +756,8 @@ pass_store_merging::terminate_and_process_all_chains () If that is the case we have to terminate any chain anchored at BASE. */ bool -pass_store_merging::terminate_all_aliasing_chains (tree dest, tree base, +pass_store_merging::terminate_all_aliasing_chains (imm_store_chain_info + **chain_info, bool var_offset_p, gimple *stmt) { @@ -750,44 +767,41 @@ pass_store_merging::terminate_all_aliasing_chains (tree dest, tree base, if (!gimple_vuse (stmt)) return false; - struct imm_store_chain_info **chain_info = NULL; - /* Check if the assignment destination (BASE) is part of a store chain. This is to catch non-constant stores to destinations that may be part of a chain. */ - if (base) + if (chain_info) { - chain_info = m_stores.get (base); - if (chain_info) + /* We have a chain at BASE and we're writing to [BASE + <variable>]. + This can interfere with any of the stores so terminate + the chain. */ + if (var_offset_p) { - /* We have a chain at BASE and we're writing to [BASE + <variable>]. - This can interfere with any of the stores so terminate - the chain. */ - if (var_offset_p) - { - terminate_and_release_chain (base); - ret = true; - } - /* Otherwise go through every store in the chain to see if it - aliases with any of them. */ - else + terminate_and_release_chain (*chain_info); + ret = true; + } + /* Otherwise go through every store in the chain to see if it + aliases with any of them. */ + else + { + struct store_immediate_info *info; + unsigned int i; + FOR_EACH_VEC_ELT ((*chain_info)->m_store_info, i, info) { - struct store_immediate_info *info; - unsigned int i; - FOR_EACH_VEC_ELT ((*chain_info)->m_store_info, i, info) + if (ref_maybe_used_by_stmt_p (stmt, + gimple_assign_lhs (info->stmt)) + || stmt_may_clobber_ref_p (stmt, + gimple_assign_lhs (info->stmt))) { - if (refs_may_alias_p (info->dest, dest)) + if (dump_file && (dump_flags & TDF_DETAILS)) { - if (dump_file && (dump_flags & TDF_DETAILS)) - { - fprintf (dump_file, - "stmt causes chain termination:\n"); - print_gimple_stmt (dump_file, stmt, 0, 0); - } - terminate_and_release_chain (base); - ret = true; - break; + fprintf (dump_file, + "stmt causes chain termination:\n"); + print_gimple_stmt (dump_file, stmt, 0, 0); } + terminate_and_release_chain (*chain_info); + ret = true; + break; } } } @@ -804,11 +818,16 @@ pass_store_merging::terminate_all_aliasing_chains (tree dest, tree base, if (chain_info && (*chain_info) == (*iter).second) continue; - tree key = (*iter).first; - if (ref_maybe_used_by_stmt_p (stmt, key) - || stmt_may_clobber_ref_p (stmt, key)) + /* We can't use the base object here as that does not reliably exist. + Build a ao_ref from the base object address (if we know the + minimum and maximum offset and the maximum size we could improve + things here). */ + ao_ref chain_ref; + ao_ref_init_from_ptr_and_size (&chain_ref, (*iter).first, NULL_TREE); + if (ref_maybe_used_by_stmt_p (stmt, &chain_ref) + || stmt_may_clobber_ref_p_1 (stmt, &chain_ref)) { - terminate_and_release_chain (key); + terminate_and_release_chain ((*iter).second); ret = true; } } @@ -821,19 +840,11 @@ pass_store_merging::terminate_all_aliasing_chains (tree dest, tree base, entry is removed after the processing in any case. */ bool -pass_store_merging::terminate_and_release_chain (tree base) +pass_store_merging::terminate_and_release_chain (imm_store_chain_info *chain_info) { - struct imm_store_chain_info **chain_info = m_stores.get (base); - - if (!chain_info) - return false; - - gcc_assert (*chain_info); - - bool ret = (*chain_info)->terminate_and_process_chain (base); - delete *chain_info; - m_stores.remove (base); - + bool ret = chain_info->terminate_and_process_chain (); + m_stores.remove (chain_info->base_addr); + delete chain_info; return ret; } @@ -870,7 +881,7 @@ imm_store_chain_info::coalesce_immediate_stores () fprintf (dump_file, "Store %u:\nbitsize:" HOST_WIDE_INT_PRINT_DEC " bitpos:" HOST_WIDE_INT_PRINT_DEC " val:\n", i, info->bitsize, info->bitpos); - print_generic_expr (dump_file, info->val, 0); + print_generic_expr (dump_file, gimple_assign_rhs1 (info->stmt), 0); fprintf (dump_file, "\n------------\n"); } @@ -1093,7 +1104,7 @@ split_group (merged_store_group *group, return true. */ bool -imm_store_chain_info::output_merged_store (tree base, merged_store_group *group) +imm_store_chain_info::output_merged_store (merged_store_group *group) { unsigned HOST_WIDE_INT start_byte_pos = group->start / BITS_PER_UNIT; @@ -1121,6 +1132,8 @@ imm_store_chain_info::output_merged_store (tree base, merged_store_group *group) unsigned int i; bool fail = false; + tree addr = force_gimple_operand_1 (unshare_expr (base_addr), &seq, + is_gimple_mem_ref_addr, NULL_TREE); FOR_EACH_VEC_ELT (split_stores, i, split_store) { unsigned HOST_WIDE_INT try_size = split_store->size; @@ -1130,8 +1143,7 @@ imm_store_chain_info::output_merged_store (tree base, merged_store_group *group) location_t loc = get_location_for_stmts (split_store->orig_stmts); tree int_type = build_nonstandard_integer_type (try_size, UNSIGNED); - SET_TYPE_ALIGN (int_type, align); - tree addr = build_fold_addr_expr (base); + int_type = build_aligned_type (int_type, align); tree dest = fold_build2 (MEM_REF, int_type, addr, build_int_cst (offset_type, try_pos)); @@ -1203,14 +1215,14 @@ imm_store_chain_info::output_merged_store (tree base, merged_store_group *group) successful. Return true iff any changes were made. */ bool -imm_store_chain_info::output_merged_stores (tree base) +imm_store_chain_info::output_merged_stores () { unsigned int i; merged_store_group *merged_store; bool ret = false; FOR_EACH_VEC_ELT (m_merged_store_groups, i, merged_store) { - if (output_merged_store (base, merged_store)) + if (output_merged_store (merged_store)) { unsigned int j; store_immediate_info *store; @@ -1240,7 +1252,7 @@ imm_store_chain_info::output_merged_stores (tree base) Return true if any changes were made. */ bool -imm_store_chain_info::terminate_and_process_chain (tree base) +imm_store_chain_info::terminate_and_process_chain () { /* Process store chain. */ bool ret = false; @@ -1248,7 +1260,7 @@ imm_store_chain_info::terminate_and_process_chain (tree base) { ret = coalesce_immediate_stores (); if (ret) - ret = output_merged_stores (base); + ret = output_merged_stores (); } /* Delete all the entries we allocated ourselves. */ @@ -1361,37 +1373,58 @@ pass_store_merging::execute (function *fun) &unsignedp, &reversep, &volatilep); /* As a future enhancement we could handle stores with the same base and offset. */ - bool invalid = offset || reversep + bool invalid = reversep || ((bitsize > MAX_BITSIZE_MODE_ANY_INT) && (TREE_CODE (rhs) != INTEGER_CST)) - || !rhs_valid_for_store_merging_p (rhs) - /* An access may not be volatile itself but base_addr may be - a volatile decl i.e. MEM[&volatile-decl]. The hashing for - tree_operand_hash won't consider such stores equal to each - other so we can't track chains on them. */ - || TREE_THIS_VOLATILE (base_addr); + || !rhs_valid_for_store_merging_p (rhs); + /* We do not want to rewrite TARGET_MEM_REFs. */ + if (TREE_CODE (base_addr) == TARGET_MEM_REF) + invalid = true; /* In some cases get_inner_reference may return a MEM_REF [ptr + byteoffset]. For the purposes of this pass canonicalize the base_addr to MEM_REF [ptr] and take byteoffset into account in the bitpos. This occurs in PR 23684 and this way we can catch more chains. */ - if (TREE_CODE (base_addr) == MEM_REF - && POINTER_TYPE_P (TREE_TYPE (TREE_OPERAND (base_addr, 0)))) + else if (TREE_CODE (base_addr) == MEM_REF) { offset_int bit_off, byte_off = mem_ref_offset (base_addr); bit_off = byte_off << LOG2_BITS_PER_UNIT; bit_off += bitpos; if (!wi::neg_p (bit_off) && wi::fits_shwi_p (bit_off)) - { - bitpos = bit_off.to_shwi (); - base_addr = build2 (MEM_REF, TREE_TYPE (base_addr), - TREE_OPERAND (base_addr, 0), - build_zero_cst (TREE_TYPE ( - TREE_OPERAND (base_addr, 1)))); - } + bitpos = bit_off.to_shwi (); else invalid = true; + base_addr = TREE_OPERAND (base_addr, 0); + } + /* get_inner_reference returns the base object, get at its + address now. */ + else + { + if (bitpos < 0) + invalid = true; + base_addr = build_fold_addr_expr (base_addr); + } + + if (! invalid + && offset != NULL_TREE) + { + /* If the access is variable offset then a base + decl has to be address-taken to be able to + emit pointer-based stores to it. + ??? We might be able to get away with + re-using the original base up to the first + variable part and then wrapping that inside + a BIT_FIELD_REF. */ + tree base = get_base_address (base_addr); + if (! base + || (DECL_P (base) + && ! TREE_ADDRESSABLE (base))) + invalid = true; + else + base_addr = build2 (POINTER_PLUS_EXPR, + TREE_TYPE (base_addr), + base_addr, offset); } struct imm_store_chain_info **chain_info @@ -1403,7 +1436,7 @@ pass_store_merging::execute (function *fun) if (chain_info) { info = new store_immediate_info ( - bitsize, bitpos, rhs, lhs, stmt, + bitsize, bitpos, stmt, (*chain_info)->m_store_info.length ()); if (dump_file && (dump_flags & TDF_DETAILS)) { @@ -1422,17 +1455,17 @@ pass_store_merging::execute (function *fun) fprintf (dump_file, "Reached maximum number of statements" " to merge:\n"); - terminate_and_release_chain (base_addr); + terminate_and_release_chain (*chain_info); } continue; } /* Store aliases any existing chain? */ - terminate_all_aliasing_chains (lhs, base_addr, false, stmt); + terminate_all_aliasing_chains (chain_info, false, stmt); /* Start a new chain. */ struct imm_store_chain_info *new_chain - = new imm_store_chain_info; - info = new store_immediate_info (bitsize, bitpos, rhs, lhs, + = new imm_store_chain_info (base_addr); + info = new store_immediate_info (bitsize, bitpos, stmt, 0); new_chain->m_store_info.safe_push (info); m_stores.put (base_addr, new_chain); @@ -1447,13 +1480,13 @@ pass_store_merging::execute (function *fun) } } else - terminate_all_aliasing_chains (lhs, base_addr, + terminate_all_aliasing_chains (chain_info, offset != NULL_TREE, stmt); continue; } - terminate_all_aliasing_chains (NULL_TREE, NULL_TREE, false, stmt); + terminate_all_aliasing_chains (NULL, false, stmt); } terminate_and_process_all_chains (); } @@ -1469,3 +1502,141 @@ make_pass_store_merging (gcc::context *ctxt) { return new pass_store_merging (ctxt); } + +#if CHECKING_P + +namespace selftest { + +/* Selftests for store merging helpers. */ + +/* Assert that all elements of the byte arrays X and Y, both of length N + are equal. */ + +static void +verify_array_eq (unsigned char *x, unsigned char *y, unsigned int n) +{ + for (unsigned int i = 0; i < n; i++) + { + if (x[i] != y[i]) + { + fprintf (stderr, "Arrays do not match. X:\n"); + dump_char_array (stderr, x, n); + fprintf (stderr, "Y:\n"); + dump_char_array (stderr, y, n); + } + ASSERT_EQ (x[i], y[i]); + } +} + +/* Test shift_bytes_in_array and that it carries bits across between + bytes correctly. */ + +static void +verify_shift_bytes_in_array (void) +{ + /* byte 1 | byte 0 + 00011111 | 11100000. */ + unsigned char orig[2] = { 0xe0, 0x1f }; + unsigned char in[2]; + memcpy (in, orig, sizeof orig); + + unsigned char expected[2] = { 0x80, 0x7f }; + shift_bytes_in_array (in, sizeof (in), 2); + verify_array_eq (in, expected, sizeof (in)); + + memcpy (in, orig, sizeof orig); + memcpy (expected, orig, sizeof orig); + /* Check that shifting by zero doesn't change anything. */ + shift_bytes_in_array (in, sizeof (in), 0); + verify_array_eq (in, expected, sizeof (in)); + +} + +/* Test shift_bytes_in_array_right and that it carries bits across between + bytes correctly. */ + +static void +verify_shift_bytes_in_array_right (void) +{ + /* byte 1 | byte 0 + 00011111 | 11100000. */ + unsigned char orig[2] = { 0x1f, 0xe0}; + unsigned char in[2]; + memcpy (in, orig, sizeof orig); + unsigned char expected[2] = { 0x07, 0xf8}; + shift_bytes_in_array_right (in, sizeof (in), 2); + verify_array_eq (in, expected, sizeof (in)); + + memcpy (in, orig, sizeof orig); + memcpy (expected, orig, sizeof orig); + /* Check that shifting by zero doesn't change anything. */ + shift_bytes_in_array_right (in, sizeof (in), 0); + verify_array_eq (in, expected, sizeof (in)); +} + +/* Test clear_bit_region that it clears exactly the bits asked and + nothing more. */ + +static void +verify_clear_bit_region (void) +{ + /* Start with all bits set and test clearing various patterns in them. */ + unsigned char orig[3] = { 0xff, 0xff, 0xff}; + unsigned char in[3]; + unsigned char expected[3]; + memcpy (in, orig, sizeof in); + + /* Check zeroing out all the bits. */ + clear_bit_region (in, 0, 3 * BITS_PER_UNIT); + expected[0] = expected[1] = expected[2] = 0; + verify_array_eq (in, expected, sizeof in); + + memcpy (in, orig, sizeof in); + /* Leave the first and last bits intact. */ + clear_bit_region (in, 1, 3 * BITS_PER_UNIT - 2); + expected[0] = 0x1; + expected[1] = 0; + expected[2] = 0x80; + verify_array_eq (in, expected, sizeof in); +} + +/* Test verify_clear_bit_region_be that it clears exactly the bits asked and + nothing more. */ + +static void +verify_clear_bit_region_be (void) +{ + /* Start with all bits set and test clearing various patterns in them. */ + unsigned char orig[3] = { 0xff, 0xff, 0xff}; + unsigned char in[3]; + unsigned char expected[3]; + memcpy (in, orig, sizeof in); + + /* Check zeroing out all the bits. */ + clear_bit_region_be (in, BITS_PER_UNIT - 1, 3 * BITS_PER_UNIT); + expected[0] = expected[1] = expected[2] = 0; + verify_array_eq (in, expected, sizeof in); + + memcpy (in, orig, sizeof in); + /* Leave the first and last bits intact. */ + clear_bit_region_be (in, BITS_PER_UNIT - 2, 3 * BITS_PER_UNIT - 2); + expected[0] = 0x80; + expected[1] = 0; + expected[2] = 0x1; + verify_array_eq (in, expected, sizeof in); +} + + +/* Run all of the selftests within this file. */ + +void +store_merging_c_tests (void) +{ + verify_shift_bytes_in_array (); + verify_shift_bytes_in_array_right (); + verify_clear_bit_region (); + verify_clear_bit_region_be (); +} + +} // namespace selftest +#endif /* CHECKING_P. */ |