diff options
Diffstat (limited to 'storage/tokudb')
90 files changed, 1906 insertions, 314 deletions
diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 9a257b8da98..57a43930eb7 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -25,7 +25,7 @@ IF (HAVE_WVLA) ENDIF() ############################################ -SET(TOKUDB_VERSION "tokudb-7.5.4") +SET(TOKUDB_VERSION "tokudb-7.5.7") SET(TOKUDB_DEB_FILES "usr/lib/mysql/plugin/ha_tokudb.so\netc/mysql/conf.d/tokudb.cnf\nusr/bin/tokuftdump\nusr/share/doc/mariadb-server-10.0/README-TOKUDB\nusr/share/doc/mariadb-server-10.0/README.md" PARENT_SCOPE) SET(USE_BDB OFF CACHE BOOL "") MARK_AS_ADVANCED(BUILDNAME) diff --git a/storage/tokudb/README.md b/storage/tokudb/README.md index 2ab2e21a5a1..0d4f09d4c86 100644 --- a/storage/tokudb/README.md +++ b/storage/tokudb/README.md @@ -30,14 +30,14 @@ working MySQL or MariaDB with Tokutek patches, and with the TokuDB storage engine, called `make.mysql.bash`. This script will download copies of the needed source code from github and build everything. -To build MySQL 5.5.40 with TokuDB 7.5.3: +To build MySQL 5.5.41 with TokuDB 7.5.5: ```sh -scripts/make.mysql.bash --mysqlbuild=mysql-5.5.40-tokudb-7.5.3-linux-x86_64 +scripts/make.mysql.bash --mysqlbuild=mysql-5.5.41-tokudb-7.5.5-linux-x86_64 ``` -To build MariaDB 5.5.40 with TokuDB 7.5.3: +To build MariaDB 5.5.41 with TokuDB 7.5.5: ```sh -scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.40-tokudb-7.5.3-linux-x86_64 +scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.41-tokudb-7.5.5-linux-x86_64 ``` Before you start, make sure you have a C++11-compatible compiler (GCC >= @@ -59,6 +59,7 @@ repositories, run this: scripts/make.mysql.debug.env.bash ``` +We use gcc from devtoolset-1.1 on CentOS 5.9 for builds. Contribute ---------- diff --git a/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.loglog.png b/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.loglog.png Binary files differindex 99b15ff642f..7b2c1c5c66d 100644 --- a/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.loglog.png +++ b/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.loglog.png diff --git a/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.png b/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.png Binary files differindex 7297013b51f..d46c053c0af 100644 --- a/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.png +++ b/storage/tokudb/doc2/sysbench.update.ma10.tokudb754.png diff --git a/storage/tokudb/ft-index/buildheader/make_tdb.cc b/storage/tokudb/ft-index/buildheader/make_tdb.cc index 88f8882df78..53706649231 100644 --- a/storage/tokudb/ft-index/buildheader/make_tdb.cc +++ b/storage/tokudb/ft-index/buildheader/make_tdb.cc @@ -572,7 +572,7 @@ static void print_db_txn_struct (void) { STRUCT_SETUP(DB_TXN, abort, "int (*%s) (DB_TXN *)"); STRUCT_SETUP(DB_TXN, api_internal,"void *%s"); STRUCT_SETUP(DB_TXN, commit, "int (*%s) (DB_TXN*, uint32_t)"); - STRUCT_SETUP(DB_TXN, prepare, "int (*%s) (DB_TXN*, uint8_t gid[DB_GID_SIZE])"); + STRUCT_SETUP(DB_TXN, prepare, "int (*%s) (DB_TXN*, uint8_t gid[DB_GID_SIZE], uint32_t flags)"); STRUCT_SETUP(DB_TXN, discard, "int (*%s) (DB_TXN*, uint32_t)"); STRUCT_SETUP(DB_TXN, id, "uint32_t (*%s) (DB_TXN *)"); STRUCT_SETUP(DB_TXN, mgrp, "DB_ENV *%s /* In TokuFT, mgrp is a DB_ENV, not a DB_TXNMGR */"); @@ -581,11 +581,13 @@ static void print_db_txn_struct (void) { "int (*txn_stat)(DB_TXN *, struct txn_stat **)", "int (*commit_with_progress)(DB_TXN*, uint32_t, TXN_PROGRESS_POLL_FUNCTION, void*)", "int (*abort_with_progress)(DB_TXN*, TXN_PROGRESS_POLL_FUNCTION, void*)", - "int (*xa_prepare) (DB_TXN*, TOKU_XA_XID *)", + "int (*xa_prepare) (DB_TXN*, TOKU_XA_XID *, uint32_t flags)", "uint64_t (*id64) (DB_TXN*)", "void (*set_client_id)(DB_TXN *, uint64_t client_id)", "uint64_t (*get_client_id)(DB_TXN *)", "bool (*is_prepared)(DB_TXN *)", + "DB_TXN *(*get_child)(DB_TXN *)", + "uint64_t (*get_start_time)(DB_TXN *)", NULL}; sort_and_dump_fields("db_txn", false, extra); } @@ -614,7 +616,7 @@ static void print_dbc_struct (void) { "int (*c_getf_set_range_reverse)(DBC *, uint32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)", "int (*c_getf_set_range_with_bound)(DBC *, uint32_t, DBT *k, DBT *k_bound, YDB_CALLBACK_FUNCTION, void *)", "int (*c_set_bounds)(DBC*, const DBT*, const DBT*, bool pre_acquire, int out_of_range_error)", - "void (*c_set_check_interrupt_callback)(DBC*, bool (*)(void*), void *)", + "void (*c_set_check_interrupt_callback)(DBC*, bool (*)(void*, uint64_t deleted_rows), void *)", "void (*c_remove_restriction)(DBC*)", "char _internal[512]", NULL}; @@ -785,7 +787,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) { printf("typedef void (*lock_timeout_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid);\n"); printf("typedef int (*iterate_row_locks_callback)(DB **db, DBT *left_key, DBT *right_key, void *extra);\n"); - printf("typedef int (*iterate_transactions_callback)(uint64_t txnid, uint64_t client_id, iterate_row_locks_callback cb, void *locks_extra, void *extra);\n"); + printf("typedef int (*iterate_transactions_callback)(DB_TXN *dbtxn, iterate_row_locks_callback cb, void *locks_extra, void *extra);\n"); printf("typedef int (*iterate_requests_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid, uint64_t start_time, void *extra);\n"); print_db_env_struct(); print_db_key_range_struct(); diff --git a/storage/tokudb/ft-index/ft/ft-internal.h b/storage/tokudb/ft-index/ft/ft-internal.h index 88fc5dca686..4a820d5682b 100644 --- a/storage/tokudb/ft-index/ft/ft-internal.h +++ b/storage/tokudb/ft-index/ft/ft-internal.h @@ -655,7 +655,7 @@ int toku_upgrade_msn_from_root_to_header(int fd, FT ft) __attribute__((nonnull)) // When lock_only is true, the callback only does optional lock tree locking. typedef int (*FT_GET_CALLBACK_FUNCTION)(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only); -typedef bool (*FT_CHECK_INTERRUPT_CALLBACK)(void *extra); +typedef bool (*FT_CHECK_INTERRUPT_CALLBACK)(void *extra, uint64_t deleted_rows); struct ft_cursor; int toku_ft_search(FT_HANDLE ft_handle, ft_search *search, FT_GET_CALLBACK_FUNCTION getf, void *getf_v, struct ft_cursor *ftcursor, bool can_bulk_fetch); diff --git a/storage/tokudb/ft-index/ft/ft-ops.cc b/storage/tokudb/ft-index/ft/ft-ops.cc index 34c9c46f1c6..b3c224afb7a 100644 --- a/storage/tokudb/ft-index/ft/ft-ops.cc +++ b/storage/tokudb/ft-index/ft/ft-ops.cc @@ -3387,7 +3387,7 @@ ok: ; idx++; if (idx >= bn->data_buffer.num_klpairs() || ((n_deleted % 64) == 0 && !search_continue(search, key, keylen))) { STATUS_INC(FT_CURSOR_SKIP_DELETED_LEAF_ENTRY, n_deleted); - if (ftcursor->interrupt_cb && ftcursor->interrupt_cb(ftcursor->interrupt_cb_extra)) { + if (ftcursor->interrupt_cb && ftcursor->interrupt_cb(ftcursor->interrupt_cb_extra, n_deleted)) { return TOKUDB_INTERRUPTED; } return DB_NOTFOUND; @@ -3396,7 +3396,7 @@ ok: ; case FT_SEARCH_RIGHT: if (idx == 0) { STATUS_INC(FT_CURSOR_SKIP_DELETED_LEAF_ENTRY, n_deleted); - if (ftcursor->interrupt_cb && ftcursor->interrupt_cb(ftcursor->interrupt_cb_extra)) { + if (ftcursor->interrupt_cb && ftcursor->interrupt_cb(ftcursor->interrupt_cb_extra, n_deleted)) { return TOKUDB_INTERRUPTED; } return DB_NOTFOUND; @@ -3410,6 +3410,8 @@ ok: ; assert_zero(r); // we just validated the index if (!le_val_is_del(le, ftcursor->is_snapshot_read, ftcursor->ttxn)) { STATUS_INC(FT_CURSOR_SKIP_DELETED_LEAF_ENTRY, n_deleted); + if (ftcursor->interrupt_cb) + ftcursor->interrupt_cb(ftcursor->interrupt_cb_extra, n_deleted); goto got_a_good_value; } } diff --git a/storage/tokudb/ft-index/ft/ft-verify.cc b/storage/tokudb/ft-index/ft/ft-verify.cc index d9606f37604..0a85136816f 100644 --- a/storage/tokudb/ft-index/ft/ft-verify.cc +++ b/storage/tokudb/ft-index/ft/ft-verify.cc @@ -160,10 +160,14 @@ get_ith_key_dbt (BASEMENTNODE bn, int i) { #define VERIFY_ASSERTION(predicate, i, string) ({ \ if(!(predicate)) { \ - (void) verbose; \ - if (true) { \ - fprintf(stderr, "%s:%d: Looking at child %d of block %" PRId64 ": %s\n", __FILE__, __LINE__, i, blocknum.b, string); \ - } \ + fprintf(stderr, "%s:%d: Looking at child %d of block %" PRId64 ": %s\n", __FILE__, __LINE__, i, blocknum.b, string); \ + result = TOKUDB_NEEDS_REPAIR; \ + if (!keep_going_on_failure) goto done; \ + }}) + +#define VERIFY_ASSERTION_BASEMENT(predicate, bn, entry, string) ({ \ + if(!(predicate)) { \ + fprintf(stderr, "%s:%d: Looking at block %" PRId64 " bn %d entry %d: %s\n", __FILE__, __LINE__, blocknum.b, bn, entry, string); \ result = TOKUDB_NEEDS_REPAIR; \ if (!keep_going_on_failure) goto done; \ }}) @@ -201,7 +205,6 @@ struct verify_message_tree_extra { int verify_message_tree(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) __attribute__((nonnull(3))); int verify_message_tree(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) { - int verbose = e->verbose; BLOCKNUM blocknum = e->blocknum; int keep_going_on_failure = e->keep_going_on_failure; int result = 0; @@ -236,7 +239,6 @@ int error_on_iter(const int32_t &UU(offset), const uint32_t UU(idx), void *UU(e) int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) __attribute__((nonnull(3))); int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) { - int verbose = e->verbose; BLOCKNUM blocknum = e->blocknum; int keep_going_on_failure = e->keep_going_on_failure; int result = 0; @@ -462,16 +464,16 @@ toku_verify_ftnode_internal(FT_HANDLE ft_handle, DBT kdbt = get_ith_key_dbt(bn, j); if (curr_less_pivot) { int compare = compare_pairs(ft_handle, curr_less_pivot, &kdbt); - VERIFY_ASSERTION(compare < 0, j, "The leafentry is >= the lower-bound pivot"); + VERIFY_ASSERTION_BASEMENT(compare < 0, i, j, "The leafentry is >= the lower-bound pivot"); } if (curr_geq_pivot) { int compare = compare_pairs(ft_handle, curr_geq_pivot, &kdbt); - VERIFY_ASSERTION(compare >= 0, j, "The leafentry is < the upper-bound pivot"); + VERIFY_ASSERTION_BASEMENT(compare >= 0, i, j, "The leafentry is < the upper-bound pivot"); } if (0 < j) { DBT prev_key_dbt = get_ith_key_dbt(bn, j-1); int compare = compare_pairs(ft_handle, &prev_key_dbt, &kdbt); - VERIFY_ASSERTION(compare < 0, j, "Adjacent leafentries are out of order"); + VERIFY_ASSERTION_BASEMENT(compare < 0, i, j, "Adjacent leafentries are out of order"); } } } diff --git a/storage/tokudb/ft-index/ft/ft.cc b/storage/tokudb/ft-index/ft/ft.cc index bf99646351a..77948dfe358 100644 --- a/storage/tokudb/ft-index/ft/ft.cc +++ b/storage/tokudb/ft-index/ft/ft.cc @@ -1064,6 +1064,11 @@ garbage_helper(BLOCKNUM blocknum, int64_t UU(size), int64_t UU(address), void *e goto exit; } } + { + float a = info->used_space, b=info->total_space; + float percentage = (1 - (a / b)) * 100; + printf("LeafNode# %d has %d BasementNodes and %2.1f%% of the allocated space is garbage\n", (int)blocknum.b, node->n_children, percentage); + } exit: toku_ftnode_free(&node); toku_free(ndd); diff --git a/storage/tokudb/ft-index/ft/logger/recover.cc b/storage/tokudb/ft-index/ft/logger/recover.cc index 8dd7bf87624..680485201da 100644 --- a/storage/tokudb/ft-index/ft/logger/recover.cc +++ b/storage/tokudb/ft-index/ft/logger/recover.cc @@ -111,7 +111,8 @@ int tokuft_recovery_trace = 0; // turn on recovery tracing, d #endif // time in seconds between recovery progress reports -#define TOKUDB_RECOVERY_PROGRESS_TIME 15 +#define TOKUFT_RECOVERY_PROGRESS_TIME 15 +time_t tokuft_recovery_progress_time = TOKUFT_RECOVERY_PROGRESS_TIME; enum ss { BACKWARD_NEWER_CHECKPOINT_END = 1, @@ -325,14 +326,12 @@ static int recover_env_init (RECOVER_ENV renv, } static void recover_env_cleanup (RECOVER_ENV renv) { - int r; - invariant_zero(renv->fmap.filenums->size()); file_map_destroy(&renv->fmap); if (renv->destroy_logger_at_end) { toku_logger_close_rollback(renv->logger); - r = toku_logger_close(&renv->logger); + int r = toku_logger_close(&renv->logger); assert(r == 0); } else { toku_logger_write_log_files(renv->logger, true); @@ -714,7 +713,7 @@ static int toku_recover_xstillopenprepared (struct logtype_xstillopenprepared *l } switch (renv->ss.ss) { case FORWARD_BETWEEN_CHECKPOINT_BEGIN_END: { - toku_txn_prepare_txn(txn, l->xa_xid); + toku_txn_prepare_txn(txn, l->xa_xid, 0); break; } case FORWARD_NEWER_CHECKPOINT_END: { @@ -749,6 +748,36 @@ static int toku_recover_backward_xbegin (struct logtype_xbegin *UU(l), RECOVER_E return 0; } +struct toku_txn_progress_extra { + time_t tlast; + LSN lsn; + const char *type; + TXNID_PAIR xid; + uint64_t last_total; +}; + +static void toku_recover_txn_progress(TOKU_TXN_PROGRESS txn_progress, void *extra) { + toku_txn_progress_extra *txn_progress_extra = static_cast<toku_txn_progress_extra *>(extra); + if (txn_progress_extra->last_total == 0) + txn_progress_extra->last_total = txn_progress->entries_total; + else + assert(txn_progress_extra->last_total == txn_progress->entries_total); + time_t tnow = time(NULL); + if (tnow - txn_progress_extra->tlast >= tokuft_recovery_progress_time) { + txn_progress_extra->tlast = tnow; + fprintf(stderr, "%.24s TokuFT ", ctime(&tnow)); + if (txn_progress_extra->lsn.lsn != 0) + fprintf(stderr, "lsn %" PRIu64 " ", txn_progress_extra->lsn.lsn); + fprintf(stderr, "%s xid %" PRIu64 ":%" PRIu64 " ", + txn_progress_extra->type, txn_progress_extra->xid.parent_id64, txn_progress_extra->xid.child_id64); + fprintf(stderr, "%" PRIu64 "/%" PRIu64 " ", + txn_progress->entries_processed, txn_progress->entries_total); + if (txn_progress->entries_total > 0) + fprintf(stderr, "%.0f%% ", ((double) txn_progress->entries_processed / (double) txn_progress->entries_total) * 100.0); + fprintf(stderr, "\n"); + } +} + static int toku_recover_xcommit (struct logtype_xcommit *l, RECOVER_ENV renv) { // find the transaction by transaction id TOKUTXN txn = NULL; @@ -756,8 +785,8 @@ static int toku_recover_xcommit (struct logtype_xcommit *l, RECOVER_ENV renv) { assert(txn!=NULL); // commit the transaction - int r = toku_txn_commit_with_lsn(txn, true, l->lsn, - NULL, NULL); + toku_txn_progress_extra extra = { time(NULL), l->lsn, "commit", l->xid }; + int r = toku_txn_commit_with_lsn(txn, true, l->lsn, toku_recover_txn_progress, &extra); assert(r == 0); // close the transaction @@ -778,7 +807,7 @@ static int toku_recover_xprepare (struct logtype_xprepare *l, RECOVER_ENV renv) assert(txn!=NULL); // Save the transaction - toku_txn_prepare_txn(txn, l->xa_xid); + toku_txn_prepare_txn(txn, l->xa_xid, 0); return 0; } @@ -799,7 +828,8 @@ static int toku_recover_xabort (struct logtype_xabort *l, RECOVER_ENV renv) { assert(txn!=NULL); // abort the transaction - r = toku_txn_abort_with_lsn(txn, l->lsn, NULL, NULL); + toku_txn_progress_extra extra = { time(NULL), l->lsn, "abort", l->xid }; + r = toku_txn_abort_with_lsn(txn, l->lsn, toku_recover_txn_progress, &extra); assert(r == 0); // close the transaction @@ -1301,7 +1331,6 @@ static int is_txn_unprepared(TOKUTXN txn, void* extra) { return 0; } - static int find_an_unprepared_txn (RECOVER_ENV renv, TOKUTXN *txnp) { TOKUTXN txn = nullptr; int r = toku_txn_manager_iter_over_live_root_txns( @@ -1326,6 +1355,7 @@ static int call_prepare_txn_callback_iter(TOKUTXN txn, void* extra) { } static void recover_abort_live_txn(TOKUTXN txn) { + fprintf(stderr, "%s %" PRIu64 "\n", __FUNCTION__, txn->txnid.parent_id64); // recursively abort all children first if (txn->child != NULL) { recover_abort_live_txn(txn->child); @@ -1333,7 +1363,8 @@ static void recover_abort_live_txn(TOKUTXN txn) { // sanity check that the recursive call successfully NULLs out txn->child invariant(txn->child == NULL); // abort the transaction - int r = toku_txn_abort_txn(txn, NULL, NULL); + toku_txn_progress_extra extra = { time(NULL), ZERO_LSN, "abort live", txn->txnid }; + int r = toku_txn_abort_txn(txn, toku_recover_txn_progress, &extra); assert(r == 0); // close the transaction @@ -1451,9 +1482,10 @@ static int do_recovery(RECOVER_ENV renv, const char *env_dir, const char *log_di // trace progress if ((i % 1000) == 0) { tnow = time(NULL); - if (tnow - tlast >= TOKUDB_RECOVERY_PROGRESS_TIME) { + if (tnow - tlast >= tokuft_recovery_progress_time) { thislsn = toku_log_entry_get_lsn(le); - fprintf(stderr, "%.24s TokuFT recovery scanning backward from %" PRIu64 " at %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, recover_state(renv)); + fprintf(stderr, "%.24s TokuFT recovery scanning backward from %" PRIu64 " at %" PRIu64 " (%s)\n", + ctime(&tnow), lastlsn.lsn, thislsn.lsn, recover_state(renv)); tlast = tnow; } } @@ -1482,16 +1514,18 @@ static int do_recovery(RECOVER_ENV renv, const char *env_dir, const char *log_di assert(le); thislsn = toku_log_entry_get_lsn(le); tnow = time(NULL); - fprintf(stderr, "%.24s TokuFT recovery starts scanning forward to %" PRIu64 " from %" PRIu64 " left %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv)); + fprintf(stderr, "%.24s TokuFT recovery starts scanning forward to %" PRIu64 " from %" PRIu64 " left %" PRIu64 " (%s)\n", + ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv)); for (unsigned i=0; 1; i++) { // trace progress if ((i % 1000) == 0) { tnow = time(NULL); - if (tnow - tlast >= TOKUDB_RECOVERY_PROGRESS_TIME) { + if (tnow - tlast >= tokuft_recovery_progress_time) { thislsn = toku_log_entry_get_lsn(le); - fprintf(stderr, "%.24s TokuFT recovery scanning forward to %" PRIu64 " at %" PRIu64 " left %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv)); + fprintf(stderr, "%.24s TokuFT recovery scanning forward to %" PRIu64 " at %" PRIu64 " left %" PRIu64 " (%s)\n", + ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv)); tlast = tnow; } } diff --git a/storage/tokudb/ft-index/ft/txn/txn.cc b/storage/tokudb/ft-index/ft/txn/txn.cc index 216cb0d8dfd..922c955a6b5 100644 --- a/storage/tokudb/ft-index/ft/txn/txn.cc +++ b/storage/tokudb/ft-index/ft/txn/txn.cc @@ -344,6 +344,7 @@ static txn_child_manager tcm; .state = TOKUTXN_LIVE, .num_pin = 0, .client_id = 0, + .start_time = time(NULL), }; TOKUTXN result = NULL; @@ -558,7 +559,7 @@ static void copy_xid (TOKU_XA_XID *dest, TOKU_XA_XID *source) { memcpy(dest->data, source->data, source->gtrid_length+source->bqual_length); } -void toku_txn_prepare_txn (TOKUTXN txn, TOKU_XA_XID *xa_xid) { +void toku_txn_prepare_txn (TOKUTXN txn, TOKU_XA_XID *xa_xid, int nosync) { if (txn->parent || toku_txn_is_read_only(txn)) { // We do not prepare children. // @@ -573,7 +574,7 @@ void toku_txn_prepare_txn (TOKUTXN txn, TOKU_XA_XID *xa_xid) { txn->state = TOKUTXN_PREPARING; toku_txn_unlock_state(txn); // Do we need to do an fsync? - txn->do_fsync = (txn->force_fsync_on_commit || txn->roll_info.num_rollentries>0); + txn->do_fsync = txn->force_fsync_on_commit || (!nosync && txn->roll_info.num_rollentries>0); copy_xid(&txn->xa_xid, xa_xid); // This list will go away with #4683, so we wn't need the ydb lock for this anymore. toku_log_xprepare(txn->logger, &txn->do_fsync_lsn, 0, txn, txn->txnid, xa_xid); @@ -787,6 +788,10 @@ void toku_txn_set_client_id(TOKUTXN txn, uint64_t client_id) { txn->client_id = client_id; } +time_t toku_txn_get_start_time(struct tokutxn *txn) { + return txn->start_time; +} + int toku_txn_reads_txnid(TXNID txnid, TOKUTXN txn) { int r = 0; TXNID oldest_live_in_snapshot = toku_get_oldest_in_live_root_txn_list(txn); diff --git a/storage/tokudb/ft-index/ft/txn/txn.h b/storage/tokudb/ft-index/ft/txn/txn.h index c458df3b5b2..4f2778bf858 100644 --- a/storage/tokudb/ft-index/ft/txn/txn.h +++ b/storage/tokudb/ft-index/ft/txn/txn.h @@ -253,6 +253,7 @@ struct tokutxn { uint32_t num_pin; // number of threads (all hot indexes) that want this // txn to not transition to commit or abort uint64_t client_id; + time_t start_time; }; typedef struct tokutxn *TOKUTXN; @@ -302,7 +303,7 @@ int toku_txn_abort_with_lsn(struct tokutxn *txn, LSN oplsn, int toku_txn_discard_txn(struct tokutxn *txn); -void toku_txn_prepare_txn (struct tokutxn *txn, TOKU_XA_XID *xid); +void toku_txn_prepare_txn (struct tokutxn *txn, TOKU_XA_XID *xid, int nosync); // Effect: Do the internal work of preparing a transaction (does not log the prepare record). void toku_txn_get_prepared_xa_xid(struct tokutxn *txn, TOKU_XA_XID *xa_xid); @@ -368,6 +369,8 @@ bool toku_txn_has_spilled_rollback(struct tokutxn *txn); uint64_t toku_txn_get_client_id(struct tokutxn *txn); void toku_txn_set_client_id(struct tokutxn *txn, uint64_t client_id); +time_t toku_txn_get_start_time(struct tokutxn *txn); + // // This function is used by the leafentry iterators. // returns TOKUDB_ACCEPT if live transaction context is allowed to read a value diff --git a/storage/tokudb/ft-index/src/tests/recover-5146.cc b/storage/tokudb/ft-index/src/tests/recover-5146.cc index c05f9effa7d..5e793830f1d 100644 --- a/storage/tokudb/ft-index/src/tests/recover-5146.cc +++ b/storage/tokudb/ft-index/src/tests/recover-5146.cc @@ -147,7 +147,7 @@ run_test(void) { uint8_t gid[DB_GID_SIZE]; memset(gid, 0, DB_GID_SIZE); gid[0]=42; - r = txn->prepare(txn, gid); CKERR(r); + r = txn->prepare(txn, gid, 0); CKERR(r); } r = env->txn_checkpoint(env, 0, 0, 0); CKERR(r); diff --git a/storage/tokudb/ft-index/src/tests/test-prepare.cc b/storage/tokudb/ft-index/src/tests/test-prepare.cc index 9033c633ea7..73d2105fa86 100644 --- a/storage/tokudb/ft-index/src/tests/test-prepare.cc +++ b/storage/tokudb/ft-index/src/tests/test-prepare.cc @@ -122,7 +122,7 @@ static void setup_env_and_prepare (DB_ENV **envp, const char *envdir, bool commi uint8_t gid[DB_GID_SIZE]; memset(gid, 0, DB_GID_SIZE); gid[0]=42; - CKERR(txn->prepare(txn, gid)); + CKERR(txn->prepare(txn, gid, 0)); if (commit) CKERR(txn->commit(txn, 0)); } diff --git a/storage/tokudb/ft-index/src/tests/test-prepare2.cc b/storage/tokudb/ft-index/src/tests/test-prepare2.cc index 8952f14cf31..12d78e4da7c 100644 --- a/storage/tokudb/ft-index/src/tests/test-prepare2.cc +++ b/storage/tokudb/ft-index/src/tests/test-prepare2.cc @@ -124,7 +124,7 @@ static void setup_env_and_prepare (DB_ENV **envp, const char *envdir, bool commi uint8_t gid[DB_GID_SIZE]; memset(gid, 0, DB_GID_SIZE); gid[0]=42; - CKERR(txn->prepare(txn, gid)); + CKERR(txn->prepare(txn, gid, 0)); if (commit) CKERR(txn->commit(txn, 0)); } diff --git a/storage/tokudb/ft-index/src/tests/test-prepare3.cc b/storage/tokudb/ft-index/src/tests/test-prepare3.cc index 3643d73f41a..30c5c728e94 100644 --- a/storage/tokudb/ft-index/src/tests/test-prepare3.cc +++ b/storage/tokudb/ft-index/src/tests/test-prepare3.cc @@ -147,7 +147,7 @@ static void setup_env_and_prepare (DB_ENV **envp, const char *envdir) { uint8_t gid[DB_GID_SIZE]; memset(gid, 0, DB_GID_SIZE); gid[0]='a'+tnum; - CKERR(txn->prepare(txn, gid)); + CKERR(txn->prepare(txn, gid, 0)); // Drop txn on the ground, since we will commit or abort it after recovery if (tnum==0) { //printf("commit %d\n", tnum); diff --git a/storage/tokudb/ft-index/src/tests/test-xa-prepare.cc b/storage/tokudb/ft-index/src/tests/test-xa-prepare.cc index e08e7361555..d3b1352f55d 100644 --- a/storage/tokudb/ft-index/src/tests/test-xa-prepare.cc +++ b/storage/tokudb/ft-index/src/tests/test-xa-prepare.cc @@ -125,7 +125,7 @@ static void setup_env_and_prepare (DB_ENV **envp, const char *envdir, bool commi .gtrid_length = 8, .bqual_length = 9}; for (int i=0; i<8+9; i++) x.data[i] = 42+i; - CKERR(txn->xa_prepare(txn, &x)); + CKERR(txn->xa_prepare(txn, &x, 0)); if (commit) CKERR(txn->commit(txn, 0)); } diff --git a/storage/tokudb/ft-index/src/tests/test5092.cc b/storage/tokudb/ft-index/src/tests/test5092.cc index 16652472b55..8ef31ae0340 100644 --- a/storage/tokudb/ft-index/src/tests/test5092.cc +++ b/storage/tokudb/ft-index/src/tests/test5092.cc @@ -118,7 +118,7 @@ static void setup_env_and_prepare (DB_ENV **envp, const char *envdir, bool commi uint8_t gid[DB_GID_SIZE]; memset(gid, 0, DB_GID_SIZE); gid[0]=42; - CKERR(txn->prepare(txn, gid)); + CKERR(txn->prepare(txn, gid, 0)); { int chk_r = db->close(db, 0); CKERR(chk_r); } if (commit) CKERR(txn->commit(txn, 0)); diff --git a/storage/tokudb/ft-index/src/tests/test_5015.cc b/storage/tokudb/ft-index/src/tests/test_5015.cc index 071b7f3660e..50900d342d0 100644 --- a/storage/tokudb/ft-index/src/tests/test_5015.cc +++ b/storage/tokudb/ft-index/src/tests/test_5015.cc @@ -133,7 +133,7 @@ test_main(int argc, char *const argv[]) { uint8_t gid[DB_GID_SIZE]; memset(gid, 0, DB_GID_SIZE); gid[0]='a'; - r = child_txn->prepare(child_txn, gid); + r = child_txn->prepare(child_txn, gid, 0); CKERR(r); r = env->txn_checkpoint(env, 0, 0, 0); diff --git a/storage/tokudb/ft-index/src/tests/test_cursor_interrupt.cc b/storage/tokudb/ft-index/src/tests/test_cursor_interrupt.cc index e992f86455b..5583b0283a6 100644 --- a/storage/tokudb/ft-index/src/tests/test_cursor_interrupt.cc +++ b/storage/tokudb/ft-index/src/tests/test_cursor_interrupt.cc @@ -95,12 +95,12 @@ PATENT RIGHTS GRANT: int num_interrupts_called; -static bool interrupt(void* extra UU()) { +static bool interrupt(void* extra UU(), uint64_t rows UU()) { num_interrupts_called++; return false; } -static bool interrupt_true(void* extra UU()) { +static bool interrupt_true(void* extra UU(), uint64_t rows UU()) { num_interrupts_called++; return true; } diff --git a/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc b/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc index dd00ddeeb9a..c104c5c8541 100644 --- a/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc +++ b/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc @@ -104,9 +104,11 @@ struct iterate_extra { bool visited_txn[3]; }; -static int iterate_callback(uint64_t txnid, uint64_t client_id, +static int iterate_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { + uint64_t txnid = txn->id64(txn); + uint64_t client_id = txn->get_client_id(txn); iterate_extra *info = reinterpret_cast<iterate_extra *>(extra); DB *db; DBT left_key, right_key; diff --git a/storage/tokudb/ft-index/src/tests/test_stress0.cc b/storage/tokudb/ft-index/src/tests/test_stress0.cc index 5dbca08db48..26192d851aa 100644 --- a/storage/tokudb/ft-index/src/tests/test_stress0.cc +++ b/storage/tokudb/ft-index/src/tests/test_stress0.cc @@ -140,9 +140,11 @@ static int UU() iterate_pending_lock_requests_op(DB_TXN *UU(txn), ARG arg, void return r; } -static int iterate_txns(uint64_t txnid, uint64_t client_id, +static int iterate_txns(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { + uint64_t txnid = txn->id64(txn); + uint64_t client_id = txn->get_client_id(txn); invariant_null(extra); invariant(txnid > 0); invariant(client_id == 0); diff --git a/storage/tokudb/ft-index/src/tests/test_stress_hot_indexing.cc b/storage/tokudb/ft-index/src/tests/test_stress_hot_indexing.cc index fe237f063ab..d70f0411e01 100644 --- a/storage/tokudb/ft-index/src/tests/test_stress_hot_indexing.cc +++ b/storage/tokudb/ft-index/src/tests/test_stress_hot_indexing.cc @@ -211,7 +211,7 @@ cleanup: gid_count++; uint32_t *hi_gid_count_p = cast_to_typeof(hi_gid_count_p) hi_gid; // make gcc --happy about -Wstrict-aliasing *hi_gid_count_p = gid_count; - int rr = hi_txn->prepare(hi_txn, hi_gid); + int rr = hi_txn->prepare(hi_txn, hi_gid, 0); CKERR(rr); if (r || (random() % 2)) { rr = hi_txn->abort(hi_txn); diff --git a/storage/tokudb/ft-index/src/tests/test_txn_close_before_prepare_commit.cc b/storage/tokudb/ft-index/src/tests/test_txn_close_before_prepare_commit.cc index e3b715c4ce7..31267b17686 100644 --- a/storage/tokudb/ft-index/src/tests/test_txn_close_before_prepare_commit.cc +++ b/storage/tokudb/ft-index/src/tests/test_txn_close_before_prepare_commit.cc @@ -130,7 +130,7 @@ test_txn_close_before_prepare_commit (void) { uint8_t gid[DB_GID_SIZE]; memset(gid, 1, DB_GID_SIZE); - r = txn->prepare(txn, gid); assert(r == 0); + r = txn->prepare(txn, gid, 0); assert(r == 0); r = txn->commit(txn, 0); assert(r == 0); r = env->close(env, 0); assert(r == 0); diff --git a/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h b/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h index 2c2525a3165..f9da1693847 100644 --- a/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h +++ b/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h @@ -593,7 +593,7 @@ static void *worker(void *arg_v) { uint64_t gid_val = txn->id64(txn); uint64_t *gid_count_p = cast_to_typeof(gid_count_p) gid; // make gcc --happy about -Wstrict-aliasing *gid_count_p = gid_val; - int rr = txn->prepare(txn, gid); + int rr = txn->prepare(txn, gid, 0); assert_zero(rr); } if (r == 0) { diff --git a/storage/tokudb/ft-index/src/tests/xa-dirty-commit.cc b/storage/tokudb/ft-index/src/tests/xa-dirty-commit.cc index 126a7c1453e..008a41274ae 100644 --- a/storage/tokudb/ft-index/src/tests/xa-dirty-commit.cc +++ b/storage/tokudb/ft-index/src/tests/xa-dirty-commit.cc @@ -136,7 +136,7 @@ static void create_prepared_txn(void) { for (int i = 0; i < 8+9; i++) { xid.data[i] = i; } - r = txn->xa_prepare(txn, &xid); + r = txn->xa_prepare(txn, &xid, 0); CKERR(r); // discard the txn so that we can close the env and run xa recovery later diff --git a/storage/tokudb/ft-index/src/tests/xa-dirty-rollback.cc b/storage/tokudb/ft-index/src/tests/xa-dirty-rollback.cc index 2d13e559050..3f0debe4116 100644 --- a/storage/tokudb/ft-index/src/tests/xa-dirty-rollback.cc +++ b/storage/tokudb/ft-index/src/tests/xa-dirty-rollback.cc @@ -136,7 +136,7 @@ static void create_prepared_txn(void) { for (int i = 0; i < 8+9; i++) { xid.data[i] = i; } - r = txn->xa_prepare(txn, &xid); + r = txn->xa_prepare(txn, &xid, 0); CKERR(r); // discard the txn so that we can close the env and run xa recovery later diff --git a/storage/tokudb/ft-index/src/tests/xa-txn-discard-abort.cc b/storage/tokudb/ft-index/src/tests/xa-txn-discard-abort.cc index 3365a1bb139..95852c68192 100644 --- a/storage/tokudb/ft-index/src/tests/xa-txn-discard-abort.cc +++ b/storage/tokudb/ft-index/src/tests/xa-txn-discard-abort.cc @@ -138,7 +138,7 @@ static void create_prepared_txn(void) { for (int i = 0; i < 8+9; i++) { xid.data[i] = i; } - r = txn->xa_prepare(txn, &xid); + r = txn->xa_prepare(txn, &xid, 0); CKERR(r); // discard the txn so that we can close the env and run xa recovery later diff --git a/storage/tokudb/ft-index/src/tests/xa-txn-discard-commit.cc b/storage/tokudb/ft-index/src/tests/xa-txn-discard-commit.cc index c4d164017ae..a21f8813f90 100644 --- a/storage/tokudb/ft-index/src/tests/xa-txn-discard-commit.cc +++ b/storage/tokudb/ft-index/src/tests/xa-txn-discard-commit.cc @@ -139,7 +139,7 @@ static void create_prepared_txn(void) { for (int i = 0; i < 8+9; i++) { xid.data[i] = i; } - r = txn->xa_prepare(txn, &xid); + r = txn->xa_prepare(txn, &xid, 0); CKERR(r); // discard the txn so that we can close the env and run xa recovery later diff --git a/storage/tokudb/ft-index/src/ydb.cc b/storage/tokudb/ft-index/src/ydb.cc index 85445a67eef..ab15a44489e 100644 --- a/storage/tokudb/ft-index/src/ydb.cc +++ b/storage/tokudb/ft-index/src/ydb.cc @@ -2492,24 +2492,21 @@ struct iter_txns_callback_extra { }; static int iter_txns_callback(TOKUTXN txn, void *extra) { + int r = 0; iter_txns_callback_extra *info = reinterpret_cast<iter_txns_callback_extra *>(extra); - DB_TXN *dbtxn = toku_txn_get_container_db_txn(txn); invariant_notnull(dbtxn); + if (db_txn_struct_i(dbtxn)->tokutxn == txn) { // make sure that the dbtxn is fully initialized + toku_mutex_lock(&db_txn_struct_i(dbtxn)->txn_mutex); + toku_pthread_rwlock_rdlock(&info->env->i->open_dbs_rwlock); - toku_mutex_lock(&db_txn_struct_i(dbtxn)->txn_mutex); - toku_pthread_rwlock_rdlock(&info->env->i->open_dbs_rwlock); - - iter_txn_row_locks_callback_extra e(info->env, &db_txn_struct_i(dbtxn)->lt_map); - const int r = info->callback(toku_txn_get_txnid(txn).parent_id64, - toku_txn_get_client_id(txn), - iter_txn_row_locks_callback, - &e, - info->extra); + iter_txn_row_locks_callback_extra e(info->env, &db_txn_struct_i(dbtxn)->lt_map); + r = info->callback(dbtxn, iter_txn_row_locks_callback, &e, info->extra); - toku_pthread_rwlock_rdunlock(&info->env->i->open_dbs_rwlock); - toku_mutex_unlock(&db_txn_struct_i(dbtxn)->txn_mutex); + toku_pthread_rwlock_rdunlock(&info->env->i->open_dbs_rwlock); + toku_mutex_unlock(&db_txn_struct_i(dbtxn)->txn_mutex); + } return r; } diff --git a/storage/tokudb/ft-index/src/ydb_cursor.cc b/storage/tokudb/ft-index/src/ydb_cursor.cc index 57f3b5808b6..eed519455fc 100644 --- a/storage/tokudb/ft-index/src/ydb_cursor.cc +++ b/storage/tokudb/ft-index/src/ydb_cursor.cc @@ -769,7 +769,7 @@ c_remove_restriction(DBC *dbc) { } static void -c_set_check_interrupt_callback(DBC* dbc, bool (*interrupt_callback)(void*), void *extra) { +c_set_check_interrupt_callback(DBC* dbc, bool (*interrupt_callback)(void*, uint64_t), void *extra) { toku_ft_cursor_set_check_interrupt_cb(dbc_ftcursor(dbc), interrupt_callback, extra); } diff --git a/storage/tokudb/ft-index/src/ydb_txn.cc b/storage/tokudb/ft-index/src/ydb_txn.cc index ce06e78b23f..885c6b713b2 100644 --- a/storage/tokudb/ft-index/src/ydb_txn.cc +++ b/storage/tokudb/ft-index/src/ydb_txn.cc @@ -240,7 +240,7 @@ static int toku_txn_abort(DB_TXN * txn, return r; } -static int toku_txn_xa_prepare (DB_TXN *txn, TOKU_XA_XID *xid) { +static int toku_txn_xa_prepare (DB_TXN *txn, TOKU_XA_XID *xid, uint32_t flags) { int r = 0; if (!txn) { r = EINVAL; @@ -273,9 +273,11 @@ static int toku_txn_xa_prepare (DB_TXN *txn, TOKU_XA_XID *xid) { HANDLE_PANICKED_ENV(txn->mgrp); } assert(!db_txn_struct_i(txn)->child); + int nosync; + nosync = (flags & DB_TXN_NOSYNC)!=0 || (db_txn_struct_i(txn)->flags&DB_TXN_NOSYNC); TOKUTXN ttxn; ttxn = db_txn_struct_i(txn)->tokutxn; - toku_txn_prepare_txn(ttxn, xid); + toku_txn_prepare_txn(ttxn, xid, nosync); TOKULOGGER logger; logger = txn->mgrp->i->logger; LSN do_fsync_lsn; @@ -292,14 +294,14 @@ exit: // requires: must hold the multi operation lock. it is // released in toku_txn_xa_prepare before the fsync. -static int toku_txn_prepare (DB_TXN *txn, uint8_t gid[DB_GID_SIZE]) { +static int toku_txn_prepare (DB_TXN *txn, uint8_t gid[DB_GID_SIZE], uint32_t flags) { TOKU_XA_XID xid; TOKU_ANNOTATE_NEW_MEMORY(&xid, sizeof(xid)); xid.formatID=0x756b6f54; // "Toku" xid.gtrid_length=DB_GID_SIZE/2; // The maximum allowed gtrid length is 64. See the XA spec in source:/import/opengroup.org/C193.pdf page 20. xid.bqual_length=DB_GID_SIZE/2; // The maximum allowed bqual length is 64. memcpy(xid.data, gid, DB_GID_SIZE); - return toku_txn_xa_prepare(txn, &xid); + return toku_txn_xa_prepare(txn, &xid, flags); } static int toku_txn_txn_stat (DB_TXN *txn, struct txn_stat **txn_stat) { @@ -427,6 +429,15 @@ static bool toku_txn_is_prepared(DB_TXN *txn) { return toku_txn_get_state(ttxn) == TOKUTXN_PREPARING; } +static DB_TXN *toku_txn_get_child(DB_TXN *txn) { + return db_txn_struct_i(txn)->child; +} + +static uint64_t toku_txn_get_start_time(DB_TXN *txn) { + TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn; + return toku_txn_get_start_time(ttxn); +} + static inline void txn_func_init(DB_TXN *txn) { #define STXN(name) txn->name = locked_txn_ ## name STXN(abort); @@ -444,6 +455,8 @@ static inline void txn_func_init(DB_TXN *txn) { #undef SUTXN txn->id64 = toku_txn_id64; txn->is_prepared = toku_txn_is_prepared; + txn->get_child = toku_txn_get_child; + txn->get_start_time = toku_txn_get_start_time; } // diff --git a/storage/tokudb/ft-index/tools/CMakeLists.txt b/storage/tokudb/ft-index/tools/CMakeLists.txt index 4ed0cb4cbdc..f745517d84e 100644 --- a/storage/tokudb/ft-index/tools/CMakeLists.txt +++ b/storage/tokudb/ft-index/tools/CMakeLists.txt @@ -1,6 +1,6 @@ set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE DONT_DEPRECATE_ERRNO) -set(tools tokudb_dump tokuftdump tdb_logprint tdb-recover ftverify ba_replay) +set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify ba_replay) foreach(tool ${tools}) add_executable(${tool} ${tool}.cc) add_dependencies(${tool} install_tdb_h) @@ -12,9 +12,6 @@ endforeach(tool) # link in math.h library just for this tool. target_link_libraries(ftverify m) -install( - TARGETS tokuftdump - DESTINATION bin - COMPONENT Server - ) +install(TARGETS tokuftdump DESTINATION ${INSTALL_BINDIR} COMPONENT Server) +install(TARGETS tokuft_logprint DESTINATION ${INSTALL_BINDIR} COMPONENT Server) diff --git a/storage/tokudb/ft-index/tools/tdb_logprint.cc b/storage/tokudb/ft-index/tools/tokuft_logprint.cc index 15a28632cfb..1dd7581b9f5 100644 --- a/storage/tokudb/ft-index/tools/tdb_logprint.cc +++ b/storage/tokudb/ft-index/tools/tokuft_logprint.cc @@ -91,8 +91,6 @@ PATENT RIGHTS GRANT: /* Dump the log from stdin to stdout. */ -#include <config.h> - #include "ft/log_header.h" #include "ft/logger/logger.h" diff --git a/storage/tokudb/ft-index/tools/tokuftdump.cc b/storage/tokudb/ft-index/tools/tokuftdump.cc index 3f73136fb5c..14c3c31a061 100644 --- a/storage/tokudb/ft-index/tools/tokuftdump.cc +++ b/storage/tokudb/ft-index/tools/tokuftdump.cc @@ -99,7 +99,10 @@ PATENT RIGHTS GRANT: #include <stdlib.h> #include <inttypes.h> #include <limits.h> - +#include <string> +#include <iostream> +#include <fstream> +#include <string.h> #include "ft/serialize/block_table.h" #include "ft/cachetable/cachetable.h" #include "ft/ft.h" @@ -108,8 +111,11 @@ PATENT RIGHTS GRANT: #include "ft/serialize/ft_node-serialize.h" #include "ft/node.h" +using namespace std; + static int do_dump_data = 1; static int do_interactive = 0; +static int do_json = 0; static int do_header = 0; static int do_fragmentation = 0; static int do_garbage = 0; @@ -118,10 +124,24 @@ static int do_rootnode = 0; static int do_node = 0; static BLOCKNUM do_node_num; static int do_tsv = 0; - static const char *arg0; static const char *fname; +//it holdes the messges count for each FT's node +typedef struct nodeMessage{ + int id; + int clean;//0=clean >=1 dirty + int *count;//holds the messages + nodeMessage *nextNode; +}NMC; +enum { maxline = 128}; + +static int printNodeMessagesToSTDout(NMC* ptr); + +static int printLevelSTDout(int *); + +static void treeToSTDout(NMC *msgs[], int height); + static void format_time(const uint64_t time_int, char *buf) { time_t timer = (time_t) time_int; ctime_r(&timer, buf); @@ -225,12 +245,204 @@ static void dump_header(FT ft) { printf(" estimated numbytes=%" PRId64 "\n", ft->in_memory_stats.numbytes); } +static int64_t getRootNode(FT ft) { + return ft->h->root_blocknum.b; +} + static int print_le(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t idx UU(), void *const ai UU()) { + unsigned int *le_index = (unsigned int *) ai; + printf("%u: ", *le_index); *le_index += 1; print_klpair(stdout, key, keylen, le); printf("\n"); return 0; } +static int getHeight(int fd, BLOCKNUM blocknum, FT ft){ + FTNODE n; + FTNODE_DISK_DATA ndd = nullptr; + ftnode_fetch_extra bfe; + bfe.create_for_full_read(ft); + int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe); + assert_zero(r); + assert(n!=0); + return n->height; +} + +static FTNODE getNode(int fd, BLOCKNUM blocknum, FT ft) { + FTNODE n; + FTNODE_DISK_DATA ndd = nullptr; + ftnode_fetch_extra bfe; + bfe.create_for_full_read(ft); + int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe); + assert_zero(r);; + return n; +} + +static int countNodes(NMC *level){ + int count=0; + NMC *ptr=level; + while(ptr!=NULL){ + count++; + ptr=ptr->nextNode; + } + return count; +} + +static int * countMessages(NMC *level){ + int *counts=new int[16]; + for(int i=0;i<16;i++){ + counts[i]=0; + } + NMC *ptr=level; + while(ptr!=NULL){ + for(int i=0;i<16;i++){ + counts[i]+=ptr->count[i]; + } + ptr=ptr->nextNode; + } + return counts; +} + +static NMC * getLast(NMC *level){ + if (level==NULL) return NULL; + NMC *ptr=level; + while(ptr->nextNode!=NULL){ + ptr=ptr->nextNode; + } + return ptr; +} + +/* + * Prints the total messages at each to STDout + */ +static int printLevelSTDout(int *count){ + int isEmpty=0; + for(int j=0;j<16;j++){ + if(count[j]>0){ + cout <<count[j]<<" "; + isEmpty++; + switch (j) { + case FT_INSERT: printf("INSERT(s) "); break; + case FT_INSERT_NO_OVERWRITE: printf("INSERT_NO_OVERWRITE(s) "); break; + case FT_DELETE_ANY: printf("DELETE_ANY(s) "); break; + case FT_ABORT_ANY: printf("ABORT_ANY(s) "); break; + case FT_COMMIT_ANY: printf("COMMIT_ANY(s) "); break; + case FT_COMMIT_BROADCAST_ALL: printf("COMMIT_BROADCAST_ALL(s) "); break; + case FT_COMMIT_BROADCAST_TXN: printf("COMMIT_BROADCAST_TXN(s) "); break; + case FT_ABORT_BROADCAST_TXN: printf("ABORT_BROADCAST_TXN(s) "); break; + case FT_OPTIMIZE: printf("OPTIMIZE(s) "); break; + case FT_OPTIMIZE_FOR_UPGRADE: printf("OPTIMIZE_FOR_UPGRADE(s) "); break; + case FT_UPDATE: printf("UPDATE(s) "); break; + case FT_UPDATE_BROADCAST_ALL: printf("UPDATE_BROADCAST_ALL(s) "); break; + } + + } + } + return isEmpty; +} + +/* + * Prints the total # of messages in a node to STD output + */ +static int printNodeMessagesToSTDout(NMC *ptr){ + cout <<"\nNode :"<<ptr->id<<" has :"; + for(int j=0;j<16;j++){ + if(ptr->count[j]>0){ + cout <<ptr->count[j]<<" "; + switch (j) { + case FT_INSERT: printf("INSERT(s) "); break; + case FT_INSERT_NO_OVERWRITE: printf("INSERT_NO_OVERWRITE(s) "); break; + case FT_DELETE_ANY: printf("DELETE_ANY(s) "); break; + case FT_ABORT_ANY: printf("ABORT_ANY(s) "); break; + case FT_COMMIT_ANY: printf("COMMIT_ANY(s) "); break; + case FT_COMMIT_BROADCAST_ALL: printf("COMMIT_BROADCAST_ALL(s) "); break; + case FT_COMMIT_BROADCAST_TXN: printf("COMMIT_BROADCAST_TXN(s) "); break; + case FT_ABORT_BROADCAST_TXN: printf("ABORT_BROADCAST_TXN(s) "); break; + case FT_OPTIMIZE: printf("OPTIMIZE(s) "); break; + case FT_OPTIMIZE_FOR_UPGRADE: printf("OPTIMIZE_FOR_UPGRADE(s) "); break; + case FT_UPDATE: printf("UPDATE(s) "); break; + case FT_UPDATE_BROADCAST_ALL: printf("UPDATE_BROADCAST_ALL(s) "); break; + } + } + } + return 1; +} + +static void levelToSTDout(NMC *list, int level){ + NMC *ptr=list; + cout <<endl<<"Height : "<<level<<endl; + while(ptr!=NULL){ + if(ptr->clean!=0){ + printNodeMessagesToSTDout(ptr); + } + else{ + cout << "\nNode : "<<ptr->id<<" has no messages"; + } + ptr=ptr->nextNode; + } + cout <<endl; +} + +/* + * prints the tree total # of nodes and total # of messages at each height in : + * STDout in human readable format + */ +static void treeToSTDout(NMC *msgs[], int height){ + for(int i=height; i>=0 ; i--){ + cout <<"At height "<<i; + int *counts=countMessages(msgs[i]); + cout <<"\n Node Count: "<< countNodes(msgs[i])<<endl; + cout <<" Messages: "; + if(printLevelSTDout(counts)==0) cout <<"0\n"; + else cout <<endl; + } +} + +//traverse through the FT and report back the count of messages in every node +static void countMessagesInFT(int fd, BLOCKNUM blocknum, FT ft,NMC *msgs[]){ + FTNODE n=getNode(fd,blocknum,ft); + + NMC *last=NULL; + if(msgs[n->height]==NULL){ + last = msgs[n->height]=new NMC; + }else { + last=getLast(msgs[n->height]); + last->nextNode=new NMC; + last=last->nextNode; + } + last->id=blocknum.b; + last->count=new int[16]; + for(int i=0;i<16;i++){ + last->count[i]=0; + } + last->clean=0; + last->nextNode=NULL; + + if (n->height==0){ + toku_ftnode_free(&n); + return; + } + for(int i=0;i<n->n_children;i++){ + NONLEAF_CHILDINFO bnc = BNC(n, i); + if (n->height==1 && n->bp[i].ptr.tag==BCT_NULL){ + cout <<n->bp[i].ptr.tag; + } + auto dump_fn=[&](const ft_msg &msg, bool UU(is_fresh)) { + enum ft_msg_type type = (enum ft_msg_type) msg.type(); + last->count[type]++; + last->clean=1; + return 0; + }; + + bnc->msg_buffer.iterate(dump_fn); + + blocknum=make_blocknum(BP_BLOCKNUM(n, i).b); + countMessagesInFT(fd,blocknum,ft, msgs); + } + + toku_ftnode_free(&n); +} + static void dump_node(int fd, BLOCKNUM blocknum, FT ft) { FTNODE n; FTNODE_DISK_DATA ndd = nullptr; @@ -254,9 +466,9 @@ static void dump_node(int fd, BLOCKNUM blocknum, FT ft) { printf(" layout_version_read_from_disk=%d\n", n->layout_version_read_from_disk); printf(" build_id=%d\n", n->build_id); printf(" max_msn_applied_to_node_on_disk=%" PRId64 " (0x%" PRIx64 ")\n", n->max_msn_applied_to_node_on_disk.msn, n->max_msn_applied_to_node_on_disk.msn); - printf(" io time %lf decompress time %lf deserialize time %lf\n", - tokutime_to_seconds(bfe.io_time), - tokutime_to_seconds(bfe.decompress_time), + printf(" io time %lf decompress time %lf deserialize time %lf\n", + tokutime_to_seconds(bfe.io_time), + tokutime_to_seconds(bfe.decompress_time), tokutime_to_seconds(bfe.deserialize_time)); printf(" n_children=%d\n", n->n_children); @@ -277,7 +489,7 @@ static void dump_node(int fd, BLOCKNUM blocknum, FT ft) { if (n->height > 0) { printf("%" PRId64 "\n", BP_BLOCKNUM(n, i).b); NONLEAF_CHILDINFO bnc = BNC(n, i); - unsigned int n_bytes = toku_bnc_nbytesinbuf(bnc); + unsigned int n_bytes = toku_bnc_nbytesinbuf(bnc); int n_entries = toku_bnc_n_entries(bnc); if (n_bytes > 0 || n_entries > 0) { printf(" buffer contains %u bytes (%d items)\n", n_bytes, n_entries); @@ -329,7 +541,8 @@ ok: printf(" n_bytes_in_buffer= %" PRIu64 "", BLB_DATA(n, i)->get_disk_size()); printf(" items_in_buffer=%u\n", BLB_DATA(n, i)->num_klpairs()); if (do_dump_data) { - BLB_DATA(n, i)->iterate<void, print_le>(NULL); + unsigned int le_index = 0; + BLB_DATA(n, i)->iterate<void, print_le>(&le_index); } } } @@ -402,8 +615,12 @@ static void dump_garbage_stats(int fd, FT ft) { uint64_t total_space = 0; uint64_t used_space = 0; toku_ft_get_garbage(ft, &total_space, &used_space); - printf("garbage total size\t%" PRIu64 "\n", total_space); - printf("garbage used size\t%" PRIu64 "\n", used_space); + printf("garbage total size :%20" PRIu64 "\n", total_space); + printf("garbage used size :%20" PRIu64 "\n", used_space); + float a=used_space,b=total_space; + + float percentage=((1-a/b)*100); + printf("Total garbage : %2.3f%%\n", percentage); } typedef struct __dump_node_extra { @@ -438,7 +655,7 @@ static void sub_block_deserialize(struct dump_sub_block *sb, unsigned char *sub_ static void verify_block(unsigned char *cp, uint64_t file_offset, uint64_t size) { // verify the header checksum const size_t node_header = 8 + sizeof (uint32_t) + sizeof (uint32_t) + sizeof (uint32_t); - + printf("%.8s layout_version=%u %u build=%d\n", cp, get_unaligned_uint32(cp+8), get_unaligned_uint32(cp+12), get_unaligned_uint32(cp+16)); unsigned char *sub_block_header = &cp[node_header]; @@ -544,7 +761,9 @@ static uint64_t getuint64(const char *f) { static void interactive_help(void) { fprintf(stderr, "help\n"); fprintf(stderr, "header\n"); - fprintf(stderr, "node NUMBER\n"); + cout <<"mr/MessagesReport [NUMBER] \n Reports messages for the level of the tree you want get more details about\n"; + cout <<"rf/readFile ft-file-name \n Switch to a different FT\n"; + fprintf(stderr, "node NUMBER \n"); fprintf(stderr, "bx OFFSET | block_translation OFFSET\n"); fprintf(stderr, "dumpdata 0|1\n"); fprintf(stderr, "fragmentation\n"); @@ -554,10 +773,160 @@ static void interactive_help(void) { fprintf(stderr, "quit\n"); } +static void freeNMC(NMC *msgs[], int height){ + for(int i=0;i<height;i++){ + if(msgs[i]!=NULL){ + delete(msgs[i]->count); + + while(msgs[i]->nextNode!=NULL){ + NMC* ptr=msgs[i]->nextNode; + msgs[i]=msgs[i]->nextNode; + delete ptr; + + } + msgs[i]=NULL; + } + } +} + +static void writeTree(NMC *msgs[],int height,char *name UU()){ + ofstream mytree ("/tmp/tree.txt",fstream::out); + if (mytree.is_open()){ + for(int i=height;i>=0;i--){ + NMC * ptr=msgs[i]; + mytree <<i<<endl; + while(ptr!=NULL){ + mytree << ptr->id<<"\t"; + if(ptr->clean!=0)mytree << "1"<<"\t"; + else mytree << "0"<<"\t"; + for(int j=0;j<15;j++)mytree << ptr->count[j]<<" "; + mytree << ptr->count[i]<<endl; + ptr=ptr->nextNode; + } + mytree <<endl; + } + } + else cout << "Unable to open file"; + mytree.close(); +} + +static void writeJson(NMC *msgs[],int height,const char *name){ + ofstream mytree (name,fstream::out); + if (mytree.is_open()){ + mytree <<"{\n \"FT\":["; + for(int i=height;i>=0;i--){ + NMC * ptr=msgs[i]; + mytree <<"{\n\"Level\": {\"Height\":\""<<i<<"\",\n \"Nodes\":["; + while(ptr!=NULL){ + mytree <<"{\"ID\":\""<< ptr->id<<"\","; + if(ptr->clean!=0){ + mytree <<"\"Messages\":["; + for(int j=0;j<16;j++) + { + mytree <<"{"; + switch (j) { + case FT_INSERT: mytree <<"\"INSERT\":\""<<ptr->count[j]<<"\""; break; + case FT_INSERT_NO_OVERWRITE: mytree <<"\"INSERT_NOVERWTE\":\""<<ptr->count[j]<<"\""; break; + case FT_DELETE_ANY: mytree <<"\"DELETE\":\""<<ptr->count[j]<<"\""; break; + case FT_ABORT_ANY: mytree <<"\"ABORT\":\""<<ptr->count[j]<<"\""; break; + case FT_COMMIT_ANY: mytree <<"\"COMMITY\":\""<<ptr->count[j]<<"\""; break; + case FT_COMMIT_BROADCAST_ALL: mytree <<"\"COMMIT_BROADCAST_ALL\":\""<<ptr->count[j]<<"\"" ; break; + case FT_COMMIT_BROADCAST_TXN: mytree <<"\"COMMIT_BROADCAST_TXN\":\""<<ptr->count[j]<<"\""; break; + case FT_ABORT_BROADCAST_TXN: mytree <<"\"ABORT_BROADCAST_TXN\":\""<<ptr->count[j]<<"\"";break; + case FT_OPTIMIZE: mytree <<"\"OPTIMIZE\":\""<<ptr->count[j]<<"\""; break; + case FT_OPTIMIZE_FOR_UPGRADE: mytree <<"\"OPTIMIZE_FOR_UPGRADE\":\""<<ptr->count[j]<<"\"";break; + case FT_UPDATE: mytree <<"\"UPDATE\":\""<<ptr->count[j]<<"\""; break; + case FT_UPDATE_BROADCAST_ALL: mytree <<"\"UPDATE_BROADCAST_ALL\":\""<<ptr->count[j]<<"\""; break; + } + mytree <<"}"; + if(j<15)mytree<<","; + } + + mytree <<"]}"; + + } + else { + mytree <<"\"Messages\":\""<< "0"<<"\"}"; + } + if(ptr->nextNode!=NULL)mytree <<",\n"; + else mytree <<"]}\n"; + ptr=ptr->nextNode; + } + mytree <<"\n}\n"; + if(i!=0)mytree <<",\n"; + } + mytree <<"\n]}\n"; + + } + else cout << "Unable to open file"; + mytree.close(); +} + +static void writeTree(NMC *msgs[],int height){ + ofstream mytree ("/tmp/tree1.txt",fstream::out); + if (mytree.is_open()){ + for(int i=height;i>=0;i--){ + NMC * ptr=msgs[i]; + mytree <<i<<endl; + while(ptr!=NULL){ + mytree << ptr->id<<","; + if(ptr->clean!=0)mytree << "1"<<","; + else mytree << "0"<<","; + for(int j=0;j<15;j++)mytree << ptr->count[j]<<","; + mytree << ptr->count[i]<<endl; + ptr=ptr->nextNode; + } + mytree <<".\""; + } + } + else cout << "Unable to open file"; + mytree.close(); +} + +static void FT_to_JSON(int fd, FT ft, CACHEFILE cf, const char * JsonFile){ + toku_ft_free(ft); + open_header(fd, &ft, cf); + int root=getRootNode(ft); + BLOCKNUM off = make_blocknum(root); + int height=getHeight(fd,off, ft); + NMC *msgs[height]; + for(int i=0;i<=height;i++){ + msgs[i]=NULL; + } + open_header(fd, &ft, cf); + root=getRootNode(ft); + off = make_blocknum(root); + countMessagesInFT(fd,off, ft,msgs); + cout <<"to STD output: \n"; + treeToSTDout(msgs,height); + writeTree(msgs,height); + cout<<"FT's json file was generated here:"; + if(JsonFile!=NULL) { + cout <<JsonFile; + writeJson(msgs,height,JsonFile); + } + else { + cout <<"./FT.json"; + writeJson(msgs,height,"./FT.json"); + } + cout<<endl; + freeNMC(msgs,height); + exit(0); +} + static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) { + toku_ft_free(ft); + open_header(fd, &ft, cf); + int root=getRootNode(ft); + BLOCKNUM off = make_blocknum(root); + int height=getHeight(fd,off, ft); + NMC *msgs[height]; + for(int i=0;i<=height;i++){ + msgs[i]=NULL; + } while (1) { - printf("ftdump>"); fflush(stdout); - enum { maxline = 64}; + printf("ftdump>"); + fflush(stdout); char line[maxline+1]; int r = readline(line, maxline); if (r == EOF) @@ -565,7 +934,7 @@ static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) { const int maxfields = 4; char *fields[maxfields]; int nfields = split_fields(line, fields, maxfields); - if (nfields == 0) + if (nfields == 0) continue; if (strcmp(fields[0], "help") == 0) { interactive_help(); @@ -573,15 +942,50 @@ static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) { toku_ft_free(ft); open_header(fd, &ft, cf); dump_header(ft); + } else if (strcmp(fields[0], "rn") == 0||strcmp(fields[0], "rootNode")==0||strcmp(fields[0], "rootnode") == 0) { + printf("Root node :%d\n",root); } else if (strcmp(fields[0], "block") == 0 && nfields == 2) { BLOCKNUM blocknum = make_blocknum(getuint64(fields[1])); dump_block(fd, blocknum, ft); + }else if ((strcmp(fields[0], "readFile") == 0 ||strcmp(fields[0], "readfile") == 0 ||strcmp(fields[0], "rf") == 0 )&& nfields == 2) { + fname=fields[1]; + fd = open(fname, O_RDWR + O_BINARY); + toku_ft_free(ft); + open_header(fd, &ft, cf); + root=getRootNode(ft); + off = make_blocknum(root); + height=getHeight(fd,off, ft); + if (fd < 0) { + fprintf(stderr, "%s: can not open the FT dump %s errno %d\n", arg0, fname, errno); + continue; + } } else if (strcmp(fields[0], "node") == 0 && nfields == 2) { - BLOCKNUM off = make_blocknum(getuint64(fields[1])); + off = make_blocknum(getuint64(fields[1])); dump_node(fd, off, ft); - } else if (strcmp(fields[0], "dumpdata") == 0 && nfields == 2) { + }else if ((strcmp(fields[0], "mr") == 0||(strcmp(fields[0], "nc")) == 0 ||strcmp(fields[0], "messagesReport") == 0 )) { + freeNMC(msgs,height); + toku_ft_free(ft); + open_header(fd, &ft, cf); + root=getRootNode(ft); + off = make_blocknum(root); + countMessagesInFT(fd,off, ft,msgs); + int level=-1; + if(nfields == 2)level=getuint64(fields[1]); + if(level>=0){ + levelToSTDout(msgs[level], level); + } + else{ + cout <<"to STD output: \n"; + treeToSTDout(msgs,height); + } + writeTree(msgs,height); + writeTree(msgs,height, NULL); + + }else if (strcmp(fields[0], "dumpdata") == 0 && nfields == 2) { + do_dump_data = strtol(fields[1], NULL, 10); - } else if (strcmp(fields[0], "block_translation") == 0 || strcmp(fields[0], "bx") == 0) { + } + else if (strcmp(fields[0], "block_translation") == 0 || strcmp(fields[0], "bx") == 0) { uint64_t offset = 0; if (nfields == 2) offset = getuint64(fields[1]); @@ -590,7 +994,7 @@ static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) { dump_fragmentation(fd, ft, do_tsv); } else if (strcmp(fields[0], "nodesizes") == 0) { dump_nodesizes(fd, ft); - } else if (strcmp(fields[0], "garbage") == 0) { + } else if (strcmp(fields[0], "garbage") == 0||strcmp(fields[0], "g") == 0) { dump_garbage_stats(fd, ft); } else if (strcmp(fields[0], "file") == 0 && nfields >= 3) { uint64_t offset = getuint64(fields[1]); @@ -604,14 +1008,18 @@ static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) { unsigned char newc = getuint64(fields[2]); set_file(fd, offset, newc); } else if (strcmp(fields[0], "quit") == 0 || strcmp(fields[0], "q") == 0) { - break; + toku_ft_free(ft); + exit(0); } } + freeNMC(msgs,height); } static int usage(void) { fprintf(stderr, "Usage: %s ", arg0); fprintf(stderr, "--interactive "); + fprintf(stderr, "--support /path/to/fractal-tree/file \n\t an interactive way to see what messages and/or switch between FTs"); + fprintf(stderr, "--json /path/to/fractal-tree/file [output json file]\n\t if left empty an FT.json will be created automatically"); fprintf(stderr, "--nodata "); fprintf(stderr, "--dumpdata 0|1 "); fprintf(stderr, "--header "); @@ -632,10 +1040,15 @@ int main (int argc, const char *const argv[]) { while (argc>0) { if (strcmp(argv[0], "--interactive") == 0 || strcmp(argv[0], "--i") == 0) { do_interactive = 1; + } + else if ((strcmp(argv[0], "--json") == 0 || strcmp(argv[0], "--s")== 0)&& argc >= 2) { + do_json = 1; + fname=argv[1]; + argc--; argv++; + break; } else if (strcmp(argv[0], "--nodata") == 0) { do_dump_data = 0; } else if (strcmp(argv[0], "--dumpdata") == 0 && argc > 1) { - argc--; argv++; do_dump_data = atoi(argv[0]); } else if (strcmp(argv[0], "--header") == 0) { do_header = 1; @@ -660,39 +1073,39 @@ int main (int argc, const char *const argv[]) { } argc--; argv++; } - if (argc != 1) - return usage(); + if (argc != 1 && do_json==0) + return usage(); int r = toku_ft_layer_init(); assert_zero(r); - - fname = argv[0]; + if(fname==NULL)fname = argv[0]; int fd = open(fname, O_RDWR + O_BINARY); if (fd < 0) { fprintf(stderr, "%s: can not open %s errno %d\n", arg0, fname, errno); return 1; } - // create a cachefile for the header CACHETABLE ct = NULL; toku_cachetable_create(&ct, 1<<25, (LSN){0}, 0); - CACHEFILE cf = NULL; r = toku_cachetable_openfd (&cf, ct, fd, fname); assert_zero(r); - FT ft = NULL; open_header(fd, &ft, cf); - + if (do_json ) { + const char *arg=argv[1]; + FT_to_JSON(fd, ft, cf,arg); + } if (do_interactive) { run_iteractive_loop(fd, ft, cf); - } else { + } + else { if (do_header) { dump_header(ft); } if (do_rootnode) { dump_node(fd, ft->h->root_blocknum, ft); - } + } if (do_node) { dump_node(fd, do_node_num, ft); } @@ -708,14 +1121,12 @@ int main (int argc, const char *const argv[]) { if (!do_header && !do_rootnode && !do_fragmentation && !do_translation_table && !do_garbage) { printf("Block translation:"); ft->blocktable.dump_translation_table(stdout); - dump_header(ft); - struct __dump_node_extra info; info.fd = fd; info.ft = ft; ft->blocktable.iterate(block_table::TRANSLATION_CHECKPOINTED, - dump_node_wrapper, &info, true, true); + dump_node_wrapper, &info, true, true); } } toku_cachefile_close(&cf, false, ZERO_LSN); diff --git a/storage/tokudb/ft-index/util/scoped_malloc.cc b/storage/tokudb/ft-index/util/scoped_malloc.cc index 551bd944beb..15d4fb3e52e 100644 --- a/storage/tokudb/ft-index/util/scoped_malloc.cc +++ b/storage/tokudb/ft-index/util/scoped_malloc.cc @@ -145,6 +145,9 @@ namespace toku { } void destroy() { +#if TOKU_SCOPED_MALLOC_DEBUG + printf("%s %p %p\n", __FUNCTION__, this, m_stack); +#endif if (m_stack != NULL) { toku_free(m_stack); m_stack = NULL; @@ -167,13 +170,17 @@ namespace toku { static void destroy_and_deregister(void *key) { invariant_notnull(key); tl_stack *st = reinterpret_cast<tl_stack *>(key); - st->destroy(); + size_t n = 0; toku_mutex_lock(&global_stack_set_mutex); - invariant_notnull(global_stack_set); - size_t n = global_stack_set->erase(st); - invariant(n == 1); + if (global_stack_set) { + n = global_stack_set->erase(st); + } toku_mutex_unlock(&global_stack_set_mutex); + + if (n == 1) { + st->destroy(); // destroy the stack if this function erased it from the set. otherwise, somebody else destroyed it. + } } // Allocate 'size' bytes and return a pointer to the first byte @@ -244,6 +251,11 @@ void toku_scoped_malloc_init(void) { } void toku_scoped_malloc_destroy(void) { + toku_scoped_malloc_destroy_key(); + toku_scoped_malloc_destroy_set(); +} + +void toku_scoped_malloc_destroy_set(void) { toku_mutex_lock(&toku::global_stack_set_mutex); invariant_notnull(toku::global_stack_set); // Destroy any tl_stacks that were registered as thread locals but did not @@ -254,10 +266,11 @@ void toku_scoped_malloc_destroy(void) { (*i)->destroy(); } delete toku::global_stack_set; + toku::global_stack_set = nullptr; toku_mutex_unlock(&toku::global_stack_set_mutex); +} - // We're deregistering the destructor key here. When this thread exits, - // the tl_stack destructor won't get called, so we need to do that first. +void toku_scoped_malloc_destroy_key(void) { int r = pthread_key_delete(toku::tl_stack_destroy_pthread_key); invariant_zero(r); } diff --git a/storage/tokudb/ft-index/util/scoped_malloc.h b/storage/tokudb/ft-index/util/scoped_malloc.h index dbd919d155e..0233b0f1aa5 100644 --- a/storage/tokudb/ft-index/util/scoped_malloc.h +++ b/storage/tokudb/ft-index/util/scoped_malloc.h @@ -151,3 +151,7 @@ void toku_scoped_malloc_init(void); void toku_scoped_malloc_destroy(void); +void toku_scoped_malloc_destroy_set(void); + +void toku_scoped_malloc_destroy_key(void); + diff --git a/storage/tokudb/ft-index/util/tests/sm-basic.cc b/storage/tokudb/ft-index/util/tests/sm-basic.cc new file mode 100644 index 00000000000..5df64294721 --- /dev/null +++ b/storage/tokudb/ft-index/util/tests/sm-basic.cc @@ -0,0 +1,127 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuFT, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +// test that basic scoped malloc works with a thread + +#ident "Copyright (c) 2015 Tokutek Inc. All rights reserved." +#include <toku_portability.h> +#include <toku_assert.h> +#include <toku_pthread.h> +#include <util/scoped_malloc.h> + +static void sm_test(void) { + toku::scoped_malloc a(1); + { + toku::scoped_malloc b(2); + { + toku::scoped_malloc c(3); + } + } +} + +static void *sm_test_f(void *arg) { + sm_test(); + return arg; +} + +int main(void) { + toku_scoped_malloc_init(); + + // run the test + toku_pthread_t tid; + int r; + r = toku_pthread_create(&tid, NULL, sm_test_f, NULL); + assert_zero(r); + void *ret; + r = toku_pthread_join(tid, &ret); + assert_zero(r); + + toku_scoped_malloc_destroy(); + + return 0; +} diff --git a/storage/tokudb/ft-index/util/tests/sm-crash-double-free.cc b/storage/tokudb/ft-index/util/tests/sm-crash-double-free.cc new file mode 100644 index 00000000000..653d4148fd0 --- /dev/null +++ b/storage/tokudb/ft-index/util/tests/sm-crash-double-free.cc @@ -0,0 +1,128 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuFT, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +// force a race between the scoped malloc global destructor and a thread variable destructor + +#ident "Copyright (c) 2015 Tokutek Inc. All rights reserved." +#define TOKU_SCOPED_MALLOC_DEBUG 1 +#include <toku_portability.h> +#include <toku_assert.h> +#include <toku_pthread.h> +#include <toku_race_tools.h> +#include <util/scoped_malloc.h> + +volatile int state = 0; + +static void sm_test(void) { + toku::scoped_malloc a(1); +} + +static void *sm_test_f(void *arg) { + sm_test(); + state = 1; + while (state != 2) sleep(1); + return arg; +} + +int main(void) { + TOKU_VALGRIND_HG_DISABLE_CHECKING(&state, sizeof state); + state = 0; + toku_scoped_malloc_init(); + toku_pthread_t tid; + int r; + r = toku_pthread_create(&tid, NULL, sm_test_f, NULL); + assert_zero(r); + void *ret; + while (state != 1) sleep(1); + toku_scoped_malloc_destroy_set(); + state = 2; + r = toku_pthread_join(tid, &ret); + assert_zero(r); + toku_scoped_malloc_destroy_key(); + return 0; +} diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index f68b50b9fdc..1b022cd3468 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -172,6 +172,31 @@ static inline uint32_t get_len_of_offsets(KEY_AND_COL_INFO* kc_info, TABLE_SHARE } +static int get_thread_query_string(my_thread_id id, String &qs) { + mysql_mutex_lock(&LOCK_thread_count); + I_List_iterator<THD> it(threads); + THD* tmp; + while ((tmp= it++)) + { + /* ID */ + if (tmp->thread_id == id) + { + /* Lock THD mutex that protects its data when looking at it. */ + mysql_mutex_lock(&tmp->LOCK_thd_data); + + /* INFO */ + if (tmp->query()) + { + qs = String(tmp->query(), tmp->query_length(), system_charset_info); + } + mysql_mutex_unlock(&tmp->LOCK_thd_data); + break; + } + } + mysql_mutex_unlock(&LOCK_thread_count); + return 0; +} + static int allocate_key_and_col_info ( TABLE_SHARE* table_share, KEY_AND_COL_INFO* kc_info) { int error; // @@ -3247,7 +3272,7 @@ void ha_tokudb::start_bulk_insert(ha_rows rows) { lock_count = 0; if ((rows == 0 || rows > 1) && share->try_table_lock) { - if (get_prelock_empty(thd) && may_table_be_empty(transaction)) { + if (get_prelock_empty(thd) && may_table_be_empty(transaction) && transaction != NULL) { if (using_ignore || is_insert_ignore(thd) || thd->lex->duplicates != DUP_ERROR || table->s->next_number_key_offset) { acquire_table_lock(transaction, lock_write); @@ -3557,8 +3582,12 @@ static void maybe_do_unique_checks_delay(THD *thd) { } } +static bool need_read_only(THD *thd) { + return opt_readonly || !THDVAR(thd, rpl_check_readonly); +} + static bool do_unique_checks(THD *thd, bool do_rpl_event) { - if (do_rpl_event && thd->slave_thread && opt_readonly && !THDVAR(thd, rpl_unique_checks)) + if (do_rpl_event && thd->slave_thread && need_read_only(thd) && !THDVAR(thd, rpl_unique_checks)) return false; else return !thd_test_options(thd, OPTION_RELAXED_UNIQUE_CHECKS); @@ -3934,13 +3963,13 @@ int ha_tokudb::write_row(uchar * record) { goto cleanup; } } - txn = create_sub_trans ? sub_trans : transaction; - + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_HANDLER_TRACE("txn %p", txn); + } if (tokudb_debug & TOKUDB_DEBUG_CHECK_KEY) { test_row_packing(record,&prim_key,&row); } - if (loader) { error = loader->put(loader, &prim_key, &row); if (error) { @@ -4214,7 +4243,7 @@ int ha_tokudb::delete_row(const uchar * record) { bool has_null; THD* thd = ha_thd(); uint curr_num_DBs; - tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); ha_statistic_increment(&SSV::ha_delete_count); @@ -4239,10 +4268,14 @@ int ha_tokudb::delete_row(const uchar * record) { goto cleanup; } + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_HANDLER_TRACE("all %p stmt %p sub_sp_level %p transaction %p", trx->all, trx->stmt, trx->sub_sp_level, transaction); + } + error = db_env->del_multiple( db_env, share->key_file[primary_key], - transaction, + transaction, &prim_key, &row, curr_num_DBs, @@ -5375,9 +5408,12 @@ int ha_tokudb::get_next(uchar* buf, int direction, DBT* key_to_compare, bool do_ } if (!error) { - tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); + THD *thd = ha_thd(); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); trx->stmt_progress.queried++; - track_progress(ha_thd()); + track_progress(thd); + if (thd_killed(thd)) + error = ER_ABORTING_CONNECTION; } cleanup: return error; @@ -6168,6 +6204,12 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) { if (error) { goto cleanup; } thd_set_ha_data(thd, tokudb_hton, trx); } + + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, + trx->tokudb_lock_count, trx->create_lock_count); + } + if (trx->all == NULL) { trx->sp_level = NULL; } @@ -6176,22 +6218,16 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) { if (lock_type == F_WRLCK) { use_write_locks = true; } - if (!trx->tokudb_lock_count++) { - if (trx->stmt) { - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("stmt already set %p %p %p %p", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level); - } - } else { - assert(trx->stmt == 0); - transaction = NULL; // Safety - error = create_txn(thd, trx); - if (error) { - trx->tokudb_lock_count--; // We didn't get the lock - goto cleanup; - } + if (!trx->stmt) { + transaction = NULL; // Safety + error = create_txn(thd, trx); + if (error) { + goto cleanup; } + trx->create_lock_count = trx->tokudb_lock_count; } transaction = trx->sub_sp_level; + trx->tokudb_lock_count++; } else { tokudb_pthread_mutex_lock(&share->mutex); @@ -6206,21 +6242,24 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) { added_rows = 0; deleted_rows = 0; share->rows_from_locked_table = 0; - if (trx->tokudb_lock_count > 0 && !--trx->tokudb_lock_count) { - if (trx->stmt) { - /* - F_UNLCK is done without a transaction commit / rollback. - This happens if the thread didn't update any rows - We must in this case commit the work to keep the row locks - */ - DBUG_PRINT("trans", ("commiting non-updating transaction")); - reset_stmt_progress(&trx->stmt_progress); - commit_txn(trx->stmt, 0); - trx->stmt = NULL; - trx->sub_sp_level = NULL; + if (trx->tokudb_lock_count > 0) { + if (--trx->tokudb_lock_count <= trx->create_lock_count) { + trx->create_lock_count = 0; + if (trx->stmt) { + /* + F_UNLCK is done without a transaction commit / rollback. + This happens if the thread didn't update any rows + We must in this case commit the work to keep the row locks + */ + DBUG_PRINT("trans", ("commiting non-updating transaction")); + reset_stmt_progress(&trx->stmt_progress); + commit_txn(trx->stmt, 0); + trx->stmt = NULL; + trx->sub_sp_level = NULL; + } } + transaction = NULL; } - transaction = NULL; } cleanup: if (tokudb_debug & TOKUDB_DEBUG_LOCK) @@ -6235,8 +6274,9 @@ cleanup: */ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) { TOKUDB_HANDLER_DBUG_ENTER("cmd %d lock %d %s", thd_sql_command(thd), lock_type, share->table_name); - if (0) + if (tokudb_debug & TOKUDB_DEBUG_LOCK) { TOKUDB_HANDLER_TRACE("q %s", thd->query()); + } int error = 0; tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); @@ -6246,6 +6286,11 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) { thd_set_ha_data(thd, tokudb_hton, trx); } + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, + trx->tokudb_lock_count, trx->create_lock_count); + } + /* note that trx->stmt may have been already initialized as start_stmt() is called for *each table* not for each storage engine, @@ -6256,9 +6301,7 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) { if (error) { goto cleanup; } - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("%p %p %p %p %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, trx->tokudb_lock_count); - } + trx->create_lock_count = trx->tokudb_lock_count; } else { if (tokudb_debug & TOKUDB_DEBUG_TXN) { @@ -7149,12 +7192,15 @@ To rename the table, make sure no transactions touch the table.", from, to); double ha_tokudb::scan_time() { TOKUDB_HANDLER_DBUG_ENTER(""); double ret_val = (double)stats.records / 3; + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %" PRIu64 " %f", (uint64_t) stats.records, ret_val); + } DBUG_RETURN(ret_val); } double ha_tokudb::keyread_time(uint index, uint ranges, ha_rows rows) { - TOKUDB_HANDLER_DBUG_ENTER(""); + TOKUDB_HANDLER_DBUG_ENTER("%u %u %" PRIu64, index, ranges, (uint64_t) rows); double ret_val; if (index == primary_key || key_is_clustering(&table->key_info[index])) { ret_val = read_time(index, ranges, rows); @@ -7172,6 +7218,9 @@ double ha_tokudb::keyread_time(uint index, uint ranges, ha_rows rows) (table->key_info[index].key_length + ref_length) + 1); ret_val = (rows + keys_per_block - 1)/ keys_per_block; + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %f", ret_val); + } DBUG_RETURN(ret_val); } @@ -7192,7 +7241,7 @@ double ha_tokudb::read_time( ha_rows rows ) { - TOKUDB_HANDLER_DBUG_ENTER(""); + TOKUDB_HANDLER_DBUG_ENTER("%u %u %" PRIu64, index, ranges, (uint64_t) rows); double total_scan; double ret_val; bool is_primary = (index == primary_key); @@ -7234,12 +7283,18 @@ double ha_tokudb::read_time( ret_val = is_clustering ? ret_val + 0.00001 : ret_val; cleanup: + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %f", ret_val); + } DBUG_RETURN(ret_val); } double ha_tokudb::index_only_read_time(uint keynr, double records) { - TOKUDB_HANDLER_DBUG_ENTER(""); + TOKUDB_HANDLER_DBUG_ENTER("%u %f", keynr, records); double ret_val = keyread_time(keynr, 1, (ha_rows)records); + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %f", ret_val); + } DBUG_RETURN(ret_val); } @@ -7257,7 +7312,7 @@ double ha_tokudb::index_only_read_time(uint keynr, double records) { // HA_POS_ERROR - Something is wrong with the index tree // ha_rows ha_tokudb::records_in_range(uint keynr, key_range* start_key, key_range* end_key) { - TOKUDB_HANDLER_DBUG_ENTER(""); + TOKUDB_HANDLER_DBUG_ENTER("%d %p %p", keynr, start_key, end_key); DBT *pleft_key, *pright_key; DBT left_key, right_key; ha_rows ret_val = HA_TOKUDB_RANGE_COUNT; @@ -7313,6 +7368,9 @@ ha_rows ha_tokudb::records_in_range(uint keynr, key_range* start_key, key_range* ret_val = (ha_rows) (rows <= 1 ? 1 : rows); cleanup: + if (tokudb_debug & TOKUDB_DEBUG_RETURN) { + TOKUDB_HANDLER_TRACE("return %" PRIu64 " %" PRIu64, (uint64_t) ret_val, rows); + } DBUG_RETURN(ret_val); } diff --git a/storage/tokudb/ha_tokudb_admin.cc b/storage/tokudb/ha_tokudb_admin.cc index c3b895bf4f4..42205c6d6be 100644 --- a/storage/tokudb/ha_tokudb_admin.cc +++ b/storage/tokudb/ha_tokudb_admin.cc @@ -89,6 +89,8 @@ PATENT RIGHTS GRANT: #ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." +#include "toku_time.h" + struct analyze_progress_extra { THD *thd; TOKUDB_SHARE *share; @@ -119,9 +121,10 @@ static int analyze_progress(void *v_extra, uint64_t rows) { progress_time = (float) (t_now - t_start) / (float) t_limit; char *write_status_msg = extra->write_status_msg; TABLE_SHARE *table_share = extra->table_share; - sprintf(write_status_msg, "%s.%s.%s %u of %u %.lf%% rows %.lf%% time", - table_share->db.str, table_share->table_name.str, extra->key_name, - extra->key_i, table_share->keys, progress_rows * 100.0, progress_time * 100.0); + sprintf(write_status_msg, "%.*s.%.*s.%s %u of %u %.lf%% rows %.lf%% time", + (int) table_share->db.length, table_share->db.str, + (int) table_share->table_name.length, table_share->table_name.str, + extra->key_name, extra->key_i, table_share->keys, progress_rows * 100.0, progress_time * 100.0); thd_proc_info(thd, write_status_msg); return 0; } @@ -154,18 +157,47 @@ int ha_tokudb::analyze(THD *thd, HA_CHECK_OPT *check_opt) { bool is_unique = false; if (i == primary_key || (key_info->flags & HA_NOSAME)) is_unique = true; + uint64_t rows = 0; + uint64_t deleted_rows = 0; int error = tokudb::analyze_card(share->key_file[i], txn, is_unique, num_key_parts, &rec_per_key[total_key_parts], - tokudb_cmp_dbt_key_parts, analyze_progress, &analyze_progress_extra); + tokudb_cmp_dbt_key_parts, analyze_progress, &analyze_progress_extra, + &rows, &deleted_rows); + sql_print_information("tokudb analyze %d %" PRIu64 " %" PRIu64, error, rows, deleted_rows); if (error != 0 && error != ETIME) { result = HA_ADMIN_FAILED; - } else { - // debug - if (tokudb_debug & TOKUDB_DEBUG_ANALYZE) { - TOKUDB_HANDLER_TRACE("%s.%s.%s", - table_share->db.str, table_share->table_name.str, i == primary_key ? "primary" : table_share->key_info[i].name); - for (uint j = 0; j < num_key_parts; j++) - TOKUDB_HANDLER_TRACE("%lu", rec_per_key[total_key_parts+j]); - } + } + if (error != 0 && rows == 0 && deleted_rows > 0) { + result = HA_ADMIN_FAILED; + } + double f = THDVAR(thd, analyze_delete_fraction); + if (result == HA_ADMIN_FAILED || (double) deleted_rows > f * (rows + deleted_rows)) { + char name[256]; int namelen; + namelen = snprintf(name, sizeof name, "%.*s.%.*s.%s", + (int) table_share->db.length, table_share->db.str, + (int) table_share->table_name.length, table_share->table_name.str, + key_name); + thd->protocol->prepare_for_resend(); + thd->protocol->store(name, namelen, system_charset_info); + thd->protocol->store("analyze", 7, system_charset_info); + thd->protocol->store("info", 4, system_charset_info); + char rowmsg[256]; int rowmsglen; + rowmsglen = snprintf(rowmsg, sizeof rowmsg, "rows processed %" PRIu64 " rows deleted %" PRIu64, rows, deleted_rows); + thd->protocol->store(rowmsg, rowmsglen, system_charset_info); + thd->protocol->write(); + + sql_print_information("tokudb analyze on %.*s %.*s", + namelen, name, rowmsglen, rowmsg); + } + if (tokudb_debug & TOKUDB_DEBUG_ANALYZE) { + char name[256]; int namelen; + namelen = snprintf(name, sizeof name, "%.*s.%.*s.%s", + (int) table_share->db.length, table_share->db.str, + (int) table_share->table_name.length, table_share->table_name.str, + key_name); + TOKUDB_HANDLER_TRACE("%.*s rows %" PRIu64 " deleted %" PRIu64, + namelen, name, rows, deleted_rows); + for (uint j = 0; j < num_key_parts; j++) + TOKUDB_HANDLER_TRACE("%lu", rec_per_key[total_key_parts+j]); } total_key_parts += num_key_parts; } @@ -186,9 +218,12 @@ typedef struct hot_optimize_context { uint progress_stage; uint current_table; uint num_tables; + float progress_limit; + uint64_t progress_last_time; + uint64_t throttle; } *HOT_OPTIMIZE_CONTEXT; -static int hot_poll_fun(void *extra, float progress) { +static int hot_optimize_progress_fun(void *extra, float progress) { HOT_OPTIMIZE_CONTEXT context = (HOT_OPTIMIZE_CONTEXT)extra; if (thd_killed(context->thd)) { sprintf(context->write_status_msg, "The process has been killed, aborting hot optimize."); @@ -207,14 +242,27 @@ static int hot_poll_fun(void *extra, float progress) { // the percentage we report here is for the current stage/db thd_progress_report(context->thd, (unsigned long long) percentage, 100); #endif - return 0; + + // throttle the optimize table + if (context->throttle) { + uint64_t time_now = toku_current_time_microsec(); + uint64_t dt = time_now - context->progress_last_time; + uint64_t throttle_time = 1000000ULL / context->throttle; + if (throttle_time > dt) { + usleep(throttle_time - dt); + } + context->progress_last_time = toku_current_time_microsec(); + } + + // return 1 if progress has reach the progress limit + return progress >= context->progress_limit; } // flatten all DB's in this table, to do so, peform hot optimize on each db int ha_tokudb::do_optimize(THD *thd) { TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name); + int error = 0; const char *orig_proc_info = tokudb_thd_get_proc_info(thd); - int error; uint curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); #ifdef HA_TOKUDB_HAS_THD_PROGRESS @@ -225,11 +273,21 @@ int ha_tokudb::do_optimize(THD *thd) { // for each DB, run optimize and hot_optimize for (uint i = 0; i < curr_num_DBs; i++) { + // only optimize the index if it matches the optimize_index_name session variable + const char *optimize_index_name = THDVAR(thd, optimize_index_name); + if (optimize_index_name) { + const char *this_index_name = i >= table_share->keys ? "primary" : table_share->key_info[i].name; + if (strcasecmp(optimize_index_name, this_index_name) != 0) { + continue; + } + } + DB* db = share->key_file[i]; error = db->optimize(db); if (error) { goto cleanup; } + struct hot_optimize_context hc; memset(&hc, 0, sizeof hc); hc.thd = thd; @@ -237,8 +295,11 @@ int ha_tokudb::do_optimize(THD *thd) { hc.ha = this; hc.current_table = i; hc.num_tables = curr_num_DBs; + hc.progress_limit = THDVAR(thd, optimize_index_fraction); + hc.progress_last_time = toku_current_time_microsec(); + hc.throttle = THDVAR(thd, optimize_throttle); uint64_t loops_run; - error = db->hot_optimize(db, NULL, NULL, hot_poll_fun, &hc, &loops_run); + error = db->hot_optimize(db, NULL, NULL, hot_optimize_progress_fun, &hc, &loops_run); if (error) { goto cleanup; } @@ -278,8 +339,10 @@ static int ha_tokudb_check_progress(void *extra, float progress) { static void ha_tokudb_check_info(THD *thd, TABLE *table, const char *msg) { if (thd->vio_ok()) { - char tablename[256]; - snprintf(tablename, sizeof tablename, "%s.%s", table->s->db.str, table->s->table_name.str); + char tablename[table->s->db.length + 1 + table->s->table_name.length + 1]; + snprintf(tablename, sizeof tablename, "%.*s.%.*s", + (int) table->s->db.length, table->s->db.str, + (int) table->s->table_name.length, table->s->table_name.str); thd->protocol->prepare_for_resend(); thd->protocol->store(tablename, strlen(tablename), system_charset_info); thd->protocol->store("check", 5, system_charset_info); @@ -328,6 +391,11 @@ int ha_tokudb::check(THD *thd, HA_CHECK_OPT *check_opt) { } struct check_context check_context = { thd }; r = db->verify_with_progress(db, ha_tokudb_check_progress, &check_context, (tokudb_debug & TOKUDB_DEBUG_CHECK) != 0, keep_going); + if (r != 0) { + char msg[32 + strlen(kname)]; + sprintf(msg, "Corrupt %s", kname); + ha_tokudb_check_info(thd, table, msg); + } snprintf(write_status_msg, sizeof write_status_msg, "%s key=%s %u result=%d", share->table_name, kname, i, r); thd_proc_info(thd, write_status_msg); if (tokudb_debug & TOKUDB_DEBUG_CHECK) { diff --git a/storage/tokudb/ha_tokudb_alter_56.cc b/storage/tokudb/ha_tokudb_alter_56.cc index cae50446fa0..213b58459bc 100644 --- a/storage/tokudb/ha_tokudb_alter_56.cc +++ b/storage/tokudb/ha_tokudb_alter_56.cc @@ -784,13 +784,16 @@ bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_i assert(trx->tokudb_lock_count > 0); // for partitioned tables, we use a single transaction to do all of the partition changes. the tokudb_lock_count // is a reference count for each of the handlers to the same transaction. obviously, we want to only abort once. - if (!--trx->tokudb_lock_count) { - abort_txn(ctx->alter_txn); - ctx->alter_txn = NULL; - trx->stmt = NULL; - trx->sub_sp_level = NULL; + if (trx->tokudb_lock_count > 0) { + if (--trx->tokudb_lock_count <= trx->create_lock_count) { + trx->create_lock_count = 0; + abort_txn(ctx->alter_txn); + ctx->alter_txn = NULL; + trx->stmt = NULL; + trx->sub_sp_level = NULL; + } + transaction = NULL; } - transaction = NULL; if (ctx->add_index_changed) { restore_add_index(table, ha_alter_info->index_add_count, ctx->incremented_num_DBs, ctx->modified_DBs); diff --git a/storage/tokudb/hatoku_defines.h b/storage/tokudb/hatoku_defines.h index 57214038500..bc3d890fffe 100644 --- a/storage/tokudb/hatoku_defines.h +++ b/storage/tokudb/hatoku_defines.h @@ -162,6 +162,9 @@ PATENT RIGHTS GRANT: #define TOKU_INCLUDE_EXTENDED_KEYS 1 #define TOKU_INCLUDE_OPTION_STRUCTS 1 #define TOKU_CLUSTERING_IS_COVERING 1 +#define TOKU_INCLUDE_LOCK_TIMEOUT_QUERY_STRING 1 +#else +#define TOKU_INCLUDE_LOCK_TIMEOUT_QUERY_STRING 1 #endif #define TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL 0 /* MariaDB 5.5 */ @@ -352,6 +355,7 @@ typedef struct st_tokudb_trx_data { DB_TXN *sp_level; DB_TXN *sub_sp_level; uint tokudb_lock_count; + uint create_lock_count; tokudb_stmt_progress stmt_progress; bool checkpoint_lock_taken; LIST *handlers; diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 99f9cc56037..a804fc80489 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -780,7 +780,7 @@ extern "C" enum durability_properties thd_get_durability_property(const MYSQL_TH #endif // Determine if an fsync is used when a transaction is committed. -static bool tokudb_fsync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) { +static bool tokudb_sync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) { #if MYSQL_VERSION_ID >= 50600 // Check the client durability property which is set during 2PC if (thd_get_durability_property(thd) == HA_IGNORE_DURABILITY) @@ -791,17 +791,19 @@ static bool tokudb_fsync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) if (txn->is_prepared(txn) && mysql_bin_log.is_open()) return false; #endif + if (tokudb_fsync_log_period > 0) + return false; return THDVAR(thd, commit_sync) != 0; } static int tokudb_commit(handlerton * hton, THD * thd, bool all) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%u", all); DBUG_PRINT("trans", ("ending transaction %s", all ? "all" : "stmt")); tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN **txn = all ? &trx->all : &trx->stmt; DB_TXN *this_txn = *txn; if (this_txn) { - uint32_t syncflag = tokudb_fsync_on_commit(thd, trx, this_txn) ? 0 : DB_TXN_NOSYNC; + uint32_t syncflag = tokudb_sync_on_commit(thd, trx, this_txn) ? 0 : DB_TXN_NOSYNC; if (tokudb_debug & TOKUDB_DEBUG_TXN) { TOKUDB_TRACE("commit trx %u txn %p syncflag %u", all, this_txn, syncflag); } @@ -811,11 +813,11 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { commit_txn_with_progress(this_txn, syncflag, thd); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_commit_after", DBUG_SUICIDE();); - if (this_txn == trx->sp_level) { - trx->sp_level = 0; - } - *txn = 0; + *txn = NULL; trx->sub_sp_level = NULL; + if (this_txn == trx->sp_level || trx->all == NULL) { + trx->sp_level = NULL; + } } else if (tokudb_debug & TOKUDB_DEBUG_TXN) { TOKUDB_TRACE("nothing to commit %d", all); @@ -825,7 +827,7 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { } static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%u", all); DBUG_PRINT("trans", ("aborting transaction %s", all ? "all" : "stmt")); tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN **txn = all ? &trx->all : &trx->stmt; @@ -836,11 +838,11 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { } tokudb_cleanup_handlers(trx, this_txn); abort_txn_with_progress(this_txn, thd); - if (this_txn == trx->sp_level) { - trx->sp_level = 0; - } - *txn = 0; + *txn = NULL; trx->sub_sp_level = NULL; + if (this_txn == trx->sp_level || trx->all == NULL) { + trx->sp_level = NULL; + } } else { if (tokudb_debug & TOKUDB_DEBUG_TXN) { @@ -852,6 +854,13 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { } #if TOKU_INCLUDE_XA +static bool tokudb_sync_on_prepare(void) { + // skip sync of log if fsync log period > 0 + if (tokudb_fsync_log_period > 0) + return false; + else + return true; +} static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { TOKUDB_DBUG_ENTER(""); @@ -866,6 +875,7 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN* txn = all ? trx->all : trx->stmt; if (txn) { + uint32_t syncflag = tokudb_sync_on_prepare() ? 0 : DB_TXN_NOSYNC; if (tokudb_debug & TOKUDB_DEBUG_TXN) { TOKUDB_TRACE("doing txn prepare:%d:%p", all, txn); } @@ -874,7 +884,7 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { thd_get_xid(thd, (MYSQL_XID*) &thd_xid); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_prepare_before", DBUG_SUICIDE();); - r = txn->xa_prepare(txn, &thd_xid); + r = txn->xa_prepare(txn, &thd_xid, syncflag); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_prepare_after", DBUG_SUICIDE();); } @@ -939,7 +949,7 @@ cleanup: #endif static int tokudb_savepoint(handlerton * hton, THD * thd, void *savepoint) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%p", savepoint); int error; SP_INFO save_info = (SP_INFO)savepoint; tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); @@ -960,6 +970,9 @@ static int tokudb_savepoint(handlerton * hton, THD * thd, void *savepoint) { trx->sp_level = save_info->txn; save_info->in_sub_stmt = false; } + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_TRACE("begin txn %p", save_info->txn); + } save_info->trx = trx; error = 0; cleanup: @@ -967,7 +980,7 @@ cleanup: } static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *savepoint) { - TOKUDB_DBUG_ENTER(""); + TOKUDB_DBUG_ENTER("%p", savepoint); int error; SP_INFO save_info = (SP_INFO)savepoint; DB_TXN* parent = NULL; @@ -975,6 +988,9 @@ static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *save tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); parent = txn_to_rollback->parent; + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_TRACE("rollback txn %p", txn_to_rollback); + } if (!(error = txn_to_rollback->abort(txn_to_rollback))) { if (save_info->in_sub_stmt) { trx->sub_sp_level = parent; @@ -988,24 +1004,27 @@ static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *save } static int tokudb_release_savepoint(handlerton * hton, THD * thd, void *savepoint) { - TOKUDB_DBUG_ENTER(""); - int error; - + TOKUDB_DBUG_ENTER("%p", savepoint); + int error = 0; SP_INFO save_info = (SP_INFO)savepoint; DB_TXN* parent = NULL; DB_TXN* txn_to_commit = save_info->txn; tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); parent = txn_to_commit->parent; - if (!(error = txn_to_commit->commit(txn_to_commit, 0))) { + if (tokudb_debug & TOKUDB_DEBUG_TXN) { + TOKUDB_TRACE("commit txn %p", txn_to_commit); + } + DB_TXN *child = txn_to_commit->get_child(txn_to_commit); + if (child == NULL && !(error = txn_to_commit->commit(txn_to_commit, 0))) { if (save_info->in_sub_stmt) { trx->sub_sp_level = parent; } else { trx->sp_level = parent; } - save_info->txn = NULL; } + save_info->txn = NULL; TOKUDB_DBUG_RETURN(error); } @@ -1393,8 +1412,6 @@ static MYSQL_SYSVAR_STR(data_dir, tokudb_data_dir, PLUGIN_VAR_READONLY, "TokuDB static MYSQL_SYSVAR_STR(version, tokudb_version, PLUGIN_VAR_READONLY, "TokuDB Version", NULL, NULL, NULL); -static MYSQL_SYSVAR_UINT(init_flags, tokudb_init_flags, PLUGIN_VAR_READONLY, "Sets TokuDB DB_ENV->open flags", NULL, NULL, tokudb_init_flags, 0, ~0U, 0); - static MYSQL_SYSVAR_UINT(write_status_frequency, tokudb_write_status_frequency, 0, "TokuDB frequency that show processlist updates status of writes", NULL, NULL, 1000, 0, ~0U, 0); static MYSQL_SYSVAR_UINT(read_status_frequency, tokudb_read_status_frequency, 0, "TokuDB frequency that show processlist updates status of reads", NULL, NULL, 10000, 0, ~0U, 0); static MYSQL_SYSVAR_INT(fs_reserve_percent, tokudb_fs_reserve_percent, PLUGIN_VAR_READONLY, "TokuDB file system space reserve (percent free required)", NULL, NULL, 5, 0, 100, 0); @@ -1431,7 +1448,6 @@ static struct st_mysql_sys_var *tokudb_system_variables[] = { MYSQL_SYSVAR(create_index_online), MYSQL_SYSVAR(disable_prefetching), MYSQL_SYSVAR(version), - MYSQL_SYSVAR(init_flags), MYSQL_SYSVAR(checkpointing_period), MYSQL_SYSVAR(prelock_empty), MYSQL_SYSVAR(checkpoint_lock), @@ -1450,6 +1466,7 @@ static struct st_mysql_sys_var *tokudb_system_variables[] = { MYSQL_SYSVAR(disable_slow_upsert), #endif MYSQL_SYSVAR(analyze_time), + MYSQL_SYSVAR(analyze_delete_fraction), MYSQL_SYSVAR(fsync_log_period), #if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL MYSQL_SYSVAR(gdb_path), @@ -1472,6 +1489,10 @@ static struct st_mysql_sys_var *tokudb_system_variables[] = { MYSQL_SYSVAR(rpl_unique_checks_delay), MYSQL_SYSVAR(rpl_lookup_rows), MYSQL_SYSVAR(rpl_lookup_rows_delay), + MYSQL_SYSVAR(rpl_check_readonly), + MYSQL_SYSVAR(optimize_index_name), + MYSQL_SYSVAR(optimize_index_fraction), + MYSQL_SYSVAR(optimize_throttle), NULL }; @@ -1693,6 +1714,8 @@ static int tokudb_fractal_tree_info(TABLE *table, THD *thd) { error = tmp_cursor->c_get(tmp_cursor, &curr_key, &curr_val, DB_NEXT); if (!error) { error = tokudb_report_fractal_tree_info_for_db(&curr_key, &curr_val, table, thd); + if (error) + error = 0; // ignore read uncommitted errors } if (!error && thd_killed(thd)) error = ER_QUERY_INTERRUPTED; @@ -1964,6 +1987,35 @@ static int tokudb_fractal_tree_block_map_done(void *p) { return 0; } +#if TOKU_INCLUDE_LOCK_TIMEOUT_QUERY_STRING +struct tokudb_search_txn_extra { + bool match_found; + uint64_t match_txn_id; + uint64_t match_client_id; +}; + +static int tokudb_search_txn_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { + uint64_t txn_id = txn->id64(txn); + uint64_t client_id = txn->get_client_id(txn); + struct tokudb_search_txn_extra *e = reinterpret_cast<struct tokudb_search_txn_extra *>(extra); + if (e->match_txn_id == txn_id) { + e->match_found = true; + e->match_client_id = client_id; + return 1; + } + return 0; +} + +static bool tokudb_txn_id_to_client_id(THD *thd, uint64_t blocking_txnid, uint64_t *blocking_client_id) { + struct tokudb_search_txn_extra e = { false, blocking_txnid, 0}; + (void) db_env->iterate_live_transactions(db_env, tokudb_search_txn_callback, &e); + if (e.match_found) { + *blocking_client_id = e.match_client_id; + } + return e.match_found; +} +#endif + static void tokudb_pretty_key(const DB *db, const DBT *key, const char *default_key, String *out) { if (key->data == NULL) { out->append(default_key); @@ -2013,8 +2065,9 @@ static void tokudb_lock_timeout_callback(DB *db, uint64_t requesting_txnid, cons // generate a JSON document with the lock timeout info String log_str; log_str.append("{"); + uint64_t mysql_thread_id = thd->thread_id; log_str.append("\"mysql_thread_id\":"); - log_str.append_ulonglong(thd->thread_id); + log_str.append_ulonglong(mysql_thread_id); log_str.append(", \"dbname\":"); log_str.append("\""); log_str.append(tokudb_get_index_name(db)); log_str.append("\""); log_str.append(", \"requesting_txnid\":"); @@ -2054,7 +2107,18 @@ static void tokudb_lock_timeout_callback(DB *db, uint64_t requesting_txnid, cons } // dump to stderr if (lock_timeout_debug & 2) { - sql_print_error("%s: %s", tokudb_hton_name, log_str.c_ptr()); + sql_print_error("%s: lock timeout %s", tokudb_hton_name, log_str.c_ptr()); + LEX_STRING *qs = thd_query_string(thd); + sql_print_error("%s: requesting_thread_id:%" PRIu64 " q:%.*s", tokudb_hton_name, mysql_thread_id, (int) qs->length, qs->str); +#if TOKU_INCLUDE_LOCK_TIMEOUT_QUERY_STRING + uint64_t blocking_thread_id = 0; + if (tokudb_txn_id_to_client_id(thd, blocking_txnid, &blocking_thread_id)) { + String blocking_qs; + if (get_thread_query_string(blocking_thread_id, blocking_qs) == 0) { + sql_print_error("%s: blocking_thread_id:%" PRIu64 " q:%.*s", tokudb_hton_name, blocking_thread_id, blocking_qs.length(), blocking_qs.c_ptr()); + } + } +#endif } } } @@ -2064,6 +2128,7 @@ static struct st_mysql_information_schema tokudb_trx_information_schema = { MYSQ static ST_FIELD_INFO tokudb_trx_field_info[] = { {"trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, {"trx_mysql_thread_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"trx_time", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} }; @@ -2072,12 +2137,17 @@ struct tokudb_trx_extra { TABLE *table; }; -static int tokudb_trx_callback(uint64_t txn_id, uint64_t client_id, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { +static int tokudb_trx_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { + uint64_t txn_id = txn->id64(txn); + uint64_t client_id = txn->get_client_id(txn); + uint64_t start_time = txn->get_start_time(txn); struct tokudb_trx_extra *e = reinterpret_cast<struct tokudb_trx_extra *>(extra); THD *thd = e->thd; TABLE *table = e->table; table->field[0]->store(txn_id, false); table->field[1]->store(client_id, false); + uint64_t tnow = (uint64_t) time(NULL); + table->field[2]->store(tnow >= start_time ? tnow - start_time : 0, false); int error = schema_table_store_record(thd, table); if (!error && thd_killed(thd)) error = ER_QUERY_INTERRUPTED; @@ -2225,7 +2295,9 @@ struct tokudb_locks_extra { TABLE *table; }; -static int tokudb_locks_callback(uint64_t txn_id, uint64_t client_id, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { +static int tokudb_locks_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { + uint64_t txn_id = txn->id64(txn); + uint64_t client_id = txn->get_client_id(txn); struct tokudb_locks_extra *e = reinterpret_cast<struct tokudb_locks_extra *>(extra); THD *thd = e->thd; TABLE *table = e->table; diff --git a/storage/tokudb/hatoku_hton.h b/storage/tokudb/hatoku_hton.h index 797e8ff48bc..71d78e57d63 100644 --- a/storage/tokudb/hatoku_hton.h +++ b/storage/tokudb/hatoku_hton.h @@ -316,16 +316,9 @@ static MYSQL_THDVAR_BOOL(disable_slow_upsert, ); #endif -static MYSQL_THDVAR_UINT(analyze_time, - 0, - "analyze time", - NULL, - NULL, - 5, // default - 0, // min - ~0U, // max - 1 // blocksize -); +static MYSQL_THDVAR_UINT(analyze_time, 0, "analyze time (seconds)", NULL /*check*/, NULL /*update*/, 5 /*default*/, 0 /*min*/, ~0U /*max*/, 1 /*blocksize*/); + +static MYSQL_THDVAR_DOUBLE(analyze_delete_fraction, 0, "fraction of rows allowed to be deleted", NULL /*check*/, NULL /*update*/, 1.0 /*def*/, 0 /*min*/, 1.0 /*max*/, 1); static void tokudb_checkpoint_lock(THD * thd); static void tokudb_checkpoint_unlock(THD * thd); @@ -430,7 +423,7 @@ static int tokudb_killed_callback(void) { return thd_killed(thd); } -static bool tokudb_killed_thd_callback(void *extra) { +static bool tokudb_killed_thd_callback(void *extra, uint64_t deleted_rows) { THD *thd = static_cast<THD *>(extra); return thd_killed(thd) != 0; } @@ -491,6 +484,15 @@ static MYSQL_THDVAR_BOOL(rpl_lookup_rows, PLUGIN_VAR_THDLOCAL, "lookup a row on static MYSQL_THDVAR_ULONGLONG(rpl_lookup_rows_delay, PLUGIN_VAR_THDLOCAL, "time in milliseconds to add to lookups on replication slave", NULL, NULL, 0 /*default*/, 0 /*min*/, ~0ULL /*max*/, 1 /*blocksize*/); +static MYSQL_THDVAR_BOOL(rpl_check_readonly, PLUGIN_VAR_THDLOCAL, "check if the slave is read only", + NULL /*check*/, NULL /*update*/, true /*default*/); + +static MYSQL_THDVAR_STR(optimize_index_name, PLUGIN_VAR_THDLOCAL + PLUGIN_VAR_MEMALLOC, "optimize index name (default all indexes)", NULL /*check*/, NULL /*update*/, NULL /*default*/); + +static MYSQL_THDVAR_DOUBLE(optimize_index_fraction, 0, "optimize index fraction (default 1.0 all)", NULL /*check*/, NULL /*update*/, 1.0 /*def*/, 0 /*min*/, 1.0 /*max*/, 1); + +static MYSQL_THDVAR_ULONGLONG(optimize_throttle, 0, "optimize throttle (default no throttle)", NULL /*check*/, NULL /*update*/, 0 /*def*/, 0 /*min*/, ~0ULL /*max*/, 1); + extern HASH tokudb_open_tables; extern pthread_mutex_t tokudb_mutex; extern uint32_t tokudb_write_status_frequency; diff --git a/storage/tokudb/mysql-test/rpl/disabled.def b/storage/tokudb/mysql-test/rpl/disabled.def index efa4be9e16c..4c1a9a3e785 100644 --- a/storage/tokudb/mysql-test/rpl/disabled.def +++ b/storage/tokudb/mysql-test/rpl/disabled.def @@ -10,3 +10,6 @@ rpl_tokudb_write_pk: unreliable, uses timestamp differences rpl_tokudb_write_pk_uc1: unreliable, uses timestamp differences rpl_tokudb_write_unique: unreliable, uses timestamp differences rpl_tokudb_write_unique_uc1: unreliable, uses timestamp differences +rpl_tokudb_read_only_ff: unreliable, uses timestamp differences +rpl_tokudb_read_only_tf: unreliable, uses timestamp differences +rpl_tokudb_read_only_tt: unreliable, uses timestamp differences diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_ff.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_ff.result new file mode 100644 index 00000000000..6db2036d933 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_ff.result @@ -0,0 +1,14 @@ +include/master-slave.inc +[connection master] +drop table if exists t; +create table t (a bigint not null, primary key(a)) engine=tokudb; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); +select unix_timestamp()-@tstart <= 10; +unix_timestamp()-@tstart <= 10 +1 +include/diff_tables.inc [master:test.t, slave:test.t] +drop table if exists t; +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_ft.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_ft.result new file mode 100644 index 00000000000..3bcd3e8ccdd --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_ft.result @@ -0,0 +1,14 @@ +include/master-slave.inc +[connection master] +drop table if exists t; +create table t (a bigint not null, primary key(a)) engine=tokudb; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); +select unix_timestamp()-@tstart <= 10; +unix_timestamp()-@tstart <= 10 +0 +include/diff_tables.inc [master:test.t, slave:test.t] +drop table if exists t; +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_tf.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_tf.result new file mode 100644 index 00000000000..6db2036d933 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_tf.result @@ -0,0 +1,14 @@ +include/master-slave.inc +[connection master] +drop table if exists t; +create table t (a bigint not null, primary key(a)) engine=tokudb; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); +select unix_timestamp()-@tstart <= 10; +unix_timestamp()-@tstart <= 10 +1 +include/diff_tables.inc [master:test.t, slave:test.t] +drop table if exists t; +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_tt.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_tt.result new file mode 100644 index 00000000000..6db2036d933 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_read_only_tt.result @@ -0,0 +1,14 @@ +include/master-slave.inc +[connection master] +drop table if exists t; +create table t (a bigint not null, primary key(a)) engine=tokudb; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); +select unix_timestamp()-@tstart <= 10; +unix_timestamp()-@tstart <= 10 +1 +include/diff_tables.inc [master:test.t, slave:test.t] +drop table if exists t; +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ff-slave.opt b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ff-slave.opt new file mode 100644 index 00000000000..b9eb687d8d5 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ff-slave.opt @@ -0,0 +1 @@ +--read-only=OFF --tokudb-rpl-check-readonly=OFF --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ff.test b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ff.test new file mode 100644 index 00000000000..c77e4b49605 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ff.test @@ -0,0 +1,53 @@ +# test replicated write rows log events on a table with a primary key. +# the slave is read only with tokudb unique checks disabled. + +source include/have_tokudb.inc; +let $engine=tokudb; +source include/have_binlog_format_row.inc; +source include/master-slave.inc; + +# initialize +connection master; +disable_warnings; +drop table if exists t; +enable_warnings; + +connection slave; +# show variables like 'read_only'; +# show variables like 'tokudb_rpl_unique_checks%'; + +# insert some rows +connection master; +# select @@binlog_format; +# select @@autocommit; +eval create table t (a bigint not null, primary key(a)) engine=$engine; +# show create table t; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +connection master; +select unix_timestamp()-@tstart <= 10; + +connection slave; +# insert into t values (5); # test read-only +# show create table t; + +# diff tables +connection master; +--let $diff_tables= master:test.t, slave:test.t +source include/diff_tables.inc; + +# cleanup +connection master; +drop table if exists t; + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; + diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ft-slave.opt b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ft-slave.opt new file mode 100644 index 00000000000..8283875e8a7 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ft-slave.opt @@ -0,0 +1 @@ +--read-only=OFF --tokudb-rpl-check-readonly=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ft.test b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ft.test new file mode 100644 index 00000000000..c77e4b49605 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_ft.test @@ -0,0 +1,53 @@ +# test replicated write rows log events on a table with a primary key. +# the slave is read only with tokudb unique checks disabled. + +source include/have_tokudb.inc; +let $engine=tokudb; +source include/have_binlog_format_row.inc; +source include/master-slave.inc; + +# initialize +connection master; +disable_warnings; +drop table if exists t; +enable_warnings; + +connection slave; +# show variables like 'read_only'; +# show variables like 'tokudb_rpl_unique_checks%'; + +# insert some rows +connection master; +# select @@binlog_format; +# select @@autocommit; +eval create table t (a bigint not null, primary key(a)) engine=$engine; +# show create table t; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +connection master; +select unix_timestamp()-@tstart <= 10; + +connection slave; +# insert into t values (5); # test read-only +# show create table t; + +# diff tables +connection master; +--let $diff_tables= master:test.t, slave:test.t +source include/diff_tables.inc; + +# cleanup +connection master; +drop table if exists t; + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; + diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tf-slave.opt b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tf-slave.opt new file mode 100644 index 00000000000..21e57d27c17 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tf-slave.opt @@ -0,0 +1 @@ +--read-only=ON --tokudb-rpl-check-readonly=OFF --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tf.test b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tf.test new file mode 100644 index 00000000000..c77e4b49605 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tf.test @@ -0,0 +1,53 @@ +# test replicated write rows log events on a table with a primary key. +# the slave is read only with tokudb unique checks disabled. + +source include/have_tokudb.inc; +let $engine=tokudb; +source include/have_binlog_format_row.inc; +source include/master-slave.inc; + +# initialize +connection master; +disable_warnings; +drop table if exists t; +enable_warnings; + +connection slave; +# show variables like 'read_only'; +# show variables like 'tokudb_rpl_unique_checks%'; + +# insert some rows +connection master; +# select @@binlog_format; +# select @@autocommit; +eval create table t (a bigint not null, primary key(a)) engine=$engine; +# show create table t; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +connection master; +select unix_timestamp()-@tstart <= 10; + +connection slave; +# insert into t values (5); # test read-only +# show create table t; + +# diff tables +connection master; +--let $diff_tables= master:test.t, slave:test.t +source include/diff_tables.inc; + +# cleanup +connection master; +drop table if exists t; + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; + diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tt-slave.opt b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tt-slave.opt new file mode 100644 index 00000000000..fd77ee0da9c --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tt-slave.opt @@ -0,0 +1 @@ +--read-only=ON --tokudb-rpl-check-readonly=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF diff --git a/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tt.test b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tt.test new file mode 100644 index 00000000000..c77e4b49605 --- /dev/null +++ b/storage/tokudb/mysql-test/rpl/t/rpl_tokudb_read_only_tt.test @@ -0,0 +1,53 @@ +# test replicated write rows log events on a table with a primary key. +# the slave is read only with tokudb unique checks disabled. + +source include/have_tokudb.inc; +let $engine=tokudb; +source include/have_binlog_format_row.inc; +source include/master-slave.inc; + +# initialize +connection master; +disable_warnings; +drop table if exists t; +enable_warnings; + +connection slave; +# show variables like 'read_only'; +# show variables like 'tokudb_rpl_unique_checks%'; + +# insert some rows +connection master; +# select @@binlog_format; +# select @@autocommit; +eval create table t (a bigint not null, primary key(a)) engine=$engine; +# show create table t; +select unix_timestamp() into @tstart; +insert into t values (1); +insert into t values (2),(3); +insert into t values (4); + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +connection master; +select unix_timestamp()-@tstart <= 10; + +connection slave; +# insert into t values (5); # test read-only +# show create table t; + +# diff tables +connection master; +--let $diff_tables= master:test.t, slave:test.t +source include/diff_tables.inc; + +# cleanup +connection master; +drop table if exists t; + +sync_slave_with_master; +# source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; + diff --git a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result index 018900c7b98..190581eddae 100644 --- a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result +++ b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result @@ -2,7 +2,7 @@ set default_storage_engine='tokudb'; set tokudb_prelock_empty=false; drop table if exists t; create table t (id int primary key); -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name @@ -19,7 +19,7 @@ TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name REQUEST_TRX_ID BLOCK_TRX_ID ./test/t-main 0001000000 0001000000 LOCK_WAITS_START_TIME test t main -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id TRX_ID MYSQL_ID TRX_ID MYSQL_ID @@ -31,7 +31,7 @@ select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name ERROR 23000: Duplicate entry '1' for key 'PRIMARY' commit; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name @@ -48,7 +48,7 @@ TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name REQUEST_TRX_ID BLOCK_TRX_ID ./test/t-main 0001000000 0001000000 LOCK_WAITS_START_TIME test t main -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id TRX_ID MYSQL_ID TRX_ID MYSQL_ID @@ -59,7 +59,7 @@ TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name commit; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name diff --git a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_timeout.result b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_timeout.result index b9fca50b507..13cdad7a438 100644 --- a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_timeout.result +++ b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_timeout.result @@ -2,7 +2,7 @@ set default_storage_engine='tokudb'; set tokudb_prelock_empty=false; drop table if exists t; create table t (id int primary key); -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name @@ -19,7 +19,7 @@ TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name REQUEST_TRX_ID BLOCK_TRX_ID ./test/t-main 0001000000 0001000000 LOCK_WAITS_START_TIME test t main -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id TRX_ID MYSQL_ID TRX_ID MYSQL_ID @@ -30,7 +30,7 @@ select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name ERROR HY000: Lock wait timeout exceeded; try restarting transaction commit; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name diff --git a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_trx.result b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_trx.result index e4c1adcca19..63e4816e16e 100644 --- a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_trx.result +++ b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_trx.result @@ -1,23 +1,23 @@ set default_storage_engine='tokudb'; set tokudb_prelock_empty=false; drop table if exists t; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id set autocommit=0; create table t (id int primary key); insert into t values (1); -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id TXN_ID_DEFAULT CLIENT_ID_DEFAULT commit; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id set autocommit=0; insert into t values (2); -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id TXN_ID_A CLIENT_ID_A commit; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id drop table t; diff --git a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test index c4f9ccefe5c..0f712000527 100644 --- a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test +++ b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test @@ -13,7 +13,7 @@ create table t (id int primary key); # verify that txn_a insert (1) blocks txn_b insert (1) and txn_b gets a duplicate key error # should be empty -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; @@ -43,7 +43,7 @@ select * from information_schema.tokudb_lock_waits; # should find the presence of two transactions replace_column 1 TRX_ID 2 MYSQL_ID; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; connection conn_a; commit; @@ -66,7 +66,7 @@ disconnect conn_b; # verify that the lock on the 2nd transaction has been released # should be be empty -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; @@ -96,7 +96,7 @@ select * from information_schema.tokudb_lock_waits; # should find the presence of two transactions replace_column 1 TRX_ID 2 MYSQL_ID; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; connection conn_a; commit; @@ -116,7 +116,7 @@ disconnect conn_b; # verify that the lock on the 2nd transaction has been released # should be be empty -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; diff --git a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_timeout.test b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_timeout.test index 75929fa3b3d..3011443fa04 100644 --- a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_timeout.test +++ b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_timeout.test @@ -10,7 +10,7 @@ enable_warnings; create table t (id int primary key); # should be empty -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; @@ -40,7 +40,7 @@ select * from information_schema.tokudb_lock_waits; # should find the presence of two transactions replace_column 1 TRX_ID 2 MYSQL_ID; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; connection conn_a; sleep 5; # sleep longer than the lock timer to force a lock timeout on txn_b @@ -61,7 +61,7 @@ disconnect conn_a; disconnect conn_b; # should be be empty -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; diff --git a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_trx.test b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_trx.test index b1d5c7e5009..d3c2636ba54 100644 --- a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_trx.test +++ b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_trx.test @@ -8,7 +8,7 @@ drop table if exists t; enable_warnings; # should be empty -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; # should have my txn let $default_id=`select connection_id()`; @@ -16,11 +16,11 @@ set autocommit=0; create table t (id int primary key); insert into t values (1); replace_column 1 TXN_ID_DEFAULT 2 CLIENT_ID_DEFAULT; -eval select * from information_schema.tokudb_trx; +eval select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; # should be empty commit; -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; connect(conn_a,localhost,root,,); let a_id=`select connection_id()`; @@ -29,13 +29,13 @@ insert into t values (2); connection default; replace_column 1 TXN_ID_A 2 CLIENT_ID_A; -eval select * from information_schema.tokudb_trx; +eval select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; connection conn_a; commit; connection default; # should be empty -select * from information_schema.tokudb_trx; +select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; disconnect conn_a; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/5733_tokudb.result b/storage/tokudb/mysql-test/tokudb_bugs/r/5733_tokudb.result index 07e8b37c263..a05587cb0a6 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/r/5733_tokudb.result +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/5733_tokudb.result @@ -10005,7 +10005,7 @@ insert into t values (9999,0); commit; explain select id from t where id>0 limit 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t index_or_range PRIMARY PRIMARY 8 NULL # Using where; Using index_or_range +1 SIMPLE t range_or_index PRIMARY PRIMARY 8 NULL # Using where; Using index explain select * from t where id>0 limit 10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t range PRIMARY PRIMARY 8 NULL # Using where diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db788-optimize-index-name.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db788-optimize-index-name.result new file mode 100644 index 00000000000..019a8299f3c --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db788-optimize-index-name.result @@ -0,0 +1,19 @@ +set default_storage_engine='tokudb'; +drop table if exists t; +create table t (a int, b int, c int, primary key(a), key(b), key(c)); +set tokudb_optimize_index_name='primary'; +optimize table t; +Table Op Msg_type Msg_text +test.t optimize note Table does not support optimize, doing recreate + analyze instead +test.t optimize status OK +set tokudb_optimize_index_name='b'; +optimize table t; +Table Op Msg_type Msg_text +test.t optimize note Table does not support optimize, doing recreate + analyze instead +test.t optimize status OK +set tokudb_optimize_index_name='c'; +optimize table t; +Table Op Msg_type Msg_text +test.t optimize note Table does not support optimize, doing recreate + analyze instead +test.t optimize status OK +drop table t; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db801.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db801.result new file mode 100644 index 00000000000..800db69ba39 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db801.result @@ -0,0 +1,18 @@ +set default_storage_engine=tokudb; +drop table if exists t; +create table t (id int not null primary key, c int not null) engine=tokudb; +insert into t values (1,0); +begin; +update t set c=10 where id=1; +update t set c=100; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +rollback; +drop table t; +create table t (id int not null primary key, c int not null) engine=tokudb partition by hash(id) partitions 1; +insert into t values (1,0); +begin; +update t set c=10 where id=1; +update t set c=100; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +rollback; +drop table t; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db805.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db805.result new file mode 100644 index 00000000000..1bc0372f1b8 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db805.result @@ -0,0 +1,18 @@ +drop table if exists t1,t3; +create table t3(a3 int,b3 decimal(0,0),c3 int,d3 int,primary key(a3,b3)) engine=TOKUDB; +LOCK TABLES t3 WRITE; +create temporary table t1(f1 int,index(f1)) engine=innodb; +INSERT INTO t1 VALUES(1),(1),(1); +select * from t1; +f1 +1 +1 +1 +ALTER TABLE t1 engine=TOKUDB; +select * from t1; +f1 +1 +1 +1 +unlock tables; +drop table t1,t3; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db806.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db806.result new file mode 100644 index 00000000000..ae87dbab281 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db806.result @@ -0,0 +1,9 @@ +drop table if exists t1,t3; +CREATE TABLE t3(a int,c int,d int)engine=TOKUDB; +lock table t3 read; +create temporary table t1 engine=tokudb as SELECT 1; +select * from t1; +1 +1 +unlock tables; +drop table t1,t3; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db811.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db811.result new file mode 100644 index 00000000000..1d26f43c9dd --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db811.result @@ -0,0 +1,14 @@ +drop table if exists t2,t3,t4; +CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB; +CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb; +CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 13; +LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE; +INSERT INTO t2(a)VALUES (REPEAT(0,1)); +ALTER TABLE t2 ADD COLUMN(c INT); +alter table t4 add column c int; +UPDATE t2 SET a=1; +select * from t2; +a b c +1 NULL NULL +unlock tables; +drop table t2,t3,t4; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db811s.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db811s.result new file mode 100644 index 00000000000..0a50e63e037 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db811s.result @@ -0,0 +1,14 @@ +drop table if exists t2,t3,t4; +CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB; +CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb; +CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 1; +LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE; +INSERT INTO t2(a)VALUES (REPEAT(0,1)); +ALTER TABLE t2 ADD COLUMN(c INT); +alter table t4 add column c int; +UPDATE t2 SET a=1; +select * from t2; +a b c +1 NULL NULL +unlock tables; +drop table t2,t3,t4; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db817.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db817.result new file mode 100644 index 00000000000..d69f0dabcb3 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db817.result @@ -0,0 +1,33 @@ +drop table if exists ti; +create table ti (id int primary key) engine=innodb; +begin; +insert into ti values (0); +savepoint b; +insert into ti values (1); +savepoint a2; +insert into ti values (2); +savepoint b; +insert into ti values (3); +rollback to a2; +commit; +select * from ti; +id +0 +1 +drop table if exists tt; +create table tt (id int primary key) engine=tokudb; +begin; +insert into tt values (0); +savepoint b; +insert into tt values (1); +savepoint a2; +insert into tt values (2); +savepoint b; +insert into tt values (3); +rollback to a2; +commit; +select * from tt; +id +0 +1 +drop table ti,tt; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db823.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db823.result new file mode 100644 index 00000000000..d94da5c0673 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db823.result @@ -0,0 +1,11 @@ +drop table if exists s,t; +create table s (id int) engine=tokudb; +lock tables s write; +create temporary table t (id int, key(id)) engine=innodb; +insert into t values (1); +alter table t engine=tokudb; +select * from t; +id +1 +unlock tables; +drop table s, t; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/memcache_dirty.result b/storage/tokudb/mysql-test/tokudb_bugs/r/memcache_dirty.result deleted file mode 100644 index 2ca26cd5c56..00000000000 --- a/storage/tokudb/mysql-test/tokudb_bugs/r/memcache_dirty.result +++ /dev/null @@ -1,14 +0,0 @@ -SET DEFAULT_STORAGE_ENGINE = 'tokudb'; -DROP TABLE IF EXISTS t1; -create table t1 (i int, j int, primary key (i))engine=TokuDB; -insert into t1 values (0,0) MEMCACHE_DIRTY 'a'; -insert into t1 values (1,0) MEMCACHE_DIRTY 'b', 'c'; -update t1 set j=j+1 where i=0 MEMCACHE_DIRTY 'a'; -update t1 set j=j+1 where i=1 MEMCACHE_DIRTY 'b', 'c'; -insert into t1 values (0,0) on duplicate key update j=j+1 MEMCACHE_DIRTY 'a'; -insert into t1 values (2,0) on duplicate key update j=j+1 MEMCACHE_DIRTY 'a', 'b'; -replace into t1 values (0,3) MEMCACHE_DIRTY 'a'; -replace into t1 values (3,3) MEMCACHE_DIRTY 'a', 'b'; -delete from t1 where i=0 MEMCACHE_DIRTY 'a'; -delete from t1 where i=1 MEMCACHE_DIRTY 'b', 'c'; -DROP TABLE t1; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/tokudb718.result b/storage/tokudb/mysql-test/tokudb_bugs/r/tokudb718.result index e63f73caf20..0cf75d40847 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/r/tokudb718.result +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/tokudb718.result @@ -3,7 +3,8 @@ drop table if exists t; create table t (id int primary key); begin; insert into t values (1),(2); -select * from information_schema.tokudb_fractal_tree_info; -ERROR HY000: Got error -30994 "Internal error < 0 (Not system error)" from storage engine TokuDB +select dictionary_name from information_schema.tokudb_fractal_tree_info; +dictionary_name +./test/t-status commit; drop table t; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/5733_tokudb.test b/storage/tokudb/mysql-test/tokudb_bugs/t/5733_tokudb.test index 2e30c839905..192004cb113 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/5733_tokudb.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/5733_tokudb.test @@ -20,9 +20,18 @@ while ($i < $n) { } commit; -# TokuDB may do index or range scan on this. Both are ok -replace_column 9 #; ---replace_result index index_or_range range index_or_range +# the plan for the following query should be a range scan. about 1 of 10 times, +# the plan is an index scan. the different scan type occurs because the query optimizer +# is handed different row counts by tokudb::records_in_range. the cost estimates made +# by the query optimizer are very close to begin with. sometimes, the cost of an index +# scan is less than the cost of a range scan. +# +# if a tokudb checkpoint occurs before this query is run, then the records_in_range +# function returns a larger than expected row estimate. +# +# column 4 is the join type (should be range or index) +# column 9 is the estimated key count +replace_column 4 range_or_index 9 #; explain select id from t where id>0 limit 10; replace_column 9 #; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db788-optimize-index-name.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db788-optimize-index-name.test new file mode 100644 index 00000000000..644f00a5862 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db788-optimize-index-name.test @@ -0,0 +1,21 @@ +# test tokudb_optimize_index_name session variable +set default_storage_engine='tokudb'; +source include/have_tokudb.inc; +disable_warnings; +drop table if exists t; +enable_warnings; + +create table t (a int, b int, c int, primary key(a), key(b), key(c)); +# optimize primary key +set tokudb_optimize_index_name='primary'; +optimize table t; + +# optimize key b +set tokudb_optimize_index_name='b'; +optimize table t; + +# optimize key c +set tokudb_optimize_index_name='c'; +optimize table t; + +drop table t; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db801.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db801.test new file mode 100644 index 00000000000..8a8fcea1496 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db801.test @@ -0,0 +1,50 @@ +# test for the DB-801 bug on mysql-5.5.41 +source include/have_tokudb.inc; +source include/have_partition.inc; +set default_storage_engine=tokudb; + +disable_warnings; +drop table if exists t; +enable_warnings; + +# run the test on a tokudb table +create table t (id int not null primary key, c int not null) engine=tokudb; + +insert into t values (1,0); + +connect(conn1,localhost,root,,); +connection default; +begin; +update t set c=10 where id=1; + +connection conn1; +--error ER_LOCK_WAIT_TIMEOUT +update t set c=100; + +connection default; +rollback; +disconnect conn1; + +drop table t; + +# run the test on a partitioned tokudb table +create table t (id int not null primary key, c int not null) engine=tokudb partition by hash(id) partitions 1; + +insert into t values (1,0); + +connect(conn1,localhost,root,,); +connection default; +begin; +update t set c=10 where id=1; + +connection conn1; +--error ER_LOCK_WAIT_TIMEOUT +update t set c=100; + +connection default; +rollback; +disconnect conn1; + +drop table t; + + diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db805.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db805.test new file mode 100644 index 00000000000..1114de6b325 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db805.test @@ -0,0 +1,17 @@ +# DB-805 test that conversion of t1 from innodb to tokudb can write rows +source include/have_tokudb.inc; +source include/have_innodb.inc; +disable_warnings; +drop table if exists t1,t3; +enable_warnings; + +create table t3(a3 int,b3 decimal(0,0),c3 int,d3 int,primary key(a3,b3)) engine=TOKUDB; +LOCK TABLES t3 WRITE; +create temporary table t1(f1 int,index(f1)) engine=innodb; +INSERT INTO t1 VALUES(1),(1),(1); +select * from t1; +ALTER TABLE t1 engine=TOKUDB; +select * from t1; +unlock tables; + +drop table t1,t3; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db806.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db806.test new file mode 100644 index 00000000000..3815e59f78c --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db806.test @@ -0,0 +1,13 @@ +# DB-806 test that lock tables and create select can write rows to the new table +source include/have_tokudb.inc; +disable_warnings; +drop table if exists t1,t3; +enable_warnings; + +CREATE TABLE t3(a int,c int,d int)engine=TOKUDB; +lock table t3 read; +create temporary table t1 engine=tokudb as SELECT 1; +select * from t1; +unlock tables; + +drop table t1,t3;
\ No newline at end of file diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db811.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db811.test new file mode 100644 index 00000000000..509f482765e --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db811.test @@ -0,0 +1,22 @@ +# DB-811 test that alter table t2 updates both the schema (FRM) and the data (tokudb files) + +source include/have_tokudb.inc; +source include/have_innodb.inc; +source include/have_partition.inc; +disable_warnings; +drop table if exists t2,t3,t4; +enable_warnings; + +CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB; +CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb; +CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 13; +LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE; +INSERT INTO t2(a)VALUES (REPEAT(0,1)); +ALTER TABLE t2 ADD COLUMN(c INT); +alter table t4 add column c int; +UPDATE t2 SET a=1; +select * from t2; +unlock tables; + +drop table t2,t3,t4; + diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db811s.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db811s.test new file mode 100644 index 00000000000..5b8c6ed79d3 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db811s.test @@ -0,0 +1,22 @@ +# DB-811 test that alter table t2 updates both the schema (FRM) and the data (tokudb files) + +source include/have_tokudb.inc; +source include/have_innodb.inc; +source include/have_partition.inc; +disable_warnings; +drop table if exists t2,t3,t4; +enable_warnings; + +CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB; +CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb; +CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 1; +LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE; +INSERT INTO t2(a)VALUES (REPEAT(0,1)); +ALTER TABLE t2 ADD COLUMN(c INT); +alter table t4 add column c int; +UPDATE t2 SET a=1; +select * from t2; +unlock tables; + +drop table t2,t3,t4; + diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db817.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db817.test new file mode 100644 index 00000000000..53c9edc3893 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db817.test @@ -0,0 +1,38 @@ +# verify that duplicate savepoint names in innodb and tokudb work the same +source include/have_innodb.inc; +source include/have_tokudb.inc; +disable_warnings; +drop table if exists ti; +enable_warnings; +create table ti (id int primary key) engine=innodb; +begin; +insert into ti values (0); +savepoint b; +insert into ti values (1); +savepoint a2; +insert into ti values (2); +savepoint b; +insert into ti values (3); +rollback to a2; +commit; +select * from ti; + +disable_warnings; +drop table if exists tt; +enable_warnings; +create table tt (id int primary key) engine=tokudb; +begin; +insert into tt values (0); +savepoint b; +insert into tt values (1); +savepoint a2; +insert into tt values (2); +savepoint b; +insert into tt values (3); +rollback to a2; +commit; +select * from tt; + +drop table ti,tt; + + diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db823.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db823.test new file mode 100644 index 00000000000..2e01c0e5797 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db823.test @@ -0,0 +1,16 @@ +# test DB-823 +# test that the conversion of table t from innodb to tokudb succeeds. +source include/have_tokudb.inc; +source include/have_innodb.inc; +disable_warnings; +drop table if exists s,t; +enable_warnings; +create table s (id int) engine=tokudb; +lock tables s write; +create temporary table t (id int, key(id)) engine=innodb; +insert into t values (1); +alter table t engine=tokudb; +select * from t; +unlock tables; +drop table s, t; + diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/memcache_dirty.test b/storage/tokudb/mysql-test/tokudb_bugs/t/memcache_dirty.test deleted file mode 100644 index e66c4cf0b6f..00000000000 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/memcache_dirty.test +++ /dev/null @@ -1,25 +0,0 @@ ---source include/have_tokudb.inc -# -# Record inconsistency. -# -# -SET DEFAULT_STORAGE_ENGINE = 'tokudb'; - ---disable_warnings -DROP TABLE IF EXISTS t1; ---enable_warnings -create table t1 (i int, j int, primary key (i))engine=TokuDB; -insert into t1 values (0,0) MEMCACHE_DIRTY 'a'; insert into t1 values (1,0) MEMCACHE_DIRTY 'b', 'c'; - -update t1 set j=j+1 where i=0 MEMCACHE_DIRTY 'a'; update t1 set j=j+1 where i=1 MEMCACHE_DIRTY 'b', 'c'; - -insert into t1 values (0,0) on duplicate key update j=j+1 MEMCACHE_DIRTY 'a'; insert into t1 values (2,0) on duplicate key update j=j+1 MEMCACHE_DIRTY 'a', 'b'; - -replace into t1 values (0,3) MEMCACHE_DIRTY 'a'; replace into t1 values (3,3) MEMCACHE_DIRTY 'a', 'b'; - -delete from t1 where i=0 MEMCACHE_DIRTY 'a'; delete from t1 where i=1 MEMCACHE_DIRTY 'b', 'c'; - - -# Final cleanup. -DROP TABLE t1; - diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb718.test b/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb718.test index 415bb7a2332..735a88afed8 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb718.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb718.test @@ -7,7 +7,6 @@ enable_warnings; create table t (id int primary key); begin; insert into t values (1),(2); ---error 1030 -select * from information_schema.tokudb_fractal_tree_info; +select dictionary_name from information_schema.tokudb_fractal_tree_info; commit; drop table t; diff --git a/storage/tokudb/scripts/common.sh b/storage/tokudb/scripts/common.sh index fc676ceeceb..fe39b9feca7 100644 --- a/storage/tokudb/scripts/common.sh +++ b/storage/tokudb/scripts/common.sh @@ -131,11 +131,20 @@ function parse_mysqlbuild() { tokudb_version=${BASH_REMATCH[6]} target_system=${BASH_REMATCH[7]} target_arch=${BASH_REMATCH[8]} + # verify targets if [ $target_system != $system ] ; then exitcode=1; fi if [ $target_arch != $arch ] ; then exitcode=1; fi + # split the version string into major.minor.patch + if [[ $mysql_version =~ ^([0-9]+)\.([0-9]+)\.([0-9]+.*) ]] ; then + mysql_version_major=${BASH_REMATCH[1]} + mysql_version_minor=${BASH_REMATCH[2]} + mysql_version_patch=${BASH_REMATCH[3]} + fi + local temp_tokudb_version=$tokudb_version + # decode enterprise if [[ $temp_tokudb_version =~ (.*)-e$ ]] ; then build_type=enterprise @@ -143,6 +152,7 @@ function parse_mysqlbuild() { else build_type=community fi + # decode debug if [[ $temp_tokudb_version =~ (.*)-debug$ ]] ; then build_debug=1 @@ -151,8 +161,9 @@ function parse_mysqlbuild() { else build_debug=0 fi + # set tag or HEAD - if [[ $temp_tokudb_version =~ ^([0-9]+)\\.([0-9]+)\\.([0-9]+) ]] ; then + if [[ $temp_tokudb_version =~ ^([0-9]+)\.([0-9]+)\.([0-9]+) ]] ; then git_tag=tokudb-$temp_tokudb_version else git_tag=HEAD @@ -160,6 +171,8 @@ function parse_mysqlbuild() { if [ -z $mysql_tree ] ; then mysql_tree=$mysql_distro-$mysql_version; fi if [ -z $jemalloc_tree ] ; then jemalloc_tree=$jemalloc_version; fi fi + + # set repository mysql_repo=$mysql_distro if [[ $mysql_version =~ ^([0-9]+\.[0-9]+) ]] ; then mysql_repo=$mysql_distro-${BASH_REMATCH[1]}; else exitcode=1; fi else @@ -174,6 +187,15 @@ function parse_mysql() { if [[ $mysql =~ ^(mysql|mariadb)-(.*)$ ]] ; then mysql_distro=${BASH_REMATCH[1]} mysql_version=${BASH_REMATCH[2]} + + # split the version string into major.minor.patch + if [[ $mysql_version =~ ^([0-9]+)\.([0-9]+)\.([0-9]+.*) ]] ; then + mysql_version_major=${BASH_REMATCH[1]} + mysql_version_minor=${BASH_REMATCH[2]} + mysql_version_patch=${BASH_REMATCH[3]} + fi + + # set repository mysql_repo=$mysql_distro if [[ $mysql_version =~ ^([0-9]+\.[0-9]+) ]] ; then mysql_repo=$mysql_distro-${BASH_REMATCH[1]}; else exitcode=1; fi exitcode=0 diff --git a/storage/tokudb/scripts/make.mysql.bash b/storage/tokudb/scripts/make.mysql.bash index 5654c1c7926..c1259797590 100755 --- a/storage/tokudb/scripts/make.mysql.bash +++ b/storage/tokudb/scripts/make.mysql.bash @@ -103,10 +103,23 @@ fi cd $mysql_distro-$mysql_version if [ $? != 0 ] ; then exit 1; fi +# extract mysql version patch number only +if [[ $mysql_version_patch =~ ^([0-9]+) ]] ; then p=${BASH_REMATCH[1]}; else p=$mysql_version_patch; fi + # install the backup source -if [ ! -d toku_backup ] ; then +tokudb_backup= +if [ $mysql_version_major -eq 5 -a $mysql_version_minor -eq 5 -a $p -le 40 ] ; then + tokudb_backup=patch github_download Tokutek/backup-$build_type $(git_tree $git_tag $backup_tree) backup-$build_type cp -r backup-$build_type/backup toku_backup +elif [ $build_type = enterprise ] ; then + tokudb_backup=plugin + github_download Tokutek/tokudb-backup-plugin $(git_tree $git_tag $backup_tree) tokudb-backup-plugin + mv tokudb-backup-plugin plugin + github_download Tokutek/backup-enterprise $(git_tree $git_tag $backup_tree) backup-enterprise + rm -rf plugin/tokudb-backup-plugin/backup + mv backup-enterprise/backup plugin/tokudb-backup-plugin + rm -rf backup-enterprise fi if [ ! -d tokudb-engine ] ; then @@ -153,6 +166,7 @@ function generate_cmake_cmd () { echo -n CC=$cc CXX=$cxx cmake \ -D BUILD_CONFIG=mysql_release \ + -D MYSQL_MAINTAINER_MODE=OFF \ -D CMAKE_BUILD_TYPE=$cmake_build_type \ -D CMAKE_TOKUDB_REVISION=$ft_revision \ -D TOKUDB_VERSION=tokudb-${tokudb_version} \ diff --git a/storage/tokudb/scripts/make.mysql.debug.env.bash b/storage/tokudb/scripts/make.mysql.debug.env.bash index b7c270cfbd7..3f8b4e6c219 100755 --- a/storage/tokudb/scripts/make.mysql.debug.env.bash +++ b/storage/tokudb/scripts/make.mysql.debug.env.bash @@ -55,14 +55,14 @@ function github_clone() { git_tag= mysql=mysql-5.5 -mysql_tree=mysql-5.5.35 +mysql_tree=mysql-5.5.41 jemalloc=jemalloc jemalloc_tree=3.6.0 tokudbengine=tokudb-engine tokudbengine_tree=master ftindex=ft-index ftindex_tree=master -backup=backup-community +backup=tokudb-backup-plugin backup_tree=master cc=gcc cxx=g++ @@ -119,9 +119,9 @@ if [ $? != 0 ] ; then exit 1; fi ln -s ../../$tokudbengine/storage/tokudb tokudb if [ $? != 0 ] ; then exit 1; fi popd -pushd $mysql_tree +pushd $mysql_tree/plugin if [ $? != 0 ] ; then exit 1; fi -ln -s ../$backup/backup toku_backup +ln -s ../../$backup $backup if [ $? != 0 ] ; then exit 1; fi popd pushd $mysql_tree/scripts @@ -148,13 +148,15 @@ fi pushd $build_dir if [ $? != 0 ] ; then exit 1; fi extra_cmake_options="-DCMAKE_LINK_DEPENDS_NO_SHARED=ON" +extra_cmake_options+=" -DBUILD_TESTING=OFF" +extra_cmake_options+=" -DMYSQL_MAINTAINER_MODE=OFF" if (( $cmake_valgrind )) ; then extra_cmake_options+=" -DUSE_VALGRIND=ON" fi if (( $cmake_debug_paranoid )) ; then extra_cmake_options+=" -DTOKU_DEBUG_PARANOID=ON" fi -CC=$cc CXX=$cxx cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=$install_dir -DBUILD_TESTING=OFF $extra_cmake_options ../$mysql_tree +CC=$cc CXX=$cxx cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=$install_dir $extra_cmake_options ../$mysql_tree if [ $? != 0 ] ; then exit 1; fi make -j4 install if [ $? != 0 ] ; then exit 1; fi diff --git a/storage/tokudb/tokudb_card.h b/storage/tokudb/tokudb_card.h index 797c705bbaf..22e6fb9b3da 100644 --- a/storage/tokudb/tokudb_card.h +++ b/storage/tokudb/tokudb_card.h @@ -218,15 +218,32 @@ namespace tokudb { return error; } + struct analyze_card_cursor_callback_extra { + int (*analyze_progress)(void *extra, uint64_t rows); + void *analyze_extra; + uint64_t *rows; + uint64_t *deleted_rows; + }; + + bool analyze_card_cursor_callback(void *extra, uint64_t deleted_rows) { + analyze_card_cursor_callback_extra *a_extra = static_cast<analyze_card_cursor_callback_extra *>(extra); + *a_extra->deleted_rows += deleted_rows; + int r = a_extra->analyze_progress(a_extra->analyze_extra, *a_extra->rows); + sql_print_information("tokudb analyze_card_cursor_callback %u %" PRIu64 " %" PRIu64, r, *a_extra->deleted_rows, deleted_rows); + return r != 0; + } + // Compute records per key for all key parts of the ith key of the table. // For each key part, put records per key part in *rec_per_key_part[key_part_index]. // Returns 0 if success, otherwise an error number. // TODO statistical dives into the FT int analyze_card(DB *db, DB_TXN *txn, bool is_unique, uint64_t num_key_parts, uint64_t *rec_per_key_part, int (*key_compare)(DB *, const DBT *, const DBT *, uint), - int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra) { + int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra, + uint64_t *return_rows, uint64_t *return_deleted_rows) { int error = 0; uint64_t rows = 0; + uint64_t deleted_rows = 0; uint64_t unique_rows[num_key_parts]; if (is_unique && num_key_parts == 1) { // dont compute for unique keys with a single part. we already know the answer. @@ -235,6 +252,8 @@ namespace tokudb { DBC *cursor = NULL; error = db->cursor(db, txn, &cursor, 0); if (error == 0) { + analyze_card_cursor_callback_extra e = { analyze_progress, progress_extra, &rows, &deleted_rows }; + cursor->c_set_check_interrupt_callback(cursor, analyze_card_cursor_callback, &e); for (uint64_t i = 0; i < num_key_parts; i++) unique_rows[i] = 1; // stop looking when the entire dictionary was analyzed, or a cap on execution time was reached, or the analyze was killed. @@ -243,8 +262,8 @@ namespace tokudb { while (1) { error = cursor->c_get(cursor, &key, 0, DB_NEXT); if (error != 0) { - if (error == DB_NOTFOUND) - error = 0; // eof is not an error + if (error == DB_NOTFOUND || error == TOKUDB_INTERRUPTED) + error = 0; // not an error break; } rows++; @@ -287,10 +306,12 @@ namespace tokudb { } } // return cardinality - if (error == 0 || error == ETIME) { - for (uint64_t i = 0; i < num_key_parts; i++) - rec_per_key_part[i] = rows / unique_rows[i]; - } + if (return_rows) + *return_rows = rows; + if (return_deleted_rows) + *return_deleted_rows = deleted_rows; + for (uint64_t i = 0; i < num_key_parts; i++) + rec_per_key_part[i] = rows / unique_rows[i]; return error; } } |
