diff options
Diffstat (limited to 'storage/tokudb/ft-index/ft/tests')
38 files changed, 346 insertions, 113 deletions
diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-5097.cc b/storage/tokudb/ft-index/ft/tests/cachetable-5097.cc index 4352499875e..7c958dd3049 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-5097.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-5097.cc @@ -103,15 +103,16 @@ static int pe_callback ( void *ftnode_pv __attribute__((__unused__)), PAIR_ATTR bytes_to_free __attribute__((__unused__)), - PAIR_ATTR* bytes_freed, - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra ) { - *bytes_freed = make_pair_attr(1); if (check_pe_callback) { pe_callback_called = true; } usleep(4*1024*1024); + finalize(make_pair_attr(1), finalize_extra); return 0; } diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-5978.cc b/storage/tokudb/ft-index/ft/tests/cachetable-5978.cc index fb8273eef72..c72d67909e1 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-5978.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-5978.cc @@ -97,7 +97,7 @@ PATENT RIGHTS GRANT: // #5978 is fixed. Here is what we do. We have four pairs with // blocknums and fullhashes of 1,2,3,4. The cachetable has only // two bucket mutexes, so 1 and 3 share a pair mutex, as do 2 and 4. -// We pin all four with expensive write locks. Then, on backgroud threads,
+// We pin all four with expensive write locks. Then, on backgroud threads, // we call get_and_pin_nonblocking on 3, where the unlockers unpins 2, and // we call get_and_pin_nonblocking on 4, where the unlockers unpins 1. Run this // enough times, and we should see a deadlock before the fix, and no deadlock diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction2.cc b/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction2.cc index 7704a5b3c63..23926241b97 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction2.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction2.cc @@ -155,15 +155,16 @@ static int pe_callback ( void *ftnode_pv, PAIR_ATTR UU(bytes_to_free), - PAIR_ATTR* bytes_freed, - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra ) { - *bytes_freed = make_pair_attr(bytes_to_free.size-1); expected_bytes_to_free--; int* CAST_FROM_VOIDP(foo, ftnode_pv); int blah = *foo; *foo = blah-1; + finalize(make_pair_attr(bytes_to_free.size-1), finalize_extra); return 0; } @@ -171,10 +172,12 @@ static int other_pe_callback ( void *ftnode_pv __attribute__((__unused__)), PAIR_ATTR bytes_to_free __attribute__((__unused__)), - PAIR_ATTR* bytes_freed __attribute__((__unused__)), - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra ) { + finalize(bytes_to_free, finalize_extra); return 0; } diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction3.cc b/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction3.cc index 99e5c3e5c5e..735bde724d0 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction3.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction3.cc @@ -169,17 +169,18 @@ static int pe_callback ( void *ftnode_pv, PAIR_ATTR UU(bytes_to_free), - PAIR_ATTR* bytes_freed, - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra ) { - *bytes_freed = make_pair_attr(bytes_to_free.size-1); usleep(1*1024*1024); if (verbose) printf("calling pe_callback\n"); expected_bytes_to_free--; int* CAST_FROM_VOIDP(foo, ftnode_pv); int blah = *foo; *foo = blah-1; + finalize(make_pair_attr(bytes_to_free.size-1), finalize_extra); return 0; } @@ -187,11 +188,12 @@ static int other_pe_callback ( void *ftnode_pv __attribute__((__unused__)), PAIR_ATTR bytes_to_free __attribute__((__unused__)), - PAIR_ATTR* bytes_freed __attribute__((__unused__)), - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra ) { - *bytes_freed = bytes_to_free; + finalize(bytes_to_free, finalize_extra); return 0; } diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction4.cc b/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction4.cc index e5db222964b..9dc1f1a5218 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction4.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-clock-eviction4.cc @@ -164,12 +164,13 @@ static int pe_callback ( void *ftnode_pv __attribute__((__unused__)), PAIR_ATTR bytes_to_free __attribute__((__unused__)), - PAIR_ATTR* bytes_freed, - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra ) { assert(false); - *bytes_freed = bytes_to_free; + finalize(bytes_to_free, finalize_extra); return 0; } diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-eviction-close-test2.cc b/storage/tokudb/ft-index/ft/tests/cachetable-eviction-close-test2.cc index 1d228bf0d6c..c8004ca1cb1 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-eviction-close-test2.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-eviction-close-test2.cc @@ -154,12 +154,13 @@ static int pe_callback ( void *ftnode_pv __attribute__((__unused__)), PAIR_ATTR bytes_to_free __attribute__((__unused__)), - PAIR_ATTR* bytes_freed, - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra ) { - *bytes_freed = make_pair_attr(bytes_to_free.size-7); sleep(2); + finalize(bytes_to_free, finalize_extra); return 0; } diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-eviction-getandpin-test2.cc b/storage/tokudb/ft-index/ft/tests/cachetable-eviction-getandpin-test2.cc index e91b59fe8f8..d65048f797a 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-eviction-getandpin-test2.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-eviction-getandpin-test2.cc @@ -110,12 +110,13 @@ static int pe_callback ( void *ftnode_pv __attribute__((__unused__)), PAIR_ATTR bytes_to_free __attribute__((__unused__)), - PAIR_ATTR* bytes_freed, - void* extraargs __attribute__((__unused__)) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR new_attr, void *extra), + void *finalize_extra ) { sleep(3); - *bytes_freed = make_pair_attr(bytes_to_free.size-7); + finalize(make_pair_attr(bytes_to_free.size - 7), finalize_extra); return 0; } diff --git a/storage/tokudb/ft-index/ft/tests/cachetable-evictor-class.cc b/storage/tokudb/ft-index/ft/tests/cachetable-evictor-class.cc index 792fced4cad..d0dff7d9570 100644 --- a/storage/tokudb/ft-index/ft/tests/cachetable-evictor-class.cc +++ b/storage/tokudb/ft-index/ft/tests/cachetable-evictor-class.cc @@ -116,7 +116,9 @@ void evictor_unit_test::init() { ZERO_STRUCT(m_cf_list); m_pl.init(); m_cf_list.init(); - m_kb = toku_kibbutz_create(1); + m_kb = NULL; + int r = toku_kibbutz_create(1, &m_kb); + assert(r == 0); } // destroy class after tests have run diff --git a/storage/tokudb/ft-index/ft/tests/ft-bfe-query.cc b/storage/tokudb/ft-index/ft/tests/ft-bfe-query.cc index e5e8dcc8643..4bf02021d23 100644 --- a/storage/tokudb/ft-index/ft/tests/ft-bfe-query.cc +++ b/storage/tokudb/ft-index/ft/tests/ft-bfe-query.cc @@ -146,7 +146,7 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,0) == PT_AVAIL); assert(BP_STATE(dn,1) == PT_AVAIL); assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_COMPRESSED); assert(BP_STATE(dn,1) == PT_COMPRESSED); assert(BP_STATE(dn,2) == PT_COMPRESSED); @@ -168,7 +168,7 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_AVAIL); assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_COMPRESSED); assert(BP_STATE(dn,2) == PT_COMPRESSED); @@ -190,7 +190,7 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_AVAIL); assert(BP_STATE(dn,2) == PT_ON_DISK); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_COMPRESSED); assert(BP_STATE(dn,2) == PT_ON_DISK); @@ -211,7 +211,7 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_ON_DISK); assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_ON_DISK); assert(BP_STATE(dn,2) == PT_COMPRESSED); @@ -232,7 +232,7 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,0) == PT_AVAIL); assert(BP_STATE(dn,1) == PT_ON_DISK); assert(BP_STATE(dn,2) == PT_ON_DISK); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_COMPRESSED); assert(BP_STATE(dn,1) == PT_ON_DISK); assert(BP_STATE(dn,2) == PT_ON_DISK); @@ -292,11 +292,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,1) == PT_ON_DISK); assert(BP_STATE(dn,2) == PT_AVAIL); // need to call this twice because we had a subset read before, that touched the clock - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_ON_DISK); assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_ON_DISK); assert(BP_STATE(dn,2) == PT_COMPRESSED); @@ -317,11 +317,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,1) == PT_AVAIL); assert(BP_STATE(dn,2) == PT_AVAIL); // need to call this twice because we had a subset read before, that touched the clock - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_COMPRESSED); assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_ON_DISK); assert(BP_STATE(dn,1) == PT_COMPRESSED); assert(BP_STATE(dn,2) == PT_COMPRESSED); @@ -341,11 +341,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) { assert(BP_STATE(dn,1) == PT_AVAIL); assert(BP_STATE(dn,2) == PT_ON_DISK); // need to call this twice because we had a subset read before, that touched the clock - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_AVAIL); assert(BP_STATE(dn,1) == PT_COMPRESSED); assert(BP_STATE(dn,2) == PT_ON_DISK); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(dn,0) == PT_COMPRESSED); assert(BP_STATE(dn,1) == PT_COMPRESSED); assert(BP_STATE(dn,2) == PT_ON_DISK); @@ -420,7 +420,8 @@ test_prefetching(void) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } diff --git a/storage/tokudb/ft-index/ft/tests/ft-clock-test.cc b/storage/tokudb/ft-index/ft/tests/ft-clock-test.cc index 606832a5df6..25daccfd120 100644 --- a/storage/tokudb/ft-index/ft/tests/ft-clock-test.cc +++ b/storage/tokudb/ft-index/ft/tests/ft-clock-test.cc @@ -154,12 +154,12 @@ test1(int fd, FT brt_h, FTNODE *dn) { // should sweep and NOT get rid of anything PAIR_ATTR attr; memset(&attr,0,sizeof(attr)); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { assert(BP_STATE(*dn,i) == PT_AVAIL); } // should sweep and get compress all - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { if (!is_leaf) { assert(BP_STATE(*dn,i) == PT_COMPRESSED); @@ -172,12 +172,12 @@ test1(int fd, FT brt_h, FTNODE *dn) { bool req = toku_ftnode_pf_req_callback(*dn, &bfe_all); assert(req); toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { assert(BP_STATE(*dn,i) == PT_AVAIL); } // should sweep and get compress all - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { if (!is_leaf) { assert(BP_STATE(*dn,i) == PT_COMPRESSED); @@ -190,15 +190,15 @@ test1(int fd, FT brt_h, FTNODE *dn) { req = toku_ftnode_pf_req_callback(*dn, &bfe_all); assert(req); toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { assert(BP_STATE(*dn,i) == PT_AVAIL); } (*dn)->dirty = 1; - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { assert(BP_STATE(*dn,i) == PT_AVAIL); } @@ -252,11 +252,11 @@ test2(int fd, FT brt_h, FTNODE *dn) { assert(!BP_SHOULD_EVICT(*dn, 1)); PAIR_ATTR attr; memset(&attr,0,sizeof(attr)); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(*dn, 0) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED); assert(BP_STATE(*dn, 1) == PT_AVAIL); assert(BP_SHOULD_EVICT(*dn, 1)); - toku_ftnode_pe_callback(*dn, attr, &attr, brt_h); + toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr); assert(BP_STATE(*dn, 1) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED); bool req = toku_ftnode_pf_req_callback(*dn, &bfe_subset); @@ -355,7 +355,8 @@ test_serialize_nonleaf(void) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); @@ -438,7 +439,8 @@ test_serialize_leaf(void) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); diff --git a/storage/tokudb/ft-index/ft/tests/ft-serialize-benchmark.cc b/storage/tokudb/ft-index/ft/tests/ft-serialize-benchmark.cc index a5cbc2f684e..42351e3108d 100644 --- a/storage/tokudb/ft-index/ft/tests/ft-serialize-benchmark.cc +++ b/storage/tokudb/ft-index/ft/tests/ft-serialize-benchmark.cc @@ -189,7 +189,8 @@ test_serialize_leaf(int valsize, int nelts, double entropy) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; brt_h->compare_fun = long_key_cmp; @@ -319,7 +320,8 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; brt_h->compare_fun = long_key_cmp; diff --git a/storage/tokudb/ft-index/ft/tests/ft-serialize-test.cc b/storage/tokudb/ft-index/ft/tests/ft-serialize-test.cc index 774948ddf95..adfd2b7efd4 100644 --- a/storage/tokudb/ft-index/ft/tests/ft-serialize-test.cc +++ b/storage/tokudb/ft-index/ft/tests/ft-serialize-test.cc @@ -187,8 +187,7 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_ // if read_none, get rid of the compressed bp's if (bft == read_none) { if ((*dn)->height == 0) { - PAIR_ATTR attr; - toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); // assert all bp's are on disk for (int i = 0; i < (*dn)->n_children; i++) { if ((*dn)->height == 0) { @@ -213,14 +212,14 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_ for (int i = 0; i < (*dn)->n_children; i++) { assert(BP_STATE(*dn,i) == PT_AVAIL); } - toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { // assert all bp's are still available, because we touched the clock assert(BP_STATE(*dn,i) == PT_AVAIL); // now assert all should be evicted assert(BP_SHOULD_EVICT(*dn, i)); } - toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), &attr, brt_h); + toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { assert(BP_STATE(*dn,i) == PT_COMPRESSED); } @@ -306,7 +305,8 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } @@ -449,7 +449,8 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } @@ -586,7 +587,8 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); @@ -733,7 +735,8 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone) TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); @@ -881,7 +884,8 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); @@ -1009,7 +1013,8 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); @@ -1134,7 +1139,8 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { TXNID_NONE, 4*1024*1024, 128*1024, - TOKU_DEFAULT_COMPRESSION_METHOD); + TOKU_DEFAULT_COMPRESSION_METHOD, + 16); brt->ft = brt_h; toku_blocktable_create_new(&brt_h->blocktable); diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test-bad-generate.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test-bad-generate.cc index d7b3504e8ad..98a8efc8393 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test-bad-generate.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test-bad-generate.cc @@ -170,7 +170,7 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail) { } FTLOADER loader; - r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, TXNID_NONE, true, 0, false); + r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false); assert(r == 0); struct rowset *rowset[nrowsets]; diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor-errors.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor-errors.cc index b4ff9265c02..93fdd308ce1 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor-errors.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor-errors.cc @@ -180,7 +180,7 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail, const char sprintf(temp, "%s/%s", testdir, "tempXXXXXX"); FTLOADER loader; - r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, TXNID_NONE, true, 0, false); + r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false); assert(r == 0); struct rowset *rowset[nrowsets]; diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor.cc index 67591c92cd1..ee4e0f99aa1 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test-extractor.cc @@ -402,7 +402,7 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) { sprintf(temp, "%s/%s", testdir, "tempXXXXXX"); FTLOADER loader; - r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, temp, ZERO_LSN, TXNID_NONE, true, 0, false); + r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, temp, ZERO_LSN, nullptr, true, 0, false); assert(r == 0); struct rowset *rowset[nrowsets]; diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test-merge-files-dbufio.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test-merge-files-dbufio.cc index 80326584c35..6f495bfb496 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test-merge-files-dbufio.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test-merge-files-dbufio.cc @@ -412,7 +412,7 @@ static void test (const char *directory, bool is_error) { bt_compare_functions, "tempxxxxxx", *lsnp, - TXNID_NONE, true, 0, false); + nullptr, true, 0, false); assert(r==0); } diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test-open.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test-open.cc index a2f39e1bfe1..f351185a525 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test-open.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test-open.cc @@ -143,7 +143,7 @@ static void test_loader_open(int ndbs) { for (i = 0; ; i++) { set_my_malloc_trigger(i+1); - r = toku_ft_loader_open(&loader, NULL, NULL, NULL, ndbs, brts, dbs, fnames, compares, "", ZERO_LSN, TXNID_NONE, true, 0, false); + r = toku_ft_loader_open(&loader, NULL, NULL, NULL, ndbs, brts, dbs, fnames, compares, "", ZERO_LSN, nullptr, true, 0, false); if (r == 0) break; } diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test-writer-errors.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test-writer-errors.cc index 5f0728cc02c..c72b6f847a3 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test-writer-errors.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test-writer-errors.cc @@ -213,7 +213,7 @@ static int write_dbfile (char *tf_template, int n, char *output_name, bool expec ft_loader_set_error_function(&bl.error_callback, NULL, NULL); ft_loader_set_poll_function(&bl.poll_callback, loader_poll_callback, NULL); - result = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD); + result = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16); toku_set_func_malloc_only(NULL); toku_set_func_realloc_only(NULL); diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test-writer.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test-writer.cc index 67bb47a7c5e..53d6bde27a0 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test-writer.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test-writer.cc @@ -262,7 +262,7 @@ static void test_write_dbfile (char *tf_template, int n, char *output_name, TXNI assert(fd>=0); if (verbose) traceit("write to file"); - r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD); + r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16); assert(r==0); r = queue_destroy(q2); diff --git a/storage/tokudb/ft-index/ft/tests/ftloader-test.cc b/storage/tokudb/ft-index/ft/tests/ftloader-test.cc index 4b7a3139adb..2fc2d309f64 100644 --- a/storage/tokudb/ft-index/ft/tests/ftloader-test.cc +++ b/storage/tokudb/ft-index/ft/tests/ftloader-test.cc @@ -425,7 +425,7 @@ static void test_merge_files (const char *tf_template, const char *output_name) int fd = open(output_name, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd>=0); - r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD); + r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16); assert(r==0); destroy_merge_fileset(&fs); diff --git a/storage/tokudb/ft-index/ft/tests/make-tree.cc b/storage/tokudb/ft-index/ft/tests/make-tree.cc index 9451c7e3448..56df485173d 100644 --- a/storage/tokudb/ft-index/ft/tests/make-tree.cc +++ b/storage/tokudb/ft-index/ft/tests/make-tree.cc @@ -124,8 +124,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) MSN msn = next_dummymsn(); // apply an insert to the leaf node + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u = {.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode,0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + toku_ft_bn_apply_cmd_once(BLB(leafnode,0), &cmd, idx, NULL, &gc_info, NULL, NULL); leafnode->max_msn_applied_to_node_on_disk = msn; diff --git a/storage/tokudb/ft-index/ft/tests/mempool-115.cc b/storage/tokudb/ft-index/ft/tests/mempool-115.cc new file mode 100644 index 00000000000..8b5d2d69195 --- /dev/null +++ b/storage/tokudb/ft-index/ft/tests/mempool-115.cc @@ -0,0 +1,196 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved." + +#include "test.h" +#include "bndata.h" + +static void +le_add_to_bn(bn_data* bn, uint32_t idx, const char *key, int keysize, const char *val, int valsize) +{ + LEAFENTRY r = NULL; + uint32_t size_needed = LE_CLEAN_MEMSIZE(valsize); + bn->get_space_for_insert( + idx, + key, + keysize, + size_needed, + &r + ); + resource_assert(r); + r->type = LE_CLEAN; + r->u.clean.vallen = valsize; + memcpy(r->u.clean.val, val, valsize); +} + +static void +le_overwrite(bn_data* bn, uint32_t idx, const char *key, int keysize, const char *val, int valsize) { + LEAFENTRY r = NULL; + uint32_t size_needed = LE_CLEAN_MEMSIZE(valsize); + bn->get_space_for_overwrite( + idx, + key, + keysize, + size_needed, // old_le_size + size_needed, + &r + ); + resource_assert(r); + r->type = LE_CLEAN; + r->u.clean.vallen = valsize; + memcpy(r->u.clean.val, val, valsize); +} + + +class bndata_bugfix_test { +public: + void + run_test(void) { + // struct ft_handle source_ft; + struct ftnode sn; + + // just copy this code from a previous test + // don't care what it does, just want to get a node up and running + sn.flags = 0x11223344; + sn.thisnodename.b = 20; + sn.layout_version = FT_LAYOUT_VERSION; + sn.layout_version_original = FT_LAYOUT_VERSION; + sn.height = 0; + sn.n_children = 2; + sn.dirty = 1; + sn.oldest_referenced_xid_known = TXNID_NONE; + MALLOC_N(sn.n_children, sn.bp); + MALLOC_N(1, sn.childkeys); + toku_memdup_dbt(&sn.childkeys[0], "b", 2); + sn.totalchildkeylens = 2; + BP_STATE(&sn,0) = PT_AVAIL; + BP_STATE(&sn,1) = PT_AVAIL; + set_BLB(&sn, 0, toku_create_empty_bn()); + set_BLB(&sn, 1, toku_create_empty_bn()); + le_add_to_bn(BLB_DATA(&sn, 0), 0, "a", 2, "aval", 5); + le_add_to_bn(BLB_DATA(&sn, 0), 1, "b", 2, "bval", 5); + le_add_to_bn(BLB_DATA(&sn, 1), 0, "x", 2, "xval", 5); + + + + // now this is the test. If I keep getting space for overwrite + // like crazy, it should expose the bug + bn_data* bnd = BLB_DATA(&sn, 0); + size_t old_size = bnd->m_buffer_mempool.size; + if (verbose) printf("frag size: %zu\n", bnd->m_buffer_mempool.frag_size); + if (verbose) printf("size: %zu\n", bnd->m_buffer_mempool.size); + for (uint32_t i = 0; i < 1000000; i++) { + le_overwrite(bnd, 0, "a", 2, "aval", 5); + } + if (verbose) printf("frag size: %zu\n", bnd->m_buffer_mempool.frag_size); + if (verbose) printf("size: %zu\n", bnd->m_buffer_mempool.size); + size_t new_size = bnd->m_buffer_mempool.size; + // just a crude test to make sure we did not grow unbounded. + // if this assert ever fails, revisit the code and see what is going + // on. It may be that some algorithm has changed. + assert(new_size < 5*old_size); + + + for (int i = 0; i < sn.n_children-1; ++i) { + toku_free(sn.childkeys[i].data); + } + for (int i = 0; i < sn.n_children; i++) { + destroy_basement_node(BLB(&sn, i)); + } + toku_free(sn.bp); + toku_free(sn.childkeys); + } +}; + +int +test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) { + bndata_bugfix_test t; + t.run_test(); + return 0; +} diff --git a/storage/tokudb/ft-index/ft/tests/msnfilter.cc b/storage/tokudb/ft-index/ft/tests/msnfilter.cc index 0b7bd426208..1c1cba86f1c 100644 --- a/storage/tokudb/ft-index/ft/tests/msnfilter.cc +++ b/storage/tokudb/ft-index/ft/tests/msnfilter.cc @@ -132,8 +132,9 @@ append_leaf(FT_HANDLE brt, FTNODE leafnode, void *key, uint32_t keylen, void *va MSN msn = next_dummymsn(); brt->ft->h->max_msn_in_ft = msn; FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &cmd, make_gc_info(false), nullptr, nullptr); + toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &cmd, &gc_info, nullptr, nullptr); { int r = toku_ft_lookup(brt, &thekey, lookup_checkf, &pair); assert(r==0); @@ -141,7 +142,7 @@ append_leaf(FT_HANDLE brt, FTNODE leafnode, void *key, uint32_t keylen, void *va } FT_MSG_S badcmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &badval }} }; - toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &badcmd, make_gc_info(false), nullptr, nullptr); + toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &badcmd, &gc_info, nullptr, nullptr); // message should be rejected for duplicate msn, row should still have original val { @@ -154,7 +155,7 @@ append_leaf(FT_HANDLE brt, FTNODE leafnode, void *key, uint32_t keylen, void *va msn = next_dummymsn(); brt->ft->h->max_msn_in_ft = msn; FT_MSG_S cmd2 = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &val2 }} }; - toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &cmd2, make_gc_info(false), nullptr, nullptr); + toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &cmd2, &gc_info, nullptr, nullptr); // message should be accepted, val should have new value { @@ -166,7 +167,7 @@ append_leaf(FT_HANDLE brt, FTNODE leafnode, void *key, uint32_t keylen, void *va // now verify that message with lesser (older) msn is rejected msn.msn = msn.msn - 10; FT_MSG_S cmd3 = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &badval } }}; - toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &cmd3, make_gc_info(false), nullptr, nullptr); + toku_ft_leaf_apply_cmd(brt->ft->compare_fun, brt->ft->update_fun, &brt->ft->cmp_descriptor, leafnode, -1, &cmd3, &gc_info, nullptr, nullptr); // message should be rejected, val should still have value in pair2 { diff --git a/storage/tokudb/ft-index/ft/tests/orthopush-flush.cc b/storage/tokudb/ft-index/ft/tests/orthopush-flush.cc index c2827ddf29b..cfac10fb921 100644 --- a/storage/tokudb/ft-index/ft/tests/orthopush-flush.cc +++ b/storage/tokudb/ft-index/ft/tests/orthopush-flush.cc @@ -96,6 +96,7 @@ PATENT RIGHTS GRANT: static TOKUTXN const null_txn = 0; static DB * const null_db = 0; static const char *fname = TOKU_TEST_FILENAME; +static txn_gc_info non_mvcc_gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); static int dummy_cmp(DB *db __attribute__((unused)), const DBT *a, const DBT *b) { @@ -217,8 +218,8 @@ insert_random_message_to_bn( *keylenp = keydbt->size; *keyp = toku_xmemdup(keydbt->data, keydbt->size); int64_t numbytes; - toku_le_apply_msg(&msg, NULL, NULL, 0, TXNID_NONE, make_gc_info(false), save, &numbytes); - toku_ft_bn_apply_cmd(t->ft->compare_fun, t->ft->update_fun, NULL, blb, &msg, TXNID_NONE, make_gc_info(false), NULL, NULL); + toku_le_apply_msg(&msg, NULL, NULL, 0, &non_mvcc_gc_info, save, &numbytes); + toku_ft_bn_apply_cmd(t->ft->compare_fun, t->ft->update_fun, NULL, blb, &msg, &non_mvcc_gc_info, NULL, NULL); if (msn.msn > blb->max_msn_applied.msn) { blb->max_msn_applied = msn; } @@ -267,12 +268,12 @@ insert_same_message_to_bns( *keylenp = keydbt->size; *keyp = toku_xmemdup(keydbt->data, keydbt->size); int64_t numbytes; - toku_le_apply_msg(&msg, NULL, NULL, 0, TXNID_NONE, make_gc_info(false), save, &numbytes); - toku_ft_bn_apply_cmd(t->ft->compare_fun, t->ft->update_fun, NULL, blb1, &msg, TXNID_NONE, make_gc_info(false), NULL, NULL); + toku_le_apply_msg(&msg, NULL, NULL, 0, &non_mvcc_gc_info, save, &numbytes); + toku_ft_bn_apply_cmd(t->ft->compare_fun, t->ft->update_fun, NULL, blb1, &msg, &non_mvcc_gc_info, NULL, NULL); if (msn.msn > blb1->max_msn_applied.msn) { blb1->max_msn_applied = msn; } - toku_ft_bn_apply_cmd(t->ft->compare_fun, t->ft->update_fun, NULL, blb2, &msg, TXNID_NONE, make_gc_info(false), NULL, NULL); + toku_ft_bn_apply_cmd(t->ft->compare_fun, t->ft->update_fun, NULL, blb2, &msg, &non_mvcc_gc_info, NULL, NULL); if (msn.msn > blb2->max_msn_applied.msn) { blb2->max_msn_applied = msn; } @@ -684,7 +685,7 @@ flush_to_leaf(FT_HANDLE t, bool make_leaf_up_to_date, bool use_flush) { if (make_leaf_up_to_date) { for (i = 0; i < num_parent_messages; ++i) { if (!parent_messages_is_fresh[i]) { - toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child, -1, parent_messages[i], make_gc_info(false), NULL, NULL); + toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL); } } for (i = 0; i < 8; ++i) { @@ -908,7 +909,7 @@ flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) { for (i = 0; i < num_parent_messages; ++i) { if (dummy_cmp(NULL, parent_messages[i]->u.id.key, &childkeys[7]) <= 0 && !parent_messages_is_fresh[i]) { - toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child, -1, parent_messages[i], make_gc_info(false), NULL, NULL); + toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL); } } for (i = 0; i < 8; ++i) { @@ -1104,8 +1105,8 @@ compare_apply_and_flush(FT_HANDLE t, bool make_leaf_up_to_date) { if (make_leaf_up_to_date) { for (i = 0; i < num_parent_messages; ++i) { if (!parent_messages_is_fresh[i]) { - toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child1, -1, parent_messages[i], make_gc_info(false), NULL, NULL); - toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child2, -1, parent_messages[i], make_gc_info(false), NULL, NULL); + toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child1, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL); + toku_ft_leaf_apply_cmd(t->ft->compare_fun, t->ft->update_fun, &t->ft->descriptor, child2, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL); } } for (i = 0; i < 8; ++i) { diff --git a/storage/tokudb/ft-index/ft/tests/test-flushes-on-cleaner.cc b/storage/tokudb/ft-index/ft/tests/test-flushes-on-cleaner.cc index d0997cca25c..291d409018a 100644 --- a/storage/tokudb/ft-index/ft/tests/test-flushes-on-cleaner.cc +++ b/storage/tokudb/ft-index/ft/tests/test-flushes-on-cleaner.cc @@ -260,8 +260,7 @@ doit (bool keep_other_bn_in_memory) { assert(node->n_children == 2); // a hack to get the basement nodes evicted for (int i = 0; i < 20; i++) { - PAIR_ATTR attr; - toku_ftnode_pe_callback(node, make_pair_attr(0xffffffff), &attr, brt->ft); + toku_ftnode_pe_callback(node, make_pair_attr(0xffffffff), brt->ft, def_pe_finalize_impl, nullptr); } // this ensures that when we do the lookups below, // that the data is read off disk diff --git a/storage/tokudb/ft-index/ft/tests/test-hot-with-bounds.cc b/storage/tokudb/ft-index/ft/tests/test-hot-with-bounds.cc index 01b9629c94d..40ed00bcdc2 100644 --- a/storage/tokudb/ft-index/ft/tests/test-hot-with-bounds.cc +++ b/storage/tokudb/ft-index/ft/tests/test-hot-with-bounds.cc @@ -205,7 +205,8 @@ doit (void) { toku_fill_dbt(&left, "g", 2); DBT right; toku_fill_dbt(&right, "n", 2); - r = toku_ft_hot_optimize(t, &left, &right, NULL, NULL); + uint64_t loops_run = 0; + r = toku_ft_hot_optimize(t, &left, &right, NULL, NULL, &loops_run); assert(r==0); // at this point, we have should have flushed diff --git a/storage/tokudb/ft-index/ft/tests/test-inc-split.cc b/storage/tokudb/ft-index/ft/tests/test-inc-split.cc index 6d0a6349b5c..cafcb496f7a 100644 --- a/storage/tokudb/ft-index/ft/tests/test-inc-split.cc +++ b/storage/tokudb/ft-index/ft/tests/test-inc-split.cc @@ -130,10 +130,10 @@ const char *fname = TOKU_TEST_FILENAME; static void doit (int ksize __attribute__((__unused__))) { - BLOCKNUM cnodes[FT_FANOUT], bnode, anode; + BLOCKNUM cnodes[16], bnode, anode; - char *keys[FT_FANOUT-1]; - int keylens[FT_FANOUT-1]; + char *keys[16-1]; + int keylens[16-1]; int i; int r; @@ -144,7 +144,7 @@ doit (int ksize __attribute__((__unused__))) { toku_testsetup_initialize(); // must precede any other toku_testsetup calls - for (i=0; i<FT_FANOUT; i++) { + for (i=0; i<16; i++) { r=toku_testsetup_leaf(t, &cnodes[i], 1, NULL, NULL); assert(r==0); char key[KSIZE+10]; @@ -156,16 +156,16 @@ doit (int ksize __attribute__((__unused__))) { } // Now we have a bunch of leaves, all of which are with 100 bytes of full. - for (i=0; i+1<FT_FANOUT; i++) { + for (i=0; i+1<16; i++) { char key[TOKU_PSIZE]; keylens[i]=1+snprintf(key, TOKU_PSIZE, "%08d", (i+1)*10000); keys[i]=toku_strdup(key); } - r = toku_testsetup_nonleaf(t, 1, &bnode, FT_FANOUT, cnodes, keys, keylens); + r = toku_testsetup_nonleaf(t, 1, &bnode, 16, cnodes, keys, keylens); assert(r==0); - for (i=0; i+1<FT_FANOUT; i++) { + for (i=0; i+1<16; i++) { toku_free(keys[i]); } diff --git a/storage/tokudb/ft-index/ft/tests/test-leafentry-nested.cc b/storage/tokudb/ft-index/ft/tests/test-leafentry-nested.cc index 2b6a1ee8488..b0a5dfe7817 100644 --- a/storage/tokudb/ft-index/ft/tests/test-leafentry-nested.cc +++ b/storage/tokudb/ft-index/ft/tests/test-leafentry-nested.cc @@ -453,12 +453,12 @@ test_le_apply(ULE ule_initial, FT_MSG msg, ULE ule_expected) { size_t result_memsize = 0; int64_t ignoreme; + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true); toku_le_apply_msg(msg, le_initial, nullptr, 0, - TXNID_NONE, - make_gc_info(true), + &gc_info, &le_result, &ignoreme); if (le_result) { @@ -751,7 +751,8 @@ static bool ule_worth_running_garbage_collection(ULE ule, TXNID oldest_reference LEAFENTRY le; int r = le_pack(ule, nullptr, 0, nullptr, 0, 0, &le); CKERR(r); invariant_notnull(le); - bool worth_running = toku_le_worth_running_garbage_collection(le, oldest_referenced_xid_known); + txn_gc_info gc_info(nullptr, oldest_referenced_xid_known, oldest_referenced_xid_known, true); + bool worth_running = toku_le_worth_running_garbage_collection(le, &gc_info); toku_free(le); return worth_running; } diff --git a/storage/tokudb/ft-index/ft/tests/test-pick-child-to-flush.cc b/storage/tokudb/ft-index/ft/tests/test-pick-child-to-flush.cc index a8c36557e00..b8b44e669f5 100644 --- a/storage/tokudb/ft-index/ft/tests/test-pick-child-to-flush.cc +++ b/storage/tokudb/ft-index/ft/tests/test-pick-child-to-flush.cc @@ -369,8 +369,7 @@ doit (void) { assert_zero(r); toku_pin_node_with_min_bfe(&node, node_internal, t); for (int i = 0; i < 20; i++) { - PAIR_ATTR attr; - toku_ftnode_pe_callback(node, make_pair_attr(0xffffffff), &attr, t->ft); + toku_ftnode_pe_callback(node, make_pair_attr(0xffffffff), t->ft, def_pe_finalize_impl, nullptr); } assert(BP_STATE(node,0) == PT_COMPRESSED); toku_unpin_ftnode(t->ft, node); diff --git a/storage/tokudb/ft-index/ft/tests/test.h b/storage/tokudb/ft-index/ft/tests/test.h index 4a1bd64354f..ad1fc7c148e 100644 --- a/storage/tokudb/ft-index/ft/tests/test.h +++ b/storage/tokudb/ft-index/ft/tests/test.h @@ -239,17 +239,21 @@ def_pe_est_callback( } static UU() int -def_pe_callback ( +def_pe_callback( void *ftnode_pv __attribute__((__unused__)), PAIR_ATTR bytes_to_free __attribute__((__unused__)), - PAIR_ATTR* bytes_freed, - void* extraargs __attribute__((__unused__)) - ) + void* extraargs __attribute__((__unused__)), + void (*finalize)(PAIR_ATTR bytes_freed, void *extra), + void *finalize_extra + ) { - *bytes_freed = bytes_to_free; + finalize(bytes_to_free, finalize_extra); return 0; } +static UU() void +def_pe_finalize_impl(PAIR_ATTR UU(bytes_freed), void *UU(extra)) { } + static UU() bool def_pf_req_callback(void* UU(ftnode_pv), void* UU(read_extraargs)) { return false; } diff --git a/storage/tokudb/ft-index/ft/tests/upgrade_test_simple.cc b/storage/tokudb/ft-index/ft/tests/upgrade_test_simple.cc index 918d48d3c5b..31811527aa2 100644 --- a/storage/tokudb/ft-index/ft/tests/upgrade_test_simple.cc +++ b/storage/tokudb/ft-index/ft/tests/upgrade_test_simple.cc @@ -144,7 +144,8 @@ static int do_hot_optimize(FT_HANDLE t, CACHETABLE UU(ct), void *extra) { float *CAST_FROM_VOIDP(fraction, extra); - int r = toku_ft_hot_optimize(t, NULL, NULL, progress, extra); + uint64_t loops_run = 0; + int r = toku_ft_hot_optimize(t, NULL, NULL, progress, extra, &loops_run); if (*fraction < 1.0) { CKERR2(r, 1); } else { diff --git a/storage/tokudb/ft-index/ft/tests/verify-bad-msn.cc b/storage/tokudb/ft-index/ft/tests/verify-bad-msn.cc index 1b90d8975fa..edb1b6262a3 100644 --- a/storage/tokudb/ft-index/ft/tests/verify-bad-msn.cc +++ b/storage/tokudb/ft-index/ft/tests/verify-bad-msn.cc @@ -128,7 +128,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); + toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, &gc_info, NULL, NULL); // Create bad tree (don't do following): // leafnode->max_msn_applied_to_node = msn; diff --git a/storage/tokudb/ft-index/ft/tests/verify-bad-pivots.cc b/storage/tokudb/ft-index/ft/tests/verify-bad-pivots.cc index 155cf67ad3e..156fc36666f 100644 --- a/storage/tokudb/ft-index/ft/tests/verify-bad-pivots.cc +++ b/storage/tokudb/ft-index/ft/tests/verify-bad-pivots.cc @@ -116,7 +116,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node MSN msn = next_dummymsn(); FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); + toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, &gc_info, NULL, NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/ft-index/ft/tests/verify-dup-in-leaf.cc b/storage/tokudb/ft-index/ft/tests/verify-dup-in-leaf.cc index c5b69bab74c..0f9f2a1000d 100644 --- a/storage/tokudb/ft-index/ft/tests/verify-dup-in-leaf.cc +++ b/storage/tokudb/ft-index/ft/tests/verify-dup-in-leaf.cc @@ -117,7 +117,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node MSN msn = next_dummymsn(); FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); + toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, &gc_info, NULL, NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/ft-index/ft/tests/verify-dup-pivots.cc b/storage/tokudb/ft-index/ft/tests/verify-dup-pivots.cc index 2a2f4079632..219f1f7da59 100644 --- a/storage/tokudb/ft-index/ft/tests/verify-dup-pivots.cc +++ b/storage/tokudb/ft-index/ft/tests/verify-dup-pivots.cc @@ -116,7 +116,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node MSN msn = next_dummymsn(); FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); + toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, &gc_info, NULL, NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/ft-index/ft/tests/verify-misrouted-msgs.cc b/storage/tokudb/ft-index/ft/tests/verify-misrouted-msgs.cc index a3290028ac4..f68b4d72e53 100644 --- a/storage/tokudb/ft-index/ft/tests/verify-misrouted-msgs.cc +++ b/storage/tokudb/ft-index/ft/tests/verify-misrouted-msgs.cc @@ -117,7 +117,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node MSN msn = next_dummymsn(); FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode,0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); + toku_ft_bn_apply_cmd_once(BLB(leafnode,0), &cmd, idx, NULL, &gc_info, NULL, NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/ft-index/ft/tests/verify-unsorted-leaf.cc b/storage/tokudb/ft-index/ft/tests/verify-unsorted-leaf.cc index ebcf429ab4f..c1e9f6f5727 100644 --- a/storage/tokudb/ft-index/ft/tests/verify-unsorted-leaf.cc +++ b/storage/tokudb/ft-index/ft/tests/verify-unsorted-leaf.cc @@ -119,7 +119,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node MSN msn = next_dummymsn(); FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); + toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, &gc_info, NULL, NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/ft-index/ft/tests/verify-unsorted-pivots.cc b/storage/tokudb/ft-index/ft/tests/verify-unsorted-pivots.cc index acadb9d8111..c209e679072 100644 --- a/storage/tokudb/ft-index/ft/tests/verify-unsorted-pivots.cc +++ b/storage/tokudb/ft-index/ft/tests/verify-unsorted-pivots.cc @@ -116,7 +116,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node MSN msn = next_dummymsn(); FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} }; - toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, TXNID_NONE, make_gc_info(false), NULL, NULL); + txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); + toku_ft_bn_apply_cmd_once(BLB(leafnode, 0), &cmd, idx, NULL, &gc_info, NULL, NULL); // dont forget to dirty the node leafnode->dirty = 1; |