diff options
Diffstat (limited to 'test/tcl/test008.tcl')
| -rw-r--r-- | test/tcl/test008.tcl | 554 |
1 files changed, 444 insertions, 110 deletions
diff --git a/test/tcl/test008.tcl b/test/tcl/test008.tcl index 70791ae6..30f1fba7 100644 --- a/test/tcl/test008.tcl +++ b/test/tcl/test008.tcl @@ -1,51 +1,56 @@ # See the file LICENSE for redistribution information. # -# Copyright (c) 1996, 2012 Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 1996, 2015 Oracle and/or its affiliates. All rights reserved. # # $Id$ # # TEST test008 -# TEST Small keys/large data +# TEST Small keys/large data with overflows or BLOB. # TEST Put/get per key # TEST Loop through keys by steps (which change) # TEST ... delete each key at step # TEST ... add each key back # TEST ... change step -# TEST Confirm that overflow pages are getting reused +# TEST Confirm that overflow pages are getting reused or blobs +# TEST are created. # TEST # TEST Take the source files and dbtest executable and enter their names as # TEST the key with their contents as data. After all are entered, begin # TEST looping through the entries; deleting some pairs and then readding them. proc test008 { method {reopen "008"} {debug 0} args} { source ./include.tcl + global alphabet + global has_crypto + global databases_in_memory - set tnum test$reopen + set testname test$reopen set args [convert_args $method $args] set omethod [convert_method $method] - if { [is_record_based $method] == 1 } { + # Test overflow case for btree and hash since only btree and hash + # have overflow pages. + # Test blob case for btree, hash and heap. + if { [is_btree $omethod] != 1 && \ + [is_hash $omethod] != 1 && [is_heap $omethod] != 1 } { puts "Test$reopen skipping for method $method" return } - puts -nonewline "$tnum: $method filename=key filecontents=data pairs" - if {$reopen == "009"} { - puts "(with close)" - } else { - puts "" - } - # Create the database and open the dictionary set txnenv 0 set eindex [lsearch -exact $args "-env"] # # If we are using an env, then testfile should just be the db name. - # Otherwise it is the test directory and the name. + # Otherwise it is the test directory and the name. Just set up + # the "basename" here; we will save both the overflow db and the + # blob db by adding $opt into the database name. if { $eindex == -1 } { - set testfile $testdir/$tnum.db + set testfile $testdir/$testname set env NULL + set blrootdir $testdir/__db_bl + set vrflags "-blob_dir $blrootdir" } else { - set testfile $tnum.db + set testfile $testname incr eindex set env [lindex $args $eindex] set txnenv [is_txnenv $env] @@ -53,147 +58,476 @@ proc test008 { method {reopen "008"} {debug 0} args} { append args " -auto_commit " } set testdir [get_home $env] + set blrtdir [$env get_blob_dir] + if { $blrtdir == "" } { + set blrtdir __db_bl + } + set blrootdir $testdir/$blrtdir + set vrflags "-env $env" + } + + # Look for incompatible configurations of blob. + set skipblob 0 + foreach conf { "-encryptaes" "-encrypt" "-compress" "-dup" "-dupsort" \ + "-read_uncommitted" "-multiversion" } { + if { [lsearch -exact $args $conf] != -1 } { + set skipblob 1 + set skipmsg "Test$reopen skipping $conf for blob" + break + } + } + if { $skipblob == 0 && $env != "NULL" && \ + [lsearch [$env get_flags] "-snapshot"] != -1 } { + set skipblob 1 + set skipmsg "Test$reopen skipping -snapshot for blob" + } + if { $skipblob == 0 && $databases_in_memory } { + set skipblob 1 + set skipmsg "Test$reopen skipping in-memory database for blob" } + if { $has_crypto == 1 && $skipblob == 0 && $env != "NULL" } { + if {[$env get_encrypt_flags] != "" } { + set skipblob 1 + set skipmsg "Test$reopen skipping security environment" + } + } + set t1 $testdir/t1 set t2 $testdir/t2 set t3 $testdir/t3 set t4 $testdir/t4 - cleanup $testdir $env - - set db [eval {berkdb_open -create -mode 0644} \ - $args {$omethod $testfile}] - error_check_good dbopen [is_valid_db $db] TRUE - - set pflags "" - set gflags "" - set txn "" - # Here is the loop where we put and get each key/data pair set file_list [get_file_list] - set count 0 - puts "\tTest$reopen.a: Initial put/get loop" - foreach f $file_list { - set names($count) $f - set key $f - - if { $txnenv == 1 } { - set t [$env txn] - error_check_good txn [is_valid_txn $t $env] TRUE - set txn "-txn $t" + set msg "filename=key filecontents=data pairs" + set pflags "" + set gflags "" + set fsflags "" + if { [is_heap $omethod] == 1 } { + set msg "filecontents=data" + set pflags -append + set fsflags -n + } + foreach opt { "overflow" "blob" } { + puts -nonewline "$testname: $method $msg $args (with $opt)" + if { $reopen == "009" } { + puts " (with close)" + } else { + puts "" } - put_file $db $txn $pflags $f - if { $txnenv == 1 } { - error_check_good txn [$t commit] 0 + + # + # Set a blob threshold so that some data items will be saved + # as blobs and others as regular data items. + # + set bflags "" + if { $opt == "blob" } { + if { $skipblob != 0 } { + puts $skipmsg + continue + } elseif { $env != "NULL" && [is_repenv $env] == 1 } { + puts "Test$reopen\ + skipping blob for replication." + continue + } elseif { [lsearch -exact $args "-chksum"] != -1 } { + set indx [lsearch -exact $args "-chksum"] + set args [lreplace $args $indx $indx] + puts "Test$reopen ignoring -chskum for blob" + } + set bflags "-blob_threshold 30" + if { $env == "NULL" } { + append bflags " -blob_dir $blrootdir" + } + } elseif { [is_heap $omethod] == 1 } { + puts "Test$reopen\ + skipping for method $method for overflow." + continue } - if { $txnenv == 1 } { - set t [$env txn] - error_check_good txn [is_valid_txn $t $env] TRUE - set txn "-txn $t" + if { $env != "NULL" } { + set testdir [get_home $env] } - get_file $db $txn $gflags $f $t4 - if { $txnenv == 1 } { - error_check_good txn [$t commit] 0 + cleanup $testdir $env + + set db [eval {berkdb_open -create -mode 0644} \ + $bflags $args $omethod $testfile-$opt.db] + error_check_good dbopen [is_valid_db $db] TRUE + if { $opt == "blob" } { + error_check_good blob_threshold \ + [$db get_blob_threshold] 30 + set blsubdir [$db get_blob_sub_dir] + set blobdir $blrootdir/$blsubdir } - error_check_good Test$reopen:diff($f,$t4) \ - [filecmp $f $t4] 0 + set txn "" - incr count - } + puts "\tTest$reopen.a: Initial put/get loop" + # When blob is enabled, first put some regular data items and + # verify there is no blob created. Then put some big data + # items and verify they are saved as blobs. + # When blob is disabled, just put some big data items so + # overflow pages are created. + set step 1 + if { $opt == "blob" } { + puts "\tTest$reopen.a$step: Put some regular data\ + items and verify there is no blob created." + incr step + if { $txnenv == 1 } { + set t [$env txn] + error_check_good txn \ + [is_valid_txn $t $env] TRUE + set txn "-txn $t" + } + for { set i 0 } { $i < 100 } { incr i } { + if { [is_heap $omethod] == 1 } { + set ret [catch {eval {$db put} \ + $txn -append {$i.$alphabet}} rids($i)] + } else { + set ret [eval {$db put} \ + $txn $pflags {$i $i.$alphabet}] + } + error_check_good db_put $ret 0 + } + if { $txnenv == 1 } { + error_check_good txn [$t commit] 0 + } + error_check_bad no_blob_files [file exists $blobdir] 1 - if {$reopen == "009"} { - error_check_good db_close [$db close] 0 + # Skip putting the blobs by cursor in heap database. + if { [is_heap $omethod] != 1 } { + puts "\tTest$reopen.a$step: Put and get\ + blobs by cursor, and verify blobs are\ + created." + incr step + if { $txnenv == 1 } { + set t [$env txn] + error_check_good txn \ + [is_valid_txn $t $env] TRUE + set txn "-txn $t" + } + set dbc [eval {$db cursor} $txn] + error_check_good cursor_open \ + [is_valid_cursor $dbc $db] TRUE + for { set i 100 } { $i < 110 } { incr i } { + error_check_good cursor_put [$dbc put \ + -keyfirst -blob $i $i.abc] 0 + set pair [$dbc get -set $i] + error_check_bad cursor_get \ + [llength $pair] 0 + error_check_good cmp_data \ + [string compare $i.abc \ + [lindex [lindex $pair 0] 1]] 0 - set db [eval {berkdb_open} $args $testfile] - error_check_good dbopen [is_valid_db $db] TRUE - } + } + error_check_good cursor_close [$dbc close] 0 + if { $txnenv == 1 } { + error_check_good txn [$t commit] 0 + } - # Now we will get step through keys again (by increments) and - # delete all the entries, then re-insert them. + set blfnum [llength \ + [glob -nocomplain $blobdir/__db.bl*]] + error_check_good blob_file_created $blfnum 10 + error_check_good blob_meta_db \ + [file exists $blobdir/__db_blob_meta.db] 1 + } - puts "\tTest$reopen.b: Delete re-add loop" - foreach i "1 2 4 8 16" { - for {set ndx 0} {$ndx < $count} { incr ndx $i} { + # + # Delete the regular data items just put since they + # do not work in dump_bin_file. + # if { $txnenv == 1 } { set t [$env txn] - error_check_good txn [is_valid_txn $t $env] TRUE + error_check_good txn \ + [is_valid_txn $t $env] TRUE set txn "-txn $t" } - set r [eval {$db del} $txn {$names($ndx)}] - error_check_good db_del:$names($ndx) $r 0 + for { set i 0 } { $i < 110 } { incr i } { + set key $i + if { [is_heap $omethod] == 1 } { + if { $i >= 100 } { + break + } + set key $rids($i) + } + set r [eval {$db del} $txn {$key}] + error_check_good db_del:$i $r 0 + } if { $txnenv == 1 } { error_check_good txn [$t commit] 0 } + if { [is_heap $omethod] == 1 } { + array unset rids + } + set blfnum [llength \ + [glob -nocomplain $blobdir/__db.bl*]] + error_check_good blob_file_deleted $blfnum 0 } - for {set ndx 0} {$ndx < $count} { incr ndx $i} { + puts "\tTest$reopen.a$step: Put some big data items." + incr step + set count 0 + foreach f $file_list { + set names($count) $f + set key $f + + if { $txnenv == 1 } { + set t [$env txn] + error_check_good txn \ + [is_valid_txn $t $env] TRUE + set txn "-txn $t" + } + if { [is_heap $omethod] == 1 } { + set rids($count) \ + [test008_heap_put_file $db $txn $f] + set key $rids($count) + } else { + put_file $db $txn $pflags $f + } + if { $txnenv == 1 } { + error_check_good txn [$t commit] 0 + } + if { $txnenv == 1 } { set t [$env txn] - error_check_good txn [is_valid_txn $t $env] TRUE + error_check_good txn \ + [is_valid_txn $t $env] TRUE set txn "-txn $t" } - put_file $db $txn $pflags $names($ndx) + get_file $db $txn $gflags $key $t4 if { $txnenv == 1 } { error_check_good txn [$t commit] 0 } + + error_check_good Test$reopen:diff($f,$t4) \ + [filecmp $f $t4] 0 + + incr count } - } + error_check_good db_sync [$db sync] 0 - if {$reopen == "009"} { - error_check_good db_close [$db close] 0 - set db [eval {berkdb_open} $args $testfile] - error_check_good dbopen [is_valid_db $db] TRUE - } + if { $opt == "blob" } { + puts "\tTest$reopen.a$step: Verify\ + blobs are created and run db_verify." + set blfnum [llength \ + [glob -nocomplain $blobdir/__db.bl*]] + error_check_good blob_file_created [expr $blfnum > 0] 1 + error_check_good blob_meta_db \ + [file exists $blobdir/__db_blob_meta.db] 1 - # Now, reopen the file and make sure the key/data pairs look right. - puts "\tTest$reopen.c: Dump contents forward" - if { $txnenv == 1 } { - set t [$env txn] - error_check_good txn [is_valid_txn $t $env] TRUE - set txn "-txn $t" - } - dump_bin_file $db $txn $t1 test008.check - if { $txnenv == 1 } { - error_check_good txn [$t commit] 0 - } + # Run verify to check the internal structure and order. + if { [catch {eval {berkdb dbverify} \ + $vrflags $testfile-$opt.db} res] } { + error "FAIL: Verification failed with $res" + } + } else { + puts "\tTest$reopen.a$step:\ + Verify overflow pages are created." + set ovf [stat_field $db stat "Overflow pages"] + error_check_good overflow_pages [expr $ovf > 0] 1 + } - set oid [open $t2.tmp w] - foreach f $file_list { - puts $oid $f - } - close $oid - filesort $t2.tmp $t2 - fileremove $t2.tmp - filesort $t1 $t3 - - error_check_good Test$reopen:diff($t3,$t2) \ - [filecmp $t3 $t2] 0 - - # Now, reopen the file and run the last test again in reverse direction. - puts "\tTest$reopen.d: Dump contents backward" - if { $txnenv == 1 } { - set t [$env txn] - error_check_good txn [is_valid_txn $t $env] TRUE - set txn "-txn $t" - } - dump_bin_file_direction $db $txn $t1 test008.check "-last" "-prev" - if { $txnenv == 1 } { - error_check_good txn [$t commit] 0 - } + if {$reopen == "009"} { + error_check_good db_close [$db close] 0 - filesort $t1 $t3 + set bflags "" + if { $opt == "blob" && $env == "NULL" } { + set bflags "-blob_dir $blrootdir" + } + set db [eval {berkdb_open} $bflags $args $testfile-$opt.db] + error_check_good dbopen [is_valid_db $db] TRUE + if { $opt == "blob" } { + error_check_good blob_threshold \ + [$db get_blob_threshold] 30 + } + } + + # + # Now we will get step through keys again (by increments) and + # delete all the entries, then re-insert them. + # + + puts "\tTest$reopen.b: Delete re-add loop" + foreach i "1 2 4 8 16" { + for {set ndx 0} {$ndx < $count} { incr ndx $i} { + if { $txnenv == 1 } { + set t [$env txn] + error_check_good txn \ + [is_valid_txn $t $env] TRUE + set txn "-txn $t" + } + set key $names($ndx) + if { [is_heap $omethod] == 1 } { + set key $rids($ndx) + } + set r [eval {$db del} $txn {$key}] + error_check_good db_del:$key $r 0 + if { $txnenv == 1 } { + error_check_good txn [$t commit] 0 + } + } + for {set ndx 0} {$ndx < $count} { incr ndx $i} { + if { $txnenv == 1 } { + set t [$env txn] + error_check_good txn \ + [is_valid_txn $t $env] TRUE + set txn "-txn $t" + } + if { [is_heap $omethod] == 1 } { + set rids($ndx) \ + [test008_heap_put_file \ + $db $txn $names($ndx)] + } else { + put_file $db $txn $pflags $names($ndx) + } + if { $txnenv == 1 } { + error_check_good txn [$t commit] 0 + } + } + } + error_check_good db_sync [$db sync] 0 - error_check_good Test$reopen:diff($t3,$t2) \ - [filecmp $t3 $t2] 0 - error_check_good close:$db [$db close] 0 + if {$reopen == "009"} { + error_check_good db_close [$db close] 0 + set db [eval {berkdb_open} $bflags $args $testfile-$opt.db] + error_check_good dbopen [is_valid_db $db] TRUE + if { $opt == "blob" } { + error_check_good blob_threshold \ + [$db get_blob_threshold] 30 + } + } + + # + # Now, reopen the file and make sure the key/data pairs + # look right. + # + puts "\tTest$reopen.c: Dump contents forward" + if { $txnenv == 1 } { + set t [$env txn] + error_check_good txn [is_valid_txn $t $env] TRUE + set txn "-txn $t" + } + if { [is_heap $omethod] == 1 } { + set nameslist [array get names] + set ridslist [array get rids] + test008_heap_dump_bin_file_direction \ + $db $txn $t1 forward $nameslist $ridslist + } else { + dump_bin_file $db $txn $t1 test008.check + } + if { $txnenv == 1 } { + error_check_good txn [$t commit] 0 + } + + set oid [open $t2.tmp w] + if { [is_heap $omethod] == 1 } { + incr count -1 + while { $count >= 0 } { + puts $oid $rids($count) + incr count -1 + } + } else { + foreach f $file_list { + puts $oid $f + } + } + close $oid + filesort $t2.tmp $t2 $fsflags + fileremove $t2.tmp + filesort $t1 $t3 $fsflags + + error_check_good Test$reopen:diff($t3,$t2) \ + [filecmp $t3 $t2] 0 + + # + # Now, reopen the file and run the last test again + # in reverse direction. + # + puts "\tTest$reopen.d: Dump contents backward" + if { $txnenv == 1 } { + set t [$env txn] + error_check_good txn [is_valid_txn $t $env] TRUE + set txn "-txn $t" + } + if { [is_heap $omethod] == 1 } { + test008_heap_dump_bin_file_direction \ + $db $txn $t1 backward $nameslist $ridslist + } else { + dump_bin_file_direction \ + $db $txn $t1 test008.check "-last" "-prev" + } + if { $txnenv == 1 } { + error_check_good txn [$t commit] 0 + } + + filesort $t1 $t3 $fsflags + + error_check_good Test$reopen:diff($t3,$t2) \ + [filecmp $t3 $t2] 0 + error_check_good close:$db [$db close] 0 + } } proc test008.check { binfile tmpfile } { - global tnum source ./include.tcl error_check_good diff($binfile,$tmpfile) \ [filecmp $binfile $tmpfile] 0 } + +proc test008_heap_put_file { db txn file } { + source ./include.tcl + + set fid [open $file r] + fconfigure $fid -translation binary + set data [read $fid] + close $fid + + set ret [catch {eval {$db put} $txn -append {$data}} res] + error_check_good put_file $ret 0 + + return $res +} + +proc test008_heap_dump_bin_file_direction { db \ + txn outfile direction fnameslist ridslist } { + source ./include.tcl + + set d1 $testdir/d1 + set outf [open $outfile w] + array set fnames $fnameslist + array set rids $ridslist + set len [llength $ridslist] + + # Now we will get each key from the DB and dump to outfile + set c [eval {$db cursor} $txn] + if { $direction == "forward" } { + set begin -first + set cont -next + } else { + set begin -last + set cont -prev + } + + for {set d [$c get $begin] } \ + { [llength $d] != 0 } {set d [$c get $cont] } { + set k [lindex [lindex $d 0] 0] + set data [lindex [lindex $d 0] 1] + set ofid [open $d1 w] + fconfigure $ofid -translation binary + puts -nonewline $ofid $data + close $ofid + + # Look for the corresponding file name based on the rid. + for { set i 0 } { $i < $len} { incr i } { + if { $k == $rids($i) } { + break + } + } + error_check_good file_rid [expr $i < $len] 1 + test008.check $fnames($i) $d1 + puts $outf $k + } + close $outf + error_check_good curs_close [$c close] 0 + fileremove -f $d1 +} |