diff options
author | Jan Lindström <jan.lindstrom@skysql.com> | 2014-12-29 21:55:20 +0200 |
---|---|---|
committer | Sergei Golubchik <serg@mariadb.org> | 2015-02-10 10:21:17 +0100 |
commit | 9cdf494197ae53fef5712ab9cbdecbed98462f1f (patch) | |
tree | 3725e0e866586f6875f4229c9a33fdb7790f6a58 /storage/xtradb | |
parent | d7d589dc01f6d70d1518b74d46fd3b75e76267f5 (diff) | |
download | mariadb-git-9cdf494197ae53fef5712ab9cbdecbed98462f1f.tar.gz |
Fixed XtraDB implementation of encryption and page encryption.
Diffstat (limited to 'storage/xtradb')
-rw-r--r-- | storage/xtradb/CMakeLists.txt | 1 | ||||
-rw-r--r-- | storage/xtradb/buf/buf0buf.cc | 141 | ||||
-rw-r--r-- | storage/xtradb/buf/buf0dblwr.cc | 23 | ||||
-rw-r--r-- | storage/xtradb/buf/buf0flu.cc | 26 | ||||
-rw-r--r-- | storage/xtradb/buf/buf0rea.cc | 4 | ||||
-rw-r--r-- | storage/xtradb/dict/dict0stats_bg.cc | 15 | ||||
-rw-r--r-- | storage/xtradb/fil/fil0crypt.cc | 3 | ||||
-rw-r--r-- | storage/xtradb/fil/fil0fil.cc | 35 | ||||
-rw-r--r-- | storage/xtradb/fil/fil0pagecompress.cc | 4 | ||||
-rw-r--r-- | storage/xtradb/fil/fil0pageencryption.cc | 628 | ||||
-rw-r--r-- | storage/xtradb/handler/ha_innodb.cc | 8 | ||||
-rw-r--r-- | storage/xtradb/include/buf0buf.h | 14 | ||||
-rw-r--r-- | storage/xtradb/include/fil0fil.h | 8 | ||||
-rw-r--r-- | storage/xtradb/include/fil0pageencryption.h | 52 | ||||
-rw-r--r-- | storage/xtradb/include/fsp0pageencryption.ic | 12 | ||||
-rw-r--r-- | storage/xtradb/include/os0file.h | 12 | ||||
-rw-r--r-- | storage/xtradb/include/os0file.ic | 3 | ||||
-rw-r--r-- | storage/xtradb/include/univ.i | 3 | ||||
-rw-r--r-- | storage/xtradb/log/log0crypt.cc | 41 | ||||
-rw-r--r-- | storage/xtradb/log/log0log.cc | 16 | ||||
-rw-r--r-- | storage/xtradb/log/log0recv.cc | 16 | ||||
-rw-r--r-- | storage/xtradb/os/os0file.cc | 139 |
22 files changed, 297 insertions, 907 deletions
diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index 121d7914083..f32bf5db360 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -363,7 +363,6 @@ SET(INNOBASE_SOURCES eval/eval0proc.cc fil/fil0fil.cc fil/fil0pagecompress.cc - fil/fil0pageencryption.cc fil/fil0crypt.cc fsp/fsp0fsp.cc fut/fut0fut.cc diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 7ae1aba31d7..37b1c65dfe1 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -59,6 +59,7 @@ Created 11/5/1995 Heikki Tuuri #include "ut0byte.h" #include "fil0pageencryption.h" +#include "fil0pagecompress.h" /* prototypes for new functions added to ha_innodb.cc */ @@ -574,7 +575,7 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { - ulint page_encrypted = fil_page_is_encrypted(read_buf); + ulint page_encrypted = fil_page_is_compressed_encrypted(read_buf) || fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; ibool crc32_inited = FALSE; @@ -1080,8 +1081,11 @@ buf_block_init( block->page.io_fix = BUF_IO_NONE; block->page.crypt_buf = NULL; block->page.crypt_buf_free = NULL; + block->page.comp_buf = NULL; + block->page.comp_buf_free = NULL; block->page.key_version = 0; + block->modify_clock = 0; #if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG @@ -3580,6 +3584,8 @@ buf_page_init_low( bpage->write_size = 0; bpage->crypt_buf = NULL; bpage->crypt_buf_free = NULL; + bpage->comp_buf = NULL; + bpage->comp_buf_free = NULL; bpage->key_version = 0; HASH_INVALIDATE(bpage, hash); @@ -5787,6 +5793,11 @@ buf_page_encrypt_before_write( buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ const byte* src_frame) /*!< in: src frame */ { + if (srv_encrypt_tables == FALSE) { + /* Encryption is disabled */ + return const_cast<byte*>(src_frame); + } + if (bpage->offset == 0) { /* Page 0 of a tablespace is not encrypted */ ut_ad(bpage->key_version == 0); @@ -5806,6 +5817,7 @@ buf_page_encrypt_before_write( } ulint zip_size = buf_page_get_zip_size(bpage); + ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; /** * TODO(jonaso): figure out more clever malloc strategy @@ -5822,18 +5834,28 @@ buf_page_encrypt_before_write( * considered a lot. */ - bpage->crypt_buf_free = (byte*)malloc(UNIV_PAGE_SIZE*2); - byte *dst_frame = bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE); + if (bpage->crypt_buf_free == NULL) { + bpage->crypt_buf_free = (byte*)malloc(page_size*2); + // TODO: Is 4k aligment enough ? + bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, page_size); + } - // encrypt page content - fil_space_encrypt(bpage->space, bpage->offset, - bpage->newest_modification, - src_frame, zip_size, dst_frame, 0); + byte *dst_frame = bpage->crypt_buf; - unsigned key_version = - mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ut_ad(key_version == 0 || key_version >= bpage->key_version); - bpage->key_version = key_version; + if (!fil_space_is_page_compressed(bpage->space)) { + // encrypt page content + fil_space_encrypt(bpage->space, bpage->offset, + bpage->newest_modification, + src_frame, zip_size, dst_frame, 0); + + unsigned key_version = + mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ut_ad(key_version == 0 || key_version >= bpage->key_version); + bpage->key_version = key_version; + } else { + // We do compression and encryption later on os0file.cc + dst_frame = (byte *)src_frame; + } // return dst_frame which will be written return dst_frame; @@ -5852,6 +5874,13 @@ buf_page_encrypt_after_write( bpage->crypt_buf_free = NULL; bpage->crypt_buf = NULL; } + + if (bpage->comp_buf_free != NULL) { + free(bpage->comp_buf_free); + bpage->comp_buf_free = NULL; + bpage->comp_buf = NULL; + } + return (TRUE); } @@ -5861,27 +5890,28 @@ Allocates memory to read in an encrypted page byte* buf_page_decrypt_before_read( /*=========================*/ - buf_page_t* bpage) /*!< in/out: buffer page to be read */ + buf_page_t* bpage, /*!< in/out: buffer page to be read */ + ulint zip_size) /*!< in: compressed page size, or 0 */ { - ulint zip_size = buf_page_get_zip_size(bpage); - - if (bpage->offset == 0) { - /* File header pages are not encrypted */ -unencrypted: - if (zip_size) - return bpage->zip.data; - else - return ((buf_block_t*) bpage)->frame; - } - - if (fil_space_check_encryption_read(bpage->space) == false) { - goto unencrypted; - } + ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - // allocate buffer to read data into - bpage->crypt_buf_free = (byte*)malloc(UNIV_PAGE_SIZE*2); - bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE); - return bpage->crypt_buf; + /* + Here we only need to allocate space for not header pages + in case of file space encryption. Table encryption is handled + later. + */ + if (!srv_encrypt_tables || bpage->offset == 0 || + fil_space_check_encryption_read(bpage->space) == false) + return zip_size ? bpage->zip.data : ((buf_block_t*) bpage)->frame; + + if (bpage->crypt_buf_free == NULL) + { + // allocate buffer to read data into + bpage->crypt_buf_free = (byte*)malloc(size*2); + // TODO: Is 4K aligment enough ? + bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, size); + } + return bpage->crypt_buf; } /********************************************************************//** @@ -5893,28 +5923,41 @@ buf_page_decrypt_after_read( buf_page_t* bpage) /*!< in/out: buffer page read from disk */ { ut_ad(bpage->key_version == 0); + ulint zip_size = buf_page_get_zip_size(bpage); + ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + + byte* dst_frame = (zip_size) ? bpage->zip.data : + ((buf_block_t*) bpage)->frame; + if (bpage->offset == 0) { /* File header pages are not encrypted */ ut_a(bpage->crypt_buf == NULL); return (TRUE); } - ulint zip_size = buf_page_get_zip_size(bpage); - ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - byte* dst_frame = (zip_size) ? bpage->zip.data : - ((buf_block_t*) bpage)->frame; const byte* src_frame = bpage->crypt_buf != NULL ? bpage->crypt_buf : dst_frame; unsigned key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); if (key_version == 0) { /* the page we read is unencrypted */ - if (dst_frame != src_frame) { - /* but we had allocated a crypt_buf */ - memcpy(dst_frame, src_frame, size); + if (fil_page_is_compressed(dst_frame)) { + if (bpage->comp_buf_free == NULL) { + bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); + // TODO: is 4k aligment enough ? + bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE); + } + + fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL); + } else { + if (dst_frame != src_frame) { + /* but we had allocated a crypt_buf */ + // TODO: Can this be avoided ? + memcpy(dst_frame, src_frame, size); + } } } else { /* the page we read is encrypted */ @@ -5923,13 +5966,27 @@ buf_page_decrypt_after_read( * malloc a buffer, copy page to it * and then decrypt from that into real page*/ bpage->crypt_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); + // TODO: is 4k aligment enough ? src_frame = bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE); memcpy(bpage->crypt_buf, dst_frame, size); } /* decrypt from src_frame to dst_frame */ fil_space_decrypt(bpage->space, src_frame, size, dst_frame); + + /* decompress from dst_frame to comp_buf and then copy to + buffer pool */ + if (page_compressed_encrypted) { + if (bpage->comp_buf_free == NULL) { + bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); + // TODO: is 4k aligment enough ? + bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE); + } + + fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL); + } } + bpage->key_version = key_version; if (bpage->crypt_buf_free != NULL) { @@ -5938,6 +5995,14 @@ buf_page_decrypt_after_read( bpage->crypt_buf = NULL; bpage->crypt_buf_free = NULL; } + + if (bpage->comp_buf_free != NULL) { + // free temp page + free(bpage->comp_buf_free); + bpage->comp_buf = NULL; + bpage->comp_buf_free = NULL; + } + return (TRUE); } diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 7b5cd4c12c2..75e86ac4326 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -524,7 +524,7 @@ buf_dblwr_process() fil_io(OS_FILE_READ, true, space_id, zip_size, page_no, 0, zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, NULL, 0); + read_buf, NULL, 0, 0); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -579,7 +579,7 @@ buf_dblwr_process() fil_io(OS_FILE_WRITE, true, space_id, zip_size, page_no, 0, zip_size ? zip_size : UNIV_PAGE_SIZE, - page, NULL, 0); + page, NULL, 0, 0); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -599,7 +599,7 @@ buf_dblwr_process() zip_size, page_no, 0, zip_size ? zip_size : UNIV_PAGE_SIZE, - page, NULL, 0); + page, NULL, 0, 0); } } } @@ -621,9 +621,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0); ut_free(unaligned_buf); } @@ -833,7 +833,8 @@ buf_dblwr_write_block_to_datafile( buf_page_get_page_no(bpage), 0, buf_page_get_zip_size(bpage), frame, - (void*) bpage, 0); + (void*) bpage, 0, + bpage->newest_modification); return; } @@ -846,7 +847,7 @@ buf_dblwr_write_block_to_datafile( fil_io(flags, sync, buf_block_get_space(block), 0, buf_block_get_page_no(block), 0, UNIV_PAGE_SIZE, frame, (void*) block, - (ulint *)&bpage->write_size); + (ulint *)&bpage->write_size, bpage->newest_modification); } /********************************************************************//** @@ -940,7 +941,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block1, 0, len, - (void*) write_buf, NULL, 0); + (void*) write_buf, NULL, 0, 0); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -956,7 +957,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block2, 0, len, - (void*) write_buf, NULL, 0); + (void*) write_buf, NULL, 0, 0); flush: /* increment the doublewrite flushed pages counter */ @@ -1189,14 +1190,14 @@ retry: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, offset, 0, UNIV_PAGE_SIZE, (void*) (buf_dblwr->write_buf - + UNIV_PAGE_SIZE * i), NULL, 0); + + UNIV_PAGE_SIZE * i), NULL, 0,bpage->newest_modification); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, offset, 0, UNIV_PAGE_SIZE, frame, - NULL, 0); + NULL, 0, bpage->newest_modification); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 571aaa2151c..c7356015788 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -955,10 +955,16 @@ buf_flush_write_block_low( if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - sync, buf_page_get_space(bpage), zip_size, - buf_page_get_page_no(bpage), 0, + sync, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, bpage, &bpage->write_size); + frame, + bpage, + &bpage->write_size, + bpage->newest_modification); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer is initialized. User can define do we use atomic writes @@ -969,10 +975,16 @@ buf_flush_write_block_low( if (awrites == ATOMIC_WRITES_ON) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - FALSE, buf_page_get_space(bpage), zip_size, - buf_page_get_page_no(bpage), 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, bpage, &bpage->write_size); + FALSE, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + frame, + bpage, + &bpage->write_size, + bpage->newest_modification); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 1bb861cf820..271fcdc12aa 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -232,14 +232,14 @@ not_to_recover: *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, zip_size, offset, 0, zip_size, - frame, bpage, 0, trx); + frame, bpage, 0, trx, 0); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, 0, offset, 0, UNIV_PAGE_SIZE, - frame, bpage, &bpage->write_size, trx); + frame, bpage, &bpage->write_size, trx, 0); } if (sync) { diff --git a/storage/xtradb/dict/dict0stats_bg.cc b/storage/xtradb/dict/dict0stats_bg.cc index ef3a8c89c08..d480e5457de 100644 --- a/storage/xtradb/dict/dict0stats_bg.cc +++ b/storage/xtradb/dict/dict0stats_bg.cc @@ -96,6 +96,21 @@ dict_stats_pool_deinit() recalc_pool.clear(); defrag_pool.clear(); + + /* + recalc_pool may still have its buffer allocated. It will free it when + its destructor is called. + The problem is, memory leak detector is run before the recalc_pool's + destructor is invoked, and will report recalc_pool's buffer as leaked + memory. To avoid that, we force recalc_pool to surrender its buffer + to empty_pool object, which will free it when leaving this function: + */ + recalc_pool_t recalc_empty_pool; + defrag_pool_t defrag_empty_pool; + memset(&recalc_empty_pool, 0, sizeof(recalc_pool_t)); + memset(&defrag_empty_pool, 0, sizeof(defrag_pool_t)); + recalc_pool.swap(recalc_empty_pool); + defrag_pool.swap(defrag_empty_pool); } /*****************************************************************//** diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index e34297f4f86..930b8c63255 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -209,8 +209,8 @@ fil_crypt_get_key(byte *dst, uint* key_length, else { // load iv + int rc = GetCryptoIV(version, (unsigned char*)iv, iv_len); - fprintf(stderr, " %d\n",rc); if (rc != CRYPT_KEY_OK) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -817,7 +817,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, || page_type == FIL_PAGE_PAGE_COMPRESSED); ulint orig_page_type=0; - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { key_version = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index e4be4f6910c..d0abced2cdb 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -25,6 +25,10 @@ Created 10/25/1995 Heikki Tuuri *******************************************************/ #include "fil0fil.h" +#include "fil0pagecompress.h" +#include "fsp0pagecompress.h" +#include "fil0pageencryption.h" +#include "fsp0pageencryption.h" #include <debug_sync.h> #include <my_dbug.h> @@ -46,6 +50,7 @@ Created 10/25/1995 Heikki Tuuri #include "page0zip.h" #include "trx0sys.h" #include "row0mysql.h" +#include "os0file.h" #ifndef UNIV_HOTBACKUP # include "buf0lru.h" # include "ibuf0ibuf.h" @@ -55,10 +60,6 @@ Created 10/25/1995 Heikki Tuuri # include "srv0srv.h" static ulint srv_data_read, srv_data_written; #endif /* !UNIV_HOTBACKUP */ -#include "fil0pagecompress.h" - -#include "fil0pageencryption.h" -#include "fsp0pageencryption.h" #include "zlib.h" #ifdef __linux__ @@ -278,7 +279,7 @@ fil_read( void* buf, /*!< in/out: buffer where to store data read; in aio this must be appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync - aio used, else ignored */ + aio used, else ignored */ ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if @@ -286,7 +287,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size)); + byte_offset, len, buf, message, write_size, 0)); } /********************************************************************//** @@ -312,17 +313,18 @@ fil_write( void* buf, /*!< in: buffer from which to write; in aio this must be appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync - aio used, else ignored */ - ulint* write_size) /*!< in/out: Actual write size initialized + aio used, else ignored */ + ulint* write_size, /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ + lsn_t lsn) /* lsn of the newest modification */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size)); + byte_offset, len, buf, message, write_size, lsn)); } /*******************************************************************//** @@ -1872,7 +1874,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0); + UNIV_PAGE_SIZE, buf, NULL, 0, 0); } mem_free(buf1); @@ -5246,7 +5248,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, space_id, NULL, 0, 0, 0, 0, 0); + node, NULL, space_id, NULL, 0, 0, 0, 0, 0, 0); #endif /* UNIV_HOTBACKUP */ if (success) { os_has_said_disk_full = FALSE; @@ -5340,7 +5342,7 @@ fil_extend_tablespaces_to_stored_len(void) single-threaded operation */ error = fil_read(TRUE, space->id, fsp_flags_get_zip_size(space->flags), - 0, 0, UNIV_PAGE_SIZE, buf, NULL); + 0, 0, UNIV_PAGE_SIZE, buf, NULL, 0); ut_a(error == DB_SUCCESS); size_in_header = fsp_get_size_low(buf); @@ -5621,13 +5623,15 @@ _fil_io( or from where to write; in aio this must be appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync - aio used, else ignored */ + aio used, else ignored */ ulint* write_size, /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - trx_t* trx) + trx_t* trx, + lsn_t lsn) /* lsn of the newest modification */ + { ulint mode; fil_space_t* space; @@ -5875,7 +5879,8 @@ _fil_io( page_compression_level, write_size, page_encrypted, - page_encryption_key); + page_encryption_key, + lsn); #else /* In mysqlbackup do normal i/o, not aio */ diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index fa25d8875ae..f80d94d9f15 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -283,10 +283,12 @@ fil_compress_page( /* Let's not compress file space header or extent descriptor */ - if ((orig_page_type == FIL_PAGE_TYPE_FSP_HDR) || (orig_page_type == FIL_PAGE_TYPE_XDES) ) { + if ((orig_page_type == FIL_PAGE_TYPE_FSP_HDR) + || (orig_page_type == FIL_PAGE_TYPE_XDES) ) { *out_len = len; return (buf); } + level = compression_level; ut_ad(fil_space_is_page_compressed(space_id)); diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc deleted file mode 100644 index 49c42615e19..00000000000 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ /dev/null @@ -1,628 +0,0 @@ -/***************************************************************************** - -Copyright (C) 2014 eperi GmbH. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ - -/***************************************************************** - @file fil/fil0pageencryption.cc - Implementation for page encryption file spaces. - - Created 08/25/2014 Ludger Göckel eperi-GmbH - Modified 11/26/2014 Jan Lindström MariaDB Corporation - ***********************************************************************/ - -#include "fil0fil.h" -#include "fil0pageencryption.h" -#include "fsp0pageencryption.h" -#include "my_dbug.h" -#include "page0zip.h" - -#include "buf0checksum.h" -#include <my_global.h> -#include <my_aes.h> -#include <math.h> - -/* - * derived from libFLAC, which is gpl v2 - */ -byte crc_table[] = { - 0x00,0x07,0x0E,0x09,0x1C,0x1B,0x12,0x15,0x38,0x3F,0x36,0x31,0x24,0x23,0x2A,0x2D,0x70,0x77,0x7E,0x79, - 0x6C,0x6B,0x62,0x65,0x48,0x4F,0x46,0x41,0x54,0x53,0x5A,0x5D,0xE0,0xE7,0xEE,0xE9,0xFC,0xFB,0xF2,0xF5, - 0xD8,0xDF,0xD6,0xD1,0xC4,0xC3,0xCA,0xCD,0x90,0x97,0x9E,0x99,0x8C,0x8B,0x82,0x85,0xA8,0xAF,0xA6,0xA1, - 0xB4,0xB3,0xBA,0xBD,0xC7,0xC0,0xC9,0xCE,0xDB,0xDC,0xD5,0xD2,0xFF,0xF8,0xF1,0xF6,0xE3,0xE4,0xED,0xEA, - 0xB7,0xB0,0xB9,0xBE,0xAB,0xAC,0xA5,0xA2,0x8F,0x88,0x81,0x86,0x93,0x94,0x9D,0x9A,0x27,0x20,0x29,0x2E, - 0x3B,0x3C,0x35,0x32,0x1F,0x18,0x11,0x16,0x03,0x04,0x0D,0x0A,0x57,0x50,0x59,0x5E,0x4B,0x4C,0x45,0x42, - 0x6F,0x68,0x61,0x66,0x73,0x74,0x7D,0x7A,0x89,0x8E,0x87,0x80,0x95,0x92,0x9B,0x9C,0xB1,0xB6,0xBF,0xB8, - 0xAD,0xAA,0xA3,0xA4,0xF9,0xFE,0xF7,0xF0,0xE5,0xE2,0xEB,0xEC,0xC1,0xC6,0xCF,0xC8,0xDD,0xDA,0xD3,0xD4, - 0x69,0x6E,0x67,0x60,0x75,0x72,0x7B,0x7C,0x51,0x56,0x5F,0x58,0x4D,0x4A,0x43,0x44,0x19,0x1E,0x17,0x10, - 0x05,0x02,0x0B,0x0C,0x21,0x26,0x2F,0x28,0x3D,0x3A,0x33,0x34,0x4E,0x49,0x40,0x47,0x52,0x55,0x5C,0x5B, - 0x76,0x71,0x78,0x7F,0x6A,0x6D,0x64,0x63,0x3E,0x39,0x30,0x37,0x22,0x25,0x2C,0x2B,0x06,0x01,0x08,0x0F, - 0x1A,0x1D,0x14,0x13,0xAE,0xA9,0xA0,0xA7,0xB2,0xB5,0xBC,0xBB,0x96,0x91,0x98,0x9F,0x8A,0x8D,0x84,0x83, - 0xDE,0xD9,0xD0,0xD7,0xC2,0xC5,0xCC,0xCB,0xE6,0xE1,0xE8,0xEF,0xFA,0xFD,0xF4,0xF3 - -}; - -/****************************************************************//** -Calculate checksum for encrypted pages -@return checksum */ -static -byte -fil_page_encryption_calc_checksum( -/*==============================*/ - unsigned char* buf, /*!<in: buffer where to calculate checksum */ - ulint len) /*!<in: buffer length */ -{ - byte crc = 0; - for (ulint i = 0; i < len; i++) { - crc = crc_table[(crc ^ buf[i]) & 0xff]; - } - return crc; -} - -/****************************************************************//** -Recalculate checksum for encrypted pages */ -static -void -do_check_sum( -/*=========*/ - ulint page_size, /*!< in: page size */ - ulint zip_size, /*!< in: compressed page size */ - byte* buf) /*!< in: buffer */ -{ - ib_uint32_t checksum = 0; - - if (zip_size) { - checksum = page_zip_calc_checksum(buf,zip_size, - static_cast<srv_checksum_algorithm_t>( - srv_checksum_algorithm)); - - mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); - return; - } - - switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { - case SRV_CHECKSUM_ALGORITHM_CRC32: - case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - checksum = buf_calc_page_crc32(buf); - break; - case SRV_CHECKSUM_ALGORITHM_INNODB: - case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: - checksum = (ib_uint32_t) buf_calc_page_new_checksum(buf); - break; - case SRV_CHECKSUM_ALGORITHM_NONE: - case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: - checksum = BUF_NO_CHECKSUM_MAGIC; - break; - /* no default so the compiler will emit a warning if new enum - is added and not handled here */ - } - - mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); - - /* old style checksum is omitted */ -} - -/****************************************************************//** - For page encrypted pages encrypt the page before actual write - operation. - - Note, that FIL_PAGE_TYPE_FSP_HDR and FIL_PAGE_TYPE_XDES type pages - are not encrypted! - - Pages are encrypted with AES/CBC/NoPadding algorithm. - - "No padding" is used to ensure, that the encrypted page does not - exceed the page size. If "no padding" is used, the input for encryption - must be of size (multiple * AES blocksize). AES Blocksize is usually 16 - (bytes). - - Everything in the page is encrypted except for the 38 byte FIL header. - Since the length of the payload is not a multiple of the AES blocksize, - and to ensure that every byte of the payload is encrypted, two encryption - operations are done. Each time with a block of adequate size as input. - 1st block contains everything from beginning of payload bytes except for - the remainder. 2nd block is of size 64 and contains the remainder and - the last (64 - sizeof(remainder)) bytes of the encrypted 1st block. - - Each encrypted page receives a new page type for PAGE_ENCRYPTION. - The original page type (2 bytes) is stored in the Checksum header of the - page (position FIL_PAGE_SPACE_OR_CHKSUM). Additionally the encryption - key identifier is stored in the Checksum Header. This uses 1 byte. - Checksum verification for encrypted pages is disabled. This checksum - should be restored after decryption. - - To be able to verify decryption in a later stage, a 1-byte checksum at - position 4 of the FIL_PAGE_SPACE_OR_CHKSUM header is stored. - For page compressed table pages the log base 2 of the length of the - encrypted data is stored. - - @return encrypted page or original page if encryption failed to be - written*/ -UNIV_INTERN -byte* -fil_encrypt_page( -/*==============*/ - ulint space_id, /*!< in: tablespace id of the table. */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: encrypted buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint encryption_key, /*!< in: encryption key */ - ulint* out_len, /*!< out: actual length of encrypted page */ - ulint* errorCode, /*!< out: an error code. set, - if page is intentionally not encrypted */ - byte* tmp_encryption_buf) /*!< in: temporary buffer or NULL */ -{ - - int err = AES_OK; - int key = 0; - uint32 data_size = 0; - ulint orig_page_type = 0; - uint32 write_size = 0; - fil_space_t* space = NULL; - byte* tmp_buf = NULL; - ulint page_len = 0; - ulint offset = 0; - - ut_ad(buf);ut_ad(out_buf); - key = encryption_key; - - *errorCode = AES_OK; - - ut_ad(fil_space_is_page_encrypted(space_id)); - fil_system_enter(); - space = fil_space_get_by_id(space_id); - fil_system_exit(); - -#ifdef UNIV_DEBUG_PAGEENCRYPTION - ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET); - fprintf(stderr, - "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu, page no %lu\n", - space_id, fil_space_name(space), len, pageno); -#endif /* UNIV_DEBUG_PAGEENCRYPTION */ - - /* read original page type */ - orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - /* Do not encrypt file space header or extend descriptor */ - if ((orig_page_type == FIL_PAGE_TYPE_FSP_HDR) - || (orig_page_type == FIL_PAGE_TYPE_XDES) ) { - *errorCode = PAGE_ENCRYPTION_WILL_NOT_ENCRYPT; - *out_len = len; - return (buf); - } - - if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { - page_len = log10((double)len)/log10((double)2); - } - - byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); - - /* data_size bytes will be encrypted at first. - * data_size will be the length of the cipher text since no padding is used.*/ - data_size = ((len - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; - - - unsigned char rkey[GetCryptoKeySize(encryption_key)]; - uint key_len = sizeof(rkey); - - unsigned char iv[16]; - uint iv_len = sizeof(iv); - - if (!HasCryptoKey(encryption_key)) { - err = PAGE_ENCRYPTION_KEY_MISSING; - } else { - int rc; - - rc = GetCryptoKey(encryption_key, rkey, key_len); - if (rc != AES_OK) - { - err = PAGE_ENCRYPTION_KEY_MISSING; - } - - rc = GetCryptoIV(encryption_key, iv, iv_len); - if (rc != AES_OK) - { - err = PAGE_ENCRYPTION_KEY_MISSING; - } - } - - /* 1st encryption: data_size bytes starting from FIL_PAGE_DATA */ - if (err == AES_OK) { - err = my_aes_encrypt_dynamic( - (uchar*) buf + FIL_PAGE_DATA, - data_size, - (uchar *) out_buf + FIL_PAGE_DATA, - &write_size, - (const unsigned char *) rkey, - key_len, - (const unsigned char *) iv, - iv_len, - 1); - - ut_ad(write_size == data_size); - - if (err == AES_OK) { - /* copy remaining bytes from input buffer to output buffer. - * Note, that this copies the final 8 bytes of a - * page, which consists of the - * Old-style checksum and the "Low 32 bits of LSN */ - memcpy(out_buf + FIL_PAGE_DATA + data_size, - buf + FIL_PAGE_DATA + data_size , - len - FIL_PAGE_DATA -data_size); - - if (tmp_encryption_buf == NULL) { - //create temporary buffer for 2nd encryption - tmp_buf = static_cast<byte *>(ut_malloc(64)); - } else { - tmp_buf = tmp_encryption_buf; - } - - /* 2nd encryption: 64 bytes from out_buf, - result length is 64 bytes */ - err = my_aes_encrypt_dynamic((uchar*)out_buf + len -offset -64, - 64, - (uchar*)tmp_buf, - &write_size, - (const unsigned char *)rkey, - key_len, - (const unsigned char *)iv, - iv_len, 1); - ut_ad(write_size == 64); - - /* copy 64 bytes from 2nd encryption to out_buf*/ - memcpy(out_buf + len - offset -64, tmp_buf, 64); - } - - } - - /* error handling */ - if (err != AES_OK) { - /* If an error occurred we leave the actual page as it was */ - - fprintf(stderr, - "InnoDB: Warning: Encryption failed for space %lu " - "name %s len %lu rt %d write %lu, error: %d\n", - space_id, fil_space_name(space), len, err, (ulint)data_size, err); - fflush(stderr); - srv_stats.pages_page_encryption_error.inc(); - *out_len = len; - - /* free temporary buffer */ - if (tmp_buf!=NULL && tmp_encryption_buf == NULL) { - ut_free(tmp_buf); - } - *errorCode = err; - - return (buf); - } - - /* Set up the page header. Copied from input buffer*/ - memcpy(out_buf, buf, FIL_PAGE_DATA); - - /* Set up the correct page type */ - mach_write_to_2(out_buf + FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - - /* The 1st checksum field is used to store original page type, etc. - * checksum check for page encrypted pages is omitted. - */ - - /* Set up the encryption key. Written to the 1st byte of - the checksum header field. This header is currently used to store data. */ - mach_write_to_1(out_buf + FIL_PAGE_SPACE_OR_CHKSUM, key); - - /* store original page type. Written to 2nd and 3rd byte - of the checksum header field */ - mach_write_to_2(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 1, orig_page_type); - - if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { - /* set byte 4 of checksum field to page length (ln(len)) */ - memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, page_len, 1); - } else { - /* set byte 4 of checksum field to checksum byte */ - memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, checksum_byte, 1); - } - -#ifdef UNIV_DEBUG - /* Verify */ - ut_ad(fil_page_is_encrypted(out_buf)); - -#endif /* UNIV_DEBUG */ - - srv_stats.pages_page_encrypted.inc(); - *out_len = len; - - /* free temporary buffer */ - if (tmp_buf!=NULL && tmp_encryption_buf == NULL) { - ut_free(tmp_buf); - } - - return (out_buf); -} - -/****************************************************************//** - For page encrypted pages decrypt the page after actual read - operation. - - See fil_encrypt_page for details, how the encryption works. - - If the decryption can be verified, original page should be completely restored. - This includes original page type, 4-byte checksum field at page start. - If it is not a page compressed table's page, decryption is verified against - a 1-byte checksum built over the plain data bytes. If this verification - fails, an error state is returned. - - @return decrypted page */ -ulint -fil_decrypt_page( -/*=============*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< in/out: buffer from which to read; in aio - this must be appropriately aligned */ - ulint len, /*!< in: length buffer, which should be decrypted.*/ - ulint* write_size, /*!< out: size of the decrypted - data. If no error occurred equal to len */ - ibool* page_compressed,/*!<out: is page compressed.*/ - byte* tmp_encryption_buf) /*!< in: temporary buffer or NULL */ -{ - int err = AES_OK; - ulint page_decryption_key; - uint32 data_size = 0; - ulint orig_page_type = 0; - uint32 tmp_write_size = 0; - ulint offset = 0; - byte *in_buf = NULL; - byte *tmp_buf = NULL; - fil_space_t* space = NULL; - - ulint page_compression_flag = 0; - - ut_ad(buf); - ut_ad(len); - - /* Before actual decrypt, make sure that page type is correct */ - ulint current_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - if ((current_page_type == FIL_PAGE_TYPE_FSP_HDR) - || (current_page_type == FIL_PAGE_TYPE_XDES)) { - /* assumed as unencrypted */ - if (write_size!=NULL) { - *write_size = len; - } - return AES_OK; - } - - if (current_page_type != FIL_PAGE_PAGE_ENCRYPTED) { - - fprintf(stderr, "InnoDB: Corruption: We try to decrypt corrupted page\n" - "InnoDB: CRC %lu type %lu.\n" - "InnoDB: len %lu\n", - mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM), - mach_read_from_2(buf + FIL_PAGE_TYPE), len); - - fflush(stderr); - return PAGE_ENCRYPTION_WRONG_PAGE_TYPE; - } - - /* 1st checksum field is used to store original page type, etc. - * checksum check for page encrypted pages is omitted. - */ - - /* read page encryption key */ - page_decryption_key = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM); - - /* Get the page type */ - orig_page_type = mach_read_from_2(buf + FIL_PAGE_SPACE_OR_CHKSUM + 1); - - /* read checksum byte */ - byte stored_checksum_byte = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM + 3); - - if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { - if (page_compressed != NULL) { - *page_compressed = 1L; - } - page_compression_flag = 1; - len = pow((double)2, (double)((int)stored_checksum_byte)); - offset = 0; - } - - data_size = ((len - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; - - - unsigned char rkey[GetCryptoKeySize(page_decryption_key)]; - uint key_len = sizeof(rkey); - - unsigned char iv[16]; - uint iv_len = sizeof(iv); - - if (!HasCryptoKey(page_decryption_key)) { - err = PAGE_ENCRYPTION_KEY_MISSING; - } else { - int rc; - - rc = GetCryptoKey(page_decryption_key, rkey, key_len); - if (rc != AES_OK) - { - err = PAGE_ENCRYPTION_KEY_MISSING; - } - - rc = GetCryptoIV(page_decryption_key, iv, iv_len); - if (rc != AES_OK) - { - err = PAGE_ENCRYPTION_KEY_MISSING; - } - } - - - if (err != AES_OK) { - /* surely key could not be determined. */ - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decrypt failed with error %d, encryption key %d.\n", - err, (int)page_decryption_key); - fflush(stderr); - - return err; - } - - if (tmp_encryption_buf == NULL) { - tmp_buf= static_cast<byte *>(ut_malloc(64)); - } else { - tmp_buf = tmp_encryption_buf; - } - - // If no buffer was given, we need to allocate temporal buffer - if (page_buf == NULL) { -#ifdef UNIV_PAGECOMPRESS_DEBUG - fprintf(stderr, - "InnoDB: Note: FIL: Encryption buffer not given, allocating...\n"); -#endif /* UNIV_PAGECOMPRESS_DEBUG */ - in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); - } else { - in_buf = page_buf; - } - - /* 1st decryption: 64 bytes */ - /* 64 bytes from data area are copied to temporary buffer. - * These are the last 64 of the (encrypted) payload */ - memcpy(tmp_buf, buf + len - offset - 64, 64); - - err = my_aes_decrypt_dynamic( - (const uchar*) tmp_buf, - 64, - (uchar *) in_buf + len - offset - 64, - &tmp_write_size, - (const unsigned char *) rkey, - key_len, - (const unsigned char *) iv, - iv_len, - 1); - - ut_ad(tmp_write_size == 64); - - /* If decrypt fails it means that page is corrupted or has an unknown key */ - if (err != AES_OK) { - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decrypt failed with error %d.\n" - "InnoDB: size %lu len %lu, key %d\n", err, (ulint)data_size, - len, (int)page_decryption_key); - fflush(stderr); - - if (tmp_encryption_buf == NULL) { - ut_free(tmp_buf); - } - - if (page_buf == NULL) { - ut_free(in_buf); - } - return err; - } - - ut_ad(tmp_write_size == 64); - - /* copy 1st part of payload from buf to in_buf */ - /* do not override result of 1st decryption */ - memcpy(in_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, len -offset -64 - FIL_PAGE_DATA); - - - /* Decrypt rest of the page */ - err = my_aes_decrypt_dynamic((uchar*) in_buf + FIL_PAGE_DATA, - data_size, - (uchar *) buf + FIL_PAGE_DATA, - &tmp_write_size, - (const unsigned char *)&rkey, - key_len, - (const unsigned char *)&iv, - iv_len, - 1); - - ut_ad(tmp_write_size = data_size); - - /* copy remaining bytes from in_buf to buf. - */ - ulint bytes_to_copy = len - FIL_PAGE_DATA - data_size - offset; - memcpy(buf + FIL_PAGE_DATA + data_size, in_buf + FIL_PAGE_DATA + data_size, bytes_to_copy); - - /* apart from header data everything is now in in_buf */ - - if (tmp_encryption_buf == NULL) { - ut_free(tmp_buf); - } - -#ifdef UNIV_PAGEENCRIPTION_DEBUG - fprintf(stderr, "InnoDB: Note: Decryption succeeded for len %lu\n", len); - fflush(stderr); -#endif - - if (page_buf == NULL) { - ut_free(in_buf); - } - - /* setting original page type */ - mach_write_to_2(buf + FIL_PAGE_TYPE, orig_page_type); - - ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET); - ulint flags = 0; - ulint zip_size = 0; - - /* please note, that page with number 0 is not encrypted */ - if (pageno == 0 ) { - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); - } else { - ulint space_id = mach_read_from_4(buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fil_system_enter(); - space = fil_space_get_by_id(space_id); - flags = fil_space_flags(space); - fil_system_exit(); - } - - if (!(page_compression_flag)) { - zip_size = fsp_flags_get_zip_size(flags); - } - - if (write_size!=NULL) { - *write_size = len; - } - - if (!(page_compression_flag)) { - byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); - - if (checksum_byte != stored_checksum_byte) { - err = PAGE_ENCRYPTION_WRONG_KEY; - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decryption verification failed with error %d," - " encryption key %d.\n", - err, (int)page_decryption_key); - fflush(stderr); - return err; - } - - /* calc check sums and write to the buffer, if page is not of type PAGE_COMPRESSED. - * if the decryption is verified, it is assumed that the - * original page was restored, re-calculating the original - * checksums should be ok - */ - do_check_sum(len, zip_size, buf); - } else { - /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ - mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); - } - - srv_stats.pages_page_decrypted.inc(); - - return err; -} - - diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index d65e2e321f2..c7e98f02702 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -11782,6 +11782,14 @@ ha_innobase::check_table_options( atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; if (options->page_encryption) { + if (srv_encrypt_tables) { + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: PAGE_ENCRYPTION not available if innodb_encrypt_tables=ON"); + return "INNODB_ENCRYPT_TABLES"; + } + if (!use_tablespace) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 10b296f6cd4..a2097b5c3b9 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1586,14 +1586,18 @@ struct buf_page_t{ if written again we check is TRIM operation needed. */ - unsigned key_version; /*!< key version for this block */ - byte* crypt_buf; /*!< for encryption the data needs to be + unsigned key_version; /*!< key version for this block */ + byte* crypt_buf; /*!< for encryption the data needs to be copied to a separate buffer before it's encrypted&written. this as a page can be read while it's being flushed */ - byte* crypt_buf_free; /*!< for encryption, allocated buffer - that is then alligned */ - + byte* crypt_buf_free; /*!< for encryption, allocated buffer + that is then alligned */ + byte* comp_buf; /*!< for compression we need + temporal buffer because page + can be read while it's being flushed */ + byte* comp_buf_free; /*!< for compression, allocated + buffer that is then alligned */ #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to buf_pool->page_hash or diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index f271522bbe9..360a26df776 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -959,8 +959,8 @@ fil_space_get_n_reserved_extents( Reads or writes data. This operation is asynchronous (aio). @return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do i/o on a tablespace which does not exist */ -#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size) \ - _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL) +#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, lsn) \ + _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL, lsn) UNIV_INTERN dberr_t @@ -996,7 +996,9 @@ _fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - trx_t* trx) + trx_t* trx, + lsn_t lsn) /* lsn of the newest modification */ + __attribute__((nonnull(8))); /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index c0c48df4e34..ada8fd25d35 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -36,26 +36,32 @@ Helper functions for encryption/decryption page data on to table space. Created 08/25/2014 ***********************************************************************/ - -/******************************PAGE_ENCRYPTION_ERROR*************************************//** +/*******************************************************************//** +Find out whether the page is page encrypted. Returns the page encryption flag of the space, or false if the space is not encrypted. The tablespace must be cached in the memory cache. @return true if page encrypted, false if not or space not found */ ibool fil_space_is_page_encrypted( -/*=========================*/ +/*========================*/ ulint id); /*!< in: space id */ - /*******************************************************************//** Find out whether the page is page encrypted @return true if page is page encrypted, false if not */ UNIV_INLINE ibool fil_page_is_encrypted( -/*===================*/ +/*==================*/ + const byte *buf); /*!< in: page */ +/*******************************************************************//** +Find out whether the page is page compressed and then encrypted +@return true if page is page compressed+encrypted, false if not */ +UNIV_INLINE +ibool +fil_page_is_compressed_encrypted( +/*=============================*/ const byte *buf); /*!< in: page */ - /*******************************************************************//** Find out whether the page can be decrypted @@ -63,40 +69,8 @@ Find out whether the page can be decrypted UNIV_INLINE ulint fil_page_encryption_status( -/*===================*/ +/*=======================*/ const byte *buf); /*!< in: page */ -/****************************************************************//** -For page encrypted pages encrypt the page before actual write -operation. -@return encrypted page to be written*/ -byte* -fil_encrypt_page( -/*==============*/ - ulint space_id, /*!< in: tablespace id of the table. */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: encrypted buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint encryption_key, /*!< in: encryption key */ - ulint* out_len, /*!< out: actual length of encrypted page */ - ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ - byte* tmp_encryption_buf); /*!< in: temporary buffer or NULL */ - -/****************************************************************//** -For page encrypted pages decrypt the page after actual read -operation. -@return decrypted page */ -ulint -fil_decrypt_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< in/out: buffer from which to read; in aio - this must be appropriately aligned */ - ulint len, /*!< in: length buffer, which should be decrypted.*/ - ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len */ - ibool* page_compressed,/*!<out: is page compressed.*/ - byte* tmp_encryption_buf); /*!< in: temporary buffer or NULL */ - #endif // fil0pageencryption_h diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic index 7ff002b203e..a2c8c3b0660 100644 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ b/storage/xtradb/include/fsp0pageencryption.ic @@ -25,6 +25,7 @@ Created 08/28/2014 #include "fsp0fsp.h" #include "fil0pageencryption.h" +#include <my_crypt_key_management.h> @@ -146,17 +147,22 @@ fil_page_encryption_status( const byte *buf) /*!< in: page */ { ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + if (page_type == FIL_PAGE_TYPE_FSP_HDR) { ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); if (fsp_flags_is_page_encrypted(flags)) { if (!HasCryptoKey(fsp_flags_get_page_encryption_key(flags))) { /* accessing table would surely fail, because no key or no key provider available */ - return PAGE_ENCRYPTION_KEY_MISSING; + if (!HasCryptoKey(fsp_flags_get_page_encryption_key(flags))) { + return PAGE_ENCRYPTION_KEY_MISSING; + } + return PAGE_ENCRYPTION_ERROR; } } } - if(page_type == FIL_PAGE_PAGE_ENCRYPTED) { - ulint key = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM); + + if(page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (!HasCryptoKey(key)) { return PAGE_ENCRYPTION_KEY_MISSING; } diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index c92cfa186fc..af3ad734f48 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -324,11 +324,11 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, \ trx, page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key) \ + page_encryption, page_encryption_key, lsn) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, \ + page_encryption, page_encryption_key, lsn, \ __FILE__, __LINE__) # define os_file_read(file, buf, offset, n, compressed) \ @@ -378,11 +378,11 @@ to original un-instrumented file I/O APIs */ # define os_aio(type, mode, name, file, buf, offset, n, message1, \ message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key) \ + page_encryption, page_encryption_key, lsn) \ os_aio_func(type, mode, name, file, buf, offset, n, \ message1, message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key) + page_encryption, page_encryption_key, lsn) # define os_file_read(file, buf, offset, n, compressed) \ os_file_read_func(file, buf, offset, n, NULL, compressed) @@ -814,6 +814,7 @@ pfs_os_aio_func( on this file space */ ulint page_encryption_key, /*!< page encryption key to be used */ + lsn_t lsn, /* lsn of the newest modification */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -1203,8 +1204,9 @@ os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key); /*!< page encryption key + ulint page_encryption_key, /*!< page encryption key to be used */ + lsn_t lsn); /* lsn of the newest modification */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in diff --git a/storage/xtradb/include/os0file.ic b/storage/xtradb/include/os0file.ic index c715d52cc13..9ab191bd460 100644 --- a/storage/xtradb/include/os0file.ic +++ b/storage/xtradb/include/os0file.ic @@ -234,6 +234,7 @@ pfs_os_aio_func( ulint page_encryption_key, /*!< page encryption key to be used */ + lsn_t lsn, /* lsn of the newest modification */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -251,7 +252,7 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, space_id, trx, page_compression, page_compression_level, write_size , - page_encryption, page_encryption_key); + page_encryption, page_encryption_key, lsn); register_pfs_file_io_end(locker, n); diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 2a01770da35..0cc56b121ed 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -497,6 +497,9 @@ typedef uint32_t ib_uint32_t; # define IB_ID_FMT UINT64PF +/* Type used for all log sequence number storage and arithmetics */ +typedef ib_uint64_t lsn_t; + #ifdef _WIN64 typedef unsigned __int64 ulint; typedef __int64 lint; diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index 43da1f10c19..0647fd04e84 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -4,7 +4,10 @@ Innodb log encrypt/decrypt Created 11/25/2013 Minli Zhu *******************************************************/ +#include "m_string.h" #include "log0crypt.h" +#include <my_crypt.h> + #include "log0log.h" #include "srv0start.h" // for srv_start_lsn #include "log0recv.h" // for recv_sys @@ -43,9 +46,6 @@ void log_init_crypt_msg_and_nonce(void) /*==============================*/ { -#ifndef HAVE_EncryptAes128Ctr - return; -#else mach_write_to_1(redo_log_crypt_msg, redo_log_purpose_byte); if (my_random_bytes(redo_log_crypt_msg + 1, PURPOSE_BYTE_LEN) != AES_OK) { @@ -64,7 +64,6 @@ log_init_crypt_msg_and_nonce(void) MY_AES_BLOCK_SIZE); abort(); } -#endif } /*********************************************************************//** @@ -77,9 +76,6 @@ log_init_crypt_key( const uint crypt_ver, /*< in: key version */ byte* key) /*< out: crypt key*/ { -#ifndef HAVE_EncryptAes128Ctr - return; -#else if (crypt_ver == UNENCRYPTED_KEY_VER) { fprintf(stderr, "\nInnodb redo log crypto: unencrypted key ver.\n\n"); @@ -105,10 +101,14 @@ log_init_crypt_key( abort(); } - int dst_len; - int rc = EncryptAes128Ecb(mysqld_key, //key - crypt_msg, MY_AES_BLOCK_SIZE, //src, srclen - key, &dst_len); //dst, &dstlen + uint32 dst_len; + my_aes_encrypt_dynamic_type func= get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); + int rc= (*func)(crypt_msg, MY_AES_BLOCK_SIZE, //src, srclen + key, &dst_len, //dst, &dstlen + (unsigned char*)&mysqld_key, sizeof(mysqld_key), + NULL, 0, + 1); + if (rc != AES_OK || dst_len != MY_AES_BLOCK_SIZE) { fprintf(stderr, @@ -116,7 +116,6 @@ log_init_crypt_key( "failed.\n"); abort(); } -#endif } /*********************************************************************//** @@ -146,12 +145,9 @@ log_blocks_crypt( byte* dst_block, /*!< out: blocks after encrypt/decrypt */ const bool is_encrypt) /*!< in: encrypt or decrypt*/ { -#ifndef HAVE_EncryptAes128Ctr - ut_error; // We should never be called -#else byte *log_block = (byte*)block; Crypt_result rc = AES_OK; - int src_len, dst_len; + uint32 src_len, dst_len; byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; ulint log_block_no, log_block_start_lsn; byte *key; @@ -185,17 +181,20 @@ log_blocks_crypt( mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn); mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); - rc = EncryptAes128Ctr(key, aes_ctr_counter, MY_AES_BLOCK_SIZE, // key, counter, block size - log_block + LOG_BLOCK_HDR_SIZE, src_len, // src, src_len - dst_block + LOG_BLOCK_HDR_SIZE, &dst_len); // dst, dst_len - ut_a(rc == CRYPT_OK); + + int rc = (* my_aes_encrypt_dynamic)(log_block + LOG_BLOCK_HDR_SIZE, src_len, + dst_block + LOG_BLOCK_HDR_SIZE, &dst_len, + (unsigned char*)key, 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + 1); + + ut_a(rc == AES_OK); ut_a(dst_len == src_len); log_block += OS_FILE_LOG_BLOCK_SIZE; dst_block += OS_FILE_LOG_BLOCK_SIZE; } return rc; -#endif } /*********************************************************************//** diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 9b33c7858bc..e007bcc3507 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1384,7 +1384,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0); + buf, group, 0, 0); srv_stats.os_log_pending_writes.dec(); } @@ -1551,7 +1551,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0); + group, 0, 0); srv_stats.os_log_pending_writes.dec(); @@ -2144,7 +2144,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0); + buf, ((byte*) group + 1), 0, 0); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2226,7 +2226,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0); } /******************************************************//** @@ -2639,7 +2639,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0); + len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0, 0); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) @@ -2771,7 +2771,7 @@ log_group_archive_file_header_write( dest_offset / UNIV_PAGE_SIZE, dest_offset % UNIV_PAGE_SIZE, 2 * OS_FILE_LOG_BLOCK_SIZE, - buf, &log_archive_io, 0); + buf, &log_archive_io, 0, 0); } /******************************************************//** @@ -2808,7 +2808,7 @@ log_group_archive_completed_header_write( dest_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, buf + LOG_FILE_ARCH_COMPLETED, - &log_archive_io, 0); + &log_archive_io, 0, 0); } /******************************************************//** @@ -2953,7 +2953,7 @@ loop: (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), ut_calc_align(len, OS_FILE_LOG_BLOCK_SIZE), buf, - &log_archive_io, 0); + &log_archive_io, 0, 0); start_lsn += len; next_offset += len; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 3491e246ba1..bb0888d3db7 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -2156,7 +2156,7 @@ recv_apply_log_recs_for_backup(void) error = fil_io(OS_FILE_READ, true, recv_addr->space, zip_size, recv_addr->page_no, 0, zip_size, - block->page.zip.data, NULL, 0); + block->page.zip.data, NULL, 0, 0); if (error == DB_SUCCESS && !buf_zip_decompress(block, TRUE)) { exit(1); @@ -2166,7 +2166,7 @@ recv_apply_log_recs_for_backup(void) recv_addr->space, 0, recv_addr->page_no, 0, UNIV_PAGE_SIZE, - block->frame, NULL, 0); + block->frame, NULL, 0, 0); } if (error != DB_SUCCESS) { @@ -2195,13 +2195,13 @@ recv_apply_log_recs_for_backup(void) recv_addr->space, zip_size, recv_addr->page_no, 0, zip_size, - block->page.zip.data, NULL, 0); + block->page.zip.data, NULL, 0, 0); } else { error = fil_io(OS_FILE_WRITE, true, recv_addr->space, 0, recv_addr->page_no, 0, UNIV_PAGE_SIZE, - block->frame, NULL, 0); + block->frame, NULL, 0, 0); } skip_this_recv_addr: recv_addr = HASH_GET_NEXT(addr_hash, recv_addr); @@ -3175,7 +3175,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0); + log_hdr_buf, max_cp_group, 0, 0); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3206,7 +3206,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0); + log_hdr_buf, max_cp_group, 0, 0); } log_hdr_log_block_size @@ -3869,7 +3869,7 @@ ask_again: /* Read the archive file header */ fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, 0, 0, - LOG_FILE_HDR_SIZE, buf, NULL, 0); + LOG_FILE_HDR_SIZE, buf, NULL, 0, 0); /* Check if the archive file header is consistent */ @@ -3943,7 +3943,7 @@ ask_again: fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, read_offset / UNIV_PAGE_SIZE, - read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0); + read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0, 0); ret = recv_scan_log_recs( (buf_pool_get_n_pages() diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 13bafeb048b..05ab1490b98 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -266,9 +266,11 @@ struct os_aio_slot_t{ byte* tmp_encryption_buf; /*!< a temporal buffer used by page encryption */ ibool page_compression_success; + /*!< TRUE if page compression was successfull, false if not */ ibool page_encryption_success; - /*!< TRUE if page compression was - successfull, false if not */ + /*!< TRUE if page encryption was successfull, false if not */ + + lsn_t lsn; /* lsn of the newest modification */ ulint file_block_size;/*!< file block size */ @@ -3148,29 +3150,6 @@ try_again: os_mutex_exit(os_file_count_mutex); if (ret && len == n) { - /* If page is encrypted we need to decrypt it first */ - if (fil_page_is_compressed_encrypted((byte *)buf) || - fil_page_is_encrypted((byte *)buf)) { - - byte *dst_frm = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); - // Decrypt the data - fil_space_decrypt((fil_space_crypt_t* ) NULL, - (byte *)buf, - n, - dst_frm); - // Copy decrypted buffer back to buf - memcpy(buf, dst_frm, n); - ut_free(dst_frm); - } - - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, len, NULL); - } - return(TRUE); } #else /* __WIN__ */ @@ -3183,29 +3162,6 @@ try_again: ret = os_file_pread(file, buf, n, offset, trx); if ((ulint) ret == n) { - /* If page is encrypted we need to decrypt it first */ - if (fil_page_is_compressed_encrypted((byte *)buf) || - fil_page_is_encrypted((byte *)buf)) { - - byte * dst_frm = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); - // Decrypt the data - fil_space_decrypt((fil_space_crypt_t*) NULL, - (byte *)buf, - n, - dst_frm); - // Copy decrypted buffer back to buf - memcpy(buf, dst_frm, n); - ut_free(dst_frm); - } - - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, n, NULL); - } - return(TRUE); } @@ -3294,28 +3250,6 @@ try_again: os_mutex_exit(os_file_count_mutex); if (ret && len == n) { - /* If page is encrypted we need to decrypt it first */ - if (fil_page_is_compressed_encrypted((byte *)buf) || - fil_page_is_encrypted((byte *)buf)) { - byte * dst_frm = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); - // Decrypt the data - fil_space_decrypt((fil_space_crypt_t* ) NULL, - (byte *)buf, - n, - dst_frm); - // Copy decrypted buffer back to buf - memcpy(buf, dst_frm, n); - ut_free(dst_frm); - } - - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, n, NULL); - } - return(TRUE); } #else /* __WIN__ */ @@ -3328,28 +3262,6 @@ try_again: ret = os_file_pread(file, buf, n, offset, NULL); if ((ulint) ret == n) { - /* If the page is encrypted we need to decrypt it first */ - if (fil_page_is_compressed_encrypted((byte *)buf) || - fil_page_is_encrypted((byte *)buf)) { - byte * dst_frm = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); - // Decrypt the data - fil_space_decrypt((fil_space_crypt_t* ) NULL, - (byte *)buf, - n, - dst_frm); - // Copy decrypted buffer back to buf - memcpy(buf, dst_frm, n); - ut_free(dst_frm); - } - - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, n, NULL); - } - return(TRUE); } #endif /* __WIN__ */ @@ -4746,11 +4658,12 @@ os_aio_array_reserve_slot( on this file space */ ulint page_encryption_key, /*!< page encryption key to be used */ - ulint* write_size)/*!< in/out: Actual write size initialized + ulint* write_size,/*!< in/out: Actual write size initialized after first successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ + lsn_t lsn) /* lsn of the newest modification */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4839,6 +4752,7 @@ found: slot->type = type; slot->buf = static_cast<byte*>(buf); slot->offset = offset; + slot->lsn = lsn; slot->io_already_done = FALSE; slot->space_id = space_id; @@ -4855,13 +4769,14 @@ found: slot->file_block_size = fil_node_get_block_size(message1); } + /* If the space is page compressed and this is write operation - then we compress the page */ - if (message1 && type == OS_FILE_WRITE && page_compression ) { + then we encrypt the page */ + if (message1 && type == OS_FILE_WRITE && page_compression) { ulint real_len = len; byte* tmp = NULL; - /* Release the array mutex while compressing */ + /* Release the array mutex while encrypting */ os_mutex_exit(array->mutex); // We allocate memory for page compressed buffer if and only @@ -4875,7 +4790,8 @@ found: #endif /* Call page compression */ - tmp = fil_compress_page(fil_node_get_space_id(slot->message1), + tmp = fil_compress_page( + fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf, len, @@ -4895,25 +4811,26 @@ found: slot->page_compression_success = FALSE; } - /* Take array mutex back */ + /* Take array mutex back, not sure if this is really needed + below */ os_mutex_enter(array->mutex); + } /* If the space is page encryption and this is write operation then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_encryption) { + if (message1 && type == OS_FILE_WRITE && page_encryption ) { /* Release the array mutex while encrypting */ os_mutex_exit(array->mutex); // We allocate memory for page encrypted buffer if and only // if it is not yet allocated. os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - /* ctr not yet supported in xtradb, lsn is null*/ + fil_space_encrypt( fil_node_get_space_id(slot->message1), slot->offset, - 0, /* QQ: Needs to be fixed to slot->lsn */ + slot->lsn, (byte *)buf, slot->len, slot->page_buf2, @@ -5213,9 +5130,9 @@ os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key) /*!< page encryption key + ulint page_encryption_key, /*!< page encryption key to be used */ - + lsn_t lsn) /* lsn of the newest modification */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5250,8 +5167,7 @@ os_aio_func( if (type == OS_FILE_READ) { ret = os_file_read_func(file, buf, offset, n, trx, page_compression); - } - else { + } else { ut_ad(!srv_read_only_mode); ut_a(type == OS_FILE_WRITE); @@ -5323,7 +5239,8 @@ try_again: slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, space_id, page_compression, page_compression_level, - page_encryption, page_encryption_key, write_size); + page_encryption, page_encryption_key, write_size, lsn); + if (type == OS_FILE_READ) { if (srv_use_native_aio) { os_n_file_reads++; @@ -5576,6 +5493,8 @@ os_aio_windows_handle( } if (fil_page_is_compressed(slot->buf)) { + /* We allocate memory for page compressed buffer if + and only if it is not yet allocated. */ os_slot_alloc_page_buf(slot); #ifdef HAVE_LZO @@ -5589,7 +5508,8 @@ os_aio_windows_handle( } else { /* OS_FILE_WRITE */ if (slot->page_compression_success && - fil_page_is_compressed(slot->page_buf)) { + (fil_page_is_compressed(slot->page_buf) || + fil_page_is_compressed_encrypted(slot->buf))) { if (srv_use_trim && os_fallocate_failed == FALSE) { // Deallocate unused blocks from file system os_file_trim(slot); @@ -5721,7 +5641,8 @@ retry: } else { /* OS_FILE_WRITE */ if (slot->page_compression_success && - fil_page_is_compressed(slot->page_buf)) { + (fil_page_is_compressed(slot->page_buf) || + fil_page_is_compressed_encrypted(slot->buf))) { ut_ad(slot->page_compression_page); if (srv_use_trim && os_fallocate_failed == FALSE) { // Deallocate unused blocks from file system |