diff options
64 files changed, 2450 insertions, 3957 deletions
diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 1f323b9d730..34bc4a199d5 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -52,10 +52,10 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ #include "ut0ut.h" #include "ut0byte.h" #include "mach0data.h" -#include "fsp0types.h" #include "rem0rec.h" #include "buf0checksum.h" /* buf_calc_page_*() */ #include "fil0fil.h" /* FIL_* */ +#include "fsp0types.h" #include "page0page.h" /* PAGE_* */ #include "page0zip.h" /* page_zip_*() */ #include "trx0undo.h" /* TRX_* */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 59cbe633dec..b2758e1253f 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -40,6 +40,7 @@ Created 11/5/1995 Heikki Tuuri #include "mem0mem.h" #include "btr0btr.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP #include "buf0buddy.h" #include "lock0lock.h" @@ -54,7 +55,6 @@ Created 11/5/1995 Heikki Tuuri #include "page0zip.h" #include "srv0mon.h" #include "buf0checksum.h" -#include "fil0pageencryption.h" #include "fil0pagecompress.h" #include "ut0byte.h" #include <new> @@ -504,7 +504,7 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { - ulint page_encrypted = fil_page_is_compressed_encrypted(read_buf) || fil_page_is_encrypted(read_buf); + ulint page_encrypted = fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; ibool crc32_inited = FALSE; @@ -5763,7 +5763,7 @@ buf_page_decrypt_after_read( unsigned key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); + bool page_compressed = fil_page_is_compressed(dst_frame); if (key_version == 0) { /* the page we read is unencrypted */ @@ -5801,7 +5801,7 @@ buf_page_decrypt_after_read( /* decompress from dst_frame to comp_buf and then copy to buffer pool */ - if (page_compressed_encrypted) { + if (page_compressed) { if (bpage->comp_buf_free == NULL) { bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); // TODO: is 4k aligment enough ? diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 477bcfe0ee3..53ef6ecb316 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -36,6 +36,8 @@ Created 2011/12/19 #include "srv0srv.h" #include "page0zip.h" #include "trx0sys.h" +#include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP @@ -385,7 +387,7 @@ buf_dblwr_init_or_load_pages( /* Read the trx sys header to check if we are using the doublewrite buffer */ off_t trx_sys_page = TRX_SYS_PAGE_NO * UNIV_PAGE_SIZE; - os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE, FALSE); + os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE); doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; @@ -430,9 +432,9 @@ buf_dblwr_init_or_load_pages( block_bytes = TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * UNIV_PAGE_SIZE; - os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes, FALSE); + os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes); os_file_read(file, buf + block_bytes, block2 * UNIV_PAGE_SIZE, - block_bytes, FALSE); + block_bytes); /* Check if any of these pages is half-written in data files, in the intended position */ @@ -530,9 +532,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, read_buf, NULL, - 0, - 0, - false); + 0); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -593,9 +593,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -620,9 +618,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); } } } @@ -644,9 +640,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL); ut_free(unaligned_buf); } @@ -860,9 +856,7 @@ buf_dblwr_write_block_to_datafile( buf_page_get_zip_size(bpage), frame, (void*) bpage, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); return; } @@ -881,9 +875,7 @@ buf_dblwr_write_block_to_datafile( UNIV_PAGE_SIZE, frame, (void*) block, - (ulint *)&bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + (ulint *)&bpage->write_size); } /********************************************************************//** @@ -977,7 +969,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block1, 0, len, - (void*) write_buf, NULL, 0, 0, false); + (void*) write_buf, NULL, 0); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -993,7 +985,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block2, 0, len, - (void*) write_buf, NULL, 0, 0, false); + (void*) write_buf, NULL, 0); flush: /* increment the doublewrite flushed pages counter */ @@ -1230,9 +1222,7 @@ retry: UNIV_PAGE_SIZE, (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ @@ -1244,9 +1234,7 @@ retry: UNIV_PAGE_SIZE, frame, NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 9bd7a7c007d..b8a34558867 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -921,9 +921,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer @@ -943,9 +941,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index e91a5da6621..5917af7ca9d 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -186,7 +186,7 @@ buf_read_page_low( *err = fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, zip_size, offset, 0, zip_size, - frame, bpage, &bpage->write_size, 0, false); + frame, bpage, &bpage->write_size); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); @@ -194,7 +194,7 @@ buf_read_page_low( | ignore_nonexistent_pages, sync, space, 0, offset, 0, UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, 0, false); + &bpage->write_size); } if (sync) { diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 9f8315a8016..aab0b1f5568 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -24,6 +24,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "fil0crypt.h" #include "srv0srv.h" #include "srv0start.h" #include "mach0data.h" @@ -35,14 +36,13 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "btr0scrub.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "ha_prototypes.h" // IB_LOG_ #include <my_crypt.h> #include <my_aes.h> #include <math.h> - +#include "fil0crypt.h" /** Mutex for keys */ UNIV_INTERN ib_mutex_t fil_crypt_key_mutex; @@ -119,67 +119,40 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; -/** - * CRYPT_SCHEME_UNENCRYPTED - * - * Used as intermediate state when convering a space from unencrypted - * to encrypted - */ -#define CRYPT_SCHEME_UNENCRYPTED 0 - -/** - * CRYPT_SCHEME_1 - * - * L = AES_ECB(KEY, IV) - * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) - */ -#define CRYPT_SCHEME_1 1 -#define CRYPT_SCHEME_1_IV_LEN 16 -// cached L given key_version -struct key_struct -{ - uint key_version; - byte key[CRYPT_SCHEME_1_IV_LEN]; -}; - -struct fil_space_rotate_state_t +/****************************************************************** +Map used AES method to crypt scheme +@return used AES crypt scheme */ +UNIV_INTERN +uint +fil_crypt_get_aes_method( +/*=====================*/ + uint aes_method) { - time_t start_time; // time when rotation started - ulint active_threads; // active threads in space - ulint next_offset; // next "free" offset - ulint max_offset; // max offset needing to be rotated - uint min_key_version_found; // min key version found but not rotated - lsn_t end_lsn; // max lsn created when rotating this space - bool starting; // initial write of IV - bool flushing; // space is being flushed at end of rotate - struct { - bool is_active; // is scrubbing active in this space - time_t last_scrub_completed; // when was last scrub completed - } scrubbing; -}; + switch (aes_method) { + case MY_AES_ALGORITHM_NONE: + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; + break; + case MY_AES_ALGORITHM_CTR: + return (uint) CRYPT_SCHEME_1_CTR; + break; + case MY_AES_ALGORITHM_CBC: + return (uint) CRYPT_SCHEME_1_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", aes_method); + ut_error; + } -struct fil_space_crypt_struct -{ - ulint type; // CRYPT_SCHEME - uint keyserver_requests; // no of key requests to key server - uint key_count; // No of initalized key-structs - key_struct keys[3]; // cached L = AES_ECB(KEY, IV) - uint min_key_version; // min key version for this space - ulint page0_offset; // byte offset on page 0 for crypt data - - ib_mutex_t mutex; // mutex protecting following variables - bool closing; // is tablespace being closed - fil_space_rotate_state_t rotate_state; - - uint iv_length; // length of IV - byte iv[1]; // IV-data -}; + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; +} /********************************************************************* Init space crypt */ UNIV_INTERN void fil_space_crypt_init() +/*==================*/ { mutex_create(fil_crypt_key_mutex_key, &fil_crypt_key_mutex, SYNC_NO_ORDER_CHECK); @@ -196,6 +169,7 @@ Cleanup space crypt */ UNIV_INTERN void fil_space_crypt_cleanup() +/*=====================*/ { os_event_free(fil_crypt_throttle_sleep_event); } @@ -204,31 +178,35 @@ fil_space_crypt_cleanup() Get key bytes for a space/key-version */ static void -fil_crypt_get_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint version, bool page_encrypted) +fil_crypt_get_key( +/*==============*/ + byte* dst, /*<! out: Key */ + uint* key_length, /*<! out: Key length */ + fil_space_crypt_t* crypt_data, /*<! in: crypt data */ + uint version) /*<! in: Key version */ { unsigned char keybuf[MY_AES_MAX_KEY_LENGTH]; mutex_enter(&crypt_data->mutex); - if (!page_encrypted) { - // Check if we already have key - for (uint i = 0; i < crypt_data->key_count; i++) { - if (crypt_data->keys[i].key_version == version) { - memcpy(dst, crypt_data->keys[i].key, - sizeof(crypt_data->keys[i].key)); - mutex_exit(&crypt_data->mutex); - return; - } + // Check if we already have key + for (uint i = 0; i < crypt_data->key_count; i++) { + if (crypt_data->keys[i].key_version == version) { + memcpy(dst, crypt_data->keys[i].key, + crypt_data->keys[i].key_length); + *key_length = crypt_data->keys[i].key_length; + mutex_exit(&crypt_data->mutex); + return; } - // Not found! - crypt_data->keyserver_requests++; + } - // Rotate keys to make room for a new - for (uint i = 1; i < array_elements(crypt_data->keys); i++) { - crypt_data->keys[i] = crypt_data->keys[i - 1]; - } - } + // Not found! + crypt_data->keyserver_requests++; + + // Rotate keys to make room for a new + for (uint i = 1; i < array_elements(crypt_data->keys); i++) { + crypt_data->keys[i] = crypt_data->keys[i - 1]; + } if (has_encryption_key(version)) { int rc; @@ -247,51 +225,45 @@ fil_crypt_get_key(byte *dst, uint* key_length, ut_error; } + /* Now compute L by encrypting IV using this key. Note + that we use random IV from crypt data. */ + const unsigned char* src = crypt_data->iv; + const int srclen = crypt_data->iv_length; + unsigned char* buf = crypt_data->keys[0].key; + uint32 buflen = CRYPT_SCHEME_1_IV_LEN; - // do ctr key initialization - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) - { - /* Now compute L by encrypting IV using this key. Note - that we use random IV from crypt data. */ - const unsigned char* src = crypt_data->iv; - const int srclen = crypt_data->iv_length; - unsigned char* buf = page_encrypted ? keybuf : crypt_data->keys[0].key; - uint32 buflen = page_encrypted ? *key_length : sizeof(crypt_data->keys[0].key); - - // call ecb explicit - my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - int rc = (*func)(src, srclen, - buf, &buflen, - (unsigned char*)keybuf, *key_length, - NULL, 0, - 1); - - if (rc != AES_OK) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt key-block " - " src: %p srclen: %d buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - src, srclen, buf, buflen, rc); - ut_error; - } + /* We use AES_ECB to encryp IV */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - if (!page_encrypted) { - crypt_data->keys[0].key_version = version; - crypt_data->key_count++; + int rc = (*func)(src, /* Data to be encrypted = IV */ + srclen, /* data length */ + buf, /* Output buffer */ + &buflen, /* Output buffer */ + keybuf, /* Key */ + *key_length, /* Key length */ + NULL, /* AES_ECB does not use IV */ + 0, /* IV-length */ + 1); /* NoPadding */ - if (crypt_data->key_count > array_elements(crypt_data->keys)) { - crypt_data->key_count = array_elements(crypt_data->keys); - } - } + if (rc != AES_OK) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to encrypt key-block " + " src: %p srclen: %d buf: %p buflen: %d." + " return-code: %d. Can't continue!\n", + src, srclen, buf, buflen, rc); + ut_error; + } - // set the key size to the aes block size because this encrypted data is the key - *key_length = MY_AES_BLOCK_SIZE; - memcpy(dst, buf, buflen); - } else { - // otherwise keybuf contains the right key - memcpy(dst, keybuf, *key_length); + crypt_data->keys[0].key_version = version; + crypt_data->key_count++; + *key_length = buflen; + crypt_data->keys[0].key_length = buflen; + + if (crypt_data->key_count > array_elements(crypt_data->keys)) { + crypt_data->key_count = array_elements(crypt_data->keys); } + memcpy(dst, buf, buflen); mutex_exit(&crypt_data->mutex); } @@ -299,8 +271,12 @@ fil_crypt_get_key(byte *dst, uint* key_length, Get key bytes for a space/latest(key-version) */ static inline void -fil_crypt_get_latest_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint *version) +fil_crypt_get_latest_key( +/*=====================*/ + byte* dst, /*!< out: Key */ + uint* key_length, /*!< out: Key length */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + uint* version) /*!< in: Key version */ { if (srv_encrypt_tables) { // used for key rotation - get the next key id from the key provider @@ -312,29 +288,33 @@ fil_crypt_get_latest_key(byte *dst, uint* key_length, } } - return fil_crypt_get_key(dst, key_length, crypt_data, *version, srv_encrypt_tables == FALSE); + return fil_crypt_get_key(dst, key_length, crypt_data, *version); } /****************************************************************** -Create a fil_space_crypt_t object */ +Create a fil_space_crypt_t object +@return crypt object */ UNIV_INTERN fil_space_crypt_t* fil_space_create_crypt_data() +/*=========================*/ { const uint iv_length = CRYPT_SCHEME_1_IV_LEN; const uint sz = sizeof(fil_space_crypt_t) + iv_length; fil_space_crypt_t* crypt_data = static_cast<fil_space_crypt_t*>(malloc(sz)); + uint aes_method = current_aes_dynamic_method; + memset(crypt_data, 0, sz); if (srv_encrypt_tables == FALSE) { - crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; crypt_data->min_key_version = 0; } else { - crypt_data->type = CRYPT_SCHEME_1; crypt_data->min_key_version = get_latest_encryption_key_version(); } + /* Set up the current AES method */ + crypt_data->type = (fil_crypt_method_t)fil_crypt_get_aes_method(aes_method); mutex_create(fil_crypt_data_mutex_key, &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; @@ -346,13 +326,18 @@ fil_space_create_crypt_data() Compare two crypt objects */ UNIV_INTERN int -fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, - const fil_space_crypt_t* crypt_data2) +fil_space_crypt_compare( +/*====================*/ + const fil_space_crypt_t* crypt_data1,/*!< in: Crypt data */ + const fil_space_crypt_t* crypt_data2)/*!< in: Crypt data */ { - ut_a(crypt_data1->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data1->type == CRYPT_SCHEME_1); - ut_a(crypt_data2->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data2->type == CRYPT_SCHEME_1); + ut_a(crypt_data1->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data1->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); + + ut_a(crypt_data2->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data2->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); ut_a(crypt_data1->iv_length == CRYPT_SCHEME_1_IV_LEN); ut_a(crypt_data2->iv_length == CRYPT_SCHEME_1_IV_LEN); @@ -365,10 +350,15 @@ fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, } /****************************************************************** -Read crypt data from a page (0) */ +Read crypt data from a page (0) +@return crypt data from page 0. */ UNIV_INTERN fil_space_crypt_t* -fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) +fil_space_read_crypt_data( +/*======================*/ + ulint space, /*!< in: file space id*/ + const byte* page, /*!< in: page 0 */ + ulint offset) /*!< in: offset */ { if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { /* crypt is not stored */ @@ -393,8 +383,10 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); - if (! (type == CRYPT_SCHEME_UNENCRYPTED || - type == CRYPT_SCHEME_1)) { + if (! (type == CRYPT_SCHEME_1_UNENCRYPTED || + type == CRYPT_SCHEME_1_CTR || + type == CRYPT_SCHEME_1_CBC)) { + ib_logf(IB_LOG_LEVEL_ERROR, "Found non sensible crypt scheme: %lu for space %lu " " offset: %lu bytes: " @@ -449,7 +441,9 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) Free a crypt data object */ UNIV_INTERN void -fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) +fil_space_destroy_crypt_data( +/*=========================*/ + fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { /* lock (and unlock) mutex to make sure no one has it locked @@ -466,10 +460,14 @@ fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) Write crypt data to a page (0) */ static void -fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, - ulint type, - byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data_low( +/*===========================*/ + fil_space_crypt_t* crypt_data, /*<! out: crypt data */ + ulint type, /*<! in: crypt scheme */ + byte* page, /*<! in: page 0 */ + ulint offset, /*<! in: offset */ + ulint maxsize, /*<! in: size of crypt data */ + mtr_t* mtr) /*<! in: minitransaction */ { ut_a(offset > 0 && offset < UNIV_PAGE_SIZE); ulint space_id = mach_read_from_4( @@ -492,10 +490,8 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len, min_key_version, MLOG_4BYTES, mtr); - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_1", - ut_error;); - byte* log_ptr = mlog_open(mtr, 11 + 12 + len); + if (log_ptr != NULL) { log_ptr = mlog_write_initial_log_record_fast( page, @@ -515,25 +511,26 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_catenate_string(mtr, crypt_data->iv, len); } - - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_2", - ut_error;); } /****************************************************************** Write crypt data to a page (0) */ UNIV_INTERN void -fil_space_write_crypt_data(ulint space, byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data( +/*=======================*/ + ulint space, /*<! in: file space */ + byte* page, /*<! in: page 0 */ + ulint offset, /*<! in: offset */ + ulint maxsize, /*<! in: size of crypt data */ + mtr_t* mtr) /*<! in: minitransaction */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); - ibool page_encrypted = fil_space_is_page_encrypted(space); /* If no crypt data is stored on memory cache for this space or space is not encrypted and encryption is not enabled, then do not continue writing crypt data to page 0. */ - if (crypt_data == NULL || (!page_encrypted && !srv_encrypt_tables)) { + if (crypt_data == NULL && !srv_encrypt_tables) { return; } @@ -542,11 +539,15 @@ fil_space_write_crypt_data(ulint space, byte* page, ulint offset, } /****************************************************************** -Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry */ +Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry +@return position on log buffer */ UNIV_INTERN byte* -fil_parse_write_crypt_data(byte* ptr, byte* end_ptr, - buf_block_t* block) +fil_parse_write_crypt_data( +/*=======================*/ + byte* ptr, /*!< in: Log entry start */ + byte* end_ptr,/*!< in: Log entry end */ + buf_block_t* block) /*!< in: buffer block */ { /* check that redo log entry is complete */ uint entry_size = @@ -569,8 +570,10 @@ fil_parse_write_crypt_data(byte* ptr, byte* end_ptr, uint len = mach_read_from_1(ptr); ptr += 1; - ut_a(type == CRYPT_SCHEME_UNENCRYPTED || - type == CRYPT_SCHEME_1); // only supported + ut_a(type == CRYPT_SCHEME_1_UNENCRYPTED || + type == CRYPT_SCHEME_1_CTR || + type == CRYPT_SCHEME_1_CBC); // supported methods + ut_a(len == CRYPT_SCHEME_1_IV_LEN); // only supported uint min_key_version = mach_read_from_4(ptr); ptr += 4; @@ -595,7 +598,10 @@ fil_parse_write_crypt_data(byte* ptr, byte* end_ptr, Clear crypt data from a page (0) */ UNIV_INTERN void -fil_space_clear_crypt_data(byte* page, ulint offset) +fil_space_clear_crypt_data( +/*=======================*/ + byte* page, /*!< in/out: Page 0 */ + ulint offset) /*!< in: Offset */ { //TODO(jonaso): pass crypt-data and read len from there ulint len = CRYPT_SCHEME_1_IV_LEN; @@ -609,7 +615,8 @@ fil_space_clear_crypt_data(byte* page, ulint offset) } /********************************************************************* -Check if page shall be encrypted before write */ +Check if page shall be encrypted before write +@return true if page should be encrypted, false if not */ UNIV_INTERN bool fil_space_check_encryption_write( @@ -633,26 +640,58 @@ fil_space_check_encryption_write( } /****************************************************************** +Map current aes method +@return AES method */ +UNIV_INTERN +uint +fil_crypt_map_aes_method( +/*=====================*/ + uint aes_method) /*!< in: AES method */ +{ + switch((fil_crypt_method_t)aes_method) { + case CRYPT_SCHEME_1_CTR: + return (uint)MY_AES_ALGORITHM_CTR; + break; + case CRYPT_SCHEME_1_CBC: + return (uint)MY_AES_ALGORITHM_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", (int)aes_method); + ut_error; + } +} + +/****************************************************************** Encrypt a page */ UNIV_INTERN void -fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, - const byte* src_frame, ulint zip_size, byte* dst_frame, ulint encryption_key) +fil_space_encrypt( +/*==============*/ + ulint space, /*!< in: Space id */ + ulint offset, /*!< in: Page offset */ + lsn_t lsn, /*!< in: lsn */ + const byte* src_frame, /*!< in: Source page to be encrypted */ + ulint zip_size, /*!< in: compressed size if + row_format compressed */ + byte* dst_frame, /*!< in: outbut buffer */ + ulint encryption_key) /*!< in: encryption key id if page + encrypted */ { fil_space_crypt_t* crypt_data=NULL; ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - // get key (L) - uint key_version = encryption_key; - byte key[MY_AES_MAX_KEY_LENGTH]; + uint key_version = (uint)encryption_key; + unsigned char key[MY_AES_MAX_KEY_LENGTH]; uint key_length=MY_AES_MAX_KEY_LENGTH; + uint aes_method; + unsigned char iv[MY_AES_BLOCK_SIZE]; ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR - || orig_page_type==FIL_PAGE_TYPE_XDES - || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED - || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + || orig_page_type==FIL_PAGE_TYPE_XDES) { + /* File space header or extent descriptor do not need to be + encrypted. */ //TODO: is this really needed ? memcpy(dst_frame, src_frame, page_size); return; @@ -667,63 +706,56 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, return; } + /* Load the IV or counter (depending to the encryption algorithm used) */ fil_crypt_get_latest_key(key, &key_length, crypt_data, &key_version); + aes_method = fil_crypt_get_aes_method(current_aes_dynamic_method); - /* Load the iv or counter (depending to the encryption algorithm used) */ - unsigned char iv[MY_AES_BLOCK_SIZE]; - - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) { - // create counter block (C) - mach_write_to_4(iv + 0, space); - ulint space_offset = mach_read_from_4( - src_frame + FIL_PAGE_OFFSET); - mach_write_to_4(iv + 4, space_offset); - mach_write_to_8(iv + 8, lsn); - } else { - // Get random IV from crypt_data - mutex_enter(&crypt_data->mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); - } + /* For AES CTR create counter and AES CBS IV */ + mach_write_to_4(iv + 0, space); + ulint space_offset = mach_read_from_4(src_frame + FIL_PAGE_OFFSET); + mach_write_to_4(iv + 4, space_offset); + mach_write_to_8(iv + 8, lsn); ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); - ibool page_encrypted = fil_space_is_page_encrypted(space); - ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint compression_algo = page_compressed ? mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) : 0; - // copy page header + /* FIL page header is not encrypted */ memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - if (page_encrypted && !page_compressed) { - // key id - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - // original page type - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2, - orig_page_type); - // new page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - } else { - // store key version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - } + /* Store key version */ + mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, key_version); - // encrypt page data + /* Calculate the start offset in a page */ ulint unencrypted_bytes = FIL_PAGE_DATA + FIL_PAGE_DATA_END; ulint srclen = page_size - unencrypted_bytes; const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; + uint32 dstlen=0; + /* For page compressed tables we encrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be encrypted. */ if (page_compressed) { - srclen = page_size - FIL_PAGE_DATA; - } - - int rc = (* my_aes_encrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); + ulint payload = mach_read_from_2(src_frame + FIL_PAGE_DATA); + mach_write_to_2(dst_frame + FIL_PAGE_DATA, payload); + srclen = payload; + src+=2; + dst+=2; + } + + /* Get encryption method */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Encrypt the data */ + int rc = (*func)(src, /* Original page */ + srclen, /* Page length */ + dst, /* Output buffer */ + &dstlen, /* Output length */ + key, /* Encryption key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV length */ + 1); /* Use noPadding */ if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -735,65 +767,35 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, ut_error; } + /* Set up the checksum */ + mach_write_to_4(dst_frame+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); + + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ if (!page_compressed) { - // copy page trailer + /* FIL page trailer is also not encrypted */ memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - - /* handle post encryption checksum */ - ib_uint32_t checksum = 0; - srv_checksum_algorithm_t algorithm = - static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); - - if (zip_size == 0) { - switch (algorithm) { - case SRV_CHECKSUM_ALGORITHM_CRC32: - case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - checksum = buf_calc_page_crc32(dst_frame); - break; - case SRV_CHECKSUM_ALGORITHM_INNODB: - case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: - checksum = (ib_uint32_t) buf_calc_page_new_checksum( - dst_frame); - break; - case SRV_CHECKSUM_ALGORITHM_NONE: - case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: - checksum = BUF_NO_CHECKSUM_MAGIC; - break; - /* no default so the compiler will emit a warning - * if new enum is added and not handled here */ - } - } else { - checksum = page_zip_calc_checksum(dst_frame, zip_size, - algorithm); - } - - // store the post-encryption checksum after the key-version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, - checksum); - } else { - /* Page compressed and encrypted tables have different - FIL_HEADER */ - ulint page_len = log10((double)page_size)/log10((double)2); - /* Set up the correct page type */ - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - /* Set up the compression algorithm */ - mach_write_to_2(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4, orig_page_type); - /* Set up the compressed size */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6, page_len); - /* Set up the compression method */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7, compression_alg); } + /* Store AES encryption method */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, + aes_method); + /* Store compression algorithm (for page compresed tables) or 0 */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6, + compression_algo); } /********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ +Check if extra buffer shall be allocated for decrypting after read +@return true if fil space has encryption data. */ UNIV_INTERN bool fil_space_check_encryption_read( -/*==============================*/ +/*=============================*/ ulint space) /*!< in: tablespace id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -810,101 +812,92 @@ fil_space_check_encryption_read( } /****************************************************************** -Decrypt a page */ +Decrypt a page +@return true if page was encrypted */ UNIV_INTERN bool -fil_space_decrypt(fil_space_crypt_t* crypt_data, - const byte* src_frame, ulint page_size, byte* dst_frame) +fil_space_decrypt( +/*==============*/ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ { ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - // key version - uint key_version; - bool page_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_ENCRYPTED); - - bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED); - - ulint orig_page_type=0; - - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - key_version = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - orig_page_type = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2); - } else { - key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } + uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); + ulint compression_algo = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6); + uint aes_method = 0; - if (key_version == 0 && !page_encrypted) { + /* Page is not encrypted if key_version is 0 */ + if (key_version == 0) { //TODO: is this really needed ? memcpy(dst_frame, src_frame, page_size); return false; /* page not decrypted */ } - // read space & offset & lsn + /* read space & offset & lsn */ ulint space = mach_read_from_4( src_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4( src_frame + FIL_PAGE_OFFSET); ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); - // copy page header + /* Copy FIL page header, it is not encrypted */ memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - // orig page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); - } - - // get key + /* Get key */ byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; - fil_crypt_get_key(key, &key_length, crypt_data, key_version, page_encrypted); - - // get the iv unsigned char iv[MY_AES_BLOCK_SIZE]; + fil_crypt_get_key(key, &key_length, crypt_data, key_version); + aes_method = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) { - // create counter block - - mach_write_to_4(iv + 0, space); - mach_write_to_4(iv + 4, offset); - mach_write_to_8(iv + 8, lsn); - } else { - // Get random IV from crypt_data - mutex_enter(&crypt_data->mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); + /* Verify used AES method */ + if ( aes_method != CRYPT_SCHEME_1_CTR && + aes_method != CRYPT_SCHEME_1_CBC) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to decrypt data-block " + " aes_method: %d unknown!\n", + aes_method); + ut_error; } + /* Create counter used as IV */ + mach_write_to_4(iv + 0, space); + mach_write_to_4(iv + 4, offset); + mach_write_to_8(iv + 8, lsn); + + /* Calculate the offset where decryption starts */ const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; + uint32 dstlen=0; ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; - ulint compression_method; + /* For page compressed tables we decrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be decrypted. */ if (page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4); - compressed_len = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6); - compression_method = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7); - } - - if (page_encrypted && !page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+2); - } - - if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - srclen = pow((double)2, (double)((int)compressed_len)) - FIL_PAGE_DATA; - } - - int rc = (* my_aes_decrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); + ulint compressed_len = mach_read_from_2(src_frame + FIL_PAGE_DATA); + src+=2; + dst+=2; + mach_write_to_2(dst_frame + FIL_PAGE_DATA, compressed_len); + srclen = compressed_len; + } + + /* Get AES method */ + my_aes_encrypt_dynamic_type func = get_aes_decrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Decrypt the data */ + int rc = (*func)(src, /* Data to be encrypted */ + srclen, /* data length */ + dst, /* Output buffer */ + &dstlen, /* Output buffer */ + key, /* Key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV-length */ + 1); /* NoPadding */ if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -916,22 +909,57 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, ut_error; } - if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - // copy page trailer + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ + if (!page_compressed) { + /* Copy FIL trailer */ memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - // clear key-version & crypt-checksum from dst - memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); - } else { - /* For page compressed tables we set up the FIL_HEADER again */ - /* setting original page type */ - mach_write_to_2(dst_frame + FIL_PAGE_TYPE, orig_page_type); - /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ - mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); - /* Set up the flush lsn to be compression algorithm */ - mach_write_to_8(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, compression_method); + /* handle post decryption checksum */ + ib_uint32_t checksum = 0; + srv_checksum_algorithm_t algorithm = + static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); + + if (page_size == UNIV_PAGE_SIZE) { + switch (algorithm) { + case SRV_CHECKSUM_ALGORITHM_CRC32: + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + checksum = buf_calc_page_crc32(dst_frame); + break; + case SRV_CHECKSUM_ALGORITHM_INNODB: + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + checksum = (ib_uint32_t) buf_calc_page_new_checksum( + dst_frame); + break; + case SRV_CHECKSUM_ALGORITHM_NONE: + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + checksum = BUF_NO_CHECKSUM_MAGIC; + break; + /* no default so the compiler will emit a warning + * if new enum is added and not handled here */ + } + } else { + checksum = page_zip_calc_checksum(dst_frame, page_size, + algorithm); + } + + mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + fprintf(stderr, "KUKKU %ld:%lu\n", mach_read_from_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM), + mach_read_from_4(dst_frame + FIL_PAGE_END_LSN_OLD_CHKSUM)); + } + + /* Clear key-version & aes_method from dst */ + memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); + + /* For page compressed tables store compression algorithm back */ + if (page_compressed) { + mach_write_to_8(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + compression_algo); } return true; /* page was decrypted */ @@ -941,8 +969,12 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, Decrypt a page */ UNIV_INTERN void -fil_space_decrypt(ulint space, - const byte* src_frame, ulint page_size, byte* dst_frame) +fil_space_decrypt( +/*==============*/ + ulint space, /*!< in: Fil space id */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ { fil_space_decrypt(fil_space_get_crypt_data(space), src_frame, page_size, dst_frame); @@ -953,8 +985,13 @@ Verify checksum for a page (iff it's encrypted) NOTE: currently this function can only be run in single threaded mode as it modifies srv_checksum_algorithm (temporarily) @return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN bool -fil_space_verify_crypt_checksum(const byte* src_frame, ulint zip_size) +fil_space_verify_crypt_checksum( +/*============================*/ + const byte* src_frame, /*!< in: page the verify */ + ulint zip_size) /*!< in: compressed size if + row_format compressed */ { // key version uint key_version = mach_read_from_4( @@ -1041,7 +1078,8 @@ struct key_state_t { Copy global key state */ static void fil_crypt_get_key_state( - key_state_t *new_state) +/*====================*/ + key_state_t *new_state) /*!< out: key state */ { if (srv_encrypt_tables == TRUE) { new_state->key_version = get_latest_encryption_key_version(); @@ -1054,9 +1092,13 @@ fil_crypt_get_key_state( } /*********************************************************************** -Check if a key needs rotation given a key_state */ +Check if a key needs rotation given a key_state +@return true if key needs rotation, false if not */ static bool -fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) +fil_crypt_needs_rotation( +/*=====================*/ + uint key_version, /*!< in: Key version */ + const key_state_t* key_state) /*!< in: Key state */ { // TODO(jonaso): Add support for rotating encrypted => unencrypted @@ -1081,9 +1123,13 @@ fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) } /*********************************************************************** -Check if a space is closing (i.e just before drop) */ -UNIV_INTERN bool -fil_crypt_is_closing(ulint space) +Check if a space is closing (i.e just before drop) +@return true if space is closing, false if not. */ +UNIV_INTERN +bool +fil_crypt_is_closing( +/*=================*/ + ulint space) /*!< in: FIL space id */ { bool closing; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1097,16 +1143,22 @@ fil_crypt_is_closing(ulint space) Start encrypting a space @return true if a pending op (fil_inc_pending_ops/fil_decr_pending_ops) is held */ -static bool -fil_crypt_start_encrypting_space(ulint space, bool *recheck) { +static +bool +fil_crypt_start_encrypting_space( +/*=============================*/ + ulint space, /*!< in: FIL space id */ + bool* recheck)/*!< out: true if recheck needed */ +{ /* we have a pending op when entering function */ bool pending_op = true; + uint aes_method = current_aes_dynamic_method; mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); - ibool page_encrypted = fil_space_is_page_encrypted(space); + ibool page_encrypted = (crypt_data != NULL); /*If spage is not encrypted and encryption is not enabled, then do not continue encrypting the space. */ @@ -1184,7 +1236,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 4 - write crypt data to page 0 */ fil_space_write_crypt_data_low(crypt_data, - CRYPT_SCHEME_1, + fil_crypt_get_aes_method(aes_method), frame, crypt_data->page0_offset, maxsize, &mtr); @@ -1232,7 +1284,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 5 - publish crypt data */ mutex_enter(&fil_crypt_threads_mutex); mutex_enter(&crypt_data->mutex); - crypt_data->type = CRYPT_SCHEME_1; + crypt_data->type = fil_crypt_get_aes_method(aes_method); ut_a(crypt_data->rotate_state.active_threads == 1); crypt_data->rotate_state.active_threads = 0; crypt_data->rotate_state.starting = false; @@ -1257,10 +1309,14 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { } /*********************************************************************** -Check if space needs rotation given a key_state */ -static bool -fil_crypt_space_needs_rotation(uint space, const key_state_t *key_state, - bool *recheck) +Check if space needs rotation given a key_state +@return true if space needs key rotation */ +static +bool +fil_crypt_space_needs_rotation( + uint space, /*!< in: FIL space id */ + const key_state_t* key_state, /*!< in: Key state */ + bool* recheck) /*!< out: needs recheck ? */ { if (fil_space_get_type(space) != FIL_TABLESPACE) { return false; @@ -1374,7 +1430,9 @@ struct rotate_thread_t { /*********************************************************************** Update global statistics with thread statistics */ static void -fil_crypt_update_total_stat(rotate_thread_t *state) +fil_crypt_update_total_stat( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { mutex_enter(&crypt_stat_mutex); crypt_stat.pages_read_from_cache += @@ -1397,9 +1455,13 @@ fil_crypt_update_total_stat(rotate_thread_t *state) /*********************************************************************** Allocate iops to thread from global setting, -used before starting to rotate a space */ -static bool -fil_crypt_alloc_iops(rotate_thread_t *state) +used before starting to rotate a space. +@return true if allocation succeeded, false if failed */ +static +bool +fil_crypt_alloc_iops( +/*=================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_ad(state->allocated_iops == 0); @@ -1429,8 +1491,11 @@ fil_crypt_alloc_iops(rotate_thread_t *state) /*********************************************************************** Reallocate iops to thread, used when inside a space */ -static void -fil_crypt_realloc_iops(rotate_thread_t *state) +static +void +fil_crypt_realloc_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_a(state->allocated_iops > 0); @@ -1519,8 +1584,11 @@ fil_crypt_realloc_iops(rotate_thread_t *state) /*********************************************************************** Return allocated iops to global */ -static void -fil_crypt_return_iops(rotate_thread_t *state) +static +void +fil_crypt_return_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { if (state->allocated_iops > 0) { uint iops = state->allocated_iops; @@ -1544,11 +1612,14 @@ fil_crypt_return_iops(rotate_thread_t *state) /*********************************************************************** Search for a space needing rotation */ +UNIV_INTERN bool fil_crypt_find_space_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state, - bool *recheck) +/*===========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state, /*!< in: Key rotation state */ + bool* recheck) /*!< out: true if recheck + needed */ { /* we need iops to start rotating */ while (!state->should_shutdown() && !fil_crypt_alloc_iops(state)) { @@ -1591,8 +1662,9 @@ Start rotating a space */ static void fil_crypt_start_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*=========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1629,12 +1701,14 @@ fil_crypt_start_rotate_space( } /*********************************************************************** -Search for batch of pages needing rotation */ +Search for batch of pages needing rotation +@return true if page needing key rotation found, false if not found */ static bool fil_crypt_find_page_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state) +/*==========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint batch = srv_alloc_time * state->allocated_iops; ulint space = state->space; @@ -1665,9 +1739,15 @@ fil_crypt_find_page_to_rotate( } /*********************************************************************** -Check if a page is uninitialized (doesn't need to be rotated) */ -static bool -fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) +Check if a page is uninitialized (doesn't need to be rotated) +@return true if page is uninitialized, false if not.*/ +static +bool +fil_crypt_is_page_uninitialized( +/*============================*/ + const byte *frame, /*!< in: Page */ + uint zip_size) /*!< in: compressed size if + row_format compressed */ { if (zip_size) { ulint stored_checksum = mach_read_from_4( @@ -1696,15 +1776,20 @@ fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) sleeptime_ms, __FILE__, __LINE__) /*********************************************************************** -Get a page and compute sleep time */ +Get a page and compute sleep time +@return page */ static buf_block_t* -fil_crypt_get_page_throttle_func(rotate_thread_t *state, - ulint space, uint zip_size, ulint offset, - mtr_t *mtr, - ulint *sleeptime_ms, - const char *file, - ulint line) +fil_crypt_get_page_throttle_func( + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction */ + ulint* sleeptime_ms, /*!< out: sleep time */ + const char* file, /*!< in: file name */ + ulint line) /*!< in: file line */ { buf_block_t* block = buf_page_try_get_func(space, offset, RW_X_LATCH, true, @@ -1753,17 +1838,22 @@ Get block and allocation status note: innodb locks fil_space_latch and then block when allocating page but locks block and then fil_space_latch when freeing page. +@return block */ static buf_block_t* btr_scrub_get_block_and_allocation_status( - rotate_thread_t *state, - ulint space, - ulint zip_size, - ulint offset, - mtr_t *mtr, +/*======================================*/ + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction + */ btr_scrub_page_allocation_status_t *allocation_status, - ulint *sleeptime_ms) + /*!< in/out: allocation status */ + ulint* sleeptime_ms) /*!< out: sleep time */ { mtr_t local_mtr; buf_block_t *block = NULL; @@ -1807,8 +1897,9 @@ Rotate one page */ static void fil_crypt_rotate_page( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint offset = state->offset; @@ -1960,8 +2051,9 @@ Rotate a batch of pages */ static void fil_crypt_rotate_pages( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint end = state->offset + state->batch; @@ -1989,7 +2081,10 @@ fil_crypt_rotate_pages( Flush rotated pages and then update page 0 */ static void -fil_crypt_flush_space(rotate_thread_t *state, ulint space) +fil_crypt_flush_space( +/*==================*/ + rotate_thread_t* state, /*!< in: Key rotation state */ + ulint space) /*!< in: FIL space id */ { fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2033,9 +2128,10 @@ fil_crypt_flush_space(rotate_thread_t *state, ulint space) RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); byte* frame = buf_block_get_frame(block); - ulint maxsize = 0; + ulint maxsize; crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + fil_space_write_crypt_data(space, frame, crypt_data->page0_offset, ULINT_MAX, &mtr); @@ -2048,8 +2144,9 @@ Complete rotating a space */ static void fil_crypt_complete_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*============================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2123,7 +2220,7 @@ A thread which monitors global key state and rotates tablespaces accordingly extern "C" UNIV_INTERN os_thread_ret_t DECLARE_THREAD(fil_crypt_thread)( -/*===============================*/ +/*=============================*/ void* arg __attribute__((unused))) /*!< in: a dummy parameter required * by os_thread_create */ { @@ -2240,13 +2337,19 @@ DECLARE_THREAD(fil_crypt_thread)( Adjust thread count for key rotation */ UNIV_INTERN void -fil_crypt_set_thread_cnt(uint new_cnt) +fil_crypt_set_thread_cnt( +/*=====================*/ + uint new_cnt) /*!< in: New key rotation thread count */ { if (new_cnt > srv_n_fil_crypt_threads) { uint add = new_cnt - srv_n_fil_crypt_threads; srv_n_fil_crypt_threads = new_cnt; for (uint i = 0; i < add; i++) { - os_thread_create(fil_crypt_thread, NULL, NULL); + os_thread_id_t rotation_thread_id; + os_thread_create(fil_crypt_thread, NULL, &rotation_thread_id); + ib_logf(IB_LOG_LEVEL_INFO, + "Creating #%d thread id %lu total threads %du\n", + i, os_thread_pf(rotation_thread_id), new_cnt); } } else if (new_cnt < srv_n_fil_crypt_threads) { srv_n_fil_crypt_threads = new_cnt; @@ -2263,7 +2366,9 @@ fil_crypt_set_thread_cnt(uint new_cnt) Adjust max key age */ UNIV_INTERN void -fil_crypt_set_rotate_key_age(uint val) +fil_crypt_set_rotate_key_age( +/*=========================*/ + uint val) /*!< in: New max key age */ { srv_fil_crypt_rotate_key_age = val; os_event_set(fil_crypt_threads_event); @@ -2273,7 +2378,9 @@ fil_crypt_set_rotate_key_age(uint val) Adjust rotation iops */ UNIV_INTERN void -fil_crypt_set_rotation_iops(uint val) +fil_crypt_set_rotation_iops( +/*========================*/ + uint val) /*!< in: New iops setting */ { srv_n_fil_crypt_iops = val; os_event_set(fil_crypt_threads_event); @@ -2284,6 +2391,7 @@ Init threads for key rotation */ UNIV_INTERN void fil_crypt_threads_init() +/*====================*/ { fil_crypt_event = os_event_create(); fil_crypt_threads_event = os_event_create(); @@ -2300,6 +2408,7 @@ End threads for key rotation */ UNIV_INTERN void fil_crypt_threads_end() +/*===================*/ { /* stop threads */ fil_crypt_set_thread_cnt(0); @@ -2309,7 +2418,9 @@ fil_crypt_threads_end() Clean up key rotation threads resources */ UNIV_INTERN void -fil_crypt_threads_cleanup() { +fil_crypt_threads_cleanup() +/*=======================*/ +{ os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); } @@ -2319,7 +2430,8 @@ Mark a space as closing */ UNIV_INTERN void fil_space_crypt_mark_space_closing( - ulint space) +/*===============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2340,7 +2452,8 @@ Wait for crypt threads to stop accessing space */ UNIV_INTERN void fil_space_crypt_close_tablespace( - ulint space) +/*=============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2352,11 +2465,14 @@ fil_space_crypt_close_tablespace( uint start = time(0); uint last = start; + mutex_enter(&crypt_data->mutex); mutex_exit(&fil_crypt_threads_mutex); crypt_data->closing = true; + uint cnt = crypt_data->rotate_state.active_threads; bool flushing = crypt_data->rotate_state.flushing; + while (cnt > 0 || flushing) { mutex_exit(&crypt_data->mutex); /* release dict mutex so that scrub threads can release their @@ -2386,11 +2502,12 @@ fil_space_crypt_close_tablespace( /********************************************************************* Get crypt status for a space (used by information_schema) return 0 if crypt data present */ +UNIV_INTERN int fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_crypt_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); @@ -2400,6 +2517,7 @@ fil_space_crypt_get_status( mutex_enter(&crypt_data->mutex); status->keyserver_requests = crypt_data->keyserver_requests; status->min_key_version = crypt_data->min_key_version; + if (crypt_data->rotate_state.active_threads > 0 || crypt_data->rotate_state.flushing) { status->rotating = true; @@ -2427,8 +2545,11 @@ fil_space_crypt_get_status( /********************************************************************* Return crypt statistics */ +UNIV_INTERN void -fil_crypt_total_stat(fil_crypt_stat_t *stat) +fil_crypt_total_stat( +/*=================*/ + fil_crypt_stat_t *stat) /*!< out: Crypt statistics */ { mutex_enter(&crypt_stat_mutex); *stat = crypt_stat; @@ -2438,11 +2559,12 @@ fil_crypt_total_stat(fil_crypt_stat_t *stat) /********************************************************************* Get scrub status for a space (used by information_schema) return 0 if data found */ +UNIV_INTERN int fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_scrub_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); memset(status, 0, sizeof(*status)); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 8b0a788b7b9..9d37de6d63b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -27,8 +27,7 @@ Created 10/25/1995 Heikki Tuuri #include "fil0fil.h" #include "fil0pagecompress.h" #include "fsp0pagecompress.h" -#include "fil0pageencryption.h" -#include "fsp0pageencryption.h" +#include "fil0crypt.h" #include <debug_sync.h> #include <my_dbug.h> @@ -284,7 +283,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0, false)); + byte_offset, len, buf, message, write_size)); } /********************************************************************//** @@ -311,18 +310,16 @@ fil_write( this must be appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn, encrypt_later)); + byte_offset, len, buf, message, write_size)); } /*******************************************************************//** @@ -648,10 +645,9 @@ fil_node_open_file( set */ page = static_cast<byte*>(ut_align(buf2, UNIV_PAGE_SIZE)); - success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE, - space->flags); + success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); - if (fil_page_encryption_status(page)) { + if (fil_page_is_encrypted(page)) { /* if page is (still) encrypted, write an error and return. * Otherwise the server would crash if decrypting is not possible. * This may be the case, if the key file could not be @@ -1156,21 +1152,6 @@ fil_space_create( ut_a(fil_system); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* by returning here it should be avoided that - * the server crashes, if someone tries to access an - * encrypted table and the encryption key is not available. - * The the table is treaded as non-existent. - */ - ib_logf(IB_LOG_LEVEL_WARN, - "Tablespace '%s' can not be opened, because " - " encryption key can not be found (space id: %lu, key %lu)\n" - , name, (ulong) id, fsp_flags_get_page_encryption_key(flags)); - return (FALSE); - } - } - /* Look for a matching tablespace and if found free it. */ do { mutex_enter(&fil_system->mutex); @@ -1836,7 +1817,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, lsn, false); + UNIV_PAGE_SIZE, buf, NULL, 0); } mem_free(buf1); @@ -1927,13 +1908,10 @@ fil_check_first_page( or the encryption key is not available, the check for reading the first page should intentionally fail with "can not decrypt" message. */ - page_is_encrypted = fil_page_encryption_status(page); - if (page_is_encrypted == PAGE_ENCRYPTION_KEY_MISSING && page_is_encrypted) { - page_is_encrypted = 1; - } else { - page_is_encrypted = 0; + page_is_encrypted = fil_page_encryption_status(page, space_id); + if (!page_is_encrypted) { if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { - fprintf(stderr, + fprintf(stderr, "InnoDB: Error: Current page size %lu != " " page size on page %lu\n", UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); @@ -1963,7 +1941,7 @@ fil_check_first_page( /* this error message is interpreted by the calling method, which is * executed if the server starts in recovery mode. */ - return(MSG_CANNOT_DECRYPT); + return(FIL_MSG_CANNOT_DECRYPT); } } @@ -2016,10 +1994,7 @@ fil_read_first_page( page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); - os_file_read(data_file, page, 0, UNIV_PAGE_SIZE, - orig_space_id != ULINT_UNDEFINED ? - fil_space_is_page_compressed(orig_space_id) : - FALSE); + os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave @@ -4082,8 +4057,7 @@ fil_user_tablespace_find_space_id( for (ulint j = 0; j < page_count; ++j) { - st = os_file_read(fsp->file, page, (j* page_size), page_size, - fsp_flags_is_page_compressed(fsp->flags)); + st = os_file_read(fsp->file, page, (j* page_size), page_size); if (!st) { ib_logf(IB_LOG_LEVEL_INFO, @@ -4229,7 +4203,7 @@ check_first_page: "%s in tablespace %s (table %s)", check_msg, fsp->filepath, tablename); fsp->success = FALSE; - if (strncmp(check_msg, MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { + if (strncmp(check_msg, FIL_MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { /* by returning here, it should be avoided, that the server crashes, * if started in recovery mode and can not decrypt tables, if * the key file can not be read. @@ -5221,7 +5195,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, 0, FALSE, 0, 0, 0, 0, false); + node, NULL, 0); #endif /* UNIV_HOTBACKUP */ @@ -5602,13 +5576,11 @@ fil_io( appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ { ulint mode; fil_space_t* space; @@ -5618,10 +5590,6 @@ fil_io( ulint wake_later; os_offset_t offset; ibool ignore_nonexistent_pages; - ibool page_compressed = FALSE; - ulint page_compression_level = 0; - ibool page_encrypted; - ulint page_encryption_key; is_log = type & OS_FILE_LOG; type = type & ~OS_FILE_LOG; @@ -5805,11 +5773,6 @@ fil_io( ut_a(byte_offset % OS_FILE_LOG_BLOCK_SIZE == 0); ut_a((len % OS_FILE_LOG_BLOCK_SIZE) == 0); - page_compressed = fsp_flags_is_page_compressed(space->flags); - page_compression_level = fsp_flags_get_page_compression_level(space->flags); - page_encrypted = fsp_flags_is_page_encrypted(space->flags); - page_encryption_key = fsp_flags_get_page_encryption_key(space->flags); - #ifdef UNIV_HOTBACKUP /* In mysqlbackup do normal i/o, not aio */ if (type == OS_FILE_READ) { @@ -5831,13 +5794,7 @@ fil_io( len, node, message, - write_size, - page_compressed, - page_compression_level, - page_encrypted, - page_encryption_key, - lsn, - encrypt_later); + write_size); #endif /* UNIV_HOTBACKUP */ @@ -6387,9 +6344,7 @@ fil_iterate( readptr = iter.crypt_io_buffer; } - if (!os_file_read(iter.file, readptr, offset, - (ulint) n_bytes, - fil_space_is_page_compressed(space_id))) { + if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); @@ -6540,8 +6495,7 @@ fil_tablespace_iterate( /* Read the first page and determine the page and zip size. */ - if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE, - dict_tf_get_page_compression(table->flags))) { + if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE)) { err = DB_IO_ERROR; @@ -6601,7 +6555,7 @@ fil_tablespace_iterate( if (iter.crypt_data != NULL) { /* clear crypt data from page 0 and write it back */ - os_file_read(file, page, 0, UNIV_PAGE_SIZE, 0); + os_file_read(file, page, 0, UNIV_PAGE_SIZE); fil_space_clear_crypt_data(page, crypt_data_offset); lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN); if (callback.get_zip_size() == 0) { @@ -6779,79 +6733,6 @@ fil_system_exit(void) mutex_exit(&fil_system->mutex); } -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->name); -} - -/*******************************************************************//** -Return space flags */ -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->flags); -} - -/*******************************************************************//** -Return page type name */ -const char* -fil_get_page_type_name( -/*===================*/ - ulint page_type) /*!< in: FIL_PAGE_TYPE */ -{ - switch(page_type) { - case FIL_PAGE_PAGE_COMPRESSED: - return (const char*)"PAGE_COMPRESSED"; - case FIL_PAGE_INDEX: - return (const char*)"INDEX"; - case FIL_PAGE_UNDO_LOG: - return (const char*)"UNDO LOG"; - case FIL_PAGE_INODE: - return (const char*)"INODE"; - case FIL_PAGE_IBUF_FREE_LIST: - return (const char*)"IBUF_FREE_LIST"; - case FIL_PAGE_TYPE_ALLOCATED: - return (const char*)"ALLOCATED"; - case FIL_PAGE_IBUF_BITMAP: - return (const char*)"IBUF_BITMAP"; - case FIL_PAGE_TYPE_SYS: - return (const char*)"SYS"; - case FIL_PAGE_TYPE_TRX_SYS: - return (const char*)"TRX_SYS"; - case FIL_PAGE_TYPE_FSP_HDR: - return (const char*)"FSP_HDR"; - case FIL_PAGE_TYPE_XDES: - return (const char*)"XDES"; - case FIL_PAGE_TYPE_BLOB: - return (const char*)"BLOB"; - case FIL_PAGE_TYPE_ZBLOB: - return (const char*)"ZBLOB"; - case FIL_PAGE_TYPE_ZBLOB2: - return (const char*)"ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return (const char*)"ORACLE PAGE COMPRESSED"; - default: - return (const char*)"PAGE TYPE CORRUPTED"; - } -} -/****************************************************************//** -Get block size from fil node -@return block size*/ -ulint -fil_node_get_block_size( -/*====================*/ - fil_node_t* node) /*!< in: Node where to get block - size */ -{ - return (node->file_block_size); -} /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index caed038b4ee..1c3db26372d 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -84,172 +84,6 @@ static ulint srv_data_read, srv_data_written; //#define UNIV_PAGECOMPRESS_DEBUG 1 /****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -static -void -fil_decompress_page_2( -/*==================*/ - byte* page_buf, /*!< out: destination buffer for - uncompressed data */ - byte* buf, /*!< in: source compressed data */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size) /*!< in/out: Actual payload size of - the compressed data. */ -{ - ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - if (page_type != FIL_PAGE_TYPE_COMPRESSED) { - /* It is not a compressed page */ - return; - } - - byte* ptr = buf + FIL_PAGE_DATA; - ulint version = mach_read_from_1(buf + FIL_PAGE_VERSION); - int err = 0; - - ut_a(version == 1); - - /* Read the original page type, before we compressed the data. */ - page_type = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_TYPE_V1); - - ulint original_len = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_SIZE_V1); - - if (original_len < UNIV_PAGE_SIZE_MIN - (FIL_PAGE_DATA + 8) - || original_len > UNIV_PAGE_SIZE_MAX - FIL_PAGE_DATA - || len < original_len + FIL_PAGE_DATA) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page. " - "Original len %lu len %lu.", - original_len, len); - - fflush(stderr); - ut_error; - - } - - ulint algorithm = mach_read_from_1(buf + FIL_PAGE_ALGORITHM_V1); - - switch(algorithm) { - case PAGE_ZLIB_ALGORITHM: { - - err = uncompress(page_buf, &len, ptr, original_len); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed " - "but uncompress failed with error %d " - " size %lu len %lu.", - err, original_len, len); - - fflush(stderr); - - ut_error; - } - - break; - } -#ifdef HAVE_LZ4 - case PAGE_LZ4_ALGORITHM: { - - err = LZ4_decompress_fast( - (const char*) ptr, (char*) (page_buf), original_len); - - if (err < 0) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %d bytes" - " size %lu len %lu.", - err, original_len, len); - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZ4 */ - -#ifdef HAVE_LZMA - case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; - size_t src_pos = 0; - size_t dst_pos = 0; - uint64_t memlimit = UINT64_MAX; - - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - ptr, - &src_pos, - original_len, - (page_buf), - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos <= 0 || dst_pos > len)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - dst_pos, original_len, len); - fflush(stderr); - - ut_error; - } - - break; - } -#endif /* HAVE_LZMA */ - -#ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - - err = lzo1x_decompress((const unsigned char *)ptr, - original_len,(unsigned char *)(page_buf), &olen, NULL); - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - olen, original_len, len); - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZO */ - - default: - ib_logf(IB_LOG_LEVEL_ERROR, - " Corruption: Page is marked as compressed " - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(algorithm)); - - fflush(stderr); - ut_error; - break; - } - - /* Leave the header alone */ - memmove(buf+FIL_PAGE_DATA, page_buf, original_len); - - mach_write_to_2(buf + FIL_PAGE_TYPE, page_type); - - ut_ad(memcmp(buf + FIL_PAGE_LSN + 4, - buf + (original_len + FIL_PAGE_DATA) - - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4) == 0); -} - -/****************************************************************//** For page compressed pages compress the page before actual write operation. @return compressed page to be written*/ @@ -289,8 +123,7 @@ fil_compress_page( if (orig_page_type == 0 || orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { *out_len = len; return (buf); } @@ -579,16 +412,6 @@ fil_decompress_page( in_buf = page_buf; } - if (ptype == FIL_PAGE_TYPE_COMPRESSED) { - - fil_decompress_page_2(in_buf, buf, len, write_size); - // Need to free temporal buffer if no buffer was given - if (page_buf == NULL) { - ut_free(in_buf); - } - return; - } - /* Before actual decompress, make sure that page type is correct */ if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index ee1f2fd9510..1cf30a56a98 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -31,6 +31,7 @@ Created 11/29/1995 Heikki Tuuri #include "buf0buf.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "mtr0log.h" #include "ut0byte.h" #include "page0page.h" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3a5f961a9bb..311fdee4d3b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -79,6 +79,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fsp0fsp.h" #include "sync0sync.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "trx0xa.h" #include "row0merge.h" #include "dict0boot.h" @@ -103,7 +104,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fts0priv.h" #include "page0zip.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) @@ -11047,8 +11047,6 @@ innobase_table_flags( modified by another thread while the table is being created. */ const ulint default_compression_level = page_zip_level; - const ulint default_encryption_key = srv_default_page_encryption_key; - *flags = 0; *flags2 = 0; @@ -11250,10 +11248,7 @@ index_bad: options->page_compressed, options->page_compression_level == 0 ? default_compression_level : options->page_compression_level, - options->atomic_writes, - options->page_encryption, - options->page_encryption_key == 0 ? - default_encryption_key : options->page_encryption_key); + options->atomic_writes); if (create_info->options & HA_LEX_CREATE_TMP_TABLE) { *flags2 |= DICT_TF2_TEMPORARY; @@ -19156,7 +19151,7 @@ static MYSQL_SYSVAR_UINT(default_page_encryption_key, srv_default_page_encryptio "Encryption key used for page encryption.", NULL, NULL, - DEFAULT_ENCRYPTION_KEY, 1, 255, 0); + FIL_DEFAULT_ENCRYPTION_KEY, 1, 255, 0); static MYSQL_SYSVAR_BOOL(scrub_log, srv_scrub_log, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 65fcb7051ef..705886736ae 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -61,6 +61,8 @@ Modified Dec 29, 2014 Jan Lindström (Added sys_semaphore_waits) #include "btr0btr.h" #include "page0zip.h" #include "sync0arr.h" +#include "fil0fil.h" +#include "fil0crypt.h" /** structure associates a name string with a file page type and/or buffer page state. */ diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 326b9e7c986..20014e05784 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -929,10 +929,8 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic + ulint atomic_writes) /*!< in: table atomic writes option value*/ - bool page_encrypted,/*!< in: table uses page encryption */ - ulint page_encryption_key) /*!< in: page encryption key */ __attribute__((nonnull)); /********************************************************************//** Convert a 32 bit integer table flags to the 32 bit integer that is diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 7c51faf844e..dd42b478c1f 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -543,9 +543,6 @@ dict_tf_is_valid( ulint data_dir = DICT_TF_HAS_DATA_DIR(flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(flags); - /* Make sure there are no bits that we do not know about. */ if (unused != 0) { fprintf(stderr, @@ -555,12 +552,10 @@ dict_tf_is_valid( "InnoDB: compact %ld atomic_blobs %ld\n" "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n" - "InnoDB: page_encryption %ld page_encryption_key %ld\n", + "InnoDB: atomic_writes %ld\n", unused, compact, atomic_blobs, unused, data_dir, zip_ssize, - page_compression, page_compression_level, atomic_writes, - page_encryption, page_encryption_key + page_compression, page_compression_level, atomic_writes ); return(false); @@ -857,9 +852,7 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic writes setup */ - bool page_encrypted, /*!< in: table uses page encryption */ - ulint page_encryption_key /*!< in: page encryption key */) + ulint atomic_writes) /*!< in: table atomic writes setup */ { atomic_writes_t awrites = (atomic_writes_t)atomic_writes; @@ -900,11 +893,6 @@ dict_tf_set( if (use_data_dir) { *flags |= (1 << DICT_TF_POS_DATA_DIR); } - - if (page_encrypted) { - *flags |= (1 << DICT_TF_POS_PAGE_ENCRYPTION) - | (page_encryption_key << DICT_TF_POS_PAGE_ENCRYPTION_KEY); - } } /********************************************************************//** @@ -927,10 +915,6 @@ dict_tf_to_fsp_flags( ulint fsp_flags; ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", @@ -958,14 +942,6 @@ dict_tf_to_fsp_flags( if page compression is used for this table. */ fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); - /* In addition, tablespace flags also contain if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION(fsp_flags, page_encryption); - - /* In addition, tablespace flags also contain page encryption key if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(fsp_flags, page_encryption_key); - /* In addition, tablespace flags also contain flag if atomic writes is used for this table */ fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); @@ -1007,8 +983,6 @@ dict_sys_tables_type_to_tf( | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY ); @@ -1045,9 +1019,7 @@ dict_tf_to_sys_tables_type( | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY); + | DICT_TF_MASK_ATOMIC_WRITES); return(type); } diff --git a/storage/innobase/include/dict0pagecompress.ic b/storage/innobase/include/dict0pagecompress.ic index a71b2b34b07..811976434a8 100644 --- a/storage/innobase/include/dict0pagecompress.ic +++ b/storage/innobase/include/dict0pagecompress.ic @@ -42,8 +42,6 @@ dict_tf_verify_flags( ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); @@ -52,9 +50,6 @@ dict_tf_verify_flags( ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - ulint fsp_page_encryption = FSP_FLAGS_GET_PAGE_ENCRYPTION(fsp_flags); - ulint fsp_page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(fsp_flags); - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", return(ULINT_UNDEFINED);); @@ -112,27 +107,6 @@ dict_tf_verify_flags( return (FALSE); } - if (page_encryption != fsp_page_encryption) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption %ld\n", - page_encryption, fsp_page_encryption); - - return (FALSE); - } - - if (page_encryption_key != fsp_page_encryption_key) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption_key %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption_key %ld\n", - page_encryption_key, fsp_page_encryption_key); - - return (FALSE); - } - - return(TRUE); } diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h new file mode 100644 index 00000000000..9d02034e4b2 --- /dev/null +++ b/storage/innobase/include/fil0crypt.h @@ -0,0 +1,394 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0crypt.h +The low-level file system encryption support functions + +Created 04/01/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0crypt_h +#define fil0crypt_h + +#define FIL_MSG_CANNOT_DECRYPT "can not decrypt" +#define FIL_ENCRYPTION_WRONG_KEY 1 +#define FIL_ENCRYPTION_WRONG_PAGE_TYPE 2 +#define FIL_ENCRYPTION_ERROR 3 +#define FIL_ENCRYPTION_KEY_MISSING 4 +#define FIL_ENCRYPTION_OK 0 +#define FIL_ENCRYPTION_WILL_NOT_ENCRYPT 5 + +/* This key will be used if nothing else is given */ +#define FIL_DEFAULT_ENCRYPTION_KEY 1 + +/** + * CRYPT_SCHEME_UNENCRYPTED + * + * Used as intermediate state when convering a space from unencrypted + * to encrypted + */ +/** + * CRYPT_SCHEME_1 + * + * AES_CTR / AES_CBC: + * L = AES_ECB(KEY, IV) + * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) + */ + +#define CRYPT_SCHEME_1 1 +#define CRYPT_SCHEME_1_IV_LEN 16 +#define CRYPT_SCHEME_UNENCRYPTED 0 + +/* Currently supported encryption methods */ +typedef enum { + CRYPT_SCHEME_1_UNENCRYPTED = 0, /*<! Space is unencrypted */ + CRYPT_SCHEME_1_CTR = 1, /*<! Space is encrypted with CTR */ + CRYPT_SCHEME_1_CBC = 4, /*<! Space is encrypted with CBC */ +} fil_crypt_method_t; + +/* Cached L or key for given key_version */ +struct key_struct +{ + uint key_version; /*!< Key version used as + identifier */ + byte key[MY_AES_MAX_KEY_LENGTH]; /*!< Cached L or key */ + uint key_length; /*!< Key length */ +}; + +struct fil_space_rotate_state_t +{ + time_t start_time; // time when rotation started + ulint active_threads; // active threads in space + ulint next_offset; // next "free" offset + ulint max_offset; // max offset needing to be rotated + uint min_key_version_found; // min key version found but not rotated + lsn_t end_lsn; // max lsn created when rotating this space + bool starting; // initial write of IV + bool flushing; // space is being flushed at end of rotate + struct { + bool is_active; // is scrubbing active in this space + time_t last_scrub_completed; // when was last scrub completed + } scrubbing; +}; + +struct fil_space_crypt_struct +{ + ulint type; // CRYPT_SCHEME + uint keyserver_requests; // no of key requests to key server + uint key_count; // No of initalized key-structs + key_struct keys[3]; // cached L = AES_ECB(KEY, IV) + uint min_key_version; // min key version for this space + ulint page0_offset; // byte offset on page 0 for crypt data + + ib_mutex_t mutex; // mutex protecting following variables + bool closing; // is tablespace being closed + fil_space_rotate_state_t rotate_state; + + uint iv_length; // length of IV + byte iv[1]; // IV-data +}; + +/* structure containing encryption specification */ +typedef struct fil_space_crypt_struct fil_space_crypt_t; + +/********************************************************************* +Init global resources needed for tablespace encryption/decryption */ +UNIV_INTERN +void +fil_space_crypt_init(); + +/********************************************************************* +Cleanup global resources needed for tablespace encryption/decryption */ +UNIV_INTERN +void +fil_space_crypt_cleanup(); + +/********************************************************************* +Create crypt data, i.e data that is used for a single tablespace */ +UNIV_INTERN +fil_space_crypt_t * +fil_space_create_crypt_data(); + +/********************************************************************* +Destroy crypt data */ +UNIV_INTERN +void +fil_space_destroy_crypt_data( +/*=========================*/ + fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */ + +/********************************************************************* +Get crypt data for a space*/ +UNIV_INTERN +fil_space_crypt_t * +fil_space_get_crypt_data( +/*======================*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Set crypt data for a space*/ +UNIV_INTERN +void +fil_space_set_crypt_data( +/*======================*/ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + +/********************************************************************* +Compare crypt data*/ +UNIV_INTERN +int +fil_space_crypt_compare( +/*======================*/ + const fil_space_crypt_t* crypt_data1, /*!< in: crypt data */ + const fil_space_crypt_t* crypt_data2); /*!< in: crypt data */ + +/********************************************************************* +Read crypt data from buffer page */ +UNIV_INTERN +fil_space_crypt_t * +fil_space_read_crypt_data( +/*======================*/ + ulint space, /*!< in: tablespace id */ + const byte* page, /*!< in: buffer page */ + ulint offset); /*!< in: offset where crypt data is stored */ + +/********************************************************************* +Write crypt data to buffer page */ +UNIV_INTERN +void +fil_space_write_crypt_data( +/*=======================*/ + ulint space, /*!< in: tablespace id */ + byte* page, /*!< in: buffer page */ + ulint offset, /*!< in: offset where to store data */ + ulint maxsize, /*!< in: max space available to store crypt data in */ + mtr_t * mtr); /*!< in: mini-transaction */ + +/********************************************************************* +Clear crypt data from page 0 (used for import tablespace) */ +UNIV_INTERN +void +fil_space_clear_crypt_data( +/*======================*/ + byte* page, /*!< in: buffer page */ + ulint offset); /*!< in: offset where crypt data is stored */ + +/********************************************************************* +Parse crypt data log record */ +UNIV_INTERN +byte* +fil_parse_write_crypt_data( +/*=======================*/ + byte* ptr, /*!< in: start of log record */ + byte* end_ptr, /*!< in: end of log record */ + buf_block_t*); /*!< in: buffer page to apply record to */ + +/********************************************************************* +Check if extra buffer shall be allocated for decrypting after read */ +UNIV_INTERN +bool +fil_space_check_encryption_read( +/*==============================*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Check if page shall be encrypted before write */ +UNIV_INTERN +bool +fil_space_check_encryption_write( +/*==============================*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Encrypt buffer page */ +UNIV_INTERN +void +fil_space_encrypt( +/*===============*/ + ulint space, /*!< in: tablespace id */ + ulint offset, /*!< in: page no */ + lsn_t lsn, /*!< in: page lsn */ + const byte* src_frame,/*!< in: page frame */ + ulint size, /*!< in: size of data to encrypt */ + byte* dst_frame, /*!< in: where to encrypt to */ + ulint page_encryption_key); /*!< in: page encryption key id if page + encrypted */ + +/********************************************************************* +Decrypt buffer page */ +UNIV_INTERN +void +fil_space_decrypt( +/*===============*/ + ulint space, /*!< in: tablespace id */ + const byte* src_frame,/*!< in: page frame */ + ulint page_size, /*!< in: size of data to encrypt */ + byte* dst_frame); /*!< in: where to decrypt to */ + + +/********************************************************************* +Decrypt buffer page +@return true if page was encrypted */ +UNIV_INTERN +bool +fil_space_decrypt( +/*===============*/ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + const byte* src_frame,/*!< in: page frame */ + ulint page_size, /*!< in: page size */ + byte* dst_frame); /*!< in: where to decrypt to */ + +/********************************************************************* +fil_space_verify_crypt_checksum +NOTE: currently this function can only be run in single threaded mode +as it modifies srv_checksum_algorithm (temporarily) +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( +/*===============*/ + const byte* src_frame,/*!< in: page frame */ + ulint zip_size); /*!< in: size of data to encrypt */ + +/********************************************************************* +Init threads for key rotation */ +UNIV_INTERN +void +fil_crypt_threads_init(); + +/********************************************************************* +Set thread count (e.g start or stops threads) used for key rotation */ +UNIV_INTERN +void +fil_crypt_set_thread_cnt( +/*=====================*/ + uint new_cnt); /*!< in: requested #threads */ + +/********************************************************************* +End threads for key rotation */ +UNIV_INTERN +void +fil_crypt_threads_end(); + +/********************************************************************* +Cleanup resources for threads for key rotation */ +UNIV_INTERN +void +fil_crypt_threads_cleanup(); + +/********************************************************************* +Set rotate key age */ +UNIV_INTERN +void +fil_crypt_set_rotate_key_age( +/*=====================*/ + uint rotate_age); /*!< in: requested rotate age */ + +/********************************************************************* +Set rotation threads iops */ +UNIV_INTERN +void +fil_crypt_set_rotation_iops( +/*=====================*/ + uint iops); /*!< in: requested iops */ + +/********************************************************************* +Mark a space as closing */ +UNIV_INTERN +void +fil_space_crypt_mark_space_closing( +/*===============*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Wait for crypt threads to stop accessing space */ +UNIV_INTERN +void +fil_space_crypt_close_tablespace( +/*===============*/ + ulint space); /*!< in: tablespace id */ + +/** Struct for retreiving info about encryption */ +struct fil_space_crypt_status_t { + ulint space; /*!< tablespace id */ + ulint scheme; /*!< encryption scheme */ + uint min_key_version; /*!< min key version */ + uint current_key_version;/*!< current key version */ + uint keyserver_requests;/*!< no of key requests to key server */ + bool rotating; /*!< is key rotation ongoing */ + bool flushing; /*!< is flush at end of rotation ongoing */ + ulint rotate_next_page_number; /*!< next page if key rotating */ + ulint rotate_max_page_number; /*!< max page if key rotating */ +}; + +/********************************************************************* +Get crypt status for a space +@return 0 if crypt data found */ +UNIV_INTERN +int +fil_space_crypt_get_status( +/*==================*/ + ulint id, /*!< in: space id */ + struct fil_space_crypt_status_t * status); /*!< out: status */ + +/** Struct for retreiving statistics about encryption key rotation */ +struct fil_crypt_stat_t { + ulint pages_read_from_cache; + ulint pages_read_from_disk; + ulint pages_modified; + ulint pages_flushed; + ulint estimated_iops; +}; + +/********************************************************************* +Get crypt rotation statistics */ +UNIV_INTERN +void +fil_crypt_total_stat( +/*==================*/ + fil_crypt_stat_t* stat); /*!< out: crypt stat */ + +/** Struct for retreiving info about scrubbing */ +struct fil_space_scrub_status_t { + ulint space; /*!< tablespace id */ + bool compressed; /*!< is space compressed */ + time_t last_scrub_completed; /*!< when was last scrub completed */ + bool scrubbing; /*!< is scrubbing ongoing */ + time_t current_scrub_started; /*!< when started current scrubbing */ + ulint current_scrub_active_threads; /*!< current scrub active threads */ + ulint current_scrub_page_number; /*!< current scrub page no */ + ulint current_scrub_max_page_number; /*!< current scrub max page no */ +}; + +/********************************************************************* +Get scrub status for a space +@return 0 if no scrub info found */ +UNIV_INTERN +int +fil_space_get_scrub_status( +/*==================*/ + ulint id, /*!< in: space id */ + struct fil_space_scrub_status_t * status); /*!< out: status */ + +#ifndef UNIV_NONINL +#include "fil0crypt.ic" +#endif + +#endif /* fil0crypt_h */ diff --git a/storage/innobase/include/fil0crypt.ic b/storage/innobase/include/fil0crypt.ic new file mode 100644 index 00000000000..b753ca5cb4f --- /dev/null +++ b/storage/innobase/include/fil0crypt.ic @@ -0,0 +1,69 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0fil.h +The low-level file system encryption support functions + +Created 04/01/2015 Jan Lindström +*******************************************************/ + +/*******************************************************************//** +Find out whether the page is page encrypted +@return true if page is page encrypted, false if not */ +UNIV_INLINE +bool +fil_page_is_encrypted( +/*==================*/ + const byte *buf) /*!< in: page */ +{ + return(mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0); +} + +/*******************************************************************//** +Find out whether the page can be decrypted. +The function for decrypting the page should already be executed before this. +@return PAGE_ENCRYPTION_KEY_MISSING if key provider is available, but key is not available + 0 if decryption should be possible +*/ +UNIV_INLINE +ulint +fil_page_encryption_status( +/*===================*/ + const byte *buf, /*!< in: page */ + ulint space_id) /*!< in: space_id */ +{ + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + + if (page_type == FIL_PAGE_TYPE_FSP_HDR) { + fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space_id); + + if (crypt_data != NULL) { + if (!has_encryption_key(crypt_data->keys[0].key_version)) { + /* accessing table would surely fail, because no key or no key provider available */ + return FIL_ENCRYPTION_KEY_MISSING; + } + } + } else { + ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + if (!has_encryption_key(key)) { + return FIL_ENCRYPTION_KEY_MISSING; + } + } + return 0; +} diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index e4f16b24392..2c6c5804f65 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -26,7 +26,6 @@ Created 10/25/1995 Heikki Tuuri #ifndef fil0fil_h #define fil0fil_h -#define MSG_CANNOT_DECRYPT "can not decrypt" #include "univ.i" #ifndef UNIV_INNOCHECKSUM @@ -136,24 +135,6 @@ extern fil_addr_t fil_addr_null; used to encrypt the page + 32-bit checksum or 64 bits of zero if no encryption */ -/** If page type is FIL_PAGE_COMPRESSED then the 8 bytes starting at -FIL_PAGE_FILE_FLUSH_LSN are broken down as follows: */ - -/** Control information version format (u8) */ -static const ulint FIL_PAGE_VERSION = FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - -/** Compression algorithm (u8) */ -static const ulint FIL_PAGE_ALGORITHM_V1 = FIL_PAGE_VERSION + 1; - -/** Original page type (u16) */ -static const ulint FIL_PAGE_ORIGINAL_TYPE_V1 = FIL_PAGE_ALGORITHM_V1 + 1; - -/** Original data size in bytes (u16)*/ -static const ulint FIL_PAGE_ORIGINAL_SIZE_V1 = FIL_PAGE_ORIGINAL_TYPE_V1 + 2; - -/** Size after compression (u16)*/ -static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; - #define FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID 34 /*!< starting from 4.1.x this contains the space id of the page */ #define FIL_PAGE_SPACE_ID FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID @@ -173,10 +154,7 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; /* @} */ /** File page types (values of FIL_PAGE_TYPE) @{ */ -#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 35631 /* page compressed + - encrypted page */ #define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< page compressed page */ -#define FIL_PAGE_PAGE_ENCRYPTED 34355 /*!< Page encrypted page */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ #define FIL_PAGE_INODE 3 /*!< Index node */ @@ -203,6 +181,9 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; #ifndef UNIV_INNOCHECKSUM +/* structure containing encryption specification */ +typedef struct fil_space_crypt_struct fil_space_crypt_t; + /** The number of fsyncs done to the log */ extern ulint fil_n_log_flushes; @@ -214,9 +195,6 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; -/* structure containing encryption specification */ -typedef struct fil_space_crypt_struct fil_space_crypt_t; - struct fsp_open_info { ibool success; /*!< Has the tablespace been opened? */ const char* check_msg; /*!< fil_check_first_page() message */ @@ -997,13 +975,11 @@ fil_io( appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt the page */ __attribute__((nonnull(8))); /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the @@ -1256,6 +1232,7 @@ fil_user_tablespace_restore_page( /*******************************************************************//** Return space flags */ +UNIV_INLINE ulint fil_space_flags( /*===========*/ @@ -1281,12 +1258,6 @@ fil_space_t* fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space); /*!< in: space */ /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ @@ -1301,270 +1272,13 @@ ulint fil_get_next_space( ulint id); /*!< in: space id */ -/********************************************************************* -Init global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_init(); - -/********************************************************************* -Cleanup global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_cleanup(); - -/********************************************************************* -Create crypt data, i.e data that is used for a single tablespace */ -fil_space_crypt_t * -fil_space_create_crypt_data(); - -/********************************************************************* -Destroy crypt data */ -UNIV_INTERN -void -fil_space_destroy_crypt_data( -/*=========================*/ - fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */ - -/********************************************************************* -Get crypt data for a space*/ -fil_space_crypt_t * -fil_space_get_crypt_data( -/*======================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Set crypt data for a space*/ -void -fil_space_set_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ - -/********************************************************************* -Compare crypt data*/ -int -fil_space_crypt_compare( -/*======================*/ - const fil_space_crypt_t* crypt_data1, /*!< in: crypt data */ - const fil_space_crypt_t* crypt_data2); /*!< in: crypt data */ - -/********************************************************************* -Read crypt data from buffer page */ -fil_space_crypt_t * -fil_space_read_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - const byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Write crypt data to buffer page */ -void -fil_space_write_crypt_data( -/*=======================*/ - ulint space, /*!< in: tablespace id */ - byte* page, /*!< in: buffer page */ - ulint offset, /*!< in: offset where to store data */ - ulint maxsize, /*!< in: max space available to store crypt data in */ - mtr_t * mtr); /*!< in: mini-transaction */ - -/********************************************************************* -Clear crypt data from page 0 (used for import tablespace) */ -void -fil_space_clear_crypt_data( -/*======================*/ - byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Parse crypt data log record */ -byte* -fil_parse_write_crypt_data( -/*=======================*/ - byte* ptr, /*!< in: start of log record */ - byte* end_ptr, /*!< in: end of log record */ - buf_block_t*); /*!< in: buffer page to apply record to */ - -/********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ -UNIV_INTERN -bool -fil_space_check_encryption_read( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Check if page shall be encrypted before write */ -UNIV_INTERN -bool -fil_space_check_encryption_write( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Encrypt buffer page */ -void -fil_space_encrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - ulint offset, /*!< in: page no */ - lsn_t lsn, /*!< in: page lsn */ - const byte* src_frame,/*!< in: page frame */ - ulint size, /*!< in: size of data to encrypt */ - byte* dst_frame, /*!< in: where to encrypt to */ - ulint page_encryption_key); /*!< in: page encryption key id if page - encrypted */ - -/********************************************************************* -Decrypt buffer page */ -void -fil_space_decrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: size of data to encrypt */ - byte* dst_frame); /*!< in: where to decrypt to */ - - -/********************************************************************* -Decrypt buffer page -@return true if page was encrypted */ -bool -fil_space_decrypt( -/*===============*/ - fil_space_crypt_t* crypt_data, /*!< in: crypt data */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: page size */ - byte* dst_frame); /*!< in: where to decrypt to */ - -/********************************************************************* -fil_space_verify_crypt_checksum -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) -@return true if page is encrypted AND OK, false otherwise */ -bool -fil_space_verify_crypt_checksum( -/*===============*/ - const byte* src_frame,/*!< in: page frame */ - ulint zip_size); /*!< in: size of data to encrypt */ - -/********************************************************************* -Init threads for key rotation */ -void -fil_crypt_threads_init(); - -/********************************************************************* -Set thread count (e.g start or stops threads) used for key rotation */ -void -fil_crypt_set_thread_cnt( -/*=====================*/ - uint new_cnt); /*!< in: requested #threads */ - -/********************************************************************* -End threads for key rotation */ -void -fil_crypt_threads_end(); - -/********************************************************************* -Cleanup resources for threads for key rotation */ -void -fil_crypt_threads_cleanup(); - -/********************************************************************* -Set rotate key age */ -void -fil_crypt_set_rotate_key_age( -/*=====================*/ - uint rotate_age); /*!< in: requested rotate age */ - -/********************************************************************* -Set rotation threads iops */ -void -fil_crypt_set_rotation_iops( -/*=====================*/ - uint iops); /*!< in: requested iops */ - -/********************************************************************* -Mark a space as closing */ -UNIV_INTERN -void -fil_space_crypt_mark_space_closing( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Wait for crypt threads to stop accessing space */ -UNIV_INTERN -void -fil_space_crypt_close_tablespace( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/** Struct for retreiving info about encryption */ -struct fil_space_crypt_status_t { - ulint space; /*!< tablespace id */ - ulint scheme; /*!< encryption scheme */ - uint min_key_version; /*!< min key version */ - uint current_key_version;/*!< current key version */ - uint keyserver_requests;/*!< no of key requests to key server */ - bool rotating; /*!< is key rotation ongoing */ - bool flushing; /*!< is flush at end of rotation ongoing */ - ulint rotate_next_page_number; /*!< next page if key rotating */ - ulint rotate_max_page_number; /*!< max page if key rotating */ -}; - -/********************************************************************* -Get crypt status for a space -@return 0 if crypt data found */ -int -fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t * status); /*!< out: status */ - -/** Struct for retreiving statistics about encryption key rotation */ -struct fil_crypt_stat_t { - ulint pages_read_from_cache; - ulint pages_read_from_disk; - ulint pages_modified; - ulint pages_flushed; - ulint estimated_iops; -}; - -/********************************************************************* -Get crypt rotation statistics */ -void -fil_crypt_total_stat( -/*==================*/ - fil_crypt_stat_t* stat); /*!< out: crypt stat */ - -/** Struct for retreiving info about scrubbing */ -struct fil_space_scrub_status_t { - ulint space; /*!< tablespace id */ - bool compressed; /*!< is space compressed */ - time_t last_scrub_completed; /*!< when was last scrub completed */ - bool scrubbing; /*!< is scrubbing ongoing */ - time_t current_scrub_started; /*!< when started current scrubbing */ - ulint current_scrub_active_threads; /*!< current scrub active threads */ - ulint current_scrub_page_number; /*!< current scrub page no */ - ulint current_scrub_max_page_number; /*!< current scrub max page no */ -}; - -/********************************************************************* -Get scrub status for a space -@return 0 if no scrub info found */ -int -fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t * status); /*!< out: status */ #endif -/*******************************************************************//** -Return page type name */ -const char* -fil_get_page_type_name( -/*===================*/ - ulint page_type); /*!< in: FIL_PAGE_TYPE */ +#ifndef UNIV_INNOCHECKSUM +#ifndef UNIV_NONINL +#include "fil0fil.ic" +#endif +#endif #endif /* fil0fil_h */ diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic new file mode 100644 index 00000000000..b1e65e6dddb --- /dev/null +++ b/storage/innobase/include/fil0fil.ic @@ -0,0 +1,108 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0fil.ic +The low-level file system support functions + +Created 31/03/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0fil_ic +#define fil0fil_ic + +/*******************************************************************//** +Return space name */ +UNIV_INLINE +char* +fil_space_name( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->name); +} + +/*******************************************************************//** +Return space flags */ +UNIV_INLINE +ulint +fil_space_flags( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->flags); +} + +/*******************************************************************//** +Return page type name */ +UNIV_INLINE +const char* +fil_get_page_type_name( +/*===================*/ + ulint page_type) /*!< in: FIL_PAGE_TYPE */ +{ + switch(page_type) { + case FIL_PAGE_PAGE_COMPRESSED: + return (const char*)"PAGE_COMPRESSED"; + case FIL_PAGE_INDEX: + return (const char*)"INDEX"; + case FIL_PAGE_UNDO_LOG: + return (const char*)"UNDO LOG"; + case FIL_PAGE_INODE: + return (const char*)"INODE"; + case FIL_PAGE_IBUF_FREE_LIST: + return (const char*)"IBUF_FREE_LIST"; + case FIL_PAGE_TYPE_ALLOCATED: + return (const char*)"ALLOCATED"; + case FIL_PAGE_IBUF_BITMAP: + return (const char*)"IBUF_BITMAP"; + case FIL_PAGE_TYPE_SYS: + return (const char*)"SYS"; + case FIL_PAGE_TYPE_TRX_SYS: + return (const char*)"TRX_SYS"; + case FIL_PAGE_TYPE_FSP_HDR: + return (const char*)"FSP_HDR"; + case FIL_PAGE_TYPE_XDES: + return (const char*)"XDES"; + case FIL_PAGE_TYPE_BLOB: + return (const char*)"BLOB"; + case FIL_PAGE_TYPE_ZBLOB: + return (const char*)"ZBLOB"; + case FIL_PAGE_TYPE_ZBLOB2: + return (const char*)"ZBLOB2"; + case FIL_PAGE_TYPE_COMPRESSED: + return (const char*)"ORACLE PAGE COMPRESSED"; + default: + return (const char*)"PAGE TYPE CORRUPTED"; + } +} + +/****************************************************************//** +Get block size from fil node +@return block size*/ +UNIV_INLINE +ulint +fil_node_get_block_size( +/*====================*/ + fil_node_t* node) /*!< in: Node where to get block + size */ +{ + return (node->file_block_size); +} + +#endif /* fil0fil_ic */ diff --git a/storage/innobase/include/fil0pageencryption.h b/storage/innobase/include/fil0pageencryption.h deleted file mode 100644 index 405b91c8c06..00000000000 --- a/storage/innobase/include/fil0pageencryption.h +++ /dev/null @@ -1,79 +0,0 @@ -/***************************************************************************** - -Copyright (C) 2014 eperi GmbH. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ - -#ifndef fil0pageencryption_h -#define fil0pageencryption_h - -#define PAGE_ENCRYPTION_WRONG_KEY 1 -#define PAGE_ENCRYPTION_WRONG_PAGE_TYPE 2 -#define PAGE_ENCRYPTION_ERROR 3 -#define PAGE_ENCRYPTION_KEY_MISSING 4 -#define PAGE_ENCRYPTION_OK 0 -#define PAGE_ENCRYPTION_WILL_NOT_ENCRYPT 5 - -/* This key will be used if nothing else is given */ -#define DEFAULT_ENCRYPTION_KEY 1 - -#include "fsp0fsp.h" -#include "fsp0pageencryption.h" - -/******************************************************************//** -@file include/fil0pageencryption.h -Helper functions for encryption/decryption page data on to table space. - -Created 08/25/2014 -***********************************************************************/ - -/*******************************************************************//** -Find out whether the page is page encrypted -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -ibool -fil_space_is_page_encrypted( -/*========================*/ - ulint id); /*!< in: space id */ - -/*******************************************************************//** -Find out whether the page is page encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*==================*/ - const byte *buf); /*!< in: page */ - -/*******************************************************************//** -Find out whether the page is page compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf); /*!< in: page */ - -/*******************************************************************//** -Find out whether the page can be decrypted -@return true if page can be decrypted, false if not. */ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*=======================*/ - const byte *buf); /*!< in: page */ - -#endif // fil0pageencryption_h diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index ee1ab4c7f55..2bac42eb081 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -59,10 +59,6 @@ is found in a remote location, not the default data directory. */ #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION 1 -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY 8 - /** Number of flag bits used to indicate atomic writes for this tablespace */ #define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 @@ -74,9 +70,7 @@ is found in a remote location, not the default data directory. */ + FSP_FLAGS_WIDTH_DATA_DIR \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) + + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) /** A mask of all the known/used bits in tablespace flags */ #define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) @@ -100,15 +94,9 @@ dictionary */ /** Zero relative shift position of the ATOMIC_WRITES field */ #define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY (FSP_FLAGS_POS_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION) /** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ + + FSP_FLAGS_WIDTH_ATOMIC_WRITES) /** Zero relative shift position of the start of the UNUSED bits */ #define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) @@ -144,14 +132,6 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Bit mask of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) /** Bit mask of the ATOMIC_WRITES field */ #define FSP_FLAGS_MASK_ATOMIC_WRITES \ ((~(~0 << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ @@ -192,14 +172,6 @@ dictionary */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ >> FSP_FLAGS_POS_ATOMIC_WRITES) -/** Return the value of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Return the value of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) /** Set a PAGE_SSIZE into the correct bits in a given tablespace flags. */ @@ -216,13 +188,6 @@ tablespace flags. */ #define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) -/** Set a PAGE_ENCRYPTION into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION(flags, encryption) \ - (flags | (encryption << FSP_FLAGS_POS_PAGE_ENCRYPTION)) -/** Set a PAGE_ENCRYPTION_KEY into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(flags, encryption_key) \ - (flags | (encryption_key << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY)) - /** Set a ATOMIC_WRITES into the correct bits in a given tablespace flags. */ #define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ diff --git a/storage/innobase/include/fsp0pageencryption.h b/storage/innobase/include/fsp0pageencryption.h deleted file mode 100644 index 52365c8e93c..00000000000 --- a/storage/innobase/include/fsp0pageencryption.h +++ /dev/null @@ -1,66 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************/ - -/******************************************************************//** -@file include/fsp0pageencryption.h -Helper functions for extracting/storing page encryption information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#ifndef FSP0PAGEENCRYPTION_H_ -#define FSP0PAGEENCRYPTION_H_ - - -#define FIL_PAGE_ENCRYPTION_AES_128 16 /*!< Encryption algorithm AES-128. */ -#define FIL_PAGE_ENCRYPTION_AES_196 24 /*!< Encryption algorithm AES-196. */ -#define FIL_PAGE_ENCRYPTION_AES_256 32 /*!< Encryption algorithm AES-256. */ - -#define FIL_PAGE_ENCRYPTED_SIZE 2 /*!< Number of bytes used to store - actual payload data size on encrypted pages. */ - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags); /*!< in: tablespace flags */ - - -#ifndef UNIV_NONINL -#include "fsp0pageencryption.ic" -#endif - - -#endif /* FSP0PAGEENCRYPTION_H_ */ diff --git a/storage/innobase/include/fsp0pageencryption.ic b/storage/innobase/include/fsp0pageencryption.ic deleted file mode 100644 index b5c3f5ab666..00000000000 --- a/storage/innobase/include/fsp0pageencryption.ic +++ /dev/null @@ -1,167 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************//** -@file include/fsp0pageencryption.ic -Implementation for helper functions for encrypting/decrypting pages -and atomic writes information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#include "fsp0fsp.h" -#include "fil0pageencryption.h" - - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION(flags)); -} - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags)); -} - - -/*******************************************************************//** -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -UNIV_INLINE -ibool -fil_space_is_page_encrypted( -/*=========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_encrypted(flags)); - } - - return(flags); -} - -/*******************************************************************//** -Returns the page encryption key of the space, or 0 if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return page compression level, ULINT_UNDEFINED if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_encryption_key( -/*=================================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_get_page_encryption_key(flags)); - } - - return(flags); -} - -/*******************************************************************//** -Find out whether the page is page is encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*==================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page is page is first compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page can be decrypted. -This is the case, if the page is already decrypted and is not the first page of the table space. -If the page is already decrypted it is not of the FIL_PAGE_PAGE_ENCRYPTED type. -if it is the first page of the table space, it is assumed that a page can be decrypted if the -key found in the flags (part of the 1st page) can be read from the key provider. -The case, if the key changed, is currently not caught. -The function for decrypting the page should already be executed before this. -@return PAGE_ENCRYPTION_KEY_MISSING if key provider is available, but key is not available - PAGE_ENCRYPTION_ERROR if other error occurred - 0 if decryption should be possible -*/ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*=====================*/ - const byte *buf) /*!< in: page */ -{ - ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (page_type == FIL_PAGE_TYPE_FSP_HDR) { - ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* accessing table would surely fail, because no key or no key provider available */ - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - } - } - - if(page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - if (!has_encryption_key(key)) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - return 0; -} diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 99490fa044e..fe6b755dc59 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -312,25 +312,21 @@ The wrapper functions have the prefix of "innodb_". */ pfs_os_file_close_func(file, __FILE__, __LINE__) # define os_aio(type, mode, name, file, buf, offset, \ - n, message1, message2, write_size, \ - page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + n, message1, message2, write_size) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, write_size, \ - page_compression, page_compression_level, \ - page_encryption, page_encryption_key, \ - lsn, encrypt, __FILE__, __LINE__) + __FILE__, __LINE__) -# define os_file_read(file, buf, offset, n, compressed) \ - pfs_os_file_read_func(file, buf, offset, n, compressed, __FILE__, __LINE__) +# define os_file_read(file, buf, offset, n) \ + pfs_os_file_read_func(file, buf, offset, n, __FILE__, __LINE__) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ +# define os_file_read_no_error_handling(file, buf, offset, n) \ pfs_os_file_read_no_error_handling_func(file, buf, offset, n, \ - compressed, __FILE__, __LINE__) + __FILE__, __LINE__) -# define os_file_write(name, file, buf, offset, n) \ - pfs_os_file_write_func(name, file, buf, offset, \ +# define os_file_write(name, file, buf, offset, n) \ + pfs_os_file_write_func(name, file, buf, offset, \ n, __FILE__, __LINE__) # define os_file_flush(file) \ @@ -362,18 +358,15 @@ to original un-instrumented file I/O APIs */ # define os_file_close(file) os_file_close_func(file) # define os_aio(type, mode, name, file, buf, offset, n, message1, \ - message2, write_size, page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + message2, write_size) \ os_aio_func(type, mode, name, file, buf, offset, n, \ - message1, message2, write_size, \ - page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn, encrypt) + message1, message2, write_size) -# define os_file_read(file, buf, offset, n, compressed) \ - os_file_read_func(file, buf, offset, n, compressed) +# define os_file_read(file, buf, offset, n) \ + os_file_read_func(file, buf, offset, n) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ - os_file_read_no_error_handling_func(file, buf, offset, n, compressed) +# define os_file_read_no_error_handling(file, buf, offset, n) \ + os_file_read_no_error_handling_func(file, buf, offset, n) # define os_file_write(name, file, buf, offset, n) \ os_file_write_func(name, file, buf, offset, n) @@ -724,8 +717,6 @@ pfs_os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -744,8 +735,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -781,16 +770,6 @@ pfs_os_aio_func( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -947,9 +926,7 @@ os_file_read_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed); /*!< in: is this file space - compressed ? */ + ulint n); /*!< in: number of bytes to read */ /*******************************************************************//** Rewind file to its start, read at most size - 1 bytes from it to str, and NUL-terminate str. All errors are silently ignored. This function is @@ -974,9 +951,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed); /*!< in: is this file space - compressed ? */ + ulint n); /*!< in: number of bytes to read */ /*******************************************************************//** NOTE! Use the corresponding macro os_file_write(), not directly this @@ -1160,21 +1135,11 @@ os_aio_func( (can be used to identify a completed aio operation); ignored if mode is OS_AIO_SYNC */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size);/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/innobase/include/os0file.ic b/storage/innobase/include/os0file.ic index 9839a841188..db525bcea19 100644 --- a/storage/innobase/include/os0file.ic +++ b/storage/innobase/include/os0file.ic @@ -220,17 +220,6 @@ pfs_os_aio_func( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< in: page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - lsn_t lsn, /*!< in: lsn of the newest - modification */ - bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -246,9 +235,7 @@ pfs_os_aio_func( src_file, src_line); result = os_aio_func(type, mode, name, file, buf, offset, - n, message1, message2, write_size, - page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn, encrypt_later); + n, message1, message2, write_size); register_pfs_file_io_end(locker, n); @@ -269,8 +256,6 @@ pfs_os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -281,7 +266,7 @@ pfs_os_file_read_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_func(file, buf, offset, n, compressed); + result = os_file_read_func(file, buf, offset, n); register_pfs_file_io_end(locker, n); @@ -304,8 +289,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -316,7 +299,7 @@ pfs_os_file_read_no_error_handling_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_no_error_handling_func(file, buf, offset, n, compressed); + result = os_file_read_no_error_handling_func(file, buf, offset, n); register_pfs_file_io_end(locker, n); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 12c05e16b6b..1c08c9fbfb4 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1276,7 +1276,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0, false); + buf, group, 0); srv_stats.os_log_pending_writes.dec(); } @@ -1443,7 +1443,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0, false); + group, 0); srv_stats.os_log_pending_writes.dec(); @@ -2011,7 +2011,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0, false); + buf, ((byte*) group + 1), 0); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2093,7 +2093,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0); } /******************************************************//** @@ -2417,7 +2417,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, NULL, 0, 0, false); + len, buf, NULL, 0); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index d0f2cd360f9..3b8c4878e32 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -49,6 +49,7 @@ Created 9/20/1997 Heikki Tuuri #include "trx0undo.h" #include "trx0rec.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP # include "buf0rea.h" # include "srv0srv.h" @@ -3101,7 +3102,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3132,7 +3133,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); } #ifdef UNIV_LOG_ARCHIVE diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index d49f5f9900d..1287ee76819 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -43,9 +43,9 @@ Created 10/21/1995 Heikki Tuuri #include "srv0srv.h" #include "srv0start.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "buf0buf.h" #include "srv0mon.h" #include "srv0srv.h" @@ -224,49 +224,14 @@ struct os_aio_slot_t{ completed */ ulint bitmap; - byte* page_compression_page; /*!< Memory allocated for - page compressed page and - freed after the write - has been completed */ - - byte* page_encryption_page; /*!< Memory allocated for - page encrypted page and - freed after the write - has been completed */ - - ibool page_compression; - ulint page_compression_level; - - ibool page_encryption; - ulint page_encryption_key; - ulint* write_size; /*!< Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - byte* page_buf; /*!< Actual page buffer for - page compressed pages, do not - free this */ - - byte* page_buf2; /*!< Actual page buffer for - page encrypted pages, do not - free this */ - byte* tmp_encryption_buf; /*!< a temporal buffer used by page encryption */ - - ibool page_compression_success; - /*!< TRUE if page compression was successfull, false if not */ - ibool page_encryption_success; - /*!< TRUE if page encryption was successfull, false if not */ - - lsn_t lsn; /* lsn of the newest modification */ - ulint file_block_size;/*!< file block size */ - bool encrypt_later; /*!< should the page be encrypted - before write */ - #ifdef WIN_ASYNC_IO HANDLE handle; /*!< handle object we need in the OVERLAPPED struct */ @@ -403,39 +368,6 @@ os_file_trim( /*=========*/ os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( - os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( - os_aio_slot_t* slot); /*!< in: slot structure */ /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ @@ -2891,9 +2823,7 @@ os_file_read_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed) /*!< in: is this file space - compressed ? */ + ulint n) /*!< in: number of bytes to read */ { #ifdef __WIN__ BOOL ret; @@ -3024,9 +2954,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed) /*!< in: is this file space - compressed ? */ + ulint n) /*!< in: number of bytes to read */ { #ifdef __WIN__ BOOL ret; @@ -4184,9 +4112,8 @@ os_aio_array_free( /*==============*/ os_aio_array_t*& array) /*!< in, own: array to free */ { - ulint i; #ifdef WIN_ASYNC_IO - + ulint i; for (i = 0; i < array->n_slots; i++) { os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); CloseHandle(slot->handle); @@ -4207,31 +4134,6 @@ os_aio_array_free( } #endif /* LINUX_NATIVE_AIO */ - for (i = 0; i < array->n_slots; i++) { - os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); - - if (slot->page_compression_page) { - ut_free(slot->page_compression_page); - slot->page_compression_page = NULL; - } - - if (slot->lzo_mem) { - ut_free(slot->lzo_mem); - slot->lzo_mem = NULL; - } - - if (slot->page_encryption_page) { - ut_free(slot->page_encryption_page); - slot->page_encryption_page = NULL; - } - - if (slot->tmp_encryption_buf) { - ut_free(slot->tmp_encryption_buf); - slot->tmp_encryption_buf = NULL; - } - } - - ut_free(array->slots); ut_free(array); @@ -4566,22 +4468,11 @@ os_aio_array_reserve_slot( to write */ os_offset_t offset, /*!< in: file offset */ ulint len, /*!< in: length of the block to read or write */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt before - writing the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4669,95 +4560,13 @@ found: slot->len = len; slot->type = type; slot->offset = offset; - slot->lsn = lsn; slot->io_already_done = FALSE; - slot->page_compression_success = FALSE; - slot->page_encryption_success = FALSE; slot->write_size = write_size; - slot->page_compression_level = page_compression_level; - slot->page_compression = page_compression; - slot->page_encryption_key = page_encryption_key; - slot->page_encryption = page_encryption; - slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); } - /* If the space is page compressed and this is write operation - then we compress the page */ - if (message1 && type == OS_FILE_WRITE && page_compression ) { - ulint real_len = len; - byte* tmp = NULL; - - /* Release the array mutex while compressing */ - os_mutex_exit(array->mutex); - - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); - -#ifdef HAVE_LZO - if (innodb_compression_algorithm == 3) { - os_slot_alloc_lzo_mem(slot); - } -#endif - - /* Call page compression */ - tmp = fil_compress_page( - fil_node_get_space_id(slot->message1), - (byte *)buf, - slot->page_buf, - len, - page_compression_level, - fil_node_get_block_size(slot->message1), - &real_len, - slot->lzo_mem - ); - - /* If compression succeeded, set up the length and buffer */ - if (tmp != buf) { - len = real_len; - buf = slot->page_buf; - slot->len = real_len; - slot->page_compression_success = TRUE; - } else { - slot->page_compression_success = FALSE; - } - - /* Take array mutex back, not sure if this is really needed - below */ - os_mutex_enter(array->mutex); - - } - - /* If the space is page encryption and this is write operation - then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && (page_encryption == 1 || encrypt_later)) { - ut_a(page_encryption == 1 || srv_encrypt_tables == 1); - /* Release the array mutex while encrypting */ - os_mutex_exit(array->mutex); - - // We allocate memory for page encrypted buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf2(slot); - - fil_space_encrypt( - fil_node_get_space_id(slot->message1), - slot->offset, - slot->lsn, - (byte *)buf, - slot->len, - slot->page_buf2, - slot->page_encryption_key); - - slot->page_encryption_success = TRUE; - buf = slot->page_buf2; - - /* Take array mutex back */ - os_mutex_enter(array->mutex); - } - slot->buf = static_cast<byte*>(buf); #ifdef WIN_ASYNC_IO @@ -5037,22 +4846,11 @@ os_aio_func( (can be used to identify a completed aio operation); ignored if mode is OS_AIO_SYNC */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt page - before write */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5104,8 +4902,7 @@ os_aio_func( and os_file_write_func() */ if (type == OS_FILE_READ) { - ret = os_file_read_func(file, buf, offset, n, - page_compression); + ret = os_file_read_func(file, buf, offset, n); } else { ut_ad(!srv_read_only_mode); @@ -5163,9 +4960,7 @@ try_again: } slot = os_aio_array_reserve_slot(type, array, message1, message2, file, - name, buf, offset, n, write_size, - page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn, encrypt_later); + name, buf, offset, n, write_size); if (type == OS_FILE_READ) { if (srv_use_native_aio) { @@ -5192,18 +4987,9 @@ try_again: if (srv_use_native_aio) { os_n_file_writes++; #ifdef WIN_ASYNC_IO - if (page_encryption && slot->page_encryption_success) { - buffer = slot->page_buf2; - n = slot->len; - } else { - if (page_compression && slot->page_compression_success) { - buffer = slot->page_buf; - n = slot->len; - } else { - buffer = buf; - } - } + n = slot->len; + buffer = buf; ret = WriteFile(file, buffer, (DWORD) n, &len, &(slot->control)); @@ -5409,27 +5195,9 @@ os_aio_windows_handle( switch (slot->type) { case OS_FILE_WRITE: - if (slot->message1 - && slot->page_encryption - && slot->page_encryption_success) { - ret_val = os_file_write(slot->name, - slot->file, - slot->page_buf2, - slot->offset, - slot->len); - } else { - if (slot->message1 - && slot->page_compression - && slot->page_compression_success) { - ret = WriteFile(slot->file, slot->page_buf, - (DWORD) slot->len, &len, - &(slot->control)); - } else { - ret = WriteFile(slot->file, slot->buf, - (DWORD) slot->len, &len, - &(slot->control)); - } - } + ret = WriteFile(slot->file, slot->buf, + (DWORD) slot->len, &len, + &(slot->control)); break; case OS_FILE_READ: ret = ReadFile(slot->file, slot->buf, @@ -5460,47 +5228,9 @@ os_aio_windows_handle( ret_val = ret && len == slot->len; } - if (slot->type == OS_FILE_READ) { - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - ut_ad(slot->message1 != NULL); - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - - // Decrypt the data - fil_space_decrypt( - fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - if (fil_page_is_compressed(slot->buf)) { - /* We allocate memory for page compressed buffer if - and only if it is not yet allocated. */ - os_slot_alloc_page_buf(slot); -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - fil_decompress_page( - slot->page_buf, - slot->buf, - slot->len, - slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } - } + if (slot->type == OS_FILE_WRITE && srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } os_aio_array_free_slot(array, slot); @@ -5592,50 +5322,9 @@ retry: /* We have not overstepped to next segment. */ ut_a(slot->pos < end_pos); - if (slot->type == OS_FILE_READ) { - /* If the page is page encrypted we encrypt */ - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - ut_ad(slot->message1 != NULL); - - // Decrypt the data - fil_space_decrypt( - fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - - /* If the table is page compressed and this - is read, we decompress before we announce - the read is complete. For writes, we free - the compressed page. */ - if (fil_page_is_compressed(slot->buf)) { - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - - fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } - } + if (slot->type == OS_FILE_WRITE && srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } /* Mark this request as completed. The error handling @@ -6080,8 +5769,7 @@ consecutive_loop: } else { ret = os_file_read( aio_slot->file, combined_buf, - aio_slot->offset, total_len, - aio_slot->page_compression); + aio_slot->offset, total_len); } srv_set_io_thread_op_info(global_segment, "file i/o done"); @@ -6653,92 +6341,6 @@ os_file_trim( } #endif /* !UNIV_HOTBACKUP */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - - if(slot->page_buf2 == NULL) { - byte* cbuf2; - byte* cbuf; - - cbuf2 = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); - cbuf = static_cast<byte *>(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_encryption_page = static_cast<byte *>(cbuf2); - slot->page_buf2 = static_cast<byte *>(cbuf); - memset(slot->page_encryption_page, 0, UNIV_PAGE_SIZE*2); - } -} - -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->page_buf == NULL) { - byte* cbuf2; - byte* cbuf; - ulint asize = UNIV_PAGE_SIZE; -#ifdef HAVE_SNAPPY - asize += snappy_max_compressed_length(asize) - UNIV_PAGE_SIZE; -#endif - - /* We allocate extra to avoid memory overwrite on compression */ - cbuf2 = static_cast<byte *>(ut_malloc(asize*2)); - cbuf = static_cast<byte *>(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_compression_page = static_cast<byte *>(cbuf2); - slot->page_buf = static_cast<byte *>(cbuf); - ut_a(slot->page_buf != NULL); - memset(slot->page_compression_page, 0, asize*2); - } -} - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if(slot->lzo_mem == NULL) { - slot->lzo_mem = static_cast<byte *>(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); - ut_a(slot->lzo_mem != NULL); - memset(slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); - } -} -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( -/*=============================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->tmp_encryption_buf == NULL) { - slot->tmp_encryption_buf = static_cast<byte *>(ut_malloc(64)); - memset(slot->tmp_encryption_buf, 0, 64); - } -} - - /***********************************************************************//** Try to get number of bytes per sector from file system. @return file block size */ diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index caed087b439..c63dbffcba9 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -2554,7 +2554,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" @@ -3385,7 +3385,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 97746aa088e..d2734ecd6b5 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -786,7 +786,7 @@ row_merge_read( #endif /* UNIV_DEBUG */ success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf, - ofs, srv_sort_buf_size, FALSE); + ofs, srv_sort_buf_size); #ifdef POSIX_FADV_DONTNEED /* Each block is read exactly once. Free up the file cache. */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 3dd65c540c3..d9fef775d40 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -57,6 +57,7 @@ Created 9/17/2000 Heikki Tuuri #include "btr0sea.h" #include "btr0defragment.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "ibuf0ibuf.h" #include "fts0fts.h" #include "fts0types.h" diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index ee88c1f9e33..78954725001 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -74,9 +74,9 @@ Created 10/8/1995 Heikki Tuuri #include "mysql/plugin.h" #include "mysql/service_thd_wait.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fil0pagecompress.h" #include "btr0scrub.h" -#include "fil0pageencryption.h" #ifdef WITH_WSREP extern int wsrep_debug; @@ -524,7 +524,7 @@ second. */ static time_t srv_last_log_flush_time; /** Default encryption key used for page encryption */ -UNIV_INTERN uint srv_default_page_encryption_key = DEFAULT_ENCRYPTION_KEY; +UNIV_INTERN uint srv_default_page_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; /** Enable semaphore request instrumentation */ UNIV_INTERN my_bool srv_instrument_semaphores = FALSE; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 3822a9abf2d..3cc66778de9 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2014, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -53,6 +53,7 @@ Created 2/16/1996 Heikki Tuuri #include "os0file.h" #include "os0thread.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fsp0fsp.h" #include "rem0rec.h" #include "mtr0mtr.h" diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 0ef97fd3e21..56b0245da93 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -40,6 +40,7 @@ Created 11/5/1995 Heikki Tuuri #include "mem0mem.h" #include "btr0btr.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP #include "buf0buddy.h" #include "lock0lock.h" @@ -57,8 +58,6 @@ Created 11/5/1995 Heikki Tuuri #include "trx0trx.h" #include "srv0start.h" #include "ut0byte.h" - -#include "fil0pageencryption.h" #include "fil0pagecompress.h" @@ -574,7 +573,7 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { - ulint page_encrypted = fil_page_is_compressed_encrypted(read_buf) || fil_page_is_encrypted(read_buf); + ulint page_encrypted = fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; ibool crc32_inited = FALSE; @@ -5938,7 +5937,7 @@ buf_page_decrypt_after_read( unsigned key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); + bool page_compressed = fil_page_is_compressed(dst_frame); if (key_version == 0) { /* the page we read is unencrypted */ @@ -5974,7 +5973,7 @@ buf_page_decrypt_after_read( /* decompress from dst_frame to comp_buf and then copy to buffer pool */ - if (page_compressed_encrypted) { + if (page_compressed) { if (bpage->comp_buf_free == NULL) { bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); // TODO: is 4k aligment enough ? diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 871f723549b..ec733dd5a73 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -36,6 +36,8 @@ Created 2011/12/19 #include "srv0srv.h" #include "page0zip.h" #include "trx0sys.h" +#include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP @@ -385,7 +387,7 @@ buf_dblwr_init_or_load_pages( /* Read the trx sys header to check if we are using the doublewrite buffer */ off_t trx_sys_page = TRX_SYS_PAGE_NO * UNIV_PAGE_SIZE; - os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE, FALSE); + os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE); doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; @@ -430,9 +432,9 @@ buf_dblwr_init_or_load_pages( block_bytes = TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * UNIV_PAGE_SIZE; - os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes, FALSE); + os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes); os_file_read(file, buf + block_bytes, block2 * UNIV_PAGE_SIZE, - block_bytes, FALSE); + block_bytes); /* Check if any of these pages is half-written in data files, in the intended position */ @@ -530,9 +532,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, read_buf, NULL, - 0, - 0, - false); + 0); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -593,9 +593,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -619,9 +617,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); } } } @@ -643,9 +639,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL); ut_free(unaligned_buf); } @@ -859,9 +855,7 @@ buf_dblwr_write_block_to_datafile( buf_page_get_zip_size(bpage), frame, (void*) bpage, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); return; } @@ -880,9 +874,7 @@ buf_dblwr_write_block_to_datafile( UNIV_PAGE_SIZE, frame, (void*) block, - (ulint *)&bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + (ulint *)&bpage->write_size); } /********************************************************************//** @@ -984,9 +976,7 @@ try_again: (void*) write_buf, NULL, - 0, - 0, - false); + 0); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -1009,9 +999,7 @@ try_again: len, (void*) write_buf, NULL, - 0, - 0, - false); + 0); flush: /* increment the doublewrite flushed pages counter */ @@ -1250,9 +1238,7 @@ retry: UNIV_PAGE_SIZE, (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ @@ -1264,9 +1250,7 @@ retry: 0, UNIV_PAGE_SIZE, frame, NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 4331db08cf5..d023e2ff227 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -963,9 +963,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer is initialized. User can define do we use atomic writes @@ -984,9 +982,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 88741f987ca..164aa3c62c1 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -232,14 +232,14 @@ not_to_recover: *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, zip_size, offset, 0, zip_size, - frame, bpage, 0, trx, 0, false); + frame, bpage, 0, trx); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, 0, offset, 0, UNIV_PAGE_SIZE, - frame, bpage, &bpage->write_size, trx, 0, false); + frame, bpage, &bpage->write_size, trx); } if (sync) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 9f8315a8016..aab0b1f5568 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -24,6 +24,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "fil0crypt.h" #include "srv0srv.h" #include "srv0start.h" #include "mach0data.h" @@ -35,14 +36,13 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "btr0scrub.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "ha_prototypes.h" // IB_LOG_ #include <my_crypt.h> #include <my_aes.h> #include <math.h> - +#include "fil0crypt.h" /** Mutex for keys */ UNIV_INTERN ib_mutex_t fil_crypt_key_mutex; @@ -119,67 +119,40 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; -/** - * CRYPT_SCHEME_UNENCRYPTED - * - * Used as intermediate state when convering a space from unencrypted - * to encrypted - */ -#define CRYPT_SCHEME_UNENCRYPTED 0 - -/** - * CRYPT_SCHEME_1 - * - * L = AES_ECB(KEY, IV) - * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) - */ -#define CRYPT_SCHEME_1 1 -#define CRYPT_SCHEME_1_IV_LEN 16 -// cached L given key_version -struct key_struct -{ - uint key_version; - byte key[CRYPT_SCHEME_1_IV_LEN]; -}; - -struct fil_space_rotate_state_t +/****************************************************************** +Map used AES method to crypt scheme +@return used AES crypt scheme */ +UNIV_INTERN +uint +fil_crypt_get_aes_method( +/*=====================*/ + uint aes_method) { - time_t start_time; // time when rotation started - ulint active_threads; // active threads in space - ulint next_offset; // next "free" offset - ulint max_offset; // max offset needing to be rotated - uint min_key_version_found; // min key version found but not rotated - lsn_t end_lsn; // max lsn created when rotating this space - bool starting; // initial write of IV - bool flushing; // space is being flushed at end of rotate - struct { - bool is_active; // is scrubbing active in this space - time_t last_scrub_completed; // when was last scrub completed - } scrubbing; -}; + switch (aes_method) { + case MY_AES_ALGORITHM_NONE: + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; + break; + case MY_AES_ALGORITHM_CTR: + return (uint) CRYPT_SCHEME_1_CTR; + break; + case MY_AES_ALGORITHM_CBC: + return (uint) CRYPT_SCHEME_1_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", aes_method); + ut_error; + } -struct fil_space_crypt_struct -{ - ulint type; // CRYPT_SCHEME - uint keyserver_requests; // no of key requests to key server - uint key_count; // No of initalized key-structs - key_struct keys[3]; // cached L = AES_ECB(KEY, IV) - uint min_key_version; // min key version for this space - ulint page0_offset; // byte offset on page 0 for crypt data - - ib_mutex_t mutex; // mutex protecting following variables - bool closing; // is tablespace being closed - fil_space_rotate_state_t rotate_state; - - uint iv_length; // length of IV - byte iv[1]; // IV-data -}; + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; +} /********************************************************************* Init space crypt */ UNIV_INTERN void fil_space_crypt_init() +/*==================*/ { mutex_create(fil_crypt_key_mutex_key, &fil_crypt_key_mutex, SYNC_NO_ORDER_CHECK); @@ -196,6 +169,7 @@ Cleanup space crypt */ UNIV_INTERN void fil_space_crypt_cleanup() +/*=====================*/ { os_event_free(fil_crypt_throttle_sleep_event); } @@ -204,31 +178,35 @@ fil_space_crypt_cleanup() Get key bytes for a space/key-version */ static void -fil_crypt_get_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint version, bool page_encrypted) +fil_crypt_get_key( +/*==============*/ + byte* dst, /*<! out: Key */ + uint* key_length, /*<! out: Key length */ + fil_space_crypt_t* crypt_data, /*<! in: crypt data */ + uint version) /*<! in: Key version */ { unsigned char keybuf[MY_AES_MAX_KEY_LENGTH]; mutex_enter(&crypt_data->mutex); - if (!page_encrypted) { - // Check if we already have key - for (uint i = 0; i < crypt_data->key_count; i++) { - if (crypt_data->keys[i].key_version == version) { - memcpy(dst, crypt_data->keys[i].key, - sizeof(crypt_data->keys[i].key)); - mutex_exit(&crypt_data->mutex); - return; - } + // Check if we already have key + for (uint i = 0; i < crypt_data->key_count; i++) { + if (crypt_data->keys[i].key_version == version) { + memcpy(dst, crypt_data->keys[i].key, + crypt_data->keys[i].key_length); + *key_length = crypt_data->keys[i].key_length; + mutex_exit(&crypt_data->mutex); + return; } - // Not found! - crypt_data->keyserver_requests++; + } - // Rotate keys to make room for a new - for (uint i = 1; i < array_elements(crypt_data->keys); i++) { - crypt_data->keys[i] = crypt_data->keys[i - 1]; - } - } + // Not found! + crypt_data->keyserver_requests++; + + // Rotate keys to make room for a new + for (uint i = 1; i < array_elements(crypt_data->keys); i++) { + crypt_data->keys[i] = crypt_data->keys[i - 1]; + } if (has_encryption_key(version)) { int rc; @@ -247,51 +225,45 @@ fil_crypt_get_key(byte *dst, uint* key_length, ut_error; } + /* Now compute L by encrypting IV using this key. Note + that we use random IV from crypt data. */ + const unsigned char* src = crypt_data->iv; + const int srclen = crypt_data->iv_length; + unsigned char* buf = crypt_data->keys[0].key; + uint32 buflen = CRYPT_SCHEME_1_IV_LEN; - // do ctr key initialization - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) - { - /* Now compute L by encrypting IV using this key. Note - that we use random IV from crypt data. */ - const unsigned char* src = crypt_data->iv; - const int srclen = crypt_data->iv_length; - unsigned char* buf = page_encrypted ? keybuf : crypt_data->keys[0].key; - uint32 buflen = page_encrypted ? *key_length : sizeof(crypt_data->keys[0].key); - - // call ecb explicit - my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - int rc = (*func)(src, srclen, - buf, &buflen, - (unsigned char*)keybuf, *key_length, - NULL, 0, - 1); - - if (rc != AES_OK) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt key-block " - " src: %p srclen: %d buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - src, srclen, buf, buflen, rc); - ut_error; - } + /* We use AES_ECB to encryp IV */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - if (!page_encrypted) { - crypt_data->keys[0].key_version = version; - crypt_data->key_count++; + int rc = (*func)(src, /* Data to be encrypted = IV */ + srclen, /* data length */ + buf, /* Output buffer */ + &buflen, /* Output buffer */ + keybuf, /* Key */ + *key_length, /* Key length */ + NULL, /* AES_ECB does not use IV */ + 0, /* IV-length */ + 1); /* NoPadding */ - if (crypt_data->key_count > array_elements(crypt_data->keys)) { - crypt_data->key_count = array_elements(crypt_data->keys); - } - } + if (rc != AES_OK) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to encrypt key-block " + " src: %p srclen: %d buf: %p buflen: %d." + " return-code: %d. Can't continue!\n", + src, srclen, buf, buflen, rc); + ut_error; + } - // set the key size to the aes block size because this encrypted data is the key - *key_length = MY_AES_BLOCK_SIZE; - memcpy(dst, buf, buflen); - } else { - // otherwise keybuf contains the right key - memcpy(dst, keybuf, *key_length); + crypt_data->keys[0].key_version = version; + crypt_data->key_count++; + *key_length = buflen; + crypt_data->keys[0].key_length = buflen; + + if (crypt_data->key_count > array_elements(crypt_data->keys)) { + crypt_data->key_count = array_elements(crypt_data->keys); } + memcpy(dst, buf, buflen); mutex_exit(&crypt_data->mutex); } @@ -299,8 +271,12 @@ fil_crypt_get_key(byte *dst, uint* key_length, Get key bytes for a space/latest(key-version) */ static inline void -fil_crypt_get_latest_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint *version) +fil_crypt_get_latest_key( +/*=====================*/ + byte* dst, /*!< out: Key */ + uint* key_length, /*!< out: Key length */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + uint* version) /*!< in: Key version */ { if (srv_encrypt_tables) { // used for key rotation - get the next key id from the key provider @@ -312,29 +288,33 @@ fil_crypt_get_latest_key(byte *dst, uint* key_length, } } - return fil_crypt_get_key(dst, key_length, crypt_data, *version, srv_encrypt_tables == FALSE); + return fil_crypt_get_key(dst, key_length, crypt_data, *version); } /****************************************************************** -Create a fil_space_crypt_t object */ +Create a fil_space_crypt_t object +@return crypt object */ UNIV_INTERN fil_space_crypt_t* fil_space_create_crypt_data() +/*=========================*/ { const uint iv_length = CRYPT_SCHEME_1_IV_LEN; const uint sz = sizeof(fil_space_crypt_t) + iv_length; fil_space_crypt_t* crypt_data = static_cast<fil_space_crypt_t*>(malloc(sz)); + uint aes_method = current_aes_dynamic_method; + memset(crypt_data, 0, sz); if (srv_encrypt_tables == FALSE) { - crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; crypt_data->min_key_version = 0; } else { - crypt_data->type = CRYPT_SCHEME_1; crypt_data->min_key_version = get_latest_encryption_key_version(); } + /* Set up the current AES method */ + crypt_data->type = (fil_crypt_method_t)fil_crypt_get_aes_method(aes_method); mutex_create(fil_crypt_data_mutex_key, &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; @@ -346,13 +326,18 @@ fil_space_create_crypt_data() Compare two crypt objects */ UNIV_INTERN int -fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, - const fil_space_crypt_t* crypt_data2) +fil_space_crypt_compare( +/*====================*/ + const fil_space_crypt_t* crypt_data1,/*!< in: Crypt data */ + const fil_space_crypt_t* crypt_data2)/*!< in: Crypt data */ { - ut_a(crypt_data1->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data1->type == CRYPT_SCHEME_1); - ut_a(crypt_data2->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data2->type == CRYPT_SCHEME_1); + ut_a(crypt_data1->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data1->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); + + ut_a(crypt_data2->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data2->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); ut_a(crypt_data1->iv_length == CRYPT_SCHEME_1_IV_LEN); ut_a(crypt_data2->iv_length == CRYPT_SCHEME_1_IV_LEN); @@ -365,10 +350,15 @@ fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, } /****************************************************************** -Read crypt data from a page (0) */ +Read crypt data from a page (0) +@return crypt data from page 0. */ UNIV_INTERN fil_space_crypt_t* -fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) +fil_space_read_crypt_data( +/*======================*/ + ulint space, /*!< in: file space id*/ + const byte* page, /*!< in: page 0 */ + ulint offset) /*!< in: offset */ { if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { /* crypt is not stored */ @@ -393,8 +383,10 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); - if (! (type == CRYPT_SCHEME_UNENCRYPTED || - type == CRYPT_SCHEME_1)) { + if (! (type == CRYPT_SCHEME_1_UNENCRYPTED || + type == CRYPT_SCHEME_1_CTR || + type == CRYPT_SCHEME_1_CBC)) { + ib_logf(IB_LOG_LEVEL_ERROR, "Found non sensible crypt scheme: %lu for space %lu " " offset: %lu bytes: " @@ -449,7 +441,9 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) Free a crypt data object */ UNIV_INTERN void -fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) +fil_space_destroy_crypt_data( +/*=========================*/ + fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { /* lock (and unlock) mutex to make sure no one has it locked @@ -466,10 +460,14 @@ fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) Write crypt data to a page (0) */ static void -fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, - ulint type, - byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data_low( +/*===========================*/ + fil_space_crypt_t* crypt_data, /*<! out: crypt data */ + ulint type, /*<! in: crypt scheme */ + byte* page, /*<! in: page 0 */ + ulint offset, /*<! in: offset */ + ulint maxsize, /*<! in: size of crypt data */ + mtr_t* mtr) /*<! in: minitransaction */ { ut_a(offset > 0 && offset < UNIV_PAGE_SIZE); ulint space_id = mach_read_from_4( @@ -492,10 +490,8 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len, min_key_version, MLOG_4BYTES, mtr); - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_1", - ut_error;); - byte* log_ptr = mlog_open(mtr, 11 + 12 + len); + if (log_ptr != NULL) { log_ptr = mlog_write_initial_log_record_fast( page, @@ -515,25 +511,26 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_catenate_string(mtr, crypt_data->iv, len); } - - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_2", - ut_error;); } /****************************************************************** Write crypt data to a page (0) */ UNIV_INTERN void -fil_space_write_crypt_data(ulint space, byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data( +/*=======================*/ + ulint space, /*<! in: file space */ + byte* page, /*<! in: page 0 */ + ulint offset, /*<! in: offset */ + ulint maxsize, /*<! in: size of crypt data */ + mtr_t* mtr) /*<! in: minitransaction */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); - ibool page_encrypted = fil_space_is_page_encrypted(space); /* If no crypt data is stored on memory cache for this space or space is not encrypted and encryption is not enabled, then do not continue writing crypt data to page 0. */ - if (crypt_data == NULL || (!page_encrypted && !srv_encrypt_tables)) { + if (crypt_data == NULL && !srv_encrypt_tables) { return; } @@ -542,11 +539,15 @@ fil_space_write_crypt_data(ulint space, byte* page, ulint offset, } /****************************************************************** -Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry */ +Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry +@return position on log buffer */ UNIV_INTERN byte* -fil_parse_write_crypt_data(byte* ptr, byte* end_ptr, - buf_block_t* block) +fil_parse_write_crypt_data( +/*=======================*/ + byte* ptr, /*!< in: Log entry start */ + byte* end_ptr,/*!< in: Log entry end */ + buf_block_t* block) /*!< in: buffer block */ { /* check that redo log entry is complete */ uint entry_size = @@ -569,8 +570,10 @@ fil_parse_write_crypt_data(byte* ptr, byte* end_ptr, uint len = mach_read_from_1(ptr); ptr += 1; - ut_a(type == CRYPT_SCHEME_UNENCRYPTED || - type == CRYPT_SCHEME_1); // only supported + ut_a(type == CRYPT_SCHEME_1_UNENCRYPTED || + type == CRYPT_SCHEME_1_CTR || + type == CRYPT_SCHEME_1_CBC); // supported methods + ut_a(len == CRYPT_SCHEME_1_IV_LEN); // only supported uint min_key_version = mach_read_from_4(ptr); ptr += 4; @@ -595,7 +598,10 @@ fil_parse_write_crypt_data(byte* ptr, byte* end_ptr, Clear crypt data from a page (0) */ UNIV_INTERN void -fil_space_clear_crypt_data(byte* page, ulint offset) +fil_space_clear_crypt_data( +/*=======================*/ + byte* page, /*!< in/out: Page 0 */ + ulint offset) /*!< in: Offset */ { //TODO(jonaso): pass crypt-data and read len from there ulint len = CRYPT_SCHEME_1_IV_LEN; @@ -609,7 +615,8 @@ fil_space_clear_crypt_data(byte* page, ulint offset) } /********************************************************************* -Check if page shall be encrypted before write */ +Check if page shall be encrypted before write +@return true if page should be encrypted, false if not */ UNIV_INTERN bool fil_space_check_encryption_write( @@ -633,26 +640,58 @@ fil_space_check_encryption_write( } /****************************************************************** +Map current aes method +@return AES method */ +UNIV_INTERN +uint +fil_crypt_map_aes_method( +/*=====================*/ + uint aes_method) /*!< in: AES method */ +{ + switch((fil_crypt_method_t)aes_method) { + case CRYPT_SCHEME_1_CTR: + return (uint)MY_AES_ALGORITHM_CTR; + break; + case CRYPT_SCHEME_1_CBC: + return (uint)MY_AES_ALGORITHM_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", (int)aes_method); + ut_error; + } +} + +/****************************************************************** Encrypt a page */ UNIV_INTERN void -fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, - const byte* src_frame, ulint zip_size, byte* dst_frame, ulint encryption_key) +fil_space_encrypt( +/*==============*/ + ulint space, /*!< in: Space id */ + ulint offset, /*!< in: Page offset */ + lsn_t lsn, /*!< in: lsn */ + const byte* src_frame, /*!< in: Source page to be encrypted */ + ulint zip_size, /*!< in: compressed size if + row_format compressed */ + byte* dst_frame, /*!< in: outbut buffer */ + ulint encryption_key) /*!< in: encryption key id if page + encrypted */ { fil_space_crypt_t* crypt_data=NULL; ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - // get key (L) - uint key_version = encryption_key; - byte key[MY_AES_MAX_KEY_LENGTH]; + uint key_version = (uint)encryption_key; + unsigned char key[MY_AES_MAX_KEY_LENGTH]; uint key_length=MY_AES_MAX_KEY_LENGTH; + uint aes_method; + unsigned char iv[MY_AES_BLOCK_SIZE]; ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR - || orig_page_type==FIL_PAGE_TYPE_XDES - || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED - || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + || orig_page_type==FIL_PAGE_TYPE_XDES) { + /* File space header or extent descriptor do not need to be + encrypted. */ //TODO: is this really needed ? memcpy(dst_frame, src_frame, page_size); return; @@ -667,63 +706,56 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, return; } + /* Load the IV or counter (depending to the encryption algorithm used) */ fil_crypt_get_latest_key(key, &key_length, crypt_data, &key_version); + aes_method = fil_crypt_get_aes_method(current_aes_dynamic_method); - /* Load the iv or counter (depending to the encryption algorithm used) */ - unsigned char iv[MY_AES_BLOCK_SIZE]; - - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) { - // create counter block (C) - mach_write_to_4(iv + 0, space); - ulint space_offset = mach_read_from_4( - src_frame + FIL_PAGE_OFFSET); - mach_write_to_4(iv + 4, space_offset); - mach_write_to_8(iv + 8, lsn); - } else { - // Get random IV from crypt_data - mutex_enter(&crypt_data->mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); - } + /* For AES CTR create counter and AES CBS IV */ + mach_write_to_4(iv + 0, space); + ulint space_offset = mach_read_from_4(src_frame + FIL_PAGE_OFFSET); + mach_write_to_4(iv + 4, space_offset); + mach_write_to_8(iv + 8, lsn); ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); - ibool page_encrypted = fil_space_is_page_encrypted(space); - ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint compression_algo = page_compressed ? mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) : 0; - // copy page header + /* FIL page header is not encrypted */ memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - if (page_encrypted && !page_compressed) { - // key id - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - // original page type - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2, - orig_page_type); - // new page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - } else { - // store key version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - } + /* Store key version */ + mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, key_version); - // encrypt page data + /* Calculate the start offset in a page */ ulint unencrypted_bytes = FIL_PAGE_DATA + FIL_PAGE_DATA_END; ulint srclen = page_size - unencrypted_bytes; const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; + uint32 dstlen=0; + /* For page compressed tables we encrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be encrypted. */ if (page_compressed) { - srclen = page_size - FIL_PAGE_DATA; - } - - int rc = (* my_aes_encrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); + ulint payload = mach_read_from_2(src_frame + FIL_PAGE_DATA); + mach_write_to_2(dst_frame + FIL_PAGE_DATA, payload); + srclen = payload; + src+=2; + dst+=2; + } + + /* Get encryption method */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Encrypt the data */ + int rc = (*func)(src, /* Original page */ + srclen, /* Page length */ + dst, /* Output buffer */ + &dstlen, /* Output length */ + key, /* Encryption key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV length */ + 1); /* Use noPadding */ if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -735,65 +767,35 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, ut_error; } + /* Set up the checksum */ + mach_write_to_4(dst_frame+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); + + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ if (!page_compressed) { - // copy page trailer + /* FIL page trailer is also not encrypted */ memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - - /* handle post encryption checksum */ - ib_uint32_t checksum = 0; - srv_checksum_algorithm_t algorithm = - static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); - - if (zip_size == 0) { - switch (algorithm) { - case SRV_CHECKSUM_ALGORITHM_CRC32: - case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - checksum = buf_calc_page_crc32(dst_frame); - break; - case SRV_CHECKSUM_ALGORITHM_INNODB: - case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: - checksum = (ib_uint32_t) buf_calc_page_new_checksum( - dst_frame); - break; - case SRV_CHECKSUM_ALGORITHM_NONE: - case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: - checksum = BUF_NO_CHECKSUM_MAGIC; - break; - /* no default so the compiler will emit a warning - * if new enum is added and not handled here */ - } - } else { - checksum = page_zip_calc_checksum(dst_frame, zip_size, - algorithm); - } - - // store the post-encryption checksum after the key-version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, - checksum); - } else { - /* Page compressed and encrypted tables have different - FIL_HEADER */ - ulint page_len = log10((double)page_size)/log10((double)2); - /* Set up the correct page type */ - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - /* Set up the compression algorithm */ - mach_write_to_2(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4, orig_page_type); - /* Set up the compressed size */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6, page_len); - /* Set up the compression method */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7, compression_alg); } + /* Store AES encryption method */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, + aes_method); + /* Store compression algorithm (for page compresed tables) or 0 */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6, + compression_algo); } /********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ +Check if extra buffer shall be allocated for decrypting after read +@return true if fil space has encryption data. */ UNIV_INTERN bool fil_space_check_encryption_read( -/*==============================*/ +/*=============================*/ ulint space) /*!< in: tablespace id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -810,101 +812,92 @@ fil_space_check_encryption_read( } /****************************************************************** -Decrypt a page */ +Decrypt a page +@return true if page was encrypted */ UNIV_INTERN bool -fil_space_decrypt(fil_space_crypt_t* crypt_data, - const byte* src_frame, ulint page_size, byte* dst_frame) +fil_space_decrypt( +/*==============*/ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ { ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - // key version - uint key_version; - bool page_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_ENCRYPTED); - - bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED); - - ulint orig_page_type=0; - - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - key_version = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - orig_page_type = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2); - } else { - key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } + uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); + ulint compression_algo = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6); + uint aes_method = 0; - if (key_version == 0 && !page_encrypted) { + /* Page is not encrypted if key_version is 0 */ + if (key_version == 0) { //TODO: is this really needed ? memcpy(dst_frame, src_frame, page_size); return false; /* page not decrypted */ } - // read space & offset & lsn + /* read space & offset & lsn */ ulint space = mach_read_from_4( src_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4( src_frame + FIL_PAGE_OFFSET); ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); - // copy page header + /* Copy FIL page header, it is not encrypted */ memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - // orig page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); - } - - // get key + /* Get key */ byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; - fil_crypt_get_key(key, &key_length, crypt_data, key_version, page_encrypted); - - // get the iv unsigned char iv[MY_AES_BLOCK_SIZE]; + fil_crypt_get_key(key, &key_length, crypt_data, key_version); + aes_method = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) { - // create counter block - - mach_write_to_4(iv + 0, space); - mach_write_to_4(iv + 4, offset); - mach_write_to_8(iv + 8, lsn); - } else { - // Get random IV from crypt_data - mutex_enter(&crypt_data->mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); + /* Verify used AES method */ + if ( aes_method != CRYPT_SCHEME_1_CTR && + aes_method != CRYPT_SCHEME_1_CBC) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to decrypt data-block " + " aes_method: %d unknown!\n", + aes_method); + ut_error; } + /* Create counter used as IV */ + mach_write_to_4(iv + 0, space); + mach_write_to_4(iv + 4, offset); + mach_write_to_8(iv + 8, lsn); + + /* Calculate the offset where decryption starts */ const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; + uint32 dstlen=0; ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; - ulint compression_method; + /* For page compressed tables we decrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be decrypted. */ if (page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4); - compressed_len = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6); - compression_method = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7); - } - - if (page_encrypted && !page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+2); - } - - if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - srclen = pow((double)2, (double)((int)compressed_len)) - FIL_PAGE_DATA; - } - - int rc = (* my_aes_decrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); + ulint compressed_len = mach_read_from_2(src_frame + FIL_PAGE_DATA); + src+=2; + dst+=2; + mach_write_to_2(dst_frame + FIL_PAGE_DATA, compressed_len); + srclen = compressed_len; + } + + /* Get AES method */ + my_aes_encrypt_dynamic_type func = get_aes_decrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Decrypt the data */ + int rc = (*func)(src, /* Data to be encrypted */ + srclen, /* data length */ + dst, /* Output buffer */ + &dstlen, /* Output buffer */ + key, /* Key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV-length */ + 1); /* NoPadding */ if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -916,22 +909,57 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, ut_error; } - if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - // copy page trailer + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ + if (!page_compressed) { + /* Copy FIL trailer */ memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - // clear key-version & crypt-checksum from dst - memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); - } else { - /* For page compressed tables we set up the FIL_HEADER again */ - /* setting original page type */ - mach_write_to_2(dst_frame + FIL_PAGE_TYPE, orig_page_type); - /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ - mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); - /* Set up the flush lsn to be compression algorithm */ - mach_write_to_8(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, compression_method); + /* handle post decryption checksum */ + ib_uint32_t checksum = 0; + srv_checksum_algorithm_t algorithm = + static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); + + if (page_size == UNIV_PAGE_SIZE) { + switch (algorithm) { + case SRV_CHECKSUM_ALGORITHM_CRC32: + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + checksum = buf_calc_page_crc32(dst_frame); + break; + case SRV_CHECKSUM_ALGORITHM_INNODB: + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + checksum = (ib_uint32_t) buf_calc_page_new_checksum( + dst_frame); + break; + case SRV_CHECKSUM_ALGORITHM_NONE: + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + checksum = BUF_NO_CHECKSUM_MAGIC; + break; + /* no default so the compiler will emit a warning + * if new enum is added and not handled here */ + } + } else { + checksum = page_zip_calc_checksum(dst_frame, page_size, + algorithm); + } + + mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + fprintf(stderr, "KUKKU %ld:%lu\n", mach_read_from_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM), + mach_read_from_4(dst_frame + FIL_PAGE_END_LSN_OLD_CHKSUM)); + } + + /* Clear key-version & aes_method from dst */ + memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); + + /* For page compressed tables store compression algorithm back */ + if (page_compressed) { + mach_write_to_8(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + compression_algo); } return true; /* page was decrypted */ @@ -941,8 +969,12 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, Decrypt a page */ UNIV_INTERN void -fil_space_decrypt(ulint space, - const byte* src_frame, ulint page_size, byte* dst_frame) +fil_space_decrypt( +/*==============*/ + ulint space, /*!< in: Fil space id */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ { fil_space_decrypt(fil_space_get_crypt_data(space), src_frame, page_size, dst_frame); @@ -953,8 +985,13 @@ Verify checksum for a page (iff it's encrypted) NOTE: currently this function can only be run in single threaded mode as it modifies srv_checksum_algorithm (temporarily) @return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN bool -fil_space_verify_crypt_checksum(const byte* src_frame, ulint zip_size) +fil_space_verify_crypt_checksum( +/*============================*/ + const byte* src_frame, /*!< in: page the verify */ + ulint zip_size) /*!< in: compressed size if + row_format compressed */ { // key version uint key_version = mach_read_from_4( @@ -1041,7 +1078,8 @@ struct key_state_t { Copy global key state */ static void fil_crypt_get_key_state( - key_state_t *new_state) +/*====================*/ + key_state_t *new_state) /*!< out: key state */ { if (srv_encrypt_tables == TRUE) { new_state->key_version = get_latest_encryption_key_version(); @@ -1054,9 +1092,13 @@ fil_crypt_get_key_state( } /*********************************************************************** -Check if a key needs rotation given a key_state */ +Check if a key needs rotation given a key_state +@return true if key needs rotation, false if not */ static bool -fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) +fil_crypt_needs_rotation( +/*=====================*/ + uint key_version, /*!< in: Key version */ + const key_state_t* key_state) /*!< in: Key state */ { // TODO(jonaso): Add support for rotating encrypted => unencrypted @@ -1081,9 +1123,13 @@ fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) } /*********************************************************************** -Check if a space is closing (i.e just before drop) */ -UNIV_INTERN bool -fil_crypt_is_closing(ulint space) +Check if a space is closing (i.e just before drop) +@return true if space is closing, false if not. */ +UNIV_INTERN +bool +fil_crypt_is_closing( +/*=================*/ + ulint space) /*!< in: FIL space id */ { bool closing; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1097,16 +1143,22 @@ fil_crypt_is_closing(ulint space) Start encrypting a space @return true if a pending op (fil_inc_pending_ops/fil_decr_pending_ops) is held */ -static bool -fil_crypt_start_encrypting_space(ulint space, bool *recheck) { +static +bool +fil_crypt_start_encrypting_space( +/*=============================*/ + ulint space, /*!< in: FIL space id */ + bool* recheck)/*!< out: true if recheck needed */ +{ /* we have a pending op when entering function */ bool pending_op = true; + uint aes_method = current_aes_dynamic_method; mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); - ibool page_encrypted = fil_space_is_page_encrypted(space); + ibool page_encrypted = (crypt_data != NULL); /*If spage is not encrypted and encryption is not enabled, then do not continue encrypting the space. */ @@ -1184,7 +1236,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 4 - write crypt data to page 0 */ fil_space_write_crypt_data_low(crypt_data, - CRYPT_SCHEME_1, + fil_crypt_get_aes_method(aes_method), frame, crypt_data->page0_offset, maxsize, &mtr); @@ -1232,7 +1284,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 5 - publish crypt data */ mutex_enter(&fil_crypt_threads_mutex); mutex_enter(&crypt_data->mutex); - crypt_data->type = CRYPT_SCHEME_1; + crypt_data->type = fil_crypt_get_aes_method(aes_method); ut_a(crypt_data->rotate_state.active_threads == 1); crypt_data->rotate_state.active_threads = 0; crypt_data->rotate_state.starting = false; @@ -1257,10 +1309,14 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { } /*********************************************************************** -Check if space needs rotation given a key_state */ -static bool -fil_crypt_space_needs_rotation(uint space, const key_state_t *key_state, - bool *recheck) +Check if space needs rotation given a key_state +@return true if space needs key rotation */ +static +bool +fil_crypt_space_needs_rotation( + uint space, /*!< in: FIL space id */ + const key_state_t* key_state, /*!< in: Key state */ + bool* recheck) /*!< out: needs recheck ? */ { if (fil_space_get_type(space) != FIL_TABLESPACE) { return false; @@ -1374,7 +1430,9 @@ struct rotate_thread_t { /*********************************************************************** Update global statistics with thread statistics */ static void -fil_crypt_update_total_stat(rotate_thread_t *state) +fil_crypt_update_total_stat( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { mutex_enter(&crypt_stat_mutex); crypt_stat.pages_read_from_cache += @@ -1397,9 +1455,13 @@ fil_crypt_update_total_stat(rotate_thread_t *state) /*********************************************************************** Allocate iops to thread from global setting, -used before starting to rotate a space */ -static bool -fil_crypt_alloc_iops(rotate_thread_t *state) +used before starting to rotate a space. +@return true if allocation succeeded, false if failed */ +static +bool +fil_crypt_alloc_iops( +/*=================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_ad(state->allocated_iops == 0); @@ -1429,8 +1491,11 @@ fil_crypt_alloc_iops(rotate_thread_t *state) /*********************************************************************** Reallocate iops to thread, used when inside a space */ -static void -fil_crypt_realloc_iops(rotate_thread_t *state) +static +void +fil_crypt_realloc_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_a(state->allocated_iops > 0); @@ -1519,8 +1584,11 @@ fil_crypt_realloc_iops(rotate_thread_t *state) /*********************************************************************** Return allocated iops to global */ -static void -fil_crypt_return_iops(rotate_thread_t *state) +static +void +fil_crypt_return_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { if (state->allocated_iops > 0) { uint iops = state->allocated_iops; @@ -1544,11 +1612,14 @@ fil_crypt_return_iops(rotate_thread_t *state) /*********************************************************************** Search for a space needing rotation */ +UNIV_INTERN bool fil_crypt_find_space_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state, - bool *recheck) +/*===========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state, /*!< in: Key rotation state */ + bool* recheck) /*!< out: true if recheck + needed */ { /* we need iops to start rotating */ while (!state->should_shutdown() && !fil_crypt_alloc_iops(state)) { @@ -1591,8 +1662,9 @@ Start rotating a space */ static void fil_crypt_start_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*=========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1629,12 +1701,14 @@ fil_crypt_start_rotate_space( } /*********************************************************************** -Search for batch of pages needing rotation */ +Search for batch of pages needing rotation +@return true if page needing key rotation found, false if not found */ static bool fil_crypt_find_page_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state) +/*==========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint batch = srv_alloc_time * state->allocated_iops; ulint space = state->space; @@ -1665,9 +1739,15 @@ fil_crypt_find_page_to_rotate( } /*********************************************************************** -Check if a page is uninitialized (doesn't need to be rotated) */ -static bool -fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) +Check if a page is uninitialized (doesn't need to be rotated) +@return true if page is uninitialized, false if not.*/ +static +bool +fil_crypt_is_page_uninitialized( +/*============================*/ + const byte *frame, /*!< in: Page */ + uint zip_size) /*!< in: compressed size if + row_format compressed */ { if (zip_size) { ulint stored_checksum = mach_read_from_4( @@ -1696,15 +1776,20 @@ fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) sleeptime_ms, __FILE__, __LINE__) /*********************************************************************** -Get a page and compute sleep time */ +Get a page and compute sleep time +@return page */ static buf_block_t* -fil_crypt_get_page_throttle_func(rotate_thread_t *state, - ulint space, uint zip_size, ulint offset, - mtr_t *mtr, - ulint *sleeptime_ms, - const char *file, - ulint line) +fil_crypt_get_page_throttle_func( + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction */ + ulint* sleeptime_ms, /*!< out: sleep time */ + const char* file, /*!< in: file name */ + ulint line) /*!< in: file line */ { buf_block_t* block = buf_page_try_get_func(space, offset, RW_X_LATCH, true, @@ -1753,17 +1838,22 @@ Get block and allocation status note: innodb locks fil_space_latch and then block when allocating page but locks block and then fil_space_latch when freeing page. +@return block */ static buf_block_t* btr_scrub_get_block_and_allocation_status( - rotate_thread_t *state, - ulint space, - ulint zip_size, - ulint offset, - mtr_t *mtr, +/*======================================*/ + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction + */ btr_scrub_page_allocation_status_t *allocation_status, - ulint *sleeptime_ms) + /*!< in/out: allocation status */ + ulint* sleeptime_ms) /*!< out: sleep time */ { mtr_t local_mtr; buf_block_t *block = NULL; @@ -1807,8 +1897,9 @@ Rotate one page */ static void fil_crypt_rotate_page( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint offset = state->offset; @@ -1960,8 +2051,9 @@ Rotate a batch of pages */ static void fil_crypt_rotate_pages( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint end = state->offset + state->batch; @@ -1989,7 +2081,10 @@ fil_crypt_rotate_pages( Flush rotated pages and then update page 0 */ static void -fil_crypt_flush_space(rotate_thread_t *state, ulint space) +fil_crypt_flush_space( +/*==================*/ + rotate_thread_t* state, /*!< in: Key rotation state */ + ulint space) /*!< in: FIL space id */ { fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2033,9 +2128,10 @@ fil_crypt_flush_space(rotate_thread_t *state, ulint space) RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); byte* frame = buf_block_get_frame(block); - ulint maxsize = 0; + ulint maxsize; crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + fil_space_write_crypt_data(space, frame, crypt_data->page0_offset, ULINT_MAX, &mtr); @@ -2048,8 +2144,9 @@ Complete rotating a space */ static void fil_crypt_complete_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*============================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2123,7 +2220,7 @@ A thread which monitors global key state and rotates tablespaces accordingly extern "C" UNIV_INTERN os_thread_ret_t DECLARE_THREAD(fil_crypt_thread)( -/*===============================*/ +/*=============================*/ void* arg __attribute__((unused))) /*!< in: a dummy parameter required * by os_thread_create */ { @@ -2240,13 +2337,19 @@ DECLARE_THREAD(fil_crypt_thread)( Adjust thread count for key rotation */ UNIV_INTERN void -fil_crypt_set_thread_cnt(uint new_cnt) +fil_crypt_set_thread_cnt( +/*=====================*/ + uint new_cnt) /*!< in: New key rotation thread count */ { if (new_cnt > srv_n_fil_crypt_threads) { uint add = new_cnt - srv_n_fil_crypt_threads; srv_n_fil_crypt_threads = new_cnt; for (uint i = 0; i < add; i++) { - os_thread_create(fil_crypt_thread, NULL, NULL); + os_thread_id_t rotation_thread_id; + os_thread_create(fil_crypt_thread, NULL, &rotation_thread_id); + ib_logf(IB_LOG_LEVEL_INFO, + "Creating #%d thread id %lu total threads %du\n", + i, os_thread_pf(rotation_thread_id), new_cnt); } } else if (new_cnt < srv_n_fil_crypt_threads) { srv_n_fil_crypt_threads = new_cnt; @@ -2263,7 +2366,9 @@ fil_crypt_set_thread_cnt(uint new_cnt) Adjust max key age */ UNIV_INTERN void -fil_crypt_set_rotate_key_age(uint val) +fil_crypt_set_rotate_key_age( +/*=========================*/ + uint val) /*!< in: New max key age */ { srv_fil_crypt_rotate_key_age = val; os_event_set(fil_crypt_threads_event); @@ -2273,7 +2378,9 @@ fil_crypt_set_rotate_key_age(uint val) Adjust rotation iops */ UNIV_INTERN void -fil_crypt_set_rotation_iops(uint val) +fil_crypt_set_rotation_iops( +/*========================*/ + uint val) /*!< in: New iops setting */ { srv_n_fil_crypt_iops = val; os_event_set(fil_crypt_threads_event); @@ -2284,6 +2391,7 @@ Init threads for key rotation */ UNIV_INTERN void fil_crypt_threads_init() +/*====================*/ { fil_crypt_event = os_event_create(); fil_crypt_threads_event = os_event_create(); @@ -2300,6 +2408,7 @@ End threads for key rotation */ UNIV_INTERN void fil_crypt_threads_end() +/*===================*/ { /* stop threads */ fil_crypt_set_thread_cnt(0); @@ -2309,7 +2418,9 @@ fil_crypt_threads_end() Clean up key rotation threads resources */ UNIV_INTERN void -fil_crypt_threads_cleanup() { +fil_crypt_threads_cleanup() +/*=======================*/ +{ os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); } @@ -2319,7 +2430,8 @@ Mark a space as closing */ UNIV_INTERN void fil_space_crypt_mark_space_closing( - ulint space) +/*===============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2340,7 +2452,8 @@ Wait for crypt threads to stop accessing space */ UNIV_INTERN void fil_space_crypt_close_tablespace( - ulint space) +/*=============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2352,11 +2465,14 @@ fil_space_crypt_close_tablespace( uint start = time(0); uint last = start; + mutex_enter(&crypt_data->mutex); mutex_exit(&fil_crypt_threads_mutex); crypt_data->closing = true; + uint cnt = crypt_data->rotate_state.active_threads; bool flushing = crypt_data->rotate_state.flushing; + while (cnt > 0 || flushing) { mutex_exit(&crypt_data->mutex); /* release dict mutex so that scrub threads can release their @@ -2386,11 +2502,12 @@ fil_space_crypt_close_tablespace( /********************************************************************* Get crypt status for a space (used by information_schema) return 0 if crypt data present */ +UNIV_INTERN int fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_crypt_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); @@ -2400,6 +2517,7 @@ fil_space_crypt_get_status( mutex_enter(&crypt_data->mutex); status->keyserver_requests = crypt_data->keyserver_requests; status->min_key_version = crypt_data->min_key_version; + if (crypt_data->rotate_state.active_threads > 0 || crypt_data->rotate_state.flushing) { status->rotating = true; @@ -2427,8 +2545,11 @@ fil_space_crypt_get_status( /********************************************************************* Return crypt statistics */ +UNIV_INTERN void -fil_crypt_total_stat(fil_crypt_stat_t *stat) +fil_crypt_total_stat( +/*=================*/ + fil_crypt_stat_t *stat) /*!< out: Crypt statistics */ { mutex_enter(&crypt_stat_mutex); *stat = crypt_stat; @@ -2438,11 +2559,12 @@ fil_crypt_total_stat(fil_crypt_stat_t *stat) /********************************************************************* Get scrub status for a space (used by information_schema) return 0 if data found */ +UNIV_INTERN int fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_scrub_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); memset(status, 0, sizeof(*status)); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 14878f59f7d..6b776edd2f9 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -27,8 +27,7 @@ Created 10/25/1995 Heikki Tuuri #include "fil0fil.h" #include "fil0pagecompress.h" #include "fsp0pagecompress.h" -#include "fil0pageencryption.h" -#include "fsp0pageencryption.h" +#include "fil0crypt.h" #include <debug_sync.h> #include <my_dbug.h> @@ -287,7 +286,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0, false)); + byte_offset, len, buf, message, write_size)); } /********************************************************************//** @@ -314,18 +313,16 @@ fil_write( this must be appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn, encrypt_later)); + byte_offset, len, buf, message, write_size)); } /*******************************************************************//** @@ -651,10 +648,9 @@ fil_node_open_file( set */ page = static_cast<byte*>(ut_align(buf2, UNIV_PAGE_SIZE)); - success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE, - space->flags); + success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); - if (fil_page_encryption_status(page)) { + if (fil_page_is_encrypted(page)) { /* if page is (still) encrypted, write an error and return. * Otherwise the server would crash if decrypting is not possible. * This may be the case, if the key file could not be @@ -1191,21 +1187,6 @@ fil_space_create( ut_a(fil_system); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* by returning here it should be avoided that - * the server crashes, if someone tries to access an - * encrypted table and the encryption key is not available. - * The the table is treaded as non-existent. - */ - ib_logf(IB_LOG_LEVEL_WARN, - "Tablespace '%s' can not be opened, because " - " encryption key can not be found (space id: %lu, key %lu)\n" - , name, (ulong) id, fsp_flags_get_page_encryption_key(flags)); - return (FALSE); - } - } - /* Look for a matching tablespace and if found free it. */ do { mutex_enter(&fil_system->mutex); @@ -1879,7 +1860,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, 0, false); + UNIV_PAGE_SIZE, buf, NULL, 0); } mem_free(buf1); @@ -1970,13 +1951,10 @@ fil_check_first_page( or the encryption key is not available, the check for reading the first page should intentionally fail with "can not decrypt" message. */ - page_is_encrypted = fil_page_encryption_status(page); - if (page_is_encrypted == PAGE_ENCRYPTION_KEY_MISSING && page_is_encrypted) { - page_is_encrypted = 1; - } else { - page_is_encrypted = 0; + page_is_encrypted = fil_page_encryption_status(page, space_id); + if (!page_is_encrypted) { if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { - fprintf(stderr, + fprintf(stderr, "InnoDB: Error: Current page size %lu != " " page size on page %lu\n", UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); @@ -2006,7 +1984,7 @@ fil_check_first_page( /* this error message is interpreted by the calling method, which is * executed if the server starts in recovery mode. */ - return(MSG_CANNOT_DECRYPT); + return(FIL_MSG_CANNOT_DECRYPT); } } @@ -2053,10 +2031,7 @@ fil_read_first_page( page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); - os_file_read(data_file, page, 0, UNIV_PAGE_SIZE, - orig_space_id != ULINT_UNDEFINED ? - fil_space_is_page_compressed(orig_space_id) : - FALSE); + os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave @@ -4106,8 +4081,7 @@ fil_user_tablespace_find_space_id( for (ulint j = 0; j < page_count; ++j) { - st = os_file_read(fsp->file, page, (j* page_size), page_size, - fsp_flags_is_page_compressed(fsp->flags)); + st = os_file_read(fsp->file, page, (j* page_size), page_size); if (!st) { ib_logf(IB_LOG_LEVEL_INFO, @@ -4250,7 +4224,7 @@ check_first_page: "%s in tablespace %s (table %s)", check_msg, fsp->filepath, tablename); fsp->success = FALSE; - if (strncmp(check_msg, MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { + if (strncmp(check_msg, FIL_MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { /* by returning here, it should be avoided, that the server crashes, * if started in recovery mode and can not decrypt tables, if * the key file can not be read. @@ -5258,7 +5232,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, space_id, NULL, 0, 0, 0, 0, 0, 0, false); + node, NULL, space_id, NULL, 0); #endif /* UNIV_HOTBACKUP */ DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", @@ -5643,9 +5617,7 @@ _fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - trx_t* trx, - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ + trx_t* trx) { ulint mode; fil_space_t* space; @@ -5655,10 +5627,6 @@ _fil_io( ulint wake_later; os_offset_t offset; ibool ignore_nonexistent_pages; - ibool page_compressed = FALSE; - ulint page_compression_level = 0; - ibool page_encrypted; - ulint page_encryption_key; is_log = type & OS_FILE_LOG; type = type & ~OS_FILE_LOG; @@ -5726,13 +5694,6 @@ _fil_io( space = fil_space_get_by_id(space_id); - page_compressed = fsp_flags_is_page_compressed(space->flags); - page_compression_level = fsp_flags_get_page_compression_level(space->flags); - - page_encrypted = fsp_flags_is_page_encrypted(space->flags); - page_encryption_key = fsp_flags_get_page_encryption_key(space->flags); - - /* If we are deleting a tablespace we don't allow any read operations on that. However, we do allow write operations. */ if (space == 0 || (type == OS_FILE_READ && space->stop_new_ops)) { @@ -5889,13 +5850,7 @@ _fil_io( message, space_id, trx, - page_compressed, - page_compression_level, - write_size, - page_encrypted, - page_encryption_key, - lsn, - encrypt_later); + write_size); #else /* In mysqlbackup do normal i/o, not aio */ @@ -6454,9 +6409,7 @@ fil_iterate( readptr = iter.crypt_io_buffer; } - if (!os_file_read(iter.file, readptr, offset, - (ulint) n_bytes, - fil_space_is_page_compressed(space_id))) { + if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); @@ -6607,8 +6560,7 @@ fil_tablespace_iterate( /* Read the first page and determine the page and zip size. */ - if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE, - dict_tf_get_page_compression(table->flags))) { + if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE)) { err = DB_IO_ERROR; @@ -6668,7 +6620,7 @@ fil_tablespace_iterate( if (iter.crypt_data != NULL) { /* clear crypt data from page 0 and write it back */ - os_file_read(file, page, 0, UNIV_PAGE_SIZE, 0); + os_file_read(file, page, 0, UNIV_PAGE_SIZE); fil_space_clear_crypt_data(page, crypt_data_offset); lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN); if (callback.get_zip_size() == 0) { @@ -6915,79 +6867,6 @@ fil_system_exit(void) mutex_exit(&fil_system->mutex); } -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->name); -} - -/*******************************************************************//** -Return space flags */ -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->flags); -} - -/*******************************************************************//** -Return page type name */ -const char* -fil_get_page_type_name( -/*===================*/ - ulint page_type) /*!< in: FIL_PAGE_TYPE */ -{ - switch(page_type) { - case FIL_PAGE_PAGE_COMPRESSED: - return (const char*)"PAGE_COMPRESSED"; - case FIL_PAGE_INDEX: - return (const char*)"INDEX"; - case FIL_PAGE_UNDO_LOG: - return (const char*)"UNDO LOG"; - case FIL_PAGE_INODE: - return (const char*)"INODE"; - case FIL_PAGE_IBUF_FREE_LIST: - return (const char*)"IBUF_FREE_LIST"; - case FIL_PAGE_TYPE_ALLOCATED: - return (const char*)"ALLOCATED"; - case FIL_PAGE_IBUF_BITMAP: - return (const char*)"IBUF_BITMAP"; - case FIL_PAGE_TYPE_SYS: - return (const char*)"SYS"; - case FIL_PAGE_TYPE_TRX_SYS: - return (const char*)"TRX_SYS"; - case FIL_PAGE_TYPE_FSP_HDR: - return (const char*)"FSP_HDR"; - case FIL_PAGE_TYPE_XDES: - return (const char*)"XDES"; - case FIL_PAGE_TYPE_BLOB: - return (const char*)"BLOB"; - case FIL_PAGE_TYPE_ZBLOB: - return (const char*)"ZBLOB"; - case FIL_PAGE_TYPE_ZBLOB2: - return (const char*)"ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return (const char*)"ORACLE PAGE COMPRESSED"; - default: - return (const char*)"PAGE TYPE CORRUPTED"; - } -} -/****************************************************************//** -Get block size from fil node -@return block size*/ -ulint -fil_node_get_block_size( -/*====================*/ - fil_node_t* node) /*!< in: Node where to get block - size */ -{ - return (node->file_block_size); -} /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 3c2edd832c4..d394ca5215c 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -84,173 +84,6 @@ static ulint srv_data_read, srv_data_written; //#define UNIV_PAGECOMPRESS_DEBUG 1 /****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -static -void -fil_decompress_page_2( -/*==================*/ - byte* page_buf, /*!< out: destination buffer for - uncompressed data */ - byte* buf, /*!< in: source compressed data */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size) /*!< in/out: Actual payload size of - the compressed data. */ -{ - ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - if (page_type != FIL_PAGE_TYPE_COMPRESSED) { - /* It is not a compressed page */ - return; - } - - byte* ptr = buf + FIL_PAGE_DATA; - ulint version = mach_read_from_1(buf + FIL_PAGE_VERSION); - int err = 0; - - ut_a(version == 1); - - /* Read the original page type, before we compressed the data. */ - page_type = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_TYPE_V1); - - ulint original_len = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_SIZE_V1); - - if (original_len < UNIV_PAGE_SIZE_MIN - (FIL_PAGE_DATA + 8) - || original_len > UNIV_PAGE_SIZE_MAX - FIL_PAGE_DATA - || len < original_len + FIL_PAGE_DATA) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page. " - "Original len %lu len %lu.", - original_len, len); - - fflush(stderr); - ut_error; - - } - - ulint algorithm = mach_read_from_1(buf + FIL_PAGE_ALGORITHM_V1); - - switch(algorithm) { - case PAGE_ZLIB_ALGORITHM: { - - err = uncompress(page_buf, &len, ptr, original_len); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed " - "but uncompress failed with error %d " - " size %lu len %lu.", - err, original_len, len); - - fflush(stderr); - - ut_error; - } - - break; - } -#ifdef HAVE_LZ4 - case PAGE_LZ4_ALGORITHM: { - - err = LZ4_decompress_fast( - (const char*) ptr, (char*) (page_buf), original_len); - - if (err < 0) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %d bytes" - " size %lu len %lu.", - err, original_len, len); - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZ4 */ - -#ifdef HAVE_LZMA - case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; - size_t src_pos = 0; - size_t dst_pos = 0; - uint64_t memlimit = UINT64_MAX; - - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - ptr, - &src_pos, - original_len, - (page_buf), - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos <= 0 || dst_pos > len)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - dst_pos, original_len, len); - fflush(stderr); - - ut_error; - } - - break; - } -#endif /* HAVE_LZMA */ - -#ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - - err = lzo1x_decompress((const unsigned char *)ptr, - original_len,(unsigned char *)(page_buf), &olen, NULL); - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - olen, original_len, len); - - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZO */ - - default: - ib_logf(IB_LOG_LEVEL_ERROR, - " Corruption: Page is marked as compressed " - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(algorithm)); - - fflush(stderr); - ut_error; - break; - } - - /* Leave the header alone */ - memmove(buf+FIL_PAGE_DATA, page_buf, original_len); - - mach_write_to_2(buf + FIL_PAGE_TYPE, page_type); - - ut_ad(memcmp(buf + FIL_PAGE_LSN + 4, - buf + (original_len + FIL_PAGE_DATA) - - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4) == 0); -} - -/****************************************************************//** For page compressed pages compress the page before actual write operation. @return compressed page to be written*/ @@ -270,7 +103,7 @@ fil_compress_page( byte* lzo_mem) /*!< in: temporal memory used by LZO */ { int err = Z_OK; - int level = 0; + int level = compression_level; ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; ulint write_size=0; /* Cache to avoid change during function execution */ @@ -290,15 +123,11 @@ fil_compress_page( if (orig_page_type == 0 || orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { *out_len = len; return (buf); } - level = compression_level; - ut_ad(fil_space_is_page_compressed(space_id)); - fil_system_enter(); fil_space_t* space = fil_space_get_by_id(space_id); fil_system_exit(); @@ -577,16 +406,6 @@ fil_decompress_page( in_buf = page_buf; } - if (ptype == FIL_PAGE_TYPE_COMPRESSED) { - - fil_decompress_page_2(in_buf, buf, len, write_size); - // Need to free temporal buffer if no buffer was given - if (page_buf == NULL) { - ut_free(in_buf); - } - return; - } - /* Before actual decompress, make sure that page type is correct */ if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 0fffe60253e..34acab04241 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -31,6 +31,7 @@ Created 11/29/1995 Heikki Tuuri #include "buf0buf.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "mtr0log.h" #include "ut0byte.h" #include "page0page.h" diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 5cc232a3b35..cb6b7d1a263 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -81,6 +81,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fsp0fsp.h" #include "sync0sync.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "trx0xa.h" #include "row0merge.h" #include "dict0boot.h" @@ -106,7 +107,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fts0priv.h" #include "page0zip.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) @@ -11557,8 +11557,6 @@ innobase_table_flags( modified by another thread while the table is being created. */ const ulint default_compression_level = page_zip_level; - const ulint default_encryption_key = srv_default_page_encryption_key; - *flags = 0; *flags2 = 0; @@ -11757,10 +11755,7 @@ index_bad: options->page_compressed, options->page_compression_level == 0 ? default_compression_level : options->page_compression_level, - options->atomic_writes, - options->page_encryption, - options->page_encryption_key == 0 ? - default_encryption_key : options->page_encryption_key); + options->atomic_writes); if (create_info->options & HA_LEX_CREATE_TMP_TABLE) { *flags2 |= DICT_TF2_TEMPORARY; @@ -20339,7 +20334,7 @@ static MYSQL_SYSVAR_UINT(default_page_encryption_key, srv_default_page_encryptio "Encryption key used for page encryption.", NULL, NULL, - DEFAULT_ENCRYPTION_KEY, 1, 255, 0); + FIL_DEFAULT_ENCRYPTION_KEY, 1, 255, 0); static MYSQL_SYSVAR_BOOL(scrub_log, srv_scrub_log, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/xtradb/handler/i_s.cc b/storage/xtradb/handler/i_s.cc index 20bc4331bc8..332af275c55 100644 --- a/storage/xtradb/handler/i_s.cc +++ b/storage/xtradb/handler/i_s.cc @@ -72,6 +72,8 @@ Modified Dec 29, 2014 Jan Lindström (Added sys_semaphore_waits) #include "btr0btr.h" #include "page0zip.h" #include "sync0arr.h" +#include "fil0fil.h" +#include "fil0crypt.h" /** structure associates a name string with a file page type and/or buffer page state. */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index ef8c9878297..0d3aac32df9 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -929,10 +929,8 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic + ulint atomic_writes) /*!< in: table atomic writes option value*/ - bool page_encrypted,/*!< in: table uses page encryption */ - ulint page_encryption_key) /*!< in: page encryption key */ __attribute__((nonnull)); /********************************************************************//** Convert a 32 bit integer table flags to the 32 bit integer that is diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index c480b3c6216..e5f4ec21bd2 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -543,9 +543,6 @@ dict_tf_is_valid( ulint data_dir = DICT_TF_HAS_DATA_DIR(flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(flags); - /* Make sure there are no bits that we do not know about. */ if (unused != 0) { @@ -556,12 +553,10 @@ dict_tf_is_valid( "InnoDB: compact %ld atomic_blobs %ld\n" "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n" - "InnoDB: page_encryption %ld page_encryption_key %ld\n", + "InnoDB: atomic_writes %ld\n", unused, compact, atomic_blobs, unused, data_dir, zip_ssize, - page_compression, page_compression_level, atomic_writes, - page_encryption, page_encryption_key + page_compression, page_compression_level, atomic_writes ); return(false); @@ -861,9 +856,7 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic writes setup */ - bool page_encrypted, /*!< in: table uses page encryption */ - ulint page_encryption_key /*!< in: page encryption key */) + ulint atomic_writes) /*!< in: table atomic writes setup */ { atomic_writes_t awrites = (atomic_writes_t)atomic_writes; @@ -904,11 +897,6 @@ dict_tf_set( *flags |= (atomic_writes << DICT_TF_POS_ATOMIC_WRITES); ut_a(dict_tf_get_atomic_writes(*flags) == awrites); - - if (page_encrypted) { - *flags |= (1 << DICT_TF_POS_PAGE_ENCRYPTION) - | (page_encryption_key << DICT_TF_POS_PAGE_ENCRYPTION_KEY); - } } /********************************************************************//** @@ -931,11 +919,6 @@ dict_tf_to_fsp_flags( ulint fsp_flags; ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - /* Keys are limited to 255 values */ - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", @@ -963,14 +946,6 @@ dict_tf_to_fsp_flags( if page compression is used for this table. */ fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); - /* In addition, tablespace flags also contain if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION(fsp_flags, page_encryption); - - /* In addition, tablespace flags also contain page encryption key if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(fsp_flags, page_encryption_key); - /* In addition, tablespace flags also contain flag if atomic writes is used for this table */ fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); @@ -1012,8 +987,6 @@ dict_sys_tables_type_to_tf( | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY ); @@ -1050,9 +1023,7 @@ dict_tf_to_sys_tables_type( | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY); + | DICT_TF_MASK_ATOMIC_WRITES); return(type); } diff --git a/storage/xtradb/include/dict0pagecompress.ic b/storage/xtradb/include/dict0pagecompress.ic index a71b2b34b07..811976434a8 100644 --- a/storage/xtradb/include/dict0pagecompress.ic +++ b/storage/xtradb/include/dict0pagecompress.ic @@ -42,8 +42,6 @@ dict_tf_verify_flags( ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); @@ -52,9 +50,6 @@ dict_tf_verify_flags( ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - ulint fsp_page_encryption = FSP_FLAGS_GET_PAGE_ENCRYPTION(fsp_flags); - ulint fsp_page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(fsp_flags); - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", return(ULINT_UNDEFINED);); @@ -112,27 +107,6 @@ dict_tf_verify_flags( return (FALSE); } - if (page_encryption != fsp_page_encryption) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption %ld\n", - page_encryption, fsp_page_encryption); - - return (FALSE); - } - - if (page_encryption_key != fsp_page_encryption_key) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption_key %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption_key %ld\n", - page_encryption_key, fsp_page_encryption_key); - - return (FALSE); - } - - return(TRUE); } diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h new file mode 100644 index 00000000000..9d02034e4b2 --- /dev/null +++ b/storage/xtradb/include/fil0crypt.h @@ -0,0 +1,394 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0crypt.h +The low-level file system encryption support functions + +Created 04/01/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0crypt_h +#define fil0crypt_h + +#define FIL_MSG_CANNOT_DECRYPT "can not decrypt" +#define FIL_ENCRYPTION_WRONG_KEY 1 +#define FIL_ENCRYPTION_WRONG_PAGE_TYPE 2 +#define FIL_ENCRYPTION_ERROR 3 +#define FIL_ENCRYPTION_KEY_MISSING 4 +#define FIL_ENCRYPTION_OK 0 +#define FIL_ENCRYPTION_WILL_NOT_ENCRYPT 5 + +/* This key will be used if nothing else is given */ +#define FIL_DEFAULT_ENCRYPTION_KEY 1 + +/** + * CRYPT_SCHEME_UNENCRYPTED + * + * Used as intermediate state when convering a space from unencrypted + * to encrypted + */ +/** + * CRYPT_SCHEME_1 + * + * AES_CTR / AES_CBC: + * L = AES_ECB(KEY, IV) + * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) + */ + +#define CRYPT_SCHEME_1 1 +#define CRYPT_SCHEME_1_IV_LEN 16 +#define CRYPT_SCHEME_UNENCRYPTED 0 + +/* Currently supported encryption methods */ +typedef enum { + CRYPT_SCHEME_1_UNENCRYPTED = 0, /*<! Space is unencrypted */ + CRYPT_SCHEME_1_CTR = 1, /*<! Space is encrypted with CTR */ + CRYPT_SCHEME_1_CBC = 4, /*<! Space is encrypted with CBC */ +} fil_crypt_method_t; + +/* Cached L or key for given key_version */ +struct key_struct +{ + uint key_version; /*!< Key version used as + identifier */ + byte key[MY_AES_MAX_KEY_LENGTH]; /*!< Cached L or key */ + uint key_length; /*!< Key length */ +}; + +struct fil_space_rotate_state_t +{ + time_t start_time; // time when rotation started + ulint active_threads; // active threads in space + ulint next_offset; // next "free" offset + ulint max_offset; // max offset needing to be rotated + uint min_key_version_found; // min key version found but not rotated + lsn_t end_lsn; // max lsn created when rotating this space + bool starting; // initial write of IV + bool flushing; // space is being flushed at end of rotate + struct { + bool is_active; // is scrubbing active in this space + time_t last_scrub_completed; // when was last scrub completed + } scrubbing; +}; + +struct fil_space_crypt_struct +{ + ulint type; // CRYPT_SCHEME + uint keyserver_requests; // no of key requests to key server + uint key_count; // No of initalized key-structs + key_struct keys[3]; // cached L = AES_ECB(KEY, IV) + uint min_key_version; // min key version for this space + ulint page0_offset; // byte offset on page 0 for crypt data + + ib_mutex_t mutex; // mutex protecting following variables + bool closing; // is tablespace being closed + fil_space_rotate_state_t rotate_state; + + uint iv_length; // length of IV + byte iv[1]; // IV-data +}; + +/* structure containing encryption specification */ +typedef struct fil_space_crypt_struct fil_space_crypt_t; + +/********************************************************************* +Init global resources needed for tablespace encryption/decryption */ +UNIV_INTERN +void +fil_space_crypt_init(); + +/********************************************************************* +Cleanup global resources needed for tablespace encryption/decryption */ +UNIV_INTERN +void +fil_space_crypt_cleanup(); + +/********************************************************************* +Create crypt data, i.e data that is used for a single tablespace */ +UNIV_INTERN +fil_space_crypt_t * +fil_space_create_crypt_data(); + +/********************************************************************* +Destroy crypt data */ +UNIV_INTERN +void +fil_space_destroy_crypt_data( +/*=========================*/ + fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */ + +/********************************************************************* +Get crypt data for a space*/ +UNIV_INTERN +fil_space_crypt_t * +fil_space_get_crypt_data( +/*======================*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Set crypt data for a space*/ +UNIV_INTERN +void +fil_space_set_crypt_data( +/*======================*/ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + +/********************************************************************* +Compare crypt data*/ +UNIV_INTERN +int +fil_space_crypt_compare( +/*======================*/ + const fil_space_crypt_t* crypt_data1, /*!< in: crypt data */ + const fil_space_crypt_t* crypt_data2); /*!< in: crypt data */ + +/********************************************************************* +Read crypt data from buffer page */ +UNIV_INTERN +fil_space_crypt_t * +fil_space_read_crypt_data( +/*======================*/ + ulint space, /*!< in: tablespace id */ + const byte* page, /*!< in: buffer page */ + ulint offset); /*!< in: offset where crypt data is stored */ + +/********************************************************************* +Write crypt data to buffer page */ +UNIV_INTERN +void +fil_space_write_crypt_data( +/*=======================*/ + ulint space, /*!< in: tablespace id */ + byte* page, /*!< in: buffer page */ + ulint offset, /*!< in: offset where to store data */ + ulint maxsize, /*!< in: max space available to store crypt data in */ + mtr_t * mtr); /*!< in: mini-transaction */ + +/********************************************************************* +Clear crypt data from page 0 (used for import tablespace) */ +UNIV_INTERN +void +fil_space_clear_crypt_data( +/*======================*/ + byte* page, /*!< in: buffer page */ + ulint offset); /*!< in: offset where crypt data is stored */ + +/********************************************************************* +Parse crypt data log record */ +UNIV_INTERN +byte* +fil_parse_write_crypt_data( +/*=======================*/ + byte* ptr, /*!< in: start of log record */ + byte* end_ptr, /*!< in: end of log record */ + buf_block_t*); /*!< in: buffer page to apply record to */ + +/********************************************************************* +Check if extra buffer shall be allocated for decrypting after read */ +UNIV_INTERN +bool +fil_space_check_encryption_read( +/*==============================*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Check if page shall be encrypted before write */ +UNIV_INTERN +bool +fil_space_check_encryption_write( +/*==============================*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Encrypt buffer page */ +UNIV_INTERN +void +fil_space_encrypt( +/*===============*/ + ulint space, /*!< in: tablespace id */ + ulint offset, /*!< in: page no */ + lsn_t lsn, /*!< in: page lsn */ + const byte* src_frame,/*!< in: page frame */ + ulint size, /*!< in: size of data to encrypt */ + byte* dst_frame, /*!< in: where to encrypt to */ + ulint page_encryption_key); /*!< in: page encryption key id if page + encrypted */ + +/********************************************************************* +Decrypt buffer page */ +UNIV_INTERN +void +fil_space_decrypt( +/*===============*/ + ulint space, /*!< in: tablespace id */ + const byte* src_frame,/*!< in: page frame */ + ulint page_size, /*!< in: size of data to encrypt */ + byte* dst_frame); /*!< in: where to decrypt to */ + + +/********************************************************************* +Decrypt buffer page +@return true if page was encrypted */ +UNIV_INTERN +bool +fil_space_decrypt( +/*===============*/ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + const byte* src_frame,/*!< in: page frame */ + ulint page_size, /*!< in: page size */ + byte* dst_frame); /*!< in: where to decrypt to */ + +/********************************************************************* +fil_space_verify_crypt_checksum +NOTE: currently this function can only be run in single threaded mode +as it modifies srv_checksum_algorithm (temporarily) +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( +/*===============*/ + const byte* src_frame,/*!< in: page frame */ + ulint zip_size); /*!< in: size of data to encrypt */ + +/********************************************************************* +Init threads for key rotation */ +UNIV_INTERN +void +fil_crypt_threads_init(); + +/********************************************************************* +Set thread count (e.g start or stops threads) used for key rotation */ +UNIV_INTERN +void +fil_crypt_set_thread_cnt( +/*=====================*/ + uint new_cnt); /*!< in: requested #threads */ + +/********************************************************************* +End threads for key rotation */ +UNIV_INTERN +void +fil_crypt_threads_end(); + +/********************************************************************* +Cleanup resources for threads for key rotation */ +UNIV_INTERN +void +fil_crypt_threads_cleanup(); + +/********************************************************************* +Set rotate key age */ +UNIV_INTERN +void +fil_crypt_set_rotate_key_age( +/*=====================*/ + uint rotate_age); /*!< in: requested rotate age */ + +/********************************************************************* +Set rotation threads iops */ +UNIV_INTERN +void +fil_crypt_set_rotation_iops( +/*=====================*/ + uint iops); /*!< in: requested iops */ + +/********************************************************************* +Mark a space as closing */ +UNIV_INTERN +void +fil_space_crypt_mark_space_closing( +/*===============*/ + ulint space); /*!< in: tablespace id */ + +/********************************************************************* +Wait for crypt threads to stop accessing space */ +UNIV_INTERN +void +fil_space_crypt_close_tablespace( +/*===============*/ + ulint space); /*!< in: tablespace id */ + +/** Struct for retreiving info about encryption */ +struct fil_space_crypt_status_t { + ulint space; /*!< tablespace id */ + ulint scheme; /*!< encryption scheme */ + uint min_key_version; /*!< min key version */ + uint current_key_version;/*!< current key version */ + uint keyserver_requests;/*!< no of key requests to key server */ + bool rotating; /*!< is key rotation ongoing */ + bool flushing; /*!< is flush at end of rotation ongoing */ + ulint rotate_next_page_number; /*!< next page if key rotating */ + ulint rotate_max_page_number; /*!< max page if key rotating */ +}; + +/********************************************************************* +Get crypt status for a space +@return 0 if crypt data found */ +UNIV_INTERN +int +fil_space_crypt_get_status( +/*==================*/ + ulint id, /*!< in: space id */ + struct fil_space_crypt_status_t * status); /*!< out: status */ + +/** Struct for retreiving statistics about encryption key rotation */ +struct fil_crypt_stat_t { + ulint pages_read_from_cache; + ulint pages_read_from_disk; + ulint pages_modified; + ulint pages_flushed; + ulint estimated_iops; +}; + +/********************************************************************* +Get crypt rotation statistics */ +UNIV_INTERN +void +fil_crypt_total_stat( +/*==================*/ + fil_crypt_stat_t* stat); /*!< out: crypt stat */ + +/** Struct for retreiving info about scrubbing */ +struct fil_space_scrub_status_t { + ulint space; /*!< tablespace id */ + bool compressed; /*!< is space compressed */ + time_t last_scrub_completed; /*!< when was last scrub completed */ + bool scrubbing; /*!< is scrubbing ongoing */ + time_t current_scrub_started; /*!< when started current scrubbing */ + ulint current_scrub_active_threads; /*!< current scrub active threads */ + ulint current_scrub_page_number; /*!< current scrub page no */ + ulint current_scrub_max_page_number; /*!< current scrub max page no */ +}; + +/********************************************************************* +Get scrub status for a space +@return 0 if no scrub info found */ +UNIV_INTERN +int +fil_space_get_scrub_status( +/*==================*/ + ulint id, /*!< in: space id */ + struct fil_space_scrub_status_t * status); /*!< out: status */ + +#ifndef UNIV_NONINL +#include "fil0crypt.ic" +#endif + +#endif /* fil0crypt_h */ diff --git a/storage/xtradb/include/fil0crypt.ic b/storage/xtradb/include/fil0crypt.ic new file mode 100644 index 00000000000..b753ca5cb4f --- /dev/null +++ b/storage/xtradb/include/fil0crypt.ic @@ -0,0 +1,69 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0fil.h +The low-level file system encryption support functions + +Created 04/01/2015 Jan Lindström +*******************************************************/ + +/*******************************************************************//** +Find out whether the page is page encrypted +@return true if page is page encrypted, false if not */ +UNIV_INLINE +bool +fil_page_is_encrypted( +/*==================*/ + const byte *buf) /*!< in: page */ +{ + return(mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0); +} + +/*******************************************************************//** +Find out whether the page can be decrypted. +The function for decrypting the page should already be executed before this. +@return PAGE_ENCRYPTION_KEY_MISSING if key provider is available, but key is not available + 0 if decryption should be possible +*/ +UNIV_INLINE +ulint +fil_page_encryption_status( +/*===================*/ + const byte *buf, /*!< in: page */ + ulint space_id) /*!< in: space_id */ +{ + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + + if (page_type == FIL_PAGE_TYPE_FSP_HDR) { + fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space_id); + + if (crypt_data != NULL) { + if (!has_encryption_key(crypt_data->keys[0].key_version)) { + /* accessing table would surely fail, because no key or no key provider available */ + return FIL_ENCRYPTION_KEY_MISSING; + } + } + } else { + ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + if (!has_encryption_key(key)) { + return FIL_ENCRYPTION_KEY_MISSING; + } + } + return 0; +} diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 2e92c2b09b2..91653c4bb7d 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -26,7 +26,6 @@ Created 10/25/1995 Heikki Tuuri #ifndef fil0fil_h #define fil0fil_h -#define MSG_CANNOT_DECRYPT "can not decrypt" #include "univ.i" #ifndef UNIV_INNOCHECKSUM @@ -132,24 +131,6 @@ extern fil_addr_t fil_addr_null; used to encrypt the page + 32-bit checksum or 64 bits of zero if no encryption */ -/** If page type is FIL_PAGE_COMPRESSED then the 8 bytes starting at -FIL_PAGE_FILE_FLUSH_LSN are broken down as follows: */ - -/** Control information version format (u8) */ -static const ulint FIL_PAGE_VERSION = FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - -/** Compression algorithm (u8) */ -static const ulint FIL_PAGE_ALGORITHM_V1 = FIL_PAGE_VERSION + 1; - -/** Original page type (u16) */ -static const ulint FIL_PAGE_ORIGINAL_TYPE_V1 = FIL_PAGE_ALGORITHM_V1 + 1; - -/** Original data size in bytes (u16)*/ -static const ulint FIL_PAGE_ORIGINAL_SIZE_V1 = FIL_PAGE_ORIGINAL_TYPE_V1 + 2; - -/** Size after compression (u16)*/ -static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; - #define FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID 34 /*!< starting from 4.1.x this contains the space id of the page */ #define FIL_PAGE_SPACE_ID FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID @@ -169,10 +150,7 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; /* @} */ /** File page types (values of FIL_PAGE_TYPE) @{ */ -#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 35631 /* page compressed + - encrypted page */ #define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< Page compressed page */ -#define FIL_PAGE_PAGE_ENCRYPTED 34355 /*!< Page encrypted page */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ #define FIL_PAGE_INODE 3 /*!< Index node */ @@ -199,6 +177,9 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; #define FIL_LOG 502 /*!< redo log */ /* @} */ +/* structure containing encryption specification */ +typedef struct fil_space_crypt_struct fil_space_crypt_t; + /** The number of fsyncs done to the log */ extern ulint fil_n_log_flushes; @@ -210,9 +191,6 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; -/* structure containing encryption specification */ -typedef struct fil_space_crypt_struct fil_space_crypt_t; - struct fsp_open_info { ibool success; /*!< Has the tablespace been opened? */ const char* check_msg; /*!< fil_check_first_page() message */ @@ -963,9 +941,6 @@ fil_space_get_n_reserved_extents( Reads or writes data. This operation is asynchronous (aio). @return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do i/o on a tablespace which does not exist */ -#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, lsn, encrypt) \ - _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL, lsn, encrypt) - UNIV_INTERN dberr_t _fil_io( @@ -1000,11 +975,14 @@ _fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - trx_t* trx, - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ + trx_t* trx) /*!< in: trx */ __attribute__((nonnull(8))); + +#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size) \ + _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL) + + /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the handler for completed requests. The aio array of pending requests is divided @@ -1298,12 +1276,6 @@ fil_space_t* fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space); /*!< in: space */ /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ @@ -1318,253 +1290,16 @@ ulint fil_get_next_space( ulint id); /*!< in: space id */ -/********************************************************************* -Init global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_init(); - -/********************************************************************* -Cleanup global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_cleanup(); - -/********************************************************************* -Create crypt data, i.e data that is used for a single tablespace */ -fil_space_crypt_t * -fil_space_create_crypt_data(); - -/********************************************************************* -Destroy crypt data */ -UNIV_INTERN -void -fil_space_destroy_crypt_data( -/*=========================*/ - fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */ - -/********************************************************************* -Get crypt data for a space*/ -fil_space_crypt_t * -fil_space_get_crypt_data( -/*======================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Set crypt data for a space*/ -void -fil_space_set_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ - -/********************************************************************* -Compare crypt data*/ -int -fil_space_crypt_compare( -/*======================*/ - const fil_space_crypt_t* crypt_data1, /*!< in: crypt data */ - const fil_space_crypt_t* crypt_data2); /*!< in: crypt data */ - -/********************************************************************* -Read crypt data from buffer page */ -fil_space_crypt_t * -fil_space_read_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - const byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Write crypt data to buffer page */ -void -fil_space_write_crypt_data( -/*=======================*/ - ulint space, /*!< in: tablespace id */ - byte* page, /*!< in: buffer page */ - ulint offset, /*!< in: offset where to store data */ - ulint maxsize, /*!< in: max space available to store crypt data in */ - mtr_t * mtr); /*!< in: mini-transaction */ - -/********************************************************************* -Clear crypt data from page 0 (used for import tablespace) */ -void -fil_space_clear_crypt_data( -/*======================*/ - byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Parse crypt data log record */ -byte* -fil_parse_write_crypt_data( -/*=======================*/ - byte* ptr, /*!< in: start of log record */ - byte* end_ptr, /*!< in: end of log record */ - buf_block_t*); /*!< in: buffer page to apply record to */ - -/********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ -UNIV_INTERN -bool -fil_space_check_encryption_read( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Check if page shall be encrypted before write */ -UNIV_INTERN -bool -fil_space_check_encryption_write( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Encrypt buffer page */ -void -fil_space_encrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - ulint offset, /*!< in: page no */ - lsn_t lsn, /*!< in: page lsn */ - const byte* src_frame,/*!< in: page frame */ - ulint size, /*!< in: size of data to encrypt */ - byte* dst_frame, /*!< in: where to encrypt to */ - ulint page_encryption_key); /*!< in: page encryption key id if page - encrypted */ - -/********************************************************************* -Decrypt buffer page */ -void -fil_space_decrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: size of data to encrypt */ - byte* dst_frame); /*!< in: where to decrypt to */ - - -/********************************************************************* -Decrypt buffer page -@return true if page was encrypted */ -bool -fil_space_decrypt( -/*===============*/ - fil_space_crypt_t* crypt_data, /*!< in: crypt data */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: page size */ - byte* dst_frame); /*!< in: where to decrypt to */ - -/********************************************************************* -fil_space_verify_crypt_checksum -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) -@return true if page is encrypted AND OK, false otherwise */ -bool -fil_space_verify_crypt_checksum( -/*===============*/ - const byte* src_frame,/*!< in: page frame */ - ulint zip_size); /*!< in: size of data to encrypt */ - -/********************************************************************* -Init threads for key rotation */ -void -fil_crypt_threads_init(); - -/********************************************************************* -Set thread count (e.g start or stops threads) used for key rotation */ -void -fil_crypt_set_thread_cnt( -/*=====================*/ - uint new_cnt); /*!< in: requested #threads */ - -/********************************************************************* -End threads for key rotation */ -void -fil_crypt_threads_end(); - -/********************************************************************* -Cleanup resources for threads for key rotation */ -void -fil_crypt_threads_cleanup(); - -/********************************************************************* -Set rotate key age */ -void -fil_crypt_set_rotate_key_age( -/*=====================*/ - uint rotate_age); /*!< in: requested rotate age */ - -/********************************************************************* -Set rotation threads iops */ -void -fil_crypt_set_rotation_iops( -/*=====================*/ - uint iops); /*!< in: requested iops */ - -/********************************************************************* -Mark a space as closing */ -UNIV_INTERN -void -fil_space_crypt_mark_space_closing( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Wait for crypt threads to stop accessing space */ -UNIV_INTERN -void -fil_space_crypt_close_tablespace( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/** Struct for retreiving info about encryption */ -struct fil_space_crypt_status_t { - ulint space; /*!< tablespace id */ - ulint scheme; /*!< encryption scheme */ - uint min_key_version; /*!< min key version */ - uint current_key_version;/*!< current key version */ - uint keyserver_requests;/*!< no of key requests to key server */ - bool rotating; /*!< is key rotation ongoing */ - bool flushing; /*!< is flush at end of rotation ongoing */ - ulint rotate_next_page_number; /*!< next page if key rotating */ - ulint rotate_max_page_number; /*!< max page if key rotating */ -}; - -/********************************************************************* -Get crypt status for a space -@return 0 if crypt data found */ -int -fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t * status); /*!< out: status */ - -/** Struct for retreiving statistics about encryption key rotation */ -struct fil_crypt_stat_t { - ulint pages_read_from_cache; - ulint pages_read_from_disk; - ulint pages_modified; - ulint pages_flushed; - ulint estimated_iops; -}; - -/********************************************************************* -Get crypt rotation statistics */ -void -fil_crypt_total_stat( -/*==================*/ - fil_crypt_stat_t* stat); /*!< out: crypt stat */ - #endif /*******************************************************************//** Return space flags */ +UNIV_INLINE ulint fil_space_flags( /*===========*/ fil_space_t* space); /*!< in: space */ - - /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ @@ -1580,30 +1315,14 @@ os_file_handle_error_no_exit( /*******************************************************************//** Return page type name */ +UNIV_INLINE const char* fil_get_page_type_name( /*===================*/ ulint page_type); /*!< in: FIL_PAGE_TYPE */ -/** Struct for retreiving info about scrubbing */ -struct fil_space_scrub_status_t { - ulint space; /*!< tablespace id */ - bool compressed; /*!< is space compressed */ - time_t last_scrub_completed; /*!< when was last scrub completed */ - bool scrubbing; /*!< is scrubbing ongoing */ - time_t current_scrub_started; /*!< when started current scrubbing */ - ulint current_scrub_active_threads; /*!< current scrub active threads */ - ulint current_scrub_page_number; /*!< current scrub page no */ - ulint current_scrub_max_page_number; /*!< current scrub max page no */ -}; - -/********************************************************************* -Get scrub status for a space -@return 0 if no scrub info found */ -int -fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t * status); /*!< out: status */ +#ifndef UNIV_NONINL +#include "fil0fil.ic" +#endif #endif /* fil0fil_h */ diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic new file mode 100644 index 00000000000..b1e65e6dddb --- /dev/null +++ b/storage/xtradb/include/fil0fil.ic @@ -0,0 +1,108 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0fil.ic +The low-level file system support functions + +Created 31/03/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0fil_ic +#define fil0fil_ic + +/*******************************************************************//** +Return space name */ +UNIV_INLINE +char* +fil_space_name( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->name); +} + +/*******************************************************************//** +Return space flags */ +UNIV_INLINE +ulint +fil_space_flags( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->flags); +} + +/*******************************************************************//** +Return page type name */ +UNIV_INLINE +const char* +fil_get_page_type_name( +/*===================*/ + ulint page_type) /*!< in: FIL_PAGE_TYPE */ +{ + switch(page_type) { + case FIL_PAGE_PAGE_COMPRESSED: + return (const char*)"PAGE_COMPRESSED"; + case FIL_PAGE_INDEX: + return (const char*)"INDEX"; + case FIL_PAGE_UNDO_LOG: + return (const char*)"UNDO LOG"; + case FIL_PAGE_INODE: + return (const char*)"INODE"; + case FIL_PAGE_IBUF_FREE_LIST: + return (const char*)"IBUF_FREE_LIST"; + case FIL_PAGE_TYPE_ALLOCATED: + return (const char*)"ALLOCATED"; + case FIL_PAGE_IBUF_BITMAP: + return (const char*)"IBUF_BITMAP"; + case FIL_PAGE_TYPE_SYS: + return (const char*)"SYS"; + case FIL_PAGE_TYPE_TRX_SYS: + return (const char*)"TRX_SYS"; + case FIL_PAGE_TYPE_FSP_HDR: + return (const char*)"FSP_HDR"; + case FIL_PAGE_TYPE_XDES: + return (const char*)"XDES"; + case FIL_PAGE_TYPE_BLOB: + return (const char*)"BLOB"; + case FIL_PAGE_TYPE_ZBLOB: + return (const char*)"ZBLOB"; + case FIL_PAGE_TYPE_ZBLOB2: + return (const char*)"ZBLOB2"; + case FIL_PAGE_TYPE_COMPRESSED: + return (const char*)"ORACLE PAGE COMPRESSED"; + default: + return (const char*)"PAGE TYPE CORRUPTED"; + } +} + +/****************************************************************//** +Get block size from fil node +@return block size*/ +UNIV_INLINE +ulint +fil_node_get_block_size( +/*====================*/ + fil_node_t* node) /*!< in: Node where to get block + size */ +{ + return (node->file_block_size); +} + +#endif /* fil0fil_ic */ diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h deleted file mode 100644 index fa2b1a5c592..00000000000 --- a/storage/xtradb/include/fil0pageencryption.h +++ /dev/null @@ -1,79 +0,0 @@ -/***************************************************************************** - -Copyright (C) 2014 eperi GmbH. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ - -#ifndef fil0pageencryption_h -#define fil0pageencryption_h - -#define PAGE_ENCRYPTION_WRONG_KEY 1 -#define PAGE_ENCRYPTION_WRONG_PAGE_TYPE 2 -#define PAGE_ENCRYPTION_ERROR 3 -#define PAGE_ENCRYPTION_KEY_MISSING 4 -#define PAGE_ENCRYPTION_OK 0 -#define PAGE_ENCRYPTION_WILL_NOT_ENCRYPT 5 - -/* This key will be used if nothing else is given */ -#define DEFAULT_ENCRYPTION_KEY 1 - -#include "fsp0fsp.h" -#include "fsp0pageencryption.h" - -/******************************************************************//** -@file include/fil0pageencryption.h -Helper functions for encryption/decryption page data on to table space. - -Created 08/25/2014 -***********************************************************************/ - -/*******************************************************************//** -Find out whether the page is page encrypted. -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -ibool -fil_space_is_page_encrypted( -/*========================*/ - ulint id); /*!< in: space id */ - -/*******************************************************************//** -Find out whether the page is page encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*==================*/ - const byte *buf); /*!< in: page */ -/*******************************************************************//** -Find out whether the page is page compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf); /*!< in: page */ - -/*******************************************************************//** -Find out whether the page can be decrypted -@return true if page can be decrypted, false if not. */ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*=======================*/ - const byte *buf); /*!< in: page */ - - -#endif // fil0pageencryption_h diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 57e5b9490b1..3d3d482a6c0 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -58,10 +58,6 @@ is found in a remote location, not the default data directory. */ #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION 1 -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY 8 - /** Number of flag bits used to indicate atomic writes for this tablespace */ #define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 @@ -73,9 +69,7 @@ is found in a remote location, not the default data directory. */ + FSP_FLAGS_WIDTH_DATA_DIR \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) + + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) /** A mask of all the known/used bits in tablespace flags */ #define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) @@ -99,15 +93,9 @@ dictionary */ /** Zero relative shift position of the ATOMIC_WRITES field */ #define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY (FSP_FLAGS_POS_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION) - /** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) /** Zero relative shift position of the start of the DATA DIR bits */ #define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) @@ -143,15 +131,6 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Bit mask of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) - /** Bit mask of the ATOMIC_WRITES field */ #define FSP_FLAGS_MASK_ATOMIC_WRITES \ ((~(~0 << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ @@ -192,14 +171,6 @@ dictionary */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ >> FSP_FLAGS_POS_ATOMIC_WRITES) -/** Return the value of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Return the value of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) /** Set a PAGE_SSIZE into the correct bits in a given tablespace flags. */ @@ -216,13 +187,6 @@ tablespace flags. */ #define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) -/** Set a PAGE_ENCRYPTION into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION(flags, encryption) \ - (flags | (encryption << FSP_FLAGS_POS_PAGE_ENCRYPTION)) -/** Set a PAGE_ENCRYPTION_KEY into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(flags, encryption_key) \ - (flags | (encryption_key << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY)) - /** Set a ATOMIC_WRITES into the correct bits in a given tablespace flags. */ #define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ diff --git a/storage/xtradb/include/fsp0pageencryption.h b/storage/xtradb/include/fsp0pageencryption.h deleted file mode 100644 index 631aa72211c..00000000000 --- a/storage/xtradb/include/fsp0pageencryption.h +++ /dev/null @@ -1,66 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************/ - -/******************************************************************//** -@file include/fsp0pageencryption.h -Helper functions for extracting/storing page encryption information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#ifndef FSP0PAGEENCRYPTION_H_ -#define FSP0PAGEENCRYPTION_H_ - -#define FIL_PAGE_ENCRYPTION_AES_128 16 /*!< Encryption algorithm AES-128. */ -#define FIL_PAGE_ENCRYPTION_AES_196 24 /*!< Encryption algorithm AES-196. */ -#define FIL_PAGE_ENCRYPTION_AES_256 32 /*!< Encryption algorithm AES-256. */ - -#define FIL_PAGE_ENCRYPTED_SIZE 2 /*!< Number of bytes used to store - actual payload data size onencrypted - pages. */ - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags); /*!< in: tablespace flags */ - - -#ifndef UNIV_NONINL -#include "fsp0pageencryption.ic" -#endif - - -#endif /* FSP0PAGEENCRYPTION_H_ */ diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic deleted file mode 100644 index d3137001fc5..00000000000 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ /dev/null @@ -1,168 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************//** -@file include/fsp0pageencryption.ic -Implementation for helper functions for encrypting/decrypting pages -and atomic writes information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#include "fsp0fsp.h" -#include "fil0pageencryption.h" - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION(flags)); -} - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags)); -} - - -/*******************************************************************//** -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -UNIV_INLINE -ibool -fil_space_is_page_encrypted( -/*=========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_encrypted(flags)); - } - - return(flags); -} - -/*******************************************************************//** -Returns the page encryption key of the space, or 0 if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return page compression level, ULINT_UNDEFINED if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_encryption_key( -/*=================================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_get_page_encryption_key(flags)); - } - - return(flags); -} - - - -/*******************************************************************//** -Find out whether the page is page encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*===================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page is page is first compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page can be decrypted. -This is the case, if the page is already decrypted and is not the first page of the table space. -If the page is already decrypted it is not of the FIL_PAGE_PAGE_ENCRYPTED type. -if it is the first page of the table space, it is assumed that a page can be decrypted if the -key found in the flags (part of the 1st page) can be read from the key provider. -The case, if the key changed, is currently not caught. -The function for decrypting the page should already be executed before this. -@return PAGE_ENCRYPTION_KEY_MISSING if key provider is available, but key is not available - PAGE_ENCRYPTION_ERROR if other error occurred - 0 if decryption should be possible -*/ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*===================*/ - const byte *buf) /*!< in: page */ -{ - ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (page_type == FIL_PAGE_TYPE_FSP_HDR) { - ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* accessing table would surely fail, because no key or no key provider available */ - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - } - } - - if(page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - if (!has_encryption_key(key)) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - return 0; -} diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index f93bf026903..154adf085b8 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -323,24 +323,21 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, \ - trx, page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + trx, write_size) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ - n, message1, message2, space_id, trx, \ - page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt, \ + n, message1, message2, space_id, trx, write_size, \ __FILE__, __LINE__) -# define os_file_read(file, buf, offset, n, compressed) \ - pfs_os_file_read_func(file, buf, offset, n, NULL, compressed, \ +# define os_file_read(file, buf, offset, n) \ + pfs_os_file_read_func(file, buf, offset, n, NULL, \ __FILE__, __LINE__) -# define os_file_read_trx(file, buf, offset, n, trx, compressed) \ - pfs_os_file_read_func(file, buf, offset, n, trx, compressed, \ +# define os_file_read_trx(file, buf, offset, n, trx) \ + pfs_os_file_read_func(file, buf, offset, n, trx, \ __FILE__, __LINE__) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ - pfs_os_file_read_no_error_handling_func(file, buf, offset, n, compressed, \ +# define os_file_read_no_error_handling(file, buf, offset, n) \ + pfs_os_file_read_no_error_handling_func(file, buf, offset, n, \ __FILE__, __LINE__) # define os_file_write(name, file, buf, offset, n) \ @@ -376,22 +373,18 @@ to original un-instrumented file I/O APIs */ # define os_file_close(file) os_file_close_func(file) # define os_aio(type, mode, name, file, buf, offset, n, message1, \ - message2, space_id, trx, \ - page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + message2, space_id, trx, write_size) \ os_aio_func(type, mode, name, file, buf, offset, n, \ - message1, message2, space_id, trx, \ - page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt) + message1, message2, space_id, trx, write_size) -# define os_file_read(file, buf, offset, n, compressed) \ - os_file_read_func(file, buf, offset, n, NULL, compressed) +# define os_file_read(file, buf, offset, n) \ + os_file_read_func(file, buf, offset, n, NULL) -# define os_file_read_trx(file, buf, offset, n, trx, compressed) \ - os_file_read_func(file, buf, offset, n, trx, compressed) +# define os_file_read_trx(file, buf, offset, n, trx) \ + os_file_read_func(file, buf, offset, n, trx) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ - os_file_read_no_error_handling_func(file, buf, offset, n, compressed) +# define os_file_read_no_error_handling(file, buf, offset, n) \ + os_file_read_no_error_handling_func(file, buf, offset, n) # define os_file_write(name, file, buf, offset, n) \ os_file_write_func(name, file, buf, offset, n) @@ -747,8 +740,6 @@ pfs_os_file_read_func( os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ trx_t* trx, /*!< in: trx */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -767,8 +758,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -801,21 +790,11 @@ pfs_os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -979,9 +958,7 @@ os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - trx_t* trx, /*!< in: trx */ - ibool compressed); /*!< in: is this file space - compressed ? */ + trx_t* trx); /*!< in: trx */ /*******************************************************************//** Rewind file to its start, read at most size - 1 bytes from it to str, and NUL-terminate str. All errors are silently ignored. This function is @@ -1006,9 +983,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed); /*!< in: is this file space - compressed ? */ + ulint n); /*!< in: number of bytes to read */ /*******************************************************************//** NOTE! Use the corresponding macro os_file_write(), not directly this @@ -1194,21 +1169,11 @@ os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size);/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/xtradb/include/os0file.ic b/storage/xtradb/include/os0file.ic index 39599ba2047..b25f1577362 100644 --- a/storage/xtradb/include/os0file.ic +++ b/storage/xtradb/include/os0file.ic @@ -220,23 +220,11 @@ pfs_os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< in: page compression - level to be used */ ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - - lsn_t lsn, /*!< in: lsn of the newest - modification */ - bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -253,8 +241,7 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, space_id, trx, - page_compression, page_compression_level, write_size , - page_encryption, page_encryption_key, lsn, encrypt_later); + write_size); register_pfs_file_io_end(locker, n); @@ -276,8 +263,6 @@ pfs_os_file_read_func( os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ trx_t* trx, - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -288,7 +273,7 @@ pfs_os_file_read_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_func(file, buf, offset, n, trx, compressed); + result = os_file_read_func(file, buf, offset, n, trx); register_pfs_file_io_end(locker, n); @@ -311,8 +296,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -323,7 +306,7 @@ pfs_os_file_read_no_error_handling_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_no_error_handling_func(file, buf, offset, n, compressed); + result = os_file_read_no_error_handling_func(file, buf, offset, n); register_pfs_file_io_end(locker, n); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 37cf18c1a0d..9b33c7858bc 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1384,7 +1384,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0, false); + buf, group, 0); srv_stats.os_log_pending_writes.dec(); } @@ -1551,7 +1551,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0, false); + group, 0); srv_stats.os_log_pending_writes.dec(); @@ -2144,7 +2144,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0, false); + buf, ((byte*) group + 1), 0); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2226,7 +2226,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0); } /******************************************************//** @@ -2639,7 +2639,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0, 0, false); + len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) @@ -2771,7 +2771,7 @@ log_group_archive_file_header_write( dest_offset / UNIV_PAGE_SIZE, dest_offset % UNIV_PAGE_SIZE, 2 * OS_FILE_LOG_BLOCK_SIZE, - buf, &log_archive_io, 0, 0, false); + buf, &log_archive_io, 0); } /******************************************************//** @@ -2808,7 +2808,7 @@ log_group_archive_completed_header_write( dest_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, buf + LOG_FILE_ARCH_COMPLETED, - &log_archive_io, 0, 0, false); + &log_archive_io, 0); } /******************************************************//** @@ -2953,7 +2953,7 @@ loop: (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), ut_calc_align(len, OS_FILE_LOG_BLOCK_SIZE), buf, - &log_archive_io, 0, 0, false); + &log_archive_io, 0); start_lsn += len; next_offset += len; diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index 60ae4a50626..e18677973b0 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -283,7 +283,7 @@ log_online_read_bitmap_page( ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); success = os_file_read(bitmap_file->file, page, bitmap_file->offset, - MODIFIED_PAGE_BLOCK_SIZE, FALSE); + MODIFIED_PAGE_BLOCK_SIZE); if (UNIV_UNLIKELY(!success)) { diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index c7fb3be9625..46f5954d44f 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -56,6 +56,7 @@ Created 9/20/1997 Heikki Tuuri #include "trx0undo.h" #include "trx0rec.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP # include "buf0rea.h" # include "srv0srv.h" @@ -3177,7 +3178,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3208,7 +3209,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); } log_hdr_log_block_size @@ -3871,7 +3872,7 @@ ask_again: /* Read the archive file header */ fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, 0, 0, - LOG_FILE_HDR_SIZE, buf, NULL, 0, 0, false); + LOG_FILE_HDR_SIZE, buf, NULL, 0); /* Check if the archive file header is consistent */ @@ -3945,7 +3946,7 @@ ask_again: fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, read_offset / UNIV_PAGE_SIZE, - read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0, 0, false); + read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0); ret = recv_scan_log_recs( (buf_pool_get_n_pages() diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index d7d224dc6dd..75750df5639 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -45,7 +45,6 @@ Created 10/21/1995 Heikki Tuuri #include "fil0fil.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "buf0buf.h" #include "btr0types.h" #include "trx0trx.h" @@ -238,21 +237,6 @@ struct os_aio_slot_t{ completed */ ulint bitmap; - byte* page_compression_page; /*!< Memory allocated for - page compressed page and - freed after the write - has been completed */ - - byte* page_encryption_page; /*!< Memory allocated for - page encrypted page and - freed after the write - has been completed */ - - ibool page_compression; - ulint page_compression_level; - - ibool page_encryption; - ulint page_encryption_key; ulint* write_size; /*!< Actual write size initialized after fist successfull trim @@ -260,31 +244,13 @@ struct os_aio_slot_t{ initialized we do not trim again if actual page size does not decrease. */ - byte* page_buf; /*!< Actual page buffer for - page compressed pages, do not - free this */ - - byte* page_buf2; /*!< Actual page buffer for - page encrypted pages, do not - free this */ - byte* tmp_encryption_buf; /*!< a temporal buffer used by page encryption */ - - ibool page_compression_success; - /*!< TRUE if page compression was successfull, false if not */ - ibool page_encryption_success; - /*!< TRUE if page encryption was successfull, false if not */ - - lsn_t lsn; /* lsn of the newest modification */ - ulint file_block_size;/*!< file block size */ - bool encrypt_later; /*!< should we encrypt the page */ #ifdef LINUX_NATIVE_AIO struct iocb control; /* Linux control block for aio */ int n_bytes; /* bytes written/read. */ int ret; /* AIO return code */ #endif /* WIN_ASYNC_IO */ - byte *lzo_mem; /* Temporal memory used by LZO */ }; /** The asynchronous i/o array structure */ @@ -401,39 +367,6 @@ os_file_trim( /*=========*/ os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( - os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( - os_aio_slot_t* slot); /*!< in: slot structure */ /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ @@ -3114,9 +3047,7 @@ os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - trx_t* trx, - ibool compressed) /*!< in: is this file space - compressed ? */ + trx_t* trx) { #ifdef __WIN__ BOOL ret; @@ -3212,9 +3143,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed) /*!< in: is this file space - compressed ? */ + ulint n) /*!< in: number of bytes to read */ { #ifdef __WIN__ BOOL ret; @@ -4275,8 +4204,6 @@ os_aio_array_free( /*==============*/ os_aio_array_t*& array) /*!< in, own: array to free */ { - ulint i; - os_mutex_free(array->mutex); os_event_free(array->not_full); os_event_free(array->is_empty); @@ -4288,31 +4215,6 @@ os_aio_array_free( } #endif /* LINUX_NATIVE_AIO */ - for (i = 0; i < array->n_slots; i++) { - os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); - - if (slot->page_compression_page) { - ut_free(slot->page_compression_page); - slot->page_compression_page = NULL; - } - - if (slot->lzo_mem) { - ut_free(slot->lzo_mem); - slot->lzo_mem = NULL; - } - - if (slot->page_encryption_page) { - ut_free(slot->page_encryption_page); - slot->page_encryption_page = NULL; - } - - if (slot->tmp_encryption_buf) { - ut_free(slot->tmp_encryption_buf); - slot->tmp_encryption_buf = NULL; - } - } - - ut_free(array->slots); ut_free(array); @@ -4658,22 +4560,11 @@ os_aio_array_reserve_slot( os_offset_t offset, /*!< in: file offset */ ulint len, /*!< in: length of the block to read or write */ ulint space_id, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< page encryption key - to be used */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after first successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest - modification */ - bool encrypt_later) /*!< in: should we encrypt the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4762,96 +4653,13 @@ found: slot->type = type; slot->buf = static_cast<byte*>(buf); slot->offset = offset; - slot->lsn = lsn; slot->io_already_done = FALSE; slot->space_id = space_id; - slot->page_compression_success = FALSE; - slot->page_encryption_success = FALSE; - slot->write_size = write_size; - slot->page_compression_level = page_compression_level; - slot->page_compression = page_compression; - slot->page_encryption_key = page_encryption_key; - slot->page_encryption = page_encryption; - slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); } - - /* If the space is page compressed and this is write operation - then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_compression) { - ulint real_len = len; - byte* tmp = NULL; - - /* Release the array mutex while encrypting */ - os_mutex_exit(array->mutex); - - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); - -#ifdef HAVE_LZO - if (innodb_compression_algorithm == 3) { - os_slot_alloc_lzo_mem(slot); - } -#endif - - /* Call page compression */ - tmp = fil_compress_page( - fil_node_get_space_id(slot->message1), - (byte *)buf, - slot->page_buf, - len, - page_compression_level, - fil_node_get_block_size(slot->message1), - &real_len, - slot->lzo_mem - ); - - /* If compression succeeded, set up the length and buffer */ - if (tmp != buf) { - len = real_len; - buf = slot->page_buf; - slot->len = real_len; - slot->page_compression_success = TRUE; - } else { - slot->page_compression_success = FALSE; - } - - /* Take array mutex back, not sure if this is really needed - below */ - os_mutex_enter(array->mutex); - - } - - /* If the space is page encryption and this is write operation - then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && (page_encryption || encrypt_later)) { - /* Release the array mutex while encrypting */ - os_mutex_exit(array->mutex); - - // We allocate memory for page encrypted buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf2(slot); - - fil_space_encrypt( - fil_node_get_space_id(slot->message1), - slot->offset, - slot->lsn, - (byte *)buf, - slot->len, - slot->page_buf2, - slot->page_encryption_key); - - slot->page_encryption_success = TRUE; - buf = slot->page_buf2; - - /* Take array mutex back */ - os_mutex_enter(array->mutex); - } - slot->buf = (byte *)buf; #ifdef WIN_ASYNC_IO @@ -5130,22 +4938,11 @@ os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt before - writing the page */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5178,8 +4975,7 @@ os_aio_func( no need to use an i/o-handler thread */ if (type == OS_FILE_READ) { - ret = os_file_read_func(file, buf, offset, n, trx, - page_compression); + ret = os_file_read_func(file, buf, offset, n, trx); } else { ut_ad(!srv_read_only_mode); ut_a(type == OS_FILE_WRITE); @@ -5247,9 +5043,7 @@ try_again: slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, space_id, - page_compression, page_compression_level, - page_encryption, page_encryption_key, - write_size, lsn, encrypt_later); + write_size); if (type == OS_FILE_READ) { if (srv_use_native_aio) { @@ -5278,15 +5072,8 @@ try_again: if (srv_use_native_aio) { os_n_file_writes++; #ifdef WIN_ASYNC_IO - if (page_encryption && slot->page_encryption_success) { - buffer = slot->page_buf2; - n = slot->len; - } else if (page_compression && slot->page_compression_success) { - buffer = slot->page_buf; - n = slot->len; - } else { - buffer = buf; - } + n = slot->len; + buffer = buf; ret = WriteFile(file, buffer, (DWORD) n, &len, &(slot->control)); @@ -5445,22 +5232,12 @@ os_aio_windows_handle( switch (slot->type) { case OS_FILE_WRITE: - if (slot->message1 && slot->page_encryption && slot->page_encryption_success) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf2, - slot->offset, slot->len); - } else { - if (slot->message1 && slot->page_compression && slot->page_compression_success) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf, + ret_val = os_file_write(slot->name, slot->file, slot->buf, slot->offset, slot->len); - } else { - ret_val = os_file_write(slot->name, slot->file, slot->buf, - slot->offset, slot->len); - } - } break; case OS_FILE_READ: ret_val = os_file_read(slot->file, slot->buf, - slot->offset, slot->len, slot->page_compression); + slot->offset, slot->len); break; default: ut_error; @@ -5485,45 +5262,10 @@ os_aio_windows_handle( ret_val = ret && len == slot->len; } - if (slot->type == OS_FILE_READ) { - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - ut_ad(slot->message1 != NULL); - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - - // Decrypt the data - fil_space_decrypt( - fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - - if (fil_page_is_compressed(slot->buf)) { - /* We allocate memory for page compressed buffer if - and only if it is not yet allocated. */ - os_slot_alloc_page_buf(slot); - -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - fil_decompress_page(slot->page_buf, slot->buf, - slot->len, slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } + if (slot->type == OS_FILE_WRITE) { + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } } @@ -5616,48 +5358,10 @@ retry: /* We have not overstepped to next segment. */ ut_a(slot->pos < end_pos); - if (slot->type == OS_FILE_READ) { - /* If the page is page encrypted we decrypt */ - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - ut_ad(slot->message1 != NULL); - - // Decrypt the data - fil_space_decrypt(fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - - /* If the table is page compressed and this - is read, we decompress before we announce - the read is complete. For writes, we free - the compressed page. */ - if (fil_page_is_compressed(slot->buf)) { - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - ut_ad(slot->page_compression_page); - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } + if (slot->type == OS_FILE_WRITE) { + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } } @@ -6140,8 +5844,7 @@ consecutive_loop: } else { ret = os_file_read( aio_slot->file, combined_buf, - aio_slot->offset, total_len, - aio_slot->page_compression); + aio_slot->offset, total_len); } srv_set_io_thread_op_info(global_segment, "file i/o done"); @@ -6714,91 +6417,6 @@ os_file_trim( } -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - - if(slot->page_buf2 == NULL) { - byte* cbuf2; - byte* cbuf; - - cbuf2 = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); - cbuf = static_cast<byte *>(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_encryption_page = static_cast<byte *>(cbuf2); - slot->page_buf2 = static_cast<byte *>(cbuf); - memset(slot->page_encryption_page, 0, UNIV_PAGE_SIZE*2); - } -} - -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->page_buf == NULL) { - byte* cbuf2; - byte* cbuf; - ulint asize = UNIV_PAGE_SIZE; -#ifdef HAVE_SNAPPY - asize += snappy_max_compressed_length(asize) - UNIV_PAGE_SIZE; -#endif - /* We allocate extra to avoid memory overwrite on - compression */ - cbuf2 = static_cast<byte *>(ut_malloc(asize*2)); - cbuf = static_cast<byte *>(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_compression_page = static_cast<byte *>(cbuf2); - slot->page_buf = static_cast<byte *>(cbuf); - ut_a(slot->page_buf != NULL); - memset(slot->page_compression_page, 0, asize*2); - } -} - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if(slot->lzo_mem == NULL) { - slot->lzo_mem = static_cast<byte *>(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); - ut_a(slot->lzo_mem != NULL); - memset(slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); - } -} -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( -/*=============================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->tmp_encryption_buf == NULL) { - slot->tmp_encryption_buf = static_cast<byte *>(ut_malloc(64)); - memset(slot->tmp_encryption_buf, 0, 64); - } -} - /***********************************************************************//** Try to get number of bytes per sector from file system. @return file block size */ diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index 5a086ccebb4..ac0e75929a4 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -2552,7 +2552,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" @@ -3380,7 +3380,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 61494bed68a..fc08a4c6f07 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -786,7 +786,7 @@ row_merge_read( #endif /* UNIV_DEBUG */ success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf, - ofs, srv_sort_buf_size, FALSE); + ofs, srv_sort_buf_size); #ifdef POSIX_FADV_DONTNEED /* Each block is read exactly once. Free up the file cache. */ diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 5b2b9ac1503..5929bb21f7b 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -57,6 +57,7 @@ Created 9/17/2000 Heikki Tuuri #include "btr0sea.h" #include "btr0defragment.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "ibuf0ibuf.h" #include "fts0fts.h" #include "fts0types.h" diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 37adca6975b..9396a713cad 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -75,10 +75,10 @@ Created 10/8/1995 Heikki Tuuri #include "mysql/plugin.h" #include "mysql/service_thd_wait.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fil0pagecompress.h" #include <my_rdtsc.h> #include "btr0scrub.h" -#include "fil0pageencryption.h" /* prototypes of new functions added to ha_innodb.cc for kill_idle_transaction */ ibool innobase_thd_is_idle(const void* thd); @@ -671,7 +671,7 @@ second. */ static time_t srv_last_log_flush_time; /** Default encryption key used for page encryption */ -UNIV_INTERN uint srv_default_page_encryption_key = DEFAULT_ENCRYPTION_KEY; +UNIV_INTERN uint srv_default_page_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; /** Enable semaphore request instrumentation */ UNIV_INTERN my_bool srv_instrument_semaphores = FALSE; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index ddf261cc236..4d2aee6f08c 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2014, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -52,6 +52,7 @@ Created 2/16/1996 Heikki Tuuri #include "os0file.h" #include "os0thread.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fsp0fsp.h" #include "rem0rec.h" #include "mtr0mtr.h" |