MDEV-17958 Keep big endian variant of innodb_checksum_algorithm=crc32 only in big-endian machine10.2-mdev-23578

Problem:
========
In MySQL 5.7, it was noticed that files are not portable between
big-endian and little-endian systems (such as SPARC and x86),
because the original implementation of innodb_checksum_algorithm=crc32
was not byte order agnostic. Basically, it weakened the
innodb_checksum_algorithm for crc32 algorithm for mariadb-10.2

Fix:
====
Keep the bug-compatible variant for crc32 algorithm only present
on big-endian systems (#ifdef WORDS_BIGENDIAN).

9 files changed, 41 insertions, 110 deletions

diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index 626ba8259e0..b0593a1ffc0 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -756,17 +756,14 @@ buf_page_is_zeroes(
 @param[in]	read_buf		database page
 @param[in]	checksum_field1		new checksum field
 @param[in]	checksum_field2		old checksum field
-@param[in]	use_legacy_big_endian   use legacy big endian algorithm
 @return true if the page is in crc32 checksum format. */
 bool
 buf_page_is_checksum_valid_crc32(
 	const byte*			read_buf,
 	ulint				checksum_field1,
-	ulint				checksum_field2,
-	bool				use_legacy_big_endian)
+	ulint				checksum_field2)
 {
-	const uint32_t	crc32 = buf_calc_page_crc32(read_buf,
-						    use_legacy_big_endian);
+	const uint32_t	crc32 = buf_calc_page_crc32(read_buf);
 
 #ifdef UNIV_INNOCHECKSUM
 	if (log_file
@@ -1081,19 +1078,11 @@ buf_page_is_corrupted(
 	const srv_checksum_algorithm_t	curr_algo =
 		static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
 
-	bool	legacy_checksum_checked = false;
-
 	switch (curr_algo) {
 	case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
 
 		if (buf_page_is_checksum_valid_crc32(read_buf,
-			checksum_field1, checksum_field2, false)) {
-			return false;
-		}
-
-		if (buf_page_is_checksum_valid_crc32(read_buf,
-			checksum_field1, checksum_field2, true)) {
-			legacy_big_endian_checksum = true;
+			checksum_field1, checksum_field2)) {
 			return false;
 		}
 
@@ -1140,24 +1129,10 @@ buf_page_is_corrupted(
 			if (srv_checksum_algorithm
 			    == SRV_CHECKSUM_ALGORITHM_CRC32) {
 
-				if (legacy_big_endian_checksum) {
-					crc32 = buf_calc_page_crc32(read_buf, true);
-					legacy_checksum_checked = true;
-				} else {
-					crc32 = buf_calc_page_crc32(read_buf, false);
-				}
-
+				crc32 = buf_calc_page_crc32(read_buf);
 				crc32_inited = true;
 
 				if (checksum_field2 != crc32
-				    && !legacy_checksum_checked) {
-					crc32 = buf_calc_page_crc32(read_buf, true);
-					if (checksum_field2 == crc32) {
-						legacy_big_endian_checksum = true;
-					}
-				}
-
-				if (checksum_field2 != crc32
 				    && checksum_field2
 				       != buf_calc_page_old_checksum(read_buf)) {
 					return true;
@@ -1186,8 +1161,7 @@ buf_page_is_corrupted(
 
 				if (!crc32_inited) {
 					crc32 = buf_calc_page_crc32(
-						read_buf,
-						legacy_big_endian_checksum);
+						read_buf);
 					crc32_inited = true;
 				}
 
@@ -1205,7 +1179,7 @@ buf_page_is_corrupted(
 
 					if (!crc32_inited) {
 						crc32 = buf_calc_page_crc32(
-							read_buf, false);
+							read_buf);
 						crc32_inited = true;
 					}
 
@@ -1262,10 +1236,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< page_zip_calc_checksum(
 				read_buf, page_size.physical(),
 				SRV_CHECKSUM_ALGORITHM_CRC32)
-			<< "/"
-			<< page_zip_calc_checksum(
-				read_buf, page_size.physical(),
-				SRV_CHECKSUM_ALGORITHM_CRC32, true)
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB)
@@ -1292,8 +1262,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 	} else {
 		const uint32_t	crc32 = buf_calc_page_crc32(read_buf);
 
-		const uint32_t	crc32_legacy = buf_calc_page_crc32(read_buf,
-								   true);
 		ulint page_type = fil_page_get_type(read_buf);
 
 		ib::info() << "Uncompressed page, stored checksum in field1 "
@@ -1302,7 +1270,7 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< ", calculated checksums for field1: "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_CRC32) << " "
-			<< crc32 << "/" << crc32_legacy
+			<< crc32
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@@ -1319,7 +1287,7 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< ", calculated checksums for field2: "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_CRC32) << " "
-			<< crc32 << "/" << crc32_legacy
+			<< crc32
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@@ -3957,10 +3925,6 @@ buf_zip_decompress(
 			<< ", crc32: "
 			<< page_zip_calc_checksum(
 				frame, size, SRV_CHECKSUM_ALGORITHM_CRC32)
-			<< "/"
-			<< page_zip_calc_checksum(
-				frame, size, SRV_CHECKSUM_ALGORITHM_CRC32,
-				true)
 			<< " innodb: "
 			<< page_zip_calc_checksum(
 				frame, size, SRV_CHECKSUM_ALGORITHM_INNODB)
diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc
index 78b49e49690..c2c3767bb59 100644
--- a/storage/innobase/buf/buf0checksum.cc
+++ b/storage/innobase/buf/buf0checksum.cc
@@ -39,21 +39,15 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
   'long unsigned int*' in initialization */
 ulong	srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
 
-/** set if we have found pages matching legacy big endian checksum */
-bool	legacy_big_endian_checksum = false;
 /** Calculates the CRC32 checksum of a page. The value is stored to the page
 when it is written to a file and also checked for a match when reading from
-the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian
-variants. Note that we must be careful to calculate the same value on 32-bit
+the file. Note that we must be careful to calculate the same value on 32-bit
 and 64-bit architectures.
 @param[in]	page			buffer page (UNIV_PAGE_SIZE bytes)
-@param[in]	use_legacy_big_endian	if true then use big endian
-byteorder when converting byte strings to integers
 @return checksum */
 uint32_t
 buf_calc_page_crc32(
-	const byte*	page,
-	bool		use_legacy_big_endian /* = false */)
+	const byte*	page)
 {
 	/* Since the field FIL_PAGE_FILE_FLUSH_LSN, and in versions <= 4.1.x
 	FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, are written outside the buffer pool
@@ -63,9 +57,11 @@ buf_calc_page_crc32(
 	checksum is stored, and also the last 8 bytes of page because
 	there we store the old formula checksum. */
 
-	ut_crc32_func_t	crc32_func = use_legacy_big_endian
-		? ut_crc32_legacy_big_endian
-		: ut_crc32;
+	ut_crc32_func_t	crc32_func = ut_crc32;
+
+#ifdef WORDS_BIGENDIAN
+	crc32_func = ut_crc32_legacy_big_endian;
+#endif /* WORDS_BIGENDIAN */
 
 	const uint32_t	c1 = crc32_func(
 		page + FIL_PAGE_OFFSET,
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 71ccca04d13..fed95cc69e8 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -2637,11 +2637,7 @@ fil_space_verify_crypt_checksum(
 		switch (algorithm) {
 		case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
 			valid = buf_page_is_checksum_valid_crc32(page, checksum1,
-								 checksum2, false);
-			if (!valid) {
-				valid = buf_page_is_checksum_valid_crc32(
-						page, checksum1, checksum2, true);
-			}
+								 checksum2);
 			break;
 		case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
 			valid = buf_page_is_checksum_valid_innodb(page, checksum1,
@@ -2654,15 +2650,9 @@ fil_space_verify_crypt_checksum(
 		case SRV_CHECKSUM_ALGORITHM_CRC32:
 		case SRV_CHECKSUM_ALGORITHM_INNODB:
 			valid = buf_page_is_checksum_valid_crc32(
-					page, checksum1, checksum2, false);
-			if (!valid) {
-				valid = buf_page_is_checksum_valid_crc32(
-						page, checksum1, checksum2,
-						true)
-					|| buf_page_is_checksum_valid_innodb(
-						page, checksum1, checksum2);
-			}
-
+					page, checksum1, checksum2)
+				|| buf_page_is_checksum_valid_innodb(
+					page, checksum1, checksum2);
 			break;
 		case SRV_CHECKSUM_ALGORITHM_NONE:
 			break;
@@ -2691,8 +2681,7 @@ fil_space_verify_crypt_checksum(
 		ib::info()
 			<< "If unencrypted: stored checksum [" << checksum1
 			<< ":" << checksum2 << "] calculated crc32 ["
-			<< buf_calc_page_crc32(page, false) << ":"
-			<< buf_calc_page_crc32(page, true) << "] innodb ["
+			<< buf_calc_page_crc32(page) << "] innodb ["
 			<< buf_calc_page_old_checksum(page) << ":"
 			<< buf_calc_page_new_checksum(page) << "] LSN "
 			<< mach_read_from_4(page + FIL_PAGE_LSN);
diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h
index 8e13b3876e4..da344a3216c 100644
--- a/storage/innobase/include/buf0buf.h
+++ b/storage/innobase/include/buf0buf.h
@@ -716,14 +716,12 @@ buf_block_unfix(
 @param[in]	read_buf		database page
 @param[in]	checksum_field1		new checksum field
 @param[in]	checksum_field2		old checksum field
-@param[in]	use_legacy_big_endian   use legacy big endian algorithm
 @return true if the page is in crc32 checksum format. */
 bool
 buf_page_is_checksum_valid_crc32(
 	const byte*			read_buf,
 	ulint				checksum_field1,
-	ulint				checksum_field2,
-	bool				use_legacy_big_endian)
+	ulint				checksum_field2)
 	MY_ATTRIBUTE((nonnull(1), warn_unused_result));
 
 /** Checks if the page is in innodb checksum format.
diff --git a/storage/innobase/include/buf0checksum.h b/storage/innobase/include/buf0checksum.h
index 0bac2b911ee..e23ab1311ca 100644
--- a/storage/innobase/include/buf0checksum.h
+++ b/storage/innobase/include/buf0checksum.h
@@ -31,17 +31,13 @@ Created Aug 11, 2011 Vasil Dimov
 
 /** Calculate the CRC32 checksum of a page. The value is stored to the page
 when it is written to a file and also checked for a match when reading from
-the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian
-variants. Note that we must be careful to calculate the same value on 32-bit
+the file. Note that we must be careful to calculate the same value on 32-bit
 and 64-bit architectures.
 @param[in]	page			buffer page (UNIV_PAGE_SIZE bytes)
-@param[in]	use_legacy_big_endian	if true then use big endian
-byteorder when converting byte strings to integers
 @return checksum */
 uint32_t
 buf_calc_page_crc32(
-	const byte*	page,
-	bool		use_legacy_big_endian = false);
+	const byte*	page);
 
 /** Calculate a checksum which is stored to the page when it is written
 to a file. Note that we must be careful to calculate the same value on
@@ -69,6 +65,5 @@ const char*
 buf_checksum_algorithm_name(srv_checksum_algorithm_t algo);
 
 extern ulong	srv_checksum_algorithm;
-extern bool	legacy_big_endian_checksum;
 
 #endif /* buf0checksum_h */
diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h
index d72d5662f78..f36d0e2996a 100644
--- a/storage/innobase/include/page0zip.h
+++ b/storage/innobase/include/page0zip.h
@@ -492,16 +492,12 @@ page_zip_parse_compress(
 @param[in]	data			compressed page
 @param[in]	size			size of compressed page
 @param[in]	algo			algorithm to use
-@param[in]	use_legacy_big_endian	only used if algo is
-SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
-then use big endian byteorder when converting byte strings to integers.
 @return page checksum */
 uint32_t
 page_zip_calc_checksum(
 	const void*			data,
 	ulint				size,
-	srv_checksum_algorithm_t	algo,
-	bool				use_legacy_big_endian = false);
+	srv_checksum_algorithm_t	algo);
 
 /**********************************************************************//**
 Verify a compressed page's checksum.
diff --git a/storage/innobase/include/ut0crc32.h b/storage/innobase/include/ut0crc32.h
index 36b389b5bd2..06910cf5d51 100644
--- a/storage/innobase/include/ut0crc32.h
+++ b/storage/innobase/include/ut0crc32.h
@@ -47,9 +47,11 @@ typedef uint32_t	(*ut_crc32_func_t)(const byte* ptr, ulint len);
 /** Pointer to CRC32 calculation function. */
 extern ut_crc32_func_t	ut_crc32;
 
+#ifdef WORDS_BIGENDIAN
 /** Pointer to CRC32 calculation function, which uses big-endian byte order
 when converting byte strings to integers internally. */
 extern ut_crc32_func_t	ut_crc32_legacy_big_endian;
+#endif /* WORDS_BIGENDIAN */
 
 /** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic,
 but very slow). */
diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc
index 82e95a79f4e..6230daf2342 100644
--- a/storage/innobase/page/page0zip.cc
+++ b/storage/innobase/page/page0zip.cc
@@ -4909,18 +4909,12 @@ corrupt:
 @param[in]	data			compressed page
 @param[in]	size			size of compressed page
 @param[in]	algo			algorithm to use
-@param[in]	use_legacy_big_endian	only used if algo is
-SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
-then use big endian byteorder when converting byte strings to integers.
-SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
-then use big endian byteorder when converting byte strings to integers.
 @return page checksum */
 uint32_t
 page_zip_calc_checksum(
 	const void*			data,
 	ulint				size,
-	srv_checksum_algorithm_t	algo,
-	bool				use_legacy_big_endian /* = false */)
+	srv_checksum_algorithm_t	algo)
 {
 	uLong		adler;
 	const Bytef*	s = static_cast<const byte*>(data);
@@ -4934,9 +4928,10 @@ page_zip_calc_checksum(
 		{
 			ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
 
-			ut_crc32_func_t	crc32_func = use_legacy_big_endian
-				? ut_crc32_legacy_big_endian
-				: ut_crc32;
+			ut_crc32_func_t	crc32_func = ut_crc32;
+#ifdef WORDS_BIGENDIAN
+			crc32_func = ut_crc32_legacy_big_endian;
+#endif
 
 			const uint32_t	crc32
 				= crc32_func(
@@ -5070,13 +5065,6 @@ page_zip_verify_checksum(
 
 	switch (curr_algo) {
 	case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
-		calc = page_zip_calc_checksum(data, size, curr_algo, true);
-		if (calc == stored) {
-			legacy_big_endian_checksum = true;
-			return TRUE;
-		}
-
-		return FALSE;
 	case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
 	case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
 		return FALSE;
@@ -5085,14 +5073,7 @@ page_zip_verify_checksum(
 			return(TRUE);
 		}
 
-		calc = page_zip_calc_checksum(data, size, curr_algo, true);
 		crc32 = calc;
-
-		if (crc32 == stored) {
-			legacy_big_endian_checksum = true;
-			return TRUE;
-		}
-
 		innodb = static_cast<ib_uint32_t>(page_zip_calc_checksum(
 			data, size, SRV_CHECKSUM_ALGORITHM_INNODB));
 		break;
diff --git a/storage/innobase/ut/ut0crc32.cc b/storage/innobase/ut/ut0crc32.cc
index cbb571e8f47..bca6f93756c 100644
--- a/storage/innobase/ut/ut0crc32.cc
+++ b/storage/innobase/ut/ut0crc32.cc
@@ -92,9 +92,11 @@ mysys/my_perf.c, contributed by Facebook under the following license.
 /** Pointer to CRC32 calculation function. */
 ut_crc32_func_t	ut_crc32;
 
+#ifdef WORDS_BIGENDIAN
 /** Pointer to CRC32 calculation function, which uses big-endian byte order
 when converting byte strings to integers internally. */
 ut_crc32_func_t	ut_crc32_legacy_big_endian;
+#endif /* WORDS_BIGENDIAN */
 
 /** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic,
 but very slow). */
@@ -396,6 +398,7 @@ ut_crc32_hw(
 	return(~crc);
 }
 
+#ifdef WORDS_BIGENDIAN
 /** Calculates CRC32 using hardware/CPU instructions.
 This function uses big endian byte ordering when converting byte sequence to
 integers.
@@ -445,6 +448,7 @@ ut_crc32_legacy_big_endian_hw(
 
 	return(~crc);
 }
+#endif /* WORDS_BIGENDIAN */
 
 /** Calculates CRC32 using hardware/CPU instructions.
 This function processes one byte at a time (very slow) and thus it does
@@ -653,6 +657,7 @@ ut_crc32_sw(
 	return(~crc);
 }
 
+#ifdef WORDS_BIGENDIAN
 /** Calculates CRC32 in software, without using CPU instructions.
 This function uses big endian byte ordering when converting byte sequence to
 integers.
@@ -704,6 +709,7 @@ ut_crc32_legacy_big_endian_sw(
 
 	return(~crc);
 }
+#endif /* WORDS_BIGENDIAN */
 
 /** Calculates CRC32 in software, without using CPU instructions.
 This function processes one byte at a time (very slow) and thus it does
@@ -736,7 +742,9 @@ ut_crc32_init()
 {
 	ut_crc32_slice8_table_init();
 	ut_crc32 = ut_crc32_sw;
+#ifdef WORDS_BIGENDIAN
 	ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_sw;
+#endif /* WORDS_BIGENDIAN */
 	ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_sw;
 	ut_crc32_implementation = "Using generic crc32 instructions";
 
@@ -770,7 +778,9 @@ ut_crc32_init()
 
 	if (features_ecx & 1 << 20) {
 		ut_crc32 = ut_crc32_hw;
+#ifdef WORDS_BIGENDIAN
 		ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw;
+#endif /* WORDS_BIGENDIAN */
 		ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_hw;
 		ut_crc32_implementation = "Using SSE2 crc32 instructions";
 	}