MDEV-30179 mariabackup --backup fails with FATAL ERROR: ... failedbb-10.6-MDEV-30179

			to copy datafile

- Mariabackup fails to copy the undo log tablespace when it undergoes
truncation. So Mariabackup should detect the redo log which does
undo tablespace truncation and also backup should read the minimum
file size of the tablespace and ignore the error while reading.

- Throw error when innodb undo tablespace read failed, but backup
doesn't find the redo log for undo tablespace truncation

4 files changed, 80 insertions, 3 deletions

diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc
index d7aca1362ed..9298271ad3d 100644
--- a/extra/mariabackup/fil_cur.cc
+++ b/extra/mariabackup/fil_cur.cc
@@ -365,6 +365,7 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t*	cursor,
 	ib_int64_t		to_read;
 	const ulint		page_size = cursor->page_size;
 	bool			defer = false;
+	static uint32_t		n_read_undo = 0;
 	xb_ad(!cursor->is_system() || page_size == srv_page_size);
 
 	cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt,
@@ -374,6 +375,7 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t*	cursor,
 		return(XB_FIL_CUR_EOF);
 	}
 
+reinit_buf:
 	if (to_read > (ib_int64_t) cursor->buf_size) {
 		to_read = (ib_int64_t) cursor->buf_size;
 	}
@@ -414,8 +416,40 @@ read_retry:
 	cursor->buf_offset = offset;
 	cursor->buf_page_no = static_cast<unsigned>(offset / page_size);
 
-	if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset,
+	DBUG_EXECUTE_IF("undo_space_trunc",
+			if (cursor->space_id == 1
+			    && (n_read_undo == 0
+			        || cursor->buf_page_no
+				   == SRV_UNDO_TABLESPACE_SIZE_IN_PAGES))
+			  goto read_fail;);
+
+	IORequest request{nullptr, nullptr, cursor->node, IORequest::READ_SYNC};
+
+	if (os_file_read(request, cursor->file, cursor->buf, offset,
 			 (ulint) to_read, nullptr) != DB_SUCCESS) {
+#ifdef UNIV_DEBUG
+read_fail:
+#endif
+		if (srv_is_undo_tablespace(cursor->space_id)) {
+			if (cursor->buf_page_no
+			    >= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) {
+				ret = XB_FIL_CUR_SKIP;
+				goto func_exit;
+			}
+
+			to_read = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES
+				  * page_size;
+
+			if (n_read_undo > 1) {
+				ret = XB_FIL_CUR_ERROR;
+				goto func_exit;
+			}
+
+			n_read_undo++;
+			space->release();
+			goto reinit_buf;
+		}
+
 		ret = XB_FIL_CUR_ERROR;
 		goto func_exit;
 	}
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 81f24337952..bfd5be59e2d 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -244,6 +244,9 @@ long innobase_file_io_threads = 4;
 ulong innobase_read_io_threads = 4;
 ulong innobase_write_io_threads = 4;
 
+/* Store the failed read of undo tablespace ids */
+std::set<space_id_t> fail_undo_ids;
+
 longlong innobase_page_size = (1LL << 14); /* 16KB */
 char*	innobase_buffer_pool_filename = NULL;
 
@@ -410,6 +413,9 @@ used to indicate whether the given deferred tablespace can
 be reconstructed. */
 static std::set<space_id_t> first_page_init_ids;
 
+/** Store the space ids of truncated undo log tablespaces */
+static std::set<space_id_t> undo_trunc_ids;
+
 // Convert non-null terminated filename to space name
 static std::string filename_to_spacename(const void *filename, size_t len);
 
@@ -926,6 +932,11 @@ static void backup_first_page_op(ulint space_id)
   first_page_init_ids.insert(space_id);
 }
 
+static void backup_undo_trunc(ulint space_id)
+{
+  undo_trunc_ids.insert(space_id);
+}
+
 /*
   Retrieve default data directory, to be used with --copy-back.
 
@@ -2851,11 +2862,21 @@ static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n,
 	}
 
 	/* The main copy loop */
-	while ((res = xb_fil_cur_read(&cursor, corrupted_pages)) ==
-		XB_FIL_CUR_SUCCESS) {
+	while (1) {
+		res = xb_fil_cur_read(&cursor, corrupted_pages);
+		if (res == XB_FIL_CUR_ERROR || res == XB_FIL_CUR_EOF) {
+			break;
+		}
+
 		if (!write_filter.process(&write_filt_ctxt, dstfile)) {
 			goto error;
 		}
+
+		if (res == XB_FIL_CUR_SKIP) {
+			fprintf(stderr, "add fail undo ids\n");
+			fail_undo_ids.insert(cursor.space_id);
+			break;
+		}
 	}
 
 	if (res == XB_FIL_CUR_ERROR) {
@@ -4449,6 +4470,16 @@ static bool xtrabackup_backup_low()
 
 	dst_log_file = NULL;
 
+	for (auto it = fail_undo_ids.begin(); it != fail_undo_ids.end();
+	     it++) {
+		if (undo_trunc_ids.find(*it) == undo_trunc_ids.end()) {
+			msg("mariabackup: Failed to read undo log "
+			    "tablespace and there is no undo "
+			    "tablespace truncation redo record.");
+			return false;
+		}
+	}
+
 	if(!xtrabackup_incremental) {
 		strcpy(metadata_type, "full-backuped");
 		metadata_from_lsn = 0;
@@ -4524,6 +4555,7 @@ static bool xtrabackup_backup_func()
 	srv_operation = SRV_OPERATION_BACKUP;
 	log_file_op = backup_file_op;
 	first_page_init = backup_first_page_op;
+	undo_space_trunc = backup_undo_trunc;
 	metadata_to_lsn = 0;
 
 	/* initialize components */
@@ -4539,6 +4571,7 @@ fail:
 
 		log_file_op = NULL;
 		first_page_init = NULL;
+		undo_space_trunc = NULL;
 		if (dst_log_file) {
 			ds_close(dst_log_file);
 			dst_log_file = NULL;
@@ -4828,6 +4861,7 @@ fail_before_log_copying_thread_start:
 	innodb_shutdown();
 	log_file_op = NULL;
 	first_page_init = NULL;
+	undo_space_trunc = NULL;
 	pthread_mutex_destroy(&backup_mutex);
 	pthread_cond_destroy(&scanned_lsn_cond);
 	if (!corrupted_pages.empty()) {
diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h
index b1c09cfa2bc..688ae21218f 100644
--- a/storage/innobase/include/log0recv.h
+++ b/storage/innobase/include/log0recv.h
@@ -96,6 +96,11 @@ extern void (*log_file_op)(ulint space_id, int type,
 @param	space_id	tablespace identifier */
 extern void (*first_page_init)(ulint space_id);
 
+/** Report an operation which does undo log tablespace truncation
+during backup
+@param	space_id	undo tablespace identifier */
+extern void (*undo_space_trunc)(ulint space_id);
+
 /** Stored redo log record */
 struct log_rec_t
 {
diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index b0651d8b8da..e3be5727895 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -1039,6 +1039,8 @@ void (*log_file_op)(ulint space_id, int type,
 
 void (*first_page_init)(ulint space_id);
 
+void (*undo_space_trunc)(ulint space_id);
+
 /** Information about initializing page contents during redo log processing.
 FIXME: Rely on recv_sys.pages! */
 class mlog_init_t
@@ -2424,6 +2426,8 @@ same_page:
                         TRX_SYS_MAX_UNDO_SPACES, "compatibility");
           truncated_undo_spaces[space_id - srv_undo_space_id_start]=
             { recovered_lsn, page_no };
+          if (undo_space_trunc)
+            undo_space_trunc(space_id);
 #endif
           last_offset= 1; /* the next record must not be same_page  */
           continue;