From 7b7e5922afa38241e0ed54aad266c54225d3f861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 10 Nov 2020 13:49:01 +0200 Subject: MDEV-24156 trx_undo_left() fails to prevent overflow trx_undo_left(): Return 0 in case of an overflow, instead of returning a negative number interpreted as a large positive number. Also, add debug assertions to check that the pointer is within the page area. This should allow us to catch bugs like MDEV-24096 easier in the future. --- storage/innobase/trx/trx0rec.cc | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 4aecc8ae610..e3e1c33b305 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -128,20 +128,18 @@ trx_undo_parse_add_undo_rec( return(ptr + len); } -/**********************************************************************//** -Calculates the free space left for extending an undo log record. +/** Calculate the free space left for extending an undo log record. +@param page undo log page +@param ptr current end of the undo page @return bytes left */ -UNIV_INLINE -ulint -trx_undo_left( -/*==========*/ - const page_t* page, /*!< in: undo log page */ - const byte* ptr) /*!< in: pointer to page */ +static ulint trx_undo_left(const page_t *page, const byte *ptr) { - /* The '- 10' is a safety margin, in case we have some small - calculation error below */ - - return(UNIV_PAGE_SIZE - (ptr - page) - 10 - FIL_PAGE_DATA_END); + ut_ad(ptr >= &page[TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE]); + /* The 10 is supposed to be an extra safety margin (and needed for + compatibility with older versions) */ + lint left= srv_page_size - (ptr - page) - (10 + FIL_PAGE_DATA_END); + ut_ad(left >= 0); + return left < 0 ? 0 : static_cast(left); } /**********************************************************************//** -- cgit v1.2.1