MDEV-19776: Assertion `to_len >= 8' failed in convert_to_printable with optimizer trace enabled

Introduced the convert_to_printable_required_length to return the correct length(taking into
consideration of dots in the case of error messages).

5 files changed, 25 insertions, 1 deletions

diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result
index c0e0b4807a1..b521447c37c 100644
--- a/mysql-test/main/opt_trace.result
+++ b/mysql-test/main/opt_trace.result
@@ -6554,4 +6554,12 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives'))
     }
 ]
 drop table t1, t0, one_k;
+#
+# MDEV-19776: Assertion `to_len >= 8' failed in convert_to_printable with optimizer trace enabled
+#
+CREATE TABLE t1 (f VARBINARY(16) NOT NULL, KEY(f));
+INSERT INTO t1 VALUES ('a'),('b');
+SET optimizer_trace = 'enabled=on';
+DELETE FROM t1 WHERE f = 'x';
+DROP TABLE t1;
 set optimizer_trace='enabled=off';
diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test
index 916b9313ca6..c1ed050062b 100644
--- a/mysql-test/main/opt_trace.test
+++ b/mysql-test/main/opt_trace.test
@@ -498,4 +498,14 @@ explain format=json select * from t1 force index(start_date) where start_date >=
 select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 drop table t1, t0, one_k;
 
+--echo #
+--echo # MDEV-19776: Assertion `to_len >= 8' failed in convert_to_printable with optimizer trace enabled
+--echo #
+
+CREATE TABLE t1 (f VARBINARY(16) NOT NULL, KEY(f));
+INSERT INTO t1 VALUES ('a'),('b');
+SET optimizer_trace = 'enabled=on';
+DELETE FROM t1 WHERE f = 'x';
+DROP TABLE t1;
+
 set optimizer_trace='enabled=off';
diff --git a/sql/field.cc b/sql/field.cc
index f533581a9df..1c874c1021a 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -6949,6 +6949,7 @@ Field_longstr::check_string_copy_error(const String_copier *copier,
 
   if (!is_stat_field)
   {
+    DBUG_ASSERT(sizeof(tmp) >= convert_to_printable_required_length(6));
     convert_to_printable(tmp, sizeof(tmp), pos, (end - pos), cs, 6);
     set_warning_truncated_wrong_value("string", tmp);
   }
diff --git a/sql/sql_string.cc b/sql/sql_string.cc
index 1b567ecb325..a4050c579d0 100644
--- a/sql/sql_string.cc
+++ b/sql/sql_string.cc
@@ -1197,10 +1197,14 @@ uint convert_to_printable(char *to, size_t to_len,
   return (uint) (t - to);
 }
 
+size_t convert_to_printable_required_length(uint len)
+{
+  return static_cast<size_t>(len) * 4 +  3/*dots*/  + 1/*trailing \0 */;
+}
 
 bool String::append_semi_hex(const char *s, uint len, CHARSET_INFO *cs)
 {
-  size_t dst_len= len * 4 + 1; //extra length for the '\0' character
+  size_t dst_len= convert_to_printable_required_length(len);
   if (reserve(dst_len))
     return true;
   uint nbytes= convert_to_printable(Ptr + str_length, dst_len, s, len, cs);
diff --git a/sql/sql_string.h b/sql/sql_string.h
index d8edf5e81f0..f21f23e42ef 100644
--- a/sql/sql_string.h
+++ b/sql/sql_string.h
@@ -126,6 +126,7 @@ size_t my_copy_with_hex_escaping(CHARSET_INFO *cs,
 uint convert_to_printable(char *to, size_t to_len,
                           const char *from, size_t from_len,
                           CHARSET_INFO *from_cs, size_t nbytes= 0);
+size_t convert_to_printable_required_length(uint len);
 
 
 class Charset