MDEV-24176 Server crashes after insert in the table with virtualbb-10.3-midenok-MDEV-24176MDEV-24176/10.3_old

column generated using date_format() and if()

When table is reopened from cache vcol_info contains stale
expression. We refresh expression via TABLE::vcol_fix_exprs() but
first we must prepare a proper context (Vcol_expr_context) and do some
preparations which meet some requirements:

1. fields must not be fixed, we unfix them with cleanup() via
cleanup_processor.

2. Item_ident must have cached_table cleared. Again, this is stale
data which was already freed. cached_table interferes with
find_field_in_tables().

We cannot clear cached_table directly in Item_ident::cleanup()
method. Lots of spaghetti logic depends on cached_table existence even
after cleanup() done.

3. If Item_ident has table_name non-NULL we trigger expr update. That
is done via Item_ident::check_vcol_func_processor() and
VCOL_TABLE_ALIAS flag.

(4-7) The below conditions are prepared by Vcol_expr_context and used in
both fix_session_expr_for_read() and TABLE::vcol_fix_exprs():

4. Any fix_expr() must be done on expr_arena as there may be new items
created. It was a bug in fix_session_expr_for_read(). It was just not
reproduced because there were no second refix. Now refix is done for
more cases so it does reproduce. Tests affected: vcol.binlog

5. Also name resolution context must be narrowed to the single table
with all crutches in place. Tests affected: vcol.update

6. sql_mode must be clean and not fail expr update.

sql_mode such as MODE_NO_BACKSLASH_ESCAPES, MODE_NO_ZERO_IN_DATE, etc
must not affect vcol expression update. If the table was created
successfully any further evaluation must not fail. Tests affected:
main.func_like

7. Warnings are disabled during expr update. It is assumed that these
warnings were printed before during initialization phase or later
during execution phase. Tests affected: vcol.wrong_arena

Dictionary: refresh, update and refix are the synonyms for
vcol_fix_exprs() or fix_session_expr_for_read() calls.

8. cleanup_excluding_fields_processor() removed. It was just a quick
hack to exclude wrongly working Item_field from processing. Now it
works due to correct execution environment (see next commit). Related
to MDEV-10355.

8 files changed, 221 insertions, 76 deletions

diff --git a/mysql-test/suite/vcol/r/vcol_syntax.result b/mysql-test/suite/vcol/r/vcol_syntax.result
index 0063f38ea36..e2df8609bc9 100644
--- a/mysql-test/suite/vcol/r/vcol_syntax.result
+++ b/mysql-test/suite/vcol/r/vcol_syntax.result
@@ -94,6 +94,20 @@ create table t1 (a int, v_a int generated always as (a));
 update t1 as x set a = 1;
 alter table t1 force;
 drop table t1;
+create table t1 (
+id int not null auto_increment primary key,
+order_date_time datetime not null,
+order_date date generated always as (convert(order_date_time, date)),
+language_id binary(16) null
+);
+update t1 as tx set order_date= null;
+alter table t1 modify column language_id binary(16) not null;
+drop table t1;
 #
-# End of 10.2 tests
+# MDEV-24176 Server crashes after insert in the table with virtual column generated using date_format() and if()
 #
+create table t1 (d1 date not null, d2 date not null,
+gd text as (concat(d1,if(d1 <> d2, date_format(d2, 'to %y-%m-%d '), ''))) );
+insert into t1(d1,d2) values
+('2020-09-01','2020-09-01'),('2020-05-01','2020-09-01');
+drop table t1;
diff --git a/mysql-test/suite/vcol/t/vcol_syntax.test b/mysql-test/suite/vcol/t/vcol_syntax.test
index 3c8a50a7f36..6a9b2701322 100644
--- a/mysql-test/suite/vcol/t/vcol_syntax.test
+++ b/mysql-test/suite/vcol/t/vcol_syntax.test
@@ -77,7 +77,25 @@ update t1 as x set a = 1;
 alter table t1 force;
 drop table t1;
 
+create table t1 (
+  id int not null auto_increment primary key,
+  order_date_time datetime not null,
+  order_date date generated always as (convert(order_date_time, date)),
+  language_id binary(16) null
+);
+
+update t1 as tx set order_date= null;
+alter table t1 modify column language_id binary(16) not null;
+# Cleanup
+drop table t1;
 
 --echo #
---echo # End of 10.2 tests
+--echo # MDEV-24176 Server crashes after insert in the table with virtual column generated using date_format() and if()
 --echo #
+create table t1 (d1 date not null, d2 date not null,
+  gd text as (concat(d1,if(d1 <> d2, date_format(d2, 'to %y-%m-%d '), ''))) );
+
+insert into t1(d1,d2) values
+  ('2020-09-01','2020-09-01'),('2020-05-01','2020-09-01');
+
+drop table t1;
diff --git a/sql/field.h b/sql/field.h
index 3be14f3ca0e..aa1180eee0b 100644
--- a/sql/field.h
+++ b/sql/field.h
@@ -532,6 +532,7 @@ static inline const char *vcol_type_name(enum_vcol_info_type type)
 #define VCOL_AUTO_INC         16
 #define VCOL_IMPOSSIBLE       32
 #define VCOL_NOT_VIRTUAL      64  /* Function can't be virtual */
+#define VCOL_TABLE_ALIAS      128
 
 #define VCOL_NOT_STRICTLY_DETERMINISTIC                       \
   (VCOL_NON_DETERMINISTIC | VCOL_TIME_FUNC | VCOL_SESSION_FUNC)
diff --git a/sql/item.cc b/sql/item.cc
index f2d20bd0e3f..b5e548b5a2a 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -835,6 +835,12 @@ bool Item_ident::remove_dependence_processor(void * arg)
   DBUG_RETURN(0);
 }
 
+bool Item_ident::cached_table_cleanup_processor(void * arg)
+{
+  cached_table= NULL;
+  return false;
+}
+
 
 bool Item_ident::collect_outer_ref_processor(void *param)
 {
diff --git a/sql/item.h b/sql/item.h
index edefb4fca26..79638ddac5c 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1736,15 +1736,8 @@ public:
 
   /*========= Item processors, to be used with Item::walk() ========*/
   virtual bool remove_dependence_processor(void *arg) { return 0; }
+  virtual bool cached_table_cleanup_processor(void * arg) { return 0; }
   virtual bool cleanup_processor(void *arg);
-  /*
-    TODO: Remove cleanup_excluding_fields_processor()
-
-    This is just a quick hack to exclude wrongly working Item_field from
-    processing. Now it works due to correct execution environment (see
-    next commit). Related to MDEV-10355
-  */
-  virtual bool cleanup_excluding_fields_processor(void *arg) { return cleanup_processor(arg); }
   virtual bool cleanup_excluding_const_fields_processor(void *arg) { return cleanup_processor(arg); }
   virtual bool collect_item_field_processor(void *arg) { return 0; }
   virtual bool collect_outer_ref_processor(void *arg) {return 0; }
@@ -2973,6 +2966,7 @@ public:
   void cleanup();
   st_select_lex *get_depended_from() const;
   bool remove_dependence_processor(void * arg);
+  bool cached_table_cleanup_processor(void * arg);
   virtual void print(String *str, enum_query_type query_type);
   virtual bool change_context_processor(void *cntx)
     { context= (Name_resolution_context *)cntx; return FALSE; }
@@ -2984,6 +2978,19 @@ public:
                             const char *db_name,
                             const char *table_name, List_iterator<Item> *it,
                             bool any_privileges);
+  bool check_vcol_func_processor(void *arg)
+  {
+    if (table_name)
+    {
+      /*
+         NOTE: alias is different in every statement, we must update it.
+         We cannot rely on alias_name_used (see NOTE above).
+      */
+      DBUG_ASSERT(field_name.str);
+      return mark_unsupported_function(field_name.str, arg, VCOL_TABLE_ALIAS);
+    }
+    return false;
+  }
 };
 
 
@@ -3192,6 +3199,8 @@ public:
   bool check_vcol_func_processor(void *arg)
   {
     context= 0;
+    if (Item_ident::check_vcol_func_processor(arg))
+      return true;
     if (field && (field->unireg_check == Field::NEXT_NUMBER))
     {
       // Auto increment fields are unsupported
@@ -3242,8 +3251,6 @@ public:
   virtual void print(String *str, enum_query_type query_type);
   bool excl_dep_on_table(table_map tab_map);
   bool excl_dep_on_grouping_fields(st_select_lex *sel);
-  bool cleanup_excluding_fields_processor(void *arg)
-  { return field ? 0 : cleanup_processor(arg); }
   bool cleanup_excluding_const_fields_processor(void *arg)
   { return field && const_item() ? 0 : cleanup_processor(arg); }
   
@@ -5102,14 +5109,6 @@ public:
   }
   bool excl_dep_on_grouping_fields(st_select_lex *sel)
   { return (*ref)->excl_dep_on_grouping_fields(sel); }
-  bool cleanup_excluding_fields_processor(void *arg)
-  {
-    Item *item= real_item();
-    if (item && item->type() == FIELD_ITEM &&
-        ((Item_field *)item)->field)
-      return 0;
-    return cleanup_processor(arg);
-  }
   bool cleanup_excluding_const_fields_processor(void *arg)
   { 
     Item *item= real_item();
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index 17ea738443e..c5f8caedbb0 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -1613,6 +1613,7 @@ bool open_table(THD *thd, TABLE_LIST *table_list, Open_table_context *ot_ctx)
 #ifdef WITH_PARTITION_STORAGE_ENGINE
   int part_names_error=0;
 #endif
+  bool from_share= false;
   DBUG_ENTER("open_table");
 
   /*
@@ -2021,6 +2022,7 @@ retry_share:
 
     /* Add table to the share's used tables list. */
     tc_add_table(thd, table);
+    from_share= true;
   }
 
   table->mdl_ticket= mdl_ticket;
@@ -2040,7 +2042,7 @@ retry_share:
   table_list->updatable= 1; // It is not derived table nor non-updatable VIEW
   table_list->table= table;
 
-  if (table->vcol_fix_exprs(thd))
+  if (!from_share && table->vcol_fix_exprs(thd))
     goto err_lock;
 
 #ifdef WITH_PARTITION_STORAGE_ENGINE
@@ -5282,46 +5284,6 @@ static void mark_real_tables_as_free_for_reuse(TABLE_LIST *table_list)
     }
 }
 
-bool TABLE::vcol_fix_exprs(THD *thd)
-{
-  if (pos_in_table_list->placeholder() || !s->vcols_need_refixing ||
-      pos_in_table_list->lock_type < TL_WRITE_ALLOW_WRITE)
-    return false;
-
-  DBUG_ASSERT(pos_in_table_list != thd->lex->first_not_own_table());
-
-  bool result= true;
-  Security_context *save_security_ctx= thd->security_ctx;
-  Query_arena *stmt_backup= thd->stmt_arena;
-  if (thd->stmt_arena->is_conventional())
-    thd->stmt_arena= expr_arena;
-
-  if (pos_in_table_list->security_ctx)
-    thd->security_ctx= pos_in_table_list->security_ctx;
-
-
-  for (Field **vf= vfield; vf && *vf; vf++)
-    if ((*vf)->vcol_info->fix_session_expr(thd))
-      goto end;
-
-  for (Field **df= default_field; df && *df; df++)
-    if ((*df)->default_value &&
-        (*df)->default_value->fix_session_expr(thd))
-      goto end;
-
-  for (Virtual_column_info **cc= check_constraints; cc && *cc; cc++)
-    if ((*cc)->fix_session_expr(thd))
-      goto end;
-
-  result= false;
-
-end:
-  thd->security_ctx= save_security_ctx;
-  thd->stmt_arena= stmt_backup;
-
-  return result;
-}
-
 
 /**
   Lock all tables in a list.
diff --git a/sql/sql_class.h b/sql/sql_class.h
index 133c0f6fec5..fe42909aa0a 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -1831,6 +1831,25 @@ private:
 };
 
 
+struct Silence_warnings : public Internal_error_handler
+{
+public:
+  virtual bool handle_condition(THD *,
+                                uint,
+                                const char* sqlstate,
+                                Sql_condition::enum_warning_level *level,
+                                const char* msg,
+                                Sql_condition ** cond_hdl)
+  {
+    *cond_hdl= NULL;
+    if (*level == Sql_condition::WARN_LEVEL_WARN)
+      return TRUE;
+
+    return FALSE;
+  }
+};
+
+
 /**
   Internal error handler to process an error from MDL_context::upgrade_lock()
   and mysql_lock_tables(). Used by implementations of HANDLER READ and
diff --git a/sql/table.cc b/sql/table.cc
index 59d9e244cf9..ed40414d1c5 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -2964,13 +2964,105 @@ bool Virtual_column_info::fix_expr(THD *thd)
 */
 bool Virtual_column_info::fix_session_expr(THD *thd)
 {
-  DBUG_ENTER("fix_session_vcol_expr");
-  if (!(flags & (VCOL_TIME_FUNC|VCOL_SESSION_FUNC)))
-    DBUG_RETURN(0);
+  // TODO: remove either this check or vcols_need_refixing
+  if (!(flags & (VCOL_TIME_FUNC|VCOL_SESSION_FUNC|VCOL_TABLE_ALIAS)))
+    return false;
 
-  expr->walk(&Item::cleanup_excluding_fields_processor, 0, 0);
+  if (expr->walk(&Item::cached_table_cleanup_processor, 0, 0))
+    return true;
+  if (expr->walk(&Item::cleanup_processor, 0, 0))
+    return true;
   DBUG_ASSERT(!expr->fixed);
-  DBUG_RETURN(fix_expr(thd));
+  if (expr->walk(&Item::change_context_processor, 0, thd->lex->current_context()))
+    return true;
+  if (fix_expr(thd))
+    return true;
+  if (expr->walk(&Item::change_context_processor, 0, NULL))
+    return true;
+  return false;
+}
+
+
+class Vcol_expr_context
+{
+  bool inited;
+  THD *thd;
+  TABLE *table;
+  LEX *old_lex;
+  LEX lex;
+  table_map old_map;
+  bool old_want_privilege;
+  Security_context *save_security_ctx;
+  sql_mode_t save_sql_mode;
+  Query_arena backup_arena;
+  Query_arena *stmt_backup;
+  Query_arena *expr_arena;
+  Silence_warnings disable_warnings;
+  bool warnings_disabled;
+
+public:
+  Vcol_expr_context(THD *_thd, TABLE *_table) :
+    inited(false),
+    thd(_thd),
+    table(_table),
+    old_lex(thd->lex),
+    old_map(table->map),
+    old_want_privilege(table->grant.want_privilege),
+    save_security_ctx(thd->security_ctx),
+    save_sql_mode(thd->variables.sql_mode),
+    stmt_backup(thd->stmt_arena),
+    expr_arena(table->expr_arena) {}
+  bool init();
+
+  ~Vcol_expr_context();
+};
+
+
+bool Vcol_expr_context::init()
+{
+  /*
+      As this is vcol expression we must narrow down name resolution to
+      single table.
+  */
+  if (init_lex_with_single_table(thd, table, &lex))
+  {
+    my_error(ER_OUT_OF_RESOURCES, MYF(0));
+    table->map= old_map;
+    return true;
+  }
+
+  thd->push_internal_handler(&disable_warnings);
+  table->grant.want_privilege= false;
+  TABLE_LIST const *tl= table->pos_in_table_list;
+
+  /* Avoid fix_outer_field() in Item_field::fix_fields(). */
+  lex.current_context()->select_lex= tl->select_lex;
+  lex.sql_command= old_lex->sql_command;
+
+  thd->set_n_backup_active_arena(expr_arena, &backup_arena);
+  thd->stmt_arena= expr_arena;
+
+  if (tl->security_ctx)
+    thd->security_ctx= tl->security_ctx;
+
+  thd->variables.sql_mode= 0;
+
+  inited= true;
+  return false;
+}
+
+Vcol_expr_context::~Vcol_expr_context()
+{
+  if (!inited)
+    return;
+  table->grant.want_privilege= old_want_privilege;
+  thd->restore_active_arena(expr_arena, &backup_arena);
+  thd->stmt_arena= stmt_backup;
+  thd->pop_internal_handler();
+  end_lex_with_single_table(thd, table, old_lex);
+  table->map= old_map;
+  thd->security_ctx= save_security_ctx;
+  thd->variables.sql_mode= save_sql_mode;
 }
 
 
@@ -2981,16 +3073,48 @@ bool Virtual_column_info::fix_session_expr(THD *thd)
 */
 bool Virtual_column_info::fix_session_expr_for_read(THD *thd, Field *field)
 {
-  DBUG_ENTER("fix_session_vcol_expr_for_read");
-  TABLE_LIST *tl= field->table->pos_in_table_list;
+  const TABLE_LIST *tl= field->table->pos_in_table_list;
   if (!tl || tl->lock_type >= TL_WRITE_ALLOW_WRITE)
-    DBUG_RETURN(0);
-  Security_context *save_security_ctx= thd->security_ctx;
-  if (tl->security_ctx)
-    thd->security_ctx= tl->security_ctx;
-  bool res= fix_session_expr(thd);
-  thd->security_ctx= save_security_ctx;
-  DBUG_RETURN(res);
+    return false;
+
+  Vcol_expr_context expr_ctx(thd, field->table);
+  if (expr_ctx.init())
+    return true;
+
+  const bool res= fix_session_expr(thd);
+  return res;
+}
+
+
+bool TABLE::vcol_fix_exprs(THD *thd)
+{
+  if (pos_in_table_list->placeholder() || !s->vcols_need_refixing)
+    return false;
+
+  Vcol_expr_context expr_ctx(thd, this);
+  if (expr_ctx.init())
+    return true;
+
+  bool result= true;
+
+  for (Field **vf= vfield; vf && *vf; vf++)
+    if ((*vf)->vcol_info->fix_session_expr(thd))
+      goto end;
+
+  for (Field **df= default_field; df && *df; df++)
+    if ((*df)->default_value &&
+        (*df)->default_value->fix_session_expr(thd))
+      goto end;
+
+  for (Virtual_column_info **cc= check_constraints; cc && *cc; cc++)
+    if ((*cc)->fix_session_expr(thd))
+      goto end;
+
+  result= false;
+
+end:
+  DBUG_ASSERT(!result || thd->get_stmt_da()->is_error());
+  return result;
 }
 
 
@@ -3071,7 +3195,7 @@ bool Virtual_column_info::fix_and_check_expr(THD *thd, TABLE *table)
   }
   flags= res.errors;
 
-  if (flags & VCOL_SESSION_FUNC)
+  if (flags & (VCOL_SESSION_FUNC|VCOL_TABLE_ALIAS))
     table->s->vcols_need_refixing= true;
 
   DBUG_RETURN(0);
@@ -3145,6 +3269,8 @@ unpack_vcol_info_from_frm(THD *thd, TABLE *table,
     table->internal_tables= sequence;
   }
 
+  lex.sql_command= old_lex->sql_command;
+
   vcol_storage.vcol_info->set_vcol_type(vcol->get_vcol_type());
   vcol_storage.vcol_info->stored_in_db=      vcol->stored_in_db;
   vcol_storage.vcol_info->name=              vcol->name;