MDEV-24953: 10.5.9 crashes with large IN() listbb-10.5-mdev24953

The problem was in and_all_keys(), the code of MDEV-9759 which calculates
the new tree weight:

First, it didn't take into account the case when

(next->next_key_part=tmp) == NULL

and dereferenced a NULL pointer when getting tmp->weight.

Second, "if (param->alloced_sel_args > SEL_ARG::MAX_SEL_ARGS) break"
could leave the loop with incorrect value of weight.

Fixed by introducing SEL_ARG::update_weight_locally() and calling it
at the end of the function. This allows to avoid caring about all the
above cases.

4 files changed, 77 insertions, 19 deletions

diff --git a/mysql-test/main/range_notembedded.result b/mysql-test/main/range_notembedded.result
index 87fa85d3ac6..0ecf47c892e 100644
--- a/mysql-test/main/range_notembedded.result
+++ b/mysql-test/main/range_notembedded.result
@@ -159,7 +159,6 @@ left(@json, 2500)
     ]
 ]
 ## Repeat the above with a bit higher max_weight:
-set @tmp9750_weight=@@optimizer_max_sel_arg_weight;
 set optimizer_max_sel_arg_weight=120;
 explain select * from t1 where 
 kp1 in (1,2,3,4,5,6,7,8,9,10) and
@@ -225,3 +224,18 @@ SELECT *
 FROM mysql.help_relation ignore index (help_topic_id)
 WHERE (help_topic_id = 8 OR help_keyword_id = 0) AND help_keyword_id != 2 AND help_topic_id >= 1900;
 help_topic_id	help_keyword_id
+#
+# MDEV-24953: 10.5.9 crashes with large IN() list
+#
+CREATE TABLE t1 (
+notification_type_id smallint(4) unsigned NOT NULL DEFAULT 0,
+item_id int(10) unsigned NOT NULL DEFAULT 0,
+item_parent_id int(10) unsigned NOT NULL DEFAULT 0,
+user_id int(10) unsigned NOT NULL DEFAULT 0,
+PRIMARY KEY (notification_type_id,item_id,item_parent_id,user_id)
+);
+insert into t1 values (1,1,1,1), (2,2,2,2), (3,3,3,3);
+# Run crashing query
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	2	NULL	3	Using where
+drop table t1;
diff --git a/mysql-test/main/range_notembedded.test b/mysql-test/main/range_notembedded.test
index a70749ced6b..5f6a05e8d91 100644
--- a/mysql-test/main/range_notembedded.test
+++ b/mysql-test/main/range_notembedded.test
@@ -82,7 +82,6 @@ set @json= json_detailed(json_extract(@trace, '$**.setup_range_conditions'));
 select left(@json, 2500);
 
 --echo ## Repeat the above with a bit higher max_weight:
-set @tmp9750_weight=@@optimizer_max_sel_arg_weight;
 set optimizer_max_sel_arg_weight=120;
 explain select * from t1 where 
   kp1 in (1,2,3,4,5,6,7,8,9,10) and
@@ -110,3 +109,34 @@ SELECT *
 FROM mysql.help_relation ignore index (help_topic_id)
 WHERE (help_topic_id = 8 OR help_keyword_id = 0) AND help_keyword_id != 2 AND help_topic_id >= 1900;
 
+--echo #
+--echo # MDEV-24953: 10.5.9 crashes with large IN() list
+--echo #
+--source include/have_sequence.inc
+
+CREATE TABLE t1 (
+  notification_type_id smallint(4) unsigned NOT NULL DEFAULT 0,
+  item_id int(10) unsigned NOT NULL DEFAULT 0,
+  item_parent_id int(10) unsigned NOT NULL DEFAULT 0,
+  user_id int(10) unsigned NOT NULL DEFAULT 0,
+  PRIMARY KEY (notification_type_id,item_id,item_parent_id,user_id)
+);
+insert into t1 values (1,1,1,1), (2,2,2,2), (3,3,3,3);
+
+let $consts=`select group_concat(concat("'",seq,"'")) from seq_1_to_4642`;
+
+--echo # Run crashing query
+--disable_query_log
+eval
+explain
+DELETE FROM t1
+WHERE
+  notification_type_id IN (3, 4, 5, 6, 23)
+  AND
+  user_id = '5044'
+  AND
+  item_parent_id IN ($consts)
+;
+--enable_query_log
+
+drop table t1;
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 3684db40242..a02b6171a20 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -9800,7 +9800,6 @@ and_all_keys(RANGE_OPT_PARAM *param, SEL_ARG *key1, SEL_ARG *key2,
     key1->right= key1->left= &null_element;
     key1->next= key1->prev= 0;
   }
-  uint new_weight= 0;
 
   for (next=key1->first(); next ; next=next->next)
   {
@@ -9813,22 +9812,21 @@ and_all_keys(RANGE_OPT_PARAM *param, SEL_ARG *key1, SEL_ARG *key2,
 	continue;
       }
       next->next_key_part=tmp;
-      new_weight += 1 + tmp->weight;
       if (use_count)
 	next->increment_use_count(use_count);
       if (param->alloced_sel_args > SEL_ARG::MAX_SEL_ARGS)
         break;
     }
     else
-    {
-      new_weight += 1 + key2->weight;
       next->next_key_part=key2;
-    }
   }
   if (!key1)
     return &null_element;			// Impossible ranges
   key1->use_count++;
-  key1->weight= new_weight;
+
+  /* Re-compute the result tree's weight. */
+  key1->update_weight_locally();
+
   key1->max_part_no= MY_MAX(key2->max_part_no, key2->part+1);
   return key1;
 }
@@ -9992,6 +9990,30 @@ get_range(SEL_ARG **e1,SEL_ARG **e2,SEL_ARG *root1)
   return 0;
 }
 
+/*
+  @brief
+    Update the tree weight.
+
+  @detail
+    Utility function to be called on a SEL_ARG tree root after doing local
+    modifications concerning changes at this key part.
+    Assumes that the weight of the graphs connected via next_key_part is
+    up to dayte.
+*/
+void SEL_ARG::update_weight_locally()
+{
+  uint new_weight= 0;
+  const SEL_ARG *sl;
+  for (sl= first(); sl ; sl= sl->next)
+  {
+    new_weight++;
+    if (sl->next_key_part)
+      new_weight += sl->next_key_part->weight;
+  }
+  weight= new_weight;
+}
+
+
 #ifndef DBUG_OFF
 /*
   Verify SEL_TREE's weight.
@@ -10728,17 +10750,7 @@ end:
   key1->use_count++;
 
   /* Re-compute the result tree's weight. */
-  {
-    uint new_weight= 0;
-    const SEL_ARG *sl;
-    for (sl= key1->first(); sl ; sl= sl->next)
-    {
-      new_weight++;
-      if (sl->next_key_part)
-        new_weight += sl->next_key_part->weight;
-    }
-    key1->weight= new_weight;
-  }
+  key1->update_weight_locally();
 
   key1->max_part_no= max_part_no;
   return key1;
diff --git a/sql/opt_range.h b/sql/opt_range.h
index 50cd43c0e85..1014176ecc5 100644
--- a/sql/opt_range.h
+++ b/sql/opt_range.h
@@ -316,6 +316,8 @@ public:
   */
   uint weight;
   enum { MAX_WEIGHT = 32000 };
+
+  void update_weight_locally();
 #ifndef DBUG_OFF
   uint verify_weight();
 #endif