12 files changed, 46 insertions, 66 deletions
diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result
index e9c7047e492..cb538a352bc 100644
--- a/mysql-test/r/mysqld--help.result
+++ b/mysql-test/r/mysqld--help.result
@@ -336,10 +336,6 @@ The following options may be given as the first argument:
  Specifies a directory to add to the ignore list when
  collecting database names from the datadir. Put a blank
  argument to reset the list accumulated so far.
- --in-subquery-conversion-threshold[=#] 
- The minimum number of scalar elements in the value list
- of IN predicate that triggers its conversion to IN
- subquery
  --init-connect=name Command(s) that are executed for each new connection
  (unless the user has SUPER privilege)
  --init-file=name    Read SQL commands from this file at startup
@@ -1415,7 +1411,6 @@ idle-transaction-timeout 0
 idle-write-transaction-timeout 0
 ignore-builtin-innodb FALSE
 ignore-db-dirs 
-in-subquery-conversion-threshold 1000
 init-connect 
 init-file (No default value)
 init-rpl-role MASTER
diff --git a/mysql-test/r/opt_tvc.result b/mysql-test/r/opt_tvc.result
index 3b03bd4ffaf..4403980c81c 100644
--- a/mysql-test/r/opt_tvc.result
+++ b/mysql-test/r/opt_tvc.result
@@ -27,7 +27,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` in (1,2)
 # set minimum number of values in VALUEs list when optimization works to 2
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 # single IN-predicate in WHERE-part
 select * from t1 where a in (1,2);
 a	b
@@ -485,7 +485,7 @@ a	b
 2	5
 deallocate prepare stmt;
 # use inside out access from tvc rows
-set @@in_subquery_conversion_threshold= default;
+set @@in_predicate_conversion_threshold= default;
 select * from t3 where a in (1,4,10);
 a	b
 1	abc
@@ -498,7 +498,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t3	range	idx	idx	5	NULL	5	100.00	Using index condition
 Warnings:
 Note	1003	select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t3` where `test`.`t3`.`a` in (1,4,10)
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 select * from t3 where a in (1,4,10);
 a	b
 1	abc
@@ -515,7 +515,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t3` semi join ((values (1),(4),(10)) `tvc_0`) where `test`.`t3`.`a` = `tvc_0`.`1`
 # use vectors in IN predeicate
-set @@in_subquery_conversion_threshold= 4;
+set @@in_predicate_conversion_threshold= 4;
 select * from t1 where (a,b) in ((1,2),(3,4));
 a	b
 1	2
@@ -527,9 +527,9 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 3	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` semi join ((values (1,2),(3,4)) `tvc_0`) where `test`.`t1`.`a` = `tvc_0`.`1` and `test`.`t1`.`b` = `tvc_0`.`2`
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 # trasformation works for the one IN predicate and doesn't work for the other
-set @@in_subquery_conversion_threshold= 5;
+set @@in_predicate_conversion_threshold= 5;
 select * from t2
 where (a,b) in ((1,2),(8,9)) and
 (a,c) in ((1,3),(8,0),(5,1));
@@ -546,7 +546,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 3	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` semi join ((values (1,3),(8,0),(5,1)) `tvc_0`) where `test`.`t2`.`a` = `tvc_0`.`1` and `test`.`t2`.`c` = `tvc_0`.`3` and (`tvc_0`.`1`,`test`.`t2`.`b`) in (<cache>((1,2)),<cache>((8,9)))
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 #
 # mdev-14281: conversion of NOT IN predicate into subquery predicate
 # 
@@ -613,4 +613,4 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` where !<expr_cache><`test`.`t2`.`a`,`test`.`t2`.`c`>(<in_optimizer>((`test`.`t2`.`a`,`test`.`t2`.`c`),(`test`.`t2`.`a`,`test`.`t2`.`c`) in ( <materialize> (/* select#2 */ select `tvc_0`.`1`,`tvc_0`.`2` from (values (1,2),(8,9),(5,1)) `tvc_0` ), <primary_index_lookup>(`test`.`t2`.`a` in <temporary table> on distinct_key where `test`.`t2`.`a` = `<subquery2>`.`1` and `test`.`t2`.`c` = `<subquery2>`.`2`))))
 drop table t1, t2, t3;
-set @@in_subquery_conversion_threshold= default;
+set @@in_predicate_conversion_threshold= default;
diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result
index 61886b2be98..e2996b963bc 100644
--- a/mysql-test/r/range.result
+++ b/mysql-test/r/range.result
@@ -1,5 +1,3 @@
-set in_subquery_conversion_threshold=10000;
-drop table if exists t1, t2, t3, t10, t100;
 CREATE TABLE t1 (
 event_date date DEFAULT '0000-00-00' NOT NULL,
 type int(11) DEFAULT '0' NOT NULL,
@@ -1064,7 +1062,9 @@ set @b= concat("explain ", @a);
 prepare stmt1 from @b;
 execute stmt1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	index	a	a	5	NULL	1003	Using where; Using index
+1	PRIMARY	t2	index	NULL	a	5	NULL	1003	Using where; Using index
+2	MATERIALIZED	<derived3>	ALL	NULL	NULL	NULL	NULL	2	
+3	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 prepare stmt1 from @a;
 execute stmt1;
 a
@@ -3007,4 +3007,3 @@ drop table t1,t2,t3;
 #
 # End of 10.2 tests
 #
-set in_subquery_conversion_threshold=default;
diff --git a/mysql-test/r/range_mrr_icp.result b/mysql-test/r/range_mrr_icp.result
index f6c523f9900..629d183bee7 100644
--- a/mysql-test/r/range_mrr_icp.result
+++ b/mysql-test/r/range_mrr_icp.result
@@ -1,7 +1,5 @@
 set @mrr_icp_extra_tmp=@@optimizer_switch;
 set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on';
-set in_subquery_conversion_threshold=10000;
-drop table if exists t1, t2, t3, t10, t100;
 CREATE TABLE t1 (
 event_date date DEFAULT '0000-00-00' NOT NULL,
 type int(11) DEFAULT '0' NOT NULL,
@@ -1066,7 +1064,9 @@ set @b= concat("explain ", @a);
 prepare stmt1 from @b;
 execute stmt1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	index	a	a	5	NULL	1003	Using where; Using index
+1	PRIMARY	t2	index	NULL	a	5	NULL	1003	Using where; Using index
+2	MATERIALIZED	<derived3>	ALL	NULL	NULL	NULL	NULL	2	
+3	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 prepare stmt1 from @a;
 execute stmt1;
 a
@@ -3019,5 +3019,4 @@ drop table t1,t2,t3;
 #
 # End of 10.2 tests
 #
-set in_subquery_conversion_threshold=default;
 set optimizer_switch=@mrr_icp_extra_tmp;
diff --git a/mysql-test/suite/sys_vars/inc/sysvars_server.inc b/mysql-test/suite/sys_vars/inc/sysvars_server.inc
index 37491315c3e..d14e905378c 100644
--- a/mysql-test/suite/sys_vars/inc/sysvars_server.inc
+++ b/mysql-test/suite/sys_vars/inc/sysvars_server.inc
@@ -19,6 +19,7 @@ select * from information_schema.system_variables
         variable_name not like 'debug%' and
         variable_name not like 'wsrep%' and
         variable_name not in (
+          'in_predicate_conversion_threshold',
           'have_openssl',
           'have_symlink',
           'hostname',
diff --git a/mysql-test/suite/sys_vars/r/sysvars_debug.result b/mysql-test/suite/sys_vars/r/sysvars_debug.result
index f50e796bff4..b544f6b50d9 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_debug.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_debug.result
@@ -1,5 +1,6 @@
 select * from information_schema.system_variables
 where variable_name like 'debug%'
+     or variable_name = 'in_predicate_conversion_threshold'
   order by variable_name;
 VARIABLE_NAME	DEBUG
 SESSION_VALUE	
@@ -85,3 +86,17 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	NULL
+VARIABLE_NAME	IN_PREDICATE_CONVERSION_THRESHOLD
+SESSION_VALUE	1000
+GLOBAL_VALUE	1000
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	1000
+VARIABLE_SCOPE	SESSION
+VARIABLE_TYPE	INT UNSIGNED
+VARIABLE_COMMENT	The minimum number of scalar elements in the value list of IN predicate that triggers its conversion to IN subquery
+NUMERIC_MIN_VALUE	0
+NUMERIC_MAX_VALUE	4294967295
+NUMERIC_BLOCK_SIZE	1
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
index 41dec216ddb..3ba61fa8915 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
@@ -1439,20 +1439,6 @@ NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
-VARIABLE_NAME	IN_SUBQUERY_CONVERSION_THRESHOLD
-SESSION_VALUE	1000
-GLOBAL_VALUE	1000
-GLOBAL_VALUE_ORIGIN	COMPILE-TIME
-DEFAULT_VALUE	1000
-VARIABLE_SCOPE	SESSION
-VARIABLE_TYPE	BIGINT UNSIGNED
-VARIABLE_COMMENT	The minimum number of scalar elements in the value list of IN predicate that triggers its conversion to IN subquery
-NUMERIC_MIN_VALUE	0
-NUMERIC_MAX_VALUE	18446744073709551615
-NUMERIC_BLOCK_SIZE	1
-ENUM_VALUE_LIST	NULL
-READ_ONLY	NO
-COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	IN_TRANSACTION
 SESSION_VALUE	0
 GLOBAL_VALUE	NULL
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
index c9aaa2b4dd5..4f5a08211f5 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
@@ -8,6 +8,7 @@ where variable_name not like 'aria%' and
 variable_name not like 'debug%' and
 variable_name not like 'wsrep%' and
 variable_name not in (
+'in_predicate_conversion_threshold',
 'have_openssl',
 'have_symlink',
 'hostname',
@@ -1565,20 +1566,6 @@ NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
-VARIABLE_NAME	IN_SUBQUERY_CONVERSION_THRESHOLD
-SESSION_VALUE	1000
-GLOBAL_VALUE	1000
-GLOBAL_VALUE_ORIGIN	COMPILE-TIME
-DEFAULT_VALUE	1000
-VARIABLE_SCOPE	SESSION
-VARIABLE_TYPE	BIGINT UNSIGNED
-VARIABLE_COMMENT	The minimum number of scalar elements in the value list of IN predicate that triggers its conversion to IN subquery
-NUMERIC_MIN_VALUE	0
-NUMERIC_MAX_VALUE	18446744073709551615
-NUMERIC_BLOCK_SIZE	1
-ENUM_VALUE_LIST	NULL
-READ_ONLY	NO
-COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	IN_TRANSACTION
 SESSION_VALUE	0
 GLOBAL_VALUE	NULL
diff --git a/mysql-test/suite/sys_vars/t/sysvars_debug.test b/mysql-test/suite/sys_vars/t/sysvars_debug.test
index fbdcbd683df..71d396e2df3 100644
--- a/mysql-test/suite/sys_vars/t/sysvars_debug.test
+++ b/mysql-test/suite/sys_vars/t/sysvars_debug.test
@@ -4,4 +4,5 @@
 --vertical_results
 select * from information_schema.system_variables
   where variable_name like 'debug%'
+     or variable_name = 'in_predicate_conversion_threshold'
   order by variable_name;
diff --git a/mysql-test/t/mysqld--help.test b/mysql-test/t/mysqld--help.test
index 520384f3ab3..27d3286685b 100644
--- a/mysql-test/t/mysqld--help.test
+++ b/mysql-test/t/mysqld--help.test
@@ -30,7 +30,7 @@ perl;
               feedback debug temp-pool ssl des-key-file xtradb sequence
               thread-concurrency super-large-pages mutex-deadlock-detector
               connect null-audit aria oqgraph sphinx thread-handling
-              test-sql-discovery query-cache-info
+              test-sql-discovery query-cache-info in-predicate-conversion-threshold
               query-response-time metadata-lock-info locales unix-socket
               wsrep file-key-management cracklib-password-check user-variables/;
 
diff --git a/mysql-test/t/opt_tvc.test b/mysql-test/t/opt_tvc.test
index 2e00308aa78..a13386deb6c 100644
--- a/mysql-test/t/opt_tvc.test
+++ b/mysql-test/t/opt_tvc.test
@@ -1,3 +1,8 @@
+#
+# MDEV-12176 Transform [NOT] IN predicate with long list of values INTO [NOT] IN subquery
+#
+source include/have_debug.inc;
+
 create table t1 (a int, b int);
 
 insert into t1
@@ -28,7 +33,7 @@ eval explain extended $query;
 
 --echo # set minimum number of values in VALUEs list when optimization works to 2
 
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 
 --echo # single IN-predicate in WHERE-part
 
@@ -251,27 +256,27 @@ deallocate prepare stmt;
 --echo # use inside out access from tvc rows
 
 let $query= select * from t3 where a in (1,4,10);
-set @@in_subquery_conversion_threshold= default;
+set @@in_predicate_conversion_threshold= default;
 eval $query;
 eval explain extended $query;
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 eval $query;
 eval explain extended $query;
 
 --echo # use vectors in IN predeicate
 
-set @@in_subquery_conversion_threshold= 4;
+set @@in_predicate_conversion_threshold= 4;
 
 let $query=
 select * from t1 where (a,b) in ((1,2),(3,4));
 
 eval $query;
 eval explain extended $query;
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 
 --echo # trasformation works for the one IN predicate and doesn't work for the other
 
-set @@in_subquery_conversion_threshold= 5;
+set @@in_predicate_conversion_threshold= 5;
 
 let $query=
 select * from t2
@@ -280,7 +285,7 @@ where (a,b) in ((1,2),(8,9)) and
 
 eval $query;
 eval explain extended $query;
-set @@in_subquery_conversion_threshold= 2;
+set @@in_predicate_conversion_threshold= 2;
 
 --echo #
 --echo # mdev-14281: conversion of NOT IN predicate into subquery predicate
@@ -312,4 +317,4 @@ eval explain extended $query;
 
 drop table t1, t2, t3;
 
-set @@in_subquery_conversion_threshold= default;
+set @@in_predicate_conversion_threshold= default;
diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test
index ca6ac61d27a..36e0e32b28b 100644
--- a/mysql-test/t/range.test
+++ b/mysql-test/t/range.test
@@ -3,12 +3,6 @@
 #
 --source include/have_innodb.inc
 
-set in_subquery_conversion_threshold=10000;
-
---disable_warnings
-drop table if exists t1, t2, t3, t10, t100;
---enable_warnings
-
 CREATE TABLE t1 (
   event_date date DEFAULT '0000-00-00' NOT NULL,
   type int(11) DEFAULT '0' NOT NULL,
@@ -2050,5 +2044,3 @@ drop table t1,t2,t3;
 --echo #
 --echo # End of 10.2 tests
 --echo #
-
-set in_subquery_conversion_threshold=default;