Fix LP BUG#772309

Analysis:
The method st_select_lex::optimize_unflattened_subqueries()
incorrectly propagated to each subquery the complete
select_options flag set for the whole query. Among other
flags in select_options, this propagated incorrectly the
STRAIGHT_JOIN flag from the upper query to the subquery.

Solution:
During EXPLAIN set only the SELECT_DESCRIBE bit in the
select_options of the subquery.

3 files changed, 49 insertions, 2 deletions

diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result
index f0ead2f3930..87903694230 100644
--- a/mysql-test/r/subselect4.result
+++ b/mysql-test/r/subselect4.result
@@ -15,7 +15,7 @@ ORDER BY count(*);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	index	NULL	a	5	NULL	2	Using where; Using index; Using temporary
 2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
-3	DEPENDENT SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	no matching row in const table
+3	DEPENDENT SUBQUERY	t3	system	NULL	NULL	NULL	NULL	0	const row not found
 # should not crash the next statement
 SELECT 1 FROM t1
 WHERE NOT EXISTS (SELECT 1 FROM t2 WHERE 1 = (SELECT MIN(t2.b) FROM t3))
@@ -1718,3 +1718,28 @@ SELECT * FROM t1 WHERE a1 IN (SELECT b1 FROM t2 WHERE b1 = b2);
 a1	a2
 set @@optimizer_switch=@save_optimizer_switch;
 drop table t1, t2;
+#                                                                                                                                                     
+# LP BUG#772309 join_tab_cmp_straight(): Assertion `!jt2->emb_sj_nest' failed in maria-5.3-mwl89 with semijoin
+#                                                                                                                                                     
+CREATE TABLE t1 ( f2 int) ;
+INSERT INTO t1 VALUES (0),(0);
+CREATE TABLE t2 ( f1 int NOT NULL ) ;
+INSERT INTO t2 VALUES (0),(0);
+CREATE TABLE t3 ( f1 int NOT NULL , f2 int) ;
+INSERT INTO t3 VALUES (0,0), (0,0);
+EXPLAIN SELECT STRAIGHT_JOIN (
+SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	DEPENDENT SUBQUERY	t3	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where; Using join buffer (flat, BNL join)
+SELECT STRAIGHT_JOIN (
+SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+(
+SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+)
+NULL
+drop table t1, t2, t3;
diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result
index b1832dadab4..136c76a198b 100644
--- a/mysql-test/r/union.result
+++ b/mysql-test/r/union.result
@@ -1632,7 +1632,7 @@ ORDER BY (SELECT a FROM t2 WHERE b = 12);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	UNION	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	
-NULL	UNION	<union1,2>	ALL	NULL	NULL	NULL	NULL	NULL	NULL	Using filesort
+NULL	UNION RESULT	<union1,2>	ALL	NULL	NULL	NULL	NULL	NULL	NULL	Using filesort
 3	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
 Warnings:
 Note	1276	Field or reference 'a' of SELECT #3 was resolved in SELECT #-1
diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test
index f0c6892d15c..c57cdf27444 100644
--- a/mysql-test/t/subselect4.test
+++ b/mysql-test/t/subselect4.test
@@ -1390,3 +1390,25 @@ SELECT * FROM t1 WHERE a1 IN (SELECT b1 FROM t2 WHERE b1 = b2);
 
 set @@optimizer_switch=@save_optimizer_switch;
 drop table t1, t2;
+
+--echo #                                                                                                                                                     
+--echo # LP BUG#772309 join_tab_cmp_straight(): Assertion `!jt2->emb_sj_nest' failed in maria-5.3-mwl89 with semijoin
+--echo #                                                                                                                                                     
+
+CREATE TABLE t1 ( f2 int) ;
+INSERT INTO t1 VALUES (0),(0);
+
+CREATE TABLE t2 ( f1 int NOT NULL ) ;
+INSERT INTO t2 VALUES (0),(0);
+
+CREATE TABLE t3 ( f1 int NOT NULL , f2 int) ;
+INSERT INTO t3 VALUES (0,0), (0,0);
+
+EXPLAIN SELECT STRAIGHT_JOIN (
+        SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+SELECT STRAIGHT_JOIN (
+        SELECT f2 FROM t1 WHERE ( f2 ) IN ( SELECT t3.f2 FROM t3 JOIN t2 ON t2.f1 = 1 )
+);
+
+drop table t1, t2, t3;