MDEV-27188: Suppress optimizer output when executing preparebb-10.7-mdev27188

- Do not write anything into Optimizer Trace at Prepare phase
- When the query gets an error at Prepare phase, make sure there
  is no trace written, either. This is important as we need to
  produce the same trace for "mtr --ps-protocol" and regular mtr run.
- For other kinds of errors, trace is still produced as it might be
  valuable.

(variant 3)

14 files changed, 236 insertions, 724 deletions

diff --git a/mysql-test/main/opt_trace,ps.rdiff b/mysql-test/main/opt_trace,ps.rdiff
deleted file mode 100644
index 3e2218de673..00000000000
--- a/mysql-test/main/opt_trace,ps.rdiff
+++ /dev/null
@@ -1,92 +0,0 @@
---- /Users/shulga/projects/mariadb/server-10.6/mysql-test/main/opt_trace.result	2021-07-21 19:17:11.000000000 +0700
-+++ /Users/shulga/projects/mariadb/server-10.6/mysql-test/main/opt_trace.reject	2021-07-21 19:17:48.000000000 +0700
-@@ -2829,14 +2829,6 @@
-                   }
-                 },
-                 {
--                  "transformation": {
--                    "select_id": 2,
--                    "from": "IN (SELECT)",
--                    "to": "semijoin",
--                    "chosen": true
--                  }
--                },
--                {
-                   "expanded_query": "/* select#2 */ select t10.pk from t10"
-                 }
-               ]
-@@ -4402,14 +4394,6 @@
-                   }
-                 },
-                 {
--                  "transformation": {
--                    "select_id": 2,
--                    "from": "IN (SELECT)",
--                    "to": "semijoin",
--                    "chosen": true
--                  }
--                },
--                {
-                   "expanded_query": "/* select#2 */ select t_inner_1.a from t1 t_inner_1 join t1 t_inner_2"
-                 }
-               ]
-@@ -4852,14 +4836,6 @@
-                   }
-                 },
-                 {
--                  "transformation": {
--                    "select_id": 2,
--                    "from": "IN (SELECT)",
--                    "to": "semijoin",
--                    "chosen": true
--                  }
--                },
--                {
-                   "expanded_query": "/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1"
-                 }
-               ]
-@@ -4879,14 +4855,6 @@
-                   }
-                 },
-                 {
--                  "transformation": {
--                    "select_id": 3,
--                    "from": "IN (SELECT)",
--                    "to": "semijoin",
--                    "chosen": true
--                  }
--                },
--                {
-                   "expanded_query": "/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4"
-                 }
-               ]
-@@ -6432,14 +6400,6 @@
-                   }
-                 },
-                 {
--                  "transformation": {
--                    "select_id": 2,
--                    "from": "IN (SELECT)",
--                    "to": "semijoin",
--                    "chosen": true
--                  }
--                },
--                {
-                   "expanded_query": "/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1"
-                 }
-               ]
-@@ -6459,14 +6419,6 @@
-                   }
-                 },
-                 {
--                  "transformation": {
--                    "select_id": 3,
--                    "from": "IN (SELECT)",
--                    "to": "semijoin",
--                    "chosen": true
--                  }
--                },
--                {
-                   "expanded_query": "/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4"
-                 }
-               ]
diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result
index b0c2a9ca4d9..18ef904633b 100644
--- a/mysql-test/main/opt_trace.result
+++ b/mysql-test/main/opt_trace.result
@@ -38,37 +38,19 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select * from v1	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select t1.a AS a,t1.b AS b from v1",
         "steps": [
           {
             "view": {
               "table": "v1",
               "select_id": 2,
-              "algorithm": "merged"
+              "algorithm": "merged",
+              "expanded_query": "/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1"
             }
           },
           {
-            "join_preparation": {
-              "select_id": 2,
-              "steps": [
-                {
-                  "expanded_query": "/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1"
-                }
-              ]
-            }
-          },
-          {
-            "expanded_query": "/* select#1 */ select t1.a AS a,t1.b AS b from v1"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "condition_processing": {
               "condition": "WHERE",
               "original_condition": "t1.a = 1",
@@ -189,37 +171,19 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select * from (select * from t1 where t1.a=1)q	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select t1.a AS a,t1.b AS b from (/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1) q",
         "steps": [
           {
             "derived": {
               "table": "q",
               "select_id": 2,
-              "algorithm": "merged"
-            }
-          },
-          {
-            "join_preparation": {
-              "select_id": 2,
-              "steps": [
-                {
-                  "expanded_query": "/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1"
-                }
-              ]
+              "algorithm": "merged",
+              "expanded_query": "/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1"
             }
           },
           {
-            "expanded_query": "/* select#1 */ select t1.a AS a,t1.b AS b from (/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1) q"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "condition_processing": {
               "condition": "WHERE",
               "original_condition": "t1.a = 1",
@@ -341,39 +305,14 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select * from v2	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "view": {
-              "table": "v2",
-              "select_id": 2,
-              "algorithm": "materialized"
-            }
-          },
-          {
-            "join_preparation": {
-              "select_id": 2,
-              "steps": [
-                {
-                  "expanded_query": "/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1 group by t1.b"
-                }
-              ]
-            }
-          },
-          {
-            "expanded_query": "/* select#1 */ select v2.a AS a,v2.b AS b from v2"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select v2.a AS a,v2.b AS b from v2",
         "steps": [
           {
             "join_optimization": {
               "select_id": 2,
+              "expanded_query": "/* select#2 */ select t1.a AS a,t1.b AS b from t1 where t1.a = 1 group by t1.b",
               "steps": [
                 {
                   "condition_processing": {
@@ -590,37 +529,19 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from v2	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select t2.a AS a from v2",
         "steps": [
           {
             "view": {
               "table": "v2",
               "select_id": 2,
-              "algorithm": "merged"
+              "algorithm": "merged",
+              "expanded_query": "/* select#2 */ select t2.a AS a from t2"
             }
           },
           {
-            "join_preparation": {
-              "select_id": 2,
-              "steps": [
-                {
-                  "expanded_query": "/* select#2 */ select t2.a AS a from t2"
-                }
-              ]
-            }
-          },
-          {
-            "expanded_query": "/* select#1 */ select t2.a AS a from v2"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "table_dependencies": [
               {
                 "table": "t2",
@@ -703,39 +624,14 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from v1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "view": {
-              "table": "v1",
-              "select_id": 2,
-              "algorithm": "materialized"
-            }
-          },
-          {
-            "join_preparation": {
-              "select_id": 2,
-              "steps": [
-                {
-                  "expanded_query": "/* select#2 */ select t1.a AS a from t1 group by t1.b"
-                }
-              ]
-            }
-          },
-          {
-            "expanded_query": "/* select#1 */ select v1.a AS a from v1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select v1.a AS a from v1",
         "steps": [
           {
             "join_optimization": {
               "select_id": 2,
+              "expanded_query": "/* select#2 */ select t1.a AS a from t1 group by t1.b",
               "steps": [
                 {
                   "table_dependencies": [
@@ -912,18 +808,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from t1,t2 where t1.a=t2.b+2 and t2.a= t1.b	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.a AS a,t1.b AS b,t1.c AS c,t2.a AS a,t2.b AS b,t2.c AS c from t1 join t2 where t1.a = t2.b + 2 and t2.a = t1.b"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.a AS a,t1.b AS b,t1.c AS c,t2.a AS a,t2.b AS b,t2.c AS c from t1 join t2 where t1.a = t2.b + 2 and t2.a = t1.b",
         "steps": [
           {
             "condition_processing": {
@@ -1163,18 +1050,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 EXPLAIN SELECT DISTINCT a FROM t1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select distinct t1.a AS a from t1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select distinct t1.a AS a from t1",
         "steps": [
           {
             "table_dependencies": [
@@ -1328,18 +1206,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 EXPLAIN SELECT MIN(d) FROM t1 where b=2 and c=3  group by a	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select min(t1.d) AS `MIN(d)` from t1 where t1.b = 2 and t1.c = 3 group by t1.a"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select min(t1.d) AS `MIN(d)` from t1 where t1.b = 2 and t1.c = 3 group by t1.a",
         "steps": [
           {
             "condition_processing": {
@@ -1527,18 +1396,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 EXPLAIN SELECT id,MIN(a),MAX(a) FROM t1 WHERE a>=20010104e0 GROUP BY id	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.`id` AS `id`,min(t1.a) AS `MIN(a)`,max(t1.a) AS `MAX(a)` from t1 where t1.a >= 20010104e0 group by t1.`id`"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.`id` AS `id`,min(t1.a) AS `MIN(a)`,max(t1.a) AS `MAX(a)` from t1 where t1.a >= 20010104e0 group by t1.`id`",
         "steps": [
           {
             "condition_processing": {
@@ -1715,18 +1575,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 EXPLAIN SELECT * FROM t1 WHERE a = 20010104e0 GROUP BY id	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.`id` AS `id`,t1.a AS a from t1 where t1.a = 20010104e0 group by t1.`id`"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.`id` AS `id`,t1.a AS a from t1 where t1.a = 20010104e0 group by t1.`id`",
         "steps": [
           {
             "condition_processing": {
@@ -1930,18 +1781,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain  select * from t1 where a=1 and b=2 order by c limit 1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.pk AS pk,t1.a AS a,t1.b AS b,t1.c AS c,t1.filler AS filler from t1 where t1.a = 1 and t1.b = 2 order by t1.c limit 1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.pk AS pk,t1.a AS a,t1.b AS b,t1.c AS c,t1.filler AS filler from t1 where t1.a = 1 and t1.b = 2 order by t1.c limit 1",
         "steps": [
           {
             "condition_processing": {
@@ -2306,18 +2148,9 @@ explain
 select t1.a from t1 left join t2 on t1.a=t2.a	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.a AS a from (t1 left join t2 on(t1.a = t2.a))"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.a AS a from (t1 left join t2 on(t1.a = t2.a))",
         "steps": [
           {
             "build_equal_items": {
@@ -2443,18 +2276,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from t1 left join t2 on t2.a=t1.a	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.a AS a,t2.a AS a,t2.b AS b from (t1 left join t2 on(t2.a = t1.a))"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.a AS a,t2.a AS a,t2.b AS b from (t1 left join t2 on(t2.a = t1.a))",
         "steps": [
           {
             "build_equal_items": {
@@ -2622,18 +2446,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select t1.a from t1 left join (t2 join t3 on t2.b=t3.b) on t2.a=t1.a and t3.a=t1.a	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.a AS a from (t1 left join (t2 join t3 on(t2.b = t3.b)) on(t2.a = t1.a and t3.a = t1.a))"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.a AS a from (t1 left join (t2 join t3 on(t2.b = t3.b)) on(t2.a = t1.a and t3.a = t1.a))",
         "steps": [
           {
             "build_equal_items": {
@@ -2812,47 +2627,20 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain extended select * from t1 where a in (select pk from t10)	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select t1.a AS a,t1.b AS b from t1 where t1.a in (/* select#2 */ select t10.pk from t10)",
         "steps": [
           {
-            "join_preparation": {
+            "transformation": {
               "select_id": 2,
-              "steps": [
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "materialization",
-                    "sjm_scan_allowed": true,
-                    "possible": true
-                  }
-                },
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "semijoin",
-                    "chosen": true
-                  }
-                },
-                {
-                  "expanded_query": "/* select#2 */ select t10.pk from t10"
-                }
-              ]
+              "from": "IN (SELECT)",
+              "to": "materialization",
+              "sjm_scan_allowed": true,
+              "possible": true
             }
           },
           {
-            "expanded_query": "/* select#1 */ select t1.a AS a,t1.b AS b from t1 where t1.a in (/* select#2 */ select t10.pk from t10)"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "transformation": {
               "select_id": 2,
               "from": "IN (SELECT)",
@@ -3136,18 +2924,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from t1 where pk = 2 and a=5 and b=1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.pk AS pk,t1.a AS a,t1.b AS b from t1 where t1.pk = 2 and t1.a = 5 and t1.b = 1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.pk AS pk,t1.a AS a,t1.b AS b from t1 where t1.pk = 2 and t1.a = 5 and t1.b = 1",
         "steps": [
           {
             "condition_processing": {
@@ -3497,18 +3276,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select f1(a) from t1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select f1(t1.a) AS `f1(a)` from t1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select f1(t1.a) AS `f1(a)` from t1",
         "steps": [
           {
             "table_dependencies": [
@@ -3594,18 +3364,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select f2(a) from t1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select f2(t1.a) AS `f2(a)` from t1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select f2(t1.a) AS `f2(a)` from t1",
         "steps": [
           {
             "table_dependencies": [
@@ -3697,7 +3458,7 @@ a
 2
 select length(trace) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 length(trace)
-2141
+2012
 set optimizer_trace_max_mem_size=100;
 select * from t1;
 a
@@ -3708,10 +3469,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select * from t1	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
-        "steps": [
-        	2041	0
+        "expanded_query": 	1912	0
 set optimizer_trace_max_mem_size=0;
 select * from t1;
 a
@@ -3719,7 +3479,7 @@ a
 2
 select * from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
-select * from t1		2141	0
+select * from t1		2012	0
 drop table t1;
 set optimizer_trace='enabled=off';
 set @@optimizer_trace_max_mem_size= @save_optimizer_trace_max_mem_size;
@@ -3810,18 +3570,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain delete t0,t1 from t0, t1 where t0.a=t1.a and t1.a<3	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select NULL AS `NULL` from t0 join t1 where t0.a = t1.a and t1.a < 3"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select NULL AS `NULL` from t0 join t1 where t0.a = t1.a and t1.a < 3",
         "steps": [
           {
             "condition_processing": {
@@ -4176,35 +3927,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from (select rand() from t1)q	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "derived": {
-              "table": "q",
-              "select_id": 2,
-              "algorithm": "merged"
-            }
-          },
-          {
-            "join_preparation": {
-              "select_id": 2,
-              "steps": [
-                {
-                  "expanded_query": "/* select#2 */ select rand() AS `rand()` from t1"
-                }
-              ]
-            }
-          },
-          {
-            "expanded_query": "/* select#1 */ select rand() AS `rand()` from (/* select#2 */ select rand() AS `rand()` from t1) q"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select rand() AS `rand()` from (/* select#2 */ select rand() AS `rand()` from t1) q",
         "steps": [
           {
             "derived": {
@@ -4217,6 +3942,7 @@ explain select * from (select rand() from t1)q	{
           {
             "join_optimization": {
               "select_id": 2,
+              "expanded_query": "/* select#2 */ select rand() AS `rand()` from t1",
               "steps": [
                 {
                   "table_dependencies": [
@@ -4385,47 +4111,20 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from t1 where a in (select t_inner_1.a from t1 t_inner_1, t1 t_inner_2)	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select t1.a AS a from t1 where t1.a in (/* select#2 */ select t_inner_1.a from t1 t_inner_1 join t1 t_inner_2)",
         "steps": [
           {
-            "join_preparation": {
+            "transformation": {
               "select_id": 2,
-              "steps": [
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "materialization",
-                    "sjm_scan_allowed": true,
-                    "possible": true
-                  }
-                },
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "semijoin",
-                    "chosen": true
-                  }
-                },
-                {
-                  "expanded_query": "/* select#2 */ select t_inner_1.a from t1 t_inner_1 join t1 t_inner_2"
-                }
-              ]
+              "from": "IN (SELECT)",
+              "to": "materialization",
+              "sjm_scan_allowed": true,
+              "possible": true
             }
           },
           {
-            "expanded_query": "/* select#1 */ select t1.a AS a from t1 where t1.a in (/* select#2 */ select t_inner_1.a from t1 t_inner_1 join t1 t_inner_2)"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "transformation": {
               "select_id": 2,
               "from": "IN (SELECT)",
@@ -4835,74 +4534,20 @@ explain select * from t1 t_outer_1,t2 t_outer_2  where t_outer_1.a in (select t_
 t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4)	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select t_outer_1.a AS a,t_outer_2.a AS a from t1 t_outer_1 join t2 t_outer_2 where t_outer_1.a in (/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1) and t_outer_2.a in (/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4)",
         "steps": [
           {
-            "join_preparation": {
+            "transformation": {
               "select_id": 2,
-              "steps": [
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "materialization",
-                    "sjm_scan_allowed": true,
-                    "possible": true
-                  }
-                },
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "semijoin",
-                    "chosen": true
-                  }
-                },
-                {
-                  "expanded_query": "/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1"
-                }
-              ]
-            }
-          },
-          {
-            "join_preparation": {
-              "select_id": 3,
-              "steps": [
-                {
-                  "transformation": {
-                    "select_id": 3,
-                    "from": "IN (SELECT)",
-                    "to": "materialization",
-                    "sjm_scan_allowed": true,
-                    "possible": true
-                  }
-                },
-                {
-                  "transformation": {
-                    "select_id": 3,
-                    "from": "IN (SELECT)",
-                    "to": "semijoin",
-                    "chosen": true
-                  }
-                },
-                {
-                  "expanded_query": "/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4"
-                }
-              ]
+              "from": "IN (SELECT)",
+              "to": "materialization",
+              "sjm_scan_allowed": true,
+              "possible": true
             }
           },
           {
-            "expanded_query": "/* select#1 */ select t_outer_1.a AS a,t_outer_2.a AS a from t1 t_outer_1 join t2 t_outer_2 where t_outer_1.a in (/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1) and t_outer_2.a in (/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4)"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "transformation": {
               "select_id": 2,
               "from": "IN (SELECT)",
@@ -4914,6 +4559,15 @@ t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4)	{
             "transformation": {
               "select_id": 3,
               "from": "IN (SELECT)",
+              "to": "materialization",
+              "sjm_scan_allowed": true,
+              "possible": true
+            }
+          },
+          {
+            "transformation": {
+              "select_id": 3,
+              "from": "IN (SELECT)",
               "to": "semijoin",
               "converted_to_semi_join": true
             }
@@ -6415,74 +6069,20 @@ explain select * from t1 t_outer_1,t2 t_outer_2  where t_outer_1.a in (select t_
 t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4)	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select t_outer_1.a AS a,t_outer_2.a AS a from t1 t_outer_1 join t2 t_outer_2 where t_outer_1.a in (/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1) and t_outer_2.a in (/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4)",
         "steps": [
           {
-            "join_preparation": {
+            "transformation": {
               "select_id": 2,
-              "steps": [
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "materialization",
-                    "sjm_scan_allowed": true,
-                    "possible": true
-                  }
-                },
-                {
-                  "transformation": {
-                    "select_id": 2,
-                    "from": "IN (SELECT)",
-                    "to": "semijoin",
-                    "chosen": true
-                  }
-                },
-                {
-                  "expanded_query": "/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1"
-                }
-              ]
-            }
-          },
-          {
-            "join_preparation": {
-              "select_id": 3,
-              "steps": [
-                {
-                  "transformation": {
-                    "select_id": 3,
-                    "from": "IN (SELECT)",
-                    "to": "materialization",
-                    "sjm_scan_allowed": true,
-                    "possible": true
-                  }
-                },
-                {
-                  "transformation": {
-                    "select_id": 3,
-                    "from": "IN (SELECT)",
-                    "to": "semijoin",
-                    "chosen": true
-                  }
-                },
-                {
-                  "expanded_query": "/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4"
-                }
-              ]
+              "from": "IN (SELECT)",
+              "to": "materialization",
+              "sjm_scan_allowed": true,
+              "possible": true
             }
           },
           {
-            "expanded_query": "/* select#1 */ select t_outer_1.a AS a,t_outer_2.a AS a from t1 t_outer_1 join t2 t_outer_2 where t_outer_1.a in (/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1) and t_outer_2.a in (/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4)"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "transformation": {
               "select_id": 2,
               "from": "IN (SELECT)",
@@ -6494,6 +6094,15 @@ t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4)	{
             "transformation": {
               "select_id": 3,
               "from": "IN (SELECT)",
+              "to": "materialization",
+              "sjm_scan_allowed": true,
+              "possible": true
+            }
+          },
+          {
+            "transformation": {
+              "select_id": 3,
+              "from": "IN (SELECT)",
               "to": "semijoin",
               "converted_to_semi_join": true
             }
@@ -8678,18 +8287,9 @@ query	trace
 SELECT 'a\0' LIMIT 0	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select 'a\0' AS `a\x00` limit 0"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select 'a\0' AS `a\x00` limit 0",
         "steps": []
       }
     },
@@ -8717,18 +8317,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select count(*) from seq_1_to_10000000	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select count(0) AS `count(*)` from seq_1_to_10000000"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select count(0) AS `count(*)` from seq_1_to_10000000",
         "steps": [
           {
             "table_dependencies": [
@@ -8855,50 +8446,6 @@ json_detailed(json_extract(trace, '$**.in_to_subquery_conversion'))
         "item": "t0.a in (1,2,3,4,5,6)",
         "conversion": 
         [
-            
-            {
-                "join_preparation": 
-                {
-                    "select_id": 2,
-                    "steps": 
-                    [
-                        
-                        {
-                            "derived": 
-                            {
-                                "table": "tvc_0",
-                                "select_id": 3,
-                                "algorithm": "materialized"
-                            }
-                        },
-                        
-                        {
-                            "transformation": 
-                            {
-                                "select_id": 2,
-                                "from": "IN (SELECT)",
-                                "to": "materialization",
-                                "sjm_scan_allowed": true,
-                                "possible": true
-                            }
-                        },
-                        
-                        {
-                            "transformation": 
-                            {
-                                "select_id": 2,
-                                "from": "IN (SELECT)",
-                                "to": "semijoin",
-                                "chosen": true
-                            }
-                        },
-                        
-                        {
-                            "expanded_query": "/* select#2 */ select tvc_0._col_1 from (values (1),(2),(3),(4),(5),(6)) tvc_0"
-                        }
-                    ]
-                }
-            }
         ]
     }
 ]
@@ -8954,7 +8501,7 @@ set @path= (select json_search(@trace, 'one', 'no predicate for first keypart'))
 set @sub_path= substr(@path, 2, locate('.best_access_path', @path)-2);
 select @sub_path;
 @sub_path
-$.steps[1].join_optimization.steps[4].considered_execution_plans[0].rest_of_plan[0]
+$.steps[0].join_optimization.steps[4].considered_execution_plans[0].rest_of_plan[0]
 select 
 json_detailed(json_extract(
 @trace,
diff --git a/mysql-test/main/opt_trace_index_merge.result b/mysql-test/main/opt_trace_index_merge.result
index f1e13586eda..b319639e9fa 100644
--- a/mysql-test/main/opt_trace_index_merge.result
+++ b/mysql-test/main/opt_trace_index_merge.result
@@ -20,18 +20,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from t1 where a=1 or b=1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.a AS a,t1.b AS b,t1.c AS c,t1.filler AS filler from t1 where t1.a = 1 or t1.b = 1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.a AS a,t1.b AS b,t1.c AS c,t1.filler AS filler from t1 where t1.a = 1 or t1.b = 1",
         "steps": [
           {
             "condition_processing": {
diff --git a/mysql-test/main/opt_trace_index_merge_innodb.result b/mysql-test/main/opt_trace_index_merge_innodb.result
index 0ddaaeae89d..bc063015e6d 100644
--- a/mysql-test/main/opt_trace_index_merge_innodb.result
+++ b/mysql-test/main/opt_trace_index_merge_innodb.result
@@ -28,18 +28,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 explain select * from t1 where pk1 != 0  and key1 = 1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select t1.pk1 AS pk1,t1.pk2 AS pk2,t1.key1 AS key1,t1.key2 AS key2 from t1 where t1.pk1 <> 0 and t1.key1 = 1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select t1.pk1 AS pk1,t1.pk2 AS pk2,t1.key1 AS key1,t1.key2 AS key2 from t1 where t1.pk1 <> 0 and t1.key1 = 1",
         "steps": [
           {
             "condition_processing": {
diff --git a/mysql-test/main/opt_trace_security.result b/mysql-test/main/opt_trace_security.result
index e1937e744a4..9223f6c0a28 100644
--- a/mysql-test/main/opt_trace_security.result
+++ b/mysql-test/main/opt_trace_security.result
@@ -16,7 +16,6 @@ select * from db1.t1;
 ERROR 42000: SELECT command denied to user 'foo'@'localhost' for table 't1'
 select * from information_schema.OPTIMIZER_TRACE;
 QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
-		0	1
 set optimizer_trace="enabled=off";
 grant select(a)  on db1.t1 to 'foo'@'%';
 set optimizer_trace="enabled=on";
@@ -51,18 +50,9 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select * from db1.t1	{
   "steps": [
     {
-      "join_preparation": {
-        "select_id": 1,
-        "steps": [
-          {
-            "expanded_query": "select db1.t1.a AS a from t1"
-          }
-        ]
-      }
-    },
-    {
       "join_optimization": {
         "select_id": 1,
+        "expanded_query": "select db1.t1.a AS a from t1",
         "steps": [
           {
             "table_dependencies": [
@@ -156,37 +146,19 @@ QUERY	TRACE	MISSING_BYTES_BEYOND_MAX_MEM_SIZE	INSUFFICIENT_PRIVILEGES
 select * from db1.v1	{
   "steps": [
     {
-      "join_preparation": {
+      "join_optimization": {
         "select_id": 1,
+        "expanded_query": "/* select#1 */ select db1.t1.a AS a from v1",
         "steps": [
           {
             "view": {
               "table": "v1",
               "select_id": 2,
-              "algorithm": "merged"
-            }
-          },
-          {
-            "join_preparation": {
-              "select_id": 2,
-              "steps": [
-                {
-                  "expanded_query": "/* select#2 */ select db1.t1.a AS a from t1"
-                }
-              ]
+              "algorithm": "merged",
+              "expanded_query": "/* select#2 */ select db1.t1.a AS a from t1"
             }
           },
           {
-            "expanded_query": "/* select#1 */ select db1.t1.a AS a from v1"
-          }
-        ]
-      }
-    },
-    {
-      "join_optimization": {
-        "select_id": 1,
-        "steps": [
-          {
             "table_dependencies": [
               {
                 "table": "t1",
diff --git a/sql/my_json_writer.h b/sql/my_json_writer.h
index d82313f996f..20f479dc6d0 100644
--- a/sql/my_json_writer.h
+++ b/sql/my_json_writer.h
@@ -224,6 +224,14 @@ public:
 
   size_t get_truncated_bytes() { return output.get_truncated_bytes(); }
 
+  /*
+    Note: this may not return exact value due to pretty-printer doing
+    buffering
+  */
+  size_t get_written_size() {
+    return output.length() + output.get_truncated_bytes();
+  }
+
   Json_writer() : 
     indent_level(0), document_start(true), element_started(false), 
     first_child(true)
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index d91557c5be2..953f2af0be8 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -549,6 +549,12 @@ bool is_materialization_applicable(THD *thd, Item_in_subselect *in_subs,
     and, depending on the rewrite, either do it, or record it to be done at a
     later phase.
 
+    NOTE
+    * This function called at Prepare phase. It should NOT do any rewrites.
+      It only collects information that's used for doing the rewrites at the
+      optimization phase.
+    * Optimizer trace is NOT yet enabled when this function is called.
+
   RETURN
     0      - OK
     Other  - Some sort of query error
@@ -703,8 +709,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
     {
       DBUG_PRINT("info", ("Subquery is semi-join conversion candidate"));
 
-      (void)subquery_types_allow_materialization(thd, in_subs);
-
       in_subs->is_flattenable_semijoin= TRUE;
 
       /* Register the subquery for further processing in flatten_subqueries() */
@@ -717,10 +721,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         if (arena)
           thd->restore_active_arena(arena, &backup);
         in_subs->is_registered_semijoin= TRUE;
-        OPT_TRACE_TRANSFORM(thd, trace_wrapper, trace_transform,
-                            select_lex->select_number,
-                            "IN (SELECT)", "semijoin");
-        trace_transform.add("chosen", true);
       }
     }
     else
@@ -1262,6 +1262,7 @@ bool convert_join_subqueries_to_semijoins(JOIN *join)
   while ((in_subq= li++))
   {
     bool remove_item= TRUE;
+    (void)subquery_types_allow_materialization(thd, in_subq);
 
     /* Stop processing if we've reached a subquery that's attached to the ON clause */
     if (in_subq->do_not_convert_to_sj)
diff --git a/sql/opt_trace.cc b/sql/opt_trace.cc
index ba9220cac44..2227519d991 100644
--- a/sql/opt_trace.cc
+++ b/sql/opt_trace.cc
@@ -106,6 +106,27 @@ inline bool sql_command_can_be_traced(enum enum_sql_command sql_command)
          sql_command == SQLCOM_UPDATE_MULTI;
 }
 
+
+void opt_trace_print_expanded_union(THD *thd, SELECT_LEX_UNIT *unit,
+                                    Json_writer_object *writer)
+{
+  DBUG_ASSERT(thd->trace_started());
+
+  StringBuffer<1024> str(system_charset_info);
+  ulonglong save_option_bits= thd->variables.option_bits;
+  thd->variables.option_bits &= ~OPTION_QUOTE_SHOW_CREATE;
+  unit->print(&str, enum_query_type(QT_TO_SYSTEM_CHARSET |
+                                    QT_SHOW_SELECT_NUMBER |
+                                    QT_ITEM_IDENT_SKIP_DB_NAMES |
+                                    QT_VIEW_INTERNAL));
+  thd->variables.option_bits= save_option_bits;
+  /*
+    The output is not very pretty lots of back-ticks, the output
+    is as the one in explain extended , lets try to improved it here.
+  */
+  writer->add("expanded_query", str.c_ptr_safe(), str.length());
+}
+
 void opt_trace_print_expanded_query(THD *thd, SELECT_LEX *select_lex,
                                     Json_writer_object *writer)
 
@@ -499,10 +520,49 @@ Opt_trace_start::Opt_trace_start(THD *thd, TABLE_LIST *tbl,
   }
 }
 
+
+/*
+  @brief
+    See "Handing Query Errors" section of comment for Opt_trace_start
+*/
+
+void Opt_trace_start::trace_heading_done()
+{
+  Json_writer *w;
+  if (traceable && (w= ctx->get_current_json()))
+    trace_heading_size= w->get_written_size();
+  else
+    trace_heading_size= 0;
+}
+
+
+/*
+  @brief
+    See "Handing Query Errors" section of comment for Opt_trace_start
+
+  @detail
+    We can't delete the trace right now, because some final writes (e.g.
+    the top-level closing '}' will still be made to it. Just set clean_me=true
+    so that it is deleted instead of saving it.
+*/
+
+void Opt_trace_start::clean_empty_trace()
+{
+  Json_writer *w;
+  if (traceable && (w= ctx->get_current_json()))
+  {
+    if (w->get_written_size() == trace_heading_size)
+      clean_me= true;
+  }
+}
+
+
 Opt_trace_start::~Opt_trace_start()
 {
   if (traceable)
   {
+    if (clean_me)
+      ctx->abort_trace();
     ctx->end();
     traceable= FALSE;
   }
diff --git a/sql/opt_trace.h b/sql/opt_trace.h
index 101fb5f707e..c6583cdab80 100644
--- a/sql/opt_trace.h
+++ b/sql/opt_trace.h
@@ -69,10 +69,34 @@ struct Opt_trace_info
   @param  query        query
   @param  length       query's length
   @param  charset      charset which was used to encode this query
+
+  @detail
+    == Lifecycle ==
+    The trace is created before the Name Resolution phase. Reasons:
+    1. This way, we can have one place where we start the trace for all kinds of
+       queries. If we tried to start tracing right before query optimization
+       starts, we would have to construct Opt_trace_start object in many
+       places: one for SELECT, for UPDATE, for DELETE, etc.
+
+    2. Privilege checking code may notify the trace that the user doesn't have
+       enough permissions to perform tracing. The trace must exist to receive
+       the notication. See missing_privilege() and the opt_trace_disable_if_...
+       functions below for details.
+
+    == Handling Query Errors ==
+    The trace is kept when query error occurs, except for the case when
+    nothing [meaningful] was traced. The second part is necessary for mtr to
+    produce the same output with and without --ps-protocol. If there is an
+    error on prepare phase, then:
+     - In --ps-protocol: PREPARE command produces no trace. The EXECUTE
+       command is not run. The trace is not generated at all.
+     - Regular SQL query: should also NOT produce any trace to match the above.
+    This is handled by trace_heading_done() and clean_empty_trace().
 */
 
 
-class Opt_trace_start {
+class Opt_trace_start
+{
  public:
   Opt_trace_start(THD *thd_arg, TABLE_LIST *tbl,
                   enum enum_sql_command sql_command,
@@ -82,8 +106,17 @@ class Opt_trace_start {
                   const CHARSET_INFO *query_charset);
   ~Opt_trace_start();
 
+  void trace_heading_done();
+  void clean_empty_trace();
  private:
   Opt_trace_context *const ctx;
+
+  /* Number of bytes written to the trace after the heading was written/ */
+  size_t trace_heading_size;
+
+  /* If true, trace should be removed (See Handling Query Errors above) */
+  bool clean_me= false;
+
   /*
     True: the query will be traced
     False: otherwise
@@ -102,6 +135,9 @@ class Opt_trace_start {
 void opt_trace_print_expanded_query(THD *thd, SELECT_LEX *select_lex,
                                     Json_writer_object *trace_object);
 
+void opt_trace_print_expanded_union(THD *thd, SELECT_LEX_UNIT *unit,
+                                    Json_writer_object *writer);
+
 void add_table_scan_values_to_trace(THD *thd, JOIN_TAB *tab);
 void trace_plan_prefix(JOIN *join, uint idx, table_map join_tables);
 void print_final_join_order(JOIN *join);
diff --git a/sql/opt_trace_context.h b/sql/opt_trace_context.h
index f578a0c67ec..ae77c94fdc2 100644
--- a/sql/opt_trace_context.h
+++ b/sql/opt_trace_context.h
@@ -114,6 +114,12 @@ public:
 
   bool is_enabled();
 
+  void abort_trace()
+  {
+    delete current_trace;
+    current_trace= NULL;
+  }
+
   void missing_privilege();
 
   static const char *flag_names[];
diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc
index 579ea34b8e4..cf878ad29c0 100644
--- a/sql/sql_derived.cc
+++ b/sql/sql_derived.cc
@@ -420,6 +420,24 @@ bool mysql_derived_merge(THD *thd, LEX *lex, TABLE_LIST *derived)
       goto exit_merge;
     }
 
+    if (unlikely(thd->trace_started()))
+    {
+      /*
+        Add to optimizer trace whether a derived table/view
+        is merged into the parent select or not.
+      */
+      OPT_TRACE_VIEWS_TRANSFORM(thd, trace_wrapper, trace_derived,
+                      derived->is_derived() ? "derived" : "view",
+                      derived->alias.str ? derived->alias.str : "<NULL>",
+                      derived->get_unit()->first_select()->select_number,
+                      derived->is_merged_derived() ? "merged" : "materialized");
+      if (derived->is_merged_derived())
+      {
+        opt_trace_print_expanded_union(thd, derived->get_unit(),
+                                       &trace_derived);
+      }
+    }
+
     /*
       exclude select lex so it doesn't show up in explain.
       do this only for derived table as for views this is already done.
@@ -822,18 +840,6 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived)
     }
   }
 
-  if (unlikely(thd->trace_started()))
-  {
-    /*
-      Add to optimizer trace whether a derived table/view
-      is merged into the parent select or not.
-    */
-    OPT_TRACE_VIEWS_TRANSFORM(thd, trace_wrapper, trace_derived,
-                    derived->is_derived() ? "derived" : "view",
-                    derived->alias.str ? derived->alias.str : "<NULL>",
-                    derived->get_unit()->first_select()->select_number,
-                    derived->is_merged_derived() ? "merged" : "materialized");
-  }
   /*
     Above cascade call of prepare is important for PS protocol, but after it
     is called we can check if we really need prepare for this derived
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index b9d3eec5a60..b19ac276267 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -3648,6 +3648,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt)
 
   Json_writer_object trace_command(thd);
   Json_writer_array trace_command_steps(thd, "steps");
+  ots.trace_heading_done();
 
   /* store old value of binlog format */
   enum_binlog_format orig_binlog_format,orig_current_stmt_binlog_format;
@@ -6154,6 +6155,9 @@ finish:
   thd->wsrep_PA_safe= true;
 #endif /* WITH_WSREP */
 
+  if (res || thd->is_error())
+    ots.clean_empty_trace();
+
   DBUG_RETURN(res || thd->is_error());
  }
 
diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc
index cc6f572ea64..b6bc9eff50e 100644
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@ -2435,17 +2435,6 @@ static bool check_prepared_statement(Prepared_statement *stmt)
   lex->first_select_lex()->context.resolve_in_table_list_only(select_lex->
                                                      get_table_list());
 
-  /*
-    For the optimizer trace, this is the symmetric, for statement preparation,
-    of what is done at statement execution (in mysql_execute_command()).
-  */
-  Opt_trace_start ots(thd, tables, lex->sql_command, &lex->var_list,
-                      thd->query(), thd->query_length(),
-                      thd->variables.character_set_client);
-
-  Json_writer_object trace_command(thd);
-  Json_writer_array trace_command_steps(thd, "steps");
-
   /* Reset warning count for each query that uses tables */
   if (tables)
     thd->get_stmt_da()->opt_clear_warning_info(thd->query_id);
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index a357d4f8c8a..45d8f54e264 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1287,11 +1287,6 @@ JOIN::prepare(TABLE_LIST *tables_init, COND *conds_init, uint og_num,
   join_list= &select_lex->top_join_list;
   union_part= unit_arg->is_unit_op();
 
-  Json_writer_object trace_wrapper(thd);
-  Json_writer_object trace_prepare(thd, "join_preparation");
-  trace_prepare.add_select_number(select_lex->select_number);
-  Json_writer_array trace_steps(thd, "steps");
-
   // simple check that we got usable conds
   dbug_print_item(conds);
 
@@ -1675,12 +1670,6 @@ JOIN::prepare(TABLE_LIST *tables_init, COND *conds_init, uint og_num,
     }
   }
 
-  if (thd->trace_started())
-  {
-    Json_writer_object trace_wrapper(thd);
-    opt_trace_print_expanded_query(thd, select_lex, &trace_wrapper);
-  }
-
   if (!procedure && result && result->prepare(fields_list, unit_arg))
     goto err;					/* purecov: inspected */
 
@@ -1985,7 +1974,11 @@ JOIN::optimize_inner()
 
   Json_writer_object trace_wrapper(thd);
   Json_writer_object trace_prepare(thd, "join_optimization");
-  trace_prepare.add_select_number(select_lex->select_number);
+  if (thd->trace_started())
+  {
+    trace_prepare.add_select_number(select_lex->select_number);
+    opt_trace_print_expanded_query(thd, select_lex, &trace_prepare);
+  }
   Json_writer_array trace_steps(thd, "steps");
 
   /*