MDEV-22401: Optimizer trace: multi-component range is not printed correctlybb-10.4-mdev22401

KEY_MULTI_RANGE::range_flag does not have correct flag bits for
per-endpoint flags (NEAR_MIN, NEAR_MAX, NO_MIN_RANGE, NO_MAX_RANGE).
It only has bits for flags that describe both endpoints.
So
- Document this.
- Switch optimizer trace to using {start|end}_key.flag values, instead.
  This fixes the bug.
- Switch records_in_column_ranges() to doing that too. (This used to
  work, because KEY_MULTI_RANGE::range_flag had correct flag value
  for the last key component, and EITS only uses one-component
  pseudo-indexes)

2 files changed, 43 insertions, 0 deletions

diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result
index 08c0895e839..1fd22e52243 100644
--- a/mysql-test/main/opt_trace.result
+++ b/mysql-test/main/opt_trace.result
@@ -8261,4 +8261,35 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.semijoin_table_pullout'))
     }
 ]
 drop table t1,t2,t3;
+#
+# MDEV-22401: Optimizer trace: multi-component range is not printed correctly
+#
+create table t1 (kp1 int, kp2 int, key(kp1, kp2));
+insert into t1 values (1,1),(1,5),(5,1),(5,5);
+set optimizer_trace=1;
+select * from t1 force index(kp1) where (kp1=2 and kp2 >=4);
+kp1	kp2
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.range_scan_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+JSON_DETAILED(JSON_EXTRACT(trace, '$**.range_scan_alternatives'))
+[
+    
+    [
+        
+        {
+            "index": "kp1",
+            "ranges": 
+            [
+                "(2,4) <= (kp1,kp2) <= (2)"
+            ],
+            "rowid_ordered": false,
+            "using_mrr": false,
+            "index_only": true,
+            "rows": 1,
+            "cost": 1.3033,
+            "chosen": false,
+            "cause": "cost"
+        }
+    ]
+]
+drop table t1;
 set optimizer_trace='enabled=off';
diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test
index 51950f00781..09b3de77848 100644
--- a/mysql-test/main/opt_trace.test
+++ b/mysql-test/main/opt_trace.test
@@ -567,4 +567,16 @@ select * from t3 where (a,a) in (select t1.a, t2.a from t1, t2 where t1.b=t2.b);
 select JSON_DETAILED(JSON_EXTRACT(trace, '$**.semijoin_table_pullout')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 
 drop table t1,t2,t3;
+
+--echo #
+--echo # MDEV-22401: Optimizer trace: multi-component range is not printed correctly
+--echo #
+
+create table t1 (kp1 int, kp2 int, key(kp1, kp2));
+insert into t1 values (1,1),(1,5),(5,1),(5,5);
+set optimizer_trace=1;
+select * from t1 force index(kp1) where (kp1=2 and kp2 >=4);
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.range_scan_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+drop table t1;
+
 set optimizer_trace='enabled=off';