Fixes some issues in Firstmatch optimization

Allows FirstMatch to handle the case where the fanout of firstmatch tables
is already less than 1.
Also Fixes LooseScan strategy to set position->{records_init, records_out}
(They were set to 0 which also caused assertion failures)

Author: Sergei Petrunia <sergey@mariadb.com>
Reviewer: Monty

3 files changed, 14 insertions, 6 deletions

diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index df70fea9b44..7c9343d77b5 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -3640,12 +3640,12 @@ bool Duplicate_weedout_picker::check_qep(JOIN *join,
 
       if (p->table->emb_sj_nest)
       {
-        sj_inner_fanout= COST_MULT(sj_inner_fanout, p->records_read);
+        sj_inner_fanout= COST_MULT(sj_inner_fanout, p->records_out);
         dups_removed_fanout |= p->table->table->map;
       }
       else
       {
-        sj_outer_fanout= COST_MULT(sj_outer_fanout, p->records_read);
+        sj_outer_fanout= COST_MULT(sj_outer_fanout, p->records_out);
         temptable_rec_size += p->table->table->file->ref_length;
       }
     }
@@ -3664,10 +3664,9 @@ bool Duplicate_weedout_picker::check_qep(JOIN *join,
                                                  sj_outer_fanout,
                                                  temptable_rec_size,
                                                  0, 0);
-    double prefix_record_count= join->positions[first_tab].prefix_record_count;
     double write_cost= (one_cost.create +
-                        prefix_record_count * sj_outer_fanout * one_cost.write);
-    double full_lookup_cost= (prefix_record_count * sj_outer_fanout *
+                        first_weedout_table_rec_count * sj_outer_fanout * one_cost.write);
+    double full_lookup_cost= (first_weedout_table_rec_count* sj_outer_fanout *
                               sj_inner_fanout * one_cost.lookup);
     *read_time= dups_cost + write_cost + full_lookup_cost;
     
@@ -3679,7 +3678,7 @@ bool Duplicate_weedout_picker::check_qep(JOIN *join,
       Json_writer_object trace(join->thd);
       trace.
         add("strategy", "DuplicateWeedout").
-        add("prefix_row_count", prefix_record_count).
+        add("prefix_row_count", first_weedout_table_rec_count).
         add("tmp_table_rows", sj_outer_fanout).
         add("sj_inner_fanout", sj_inner_fanout).
         add("rows", *record_count).
diff --git a/sql/opt_subselect.h b/sql/opt_subselect.h
index 2eb5ff56b02..f59ce0dbb85 100644
--- a/sql/opt_subselect.h
+++ b/sql/opt_subselect.h
@@ -300,6 +300,7 @@ public:
     if (best_loose_scan_cost != DBL_MAX)
     {
       pos->records_read=    best_loose_scan_records;
+      pos->records_init= pos->records_out= pos->records_read;
       pos->key=             best_loose_scan_start_key;
       pos->cond_selectivity= 1.0;
       pos->loosescan_picker.loosescan_key=   best_loose_scan_key;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 62824c44f89..583839fd66d 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -18803,6 +18803,14 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab,
   join->cur_sj_inner_tables= save_cur_sj_inner_tables;
 
   *reopt_cost= cost;
+  if (rec_count < *outer_rec_count)
+  {
+    /*
+      The tables inside the subquery produce smaller fanout than outer tables.
+      This can happen in edge cases.
+    */
+    *outer_rec_count= rec_count;
+  }
 }