merged

BitKeeper/etc/ignore:
  auto-union
myisam/myisamdef.h:
  Auto merged
sql/ha_berkeley.cc:
  Auto merged
sql/ha_innodb.h:
  Auto merged
sql/ha_myisam.cc:
  Auto merged
sql/handler.h:
  Auto merged
sql/item.cc:
  Auto merged
sql/item_func.cc:
  Auto merged
sql/mysql_priv.h:
  Auto merged
sql/mysqld.cc:
  Auto merged
sql/opt_range.cc:
  Auto merged
sql/slave.cc:
  Auto merged
sql/sql_acl.cc:
  Auto merged
sql/sql_base.cc:
  Auto merged
sql/sql_show.cc:
  Auto merged
sql/sql_test.cc:
  Auto merged
sql/table.cc:
  Auto merged

9 files changed, 175 insertions, 49 deletions

diff --git a/myisam/ft_boolean_search.c b/myisam/ft_boolean_search.c
index 2fbd6490cfd..104acf02324 100644
--- a/myisam/ft_boolean_search.c
+++ b/myisam/ft_boolean_search.c
@@ -168,7 +168,11 @@ static void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
         ftbw->word[0]=w.len;
         if (param.yesno > 0) up->ythresh++;
         queue_insert(& ftb->queue, (byte *)ftbw);
+#ifdef TO_BE_REMOVED
+        /* after removing the following line,
+           ftb->with_scan handling can be simplified (no longer a bitmap) */
         ftb->with_scan|=(param.trunc & FTB_FLAG_TRUNC);
+#endif
         break;
       case 2: /* left bracket */
         ftbe=(FTB_EXPR *)alloc_root(&ftb->mem_root, sizeof(FTB_EXPR));
@@ -387,25 +391,34 @@ FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query,
 }
 
 
-/* returns 1 if str0 contain str1 */
+/* returns 1 if str0 ~= /\<str1\>/ */
 static int _ftb_strstr(const byte *s0, const byte *e0,
                 const byte *s1, const byte *e1,
                 CHARSET_INFO *cs)
 {
-  const byte *p;
+  const byte *p0, *p1;
+  my_bool s_after, e_before;
 
-  while (s0 < e0)
+  s_after=true_word_char(cs, s1[0]);
+  e_before=true_word_char(cs, e1[-1]);
+  p0=s0;
+
+  while (p0 < e0)
   {
-    while (s0 < e0 && cs->to_upper[(uint) (uchar) *s0++] !=
+    while (p0 < e0 && cs->to_upper[(uint) (uchar) *p0++] !=
 	   cs->to_upper[(uint) (uchar) *s1])
       /* no-op */;
-    if (s0 >= e0)
+    if (p0 >= e0)
       return 0;
-    p=s1+1;
-    while (s0 < e0 && p < e1 && cs->to_upper[(uint) (uchar) *s0] ==
-	   cs->to_upper[(uint) (uchar) *p])
-      s0++, p++;
-    if (p >= e1)
+
+    if (s_after && p0-1 > s0 && true_word_char(cs, p0[-2]))
+      continue;
+
+    p1=s1+1;
+    while (p0 < e0 && p1 < e1 && cs->to_upper[(uint) (uchar) *p0] ==
+	   cs->to_upper[(uint) (uchar) *p1])
+      p0++, p1++;
+    if (p1 == e1 && (!e_before || p0 == e0 || !true_word_char(cs, p0[0])))
       return 1;
   }
   return 0;
diff --git a/myisam/ft_parser.c b/myisam/ft_parser.c
index 14c67333734..57b379cfac0 100644
--- a/myisam/ft_parser.c
+++ b/myisam/ft_parser.c
@@ -73,15 +73,6 @@ FT_WORD * ft_linearize(TREE *wtree)
   DBUG_RETURN(wlist);
 }
 
-#define true_word_char(s,X)	(my_isalnum(s,X) || (X)=='_')
-#ifdef HYPHEN_IS_DELIM
-#define misc_word_char(X)	((X)=='\'')
-#else
-#define misc_word_char(X)	((X)=='\'' || (X)=='-')
-#endif
-#define word_char(s,X)		(true_word_char(s,X) || misc_word_char(X))
-
-
 /* returns:
  * 0 - eof
  * 1 - word found
diff --git a/myisam/ft_update.c b/myisam/ft_update.c
index cdf3b306087..8423b6898cd 100644
--- a/myisam/ft_update.c
+++ b/myisam/ft_update.c
@@ -21,13 +21,6 @@
 #include "ftdefs.h"
 #include <math.h>
 
-/**************************************************************
-   This is to make ft-code to ignore keyseg.length at all     *
-   and to index the whole VARCHAR/BLOB instead...             */
-#undef set_if_smaller
-#define set_if_smaller(A,B)                          /* no op */
-/**************************************************************/
-
 void _mi_ft_segiterator_init(MI_INFO *info, uint keynr, const byte *record,
 			     FT_SEG_ITERATOR *ftsi)
 {
@@ -88,7 +81,6 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
   {
     ftsi->len=uint2korr(ftsi->pos);
     ftsi->pos+=2;				 /* Skip VARCHAR length */
-    set_if_smaller(ftsi->len,ftsi->seg->length);
     DBUG_RETURN(1);
   }
   if (ftsi->seg->flag & HA_BLOB_PART)
@@ -96,7 +88,6 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
     ftsi->len=_mi_calc_blob_length(ftsi->seg->bit_start,ftsi->pos);
     memcpy_fixed((char*) &ftsi->pos, ftsi->pos+ftsi->seg->bit_start,
 		 sizeof(char*));
-    set_if_smaller(ftsi->len,ftsi->seg->length);
     DBUG_RETURN(1);
   }
   ftsi->len=ftsi->seg->length;
@@ -305,3 +296,53 @@ uint _ft_make_key(MI_INFO *info, uint keynr, byte *keybuf, FT_WORD *wptr,
   memcpy(buf+HA_FT_WLEN+2,wptr->pos,wptr->len);
   DBUG_RETURN(_mi_make_key(info,keynr,(uchar*) keybuf,buf,filepos));
 }
+
+/*
+  convert key value to ft2
+*/
+uint _mi_ft_convert_to_ft2(MI_INFO *info, uint keynr, uchar *key)
+{
+  my_off_t root;
+  DYNAMIC_ARRAY *da=info->ft1_to_ft2;
+  MI_KEYDEF *keyinfo=&info->s->ft2_keyinfo;
+  uchar *key_ptr=dynamic_array_ptr(da, 0), *end;
+  uint length, key_length;
+  DBUG_ENTER("_mi_ft_convert_to_ft2");
+
+  /* we'll generate one pageful at once, and insert the rest one-by-one */
+  /* calculating the length of this page ...*/
+  length=(keyinfo->block_length-2) / keyinfo->keylength;
+  set_if_smaller(length, da->elements);
+  length=length * keyinfo->keylength;
+
+  get_key_full_length_rdonly(key_length, key);
+  while (_mi_ck_delete(info, keynr, key, key_length) == 0)
+    /* nothing to do here.
+       _mi_ck_delete() will populate info->ft1_to_ft2 with deleted keys
+     */;
+
+  /* creating pageful of keys */
+  mi_putint(info->buff,length+2,0);
+  memcpy(info->buff+2, key_ptr, length);
+  info->buff_used=info->page_changed=1;           /* info->buff is used */
+  if ((root= _mi_new(info,keyinfo)) == HA_OFFSET_ERROR ||
+      _mi_write_keypage(info,keyinfo,root,info->buff))
+    DBUG_RETURN(-1);
+
+  /* inserting the rest of key values */
+  end=dynamic_array_ptr(da, da->elements);
+  for (key_ptr+=length; key_ptr < end; key_ptr+=keyinfo->keylength)
+    if(_mi_ck_real_write_btree(info, keyinfo, key_ptr, 0, &root, SEARCH_SAME))
+      DBUG_RETURN(-1);
+
+  /* now, writing the word key entry */
+  ft_intXstore(key+key_length, -da->elements);
+  _mi_dpointer(info, key+key_length+HA_FT_WLEN, root);
+
+  DBUG_RETURN(_mi_ck_real_write_btree(info,
+                                     info->s->keyinfo+keynr,
+                                     key, 0,
+                                     &info->s->state.key_root[keynr],
+                                     SEARCH_SAME));
+}
+
diff --git a/myisam/ftdefs.h b/myisam/ftdefs.h
index 88d7e79937b..c0a19262dcf 100644
--- a/myisam/ftdefs.h
+++ b/myisam/ftdefs.h
@@ -22,8 +22,9 @@
 #include <m_ctype.h>
 #include <my_tree.h>
 
-#define HYPHEN_IS_DELIM
-#define HYPHEN_IS_CONCAT     /* not used for now */
+#define true_word_char(s,X)	(my_isalnum(s,X) || (X)=='_')
+#define misc_word_char(X)	((X)=='\'')
+#define word_char(s,X)		(true_word_char(s,X) || misc_word_char(X))
 
 #define COMPILE_STOPWORDS_IN
 
diff --git a/myisam/fulltext.h b/myisam/fulltext.h
index ec267eb3e86..d8c74d4e94b 100644
--- a/myisam/fulltext.h
+++ b/myisam/fulltext.h
@@ -34,3 +34,5 @@ int  _mi_ft_cmp(MI_INFO *, uint, const byte *, const byte *);
 int  _mi_ft_add(MI_INFO *, uint, byte *, const byte *, my_off_t);
 int  _mi_ft_del(MI_INFO *, uint, byte *, const byte *, my_off_t);
 
+uint _mi_ft_convert_to_ft2(MI_INFO *, uint, uchar *);
+
diff --git a/myisam/mi_delete.c b/myisam/mi_delete.c
index 2ab5c5d0319..d8e1aef5eb6 100644
--- a/myisam/mi_delete.c
+++ b/myisam/mi_delete.c
@@ -18,6 +18,7 @@
 
 #include "fulltext.h"
 #include "rt_index.h"
+#include <assert.h>
 
 #ifdef	__WIN__
 #include <errno.h>
@@ -231,13 +232,22 @@ static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
 
     get_key_full_length_rdonly(off, lastkey);
     subkeys=ft_sintXkorr(lastkey+off);
+    DBUG_ASSERT(info->ft1_to_ft2==0 || subkeys >=0);
     comp_flag=SEARCH_SAME;
     if (subkeys >= 0)
     {
       /* normal word, one-level tree structure */
-      DBUG_PRINT("info",("FT1"));
-      flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key,USE_WHOLE_KEY,
-                                  comp_flag, &keypos, lastkey, &last_key);
+      if (info->ft1_to_ft2)
+      {
+        /* we're in ft1->ft2 conversion mode. Saving key data */
+        insert_dynamic(info->ft1_to_ft2, lastkey+off);
+      }
+      else
+      {
+        /* we need exact match only if not in ft1->ft2 conversion mode */
+        flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key,USE_WHOLE_KEY,
+                                    comp_flag, &keypos, lastkey, &last_key);
+      }
       /* fall through to normal delete */
     }
     else
@@ -252,13 +262,11 @@ static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
       if (subkeys == -1)
       {
         /* the last entry in sub-tree */
-        DBUG_PRINT("info",("FT2: the last entry"));
         _mi_dispose(info, keyinfo, root);
         /* fall through to normal delete */
       }
       else
       {
-        DBUG_PRINT("info",("FT2: going down"));
         keyinfo=&info->s->ft2_keyinfo;
         kpos-=keyinfo->keylength+nod_flag; /* we'll modify key entry 'in vivo' */
         key+=off;
diff --git a/myisam/mi_open.c b/myisam/mi_open.c
index c4b24acdb77..744bb9bb3b6 100644
--- a/myisam/mi_open.c
+++ b/myisam/mi_open.c
@@ -513,8 +513,8 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 		       NullS))
     goto err;
   errpos=6;
-  
-  if (!have_rtree) 
+
+  if (!have_rtree)
     info.rtree_recursion_state= NULL;
 
   strmov(info.filename,org_name);
@@ -536,6 +536,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
   info.lock_type=F_UNLCK;
   info.quick_mode=0;
   info.bulk_insert=0;
+  info.ft1_to_ft2=0;
   info.errkey= -1;
   info.page_changed=1;
   pthread_mutex_lock(&share->intern_lock);
@@ -1112,7 +1113,7 @@ char *mi_recinfo_read(char *ptr, MI_COLUMNDEF *recinfo)
 
 /**************************************************************************
 Open data file with or without RAID
-We can't use dup() here as the data file descriptors need to have different 
+We can't use dup() here as the data file descriptors need to have different
 active seek-positions.
 
 The argument file_to_dup is here for the future if there would on some OS
diff --git a/myisam/mi_write.c b/myisam/mi_write.c
index 8e0b7e3530c..b6a7bf50dd0 100644
--- a/myisam/mi_write.c
+++ b/myisam/mi_write.c
@@ -18,6 +18,7 @@
 
 #include "fulltext.h"
 #include "rt_index.h"
+#include <assert.h>
 
 #ifdef	__WIN__
 #include <errno.h>
@@ -124,7 +125,7 @@ int mi_write(MI_INFO *info, byte *record)
       else
       {
         if (share->keyinfo[i].ck_insert(info,i,buff,
-        			_mi_make_key(info,i,buff,record,filepos)))
+			_mi_make_key(info,i,buff,record,filepos)))
         {
           if (local_lock_tree)
             rw_unlock(&share->key_root_lock[i]);
@@ -264,13 +265,32 @@ int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key,
   else
     comp_flag=SEARCH_SAME;			/* Keys in rec-pos order */
 
+  error=_mi_ck_real_write_btree(info, keyinfo, key, key_length,
+                                root, comp_flag);
+  if (info->ft1_to_ft2)
+  {
+    if (!error)
+      error= _mi_ft_convert_to_ft2(info, keynr, key);
+    delete_dynamic(info->ft1_to_ft2);
+    my_free(info->ft1_to_ft2, MYF(0));
+    info->ft1_to_ft2=0;
+  }
+  DBUG_RETURN(error);
+} /* _mi_ck_write_btree */
+
+int _mi_ck_real_write_btree(MI_INFO *info, MI_KEYDEF *keyinfo,
+    uchar *key, uint key_length, my_off_t *root, uint comp_flag)
+{
+  int error;
+  DBUG_ENTER("_mi_ck_real_write_btree");
+  /* key_length parameter is used only if comp_flag is SEARCH_FIND */
   if (*root == HA_OFFSET_ERROR ||
       (error=w_search(info, keyinfo, comp_flag, key, key_length,
 		      *root, (uchar *) 0, (uchar*) 0,
 		      (my_off_t) 0, 1)) > 0)
     error=_mi_enlarge_root(info,keyinfo,key,root);
   DBUG_RETURN(error);
-} /* _mi_ck_write_btree */
+} /* _mi_ck_real_write_btree */
 
 
 	/* Make a new root with key as only pointer */
@@ -359,13 +379,11 @@ static int w_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
         keyinfo=&info->s->ft2_keyinfo;
         key+=off;
         keypos-=keyinfo->keylength; /* we'll modify key entry 'in vivo' */
-        if ((error=w_search(info, keyinfo, comp_flag, key, HA_FT_WLEN, root,
-                            (uchar *) 0, (uchar*) 0, (my_off_t) 0, 1)) > 0)
-        {
-          error=_mi_enlarge_root(info, keyinfo, key, &root);
-          _mi_dpointer(info, keypos+HA_FT_WLEN, root);
-        }
+        error=_mi_ck_real_write_btree(info, keyinfo, key, 0,
+                                      &root, comp_flag);
+        _mi_dpointer(info, keypos+HA_FT_WLEN, root);
         subkeys--; /* should there be underflow protection ? */
+        DBUG_ASSERT(subkeys < 0);
         ft_intXstore(keypos, subkeys);
         if (!error)
           error=_mi_write_keypage(info,keyinfo,page,temp_buff);
@@ -410,7 +428,6 @@ int _mi_insert(register MI_INFO *info, register MI_KEYDEF *keyinfo,
 	       uchar *key, uchar *anc_buff, uchar *key_pos, uchar *key_buff,
                uchar *father_buff, uchar *father_key_pos, my_off_t father_page,
 	       my_bool insert_last)
-
 {
   uint a_length,nod_flag;
   int t_length;
@@ -464,8 +481,56 @@ int _mi_insert(register MI_INFO *info, register MI_KEYDEF *keyinfo,
   a_length+=t_length;
   mi_putint(anc_buff,a_length,nod_flag);
   if (a_length <= keyinfo->block_length)
+  {
+    if (keyinfo->block_length - a_length < 32 &&
+        keyinfo->flag & HA_FULLTEXT && key_pos == endpos &&
+        info->s->base.key_reflength <= info->s->base.rec_reflength &&
+        info->s->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD))
+    {
+      /*
+        Normal word. One-level tree. Page is almost full.
+        Let's consider converting.
+        We'll compare 'key' and the first key at anc_buff
+       */
+      uchar *a=key, *b=anc_buff+2+nod_flag;
+      uint alen, blen, ft2len=info->s->ft2_keyinfo.keylength;
+      /* the very first key on the page is always unpacked */
+      DBUG_ASSERT((*b & 128) == 0);
+#if HA_FT_MAXLEN >= 127
+      blen= mi_uint2korr(b); b+=2;
+#else
+      blen= *b++;
+#endif
+      get_key_length(alen,a);
+      DBUG_ASSERT(info->ft1_to_ft2==0);
+      if (alen == blen &&
+          mi_compare_text(keyinfo->seg->charset, a, alen, b, blen, 0)==0)
+      {
+        /* yup. converting */
+        info->ft1_to_ft2=(DYNAMIC_ARRAY *)
+          my_malloc(sizeof(DYNAMIC_ARRAY), MYF(MY_WME));
+        my_init_dynamic_array(info->ft1_to_ft2, ft2len, 300, 50);
+
+        /*
+          now, adding all keys from the page to dynarray
+          if the page is a leaf (if not keys will be deleted later)
+        */
+        if (!nod_flag)
+        {
+          /* let's leave the first key on the page, though, because
+             we cannot easily dispatch an empty page here */
+          b+=blen+ft2len+2;
+          for (a=anc_buff+a_length ; b < a ; b+=ft2len+2)
+            insert_dynamic(info->ft1_to_ft2, b);
+
+          /* fixing the page's length - it contains only one key now */
+          mi_putint(anc_buff,2+blen+ft2len+2,0);
+        }
+        /* the rest will be done when we're back from recursion */
+      }
+    }
     DBUG_RETURN(0);				/* There is room on page */
-
+  }
   /* Page is full */
   if (nod_flag)
     insert_last=0;
diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h
index 9844bb7b36d..2c5d5eca756 100644
--- a/myisam/myisamdef.h
+++ b/myisam/myisamdef.h
@@ -222,7 +222,8 @@ struct st_myisam_info {
   MI_BLOB     *blobs;			/* Pointer to blobs */
   MI_BIT_BUFF  bit_buff;
   /* accumulate indexfile changes between write's */
-  TREE	      *bulk_insert;
+  TREE	        *bulk_insert;
+  DYNAMIC_ARRAY *ft1_to_ft2;            /* used only in ft1->ft2 conversion */
   char *filename;			/* parameter to open filename */
   uchar *buff,				/* Temp area for key */
 	*lastkey,*lastkey2;		/* Last used search key */
@@ -464,6 +465,9 @@ extern int _mi_delete_static_record(MI_INFO *info);
 extern int _mi_cmp_static_record(MI_INFO *info,const byte *record);
 extern int _mi_read_rnd_static_record(MI_INFO*, byte *,my_off_t, my_bool);
 extern int _mi_ck_write(MI_INFO *info,uint keynr,uchar *key,uint length);
+extern int _mi_ck_real_write_btree(MI_INFO *info, MI_KEYDEF *keyinfo,
+                                   uchar *key, uint key_length,
+                                   my_off_t *root, uint comp_flag);
 extern int _mi_enlarge_root(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, my_off_t *root);
 extern int _mi_insert(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,
 		      uchar *anc_buff,uchar *key_pos,uchar *key_buff,