BUG#11754145 - 45702: IMPOSSIBE TO SPECIFY MYISAM_SORT_BUFFER > 4GB

                      ON 64 BIT MACHINES

PROBLEM: When sorting index during repair of
         myisam tables, due to improper casting
         of buffer size variables value of myisam_
         sort_buffer_size is not set greater than
         4GB.

SOLUTION: Proper casting of buffer size variable.
          myisam_buffer_size changed to unsigned
          long long to handle size > 4GB on
          linux as well as windows.

6 files changed, 20 insertions, 11 deletions

diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc
index 47ca430d160..4f195c63e3e 100644
--- a/storage/myisam/ha_myisam.cc
+++ b/storage/myisam/ha_myisam.cc
@@ -77,10 +77,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
   "disables parallel repair", NULL, NULL,
   1, 1, ULONG_MAX, 1);
 
-static MYSQL_THDVAR_ULONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
+static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
   "The buffer that is allocated when sorting the index when doing "
   "a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL,
-  8192*1024, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD), ULONG_MAX, 1);
+  8192 * 1024, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD), SIZE_T_MAX, 1);
 
 static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG,
   "Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE);
diff --git a/storage/myisam/ha_myisam.h b/storage/myisam/ha_myisam.h
index 7e96b54c57f..94ee8d3b5f3 100644
--- a/storage/myisam/ha_myisam.h
+++ b/storage/myisam/ha_myisam.h
@@ -35,7 +35,6 @@ typedef struct st_ha_create_information HA_CREATE_INFO;
 #define HA_RECOVER_QUICK	8	/* Don't check rows in data file */
 #define HA_RECOVER_OFF         16	/* No automatic recover */
 
-extern ulong myisam_sort_buffer_size;
 extern TYPELIB myisam_recover_typelib;
 extern const char *myisam_recover_names[];
 extern ulonglong myisam_recover_options;
diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c
index 77cd66a4811..c0540ae086d 100644
--- a/storage/myisam/mi_check.c
+++ b/storage/myisam/mi_check.c
@@ -2429,7 +2429,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
 
     if (_create_index_by_sort(&sort_param,
 			      (my_bool) (!(param->testflag & T_VERBOSE)),
-			      (uint) param->sort_buffer_length))
+                              param->sort_buffer_length))
     {
       param->retry_repair=1;
       goto err;
diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c
index d56f1e18930..22fbadfa86a 100644
--- a/storage/myisam/myisamchk.c
+++ b/storage/myisam/myisamchk.c
@@ -297,9 +297,9 @@ static struct my_option my_long_options[] =
     INT_MAX32, (long) MALLOC_OVERHEAD, (long) 1L, 0},
   { "sort_buffer_size", OPT_SORT_BUFFER_SIZE, "",
     &check_param.sort_buffer_length,
-    &check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
+    &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG,
     (long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD),
-    ULONG_MAX, (long) MALLOC_OVERHEAD, (long) 1L, 0},
+    SIZE_T_MAX, (long) MALLOC_OVERHEAD, (long) 1L, 0},
   { "sort_key_blocks", OPT_SORT_KEY_BLOCKS, "",
     &check_param.sort_key_blocks,
     &check_param.sort_key_blocks, 0, GET_ULONG, REQUIRED_ARG,
diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h
index 6b4fc8ea59f..a2fd073f002 100644
--- a/storage/myisam/myisamdef.h
+++ b/storage/myisam/myisamdef.h
@@ -320,9 +320,10 @@ typedef struct st_mi_sort_param
   */
   ulonglong unique[MI_MAX_KEY_SEG+1];
   ulonglong notnull[MI_MAX_KEY_SEG+1];
+  ulonglong sortbuff_size;
 
   my_off_t pos,max_pos,filepos,start_recpos;
-  uint key, key_length,real_key_length,sortbuff_size;
+  uint key, key_length,real_key_length;
   uint maxbuffers, keys, find_length, sort_keys_length;
   my_bool fix_datafile, master;
   my_bool calc_checksum;                /* calculate table checksum */
@@ -771,7 +772,7 @@ pthread_handler_t thr_find_all_keys(void *arg);
 int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file);
 
 int sort_write_record(MI_SORT_PARAM *sort_param);
-int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ulong);
+int _create_index_by_sort(MI_SORT_PARAM *info, my_bool no_messages, ulonglong);
 
 #ifdef __cplusplus
 }
diff --git a/storage/myisam/sort.c b/storage/myisam/sort.c
index 9b628ee2842..70dd0579c79 100644
--- a/storage/myisam/sort.c
+++ b/storage/myisam/sort.c
@@ -99,10 +99,11 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
 */
 
 int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
-			  ulong sortbuff_size)
+			  ulonglong sortbuff_size)
 {
   int error,maxbuffer,skr;
-  uint memavl,old_memavl,keys,sort_length;
+  uint sort_length, keys;
+  ulonglong memavl, old_memavl;
   DYNAMIC_ARRAY buffpek;
   ha_rows records;
   uchar **sort_keys;
@@ -134,6 +135,9 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
   sort_length=	info->key_length;
   LINT_INIT(keys);
 
+  if ((memavl - sizeof(BUFFPEK)) / (sort_length + sizeof(char *)) > UINT_MAX32)
+    memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
+
   while (memavl >= MIN_SORT_BUFFER)
   {
     if ((records < UINT_MAX32) && 
@@ -308,7 +312,8 @@ pthread_handler_t thr_find_all_keys(void *arg)
 {
   MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg;
   int error;
-  uint memavl,old_memavl,keys,sort_length;
+  ulonglong memavl, old_memavl;
+  uint keys, sort_length;
   uint idx, maxbuffer;
   uchar **sort_keys=0;
 
@@ -349,6 +354,10 @@ pthread_handler_t thr_find_all_keys(void *arg)
     sort_length=  sort_param->key_length;
     maxbuffer=    1;
 
+    if ((memavl - sizeof(BUFFPEK)) / (sort_length +
+                                      sizeof(char *)) > UINT_MAX32)
+      memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));    
+
     while (memavl >= MIN_SORT_BUFFER)
     {
       if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <=