Fix crash caused by calling DES_ENCRYPT() without the --des-key-file

option having been passed to the server. (Bug #11643)

6 files changed, 37 insertions, 7 deletions

diff --git a/BitKeeper/etc/config b/BitKeeper/etc/config
index c609fcdbd49..f1e8e29fbfb 100644
--- a/BitKeeper/etc/config
+++ b/BitKeeper/etc/config
@@ -24,7 +24,7 @@ description: MySQL - fast and reliable SQL database
 # repository is commercial it can be an internal email address or "none"
 # to disable logging.
 # 
-logging:  logging@openlogging.org
+logging: none
 # 
 # If this field is set, all checkins will appear to be made by this user,
 # in effect making this a single user package.  Single user packages are
diff --git a/mysql-test/r/func_des_encrypt.result b/mysql-test/r/func_des_encrypt.result
new file mode 100644
index 00000000000..46b30bdab58
--- /dev/null
+++ b/mysql-test/r/func_des_encrypt.result
@@ -0,0 +1,3 @@
+select des_encrypt('hello');
+des_encrypt('hello')
+€Ö2nV“Ø}
diff --git a/mysql-test/t/func_des_encrypt.test b/mysql-test/t/func_des_encrypt.test
new file mode 100644
index 00000000000..201a0051c58
--- /dev/null
+++ b/mysql-test/t/func_des_encrypt.test
@@ -0,0 +1,9 @@
+-- source include/have_openssl.inc
+
+# This test can't be in func_encrypt.test, because it requires
+# --des-key-file to not be set.
+
+#
+# Bug #11643: des_encrypt() causes server to die
+#
+select des_encrypt('hello');
diff --git a/sql/des_key_file.cc b/sql/des_key_file.cc
index c6b4c5f2c34..558e3f16ad2 100644
--- a/sql/des_key_file.cc
+++ b/sql/des_key_file.cc
@@ -22,7 +22,17 @@
 struct st_des_keyschedule des_keyschedule[10];
 uint   des_default_key;
 pthread_mutex_t LOCK_des_key_file;
-static int initialized;
+static int initialized= 0;
+
+void
+init_des_key_file()
+{
+  if (!initialized)
+  {
+    initialized=1;
+    pthread_mutex_init(&LOCK_des_key_file,MY_MUTEX_INIT_FAST);
+  }
+}
 
 /*
  Function which loads DES keys from plaintext file into memory on MySQL
@@ -45,11 +55,7 @@ load_des_key_file(const char *file_name)
   DBUG_ENTER("load_des_key_file");
   DBUG_PRINT("enter",("name: %s",file_name));
 
-  if (!initialized)
-  {
-    initialized=1;
-    pthread_mutex_init(&LOCK_des_key_file,MY_MUTEX_INIT_FAST);
-  }
+  init_des_key_file();
 
   VOID(pthread_mutex_lock(&LOCK_des_key_file));
   if ((file=my_open(file_name,O_RDONLY | O_BINARY ,MYF(MY_WME))) < 0 ||
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index 881a8a7c915..7fb27470b8e 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -388,6 +388,9 @@ String *Item_func_des_encrypt::val_str(String *str)
 
   if (arg_count == 1)
   {
+    /* Make sure LOCK_des_key_file was initialized. */
+    init_des_key_file();
+
     /* Protect against someone doing FLUSH DES_KEY_FILE */
     VOID(pthread_mutex_lock(&LOCK_des_key_file));
     keyschedule= des_keyschedule[key_number=des_default_key];
@@ -398,6 +401,10 @@ String *Item_func_des_encrypt::val_str(String *str)
     key_number= (uint) args[1]->val_int();
     if (key_number > 9)
       goto error;
+
+    /* Make sure LOCK_des_key_file was initialized. */
+    init_des_key_file();
+
     VOID(pthread_mutex_lock(&LOCK_des_key_file));
     keyschedule= des_keyschedule[key_number];
     VOID(pthread_mutex_unlock(&LOCK_des_key_file));
@@ -485,6 +492,10 @@ String *Item_func_des_decrypt::val_str(String *str)
     // Check if automatic key and that we have privilege to uncompress using it
     if (!(current_thd->master_access & SUPER_ACL) || key_number > 9)
       goto error;
+
+    /* Make sure LOCK_des_key_file was initialized. */
+    init_des_key_file();
+
     VOID(pthread_mutex_lock(&LOCK_des_key_file));
     keyschedule= des_keyschedule[key_number];
     VOID(pthread_mutex_unlock(&LOCK_des_key_file));
diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h
index cc58e34d582..5c97269b5ce 100644
--- a/sql/mysql_priv.h
+++ b/sql/mysql_priv.h
@@ -624,6 +624,7 @@ extern char *des_key_file;
 extern struct st_des_keyschedule des_keyschedule[10];
 extern uint des_default_key;
 extern pthread_mutex_t LOCK_des_key_file;
+void init_des_key_file();
 bool load_des_key_file(const char *file_name);
 void free_des_key_file();
 #endif /* HAVE_OPENSSL */