Many files:

  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again


sql/ha_innodb.cc:
  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again
innobase/data/data0type.c:
  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again
innobase/dict/dict0crea.c:
  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again
innobase/dict/dict0load.c:
  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again
innobase/include/data0type.h:
  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again
innobase/include/data0type.ic:
  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again
innobase/rem/rem0cmp.c:
  Multiple charset support for InnoDB; note that if you create tables and the default charset-collation is not latin1_swedish_ci, then you cannot use those tables if you downgrade to 4.0 again

6 files changed, 272 insertions, 44 deletions

diff --git a/innobase/data/data0type.c b/innobase/data/data0type.c
index f505cdfb0a2..39c80b773a2 100644
--- a/innobase/data/data0type.c
+++ b/innobase/data/data0type.c
@@ -12,6 +12,13 @@ Created 1/16/1996 Heikki Tuuri
 #include "data0type.ic"
 #endif
 
+/* At the database startup we store the default-charset collation number of
+this MySQL installation to this global variable. If we have < 4.1.2 format
+column definitions, or records in the insert buffer, we use this
+charset-collation code for them. */
+
+ulint	data_mysql_default_charset_coll		= 99999999;
+
 dtype_t		dtype_binary_val = {DATA_BINARY, 0, 0, 0, 0};
 dtype_t* 	dtype_binary 	= &dtype_binary_val;
 
diff --git a/innobase/dict/dict0crea.c b/innobase/dict/dict0crea.c
index d6b1b7261ad..5fc2f26fc31 100644
--- a/innobase/dict/dict0crea.c
+++ b/innobase/dict/dict0crea.c
@@ -1315,7 +1315,7 @@ loop:
 
 	if (error == DB_DUPLICATE_KEY) {
 		mutex_enter(&dict_foreign_err_mutex);
-		ut_sprintf_timestamp(dict_foreign_err_buf);
+		ut_sprintf_timestamp(ebuf);
 		sprintf(ebuf + strlen(ebuf),
 " Error in foreign key constraint creation for table %.500s.\n"
 "A foreign key constraint of name %.500s\n"
diff --git a/innobase/dict/dict0load.c b/innobase/dict/dict0load.c
index e9076db08f3..07c4ef3c683 100644
--- a/innobase/dict/dict0load.c
+++ b/innobase/dict/dict0load.c
@@ -360,6 +360,15 @@ dict_load_columns(
 		field = rec_get_nth_field(rec, 6, &len);
 		prtype = mach_read_from_4(field);
 
+		if (dtype_is_non_binary_string_type(mtype, prtype)
+		    && dtype_get_charset_coll(prtype) == 0) {
+			/* This is a non-binary string type, and the table
+			was created with < 4.1.2. Use the default charset. */
+
+			prtype = dtype_form_prtype(prtype,
+					data_mysql_default_charset_coll);
+		}
+
 		field = rec_get_nth_field(rec, 7, &len);
 		col_len = mach_read_from_4(field);
 
diff --git a/innobase/include/data0type.h b/innobase/include/data0type.h
index f202230bb94..0b8d77ee542 100644
--- a/innobase/include/data0type.h
+++ b/innobase/include/data0type.h
@@ -11,6 +11,8 @@ Created 1/16/1996 Heikki Tuuri
 
 #include "univ.i"
 
+extern ulint	data_mysql_default_charset_coll;
+
 /* SQL data type struct */
 typedef struct dtype_struct		dtype_t;
 
@@ -18,31 +20,62 @@ typedef struct dtype_struct		dtype_t;
 data type */
 extern dtype_t* 	dtype_binary;
 
-/* Data main types of SQL data */
-#define	DATA_VARCHAR	1	/* character varying */
-#define DATA_CHAR	2	/* fixed length character */
+/*-------------------------------------------*/
+/* The 'MAIN TYPE' of a column */
+#define	DATA_VARCHAR	1	/* character varying of the
+				latin1_swedish_ci charset-collation */
+#define DATA_CHAR	2	/* fixed length character of the
+				latin1_swedish_ci charset-collation */
 #define DATA_FIXBINARY	3	/* binary string of fixed length */
 #define DATA_BINARY	4	/* binary string */
-#define DATA_BLOB	5	/* binary large object, or a TEXT type; if
-				prtype & DATA_NONLATIN1 != 0 the data must
-				be compared by MySQL as a whole field; if
-				prtype & DATA_BINARY_TYPE == 0, then this is
-				actually a TEXT column */
+#define DATA_BLOB	5	/* binary large object, or a TEXT type;
+				if prtype & DATA_BINARY_TYPE == 0, then this is
+				actually a TEXT column; see also below about
+				the flag DATA_NONLATIN1 */
 #define	DATA_INT	6	/* integer: can be any size 1 - 8 bytes */
 #define	DATA_SYS_CHILD	7	/* address of the child page in node pointer */
 #define	DATA_SYS	8	/* system column */
+
 /* Data types >= DATA_FLOAT must be compared using the whole field, not as
 binary strings */
+
 #define DATA_FLOAT	9
 #define DATA_DOUBLE	10
 #define DATA_DECIMAL	11	/* decimal number stored as an ASCII string */
-#define	DATA_VARMYSQL	12	/* non-latin1 varying length char */
-#define	DATA_MYSQL	13	/* non-latin1 fixed length char */
+#define	DATA_VARMYSQL	12	/* any charset varying length char */
+#define	DATA_MYSQL	13	/* any charset fixed length char */
+				/* NOTE that 4.1.1 used DATA_MYSQL and
+				DATA_VARMYSQL for all character sets, and the
+				charset-collation for tables created with it
+				can also be latin1_swedish_ci */
 #define DATA_MTYPE_MAX	63	/* dtype_store_for_order_and_null_size()
 				requires the values are <= 63 */
 /*-------------------------------------------*/
-/* In the lowest byte in the precise type we store the MySQL type code
-(not applicable for system columns). */
+/* The 'PRECISE TYPE' of a column */
+/*
+Tables created by a MySQL user have the following convention:
+
+- In the least significant byte in the precise type we store the MySQL type
+code (not applicable for system columns).
+
+- In the second least significant byte we OR flags DATA_NOT_NULL,
+DATA_UNSIGNED, DATA_BINARY_TYPE, DATA_NONLATIN1.
+
+- In the third least significant byte of the precise type of string types we
+store the MySQL charset-collation code. In DATA_BLOB columns created with
+< 4.0.14 we do not actually know if it is a BLOB or a TEXT column. Since there
+are no indexes on prefixes of BLOB or TEXT columns in < 4.0.14, this is no
+problem, though.
+
+Note that versions < 4.1.2 or < 5.0.1 did not store the charset code to the
+precise type, since the charset was always the default charset of the MySQL
+installation. If the stored charset code is 0 in the system table SYS_COLUMNS
+of InnoDB, that means that the default charset of this MySQL installation
+should be used.
+
+InnoDB's own internal system tables have different precise types for their
+columns, and for them the precise type is usually not used at all.
+*/
 
 #define DATA_ENGLISH    4       /* English language character string: this
 				is a relic from pre-MySQL time and only used
@@ -69,7 +102,7 @@ be less than 256 */
 #define DATA_MIX_ID_LEN	9	/* maximum stored length for mix id (in a
 				compressed dulint form) */
 #define	DATA_N_SYS_COLS 4 	/* number of system columns defined above */
-/*-------------------------------------------*/
+
 /* Flags ORed to the precise data type */
 #define DATA_NOT_NULL	256	/* this is ORed to the precise type when
 				the column is declared as NOT NULL */
@@ -79,20 +112,58 @@ be less than 256 */
 				string, this is ORed to the precise type:
 				this only holds for tables created with
 				>= MySQL-4.0.14 */
-#define	DATA_NONLATIN1 2048	/* if the data type is a DATA_BLOB (actually
-				TEXT) of a non-latin1 type, this is ORed to
-				the precise type: this only holds for tables
-				created with >= MySQL-4.0.14 */
+#define	DATA_NONLATIN1 2048	/* If the data type is DATA_BLOB with
+				the prtype & DATA_BINARY_TYPE == 0, that is,
+				TEXT, then in versions 4.0.14 - 4.0.xx this
+				flag is set to 1, if the charset is not
+				latin1. In version 4.1.1 this was set
+				to 1 for all TEXT columns. In versions >= 4.1.2
+				this is set to 1 if the charset-collation of a
+				TEXT column is not latin1_swedish_ci. */
 /*-------------------------------------------*/
 
 /* This many bytes we need to store the type information affecting the
 alphabetical order for a single field and decide the storage size of an
 SQL null*/
-#define DATA_ORDER_NULL_TYPE_BUF_SIZE	4
-/* In the >= 4.1.x storage format we need 2 bytes more for the charset */
+#define DATA_ORDER_NULL_TYPE_BUF_SIZE		4
+/* In the >= 4.1.x storage format we add 2 bytes more so that we can also
+store the charset-collation number; one byte is left unused, though */
 #define DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE	6
 
 /*************************************************************************
+Checks if a data main type is a string type. Also a BLOB is considered a
+string type. */
+UNIV_INLINE
+ibool
+dtype_is_string_type(
+/*=================*/
+			/* out: TRUE if string type */
+	ulint	mtype);	/* in: InnoDB main data type code: DATA_CHAR, ... */
+/*************************************************************************
+Checks if a type is a binary string type. Note that for tables created with
+< 4.0.14, we do not know if a DATA_BLOB column is a BLOB or a TEXT column. For
+those DATA_BLOB columns this function currently returns FALSE. */
+UNIV_INLINE
+ibool
+dtype_is_binary_string_type(
+/*========================*/
+			/* out: TRUE if binary string type */
+	ulint	mtype,	/* in: main data type */
+	ulint	prtype);/* in: precise type */
+/*************************************************************************
+Checks if a type is a non-binary string type. That is, dtype_is_string_type is
+TRUE and dtype_is_binary_string_type is FALSE. Note that for tables created
+with < 4.0.14, we do not know if a DATA_BLOB column is a BLOB or a TEXT column.
+For those DATA_BLOB columns this function currently returns FALSE. */
+
+UNIV_INLINE
+ibool
+dtype_is_non_binary_string_type(
+/*============================*/
+			/* out: TRUE if non-binary string type */
+	ulint	mtype,	/* in: main data type */
+	ulint	prtype);/* in: precise type */
+/*************************************************************************
 Sets a data type structure. */
 UNIV_INLINE
 void
@@ -126,6 +197,22 @@ dtype_get_prtype(
 /*=============*/
 	dtype_t*	type);
 /*************************************************************************
+Gets the MySQL charset-collation code for MySQL string types. */
+UNIV_INLINE
+ulint
+dtype_get_charset_coll(
+/*===================*/
+	ulint	prtype);/* in: precise data type */
+/*************************************************************************
+Forms a precise type from the < 4.1.2 format precise type plus the
+charset-collation code. */
+ulint
+dtype_form_prtype(
+/*==============*/
+	ulint	old_prtype,	/* in: the MySQL type code and the flags
+				DATA_NONLATIN1 etc. */
+	ulint	charset_coll);	/* in: MySQL charset-collation code */
+/*************************************************************************
 Gets the type length. */
 UNIV_INLINE
 ulint
@@ -225,9 +312,8 @@ dtype_print(
 struct dtype_struct{
 	ulint	mtype;		/* main data type */
 	ulint	prtype;		/* precise type; MySQL data type */
-	ulint	chrset;		/* MySQL character set code */
 
-	/* remaining two fields do not affect alphabetical ordering: */
+	/* the remaining two fields do not affect alphabetical ordering: */
 
 	ulint	len;		/* length */
 	ulint	prec;		/* precision */
diff --git a/innobase/include/data0type.ic b/innobase/include/data0type.ic
index 5d39b3e430b..51a1d593d4b 100644
--- a/innobase/include/data0type.ic
+++ b/innobase/include/data0type.ic
@@ -9,6 +9,70 @@ Created 1/16/1996 Heikki Tuuri
 #include "mach0data.h"
 
 /*************************************************************************
+Checks if a data main type is a string type. Also a BLOB is considered a
+string type. */
+
+ibool
+dtype_is_string_type(
+/*=================*/
+			/* out: TRUE if string type */
+	ulint	mtype)	/* in: InnoDB main data type code: DATA_CHAR, ... */
+{
+ 	if (mtype <= DATA_BLOB
+	    || mtype == DATA_MYSQL
+	    || mtype == DATA_VARMYSQL) {
+
+		return(TRUE);
+	}
+
+	return(FALSE);
+}
+
+/*************************************************************************
+Checks if a type is a binary string type. Note that for tables created with
+< 4.0.14, we do not know if a DATA_BLOB column is a BLOB or a TEXT column. For
+those DATA_BLOB columns this function currently returns FALSE. */
+UNIV_INLINE
+ibool
+dtype_is_binary_string_type(
+/*========================*/
+			/* out: TRUE if binary string type */
+	ulint	mtype,	/* in: main data type */
+	ulint	prtype)	/* in: precise type */
+{
+        if ((mtype == DATA_FIXBINARY)
+	    || (mtype == DATA_BINARY)
+	    || (mtype == DATA_BLOB && (prtype & DATA_BINARY_TYPE))) {
+
+		return(TRUE);
+	}
+
+	return(FALSE);
+}
+
+/*************************************************************************
+Checks if a type is a non-binary string type. That is, dtype_is_string_type is
+TRUE and dtype_is_binary_string_type is FALSE. Note that for tables created
+with < 4.0.14, we do not know if a DATA_BLOB column is a BLOB or a TEXT column.
+For those DATA_BLOB columns this function currently returns FALSE. */
+UNIV_INLINE
+ibool
+dtype_is_non_binary_string_type(
+/*============================*/
+			/* out: TRUE if non-binary string type */
+	ulint	mtype,	/* in: main data type */
+	ulint	prtype)	/* in: precise type */
+{
+	if (dtype_is_string_type(mtype) == TRUE
+	    && dtype_is_binary_string_type(mtype, prtype) == FALSE) {
+		
+		return(TRUE);
+	}
+
+	return(FALSE);
+}
+
+/*************************************************************************
 Sets a data type structure. */
 UNIV_INLINE
 void
@@ -27,7 +91,6 @@ dtype_set(
 	type->prtype = prtype;
 	type->len = len;
 	type->prec = prec;
-	type->chrset = 0;
 
 	ut_ad(dtype_validate(type));
 }
@@ -73,6 +136,33 @@ dtype_get_prtype(
 }
 
 /*************************************************************************
+Gets the MySQL charset-collation code for MySQL string types. */
+UNIV_INLINE
+ulint
+dtype_get_charset_coll(
+/*===================*/
+	ulint	prtype)	/* in: precise data type */
+{
+	return((prtype >> 16) & 0xFF);
+}
+
+/*************************************************************************
+Forms a precise type from the < 4.1.2 format precise type plus the
+charset-collation code. */
+ulint
+dtype_form_prtype(
+/*==============*/
+	ulint	old_prtype,	/* in: the MySQL type code and the flags
+				DATA_NONLATIN1 etc. */
+	ulint	charset_coll)	/* in: MySQL charset-collation code */
+{
+	ut_a(old_prtype < 256 * 256);
+	ut_a(charset_coll < 256);
+
+	return(old_prtype + (charset_coll << 16));
+}
+
+/*************************************************************************
 Gets the type length. */
 UNIV_INLINE
 ulint
@@ -155,12 +245,16 @@ dtype_new_store_for_order_and_null_size(
 
 	mach_write_to_2(buf + 2, type->len & 0xFFFFUL);
 
-	mach_write_to_2(buf + 4, type->chrset & 0xFFFFUL);
+	mach_write_to_2(buf + 4, dtype_get_charset_coll(type->prtype));
+
+	/* Note that the second last byte is left unused, because the
+	charset-collation code is always < 256 */
 }
 
 /**************************************************************************
 Reads to a type the stored information which determines its alphabetical
-ordering and the storage size of an SQL NULL value. */
+ordering and the storage size of an SQL NULL value. This is the < 4.1.x
+storage format. */
 UNIV_INLINE
 void
 dtype_read_for_order_and_null_size(
@@ -182,12 +276,15 @@ dtype_read_for_order_and_null_size(
 	}
 
 	type->len = mach_read_from_2(buf + 2);
+		
+	type->prtype = dtype_form_prtype(type->prtype,
+					data_mysql_default_charset_coll);
 }	
 
 /**************************************************************************
 Reads to a type the stored information which determines its alphabetical
-ordering and the storage size of an SQL NULL value. This is the 4.1.x storage
-format. */
+ordering and the storage size of an SQL NULL value. This is the >= 4.1.x
+storage format. */
 UNIV_INLINE
 void
 dtype_new_read_for_order_and_null_size(
@@ -195,6 +292,8 @@ dtype_new_read_for_order_and_null_size(
 	dtype_t*	type,	/* in: type struct */
 	byte*		buf)	/* in: buffer for stored type order info */
 {
+	ulint	charset_coll;
+
 	ut_ad(6 == DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE);
 	
 	type->mtype = buf[0] & 63;
@@ -210,8 +309,26 @@ dtype_new_read_for_order_and_null_size(
 
 	type->len = mach_read_from_2(buf + 2);
 
-	type->chrset = mach_read_from_2(buf + 4);
-}	
+	mach_read_from_2(buf + 4);
+
+	charset_coll = mach_read_from_2(buf + 4);
+
+	if (dtype_is_string_type(type->mtype)) {
+		ut_a(charset_coll < 256);
+
+		if (charset_coll == 0) {
+			/* This insert buffer record was inserted with MySQL
+			version < 4.1.2, and the charset-collation code was not
+			explicitly stored to dtype->prtype at that time. It
+			must be the default charset-collation of this MySQL
+			installation. */
+
+			charset_coll = data_mysql_default_charset_coll;
+		}
+		
+		type->prtype = dtype_form_prtype(type->prtype, charset_coll);
+	}						
+}
 
 /***************************************************************************
 Returns the size of a fixed size data type, 0 if not a fixed size type. */
diff --git a/innobase/rem/rem0cmp.c b/innobase/rem/rem0cmp.c
index abfba3a31c9..b6f32c83b35 100644
--- a/innobase/rem/rem0cmp.c
+++ b/innobase/rem/rem0cmp.c
@@ -61,10 +61,11 @@ must be a copy of the the one in ha_innobase.cc! */
 
 int
 innobase_mysql_cmp(
-/*===============*/	
+/*===============*/
 					/* out: 1, 0, -1, if a is greater,
 					equal, less than b, respectively */
-	int		mysql_type,	/* in: MySQL type */ 
+	int		mysql_type,	/* in: MySQL type */
+	uint		charset_number,	/* in: number of the charset */
 	unsigned char*	a,		/* in: data field */
 	unsigned int	a_length,	/* in: data field length,
 					not UNIV_SQL_NULL */
@@ -97,16 +98,28 @@ cmp_types_are_equal(
 	dtype_t*	type1,	/* in: type 1 */
 	dtype_t*	type2)	/* in: type 2 */
 {
-        if ((type1->mtype == DATA_VARCHAR && type2->mtype == DATA_CHAR)
-          || (type1->mtype == DATA_CHAR && type2->mtype == DATA_VARCHAR)
-          || (type1->mtype == DATA_FIXBINARY && type2->mtype == DATA_BINARY)
-          || (type1->mtype == DATA_BINARY && type2->mtype == DATA_FIXBINARY)
-          || (type1->mtype == DATA_MYSQL && type2->mtype == DATA_VARMYSQL)
-          || (type1->mtype == DATA_VARMYSQL && type2->mtype == DATA_MYSQL)) {
-
-                return(TRUE);
+	if (dtype_is_non_binary_string_type(type1->mtype, type1->prtype)
+	    && dtype_is_non_binary_string_type(type2->mtype, type2->prtype)) {
+
+		/* Both are non-binary string types: they can be compared if
+		and only if the charset-collation is the same */
+
+		if (dtype_get_charset_coll(type1->prtype)
+				== dtype_get_charset_coll(type2->prtype)) {
+			return(TRUE);
+		}
+
+		return(FALSE);
         }
 
+	if (dtype_is_binary_string_type(type1->mtype, type1->prtype)
+	    && dtype_is_binary_string_type(type2->mtype, type2->prtype)) {
+
+		/* Both are binary string types: they can be compared */
+
+		return(TRUE);
+	}
+	
         if (type1->mtype != type2->mtype) {
 
 		return(FALSE);
@@ -128,11 +141,6 @@ cmp_types_are_equal(
 		return(FALSE);
 	}
 
-	if (type1->mtype == DATA_BLOB && (type1->prtype & DATA_BINARY_TYPE)
-			           != (type2->prtype & DATA_BINARY_TYPE)) {
-	        return(FALSE);
-	} 
-
 	return(TRUE);
 }
 
@@ -269,6 +277,7 @@ cmp_whole_field(
 
 		return(innobase_mysql_cmp(
 				(int)(type->prtype & DATA_MYSQL_TYPE_MASK),
+				(uint)dtype_get_charset_coll(type->prtype),
 				a, a_length, b, b_length));
 	default:
 	        fprintf(stderr,