MDEV-5241: Collation incompatibilities with MySQL-5.6

- Character set code & tests from Alexander Barkov
- Integration with ALTER TABLE, REPAIR and open_table from Monty

The problem was that MySQL 5.6 added some croatian and vitanamese character set collations that are incompatible with MariaDB.

The fix is to move the MariaDB conflicting collation numbers out of the region that MySQL is likely to use.
mysql_upgrade, REPAIR TABLE or ALTER TABLE will fix the collations.
If one tries to access and old incompatible table, one will get the error "Table upgrade required...."
After this patch, MariaDB supports all the MySQL character set collations and the old MariaDB croatian collations, which are closer to the latest standard than the MySQL versions.

New character sets:
ucs2_croatian_mysql561_uca_ci
utf8_croatian_mysql561_uca_ci
utf16_croatian_mysql561_uca_ci
utf32_croatian_mysql561_uca_ci
utf8mb4_croatian_mysql561_uca_ci

Other things:
- Fixed some compiler warnings
- mysql_upgrade prints information about repaired tables.
- Increased version number

VERSION:
  Increased VERSION number
client/mysqlcheck.c:
  Print repaired table name when using --verbose
include/m_ctype.h:
  Add new MariaDB collation regions that are not likely to conflict with MySQL
include/my_base.h:
  Added flag to detect if table was opened for ALTER TABLE
mysql-test/r/ctype_ldml.result:
  Updated result
mysql-test/r/ctype_uca.result:
  Updated result
mysql-test/r/ctype_upgrade.result:
  Updated result
mysql-test/r/ctype_utf16_uca.result:
  Updated result
mysql-test/r/ctype_utf32_uca.result:
  Updated result
mysql-test/r/ctype_utf8mb4_uca.result:
  Updated result
mysql-test/std_data/ctype_upgrade:
  Test files for testing upgrading of conflicting collations
mysql-test/suite/engines/funcs/r/db_alter_collate_ascii.result:
  New collations added
mysql-test/suite/engines/funcs/r/db_alter_collate_utf8.result:
  New collations added
mysql-test/suite/innodb/r/innodb_ctype_ldml.result:
  Updated test result
mysql-test/suite/innodb/t/innodb_ctype_ldml.test:
  Updated test result
mysql-test/suite/plugins/r/show_all_plugins.result:
  Updated version number
mysql-test/suite/roles/create_and_drop_role_invalid_user_table.result:
  Updated version number
mysql-test/t/ctype_ldml.test:
  Updated test
mysql-test/t/ctype_uca.test:
  Testing of new collations
mysql-test/t/ctype_upgrade.test:
  Testing of upgrading tables with old collations
  The test ensures that:
  - We will get an error if we try to open a table with old collations.
  - CHECK TABLE will detect that the table needs to be upgraded.
  - ALTER TABLE and REPAIR will fix the table.
  - mysql_upgrade works as expected
mysql-test/t/ctype_utf16_uca.test:
  Testing of new collations
mysql-test/t/ctype_utf32_uca.test:
  Testing of new collations
mysql-test/t/ctype_utf8mb4_uca.test:
  Testing of new collations
mysys/charset-def.c:
  Added new character sets
mysys/charset.c:
  Always give an error, if requested, if a character set didn't exist
sql/handler.cc:
  - Added upgrade_collation() to check if collation is compatible with old version
  - check_collation_compatibility() checks if we are using an old collation from MariaDB 5.5 or MySQL 5.6
  - ha_check_for_upgrade() returns HA_ADMIN_NEEDS_ALTER if we have an incompatible collation
sql/handler.h:
  Added new prototypes
sql/sql_table.cc:
  - Mark that tables are opened for ALTER TABLE
  - If table needs to be upgraded, ensure we are not using online alter table.
sql/table.cc:
  - If we are using an old incompatible collation, change to use the new one and mark table as incompatible.
  - Give an error if we try to open an incompatible table.
sql/table.h:
  Added error that table needs to be rebuild
storage/connect/ha_connect.cc:
  Fixed compiler warning
strings/ctype-uca.c:
  New character sets

5 files changed, 109 insertions, 12 deletions

diff --git a/sql/handler.cc b/sql/handler.cc
index dece51839c7..28422794969 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -3534,6 +3534,38 @@ bool handler::get_error_message(int error, String* buf)
   return FALSE;
 }
 
+/**
+   Check if a collation has changed number
+
+   @param mysql_version
+   @param current collation number
+
+   @retval new collation number (same as current collation number of no change)
+*/
+
+uint upgrade_collation(ulong mysql_version, uint cs_number)
+{
+  if (mysql_version >= 50300 && mysql_version <= 50399)
+  {
+    switch (cs_number) {
+    case 149: return MY_PAGE2_COLLATION_ID_UCS2;   // ucs2_crotian_ci
+    case 213: return MY_PAGE2_COLLATION_ID_UTF8;   // utf8_crotian_ci
+    }
+  }
+  if ((mysql_version >= 50500 && mysql_version <= 50599) ||
+      (mysql_version >= 100000 && mysql_version <= 100005))
+  {
+    switch (cs_number) {
+    case 149: return MY_PAGE2_COLLATION_ID_UCS2;   // ucs2_crotian_ci
+    case 213: return MY_PAGE2_COLLATION_ID_UTF8;   // utf8_crotian_ci
+    case 214: return MY_PAGE2_COLLATION_ID_UTF32;  // utf32_croatian_ci
+    case 215: return MY_PAGE2_COLLATION_ID_UTF16;  // utf16_croatian_ci
+    case 245: return MY_PAGE2_COLLATION_ID_UTF8MB4;// utf8mb4_croatian_ci
+    }
+  }
+  return cs_number;
+}
+
 
 /**
   Check for incompatible collation changes.
@@ -3575,9 +3607,29 @@ int handler::check_collation_compatibility()
              (cs_number == 33 || /* utf8_general_ci - bug #27877 */
               cs_number == 35))) /* ucs2_general_ci - bug #27877 */
           return HA_ADMIN_NEEDS_UPGRADE;
-      }  
-    }  
-  }  
+      }
+    }
+  }
+
+  if (mysql_version < 100006)
+  {
+    /*
+      Check if we are using collations from that has changed numbering.
+      This happend at least between MariaDB 5.5 and MariaDB 10.0 as MySQL
+      added conflicting numbers.
+    */
+  
+    if (table->s->table_charset->number !=
+        upgrade_collation(mysql_version, table->s->table_charset->number))
+      return HA_ADMIN_NEEDS_ALTER;        
+    
+    for (Field **field= table->field; (*field); field++)
+    {
+      if ((*field)->charset()->number !=
+          upgrade_collation(mysql_version, (*field)->charset()->number))
+        return HA_ADMIN_NEEDS_ALTER;
+    }
+  }
   return 0;
 }
 
@@ -3588,6 +3640,9 @@ int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt)
   KEY *keyinfo, *keyend;
   KEY_PART_INFO *keypart, *keypartend;
 
+  if (table->s->incompatible_version)
+    return HA_ADMIN_NEEDS_ALTER;
+
   if (!table->s->mysql_version)
   {
     /* check for blob-in-key error */
diff --git a/sql/handler.h b/sql/handler.h
index 360120615f3..0c1ae1d01c2 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -4055,4 +4055,5 @@ inline const char *table_case_name(HA_CREATE_INFO *info, const char *name)
 
 void print_keydup_error(TABLE *table, KEY *key, const char *msg, myf errflag);
 void print_keydup_error(TABLE *table, KEY *key, myf errflag);
+uint upgrade_collation(ulong mysql_version, uint cs_number);
 #endif
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index f78f1b56489..df33c3535b5 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -6740,7 +6740,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
   List<Key_part_spec> key_parts;
   uint db_create_options= (table->s->db_create_options
                            & ~(HA_OPTION_PACK_RECORD));
-  uint used_fields= create_info->used_fields;
+  uint used_fields;
   KEY *key_info=table->key_info;
   bool rc= TRUE;
   bool modified_primary_key= FALSE;
@@ -6748,6 +6748,14 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
   Field **f_ptr,*field;
   DBUG_ENTER("mysql_prepare_alter_table");
 
+  /*
+    Merge incompatible changes flag in case of upgrade of a table from an
+    old MariaDB or MySQL version.  This ensures that we don't try to do an
+    online alter table if field packing or character set changes are required.
+  */
+  create_info->used_fields|= table->s->incompatible_version;
+  used_fields= create_info->used_fields;
+
   create_info->varchar= FALSE;
   /* Let new create options override the old ones */
   if (!(used_fields & HA_CREATE_USED_MIN_ROWS))
@@ -7732,8 +7740,11 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name,
 
   DEBUG_SYNC(thd, "alter_table_before_open_tables");
   uint tables_opened;
+
+  thd->open_options|= HA_OPEN_FOR_ALTER;
   bool error= open_tables(thd, &table_list, &tables_opened, 0,
                           &alter_prelocking_strategy);
+  thd->open_options&= ~HA_OPEN_FOR_ALTER;
 
   DEBUG_SYNC(thd, "alter_opened_table");
 
diff --git a/sql/table.cc b/sql/table.cc
index eed1de9bb35..c8c43ea3de1 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -1009,12 +1009,18 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
   share->null_field_first= 0;
   if (!frm_image[32])				// New frm file in 3.23
   {
+    uint cs_org= (((uint) frm_image[41]) << 8) + (uint) frm_image[38];
+    uint cs_new= upgrade_collation(share->mysql_version, cs_org);
+    if (cs_org != cs_new)
+      share->incompatible_version|= HA_CREATE_USED_CHARSET;
+    
     share->avg_row_length= uint4korr(frm_image+34);
     share->transactional= (ha_choice) (frm_image[39] & 3);
     share->page_checksum= (ha_choice) ((frm_image[39] >> 2) & 3);
     share->row_type= (enum row_type) frm_image[40];
-    share->table_charset= get_charset((((uint) frm_image[41]) << 8) + 
-                                        (uint) frm_image[38], MYF(0));
+
+    if (cs_new && !(share->table_charset= get_charset(cs_new, MYF(MY_WME))))
+      goto err;
     share->null_field_first= 1;
     share->stats_sample_pages= uint2korr(frm_image+42);
     share->stats_auto_recalc= (enum_stats_auto_recalc)(frm_image[44]);
@@ -1032,6 +1038,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
     }
     share->table_charset= default_charset_info;
   }
+
   share->db_record_offset= 1;
   share->max_rows= uint4korr(frm_image+18);
   share->min_rows= uint4korr(frm_image+22);
@@ -1418,16 +1425,19 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
       }
       else
       {
-        uint csid= strpos[14] + (((uint) strpos[11]) << 8);
-        if (!csid)
+        uint cs_org= strpos[14] + (((uint) strpos[11]) << 8);
+        uint cs_new= upgrade_collation(share->mysql_version, cs_org);
+        if (cs_org != cs_new)
+          share->incompatible_version|= HA_CREATE_USED_CHARSET;
+        if (!cs_new)
           charset= &my_charset_bin;
-        else if (!(charset= get_charset(csid, MYF(0))))
+        else if (!(charset= get_charset(cs_new, MYF(0))))
         {
-          const char *csname= get_charset_name((uint) csid);
+          const char *csname= get_charset_name((uint) cs_new);
           char tmp[10];
           if (!csname || csname[0] =='?')
           {
-            my_snprintf(tmp, sizeof(tmp), "#%d", csid);
+            my_snprintf(tmp, sizeof(tmp), "#%d", cs_new);
             csname= tmp;
           }
           my_printf_error(ER_UNKNOWN_COLLATION,
@@ -2489,6 +2499,13 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share,
   outparam->db_stat= db_stat;
   outparam->write_row_record= NULL;
 
+  if (share->incompatible_version &&
+      !(ha_open_flags & (HA_OPEN_FOR_ALTER | HA_OPEN_FOR_REPAIR)))
+  {
+    /* one needs to run mysql_upgrade on the table */
+    error= OPEN_FRM_NEEDS_REBUILD;
+    goto err;
+  }
   init_sql_alloc(&outparam->mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(0));
 
   if (outparam->alias.copy(alias, strlen(alias), table_alias_charset))
@@ -3012,6 +3029,11 @@ void open_table_error(TABLE_SHARE *share, enum open_frm_error error,
     strxmov(buff, share->normalized_path.str, reg_ext, NullS);
     my_error(ER_ERROR_ON_READ, errortype, buff, db_errno);
     break;
+  case OPEN_FRM_NEEDS_REBUILD:
+    strxnmov(buff, sizeof(buff)-1,
+             share->db.str, ".", share->table_name.str, NullS);
+    my_error(ER_TABLE_NEEDS_REBUILD, errortype, buff);
+    break;
   }
   DBUG_VOID_RETURN;
 } /* open_table_error */
diff --git a/sql/table.h b/sql/table.h
index ce7032b4c60..301b98a5ec2 100644
--- a/sql/table.h
+++ b/sql/table.h
@@ -560,7 +560,8 @@ enum open_frm_error {
   OPEN_FRM_DISCOVER,
   OPEN_FRM_ERROR_ALREADY_ISSUED,
   OPEN_FRM_NOT_A_VIEW,
-  OPEN_FRM_NOT_A_TABLE
+  OPEN_FRM_NOT_A_TABLE,
+  OPEN_FRM_NEEDS_REBUILD
 };
 
 /**
@@ -734,6 +735,13 @@ struct TABLE_SHARE
   ulong table_map_id;                   /* for row-based replication */
 
   /*
+    Things that are incompatible between the stored version and the
+    current version. This is a set of HA_CREATE... bits that can be used
+    to modify create_info->used_fields for ALTER TABLE.
+  */
+  ulong incompatible_version;
+
+  /*
     Cache for row-based replication table share checks that does not
     need to be repeated. Possible values are: -1 when cache value is
     not calculated yet, 0 when table *shall not* be replicated, 1 when