MDEV-6245 Certain compressed tables with myisampack are corrupted by "CHECK TABLE"

- Fixed bug that we where using wrong checksum algorithm when using VARCHAR with fixed lenth rows
- Ensure in myisampack that HA_OPTION_NULL_FIELDS is set for tables with null fields.

mysql-test/r/myisampack.result:
  Updated results
mysql-test/t/myisampack.test:
  Added more tests
storage/myisam/mi_open.c:
  Use correct checksum algorithm when we have VARCHAR fields with fixed length records
storage/myisam/myisampack.c:
  Ensure HA_OPTION_NULL_FIELDS is set for tables with null fields.
  (This was not set by default for not compressed tables without checksums to keep MyISAM tables compatible with MySQL)

4 files changed, 113 insertions, 6 deletions

diff --git a/mysql-test/r/myisampack.result b/mysql-test/r/myisampack.result
index 56f61ccdf47..aeb12c1f68e 100644
--- a/mysql-test/r/myisampack.result
+++ b/mysql-test/r/myisampack.result
@@ -150,3 +150,57 @@ CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 DROP TABLE t1;
+create table `t1` (`id` varchar(15) DEFAULT NULL) ENGINE=MyISAM ROW_FORMAT=FIXED;
+insert into t1 values ('aaa'),('bbb'),('ccc'),('ddd'),('eee');
+insert into t1 (select * from t1);
+insert into t1 (select * from t1);
+insert into t1 (select * from t1);
+insert into t1 (select * from t1);
+checksum table t1;
+Table	Checksum
+test.t1	2696656816
+insert into t1 values(NULL);
+checksum table t1;
+Table	Checksum
+test.t1	2679879600
+flush table t1;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+checksum table t1;
+Table	Checksum
+test.t1	2679879600
+alter table t1 checksum=1 row_format=fixed;
+checksum table t1;
+Table	Checksum
+test.t1	2679879600
+flush table t1;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+checksum table t1;
+Table	Checksum
+test.t1	2679879600
+alter table t1 row_format=dynamic checksum=0;
+checksum table t1;
+Table	Checksum
+test.t1	2330021136
+flush table t1;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+checksum table t1;
+Table	Checksum
+test.t1	2330021136
+alter table t1 checksum=1 row_format=dynamic;
+checksum table t1;
+Table	Checksum
+test.t1	2330021136
+flush table t1;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+checksum table t1;
+Table	Checksum
+test.t1	2330021136
+drop table t1;
diff --git a/mysql-test/t/myisampack.test b/mysql-test/t/myisampack.test
index afa417822e6..cc9429a1707 100644
--- a/mysql-test/t/myisampack.test
+++ b/mysql-test/t/myisampack.test
@@ -267,3 +267,50 @@ FLUSH TABLE t1;
 --exec $MYISAMCHK -soq $MYSQLD_DATADIR/test/t1
 CHECK TABLE t1;
 DROP TABLE t1;
+
+#
+# MDEV-6245 Certain compressed tables with myisampack are corrupted by
+# "CHECK TABLE"
+#
+# Issue was that checksum failed for tables with NULL and VARCHAR fields
+#
+
+create table `t1` (`id` varchar(15) DEFAULT NULL) ENGINE=MyISAM ROW_FORMAT=FIXED;
+insert into t1 values ('aaa'),('bbb'),('ccc'),('ddd'),('eee');
+insert into t1 (select * from t1);
+insert into t1 (select * from t1);
+insert into t1 (select * from t1);
+insert into t1 (select * from t1);
+checksum table t1;
+insert into t1 values(NULL);
+checksum table t1;
+flush table t1;
+--exec $MYISAMPACK -sf $MYSQLD_DATADIR/test/t1
+--exec $MYISAMCHK -srnq $MYSQLD_DATADIR/test/t1
+check table t1;
+checksum table t1;
+alter table t1 checksum=1 row_format=fixed;
+checksum table t1;
+flush table t1;
+--exec $MYISAMPACK -sf $MYSQLD_DATADIR/test/t1
+--exec $MYISAMCHK -srnq $MYSQLD_DATADIR/test/t1
+check table t1;
+checksum table t1;
+
+# Testing with row_format=dynamic
+
+alter table t1 row_format=dynamic checksum=0;
+checksum table t1;
+flush table t1;
+--exec $MYISAMPACK -sf $MYSQLD_DATADIR/test/t1
+--exec $MYISAMCHK -srnq $MYSQLD_DATADIR/test/t1
+check table t1;
+checksum table t1;
+alter table t1 checksum=1 row_format=dynamic;
+checksum table t1;
+flush table t1;
+--exec $MYISAMPACK -sf $MYSQLD_DATADIR/test/t1
+--exec $MYISAMCHK -srnq $MYSQLD_DATADIR/test/t1
+check table t1;
+checksum table t1;
+drop table t1;
diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c
index 22225303bae..94bc92088ad 100644
--- a/storage/myisam/mi_open.c
+++ b/storage/myisam/mi_open.c
@@ -458,13 +458,11 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 	share->blobs[j].offset=offset;
 	j++;
       }
-#if MYSQL_VERSION_ID <= 60100
-      /* This is to detect old checksum option */
+      /* This is to detect how to calculate checksums */
       if (share->rec[i].null_bit)
         share->has_null_fields= 1;
       if (share->rec[i].type == FIELD_VARCHAR)
         share->has_varchar_fields= 1;
-#endif
       offset+=share->rec[i].length;
     }
     share->rec[i].type=(int) FIELD_LAST;	/* End marker */
@@ -754,7 +752,8 @@ void mi_setup_functions(register MYISAM_SHARE *share)
     share->read_record=_mi_read_pack_record;
     share->read_rnd=_mi_read_rnd_pack_record;
     if ((share->options &
-              (HA_OPTION_PACK_RECORD | HA_OPTION_NULL_FIELDS)))
+              (HA_OPTION_PACK_RECORD | HA_OPTION_NULL_FIELDS)) ||
+        share->has_varchar_fields)
       share->calc_checksum= mi_checksum;
     else
       share->calc_checksum= mi_static_checksum;
diff --git a/storage/myisam/myisampack.c b/storage/myisam/myisampack.c
index 92ee1dde84c..50622c464ca 100644
--- a/storage/myisam/myisampack.c
+++ b/storage/myisam/myisampack.c
@@ -415,6 +415,11 @@ static MI_INFO *open_isam_file(char *name,int mode)
     if (verbose)
       puts("Recompressing already compressed table");
     share->options&= ~HA_OPTION_READ_ONLY_DATA; /* We are modifing it */
+
+    /* We want to use the new checksums if we have null fields */
+    if (share->has_null_fields)
+      share->options|= HA_OPTION_NULL_FIELDS;
+    
   }
   if (! force_pack && share->state.state.records != 0 &&
       (share->state.state.records <= 1 ||
@@ -2964,7 +2969,8 @@ static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length,
   uint key;
   DBUG_ENTER("save_state");
 
-  options|= HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA;
+  options|= (HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA |
+             (share->options & HA_OPTION_NULL_FIELDS));
   mi_int2store(share->state.header.options,options);
 
   share->state.state.data_file_length=new_length;
@@ -3013,7 +3019,8 @@ static int save_state_mrg(File file,PACK_MRG_INFO *mrg,my_off_t new_length,
 
   state= isam_file->s->state;
   options= (mi_uint2korr(state.header.options) | HA_OPTION_COMPRESS_RECORD |
-	    HA_OPTION_READ_ONLY_DATA);
+	    HA_OPTION_READ_ONLY_DATA |
+            (isam_file->s->options & HA_OPTION_NULL_FIELDS));
   mi_int2store(state.header.options,options);
   state.state.data_file_length=new_length;
   state.state.del=0;