BUG#51342 - more xid crashing

SET autocommit=1 while XA transaction is active may
cause various side effects, including memory corruption
and server crash.

The problem is that SET autocommit=1 and further queries
attempt to commit local transaction, whereas XA transaction
is still active.

As local and XA transactions are mutually exclusive, this
patch forbids enabling autocommit mode while XA transaction
is active.

mysql-test/r/xa.result:
  A test case for BUG#51342.
mysql-test/t/xa.test:
  A test case for BUG#51342.
sql/set_var.cc:
  Forbid enabling autocommit mode while XA transaction is
  active.

3 files changed, 40 insertions, 0 deletions

diff --git a/mysql-test/r/xa.result b/mysql-test/r/xa.result
index 592cf07522b..acf219688e1 100644
--- a/mysql-test/r/xa.result
+++ b/mysql-test/r/xa.result
@@ -74,4 +74,21 @@ ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was det
 xa rollback 'a','c';
 xa start 'a','c';
 drop table t1;
+#
+# BUG#51342 - more xid crashing
+#
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+XA START 'x';
+SET SESSION autocommit=0;
+INSERT INTO t1 VALUES(1);
+SET SESSION autocommit=1;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ACTIVE state
+SELECT @@autocommit;
+@@autocommit
+0
+INSERT INTO t1 VALUES(1);
+XA END 'x';
+XA COMMIT 'x' ONE PHASE;
+DROP TABLE t1;
+SET SESSION autocommit=DEFAULT;
 End of 5.0 tests
diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test
index 04ecf518577..d5def009172 100644
--- a/mysql-test/t/xa.test
+++ b/mysql-test/t/xa.test
@@ -122,6 +122,22 @@ xa start 'a','c';
 --connection default
 drop table t1;
 
+--echo #
+--echo # BUG#51342 - more xid crashing
+--echo #
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+XA START 'x';
+SET SESSION autocommit=0;
+INSERT INTO t1 VALUES(1);
+--error ER_XAER_RMFAIL
+SET SESSION autocommit=1;
+SELECT @@autocommit;
+INSERT INTO t1 VALUES(1);
+XA END 'x';
+XA COMMIT 'x' ONE PHASE;
+DROP TABLE t1;
+SET SESSION autocommit=DEFAULT;
+
 --echo End of 5.0 tests
 
 # Wait till all disconnects are completed
diff --git a/sql/set_var.cc b/sql/set_var.cc
index 4871afd2c56..8e032d44a62 100644
--- a/sql/set_var.cc
+++ b/sql/set_var.cc
@@ -3065,6 +3065,13 @@ static bool set_option_autocommit(THD *thd, set_var *var)
     if ((org_options & OPTION_NOT_AUTOCOMMIT))
     {
       /* We changed to auto_commit mode */
+      if (thd->transaction.xid_state.xa_state != XA_NOTR)
+      {
+        thd->options= org_options;
+        my_error(ER_XAER_RMFAIL, MYF(0),
+                 xa_state_names[thd->transaction.xid_state.xa_state]);
+        return 1;
+      }
       thd->options&= ~OPTION_BEGIN;
       thd->transaction.all.modified_non_trans_table= FALSE;
       thd->server_status|= SERVER_STATUS_AUTOCOMMIT;