bug#8786 - ndb autodiscover, sometimes fails

  remove dict forwarding
  add api retries on NotMaster
  


ndb/include/kernel/signaldata/AlterIndx.hpp:
  Add NotMaster error code
ndb/include/kernel/signaldata/BuildIndx.hpp:
  Add NotMaster error code
ndb/include/kernel/signaldata/CreateIndx.hpp:
  Add NotMaster error code
ndb/include/kernel/signaldata/CreateTrig.hpp:
  Add NotMaster error code
ndb/include/kernel/signaldata/DropIndx.hpp:
  Add NotMaster error code
ndb/src/kernel/blocks/dbdict/Dbdict.cpp:
  Never forward requests,
    instead REF to API who will retry towards correct node
ndb/src/ndbapi/NdbDictionaryImpl.cpp:
  1) Set error code for timeout
  2) Handle NotMaster with retry in all DICT requests
ndb/src/ndbapi/ndb_cluster_connection.cpp:
  Fix so that input values is in seconds and not 100ms's
sql/ha_ndbcluster.cc:
  Wait 3 sec for all nodes to connect...

9 files changed, 68 insertions, 27 deletions

diff --git a/ndb/include/kernel/signaldata/AlterIndx.hpp b/ndb/include/kernel/signaldata/AlterIndx.hpp
index 1f464ded010..4c8bb465cf1 100644
--- a/ndb/include/kernel/signaldata/AlterIndx.hpp
+++ b/ndb/include/kernel/signaldata/AlterIndx.hpp
@@ -181,6 +181,7 @@ public:
   enum ErrorCode {
     NoError = 0,
     Busy = 701,
+    NotMaster = 702,
     IndexNotFound = 4243,
     IndexExists = 4244,
     BadRequestType = 4247,
diff --git a/ndb/include/kernel/signaldata/BuildIndx.hpp b/ndb/include/kernel/signaldata/BuildIndx.hpp
index 29dfaeb79a6..e8eca03ea81 100644
--- a/ndb/include/kernel/signaldata/BuildIndx.hpp
+++ b/ndb/include/kernel/signaldata/BuildIndx.hpp
@@ -234,6 +234,7 @@ public:
   enum ErrorCode {
     NoError = 0,
     Busy = 701,
+    NotMaster = 702,
     BadRequestType = 4247,
     InvalidPrimaryTable = 4249,
     InvalidIndexType = 4250,
diff --git a/ndb/include/kernel/signaldata/CreateIndx.hpp b/ndb/include/kernel/signaldata/CreateIndx.hpp
index 3e277b38dea..2751cfbbbfa 100644
--- a/ndb/include/kernel/signaldata/CreateIndx.hpp
+++ b/ndb/include/kernel/signaldata/CreateIndx.hpp
@@ -184,7 +184,7 @@ public:
 /**
  * CreateIndxRef.
  */
-class CreateIndxRef {
+struct CreateIndxRef {
   friend bool printCREATE_INDX_REF(FILE*, const Uint32*, Uint32, Uint16);
 
 public:
@@ -210,7 +210,6 @@ public:
     InvalidAttributeOrder = 4255
   };
 
-private:
   CreateIndxConf m_conf;
   //Uint32 m_userRef;
   //Uint32 m_connectionPtr;
diff --git a/ndb/include/kernel/signaldata/CreateTrig.hpp b/ndb/include/kernel/signaldata/CreateTrig.hpp
index a8de9e50dd4..4af9cb57f68 100644
--- a/ndb/include/kernel/signaldata/CreateTrig.hpp
+++ b/ndb/include/kernel/signaldata/CreateTrig.hpp
@@ -288,6 +288,7 @@ public:
   enum ErrorCode {
     NoError = 0,
     Busy = 701,
+    NotMaster = 702,
     TriggerNameTooLong = 4236,
     TooManyTriggers = 4237,
     TriggerNotFound = 4238,
diff --git a/ndb/include/kernel/signaldata/DropIndx.hpp b/ndb/include/kernel/signaldata/DropIndx.hpp
index 0c0cf31aec8..247fbdf914e 100644
--- a/ndb/include/kernel/signaldata/DropIndx.hpp
+++ b/ndb/include/kernel/signaldata/DropIndx.hpp
@@ -160,7 +160,7 @@ public:
 /**
  * DropIndxRef.
  */
-class DropIndxRef {
+struct DropIndxRef {
   friend bool printDROP_INDX_REF(FILE*, const Uint32*, Uint32, Uint16);
 
 public:
@@ -168,6 +168,7 @@ public:
     NoError = 0,
     InvalidIndexVersion = 241,
     Busy = 701,
+    NotMaster = 702,
     IndexNotFound = 4243,
     BadRequestType = 4247,
     InvalidName = 4248,
@@ -175,7 +176,6 @@ public:
   };
   STATIC_CONST( SignalLength = DropIndxConf::SignalLength + 3 );
 
-private:
   DropIndxConf m_conf;
   //Uint32 m_userRef;
   //Uint32 m_connectionPtr;
diff --git a/ndb/src/kernel/blocks/dbdict/Dbdict.cpp b/ndb/src/kernel/blocks/dbdict/Dbdict.cpp
index 432b2617f65..2b2748eb165 100644
--- a/ndb/src/kernel/blocks/dbdict/Dbdict.cpp
+++ b/ndb/src/kernel/blocks/dbdict/Dbdict.cpp
@@ -6053,11 +6053,21 @@ Dbdict::execCREATE_INDX_REQ(Signal* signal)
       jam();
       if (getOwnNodeId() != c_masterNodeId) {
         jam();
-        // forward to DICT master
-        sendSignal(calcDictBlockRef(c_masterNodeId), GSN_CREATE_INDX_REQ,
-            signal, signal->getLength(), JBB);
-        return;
+	
+	releaseSections(signal);
+	OpCreateIndex opBusy;
+	opPtr.p = &opBusy;
+	opPtr.p->save(req);
+	opPtr.p->m_isMaster = (senderRef == reference());
+	opPtr.p->key = 0;
+	opPtr.p->m_requestType = CreateIndxReq::RT_DICT_PREPARE;
+	opPtr.p->m_errorCode = CreateIndxRef::NotMaster;
+	opPtr.p->m_errorLine = __LINE__;
+	opPtr.p->m_errorNode = c_masterNodeId;
+	createIndex_sendReply(signal, opPtr, true);
+	return;
       }
+      
       // forward initial request plus operation key to all
       req->setOpKey(++c_opRecordSequence);
       NodeReceiverGroup rg(DBDICT, c_aliveNodes);
@@ -6588,10 +6598,9 @@ Dbdict::execDROP_INDX_REQ(Signal* signal)
       jam();
       if (getOwnNodeId() != c_masterNodeId) {
         jam();
-        // forward to DICT master
-        sendSignal(calcDictBlockRef(c_masterNodeId), GSN_DROP_INDX_REQ,
-            signal, signal->getLength(), JBB);
-        return;
+
+	err = DropIndxRef::NotMaster;
+	goto error;
       }
       // forward initial request plus operation key to all
       Uint32 indexId= req->getIndexId();
@@ -6679,6 +6688,7 @@ error:
   opPtr.p->save(req);
   opPtr.p->m_errorCode = (DropIndxRef::ErrorCode)err;
   opPtr.p->m_errorLine = __LINE__;
+  opPtr.p->m_errorNode = c_masterNodeId;
   dropIndex_sendReply(signal, opPtr, true);
 }
 
@@ -9117,9 +9127,15 @@ Dbdict::execALTER_INDX_REQ(Signal* signal)
       jam();
       if (! isLocal && getOwnNodeId() != c_masterNodeId) {
         jam();
-        // forward to DICT master
-        sendSignal(calcDictBlockRef(c_masterNodeId), GSN_ALTER_INDX_REQ,
-            signal, signal->getLength(), JBB);
+
+	releaseSections(signal);
+	OpAlterIndex opBad;
+	opPtr.p = &opBad;
+	opPtr.p->save(req);
+	opPtr.p->m_errorCode = AlterIndxRef::NotMaster;
+	opPtr.p->m_errorLine = __LINE__;
+	opPtr.p->m_errorNode = c_masterNodeId;
+	alterIndex_sendReply(signal, opPtr, true);
         return;
       }
       // forward initial request plus operation key to all
@@ -9797,9 +9813,15 @@ Dbdict::execBUILDINDXREQ(Signal* signal)
       jam();
       if (getOwnNodeId() != c_masterNodeId) {
         jam();
-        // forward to DICT master
-        sendSignal(calcDictBlockRef(c_masterNodeId), GSN_BUILDINDXREQ,
-            signal, signal->getLength(), JBB);
+
+	releaseSections(signal);
+	OpBuildIndex opBad;
+	opPtr.p = &opBad;
+	opPtr.p->save(req);
+	opPtr.p->m_errorCode = BuildIndxRef::NotMaster;
+	opPtr.p->m_errorLine = __LINE__;
+	opPtr.p->m_errorNode = c_masterNodeId;
+	buildIndex_sendReply(signal, opPtr, true);
         return;
       }
       // forward initial request plus operation key to all
@@ -10263,9 +10285,15 @@ Dbdict::execCREATE_TRIG_REQ(Signal* signal)
       jam();
       if (! isLocal && getOwnNodeId() != c_masterNodeId) {
         jam();
-        // forward to DICT master
-        sendSignal(calcDictBlockRef(c_masterNodeId), GSN_CREATE_TRIG_REQ,
-            signal, signal->getLength(), JBB);
+
+	releaseSections(signal);
+	OpCreateTrigger opBad;
+	opPtr.p = &opBad;
+	opPtr.p->save(req);
+	opPtr.p->m_errorCode = CreateTrigRef::NotMaster;
+	opPtr.p->m_errorLine = __LINE__;
+	opPtr.p->m_errorNode = c_masterNodeId;
+	createTrigger_sendReply(signal,  opPtr, true);
         return;
       }
       // forward initial request plus operation key to all
diff --git a/ndb/src/ndbapi/NdbDictionaryImpl.cpp b/ndb/src/ndbapi/NdbDictionaryImpl.cpp
index 9f6ed144fb0..3d92f0e010f 100644
--- a/ndb/src/ndbapi/NdbDictionaryImpl.cpp
+++ b/ndb/src/ndbapi/NdbDictionaryImpl.cpp
@@ -945,6 +945,12 @@ NdbDictInterface::dictSignal(NdbApiSignal* signal,
     if(m_waiter.m_state == WAIT_NODE_FAILURE)
       continue;
 
+    if(m_waiter.m_state == WST_WAIT_TIMEOUT)
+    {
+      m_error.code = 4008;
+      DBUG_RETURN(-1);
+    }
+    
     if ( (temporaryMask & m_error.code) != 0 ) {
       continue;
     }
@@ -2091,8 +2097,8 @@ int
 NdbDictInterface::createIndex(NdbApiSignal* signal, 
 			      LinearSectionPtr ptr[3])
 {
-  const int noErrCodes = 1;
-  int errCodes[noErrCodes] = {CreateIndxRef::Busy};
+  const int noErrCodes = 2;
+  int errCodes[noErrCodes] = {CreateIndxRef::Busy, CreateIndxRef::NotMaster};
   return dictSignal(signal,ptr,2,
 		    1 /*use masternode id*/,
 		    100,
@@ -2116,6 +2122,8 @@ NdbDictInterface::execCREATE_INDX_REF(NdbApiSignal * signal,
 {
   const CreateIndxRef* const ref = CAST_CONSTPTR(CreateIndxRef, signal->getDataPtr());
   m_error.code = ref->getErrorCode();
+  if(m_error.code == ref->NotMaster)
+    m_masterNodeId= ref->m_errorNode;
   m_waiter.signal(NO_WAIT);  
 }
 
@@ -2212,8 +2220,8 @@ NdbDictInterface::dropIndex(const NdbIndexImpl & impl,
 int
 NdbDictInterface::dropIndex(NdbApiSignal* signal, LinearSectionPtr ptr[3])
 {
-  const int noErrCodes = 1;
-  int errCodes[noErrCodes] = {DropIndxRef::Busy};
+  const int noErrCodes = 2;
+  int errCodes[noErrCodes] = {DropIndxRef::Busy, DropIndxRef::NotMaster};
   int r = dictSignal(signal,NULL,0,
 		     1/*Use masternode id*/,
 		     100,
@@ -2240,6 +2248,8 @@ NdbDictInterface::execDROP_INDX_REF(NdbApiSignal * signal,
 {
   const DropIndxRef* const ref = CAST_CONSTPTR(DropIndxRef, signal->getDataPtr());
   m_error.code = ref->getErrorCode();
+  if(m_error.code == ref->NotMaster)
+    m_masterNodeId= ref->m_errorNode;
   m_waiter.signal(NO_WAIT);  
 }
 
diff --git a/ndb/src/ndbapi/ndb_cluster_connection.cpp b/ndb/src/ndbapi/ndb_cluster_connection.cpp
index 5df707e211d..4650a719542 100644
--- a/ndb/src/ndbapi/ndb_cluster_connection.cpp
+++ b/ndb/src/ndbapi/ndb_cluster_connection.cpp
@@ -222,7 +222,8 @@ Ndb_cluster_connection::wait_until_ready(int timeout,
     else if (foundAliveNode > 0)
     {
       noChecksSinceFirstAliveFound++;
-      if (noChecksSinceFirstAliveFound > timeout_after_first_alive)
+      // 100 ms delay -> 10*
+      if (noChecksSinceFirstAliveFound > 10*timeout_after_first_alive)
 	DBUG_RETURN(1);
     }
     else if (secondsCounter >= timeout)
diff --git a/sql/ha_ndbcluster.cc b/sql/ha_ndbcluster.cc
index a959cbaf434..1b771d7cfc8 100644
--- a/sql/ha_ndbcluster.cc
+++ b/sql/ha_ndbcluster.cc
@@ -4339,7 +4339,7 @@ bool ndbcluster_init()
     DBUG_PRINT("info",("NDBCLUSTER storage engine at %s on port %d",
 		       g_ndb_cluster_connection->get_connected_host(),
 		       g_ndb_cluster_connection->get_connected_port()));
-    g_ndb_cluster_connection->wait_until_ready(10,0);
+    g_ndb_cluster_connection->wait_until_ready(10,3);
   } 
   else if(res == 1)
   {