Edit debugger so win64interrupt is called

when trying to debug a 64bit application under windows.

Task-number: QTCREATORBUG-2521
Change-Id: I38922a6bed09640ce88184e6913a9fbb1d7433de
Reviewed-by: Joerg Bornemann <joerg.bornemann@nokia.com>
Reviewed-by: hjk <qthjk@ovi.com>

4 files changed, 48 insertions, 6 deletions

diff --git a/src/plugins/debugger/cdb/cdbengine.cpp b/src/plugins/debugger/cdb/cdbengine.cpp
index 158ba8899c..e624fe0ac0 100644
--- a/src/plugins/debugger/cdb/cdbengine.cpp
+++ b/src/plugins/debugger/cdb/cdbengine.cpp
@@ -1182,7 +1182,7 @@ void CdbEngine::doInterruptInferior(SpecialStopMode sm)
     m_specialStopMode = sm;
     QString errorMessage;
     showMessage(QString::fromLatin1("Interrupting process %1...").arg(inferiorPid()), LogMisc);
-    if (!winDebugBreakProcess(inferiorPid(), &errorMessage)) {
+    if (!winDebugBreakProcess(inferiorPid(), &errorMessage, Utils::winIs64BitBinary(cdbBinary(startParameters())))) {
         m_specialStopMode = oldSpecialMode;
         showMessage(QString::fromLatin1("Cannot interrupt process %1: %2").arg(inferiorPid()).arg(errorMessage), LogError);
     }
diff --git a/src/plugins/debugger/procinterrupt.cpp b/src/plugins/debugger/procinterrupt.cpp
index 291d50d60d..cf5a49150d 100644
--- a/src/plugins/debugger/procinterrupt.cpp
+++ b/src/plugins/debugger/procinterrupt.cpp
@@ -33,6 +33,7 @@
 #include "procinterrupt.h"
 
 #include <QtCore/QProcess> // makes kill visible on Windows.
+#include <QtCore/QFile>
 
 using namespace Debugger::Internal;
 
@@ -40,8 +41,27 @@ using namespace Debugger::Internal;
 
 #define _WIN32_WINNT 0x0501 /* WinXP, needed for DebugBreakProcess() */
 
+#include <utils/winutils.h>
 #include <windows.h>
 
+static BOOL isWow64Process(HANDLE hproc)
+{
+    BOOL ret = false;
+    typedef BOOL (WINAPI *LPFN_ISWOW64PROCESS) (HANDLE, PBOOL);
+    LPFN_ISWOW64PROCESS fnIsWow64Process = NULL;
+    HMODULE hModule = GetModuleHandle(L"kernel32.dll");
+    if (hModule == NULL)
+        return false;
+
+    fnIsWow64Process = reinterpret_cast<LPFN_ISWOW64PROCESS>(GetProcAddress(hModule, "IsWow64Process"));
+    if (fnIsWow64Process == NULL)
+        return false;
+
+    if (!fnIsWow64Process(hproc, &ret))
+        return false;
+    return ret;
+}
+
 bool Debugger::Internal::interruptProcess(int pID)
 {
     if (pID <= 0)
@@ -51,10 +71,18 @@ bool Debugger::Internal::interruptProcess(int pID)
     if (hproc == NULL)
         return false;
 
-    bool ok = DebugBreakProcess(hproc) != 0;
+    BOOL proc64bit = false;
 
-    CloseHandle(hproc);
+    if (Utils::winIs64BitSystem())
+        proc64bit = !isWow64Process(hproc);
 
+    bool ok = false;
+    if (proc64bit)
+        ok = !QProcess::execute(QCoreApplication::applicationDirPath() + QString::fromLatin1("/win64interrupt.exe %1").arg(pID));
+    else
+        ok = !DebugBreakProcess(hproc);
+
+    CloseHandle(hproc);
     return ok;
 }
 
diff --git a/src/plugins/debugger/shared/hostutils.cpp b/src/plugins/debugger/shared/hostutils.cpp
index 671e94336d..7e67648eab 100644
--- a/src/plugins/debugger/shared/hostutils.cpp
+++ b/src/plugins/debugger/shared/hostutils.cpp
@@ -146,7 +146,7 @@ bool winResumeThread(unsigned long dwThreadId, QString *errorMessage)
 }
 
 // Open the process and break into it
-bool winDebugBreakProcess(unsigned long  pid, QString *errorMessage)
+bool winDebugBreakProcess(unsigned long  pid, QString *errorMessage, bool isCdb64bit)
 {
     bool ok = false;
     HANDLE inferior = NULL;
@@ -160,7 +160,21 @@ bool winDebugBreakProcess(unsigned long  pid, QString *errorMessage)
                     arg(pid).arg(Utils::winErrorMessage(GetLastError()));
             break;
         }
-        if (!DebugBreakProcess(inferior)) {
+
+        if (isCdb64bit) {
+            switch (QProcess::execute(QCoreApplication::applicationDirPath() + QString::fromLatin1("/win64interrupt.exe %1").arg(pid))) {
+            case -2:
+                *errorMessage = QString::fromLatin1("Cannot start win64interrupt.exe. Check src/tools/win64interrupt/win64interrupt.c for more information.");
+                break;
+            case 0:
+                ok = true;
+                break;
+            default:
+                *errorMessage = QString::fromLatin1("win64interrupt.exe could not break the process with the pid %1.").arg(pid);
+                break;
+            }
+            break;
+        } else if (!DebugBreakProcess(inferior)) {
             *errorMessage = QString::fromLatin1("DebugBreakProcess failed: %1").arg(Utils::winErrorMessage(GetLastError()));
             break;
         }
diff --git a/src/plugins/debugger/shared/hostutils.h b/src/plugins/debugger/shared/hostutils.h
index 2e8e311feb..4fa3036ea0 100644
--- a/src/plugins/debugger/shared/hostutils.h
+++ b/src/plugins/debugger/shared/hostutils.h
@@ -60,7 +60,7 @@ QList<ProcData> hostProcessList();
 bool winResumeThread(unsigned long dwThreadId, QString *errorMessage);
 
 // Open a process by PID and break into it.
-bool winDebugBreakProcess(unsigned long  pid, QString *errorMessage);
+bool winDebugBreakProcess(unsigned long  pid, QString *errorMessage, bool isCdb64bit);
 
 unsigned long winGetCurrentProcessId();