qtcdebugger: Register as post-mortem debugger for 64bit applications as well.

Add mode argument to registry access helpers allowing to access the 64bit
registry from 32bit applications and vice versa using special REGSAM
values, similar to the new QSettings formats introduced in Qt 5.7.

This allows for setting the 64bit post-mortem debugger when running
as a 32bit application on 64bit Windows. The -wow argument is extended
to indicate accessing the 64bit node from 32bit.

Task-number: QTCREATORBUG-16386
Change-Id: I7f003673777e4b8c1b259ba1905a4207b4ce0b43
Reviewed-by: David Schulz <david.schulz@theqtcompany.com>

3 files changed, 60 insertions, 8 deletions

diff --git a/src/shared/registryaccess/registryaccess.cpp b/src/shared/registryaccess/registryaccess.cpp
index 8acf5eeb90..e37185e507 100644
--- a/src/shared/registryaccess/registryaccess.cpp
+++ b/src/shared/registryaccess/registryaccess.cpp
@@ -98,6 +98,7 @@ bool openRegistryKey(HKEY category, // HKEY_LOCAL_MACHINE, etc.
                      const WCHAR *key,
                      bool readWrite,
                      HKEY *keyHandle,
+                     AccessMode mode,
                      QString *errorMessage)
 {
     Q_UNUSED(debuggerRegistryKeyC);  // avoid warning from MinGW
@@ -105,6 +106,16 @@ bool openRegistryKey(HKEY category, // HKEY_LOCAL_MACHINE, etc.
     REGSAM accessRights = KEY_READ;
     if (readWrite)
          accessRights |= KEY_SET_VALUE;
+    switch (mode) {
+    case RegistryAccess::DefaultAccessMode:
+        break;
+    case RegistryAccess::Registry32Mode:
+        accessRights |= KEY_WOW64_32KEY;
+        break;
+    case RegistryAccess::Registry64Mode:
+        accessRights |= KEY_WOW64_64KEY;
+        break;
+    }
     const LONG rc = RegOpenKeyEx(category, key, 0, accessRights, keyHandle);
     if (rc != ERROR_SUCCESS) {
         *errorMessage = msgFunctionFailed("RegOpenKeyEx", rc);
diff --git a/src/shared/registryaccess/registryaccess.h b/src/shared/registryaccess/registryaccess.h
index 7b65160d60..dae0c5eda9 100644
--- a/src/shared/registryaccess/registryaccess.h
+++ b/src/shared/registryaccess/registryaccess.h
@@ -37,6 +37,12 @@
 
 namespace RegistryAccess {
 
+enum AccessMode {
+    DefaultAccessMode,
+    Registry32Mode = 0x2, // Corresponds to QSettings::Registry32Format (5.7)
+    Registry64Mode = 0x4  // Corresponds to QSettings::Registry64Format (5.7)
+};
+
 static const char *debuggerApplicationFileC = "qtcdebugger";
 static const WCHAR *debuggerRegistryKeyC = L"Software\\Microsoft\\Windows NT\\CurrentVersion\\AeDebug";
 static const WCHAR *debuggerRegistryValueNameC = L"Debugger";
@@ -68,8 +74,12 @@ bool openRegistryKey(HKEY category, // HKEY_LOCAL_MACHINE, etc.
                      const WCHAR *key,
                      bool readWrite,
                      HKEY *keyHandle,
+                     AccessMode mode,
                      QString *errorMessage);
 
+inline bool openRegistryKey(HKEY category, const WCHAR *key, bool readWrite, HKEY *keyHandle, QString *errorMessage)
+{ return openRegistryKey(category, key, readWrite, keyHandle, DefaultAccessMode, errorMessage); }
+
 QString debuggerCall(const QString &additionalOption = QString());
 
 bool isRegistered(HKEY handle, const QString &call, QString *errorMessage, QString *oldDebugger = 0);
diff --git a/src/tools/qtcdebugger/main.cpp b/src/tools/qtcdebugger/main.cpp
index 73f781135c..b58b012041 100644
--- a/src/tools/qtcdebugger/main.cpp
+++ b/src/tools/qtcdebugger/main.cpp
@@ -70,6 +70,9 @@ static const char creatorBinaryC[] = "qtcreator.exe";
 enum Mode { HelpMode, RegisterMode, UnregisterMode, PromptMode, ForceCreatorMode, ForceDefaultMode };
 
 Mode optMode = PromptMode;
+// WOW: Indicates registry key access mode:
+// - Accessing 32bit using a 64bit built Qt Creator or,
+// - Accessing 64bit using a 32bit built Qt Creator on 64bit Windows
 bool optIsWow = false;
 bool noguiMode = false;
 unsigned long argProcessId = 0;
@@ -172,6 +175,14 @@ static void usage(const QString &binary, const QString &message = QString())
     msgBox.exec();
 }
 
+static bool is64BitWindowsSystem() // Courtesy utils library
+{
+    SYSTEM_INFO systemInfo;
+    GetNativeSystemInfo(&systemInfo);
+    return systemInfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64
+        || systemInfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_IA64;
+}
+
 // ------- Registry helpers
 
 static inline bool registryWriteBinaryKey(HKEY handle,
@@ -303,8 +314,15 @@ bool readDefaultDebugger(QString *defaultDebugger,
 {
     bool success = false;
     HKEY handle;
-    if (openRegistryKey(HKEY_LOCAL_MACHINE, optIsWow ? debuggerWow32RegistryKeyC : debuggerRegistryKeyC,
-                        false, &handle, errorMessage)) {
+    const RegistryAccess::AccessMode accessMode = optIsWow
+#ifdef Q_OS_WIN64
+        ? RegistryAccess::Registry32Mode
+#else
+        ? RegistryAccess::Registry64Mode
+#endif
+        : RegistryAccess::DefaultAccessMode;
+
+    if (openRegistryKey(HKEY_LOCAL_MACHINE, debuggerRegistryKeyC, false, &handle, accessMode, errorMessage)) {
         success = registryReadStringKey(handle, debuggerRegistryDefaultValueNameC,
                                         defaultDebugger, errorMessage);
         RegCloseKey(handle);
@@ -372,12 +390,13 @@ bool chooseDebugger(QString *errorMessage)
 
 static bool registerDebuggerKey(const WCHAR *key,
                                 const QString &call,
+                                RegistryAccess::AccessMode access,
                                 QString *errorMessage)
 {
     HKEY handle = 0;
     bool success = false;
     do {
-        if (!openRegistryKey(HKEY_LOCAL_MACHINE, key, true, &handle, errorMessage))
+        if (!openRegistryKey(HKEY_LOCAL_MACHINE, key, true, &handle, access, errorMessage))
             break;
         // Save old key, which might be missing
         QString oldDebugger;
@@ -401,11 +420,16 @@ static bool registerDebuggerKey(const WCHAR *key,
 
 bool install(QString *errorMessage)
 {
-    if (!registerDebuggerKey(debuggerRegistryKeyC, debuggerCall(), errorMessage))
+    if (!registerDebuggerKey(debuggerRegistryKeyC, debuggerCall(), RegistryAccess::DefaultAccessMode, errorMessage))
         return false;
 #ifdef Q_OS_WIN64
-    if (!registerDebuggerKey(debuggerWow32RegistryKeyC, debuggerCall(QLatin1String("-wow")), errorMessage))
+    if (!registerDebuggerKey(debuggerRegistryKeyC, debuggerCall(QLatin1String("-wow")), RegistryAccess::Registry32Mode, errorMessage))
         return false;
+#else
+    if (is64BitWindowsSystem()) {
+        if (!registerDebuggerKey(debuggerRegistryKeyC, debuggerCall(QLatin1String("-wow")), RegistryAccess::Registry64Mode, errorMessage))
+            return false;
+    }
 #endif
     return true;
 }
@@ -413,12 +437,13 @@ bool install(QString *errorMessage)
 // Unregister helper: Restore the original debugger key
 static bool unregisterDebuggerKey(const WCHAR *key,
                                   const QString &call,
+                                  RegistryAccess::AccessMode access,
                                   QString *errorMessage)
 {
     HKEY handle = 0;
     bool success = false;
     do {
-        if (!openRegistryKey(HKEY_LOCAL_MACHINE, key, true, &handle, errorMessage))
+        if (!openRegistryKey(HKEY_LOCAL_MACHINE, key, true, &handle, access, errorMessage))
             break;
         QString debugger;
         if (!isRegistered(handle, call, errorMessage, &debugger) && !debugger.isEmpty()) {
@@ -448,12 +473,18 @@ static bool unregisterDebuggerKey(const WCHAR *key,
 
 bool uninstall(QString *errorMessage)
 {
-    if (!unregisterDebuggerKey(debuggerRegistryKeyC, debuggerCall(), errorMessage))
+    if (!unregisterDebuggerKey(debuggerRegistryKeyC, debuggerCall(), RegistryAccess::DefaultAccessMode, errorMessage))
         return false;
 #ifdef Q_OS_WIN64
-    if (!unregisterDebuggerKey(debuggerWow32RegistryKeyC, debuggerCall(QLatin1String("-wow")), errorMessage))
+    if (!unregisterDebuggerKey(debuggerRegistryKeyC, debuggerCall(QLatin1String("-wow")), RegistryAccess::Registry32Mode, errorMessage))
         return false;
+#else
+    if (is64BitWindowsSystem()) {
+        if (!unregisterDebuggerKey(debuggerRegistryKeyC, debuggerCall(QLatin1String("-wow")), RegistryAccess::Registry64Mode, errorMessage))
+            return false;
+    }
 #endif
+
     return true;
 }