Windows QPA: also consider window flags when judging frameless or not

It's possible that the window has the WS_THICKFRAME flag but the
Qt::FramelessWindowHint is enabled at the same time. That usually
indicates the user is customizing the window, but it will also
confuse QPA's current logic. We can make the logic more roboust
by reading the Qt window flags as well.

Change-Id: I9894e312a92ef7f5fc0aa93974f216b67fe2cf29
Reviewed-by: Oliver Wolff <oliver.wolff@qt.io>
(cherry picked from commit 17ab11ce493477f891916351b6ac0f0a8ad28ca6)

3 files changed, 43 insertions, 17 deletions

diff --git a/src/plugins/platforms/windows/qwindowsintegration.cpp b/src/plugins/platforms/windows/qwindowsintegration.cpp
index 22de3236ea..9af99abfad 100644
--- a/src/plugins/platforms/windows/qwindowsintegration.cpp
+++ b/src/plugins/platforms/windows/qwindowsintegration.cpp
@@ -329,10 +329,12 @@ QPlatformWindow *QWindowsIntegration::createPlatformWindow(QWindow *window) cons
     requested.geometry = window->isTopLevel()
         ? QHighDpi::toNativePixels(window->geometry(), window)
         : QHighDpi::toNativeLocalPosition(window->geometry(), window);
-    // Apply custom margins (see  QWindowsWindow::setCustomMargins())).
-    const QVariant customMarginsV = window->property("_q_windowsCustomMargins");
-    if (customMarginsV.isValid())
-        requested.customMargins = qvariant_cast<QMargins>(customMarginsV);
+    if (!(requested.flags & Qt::FramelessWindowHint)) {
+        // Apply custom margins (see  QWindowsWindow::setCustomMargins())).
+        const QVariant customMarginsV = window->property("_q_windowsCustomMargins");
+        if (customMarginsV.isValid())
+            requested.customMargins = qvariant_cast<QMargins>(customMarginsV);
+    }
 
     QWindowsWindowData obtained =
         QWindowsWindowData::create(window, requested,
diff --git a/src/plugins/platforms/windows/qwindowswindow.cpp b/src/plugins/platforms/windows/qwindowswindow.cpp
index 5d20f59c06..d1a017b6cb 100644
--- a/src/plugins/platforms/windows/qwindowswindow.cpp
+++ b/src/plugins/platforms/windows/qwindowswindow.cpp
@@ -889,7 +889,8 @@ QWindowsWindowData
                                                                        style, exStyle));
     QWindowsContext::instance()->setWindowCreationContext(context);
 
-    const bool hasFrame = (style & (WS_DLGFRAME | WS_THICKFRAME));
+    const bool hasFrame = (style & (WS_DLGFRAME | WS_THICKFRAME))
+            && !(result.flags & Qt::FramelessWindowHint);
     QMargins invMargins = topLevel && hasFrame && QWindowsGeometryHint::positionIncludesFrame(w)
             ? invisibleMargins(QPoint(context->frameX, context->frameY)) : QMargins();
 
@@ -1407,13 +1408,16 @@ QWindowCreationContext::QWindowCreationContext(const QWindow *w, const QScreen *
     requestedGeometry(geometry),
     obtainedPos(geometryIn.topLeft()),
     obtainedSize(geometryIn.size()),
-    margins(QWindowsGeometryHint::frame(w, geometry, style, exStyle)),
-    customMargins(cm)
+    margins(QWindowsGeometryHint::frame(w, geometry, style, exStyle))
 {
     // Geometry of toplevels does not consider window frames.
     // TODO: No concept of WA_wasMoved yet that would indicate a
     // CW_USEDEFAULT unless set. For now, assume that 0,0 means 'default'
     // for toplevels.
+
+    if (!(w->flags() & Qt::FramelessWindowHint))
+        customMargins = cm;
+
     if (geometry.isValid()
         || !qt_window_private(const_cast<QWindow *>(w))->resizeAutomatic) {
         frameX = geometry.x();
@@ -1968,12 +1972,14 @@ void QWindowsWindow::handleDpiScaledSize(WPARAM wParam, LPARAM lParam, LRESULT *
     const qreal scale = QHighDpiScaling::roundScaleFactor(qreal(dpi) / QWindowsScreen::baseDpi) /
                         QHighDpiScaling::roundScaleFactor(qreal(savedDpi()) / QWindowsScreen::baseDpi);
     const QMargins margins = QWindowsGeometryHint::frame(window(), style(), exStyle(), dpi);
-    // We need to update the custom margins to match the current DPI, because
-    // we don't want our users manually hook into this message just to set a
-    // new margin, but here we can't call setCustomMargins() directly, that
-    // function will change the window geometry which conflicts with what we
-    // are currently doing.
-    m_data.customMargins *= scale;
+    if (!(m_data.flags & Qt::FramelessWindowHint)) {
+        // We need to update the custom margins to match the current DPI, because
+        // we don't want our users manually hook into this message just to set a
+        // new margin, but here we can't call setCustomMargins() directly, that
+        // function will change the window geometry which conflicts with what we
+        // are currently doing.
+        m_data.customMargins *= scale;
+    }
     const QSize windowSize = (geometry().size() * scale).grownBy(margins + customMargins());
     SIZE *size = reinterpret_cast<SIZE *>(lParam);
     size->cx = windowSize.width();
@@ -2124,7 +2130,7 @@ void QWindowsWindow::setGeometry(const QRect &rectIn)
         if (m_data.geometry != rect && (isVisible() || QLibraryInfo::isDebugBuild())) {
             const auto warning =
                 msgUnableToSetGeometry(this, rectIn, m_data.geometry,
-                                       m_data.fullFrameMargins, m_data.customMargins);
+                                       fullFrameMargins(), customMargins());
             qWarning("%s: %s", __FUNCTION__, qPrintable(warning));
         }
     } else {
@@ -2393,7 +2399,8 @@ QWindowsWindowData QWindowsWindow::setWindowFlags_sys(Qt::WindowFlags wt,
     QWindowsWindowData result = m_data;
     result.flags = creationData.flags;
     result.embedded = creationData.embedded;
-    result.hasFrame = (creationData.style & (WS_DLGFRAME | WS_THICKFRAME));
+    result.hasFrame = (creationData.style & (WS_DLGFRAME | WS_THICKFRAME))
+            && !(creationData.flags & Qt::FramelessWindowHint);
     return result;
 }
 
@@ -2683,6 +2690,8 @@ bool QWindowsWindow::handleGeometryChanging(MSG *message) const
 
 void QWindowsWindow::setFullFrameMargins(const QMargins &newMargins)
 {
+    if (m_data.flags & Qt::FramelessWindowHint)
+        return;
     if (m_data.fullFrameMargins != newMargins) {
         qCDebug(lcQpaWindows) << __FUNCTION__ << window() <<  m_data.fullFrameMargins  << "->" << newMargins;
         m_data.fullFrameMargins = newMargins;
@@ -2700,12 +2709,14 @@ void QWindowsWindow::updateFullFrameMargins()
 
 void QWindowsWindow::calculateFullFrameMargins()
 {
+    if (m_data.flags & Qt::FramelessWindowHint)
+        return;
     // Normally obtained from WM_NCCALCSIZE. This calculation only works
     // when no native menu is present.
     const auto systemMargins = testFlag(DisableNonClientScaling)
         ? QWindowsGeometryHint::frameOnPrimaryScreen(window(), m_data.hwnd)
         : frameMargins_sys();
-    setFullFrameMargins(systemMargins + m_data.customMargins);
+    setFullFrameMargins(systemMargins + customMargins());
 }
 
 QMargins QWindowsWindow::frameMargins() const
@@ -2718,6 +2729,8 @@ QMargins QWindowsWindow::frameMargins() const
 
 QMargins QWindowsWindow::fullFrameMargins() const
 {
+    if (m_data.flags & Qt::FramelessWindowHint)
+        return {};
     return m_data.fullFrameMargins;
 }
 
@@ -3204,6 +3217,13 @@ void QWindowsWindow::setMenuBar(QWindowsMenuBar *mb)
     m_menuBar = mb;
 }
 
+QMargins QWindowsWindow::customMargins() const
+{
+    if (m_data.flags & Qt::FramelessWindowHint)
+        return {};
+    return m_data.customMargins;
+}
+
 /*!
     \brief Sets custom margins to be added to the default margins determined by
     the windows style in the handling of the WM_NCCALCSIZE message.
@@ -3216,6 +3236,10 @@ void QWindowsWindow::setMenuBar(QWindowsMenuBar *mb)
 
 void QWindowsWindow::setCustomMargins(const QMargins &newCustomMargins)
 {
+    if (m_data.flags & Qt::FramelessWindowHint) {
+        qCWarning(lcQpaWindows) << "You should not set custom margins for a frameless window.";
+        return;
+    }
     if (newCustomMargins != m_data.customMargins) {
         const QMargins oldCustomMargins = m_data.customMargins;
         m_data.customMargins = newCustomMargins;
diff --git a/src/plugins/platforms/windows/qwindowswindow.h b/src/plugins/platforms/windows/qwindowswindow.h
index eee13c2281..49e80522b4 100644
--- a/src/plugins/platforms/windows/qwindowswindow.h
+++ b/src/plugins/platforms/windows/qwindowswindow.h
@@ -272,7 +272,7 @@ public:
     QWindowsMenuBar *menuBar() const;
     void setMenuBar(QWindowsMenuBar *mb);
 
-    QMargins customMargins() const override { return m_data.customMargins; }
+    QMargins customMargins() const override;
     void setCustomMargins(const QMargins &m) override;
 
     void setStyle(unsigned s) const;