Windows: Update layered state on creation instead of backingstore flush

The WS_EX_LAYERED state of a window can be determined on creation, and
the logic to do so should be centralized to QWindowsWindow::setWindowLayered,
so that we don't have divergence.

This fixes an issue where child windows would only support transparency
if they had Qt::FramelessWindowHint set, as the QWindowBackingStore had
a different idea about when to enable WS_EX_LAYERED than QWindowsWindow.

Task-number: QTBUG-122590
Task-number: QTBUG-135859
Pick-to: 6.9 6.8
Change-Id: I453967a5a2ce8974cdd1dbf6e36327e97384c33d
Reviewed-by: Oliver Wolff <[email protected]>
Reviewed-by: Zhao Yuhang <[email protected]>
Reviewed-by: Pavel Dubsky <[email protected]>

3 files changed, 5 insertions, 14 deletions

diff --git a/src/plugins/platforms/windows/qwindowsbackingstore.cpp b/src/plugins/platforms/windows/qwindowsbackingstore.cpp
index 07918f6094d..d4e3cf3ccca 100644
--- a/src/plugins/platforms/windows/qwindowsbackingstore.cpp
+++ b/src/plugins/platforms/windows/qwindowsbackingstore.cpp
@@ -50,9 +50,7 @@ void QWindowsBackingStore::flush(QWindow *window, const QRegion &region,
     QWindowsWindow *rw = QWindowsWindow::windowsWindowOf(window);
     Q_ASSERT(rw);
 
-    const bool hasAlpha = rw->format().hasAlpha();
-    const Qt::WindowFlags flags = window->flags();
-    if ((flags & Qt::FramelessWindowHint) && QWindowsWindow::setWindowLayered(rw->handle(), flags, hasAlpha, rw->opacity()) && hasAlpha) {
+    if (rw->isLayered()) {
         // Windows with alpha: Use blend function to update.
         QRect r = QHighDpi::toNativePixels(window->frameGeometry(), window);
         QMargins frameMargins = rw->frameMargins();
diff --git a/src/plugins/platforms/windows/qwindowswindow.cpp b/src/plugins/platforms/windows/qwindowswindow.cpp
index 22277af52bb..d11e6289566 100644
--- a/src/plugins/platforms/windows/qwindowswindow.cpp
+++ b/src/plugins/platforms/windows/qwindowswindow.cpp
@@ -538,14 +538,6 @@ static void setWindowOpacity(HWND hwnd, Qt::WindowFlags flags, bool hasAlpha, bo
     }
 }
 
-static inline void updateGLWindowSettings(const QWindow *w, HWND hwnd, Qt::WindowFlags flags, qreal opacity)
-{
-    const bool isAccelerated = windowIsAccelerated(w);
-    const bool hasAlpha = w->format().hasAlpha();
-
-    setWindowOpacity(hwnd, flags, hasAlpha, isAccelerated, opacity);
-}
-
 [[nodiscard]] static inline int getResizeBorderThickness(const UINT dpi)
 {
     // The width of the padded border will always be 0 if DWM composition is
@@ -1046,10 +1038,13 @@ void WindowCreationData::initialize(const QWindow *w, HWND hwnd, bool frameChang
             MARGINS margins = { -1, -1, -1, -1 };
             DwmExtendFrameIntoClientArea(hwnd, &margins);
         }
-        updateGLWindowSettings(w, hwnd, flags, opacityLevel);
     } else { // child.
         SetWindowPos(hwnd, HWND_TOP, 0, 0, 0, 0, swpFlags);
     }
+
+    const bool isAccelerated = windowIsAccelerated(w);
+    const bool hasAlpha = w->format().hasAlpha();
+    setWindowOpacity(hwnd, flags, hasAlpha, isAccelerated, opacityLevel);
 }
 
 
diff --git a/tests/manual/embeddedwindows/main.cpp b/tests/manual/embeddedwindows/main.cpp
index e34c7206eaa..b58370d1f3a 100644
--- a/tests/manual/embeddedwindows/main.cpp
+++ b/tests/manual/embeddedwindows/main.cpp
@@ -69,8 +69,6 @@ int main(int argc, char *argv[])
     QSurfaceFormat format = transparentChildWindow->format();
     format.setAlphaBufferSize(8);
     transparentChildWindow->setFormat(format);
-    // FIXME: Windows requires this, even for child windows
-    transparentChildWindow->setFlag(Qt::FramelessWindowHint);
     transparentChildWindow->setParent(&window);
     transparentChildWindow->setGeometry(350, 50, 100, 100);
     transparentChildWindow->showNormal();