Qt: Force layer backing on Big Sur when needed.

Make sure NSView.wantsLayer is true by setting QT_MAC_WANTS_LAYER=1 at
startup if we're running on Big Sur and we were built with a version of
Qt susceptible to QTBUG-87014. Fixes #17075?


(cherry picked from commit d4b40c00472d0d1eb032725f27f1c827943d8546)

1 files changed, 56 insertions, 2 deletions

diff --git a/ui/qt/main.cpp b/ui/qt/main.cpp
index 8ba05cfa17..8ba244f682 100644
--- a/ui/qt/main.cpp
+++ b/ui/qt/main.cpp
@@ -365,7 +365,7 @@ check_and_warn_user_startup()
 // path except our application directory.
 
 static inline void
-reset_library_path(void)
+win32_reset_library_path(void)
 {
     QString app_path = QDir(get_progfile_dir()).path();
     foreach (QString path, QCoreApplication::libraryPaths()) {
@@ -375,6 +375,56 @@ reset_library_path(void)
 }
 #endif
 
+#ifdef Q_OS_MAC
+// Try to work around
+//
+//     https://gitlab.com/wireshark/wireshark/-/issues/17075
+//
+// aka
+//
+//     https://bugreports.qt.io/browse/QTBUG-87014
+//
+// The fix at
+//
+//     https://codereview.qt-project.org/c/qt/qtbase/+/322228/3/src/plugins/platforms/cocoa/qnsview_drawing.mm
+//
+// enables layer backing if we're running on Big Sur OR we're running on
+// Catalina AND we were built with the Catalina SDK. Enable layer backing
+// here by setting QT_MAC_WANTS_LAYER=1, but only if we're running on Big
+// Sur and our version of Qt doesn't have a fix for QTBUG-87014.
+#include <QOperatingSystemVersion>
+static inline void
+macos_enable_layer_backing(void)
+{
+    // At the time of this writing, the QTBUG-87014 for layerEnabledByMacOS is...
+    //
+    // ...in https://github.com/qt/qtbase/blob/5.12/src/plugins/platforms/cocoa/qnsview_drawing.mm
+    // ...not in https://github.com/qt/qtbase/blob/5.12.10/src/plugins/platforms/cocoa/qnsview_drawing.mm
+    // ...in https://github.com/qt/qtbase/blob/5.15/src/plugins/platforms/cocoa/qnsview_drawing.mm
+    // ...not in https://github.com/qt/qtbase/blob/5.15.2/src/plugins/platforms/cocoa/qnsview_drawing.mm
+    // ...not in https://github.com/qt/qtbase/blob/6.0/src/plugins/platforms/cocoa/qnsview_drawing.mm
+    // ...not in https://github.com/qt/qtbase/blob/6.0.0/src/plugins/platforms/cocoa/qnsview_drawing.mm
+    //
+    // We'll assume that it will be fixed in 5.12.11, 5.15.3, and 6.0.1.
+    // Note that we only ship LTS versions of Qt with our macOS packages.
+    // Feel free to add other versions if needed.
+#if  \
+        (QT_VERSION >= QT_VERSION_CHECK(5, 12, 0) && QT_VERSION < QT_VERSION_CHECK(5, 12, 11) \
+        || (QT_VERSION >= QT_VERSION_CHECK(5, 15, 0) &&  QT_VERSION < QT_VERSION_CHECK(5, 15, 3)) \
+        || (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) &&  QT_VERSION < QT_VERSION_CHECK(6, 0, 1)) \
+    )
+    QOperatingSystemVersion os_ver = QOperatingSystemVersion::current();
+    int major_ver = os_ver.majorVersion();
+    int minor_ver = os_ver.minorVersion();
+    if ( (major_ver == 10 && minor_ver >= 16) || major_ver >= 11 ) {
+        if (qgetenv("QT_MAC_WANTS_LAYER").isEmpty()) {
+            qputenv("QT_MAC_WANTS_LAYER", "1");
+        }
+    }
+#endif
+}
+#endif
+
 /* And now our feature presentation... [ fade to music ] */
 int main(int argc, char *qt_argv[])
 {
@@ -435,6 +485,10 @@ int main(int argc, char *qt_argv[])
     cf_set_max_records(53000000);
 #endif
 
+#ifdef Q_OS_MAC
+    macos_enable_layer_backing();
+#endif
+
     /* Enable destinations for logging earlier in startup */
     set_console_log_handler();
     qInstallMessageHandler(g_log_message_handler);
@@ -579,7 +633,7 @@ int main(int argc, char *qt_argv[])
     commandline_early_options(argc, argv);
 
 #ifdef _WIN32
-    reset_library_path();
+    win32_reset_library_path();
 #endif
 
     // Handle DPI scaling on Windows. This causes problems in at least