Add a few improvements to the baseline testing framework

1) Add a QBASELINETEST_MAIN macro as a replacement for QTEST_MAIN,
relieving the clients of reproducing the kludgy workaround

2) Add a -server command line option to baseline tests, as an
alternative to specifying the server in an environment variable

3) Fix command line parsing so that it exits on syntax errors.

Change-Id: I36f38267143a308e971e2e7b2fdbe4be44370043
Reviewed-by: Volker Hilsheimer <[email protected]>

8 files changed, 42 insertions, 58 deletions

diff --git a/tests/baseline/painting/tst_baseline_painting.cpp b/tests/baseline/painting/tst_baseline_painting.cpp
index f486b334303..c8de08e7d28 100644
--- a/tests/baseline/painting/tst_baseline_painting.cpp
+++ b/tests/baseline/painting/tst_baseline_painting.cpp
@@ -473,17 +473,6 @@ void tst_Lancelot::paint(QPaintDevice *device, GraphicsEngine engine, QImage::Fo
     p.end();
 }
 
-#define main _realmain
-QTEST_MAIN(tst_Lancelot)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Lancelot);
 
 #include "tst_baseline_painting.moc"
diff --git a/tests/baseline/shared/baselineprotocol.cpp b/tests/baseline/shared/baselineprotocol.cpp
index 6a38e718317..311b003c075 100644
--- a/tests/baseline/shared/baselineprotocol.cpp
+++ b/tests/baseline/shared/baselineprotocol.cpp
@@ -261,18 +261,21 @@ bool BaselineProtocol::disconnect()
 }
 
 
-bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo& clientInfo)
+bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo &clientInfo, const QString &server)
 {
     errMsg.clear();
-    QByteArray serverName(qgetenv("QT_LANCELOT_SERVER"));
-    if (serverName.isNull())
-        serverName = "lancelot.test.qt-project.org";
+    QString serverName = server;
+    if (serverName.isEmpty()) {
+        serverName = qEnvironmentVariable("QT_LANCELOT_SERVER");
+        if (serverName.isEmpty())
+            serverName = QStringLiteral("lancelot.test.qt-project.org");
+    }
 
     socket.connectToHost(serverName, ServerPort);
     if (!socket.waitForConnected(Timeout)) {
         QThread::sleep(std::chrono::seconds{3});  // Wait a bit and try again, the server might just be restarting
         if (!socket.waitForConnected(Timeout)) {
-            errMsg += QLS("TCP connectToHost failed. Host:") + QLS(serverName) + QLS(" port:") + QString::number(ServerPort);
+            errMsg += QLS("TCP connectToHost failed. Host:") + serverName + QLS(" port:") + QString::number(ServerPort);
             return false;
         }
     }
diff --git a/tests/baseline/shared/baselineprotocol.h b/tests/baseline/shared/baselineprotocol.h
index 598a0cd3afd..93d416fcd8f 100644
--- a/tests/baseline/shared/baselineprotocol.h
+++ b/tests/baseline/shared/baselineprotocol.h
@@ -115,7 +115,8 @@ public:
     // For client:
 
     // For advanced client:
-    bool connect(const QString &testCase, bool *dryrun = nullptr, const PlatformInfo& clientInfo = PlatformInfo());
+    bool connect(const QString &testCase, bool *dryrun = nullptr,
+                 const PlatformInfo &clientInfo = PlatformInfo(), const QString &server = QString());
     bool disconnect();
     bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList);
     bool submitMatch(const ImageItem &item, QByteArray *serverMsg);
diff --git a/tests/baseline/shared/qbaselinetest.cpp b/tests/baseline/shared/qbaselinetest.cpp
index e41b8d53216..964266f1b4d 100644
--- a/tests/baseline/shared/qbaselinetest.cpp
+++ b/tests/baseline/shared/qbaselinetest.cpp
@@ -11,6 +11,7 @@
 namespace QBaselineTest {
 
 static char *fargv[MAXCMDLINEARGS];
+static QString server;
 static bool simfail = false;
 static PlatformInfo customInfo;
 static bool customAutoModeSet = false;
@@ -33,6 +34,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         return;
 
     bool showHelp = false;
+    bool abortOnHelp = true;
 
     int fargc = 0;
     int numArgs = *argcp;
@@ -41,7 +43,16 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         QByteArray arg = (*argvp)[i];
         QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr;
 
-        if (arg == "-simfail") {
+        if (arg == "-server") {
+            i++;
+            if (!nextArg.isEmpty()) {
+                server = QString::fromLocal8Bit(nextArg);
+            } else {
+                qWarning() << "-server requires parameter";
+                showHelp = true;
+                break;
+            }
+        } else if (arg == "-simfail") {
             simfail = true;
         } else if (arg == "-fuzzlevel") {
             i++;
@@ -77,10 +88,13 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
             }
             customInfo.addOverride(key, value);
         } else {
-            if ( (arg == "-help") || (arg == "--help") )
+            if ( (arg == "-help") || (arg == "--help") ) {
                 showHelp = true;
+                abortOnHelp = false;
+            }
             if (fargc >= MAXCMDLINEARGS) {
                 qWarning() << "Too many command line arguments!";
+                showHelp = true;
                 break;
             }
             fargv[fargc++] = (*argvp)[i];
@@ -93,7 +107,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         // TBD: arrange for this to be printed *after* QTest's help
         QTextStream out(stdout);
         out << "\n Baseline testing (lancelot) options:\n";
-        out << " -simfail            : Force an image comparison mismatch. For testing purposes.\n";
+        out << " -server <host>      : Set the network baseline server to connect to.\n";
+        out << "                       The default is taken from the environment variable QT_LANCELOT_SERVER.\n";
+        out << " -simfail            : Force an image comparison mismatch. For development purposes.\n";
         out << " -fuzzlevel <int>    : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
         out << " -auto               : Inform server that this run is done by a daemon, CI system or similar.\n";
         out << " -adhoc (default)    : The inverse of -auto; this run is done by human, e.g. for testing.\n";
@@ -104,6 +120,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         out << "                       for example: -compareto QtVersion=4.8.0\n";
         out << "                       Multiple -compareto client specifications may be given.\n";
         out << "\n";
+        out.flush();
+        if (abortOnHelp)
+            std::exit(1);
     }
 }
 
@@ -182,7 +201,7 @@ bool connect(QByteArray *msg, bool *error)
         return false;
     }
 
-    if (!proto.connect(testCase, &dryRunMode, clientInfo)) {
+    if (!proto.connect(testCase, &dryRunMode, clientInfo, server)) {
         *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
         *error = true;
         return false;
diff --git a/tests/baseline/shared/qbaselinetest.h b/tests/baseline/shared/qbaselinetest.h
index f120e2bcd8f..73bb5dfda9b 100644
--- a/tests/baseline/shared/qbaselinetest.h
+++ b/tests/baseline/shared/qbaselinetest.h
@@ -66,4 +66,9 @@ do {\
         QSKIP("Blacklisted on baseline server.");\
 } while (0)
 
+#define QBASELINETEST_MAIN(TestObject) QTEST_MAIN_WRAPPER(TestObject, \
+    QHashSeed::setDeterministicGlobalSeed(); \
+    QBaselineTest::handleCmdLineArgs(&argc, &argv); \
+    QTEST_MAIN_SETUP())
+
 #endif // BASELINETEST_H
diff --git a/tests/baseline/stylesheet/tst_baseline_stylesheet.cpp b/tests/baseline/stylesheet/tst_baseline_stylesheet.cpp
index 67a618988be..1028c68250f 100644
--- a/tests/baseline/stylesheet/tst_baseline_stylesheet.cpp
+++ b/tests/baseline/stylesheet/tst_baseline_stylesheet.cpp
@@ -224,17 +224,6 @@ void tst_Stylesheet::tst_QHeaderView()
     QBASELINE_TEST(takeSnapshot());
 }
 
-#define main _realmain
-QTEST_MAIN(tst_Stylesheet)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Stylesheet)
 
 #include "tst_baseline_stylesheet.moc"
diff --git a/tests/baseline/text/tst_baseline_text.cpp b/tests/baseline/text/tst_baseline_text.cpp
index 59a5f478a52..45c709b92f5 100644
--- a/tests/baseline/text/tst_baseline_text.cpp
+++ b/tests/baseline/text/tst_baseline_text.cpp
@@ -103,17 +103,6 @@ void tst_Text::tst_differentScriptsBackgrounds()
 }
 
 
-#define main _realmain
-QTEST_MAIN(tst_Text)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Text)
 
 #include "tst_baseline_text.moc"
diff --git a/tests/baseline/widgets/tst_baseline_widgets.cpp b/tests/baseline/widgets/tst_baseline_widgets.cpp
index 8a763eb8fa3..ffabf9e07eb 100644
--- a/tests/baseline/widgets/tst_baseline_widgets.cpp
+++ b/tests/baseline/widgets/tst_baseline_widgets.cpp
@@ -1275,17 +1275,6 @@ void tst_Widgets::tst_QLCDNumber()
     QBASELINE_CHECK_DEFERRED(takeSnapshot(), "lcdnumber");
 }
 
-#define main _realmain
-QTEST_MAIN(tst_Widgets)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Widgets)
 
 #include "tst_baseline_widgets.moc"