/**************************************************************************** ** ** Copyright (C) 2014 Digia Plc ** All rights reserved. ** For any questions to Digia, please use contact form at http://qt.digia.com ** ** This file is part of the Qt Creator Enterprise Auto Test Add-on. ** ** Licensees holding valid Qt Enterprise licenses may use this file in ** accordance with the Qt Enterprise License Agreement provided with the ** Software or, alternatively, in accordance with the terms contained in ** a written agreement between you and Digia. ** ** If you have questions regarding the use of this file, please use ** contact form at http://qt.digia.com ** ****************************************************************************/ #include "autotestconstants.h" #include "autotestplugin.h" #include "testresultspane.h" #include "testrunner.h" #include "testsettings.h" #include // REMOVE #include #include #include #include #include #include #include #include #include #include namespace Autotest { namespace Internal { static TestRunner *m_instance = 0; static QProcess *m_runner = 0; static QFutureInterface *m_currentFuture = 0; TestRunner *TestRunner::instance() { if (!m_instance) m_instance = new TestRunner; return m_instance; } TestRunner::TestRunner(QObject *parent) : QObject(parent), m_building(false), m_executingTests(false) { } TestRunner::~TestRunner() { qDeleteAll(m_selectedTests); m_selectedTests.clear(); m_instance = 0; if (m_runner) delete m_runner; } void TestRunner::setSelectedTests(const QList &selected) { qDeleteAll(m_selectedTests); m_selectedTests.clear(); m_selectedTests = selected; } /******************** XML line parser helper ********************/ static bool xmlStartsWith(const QString &code, const QString &start, QString &result) { if (code.startsWith(start)) { result = code.mid(start.length()); result = result.left(result.indexOf(QLatin1Char('"'))); result = result.left(result.indexOf(QLatin1String(""), index) - index); return !result.isEmpty(); } return false; } static bool xmlExtractTypeFileLine(const QString &code, const QString &tagStart, ResultType &result, QString &file, int &line) { if (code.startsWith(tagStart)) { int start = code.indexOf(QLatin1String(" type=\"")) + 7; result = TestResult::resultFromString( code.mid(start, code.indexOf(QLatin1Char('"'), start) - start)); start = code.indexOf(QLatin1String(" file=\"")) + 7; file = code.mid(start, code.indexOf(QLatin1Char('"'), start) - start); start = code.indexOf(QLatin1String(" line=\"")) + 7; line = code.mid(start, code.indexOf(QLatin1Char('"'), start) - start).toInt(); return true; } return false; } // adapted from qplaintestlogger.cpp static QString formatResult(double value) { if (value < 0 || value == NAN) return QLatin1String("NAN"); if (value == 0) return QLatin1String("0"); int significantDigits = 0; qreal divisor = 1; while (value / divisor >= 1) { divisor *= 10; ++significantDigits; } QString beforeDecimalPoint = QString::number(value, 'f', 0); QString afterDecimalPoint = QString::number(value, 'f', 20); afterDecimalPoint.remove(0, beforeDecimalPoint.count() + 1); const int beforeUse = qMin(beforeDecimalPoint.count(), significantDigits); const int beforeRemove = beforeDecimalPoint.count() - beforeUse; beforeDecimalPoint.chop(beforeRemove); for (int i = 0; i < beforeRemove; ++i) beforeDecimalPoint.append(QLatin1Char('0')); int afterUse = significantDigits - beforeUse; if (beforeDecimalPoint == QLatin1String("0") && !afterDecimalPoint.isEmpty()) { ++afterUse; int i = 0; while (i < afterDecimalPoint.count() && afterDecimalPoint.at(i) == QLatin1Char('0')) ++i; afterUse += i; } const int afterRemove = afterDecimalPoint.count() - afterUse; afterDecimalPoint.chop(afterRemove); QString result = beforeDecimalPoint; if (afterUse > 0) result.append(QLatin1Char('.')); result += afterDecimalPoint; return result; } static bool xmlExtractBenchmarkInformation(const QString &code, const QString &tagStart, QString &description) { if (code.startsWith(tagStart)) { int start = code.indexOf(QLatin1String(" metric=\"")) + 9; const QString metric = code.mid(start, code.indexOf(QLatin1Char('"'), start) - start); start = code.indexOf(QLatin1String(" value=\"")) + 8; const double value = code.mid(start, code.indexOf(QLatin1Char('"'), start) - start).toDouble(); start = code.indexOf(QLatin1String(" iterations=\"")) + 13; const int iterations = code.mid(start, code.indexOf(QLatin1Char('"'), start) - start).toInt(); QString metricsTxt; if (metric == QLatin1String("WalltimeMilliseconds")) // default metricsTxt = QLatin1String("msecs"); else if (metric == QLatin1String("CPUTicks")) // -tickcounter metricsTxt = QLatin1String("CPU ticks"); else if (metric == QLatin1String("Events")) // -eventcounter metricsTxt = QLatin1String("events"); else if (metric == QLatin1String("InstructionReads")) // -callgrind metricsTxt = QLatin1String("instruction reads"); else if (metric == QLatin1String("CPUCycles")) // -perf metricsTxt = QLatin1String("CPU cycles"); description = QObject::tr("%1 %2 per iteration (total: %3, iterations: %4)") .arg(formatResult(value)) .arg(metricsTxt) .arg(formatResult(value * (double)iterations)) .arg(iterations); return true; } return false; } /****************** XML line parser helper end ******************/ void processOutput() { if (!m_runner) return; static QString className; static QString testCase; static QString dataTag; static ResultType result = ResultType::UNKNOWN; static QString description; static QString file; static int lineNumber = 0; static QString duration; static bool readingDescription = false; static QString qtVersion; static QString qtestVersion; static QString bmDescription; while (m_runner->canReadLine()) { // TODO Qt5 uses UTF-8 - while Qt4 uses ISO-8859-1 - could this be a problem? const QString line = QString::fromUtf8(m_runner->readLine()).trimmed(); if (line.isEmpty() || line.startsWith(QLatin1String("addTestResult( TestResult(className, testCase, QString(), ResultType::MESSAGE_CURRENT_TEST, QObject::tr("Entering Test Function %1::%2") .arg(className).arg(testCase))); continue; } if (xmlStartsWith(line, QLatin1String(""), dataTag)) continue; if (xmlCData(line, QLatin1String(""), description)) { if (!line.endsWith(QLatin1String(""))) readingDescription = true; continue; } if (xmlExtractTypeFileLine(line, QLatin1String(""))) { TestResult testResult(className, testCase, dataTag, result, description); if (!file.isEmpty()) file = QFileInfo(m_runner->workingDirectory(), file).canonicalFilePath(); testResult.setFileName(file); testResult.setLine(lineNumber); TestResultsPane::instance()->addTestResult(testResult); } continue; } if (xmlExtractBenchmarkInformation(line, QLatin1String("addTestResult(testResult); continue; } if (line == QLatin1String("") || line == QLatin1String("")) { TestResult testResult(className, testCase, dataTag, result, description); if (!file.isEmpty()) file = QFileInfo(m_runner->workingDirectory(), file).canonicalFilePath(); testResult.setFileName(file); testResult.setLine(lineNumber); TestResultsPane::instance()->addTestResult(testResult); description = QString(); } else if (line == QLatin1String("") && !duration.isEmpty()) { TestResult testResult(className, testCase, QString(), ResultType::MESSAGE_INTERNAL, QObject::tr("execution took %1ms").arg(duration)); TestResultsPane::instance()->addTestResult(testResult); m_currentFuture->setProgressValue(m_currentFuture->progressValue() + 1); } else if (line == QLatin1String("") && !duration.isEmpty()) { TestResult testResult(className, QString(), QString(), ResultType::MESSAGE_INTERNAL, QObject::tr("Test execution took %1ms").arg(duration)); TestResultsPane::instance()->addTestResult(testResult); } else if (readingDescription) { if (line.endsWith(QLatin1String("]]>"))) { description.append(QLatin1Char('\n')); description.append(line.left(line.indexOf(QLatin1String("]]>")))); readingDescription = false; } else { description.append(QLatin1Char('\n')); description.append(line); } } else if (xmlStartsWith(line, QLatin1String(""), qtVersion)) { TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_INTERNAL, QObject::tr("Qt Version: %1").arg(qtVersion))); } else if (xmlStartsWith(line, QLatin1String(""), qtestVersion)) { TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_INTERNAL, QObject::tr("QTest Version: %1").arg(qtestVersion))); } else { // qDebug() << "Unhandled line:" << line; // TODO remove } } } static QString which(const QString &path, const QString &cmd) { if (path.isEmpty() || cmd.isEmpty()) return QString(); QStringList paths; #ifdef Q_OS_WIN paths = path.split(QLatin1Char(';')); #else paths = path.split(QLatin1Char(':')); #endif foreach (const QString &p, paths) { const QString fName = p + QDir::separator() + cmd; QFileInfo fi(fName); if (fi.exists() && fi.isExecutable()) return fName; #ifdef Q_OS_WIN fi = QFileInfo(fName + QLatin1String(".exe")); if (fi.exists()) return fi.absoluteFilePath(); fi = QFileInfo(fName + QLatin1String(".bat")); if (fi.exists()) return fi.absoluteFilePath(); fi = QFileInfo(fName + QLatin1String(".cmd")); if (fi.exists()) return fi.absoluteFilePath(); #endif } return QString(); } bool performExec(const QString &cmd, const QStringList &args, const QString &workingDir, const Utils::Environment &env, int timeout) { QString runCmd; if (!QDir::toNativeSeparators(cmd).contains(QDir::separator())) { if (env.hasKey(QLatin1String("PATH"))) runCmd = which(env.value(QLatin1String("PATH")), cmd); } else if (QFileInfo(cmd).exists()) { runCmd = cmd; } if (runCmd.isEmpty()) { TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_FATAL, QObject::tr("*** Could not find command '%1' ***").arg(cmd))); return false; } m_runner->setWorkingDirectory(workingDir); m_runner->setProcessEnvironment(env.toProcessEnvironment()); QTime executionTimer; if (args.count()) { m_runner->start(runCmd, args); } else { m_runner->start(runCmd); } bool ok = m_runner->waitForStarted(); executionTimer.start(); if (ok) { while (m_runner->state() == QProcess::Running && executionTimer.elapsed() < timeout) { if (m_currentFuture->isCanceled()) { m_runner->kill(); m_runner->waitForFinished(); TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_FATAL, QObject::tr("*** Test Run canceled by user ***"))); } qApp->processEvents(); } } if (ok && executionTimer.elapsed() < timeout) { return m_runner->exitCode() == 0; } else { if (m_runner->state() != QProcess::NotRunning) { m_runner->kill(); m_runner->waitForFinished(); TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_FATAL, QObject::tr("*** Test Case canceled due to timeout ***\n" "Maybe raise the timeout?"))); } return false; } } void performTestRun(QFutureInterface &future, const QList selectedTests) { int testCaseCount = 0; foreach (const TestConfiguration *config, selectedTests) testCaseCount += config->testCaseCount(); m_currentFuture = &future; m_runner = new QProcess; m_runner->setReadChannelMode(QProcess::MergedChannels); m_runner->setReadChannel(QProcess::StandardOutput); QObject::connect(m_runner, &QProcess::readyReadStandardOutput, &processOutput); future.setProgressRange(0, testCaseCount); future.setProgressValue(0); const QSharedPointer settings = AutotestPlugin::instance()->settings(); const int timeout = settings->timeout; const QString metricsOption = TestSettings::metricsTypeToOption(settings->metrics); foreach (const TestConfiguration *tc, selectedTests) { if (future.isCanceled()) break; QString cmd = tc->targetFile(); QString workDir = tc->workingDirectory(); QStringList args; Utils::Environment env = tc->environment(); args << QLatin1String("-xml"); if (!metricsOption.isEmpty()) args << metricsOption; if (tc->testCases().count()) args << tc->testCases(); performExec(cmd, args, workDir, env, timeout); } future.setProgressValue(testCaseCount); delete m_runner; m_runner = 0; m_currentFuture = 0; } void TestRunner::runTests() { // clear old log and output pane TestResultsPane::instance()->clearContents(); // handle faulty test configurations QList toBeRemoved; foreach (TestConfiguration *config, m_selectedTests) if (!config->project()) { toBeRemoved.append(config); TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_WARN, tr("*** Project is null for '%1' - removing from Test Run ***\n" "This might be the case for a faulty environment or similar." ).arg(config->displayName()))); } foreach (TestConfiguration *config, toBeRemoved) { m_selectedTests.removeOne(config); delete config; } if (m_selectedTests.empty()) { TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_WARN, tr("*** No tests selected - canceling Test Run ***"))); return; } ProjectExplorer::Project *project = m_selectedTests.at(0)->project(); if (!project) { TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_WARN, tr("*** Project is null - canceling Test Run ***\n" "Actually only Desktop kits are supported - make sure the " "current active kit is a Desktop kit."))); return; } ProjectExplorer::ProjectExplorerPlugin *pep = ProjectExplorer::ProjectExplorerPlugin::instance(); ProjectExplorer::Internal::ProjectExplorerSettings pes = pep->projectExplorerSettings(); if (pes.buildBeforeDeploy) { if (!project->hasActiveBuildSettings()) { TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_FATAL, tr("*** Project is not configured - canceling Test Run ***"))); return; } buildProject(project); while (m_building) { qApp->processEvents(); } if (!m_buildSucceeded) { TestResultsPane::instance()->addTestResult( TestResult(QString(), QString(), QString(), ResultType::MESSAGE_FATAL, tr("*** Build failed - canceling Test Run ***"))); return; } } m_executingTests = true; emit testRunStarted(); QFuture future = QtConcurrent::run(&performTestRun , m_selectedTests); Core::FutureProgress *progress = Core::ProgressManager::addTask(future, tr("Running Tests"), Autotest::Constants::TASK_INDEX); connect(progress, &Core::FutureProgress::finished, TestRunner::instance(), &TestRunner::onFinished); } void TestRunner::buildProject(ProjectExplorer::Project *project) { m_building = true; m_buildSucceeded = false; ProjectExplorer::BuildManager *mgr = static_cast( ProjectExplorer::BuildManager::instance()); ProjectExplorer::ProjectExplorerPlugin *pep = ProjectExplorer::ProjectExplorerPlugin::instance(); connect(mgr, &ProjectExplorer::BuildManager::buildQueueFinished, this, &TestRunner::buildFinished); pep->buildProject(project); } void TestRunner::buildFinished(bool success) { ProjectExplorer::BuildManager *mgr = static_cast( ProjectExplorer::BuildManager::instance()); disconnect(mgr, &ProjectExplorer::BuildManager::buildQueueFinished, this, &TestRunner::buildFinished); m_building = false; m_buildSucceeded = success; } void TestRunner::onFinished() { m_executingTests = false; emit testRunFinished(); } void TestRunner::stopTestRun() { if (m_runner && m_runner->state() != QProcess::NotRunning && m_currentFuture) m_currentFuture->cancel(); } } // namespace Internal } // namespace Autotest