Add benchmarking tool.

Takes two qbs repo states, runs callgrind and massif on different phases
of the build process for each of them and informs the user of the
relative performance of these operations.
I suggest running this tool before committing anything that could
conceivably impact performance. Note that we have similar regression
detection functionality on our build machines, but it's better to spot
such problems before pushing a change.
Example output (for commits 04df1532c4 and f53d724eec building qbs itself):
========== Performance data for Resolving ==========
    Old instruction count: 1931242543
    New instruction count: 1457438375
    Relative change: -25 %
    Old peak memory usage: 1710721 Bytes
    New peak memory usage: 1509879 Bytes
    Relative change: -12 %
========== Performance data for Rule Execution ==========
    Old instruction count: 2924308821
    New instruction count: 2890322188
    Relative change: -2 %
    Old peak memory usage: 2520740 Bytes
    New peak memory usage: 2488058 Bytes
    Relative change: -2 %
========== Performance data for Null Build ==========
    Old instruction count: 572213104
    New instruction count: 517813513
    Relative change: -10 %
    Old peak memory usage: 1964455 Bytes
    New peak memory usage: 1774279 Bytes
    Relative change: -10 %

Change-Id: I1f05c647d204b6cacf3539c8ecbf13633b757cf7
Reviewed-by: Joerg Bornemann <joerg.bornemann@theqtcompany.com>

13 files changed, 1083 insertions, 0 deletions

diff --git a/tests/benchmarker/activities.h b/tests/benchmarker/activities.h
new file mode 100644
index 000000000..93c27abbc
--- /dev/null
+++ b/tests/benchmarker/activities.h
@@ -0,0 +1,44 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_BENCHMARKER_ACTIVITY_H
+#define QBS_BENCHMARKER_ACTIVITY_H
+
+#include <QFlags>
+
+namespace qbsBenchmarker {
+
+enum Activity { ActivityResolving = 1, ActivityRuleExecution = 2, ActivityNullBuild = 4 };
+Q_DECLARE_FLAGS(Activities, Activity)
+Q_DECLARE_OPERATORS_FOR_FLAGS(Activities)
+
+} // namespace qbsBenchmarker
+
+#endif // Include guard.
+
diff --git a/tests/benchmarker/benchmarker-main.cpp b/tests/benchmarker/benchmarker-main.cpp
new file mode 100644
index 000000000..b4bcbb038
--- /dev/null
+++ b/tests/benchmarker/benchmarker-main.cpp
@@ -0,0 +1,109 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#include "benchmarker.h"
+#include "commandlineparser.h"
+#include "exception.h"
+
+#include <QCoreApplication>
+
+#include <cstdlib>
+#include <iostream>
+
+using namespace qbsBenchmarker;
+
+static QByteArray relativeChange(qint64 oldVal, qint64 newVal)
+{
+    QByteArray change = newVal == 0
+            ? "0" : QByteArray::number(std::abs(newVal * 100 / oldVal - 100));
+    change += " %";
+    if (oldVal > newVal)
+        change.prepend('-');
+    else if (oldVal < newVal)
+        change.prepend('+');
+    return change;
+}
+
+static void printResults(Activity activity, const BenchmarkResults &results)
+{
+    std::cout << "========== Performance data for ";
+    switch (activity) {
+    case ActivityResolving:
+        std::cout << "Resolving";
+        break;
+    case ActivityRuleExecution:
+        std::cout << "Rule Execution";
+        break;
+    case ActivityNullBuild:
+        std::cout << "Null Build";
+        break;
+    }
+    std::cout << " ==========" << std::endl;
+    const BenchmarkResult result = results.value(activity);
+    const char * const indent = "    ";
+    std::cout << indent << "Old instruction count: " << result.oldInstructionCount << std::endl;
+    std::cout << indent << "New instruction count: " << result.newInstructionCount << std::endl;
+    std::cout << indent << "Relative change: "
+              << relativeChange(result.oldInstructionCount, result.newInstructionCount).constData()
+              << std::endl;
+    std::cout << indent << "Old peak memory usage: " << result.oldPeakMemoryUsage << " Bytes"
+              << std::endl;
+    std::cout << indent
+              << "New peak memory usage: " << result.newPeakMemoryUsage << " Bytes" << std::endl;
+    std::cout << indent << "Relative change: "
+              << relativeChange(result.oldPeakMemoryUsage, result.newPeakMemoryUsage).constData()
+              << std::endl;
+}
+
+static void printResults(Activities activities, const BenchmarkResults &results)
+{
+    if (activities & ActivityResolving)
+        printResults(ActivityResolving, results);
+    if (activities & ActivityRuleExecution)
+        printResults(ActivityRuleExecution, results);
+    if (activities & ActivityNullBuild)
+        printResults(ActivityNullBuild, results);
+}
+
+int main(int argc, char *argv[])
+{
+    try {
+        QCoreApplication app(argc, argv);
+        CommandLineParser clParser;
+        clParser.parse();
+        Benchmarker benchmarker(clParser.activies(), clParser.oldCommit(), clParser.newCommit(),
+                                clParser.testProjectFilePath(), clParser.qbsRepoDirPath());
+        benchmarker.benchmark();
+        printResults(clParser.activies(), benchmarker.results());
+    } catch (const Exception &e) {
+        std::cerr << qPrintable(e.description()) << std::endl;
+        return EXIT_FAILURE;
+    }
+}
diff --git a/tests/benchmarker/benchmarker.cpp b/tests/benchmarker/benchmarker.cpp
new file mode 100644
index 000000000..3c1ed4c97
--- /dev/null
+++ b/tests/benchmarker/benchmarker.cpp
@@ -0,0 +1,113 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#include "benchmarker.h"
+
+#include "exception.h"
+#include "runsupport.h"
+#include "valgrindrunner.h"
+
+#include <QtConcurrent>
+
+#include <iostream>
+
+namespace qbsBenchmarker {
+
+Benchmarker::Benchmarker(Activities activities, const QString &oldCommit, const QString &newCommit,
+                         const QString &testProject, const QString &qbsRepo)
+    : m_activities(activities)
+    , m_oldCommit(oldCommit)
+    , m_newCommit(newCommit)
+    , m_testProject(testProject)
+    , m_qbsRepo(qbsRepo)
+{
+}
+
+Benchmarker::~Benchmarker()
+{
+    if (!m_commitToRestore.isEmpty()) {
+        try {
+            runProcess(QStringList() << "git" << "checkout" << m_commitToRestore, m_qbsRepo);
+        } catch (const Exception &e) {
+            qDebug("Failed to restore original commit %s: %s", qPrintable(m_commitToRestore),
+                   qPrintable(e.description()));
+        }
+    }
+}
+
+void Benchmarker::benchmark()
+{
+    rememberCurrentRepoState();
+    runProcess(QStringList() << "git" << "checkout" << m_oldCommit, m_qbsRepo);
+    const QString oldQbsBuildDir = m_baseOutputDir.path() + "/old-qbs-build";
+    std::cout << "Building from old repo state..." << std::endl;
+    buildQbs(oldQbsBuildDir);
+    runProcess(QStringList() << "git" << "checkout" << m_newCommit, m_qbsRepo);
+    const QString newQbsBuildDir = m_baseOutputDir.path() + "/new-qbs-build";
+    std::cout << "Building from new repo state..." << std::endl;
+    buildQbs(newQbsBuildDir);
+    std::cout << "Now running valgrind. This can take a while." << std::endl;
+
+    ValgrindRunner oldDataRetriever(m_activities, m_testProject, oldQbsBuildDir,
+                                    m_baseOutputDir.path() + "/old-stuff");
+    ValgrindRunner newDataRetriever(m_activities, m_testProject, newQbsBuildDir,
+                                    m_baseOutputDir.path() + "/new-stuff");
+    QFuture<void> oldFuture = QtConcurrent::run(&oldDataRetriever, &ValgrindRunner::run);
+    QFuture<void> newFuture = QtConcurrent::run(&newDataRetriever, &ValgrindRunner::run);
+    oldFuture.waitForFinished();
+    foreach (const ValgrindResult &valgrindResult, oldDataRetriever.results()) {
+        BenchmarkResult &benchmarkResult = m_results[valgrindResult.activity];
+        benchmarkResult.oldInstructionCount = valgrindResult.instructionCount;
+        benchmarkResult.oldPeakMemoryUsage = valgrindResult.peakMemoryUsage;
+    }
+    newFuture.waitForFinished();
+    foreach (const ValgrindResult &valgrindResult, newDataRetriever.results()) {
+        BenchmarkResult &benchmarkResult = m_results[valgrindResult.activity];
+        benchmarkResult.newInstructionCount = valgrindResult.instructionCount;
+        benchmarkResult.newPeakMemoryUsage = valgrindResult.peakMemoryUsage;
+    }
+    std::cout << "Done!" << std::endl;
+}
+
+void Benchmarker::rememberCurrentRepoState()
+{
+    QByteArray commit;
+    runProcess(QStringList() << "git" << "describe" << "HEAD", m_qbsRepo, &commit);
+    m_commitToRestore = QString::fromLatin1(commit);
+}
+
+void Benchmarker::buildQbs(const QString &buildDir) const
+{
+    if (!QDir::root().mkpath(buildDir))
+        throw Exception(QString::fromLatin1("Failed to create directory '%1'.").arg(buildDir));
+    runProcess(QStringList() << "qmake" << m_qbsRepo + "/qbs.pro", buildDir);
+    runProcess(QStringList() << "make" << "-s", buildDir);
+}
+
+} // namespace qbsBenchmarker
diff --git a/tests/benchmarker/benchmarker.h b/tests/benchmarker/benchmarker.h
new file mode 100644
index 000000000..1e30f5676
--- /dev/null
+++ b/tests/benchmarker/benchmarker.h
@@ -0,0 +1,82 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_BENCHMARKER_BENCHMARKER_H
+#define QBS_BENCHMARKER_BENCHMARKER_H
+
+#include "activities.h"
+
+#include <QHash>
+#include <QString>
+#include <QTemporaryDir>
+
+QT_BEGIN_NAMESPACE
+class QStringList;
+QT_END_NAMESPACE
+
+namespace qbsBenchmarker {
+
+class BenchmarkResult
+{
+public:
+    qint64 oldInstructionCount;
+    qint64 newInstructionCount;
+    qint64 oldPeakMemoryUsage;
+    qint64 newPeakMemoryUsage;
+};
+typedef QHash<Activity, BenchmarkResult> BenchmarkResults;
+
+class Benchmarker
+{
+public:
+    Benchmarker(Activities activities, const QString &oldCommit, const QString &newCommit,
+                const QString &testProject, const QString &qbsRepo);
+    ~Benchmarker();
+
+    void benchmark();
+
+    BenchmarkResults results() const { return m_results; }
+
+private:
+    void rememberCurrentRepoState();
+    void buildQbs(const QString &buildDir) const;
+
+    const Activities m_activities;
+    const QString m_oldCommit;
+    const QString m_newCommit;
+    const QString m_testProject;
+    const QString m_qbsRepo;
+    QString m_commitToRestore;
+    QTemporaryDir m_baseOutputDir;
+    BenchmarkResults m_results;
+};
+
+} // namespace qbsBenchmarker
+
+#endif // Include guard.
diff --git a/tests/benchmarker/benchmarker.pro b/tests/benchmarker/benchmarker.pro
new file mode 100644
index 000000000..4725e6684
--- /dev/null
+++ b/tests/benchmarker/benchmarker.pro
@@ -0,0 +1,20 @@
+TARGET = qbs_benchmarker
+DESTDIR = ../../bin
+CONFIG += console
+CONFIG -= app_bundle
+CONFIG += c++11
+QT += concurrent
+SOURCES = \
+    benchmarker-main.cpp \
+    benchmarker.cpp \
+    commandlineparser.cpp \
+    runsupport.cpp \
+    valgrindrunner.cpp
+
+HEADERS = \
+    activities.h \
+    benchmarker.h \
+    commandlineparser.h \
+    exception.h \
+    runsupport.h \
+    valgrindrunner.h
diff --git a/tests/benchmarker/benchmarker.qbs b/tests/benchmarker/benchmarker.qbs
new file mode 100644
index 000000000..8dec56d60
--- /dev/null
+++ b/tests/benchmarker/benchmarker.qbs
@@ -0,0 +1,23 @@
+import qbs
+
+QtApplication {
+    name: "qbs_benchmarker"
+    destinationDirectory: "bin"
+    type: "application"
+    consoleApplication: true
+    cpp.cxxLanguageVersion: "c++11"
+    Depends { name: "Qt.concurrent" }
+    files: [
+        "activities.h",
+        "benchmarker-main.cpp",
+        "benchmarker.cpp",
+        "benchmarker.h",
+        "commandlineparser.cpp",
+        "commandlineparser.h",
+        "exception.h",
+        "runsupport.cpp",
+        "runsupport.h",
+        "valgrindrunner.cpp",
+        "valgrindrunner.h",
+    ]
+}
diff --git a/tests/benchmarker/commandlineparser.cpp b/tests/benchmarker/commandlineparser.cpp
new file mode 100644
index 000000000..3b8452f8d
--- /dev/null
+++ b/tests/benchmarker/commandlineparser.cpp
@@ -0,0 +1,117 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#include "commandlineparser.h"
+
+#include "exception.h"
+
+#include <QCommandLineOption>
+#include <QCommandLineParser>
+#include <QCoreApplication>
+#include <QFileInfo>
+
+namespace qbsBenchmarker {
+
+static QString resolveActivity() { return "resolving"; }
+static QString ruleExecutionActivity() { return "rule-execution"; }
+static QString nullBuildActivity() { return "null-build"; }
+static QString allActivities() { return "all"; }
+
+CommandLineParser::CommandLineParser()
+{
+}
+
+void CommandLineParser::parse()
+{
+    QCommandLineParser parser;
+    parser.setApplicationDescription("This tool aims to detect qbs performance regressions "
+                                     "using valgrind.");
+    parser.addHelpOption();
+    QCommandLineOption oldCommitOption("old-commit", "The old qbs commit.", "old commit");
+    parser.addOption(oldCommitOption);
+    QCommandLineOption newCommitOption("new-commit", "The new qbs commit.", "new commit");
+    parser.addOption(newCommitOption);
+    QCommandLineOption testProjectOption("test-project",
+            "The example project to use for the benchmark.", "project file path");
+    parser.addOption(testProjectOption);
+    QCommandLineOption qbsRepoOption("qbs-repo", "The qbs repository.", "repo path");
+    parser.addOption(qbsRepoOption);
+    QCommandLineOption activitiesOption("activities",
+            QString::fromLatin1("The activities to benchmark. Possible values (CSV): %1,%2,%3,%4")
+                    .arg(resolveActivity(), ruleExecutionActivity(), nullBuildActivity(),
+                         allActivities()), "activities", allActivities());
+    parser.addOption(activitiesOption);
+    parser.process(*QCoreApplication::instance());
+    QList<QCommandLineOption> mandatoryOptions = QList<QCommandLineOption>()
+            << oldCommitOption << newCommitOption << testProjectOption << qbsRepoOption;
+    foreach (const QCommandLineOption &o, mandatoryOptions) {
+        if (!parser.isSet(o))
+            throwException(o.names().first(), parser.helpText());
+        if (parser.value(o).isEmpty())
+            throwException(o.names().first(), QString(), parser.helpText());
+    }
+    m_oldCommit = parser.value(oldCommitOption);
+    m_newCommit = parser.value(newCommitOption);
+    m_testProjectFilePath = parser.value(testProjectOption);
+    m_qbsRepoDirPath = parser.value(qbsRepoOption);
+    const QStringList activitiesList = parser.value(activitiesOption).split(',');
+    m_activities = 0;
+    foreach (const QString &activityString, activitiesList) {
+        if (activityString == allActivities()) {
+            m_activities = ActivityResolving | ActivityRuleExecution | ActivityNullBuild;
+            break;
+        } else if (activityString == resolveActivity()) {
+            m_activities = ActivityResolving;
+        } else if (activityString == ruleExecutionActivity()) {
+            m_activities |= ActivityRuleExecution;
+        } else if (activityString == nullBuildActivity()) {
+            m_activities |= ActivityNullBuild;
+        } else {
+            throwException(activitiesOption.names().first(), activityString, parser.helpText());
+        }
+    }
+}
+
+void CommandLineParser::throwException(const QString &optionName, const QString &illegalValue,
+                                       const QString &helpText)
+{
+    const QString errorText(QString::fromLatin1("Error parsing command line: Illegal value '%1' "
+            "for option '--%2'.\n%3").arg(illegalValue, optionName, helpText));
+    throw Exception(errorText);
+}
+
+void CommandLineParser::throwException(const QString &missingOption, const QString &helpText)
+{
+    const QString errorText(QString::fromLatin1("Error parsing command line: Missing mandatory "
+            "option '--%1'.\n%3").arg(missingOption, helpText));
+    throw Exception(errorText);
+}
+
+
+} // namespace qbsBenchmarker
diff --git a/tests/benchmarker/commandlineparser.h b/tests/benchmarker/commandlineparser.h
new file mode 100644
index 000000000..8d46b02cc
--- /dev/null
+++ b/tests/benchmarker/commandlineparser.h
@@ -0,0 +1,66 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_BENCHMARKER_COMMANDLINEPARSER_H
+#define QBS_BENCHMARKER_COMMANDLINEPARSER_H
+
+#include "activities.h"
+
+#include <QStringList>
+
+namespace qbsBenchmarker {
+
+class CommandLineParser
+{
+public:
+    CommandLineParser();
+
+    void parse();
+
+    Activities activies() const { return m_activities; }
+    QString oldCommit() const { return m_oldCommit; }
+    QString newCommit() const { return m_newCommit; }
+    QString testProjectFilePath() const { return m_testProjectFilePath; }
+    QString qbsRepoDirPath() const { return m_qbsRepoDirPath; }
+
+private:
+    Q_NORETURN void throwException(const QString &optionName, const QString &illegalValue,
+                                   const QString &helpText);
+    Q_NORETURN void throwException(const QString &missingOption, const QString &helpText);
+
+    Activities m_activities;
+    QString m_oldCommit;
+    QString m_newCommit;
+    QString m_testProjectFilePath;
+    QString m_qbsRepoDirPath;
+};
+
+} // namespace qbsBenchmarker
+
+#endif // Include guard.
diff --git a/tests/benchmarker/exception.h b/tests/benchmarker/exception.h
new file mode 100644
index 000000000..623a04257
--- /dev/null
+++ b/tests/benchmarker/exception.h
@@ -0,0 +1,55 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_BENCHMARKER_EXCEPTION_H
+#define QBS_BENCHMARKER_EXCEPTION_H
+
+#include <QException>
+#include <QString>
+
+namespace qbsBenchmarker {
+
+class Exception : public QException {
+public:
+    explicit Exception(const QString &description) : m_description(description) {}
+    ~Exception() throw() { }
+
+    QString description() const { return m_description; }
+
+private:
+    void raise() const { throw *this; }
+    Exception *clone() const { return new Exception(*this); }
+
+    QString m_description;
+};
+
+} // namespace qbsBenchmarker
+
+#endif // Include guard.
+
diff --git a/tests/benchmarker/runsupport.cpp b/tests/benchmarker/runsupport.cpp
new file mode 100644
index 000000000..0d6f602a1
--- /dev/null
+++ b/tests/benchmarker/runsupport.cpp
@@ -0,0 +1,70 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#include "runsupport.h"
+
+#include "exception.h"
+
+#include <QByteArray>
+#include <QProcess>
+#include <QString>
+#include <QStringList>
+
+namespace qbsBenchmarker {
+
+void runProcess(const QStringList &commandLine, const QString &workingDir, QByteArray *output)
+{
+    QStringList args = commandLine;
+    const QString command = args.takeFirst();
+    QProcess p;
+    if (!workingDir.isEmpty())
+        p.setWorkingDirectory(workingDir);
+    p.start(command, args);
+    if (!p.waitForStarted())
+        throw Exception(QString::fromLatin1("Process '%1' failed to start.").arg(command));
+    p.waitForFinished(-1);
+    if (p.exitStatus() != QProcess::NormalExit) {
+        throw Exception(QString::fromLatin1("Error running '%1': %2")
+                        .arg(command, p.errorString()));
+    }
+    if (p.exitCode() != 0) {
+        QString errorString = QString::fromLatin1("Command '%1' finished with exit code %2.")
+                .arg(command).arg(p.exitCode());
+        const QByteArray stdErr = p.readAllStandardError();
+        if (!stdErr.isEmpty()) {
+            errorString += QString::fromLatin1("\nStandard error output was: '%1'")
+                    .arg(QString::fromLocal8Bit(stdErr));
+        }
+        throw Exception(errorString);
+    }
+    if (output)
+        *output = p.readAllStandardOutput().trimmed();
+}
+
+} // namespace qbsBenchmarker
diff --git a/tests/benchmarker/runsupport.h b/tests/benchmarker/runsupport.h
new file mode 100644
index 000000000..fa21304e5
--- /dev/null
+++ b/tests/benchmarker/runsupport.h
@@ -0,0 +1,49 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_BENCHMARKER_RUNSUPPORT_H
+#define QBS_BENCHMARKER_RUNSUPPORT_H
+
+#include <QtGlobal>
+#include <QString>
+
+QT_BEGIN_NAMESPACE
+class QByteArray;
+class QStringList;
+QT_END_NAMESPACE
+
+namespace qbsBenchmarker {
+
+void runProcess(const QStringList &commandLine, const QString& workingDir = QString(),
+                QByteArray *output = 0);
+
+} // namespace qbsBenchmarker
+
+#endif // Include guard.
+
diff --git a/tests/benchmarker/valgrindrunner.cpp b/tests/benchmarker/valgrindrunner.cpp
new file mode 100644
index 000000000..6d97c93e9
--- /dev/null
+++ b/tests/benchmarker/valgrindrunner.cpp
@@ -0,0 +1,243 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#include "valgrindrunner.h"
+
+#include "exception.h"
+#include "runsupport.h"
+
+#include <QBuffer>
+#include <QDir>
+#include <QFile>
+#include <QMutexLocker>
+#include <QStringList>
+#include <QtConcurrent>
+
+namespace qbsBenchmarker {
+
+ValgrindRunner::ValgrindRunner(Activities activities, const QString &testProject,
+                               const QString &qbsBuildDir, const QString &baseOutputDir)
+    : m_activities(activities)
+    , m_testProject(testProject)
+    , m_qbsBinary(qbsBuildDir + "/bin/qbs")
+    , m_baseOutputDir(baseOutputDir)
+{
+    if (!QDir::root().mkpath(m_baseOutputDir))
+        throw Exception(QString::fromLatin1("Failed to create directory '%1'.").arg(baseOutputDir));
+}
+
+void ValgrindRunner::run()
+{
+    QList<QFuture<void>> futures;
+    if (m_activities & ActivityResolving)
+        futures << QtConcurrent::run(this, &ValgrindRunner::traceResolving);
+    if (m_activities & ActivityRuleExecution)
+        futures << QtConcurrent::run(this, &ValgrindRunner::traceRuleExecution);
+    if (m_activities & ActivityNullBuild)
+        futures << QtConcurrent::run(this, &ValgrindRunner::traceNullBuild);
+    while (!futures.isEmpty())
+        futures.takeFirst().waitForFinished();
+}
+
+void ValgrindRunner::traceResolving()
+{
+    const QString buildDirCallgrind = m_baseOutputDir + "/build-dir.resolving.callgrind";
+    const QString buildDirMassif = m_baseOutputDir + "/build-dir.resolving.massif";
+    traceActivity(ActivityResolving, buildDirCallgrind, buildDirMassif);
+}
+
+void ValgrindRunner::traceRuleExecution()
+{
+    const QString buildDirCallgrind = m_baseOutputDir + "/build-dir.rule-execution.callgrind";
+    const QString buildDirMassif = m_baseOutputDir + "/build-dir.rule-execution.massif";
+    runProcess(qbsCommandLine("resolve", buildDirCallgrind, false));
+    runProcess(qbsCommandLine("resolve", buildDirMassif, false));
+    traceActivity(ActivityRuleExecution, buildDirCallgrind, buildDirMassif);
+}
+
+void ValgrindRunner::traceNullBuild()
+{
+    const QString buildDirCallgrind = m_baseOutputDir + "/build-dir.null-build.callgrind";
+    const QString buildDirMassif = m_baseOutputDir + "/build-dir.null-build.massif";
+    runProcess(qbsCommandLine("build", buildDirCallgrind, false));
+    runProcess(qbsCommandLine("build", buildDirMassif, false));
+    traceActivity(ActivityNullBuild, buildDirCallgrind, buildDirMassif);
+}
+
+void ValgrindRunner::traceActivity(Activity activity, const QString &buildDirCallgrind,
+                                   const QString &buildDirMassif)
+{
+    QString activityString;
+    QString qbsCommand;
+    bool dryRun;
+    switch (activity) {
+    case ActivityResolving:
+        activityString = "resolving";
+        qbsCommand = "resolve";
+        dryRun = false;
+        break;
+    case ActivityRuleExecution:
+        activityString = "rule-execution";
+        qbsCommand = "build";
+        dryRun = true;
+        break;
+    case ActivityNullBuild:
+        activityString = "null-build";
+        qbsCommand = "build";
+        dryRun = false;
+        break;
+    }
+
+    const QString outFileCallgrind = m_baseOutputDir + "/outfile." + activityString + ".callgrind";
+    const QString outFileMassif = m_baseOutputDir + "/outfile." + activityString + ".massif";
+    QFuture<qint64> callGrindFuture = QtConcurrent::run(this, &ValgrindRunner::runCallgrind,
+            qbsCommand, buildDirCallgrind, dryRun, outFileCallgrind);
+    QFuture<qint64> massifFuture = QtConcurrent::run(this, &ValgrindRunner::runMassif, qbsCommand,
+            buildDirMassif, dryRun, outFileMassif);
+    callGrindFuture.waitForFinished();
+    massifFuture.waitForFinished();
+    addToResults(ValgrindResult(activity, callGrindFuture.result(), massifFuture.result()));
+}
+
+QStringList ValgrindRunner::qbsCommandLine(const QString &command, const QString &buildDir,
+                                           bool dryRun) const
+{
+    QStringList commandLine = QStringList() << m_qbsBinary << command << "-qq" << "-d" << buildDir
+                                            << "-f" << m_testProject;
+    if (dryRun)
+        commandLine << "--dry-run";
+    return commandLine;
+}
+
+QStringList ValgrindRunner::wrapForValgrind(const QStringList &commandLine, const QString &tool,
+                                            const QString &outFile) const
+{
+    return QStringList() << "valgrind" << "--smc-check=all" << ("--tool=" + tool)
+                         << ("--" + tool + "-out-file=" + outFile) << commandLine;
+}
+
+QStringList ValgrindRunner::valgrindCommandLine(const QString &qbsCommand, const QString &buildDir,
+        bool dryRun, const QString &tool, const QString &outFile) const
+{
+    return wrapForValgrind(qbsCommandLine(qbsCommand, buildDir, dryRun), tool, outFile);
+}
+
+void ValgrindRunner::addToResults(const ValgrindResult &result)
+{
+    QMutexLocker locker(&m_resultsMutex);
+    m_results << result;
+}
+
+qint64 ValgrindRunner::runCallgrind(const QString &qbsCommand, const QString &buildDir,
+                                     bool dryRun, const QString &outFile)
+{
+    runProcess(valgrindCommandLine(qbsCommand, buildDir, dryRun, "callgrind", outFile));
+    QFile f(outFile);
+    if (!f.open(QIODevice::ReadOnly)) {
+        throw Exception(QString::fromLatin1("Failed to open file '%1': %2")
+                        .arg(outFile, f.errorString()));
+    }
+    while (!f.atEnd()) {
+        const QByteArray line = f.readLine().trimmed();
+        static const QByteArray magicString = "summary: ";
+        if (!line.startsWith(magicString))
+            continue;
+        const QByteArray icString = line.mid(magicString.count());
+        bool ok;
+        const qint64 iCount = icString.toLongLong(&ok);
+        if (!ok) {
+            throw Exception(QString::fromLatin1("Unexpected line in callgrind output file "
+                                                "'%1': '%2'.")
+                            .arg(outFile, QString::fromLocal8Bit(line)));
+        }
+        return iCount;
+    }
+
+    throw Exception(QString::fromLatin1("Failed to find summary line in callgrind "
+                                        "output file '%1'.").arg(outFile));
+}
+
+qint64 ValgrindRunner::runMassif(const QString &qbsCommand, const QString &buildDir, bool dryRun,
+                                  const QString &outFile)
+{
+    runProcess(valgrindCommandLine(qbsCommand, buildDir, dryRun, "massif", outFile));
+    QByteArray ms_printOutput;
+    runProcess(QStringList() << "ms_print" << outFile, QString(), &ms_printOutput);
+    QBuffer buffer(&ms_printOutput);
+    buffer.open(QIODevice::ReadOnly);
+    QByteArray peakSnapshot;
+    const QString exceptionStringPattern = QString::fromLatin1("Failed to extract peak memory "
+            "usage from file '%1': %2").arg(outFile);
+    while (!buffer.atEnd()) {
+        const QByteArray line = buffer.readLine();
+        static const QByteArray magicString = " (peak)";
+        const int magicStringOffset = line.indexOf(magicString);
+        if (magicStringOffset == -1)
+            continue;
+        int delimiterOffset = line.lastIndexOf(',', magicStringOffset);
+        if (delimiterOffset == -1)
+            delimiterOffset = line.lastIndexOf('[', magicStringOffset);
+        if (delimiterOffset == -1) {
+            const QString details = QString::fromLatin1("Failed to extract peak snapshot from "
+                    "line '%1'.").arg(QString::fromLocal8Bit(line));
+            throw Exception(exceptionStringPattern.arg(details));
+        }
+        peakSnapshot = line.mid(delimiterOffset + 1, magicStringOffset - delimiterOffset).trimmed();
+        break;
+    }
+    if (peakSnapshot.isEmpty())
+        throw Exception(exceptionStringPattern.arg("No peak marker found"));
+    while (!buffer.atEnd()) {
+        const QByteArray line = buffer.readLine().simplified();
+        if (!line.startsWith(peakSnapshot + ' '))
+            continue;
+        const QList<QByteArray> entries = line.split(' ');
+        if (entries.count() != 6) {
+            const QString details = QString::fromLatin1("Expected 6 entries in line '%1', but "
+                    "there are %2.").arg(QString::fromLocal8Bit(line)).arg(entries.count());
+            throw Exception(exceptionStringPattern.arg(details));
+        }
+        QByteArray peakMemoryString = entries.at(4);
+        peakMemoryString.replace(',', QByteArray());
+        bool ok;
+        qint64 peakMemoryUsage = peakMemoryString.toLongLong(&ok);
+        if (!ok) {
+            const QString details = QString::fromLatin1("Failed to parse peak memory value '%1' "
+                    "as a number.").arg(QString::fromLocal8Bit(peakMemoryString));
+            throw Exception(exceptionStringPattern.arg(details));
+        }
+        return peakMemoryUsage;
+    }
+
+    const QString details = QString::fromLatin1("Failed to find snapshot '%1'.")
+            .arg(QString::fromLocal8Bit(peakSnapshot));
+    throw Exception(exceptionStringPattern.arg(details));
+}
+
+} // namespace qbsBenchmarker
diff --git a/tests/benchmarker/valgrindrunner.h b/tests/benchmarker/valgrindrunner.h
new file mode 100644
index 000000000..e8f3ce6be
--- /dev/null
+++ b/tests/benchmarker/valgrindrunner.h
@@ -0,0 +1,92 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of the Qt Build Suite.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company.  For licensing terms and
+** conditions see http://www.qt.io/terms-conditions.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+#ifndef QBS_BENCHMARKER_BENCHMARKRUNNER_H
+#define QBS_BENCHMARKER_BENCHMARKRUNNER_H
+
+#include "activities.h"
+
+#include <QList>
+#include <QMutex>
+#include <QString>
+
+QT_BEGIN_NAMESPACE
+class QStringList;
+QT_END_NAMESPACE
+
+namespace qbsBenchmarker {
+
+class ValgrindResult
+{
+public:
+    ValgrindResult(Activity a, qint64 ic, qint64 mem)
+        : activity(a), instructionCount(ic), peakMemoryUsage(mem) {}
+
+    Activity activity;
+    qint64 instructionCount;
+    qint64 peakMemoryUsage;
+};
+
+class ValgrindRunner
+{
+public:
+    ValgrindRunner(Activities activities, const QString &testProject, const QString &qbsBuildDir,
+                    const QString &baseOutputDir);
+
+    void run();
+    QList<ValgrindResult> results() const { return m_results; }
+
+private:
+    void traceResolving();
+    void traceRuleExecution();
+    void traceNullBuild();
+    void traceActivity(Activity activity, const QString &buildDirCallgrind,
+                       const QString &buildDirMassif);
+    QStringList qbsCommandLine(const QString &command, const QString &buildDir, bool dryRun) const;
+    QStringList wrapForValgrind(const QStringList &commandLine, const QString &tool,
+                                const QString &outFile) const;
+    QStringList valgrindCommandLine(const QString &qbsCommand, const QString &buildDir, bool dryRun,
+            const QString &tool, const QString &outFile) const;
+    void addToResults(const ValgrindResult &results);
+    qint64 runCallgrind(const QString &qbsCommand, const QString &buildDir, bool dryRun,
+                         const QString &outFile);
+    qint64 runMassif(const QString &qbsCommand, const QString &buildDir, bool dryRun,
+                      const QString &outFile);
+
+    const Activities m_activities;
+    const QString m_testProject;
+    const QString m_qbsBinary;
+    const QString m_baseOutputDir;
+    QList<ValgrindResult> m_results;
+    QMutex m_resultsMutex;
+};
+
+} // namespace qbsBenchmarker
+
+#endif // Include guard.