[test-runner] Move probe related command line args to manifest

- Move probe related command line arguments to manifest, example:

  "probes": ["probeGFX", "probeNetwork", "probeMemory"],
  "metric_path": "metrics/linux"

- Use manifest file name as a postfix for a result page
- Move injected probes 'begin' section before map object creation
- Generalize artifact storing shell script

6 files changed, 49 insertions, 41 deletions

diff --git a/circle.yml b/circle.yml
index 8ad976c37d..65096d083a 100644
--- a/circle.yml
+++ b/circle.yml
@@ -267,9 +267,7 @@ commands:
             mkdir -p /tmp/tests/clang-tidy
             if [ -f clang-tidy.log ]; then cp clang-tidy.log /tmp/tests/clang-tidy; fi
             mkdir -p /tmp/tests/render
-            if [ -f render-test/tests_index.html ]; then cp render-test/tests_index.html /tmp/tests/render; fi
-            if [ -f render-test/render-tests_index.html ]; then cp render-test/render-tests_index.html /tmp/tests/render; fi
-            if [ -f render-test/query-tests_index.html ]; then cp render-test/query-tests_index.html /tmp/tests/render; fi
+            if ls render-test/*.html 1> /dev/null 2>&1; then cp render-test/*.html /tmp/tests/render; fi
             mkdir -p /tmp/tests/coredumps
             if ls core* 1> /dev/null 2>&1; then cp core* /tmp/tests/coredumps; fi
             mkdir -p /tmp/tests/valgrind
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index 01d2246685..bcc10c91d4 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -44,6 +44,10 @@ const std::string& Manifest::getResultPath() const {
     return resultPath;
 }
 
+const std::set<std::string>& Manifest::getProbes() const {
+    return probes;
+}
+
 void Manifest::doShuffle(uint32_t seed) {
     std::seed_seq sequence{seed};
     std::mt19937 shuffler(sequence);
@@ -389,6 +393,25 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
         manifest.ignores = parseIgnores(ignorePaths);
     }
 
+    if (document.HasMember("probes")) {
+        const auto& probesValue = document["probes"];
+        if (!probesValue.IsArray()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Provided probes inside the manifest file: %s is not a valid array",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        for (const auto& value : probesValue.GetArray()) {
+            if (!value.IsString()) {
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid probe type is provoided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
+            }
+            manifest.probes.emplace(value.GetString());
+        }
+    }
+
     manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string();
     if (manifest.testRootPath.back() == '/') {
         manifest.testRootPath.pop_back();
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
index c4672fb4c5..8e2c45efde 100644
--- a/render-test/manifest_parser.hpp
+++ b/render-test/manifest_parser.hpp
@@ -20,6 +20,7 @@ public:
     const std::string& getAssetPath() const;
     const std::string& getManifestPath() const;
     const std::string& getResultPath() const;
+    const std::set<std::string>& getProbes() const;
     void doShuffle(uint32_t seed);
 
     std::string localizeURL(const std::string& url) const;
@@ -48,6 +49,7 @@ private:
     std::string resultPath;
     std::vector<std::pair<std::string, std::string>> ignores;
     std::vector<TestPaths> testPaths;
+    std::set<std::string> probes;
 };
 
 class ManifestParser {
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 657e507d1a..4d18902083 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -40,14 +40,10 @@ void operator delete(void* ptr, size_t) noexcept {
 #endif
 
 namespace {
-using ArgumentsTuple =
-    std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string, std::set<std::string>>;
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
 ArgumentsTuple parseArguments(int argc, char** argv) {
     args::ArgumentParser argumentParser("Mapbox GL Test Runner");
 
-    static const std::unordered_map<std::string, std::string> probeMap{
-        {"memory", "probeMemory"}, {"network", "probeNetwork"}, {"gfx", "probeGFX"}};
-
     args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
 
     args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
@@ -57,12 +53,6 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
     args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
     args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
-    args::MapFlagList<std::string, std::string> probes(
-        argumentParser,
-        "probe",
-        "Probe to be injected into a test. Supported values are: [memory, gfx, network]",
-        {"probe"},
-        probeMap);
 
     try {
         argumentParser.ParseCLI(argc, argv);
@@ -96,9 +86,6 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         exit(4);
     }
 
-    const auto& probeValues = args::get(probes);
-    std::set<std::string> injectedProbes(probeValues.begin(), probeValues.end());
-
     auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
     auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
     const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
@@ -108,8 +95,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
                           seed,
                           manifestPath.string(),
                           std::move(testNames),
-                          std::move(testFilter),
-                          std::move(injectedProbes)};
+                          std::move(testFilter)};
 }
 } // namespace
 namespace mbgl {
@@ -121,10 +107,8 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
     std::string manifestPath;
     std::vector<std::string> testNames;
     std::string testFilter;
-    std::set<std::string> injectedProbes;
 
-    std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter, injectedProbes) =
-        parseArguments(argc, argv);
+    std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
     auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
     if (!manifestData) {
         exit(5);
@@ -175,7 +159,7 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
 
         bool errored = !metadata.errorMessage.empty();
         if (!errored) {
-            errored = !runner.run(metadata, injectedProbes) || !metadata.errorMessage.empty();
+            errored = !runner.run(metadata) || !metadata.errorMessage.empty();
         }
 
         bool passed =
@@ -218,8 +202,10 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
             testStatus();
         }
     }
-    const auto resultPath =
-        manifest.getResultPath() + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
+
+    const auto manifestName = std::string("_").append(mbgl::filesystem::path(manifestPath).stem());
+    const auto resultPath = manifest.getResultPath() + "/" + (testNames.empty() ? "render-tests" : testNames.front()) +
+                            manifestName + "_index.html";
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
     mbgl::util::write_file(resultPath, resultsHTML);
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 5a28576d85..c58095dd14 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -1162,10 +1162,17 @@ TestRunner::Impl::Impl(const TestMetadata& metadata)
 
 TestRunner::Impl::~Impl() {}
 
-bool TestRunner::run(TestMetadata& metadata, const std::set<std::string>& injectedProbes) {
+bool TestRunner::run(TestMetadata& metadata) {
     AllocationIndex::setActive(false);
     AllocationIndex::reset();
     ProxyFileSource::setTrackingActive(false);
+
+    RunContext ctx{};
+    // Run 'begin' probes provided via command line arguments.
+    if (!runInjectedProbesBegin(metadata, ctx)) {
+        return false;
+    }
+
     std::string key = mbgl::util::toString(uint32_t(metadata.mapMode))
         + "/" + mbgl::util::toString(metadata.pixelRatio)
         + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
@@ -1177,13 +1184,6 @@ bool TestRunner::run(TestMetadata& metadata, const std::set<std::string>& inject
     auto& frontend = maps[key]->frontend;
     auto& map = maps[key]->map;
 
-    RunContext ctx{};
-
-    // Run 'begin' probes provided via command line arguments.
-    if (!runInjectedProbesBegin(metadata, injectedProbes, ctx)) {
-        return false;
-    }
-
     frontend.setSize(metadata.size);
     map.setSize(metadata.size);
 
@@ -1203,7 +1203,7 @@ bool TestRunner::run(TestMetadata& metadata, const std::set<std::string>& inject
     }
 
     // Run 'end' probes provided via command line arguments
-    if (!runInjectedProbesEnd(metadata, injectedProbes, ctx, result.stats)) {
+    if (!runInjectedProbesEnd(metadata, ctx, result.stats)) {
         return false;
     }
 
@@ -1238,7 +1238,7 @@ bool runInjectedProbe(TestMetadata& metadata,
     return true;
 }
 
-bool TestRunner::runInjectedProbesBegin(TestMetadata& metadata, const std::set<std::string>& probes, RunContext& ctx) {
+bool TestRunner::runInjectedProbesBegin(TestMetadata& metadata, RunContext& ctx) {
     const std::string mark = " - default - start";
     static const InjectedProbeMap beginInjectedProbeMap = {
         {// Injected memory probe begin
@@ -1269,11 +1269,10 @@ bool TestRunner::runInjectedProbesBegin(TestMetadata& metadata, const std::set<s
                  std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
          }}};
 
-    return runInjectedProbe(metadata, probes, ctx, beginInjectedProbeMap);
+    return runInjectedProbe(metadata, manifest.getProbes(), ctx, beginInjectedProbeMap);
 }
 
 bool TestRunner::runInjectedProbesEnd(TestMetadata& metadata,
-                                      const std::set<std::string>& probes,
                                       RunContext& ctx,
                                       mbgl::gfx::RenderingStats stats) {
     const std::string mark = " - default - end";
@@ -1319,7 +1318,7 @@ bool TestRunner::runInjectedProbesEnd(TestMetadata& metadata,
              ProxyFileSource::setTrackingActive(false);
          }}};
 
-    return runInjectedProbe(metadata, probes, ctx, endInjectedProbeMap);
+    return runInjectedProbe(metadata, manifest.getProbes(), ctx, endInjectedProbeMap);
 }
 
 void TestRunner::reset() {
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 5dd529f057..2503372602 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -15,7 +15,7 @@ struct TestMetadata;
 class TestRunner {
 public:
     explicit TestRunner(Manifest);
-    bool run(TestMetadata&, const std::set<std::string>&);
+    bool run(TestMetadata&);
     void reset();
 
     // Manifest
@@ -24,8 +24,8 @@ public:
 
 private:
     bool runOperations(const std::string& key, TestMetadata&, RunContext&);
-    bool runInjectedProbesBegin(TestMetadata&, const std::set<std::string>&, RunContext&);
-    bool runInjectedProbesEnd(TestMetadata&, const std::set<std::string>&, RunContext&, mbgl::gfx::RenderingStats);
+    bool runInjectedProbesBegin(TestMetadata&, RunContext&);
+    bool runInjectedProbesEnd(TestMetadata&, RunContext&, mbgl::gfx::RenderingStats);
 
     bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
                                std::vector<mbgl::Feature>&& features,