[test-runner] Move probe related command line args to manifest

- Move probe related command line arguments to manifest, example:

  "probes": ["probeGFX", "probeNetwork", "probeMemory"],
  "metric_path": "metrics/linux"

- Use manifest file name as a postfix for a result page
- Move injected probes 'begin' section before map object creation
- Generalize artifact storing shell script

1 files changed, 8 insertions, 22 deletions

diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 657e507d1a..4d18902083 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -40,14 +40,10 @@ void operator delete(void* ptr, size_t) noexcept {
 #endif
 
 namespace {
-using ArgumentsTuple =
-    std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string, std::set<std::string>>;
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
 ArgumentsTuple parseArguments(int argc, char** argv) {
     args::ArgumentParser argumentParser("Mapbox GL Test Runner");
 
-    static const std::unordered_map<std::string, std::string> probeMap{
-        {"memory", "probeMemory"}, {"network", "probeNetwork"}, {"gfx", "probeGFX"}};
-
     args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
 
     args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
@@ -57,12 +53,6 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
     args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
     args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
-    args::MapFlagList<std::string, std::string> probes(
-        argumentParser,
-        "probe",
-        "Probe to be injected into a test. Supported values are: [memory, gfx, network]",
-        {"probe"},
-        probeMap);
 
     try {
         argumentParser.ParseCLI(argc, argv);
@@ -96,9 +86,6 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         exit(4);
     }
 
-    const auto& probeValues = args::get(probes);
-    std::set<std::string> injectedProbes(probeValues.begin(), probeValues.end());
-
     auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
     auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
     const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
@@ -108,8 +95,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
                           seed,
                           manifestPath.string(),
                           std::move(testNames),
-                          std::move(testFilter),
-                          std::move(injectedProbes)};
+                          std::move(testFilter)};
 }
 } // namespace
 namespace mbgl {
@@ -121,10 +107,8 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
     std::string manifestPath;
     std::vector<std::string> testNames;
     std::string testFilter;
-    std::set<std::string> injectedProbes;
 
-    std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter, injectedProbes) =
-        parseArguments(argc, argv);
+    std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
     auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
     if (!manifestData) {
         exit(5);
@@ -175,7 +159,7 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
 
         bool errored = !metadata.errorMessage.empty();
         if (!errored) {
-            errored = !runner.run(metadata, injectedProbes) || !metadata.errorMessage.empty();
+            errored = !runner.run(metadata) || !metadata.errorMessage.empty();
         }
 
         bool passed =
@@ -218,8 +202,10 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
             testStatus();
         }
     }
-    const auto resultPath =
-        manifest.getResultPath() + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
+
+    const auto manifestName = std::string("_").append(mbgl::filesystem::path(manifestPath).stem());
+    const auto resultPath = manifest.getResultPath() + "/" + (testNames.empty() ? "render-tests" : testNames.front()) +
+                            manifestName + "_index.html";
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
     mbgl::util::write_file(resultPath, resultsHTML);