[test-runner] Allow to define filter inside manifest file

Command line option is still available and allows to override filter
that is defined in a manifest file.

7 files changed, 22 insertions, 16 deletions

diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake
index 39777c93f6..fdbe8a9171 100644
--- a/next/platform/linux/linux.cmake
+++ b/next/platform/linux/linux.cmake
@@ -147,11 +147,7 @@ else()
     )
     add_test(
         NAME mbgl-render-test-probe-memory
-        COMMAND
-            mbgl-render-test-runner
-            render-tests
-            --manifestPath=${MBGL_ROOT}/render-test/linux-manifest-probe-memory.json
-            --filter=real-world
+        COMMAND mbgl-render-test-runner render-tests --manifestPath=${MBGL_ROOT}/render-test/linux-manifest-probe-memory.json
     )
 endif()
 
diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp
index 0d2400ac74..d0af8d21a6 100644
--- a/platform/android/src/test/render_test_runner.cpp
+++ b/platform/android/src/test/render_test_runner.cpp
@@ -252,10 +252,10 @@ void android_main(struct android_app* app) {
     } else {
         unZipFile(env, zipFile, storagePath);
 
-        auto runTestWithManifest = [&storagePath, &app, &outFd, &outEvents, &source](
-                                       const std::string manifest, const std::string filter = {}) -> bool {
+        auto runTestWithManifest =
+            [&storagePath, &app, &outFd, &outEvents, &source](const std::string& manifest) -> bool {
             const std::string configFile = storagePath + manifest;
-            std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", configFile, filter};
+            std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", configFile};
             std::vector<char*> argv;
             for (const auto& arg : arguments) {
                 argv.push_back(const_cast<char*>(arg.data()));
@@ -279,7 +279,7 @@ void android_main(struct android_app* app) {
             return result;
         };
         runTestWithManifest("/android-manifest-probe-network-gfx.json");
-        runTestWithManifest("/android-manifest-probe-memory.json", "--filter=real-world");
+        runTestWithManifest("/android-manifest-probe-memory.json");
         mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
         changeState(env, app);
     }
diff --git a/render-test/android-manifest-probe-memory.json b/render-test/android-manifest-probe-memory.json
index bc94465761..a3562088f4 100644
--- a/render-test/android-manifest-probe-memory.json
+++ b/render-test/android-manifest-probe-memory.json
@@ -6,5 +6,6 @@
     "asset_path": "mapbox-gl-js/test/integration",
     "result_path":"/sdcard/",
     "probes": ["probeMemory"],
-    "metric_path":"render-test/metrics/android"
+    "metric_path":"render-test/metrics/android",
+    "filter": "real-world"
 }  
\ No newline at end of file
diff --git a/render-test/linux-manifest-probe-memory.json b/render-test/linux-manifest-probe-memory.json
index 1f9fa1ca5c..25b59e4a44 100644
--- a/render-test/linux-manifest-probe-memory.json
+++ b/render-test/linux-manifest-probe-memory.json
@@ -5,5 +5,6 @@
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration",
     "probes": ["probeMemory"],
-    "metric_path":"metrics/linux-gcc8"
+    "metric_path":"metrics/linux-gcc8",
+    "filter": "real-world"
 }
\ No newline at end of file
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index bcc10c91d4..e3217f0f50 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -257,7 +257,7 @@ mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::
 
 mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manifestPath,
                                                        const std::vector<std::string>& testNames,
-                                                       const std::string& testFilter) {
+                                                       std::string testFilter) {
     Manifest manifest;
     const auto filePath = mbgl::filesystem::path(manifestPath);
     manifest.manifestPath = manifestPath.substr(0, manifestPath.find(filePath.filename()));
@@ -412,6 +412,17 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
         }
     }
 
+    if (testFilter.empty() && document.HasMember("filter")) {
+        const auto& filterValue = document["filter"];
+        if (!filterValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid filter is provoided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+
+        testFilter = filterValue.GetString();
+    }
+
     manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string();
     if (manifest.testRootPath.back() == '/') {
         manifest.testRootPath.pop_back();
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
index 8e2c45efde..cf5644d316 100644
--- a/render-test/manifest_parser.hpp
+++ b/render-test/manifest_parser.hpp
@@ -56,5 +56,5 @@ class ManifestParser {
 public:
     static mbgl::optional<Manifest> parseManifest(const std::string& manifestPath,
                                                   const std::vector<std::string>& testNames,
-                                                  const std::string& testFilter);
+                                                  std::string testFilter);
 };
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index b0b2c745ec..89536a0605 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -101,9 +101,6 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         mbgl::Log::Info(mbgl::Event::General, stream.str());
         mbgl::Log::Error(mbgl::Event::General, e.what());
         exit(2);
-    } catch (const std::regex_error& e) {
-        mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
-        exit(3);
     }
 
     mbgl::filesystem::path manifestPath{testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH}};