From 429292fb9898cbcf3f2a0b644e01277473124252 Mon Sep 17 00:00:00 2001 From: zmiao Date: Fri, 24 Jan 2020 15:45:47 +0200 Subject: [test-runner] Add iOS RenderTestRunner test metrics (#16139) * [test-runner] Add iOS RenderTestRunner test metrics * [test-runner] Add ios probe metrics --- render-test/ios/ios_test_runner.cpp | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'render-test/ios/ios_test_runner.cpp') diff --git a/render-test/ios/ios_test_runner.cpp b/render-test/ios/ios_test_runner.cpp index 842318c64a..ceb44440aa 100644 --- a/render-test/ios/ios_test_runner.cpp +++ b/render-test/ios/ios_test_runner.cpp @@ -9,9 +9,9 @@ #define EXPORT __attribute__((visibility("default"))) EXPORT -bool TestRunner::startTest(const std::string& manifest) { +bool TestRunner::startTest(const std::string& manifestBasePath) { auto runTestWithManifest = [](const std::string& manifest) -> bool { - std::vector arguments = {"mbgl-render-test-runner", "-p", manifest}; + std::vector arguments = {"mbgl-render-test-runner", "-p", manifest, "-u", "rebaseline"}; std::vector argv; for (const auto& arg : arguments) { argv.push_back(const_cast(arg.data())); @@ -24,17 +24,26 @@ bool TestRunner::startTest(const std::string& manifest) { }; mbgl::Log::Info(mbgl::Event::General, "Start running RenderTestRunner with manifest: '%s'", manifest.c_str()); - auto result = mbgl::runRenderTests(static_cast(argv.size() - 1), argv.data(), testStatus) == 0; - mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str()); - return result; + auto result = mbgl::runRenderTests(static_cast(argv.size() - 1), argv.data(), testStatus); + + mbgl::Log::Info(mbgl::Event::General, + "End running RenderTestRunner with manifest: '%s' with result value %d", + manifest.c_str(), + result); + return result == 0; }; - auto ret = false; + bool status = false; try { - ret = runTestWithManifest(manifest); + status = runTestWithManifest(manifestBasePath + "/next-ios-render-test-runner-style.json"); + status = runTestWithManifest(manifestBasePath + "/next-ios-render-test-runner-metrics.json") && status; } catch (...) { - mbgl::Log::Info(mbgl::Event::General, "testFailed"); + mbgl::Log::Info(mbgl::Event::General, "Failed with exception"); } + mbgl::Log::Info(mbgl::Event::General, "All tests are finished!"); - return ret; + if (!status) { + mbgl::Log::Info(mbgl::Event::General, "There are failing test cases"); + } + return status; } -- cgit v1.2.1