From b53093b5221efb6f98f712f734b823ffce1400e1 Mon Sep 17 00:00:00 2001 From: Juha Alanen Date: Thu, 10 Oct 2019 12:17:34 +0300 Subject: [render-test] Add support for collecting network metrics --- next/render-test/CMakeLists.txt | 3 +- render-test/file_source.cpp | 65 +++++++++++++++++++++++++++++++++++++++++ render-test/file_source.hpp | 40 +++++++++++++++++++++++++ render-test/metadata.hpp | 11 ++++++- render-test/parser.cpp | 34 +++++++++++++++++++++ render-test/runner.cpp | 53 +++++++++++++++++++++++++++++++++ 6 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 render-test/file_source.cpp create mode 100644 render-test/file_source.hpp diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt index 2e1c5438ed..b3aa20ba62 100644 --- a/next/render-test/CMakeLists.txt +++ b/next/render-test/CMakeLists.txt @@ -4,7 +4,8 @@ add_library( ${MBGL_ROOT}/expression-test/test_runner_common.hpp ${MBGL_ROOT}/render-test/allocation_index.cpp ${MBGL_ROOT}/render-test/allocation_index.hpp - ${MBGL_ROOT}/render-test/filesystem.hpp + ${MBGL_ROOT}/render-test/file_source.cpp + ${MBGL_ROOT}/render-test/file_source.hpp ${MBGL_ROOT}/render-test/filesystem.hpp ${MBGL_ROOT}/render-test/include/mbgl/render_test.hpp ${MBGL_ROOT}/render-test/metadata.hpp diff --git a/render-test/file_source.cpp b/render-test/file_source.cpp new file mode 100644 index 0000000000..0968f1d2f0 --- /dev/null +++ b/render-test/file_source.cpp @@ -0,0 +1,65 @@ +#include + +#include "file_source.hpp" + +namespace mbgl { + +std::atomic_size_t requestCount{0}; +std::atomic_size_t transferredSize{0}; +std::atomic_bool active{false}; + +ProxyFileSource::ProxyFileSource(const std::string& cachePath, + const std::string& assetPath, + bool supportCacheOnlyRequests_) + : DefaultFileSource(cachePath, assetPath, supportCacheOnlyRequests_) {} + +ProxyFileSource::ProxyFileSource(const std::string& cachePath, + std::unique_ptr&& assetFileSource_, + bool supportCacheOnlyRequests_) + : DefaultFileSource(cachePath, std::move(assetFileSource_), supportCacheOnlyRequests_) {} + +ProxyFileSource::~ProxyFileSource() = default; + +std::unique_ptr ProxyFileSource::request(const Resource& resource, Callback callback) { + auto result = DefaultFileSource::request(resource, [=](Response response) { + std::size_t size = response.data != nullptr ? response.data->size() : 0; + if (active) { + requestCount++; + transferredSize += size; + } + callback(response); + }); + return result; +} + +std::shared_ptr FileSource::createPlatformFileSource(const ResourceOptions& options) { + auto fileSource = std::make_shared( + options.cachePath(), options.assetPath(), options.supportsCacheOnlyRequests()); + fileSource->setAccessToken(options.accessToken()); + fileSource->setAPIBaseURL(options.baseURL()); + return fileSource; +} + +// static +void ProxyFileSource::setTrackingActive(bool active_) { + active = active_; + requestCount = 0; + transferredSize = 0; +} + +// static +bool ProxyFileSource::isTrackingActive() { + return active; +} + +// static +size_t ProxyFileSource::getRequestCount() { + return requestCount; +} + +// static +size_t ProxyFileSource::getTransferredSize() { + return transferredSize; +} + +} // namespace mbgl diff --git a/render-test/file_source.hpp b/render-test/file_source.hpp new file mode 100644 index 0000000000..58acf7b6ad --- /dev/null +++ b/render-test/file_source.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include + +namespace mbgl { + +class ProxyFileSource : public DefaultFileSource { +public: + ProxyFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests = true); + ProxyFileSource(const std::string& cachePath, + std::unique_ptr&& assetFileSource, + bool supportCacheOnlyRequests = true); + ~ProxyFileSource(); + + std::unique_ptr request(const Resource&, Callback); + + /** + * @brief Starts/stops metrics tracking. + */ + static void setTrackingActive(bool); + /** + * @brief Returns metrics tracking status. + */ + static bool isTrackingActive(); + /** + * @brief Returns the total amount of requests. + * + * @return size_t + */ + static size_t getRequestCount(); + + /** + * @brief Returns the size of transferred data (in bytes). + * + * @return size_t + */ + static size_t getTransferredSize(); +}; + +} // namespace mbgl diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index bd26e6a7ba..c26fde1d19 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -57,11 +57,20 @@ struct MemoryProbe { size_t allocations; }; +struct NetworkProbe { + NetworkProbe() = default; + NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {} + + size_t requests; + size_t transferred; +}; + class TestMetrics { public: - bool isEmpty() const { return fileSize.empty() && memory.empty(); } + bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty(); } std::map fileSize; std::map memory; + std::map network; }; struct TestMetadata { diff --git a/render-test/parser.cpp b/render-test/parser.cpp index f4e54493eb..41c9b83298 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -304,6 +304,21 @@ std::string serializeMetrics(const TestMetrics& metrics) { writer.EndArray(); } + // Start network section + if (!metrics.network.empty()) { + writer.Key("network"); + writer.StartArray(); + for (const auto& networkProbe : metrics.network) { + assert(!networkProbe.first.empty()); + writer.StartArray(); + writer.String(networkProbe.first.c_str()); + writer.Uint64(networkProbe.second.requests); + writer.Uint64(networkProbe.second.transferred); + writer.EndArray(); + } + writer.EndArray(); + } + writer.EndObject(); return s.GetString(); @@ -500,6 +515,25 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { } } + if (document.HasMember("network")) { + const mbgl::JSValue& networkValue = document["network"]; + assert(networkValue.IsArray()); + for (auto& probeValue : networkValue.GetArray()) { + assert(probeValue.IsArray()); + assert(probeValue.Size() >= 3u); + assert(probeValue[0].IsString()); + assert(probeValue[1].IsNumber()); + assert(probeValue[2].IsNumber()); + + std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()}; + assert(!mark.empty()); + + result.network.emplace(std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64())); + } + } + return result; } diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 3594c9488b..810806d231 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -21,6 +21,7 @@ #include <../expression-test/test_runner_common.hpp> #include "allocation_index.hpp" +#include "file_source.hpp" #include "metadata.hpp" #include "parser.hpp" #include "runner.hpp" @@ -294,6 +295,35 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, } } + // Check network metrics. + for (const auto& expected : metadata.expectedMetrics.network) { + auto actual = metadata.metrics.network.find(expected.first); + if (actual == metadata.metrics.network.end()) { + metadata.errorMessage = "Failed to find network probe: " + expected.first; + return false; + } + bool failed = false; + if (actual->second.requests != expected.second.requests) { + std::stringstream ss; + ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests + << ", expected is " << expected.second.requests << ". "; + + metadata.errorMessage = ss.str(); + failed = true; + } + if (actual->second.transferred != expected.second.transferred) { + std::stringstream ss; + ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred + << " bytes, expected is " << expected.second.transferred << " bytes."; + + metadata.errorMessage += ss.str(); + failed = true; + } + if (failed) { + return false; + } + } + return true; } @@ -343,6 +373,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { static const std::string memoryProbeOp("probeMemory"); static const std::string memoryProbeStartOp("probeMemoryStart"); static const std::string memoryProbeEndOp("probeMemoryEnd"); + static const std::string networkProbeOp("probeNetwork"); + static const std::string networkProbeStartOp("probeNetworkStart"); + static const std::string networkProbeEndOp("probeNetworkEnd"); static const std::string setFeatureStateOp("setFeatureState"); static const std::string getFeatureStateOp("getFeatureState"); static const std::string removeFeatureStateOp("removeFeatureState"); @@ -619,6 +652,25 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { assert(AllocationIndex::isActive()); AllocationIndex::setActive(false); AllocationIndex::reset(); + } else if (operationArray[0].GetString() == networkProbeStartOp) { + // probeNetworkStart + assert(!ProxyFileSource::isTrackingActive()); + ProxyFileSource::setTrackingActive(true); + } else if (operationArray[0].GetString() == networkProbeOp) { + // probeNetwork + assert(ProxyFileSource::isTrackingActive()); + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + + metadata.metrics.network.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize())); + } else if (operationArray[0].GetString() == networkProbeEndOp) { + // probeNetworkEnd + assert(ProxyFileSource::isTrackingActive()); + ProxyFileSource::setTrackingActive(false); } else if (operationArray[0].GetString() == setFeatureStateOp) { // setFeatureState assert(operationArray.Size() >= 3u); @@ -786,6 +838,7 @@ TestRunner::Impl::Impl(const TestMetadata& metadata) bool TestRunner::run(TestMetadata& metadata) { AllocationIndex::setActive(false); AllocationIndex::reset(); + ProxyFileSource::setTrackingActive(false); std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) + "/" + mbgl::util::toString(metadata.pixelRatio) + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions)); -- cgit v1.2.1