[render-test] Add support for collecting network metrics

6 files changed, 204 insertions, 2 deletions

diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt
index 2e1c5438ed..b3aa20ba62 100644
--- a/next/render-test/CMakeLists.txt
+++ b/next/render-test/CMakeLists.txt
@@ -4,7 +4,8 @@ add_library(
     ${MBGL_ROOT}/expression-test/test_runner_common.hpp
     ${MBGL_ROOT}/render-test/allocation_index.cpp
     ${MBGL_ROOT}/render-test/allocation_index.hpp
-    ${MBGL_ROOT}/render-test/filesystem.hpp
+    ${MBGL_ROOT}/render-test/file_source.cpp
+    ${MBGL_ROOT}/render-test/file_source.hpp
     ${MBGL_ROOT}/render-test/filesystem.hpp
     ${MBGL_ROOT}/render-test/include/mbgl/render_test.hpp
     ${MBGL_ROOT}/render-test/metadata.hpp
diff --git a/render-test/file_source.cpp b/render-test/file_source.cpp
new file mode 100644
index 0000000000..0968f1d2f0
--- /dev/null
+++ b/render-test/file_source.cpp
@@ -0,0 +1,65 @@
+#include <mbgl/storage/resource_options.hpp>
+
+#include "file_source.hpp"
+
+namespace mbgl {
+
+std::atomic_size_t requestCount{0};
+std::atomic_size_t transferredSize{0};
+std::atomic_bool active{false};
+
+ProxyFileSource::ProxyFileSource(const std::string& cachePath,
+                                 const std::string& assetPath,
+                                 bool supportCacheOnlyRequests_)
+    : DefaultFileSource(cachePath, assetPath, supportCacheOnlyRequests_) {}
+
+ProxyFileSource::ProxyFileSource(const std::string& cachePath,
+                                 std::unique_ptr<FileSource>&& assetFileSource_,
+                                 bool supportCacheOnlyRequests_)
+    : DefaultFileSource(cachePath, std::move(assetFileSource_), supportCacheOnlyRequests_) {}
+
+ProxyFileSource::~ProxyFileSource() = default;
+
+std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource, Callback callback) {
+    auto result = DefaultFileSource::request(resource, [=](Response response) {
+        std::size_t size = response.data != nullptr ? response.data->size() : 0;
+        if (active) {
+            requestCount++;
+            transferredSize += size;
+        }
+        callback(response);
+    });
+    return result;
+}
+
+std::shared_ptr<FileSource> FileSource::createPlatformFileSource(const ResourceOptions& options) {
+    auto fileSource = std::make_shared<ProxyFileSource>(
+        options.cachePath(), options.assetPath(), options.supportsCacheOnlyRequests());
+    fileSource->setAccessToken(options.accessToken());
+    fileSource->setAPIBaseURL(options.baseURL());
+    return fileSource;
+}
+
+// static
+void ProxyFileSource::setTrackingActive(bool active_) {
+    active = active_;
+    requestCount = 0;
+    transferredSize = 0;
+}
+
+// static
+bool ProxyFileSource::isTrackingActive() {
+    return active;
+}
+
+// static
+size_t ProxyFileSource::getRequestCount() {
+    return requestCount;
+}
+
+// static
+size_t ProxyFileSource::getTransferredSize() {
+    return transferredSize;
+}
+
+} // namespace mbgl
diff --git a/render-test/file_source.hpp b/render-test/file_source.hpp
new file mode 100644
index 0000000000..58acf7b6ad
--- /dev/null
+++ b/render-test/file_source.hpp
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <mbgl/storage/default_file_source.hpp>
+
+namespace mbgl {
+
+class ProxyFileSource : public DefaultFileSource {
+public:
+    ProxyFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests = true);
+    ProxyFileSource(const std::string& cachePath,
+                    std::unique_ptr<FileSource>&& assetFileSource,
+                    bool supportCacheOnlyRequests = true);
+    ~ProxyFileSource();
+
+    std::unique_ptr<AsyncRequest> request(const Resource&, Callback);
+
+    /**
+     * @brief Starts/stops metrics tracking.
+     */
+    static void setTrackingActive(bool);
+    /**
+     * @brief Returns metrics tracking status.
+     */
+    static bool isTrackingActive();
+    /**
+     * @brief Returns the total amount of requests.
+     *
+     * @return size_t
+     */
+    static size_t getRequestCount();
+
+    /**
+     * @brief Returns the size of transferred data (in bytes).
+     *
+     * @return size_t
+     */
+    static size_t getTransferredSize();
+};
+
+} // namespace mbgl
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index bd26e6a7ba..c26fde1d19 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -57,11 +57,20 @@ struct MemoryProbe {
     size_t allocations;
 };
 
+struct NetworkProbe {
+    NetworkProbe() = default;
+    NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {}
+
+    size_t requests;
+    size_t transferred;
+};
+
 class TestMetrics {
 public:
-    bool isEmpty() const { return fileSize.empty() && memory.empty(); }
+    bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty(); }
     std::map<std::string, FileSizeProbe> fileSize;
     std::map<std::string, MemoryProbe> memory;
+    std::map<std::string, NetworkProbe> network;
 };
 
 struct TestMetadata {
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index f4e54493eb..41c9b83298 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -304,6 +304,21 @@ std::string serializeMetrics(const TestMetrics& metrics) {
         writer.EndArray();
     }
 
+    // Start network section
+    if (!metrics.network.empty()) {
+        writer.Key("network");
+        writer.StartArray();
+        for (const auto& networkProbe : metrics.network) {
+            assert(!networkProbe.first.empty());
+            writer.StartArray();
+            writer.String(networkProbe.first.c_str());
+            writer.Uint64(networkProbe.second.requests);
+            writer.Uint64(networkProbe.second.transferred);
+            writer.EndArray();
+        }
+        writer.EndArray();
+    }
+
     writer.EndObject();
 
     return s.GetString();
@@ -500,6 +515,25 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
         }
     }
 
+    if (document.HasMember("network")) {
+        const mbgl::JSValue& networkValue = document["network"];
+        assert(networkValue.IsArray());
+        for (auto& probeValue : networkValue.GetArray()) {
+            assert(probeValue.IsArray());
+            assert(probeValue.Size() >= 3u);
+            assert(probeValue[0].IsString());
+            assert(probeValue[1].IsNumber());
+            assert(probeValue[2].IsNumber());
+
+            std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
+            assert(!mark.empty());
+
+            result.network.emplace(std::piecewise_construct,
+                                   std::forward_as_tuple(std::move(mark)),
+                                   std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
+        }
+    }
+
     return result;
 }
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 3594c9488b..810806d231 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -21,6 +21,7 @@
 
 #include <../expression-test/test_runner_common.hpp>
 #include "allocation_index.hpp"
+#include "file_source.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
@@ -294,6 +295,35 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
         }
     }
 
+    // Check network metrics.
+    for (const auto& expected : metadata.expectedMetrics.network) {
+        auto actual = metadata.metrics.network.find(expected.first);
+        if (actual == metadata.metrics.network.end()) {
+            metadata.errorMessage = "Failed to find network probe: " + expected.first;
+            return false;
+        }
+        bool failed = false;
+        if (actual->second.requests != expected.second.requests) {
+            std::stringstream ss;
+            ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests
+               << ", expected is " << expected.second.requests << ". ";
+
+            metadata.errorMessage = ss.str();
+            failed = true;
+        }
+        if (actual->second.transferred != expected.second.transferred) {
+            std::stringstream ss;
+            ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred
+               << " bytes, expected is " << expected.second.transferred << " bytes.";
+
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+        if (failed) {
+            return false;
+        }
+    }
+
     return true;
 }
 
@@ -343,6 +373,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
     static const std::string memoryProbeOp("probeMemory");
     static const std::string memoryProbeStartOp("probeMemoryStart");
     static const std::string memoryProbeEndOp("probeMemoryEnd");
+    static const std::string networkProbeOp("probeNetwork");
+    static const std::string networkProbeStartOp("probeNetworkStart");
+    static const std::string networkProbeEndOp("probeNetworkEnd");
     static const std::string setFeatureStateOp("setFeatureState");
     static const std::string getFeatureStateOp("getFeatureState");
     static const std::string removeFeatureStateOp("removeFeatureState");
@@ -619,6 +652,25 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         assert(AllocationIndex::isActive());
         AllocationIndex::setActive(false);
         AllocationIndex::reset();
+    } else if (operationArray[0].GetString() == networkProbeStartOp) {
+        // probeNetworkStart
+        assert(!ProxyFileSource::isTrackingActive());
+        ProxyFileSource::setTrackingActive(true);
+    } else if (operationArray[0].GetString() == networkProbeOp) {
+        // probeNetwork
+        assert(ProxyFileSource::isTrackingActive());
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsString());
+        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+
+        metadata.metrics.network.emplace(
+            std::piecewise_construct,
+            std::forward_as_tuple(std::move(mark)),
+            std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
+    } else if (operationArray[0].GetString() == networkProbeEndOp) {
+        // probeNetworkEnd
+        assert(ProxyFileSource::isTrackingActive());
+        ProxyFileSource::setTrackingActive(false);
     } else if (operationArray[0].GetString() == setFeatureStateOp) {
         // setFeatureState
         assert(operationArray.Size() >= 3u);
@@ -786,6 +838,7 @@ TestRunner::Impl::Impl(const TestMetadata& metadata)
 bool TestRunner::run(TestMetadata& metadata) {
     AllocationIndex::setActive(false);
     AllocationIndex::reset();
+    ProxyFileSource::setTrackingActive(false);
     std::string key = mbgl::util::toString(uint32_t(metadata.mapMode))
         + "/" + mbgl::util::toString(metadata.pixelRatio)
         + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));