[test runner] Tests for memory size metrics

11 files changed, 163 insertions, 2 deletions

diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 925599dfb8..401b060fd6 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -292,7 +292,7 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
         if (!passed) {
             std::stringstream ss;
             ss << "Number of allocations at probe \"" << expected.first << "\" is " << actual->second.allocations
-               << ", expected is " << expected.second.allocations << "±" << delta << " allocations.";
+               << ", expected is " << expected.second.allocations << "±" << std::round(delta) << " allocations.";
 
             metadata.errorMessage = ss.str();
             return false;
diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/expected.png b/render-test/tests/memory/fail-memory-size-is-too-big/expected.png
new file mode 100644
index 0000000000..0858c19f05
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-big/expected.png
diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/metrics.json b/render-test/tests/memory/fail-memory-size-is-too-big/metrics.json
new file mode 100644
index 0000000000..69832a1d9f
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-big/metrics.json
@@ -0,0 +1 @@
+{"memory":[["after setZoom 0.9",33271,62],["end",40000,118],["start",0,0]]}
diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/style.json b/render-test/tests/memory/fail-memory-size-is-too-big/style.json
new file mode 100644
index 0000000000..b2805ef34f
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-big/style.json
@@ -0,0 +1,52 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "width": 64,
+      "height": 64,
+      "operations": [
+        [ "wait" ],
+        [ "probeMemoryStart" ],
+        [ "probeMemory", "start" ],
+        [
+          "setZoom",
+          0.9
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "after setZoom 0.9", 0.005 ],
+        [
+          "setLayerZoomRange",
+          "circle",
+          1,
+          2
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "end", 0.005 ],
+        [ "probeMemoryEnd" ]
+      ]
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/expected.png b/render-test/tests/memory/fail-memory-size-is-too-small/expected.png
new file mode 100644
index 0000000000..0858c19f05
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-small/expected.png
diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/metrics.json b/render-test/tests/memory/fail-memory-size-is-too-small/metrics.json
new file mode 100644
index 0000000000..51cdc6267c
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-small/metrics.json
@@ -0,0 +1 @@
+{"memory":[["after setZoom 0.9",33271,62],["end",60000,118],["start",0,0]]}
diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/style.json b/render-test/tests/memory/fail-memory-size-is-too-small/style.json
new file mode 100644
index 0000000000..b2805ef34f
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-small/style.json
@@ -0,0 +1,52 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "width": 64,
+      "height": 64,
+      "operations": [
+        [ "wait" ],
+        [ "probeMemoryStart" ],
+        [ "probeMemory", "start" ],
+        [
+          "setZoom",
+          0.9
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "after setZoom 0.9", 0.005 ],
+        [
+          "setLayerZoomRange",
+          "circle",
+          1,
+          2
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "end", 0.005 ],
+        [ "probeMemoryEnd" ]
+      ]
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/memory/pass-memory-size-is-same/expected.png b/render-test/tests/memory/pass-memory-size-is-same/expected.png
new file mode 100644
index 0000000000..0858c19f05
--- /dev/null
+++ b/render-test/tests/memory/pass-memory-size-is-same/expected.png
diff --git a/render-test/tests/memory/pass-memory-size-is-same/metrics.json b/render-test/tests/memory/pass-memory-size-is-same/metrics.json
new file mode 100644
index 0000000000..9ed24a99c7
--- /dev/null
+++ b/render-test/tests/memory/pass-memory-size-is-same/metrics.json
@@ -0,0 +1 @@
+{"memory":[["after setZoom 0.9",33271,62],["end",47833,118],["start",0,0]]}
\ No newline at end of file
diff --git a/render-test/tests/memory/pass-memory-size-is-same/style.json b/render-test/tests/memory/pass-memory-size-is-same/style.json
new file mode 100644
index 0000000000..b2805ef34f
--- /dev/null
+++ b/render-test/tests/memory/pass-memory-size-is-same/style.json
@@ -0,0 +1,52 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "width": 64,
+      "height": 64,
+      "operations": [
+        [ "wait" ],
+        [ "probeMemoryStart" ],
+        [ "probeMemory", "start" ],
+        [
+          "setZoom",
+          0.9
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "after setZoom 0.9", 0.005 ],
+        [
+          "setLayerZoomRange",
+          "circle",
+          1,
+          2
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "end", 0.005 ],
+        [ "probeMemoryEnd" ]
+      ]
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/should-fail.json b/render-test/tests/should-fail.json
index d80b419605..f08497b07f 100644
--- a/render-test/tests/should-fail.json
+++ b/render-test/tests/should-fail.json
@@ -5,5 +5,7 @@
   "tests/file-size/fail-file-not-found": "Should fail, file not found.",
   "tests/network/fail-requests": "Should fail, number of requests higher than expected.",
   "tests/network/fail-transferred": "Should fail, amount of transferred data higher than expected.",
-  "tests/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected."
+  "tests/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.",
+  "tests/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.",
+  "tests/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected."
 }