[core] Log Memory.Footprint test results (#8170)

* Log Memory.Footprint test output

* On CI, only run Memory.Footprint test with libjemalloc

6 files changed, 215 insertions, 30 deletions

diff --git a/.travis.yml b/.travis.yml
index d41cfa83a4..3b13902776 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -173,7 +173,8 @@ matrix:
         - mapbox_export_mesa_library_path
       script:
         - make qt-app
-        - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so make run-qt-test-Memory.*:*.Load
+        - GTEST_OUTPUT=xml LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so make run-qt-test-Memory.*:*.Load
+        - scripts/log_memory_benchmarks.sh test_detail.xml "Platform=Linux,Compiler=${_CC},Arch=$(uname -m)"
 
     # Qt 5 - GCC 5 - Release
     - os: linux
diff --git a/cmake/test-files.cmake b/cmake/test-files.cmake
index b2bff6c72a..59929bbb70 100644
--- a/cmake/test-files.cmake
+++ b/cmake/test-files.cmake
@@ -48,6 +48,8 @@ set(MBGL_TEST_FILES
     test/src/mbgl/test/fake_file_source.hpp
     test/src/mbgl/test/fixture_log_observer.cpp
     test/src/mbgl/test/fixture_log_observer.hpp
+    test/src/mbgl/test/getrss.cpp
+    test/src/mbgl/test/getrss.hpp
     test/src/mbgl/test/stub_file_source.cpp
     test/src/mbgl/test/stub_file_source.hpp
     test/src/mbgl/test/stub_geometry_tile_feature.hpp
diff --git a/scripts/log_memory_benchmarks.sh b/scripts/log_memory_benchmarks.sh
new file mode 100755
index 0000000000..1f213caa92
--- /dev/null
+++ b/scripts/log_memory_benchmarks.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+set -u
+
+# Logs metrics on memory usage to CloudWatch
+
+GTEST_OUTPUT=$1
+DIMENSIONS=$2
+
+if [ -z "${DIMENSIONS}" ]; then
+    echo "* No dimensions specified for memory benchmarks"
+    exit 1
+fi
+
+function reportAttributeValue {
+    ATTR_NAME=$1
+    ATTR_UNITS=$2
+    ATTR_VALUE=$(cat $GTEST_OUTPUT | grep -o "$ATTR_NAME=\"[^\"]*" | sed "s/$ATTR_NAME=\"//")
+    if [ ${CLOUDWATCH:-} ]; then
+        echo "* Reporting $ATTR_NAME = $ATTR_VALUE $ATTR_UNITS for '${DIMENSIONS}'"
+        aws --region us-east-1 cloudwatch put-metric-data \
+            --namespace "Mapbox/GL" \
+            --metric-name "$ATTR_NAME" \
+            --unit "$ATTR_UNITS" \
+            --value ${ATTR_VALUE} \
+            --dimensions "${DIMENSIONS}"
+    else
+        echo "* Measured $ATTR_NAME = $ATTR_VALUE $ATTR_UNITS for '${DIMENSIONS}'"
+    fi
+}
+
+if [ -f "${GTEST_OUTPUT}" ]; then
+    reportAttributeValue vectorFootprint Bytes
+    reportAttributeValue rasterFootprint Bytes
+else
+    echo "* File '${GTEST_OUTPUT}' does not exist"
+fi
diff --git a/test/src/mbgl/test/getrss.cpp b/test/src/mbgl/test/getrss.cpp
new file mode 100644
index 0000000000..9f57ad8e7b
--- /dev/null
+++ b/test/src/mbgl/test/getrss.cpp
@@ -0,0 +1,102 @@
+#include <mbgl/test/getrss.hpp>
+
+/*
+ * Adapted from
+ * http://nadeausoftware.com/articles/2012/07/c_c_tip_how_get_process_resident_set_size_physical_memory_use
+ *
+ * Author:  David Robert Nadeau
+ * Site:    http://NadeauSoftware.com/
+ * License: Creative Commons Attribution 3.0 Unported License
+ *          http://creativecommons.org/licenses/by/3.0/deed.en_US
+ */
+
+namespace mbgl {
+namespace test {
+
+/**
+ * Returns the peak (maximum so far) resident set size (physical
+ * memory use) measured in bytes, or zero if the value cannot be
+ * determined on this OS.
+ */
+size_t getPeakRSS( )
+{
+#if defined(_WIN32)
+	/* Windows -------------------------------------------------- */
+	PROCESS_MEMORY_COUNTERS info;
+	GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) );
+	return (size_t)info.PeakWorkingSetSize;
+
+#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__)))
+	/* AIX and Solaris ------------------------------------------ */
+	struct psinfo psinfo;
+	int fd = -1;
+	if ( (fd = open( "/proc/self/psinfo", O_RDONLY )) == -1 )
+		return (size_t)0L;		/* Can't open? */
+	if ( read( fd, &psinfo, sizeof(psinfo) ) != sizeof(psinfo) )
+	{
+		close( fd );
+		return (size_t)0L;		/* Can't read? */
+	}
+	close( fd );
+	return (size_t)(psinfo.pr_rssize * 1024L);
+
+#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
+	/* BSD, Linux, and OSX -------------------------------------- */
+	struct rusage rusage;
+	getrusage( RUSAGE_SELF, &rusage );
+#if defined(__APPLE__) && defined(__MACH__)
+	return (size_t)rusage.ru_maxrss;
+#else
+	return (size_t)(rusage.ru_maxrss * 1024L);
+#endif
+
+#else
+	/* Unknown OS ----------------------------------------------- */
+	return (size_t)0L;			/* Unsupported. */
+#endif
+}
+
+/**
+ * Returns the current resident set size (physical memory use) measured
+ * in bytes, or zero if the value cannot be determined on this OS.
+ */
+size_t getCurrentRSS( )
+{
+#if defined(_WIN32)
+	/* Windows -------------------------------------------------- */
+	PROCESS_MEMORY_COUNTERS info;
+	GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) );
+	return (size_t)info.WorkingSetSize;
+
+#elif defined(__APPLE__) && defined(__MACH__)
+	/* OSX ------------------------------------------------------ */
+	struct mach_task_basic_info info;
+	mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+	if ( task_info( mach_task_self( ), MACH_TASK_BASIC_INFO,
+		(task_info_t)&info, &infoCount ) != KERN_SUCCESS )
+		return (size_t)0L;		/* Can't access? */
+	return (size_t)info.resident_size;
+
+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
+	/* Linux ---------------------------------------------------- */
+	long rss = 0L;
+	FILE* fp = NULL;
+	if ( (fp = fopen( "/proc/self/statm", "r" )) == NULL )
+		return (size_t)0L;		/* Can't open? */
+	if ( fscanf( fp, "%*s%ld", &rss ) != 1 )
+	{
+		fclose( fp );
+		return (size_t)0L;		/* Can't read? */
+	}
+	fclose( fp );
+	return (size_t)rss * (size_t)sysconf( _SC_PAGESIZE);
+
+#else
+	/* AIX, BSD, Solaris, and Unknown OS ------------------------ */
+	return (size_t)0L;			/* Unsupported. */
+#endif
+}
+
+} // namespace test
+} // namespace mbgl
+
diff --git a/test/src/mbgl/test/getrss.hpp b/test/src/mbgl/test/getrss.hpp
new file mode 100644
index 0000000000..a4420c4b5f
--- /dev/null
+++ b/test/src/mbgl/test/getrss.hpp
@@ -0,0 +1,45 @@
+#if defined(_WIN32)
+#include <windows.h>
+#include <psapi.h>
+
+#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
+#include <unistd.h>
+#include <sys/resource.h>
+
+#if defined(__APPLE__) && defined(__MACH__)
+#include <mach/mach.h>
+#include <mach/message.h>  // for mach_port_t
+#include <mach/task_info.h>
+
+#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__)))
+#include <fcntl.h>
+#include <procfs.h>
+
+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
+#include <stdio.h>
+
+#endif
+
+#else
+#error "Cannot define getPeakRSS( ) or getCurrentRSS( ) for an unknown OS."
+#endif
+
+namespace mbgl {
+namespace test {
+        
+
+/**
+ * Returns the peak (maximum so far) resident set size (physical
+ * memory use) measured in bytes, or zero if the value cannot be
+ * determined on this OS.
+ */
+size_t getPeakRSS();
+
+/**
+ * Returns the current resident set size (physical memory use) measured
+ * in bytes, or zero if the value cannot be determined on this OS.
+ */
+size_t getCurrentRSS();
+
+}
+}
diff --git a/test/util/memory.test.cpp b/test/util/memory.test.cpp
index 984e7a3e24..d49c49018f 100644
--- a/test/util/memory.test.cpp
+++ b/test/util/memory.test.cpp
@@ -1,4 +1,5 @@
 #include <mbgl/test/stub_file_source.hpp>
+#include <mbgl/test/getrss.hpp>
 #include <mbgl/test/util.hpp>
 
 #include <mbgl/map/map.hpp>
@@ -21,29 +22,6 @@
 using namespace mbgl;
 using namespace std::literals::string_literals;
 
-long getRSS() {
-    auto statm = util::read_file("/proc/self/statm");
-
-    std::vector<std::string> stats;
-    std::istringstream stream(statm);
-
-    std::copy(std::istream_iterator<std::string>(stream),
-        std::istream_iterator<std::string>(),
-        std::back_inserter(stats));
-
-    return std::stol(stats[1]) * getpagesize();
-}
-
-bool isUsingJemalloc() {
-    const char* preload = getenv("LD_PRELOAD");
-
-    if (preload) {
-        return std::string(preload).find("libjemalloc.so") != std::string::npos;
-    } else {
-        return false;
-    }
-}
-
 class MemoryTest {
 public:
     MemoryTest() {
@@ -109,16 +87,31 @@ TEST(Memory, Raster) {
     test::render(map, test.view);
 }
 
+/**
+On CI, we only run the memory footprint test in the Qt build, because it uses
+jemalloc, which yields more consistent memory usage results.  To force it to
+run locally, use `DO_MEMORY_FOOTPRINT=1 make run-test-Memory.Footprint.
+*/
+bool shouldRunFootprint() {
+    const char* preload = getenv("LD_PRELOAD");
+    
+    if (preload) {
+        return std::string(preload).find("libjemalloc.so") != std::string::npos;
+    } else {
+        return getenv("DO_MEMORY_FOOTPRINT");
+    }
+}
+
 // This test will measure the size of a Map object
 // after rendering a raster and a vector style. The
 // idea is to try to keep the memory footprint within
 // reasonable limits, so this test acts more like a
 // safeguard.
 TEST(Memory, Footprint) {
-    if (!isUsingJemalloc()) {
+    if (!shouldRunFootprint()) {
         return;
     }
-
+    
     MemoryTest test;
 
     auto renderMap = [&](Map& map, const char* style){
@@ -141,7 +134,7 @@ TEST(Memory, Footprint) {
     std::vector<std::unique_ptr<Map>> maps;
     unsigned runs = 15;
 
-    long vectorInitialRSS = getRSS();
+    long vectorInitialRSS = mbgl::test::getCurrentRSS();
     for (unsigned i = 0; i < runs; ++i) {
         auto vector = std::make_unique<Map>(test.backend, Size{ 256, 256 }, 2, test.fileSource,
                                             test.threadPool, MapMode::Still);
@@ -149,9 +142,9 @@ TEST(Memory, Footprint) {
         maps.push_back(std::move(vector));
     };
 
-    double vectorFootprint = (getRSS() - vectorInitialRSS) / double(runs);
+    double vectorFootprint = (mbgl::test::getCurrentRSS() - vectorInitialRSS) / double(runs);
 
-    long rasterInitialRSS = getRSS();
+    long rasterInitialRSS = mbgl::test::getCurrentRSS();
     for (unsigned i = 0; i < runs; ++i) {
         auto raster = std::make_unique<Map>(test.backend, Size{ 256, 256 }, 2, test.fileSource,
                                             test.threadPool, MapMode::Still);
@@ -159,7 +152,10 @@ TEST(Memory, Footprint) {
         maps.push_back(std::move(raster));
     };
 
-    double rasterFootprint = (getRSS() - rasterInitialRSS) / double(runs);
+    double rasterFootprint = (mbgl::test::getCurrentRSS() - rasterInitialRSS) / double(runs);
+    
+    RecordProperty("vectorFootprint", vectorFootprint);
+    RecordProperty("rasterFootprint", rasterFootprint);
 
     ASSERT_LT(vectorFootprint, 65 * 1024 * 1024) << "\
         mbgl::Map footprint over 65MB for vector styles.";