[android] improve test runner by allowing selective execution of tests

5 files changed, 95 insertions, 36 deletions

diff --git a/Makefile b/Makefile
index 5469fb1f70..1bb7eaf211 100644
--- a/Makefile
+++ b/Makefile
@@ -478,6 +478,8 @@ MBGL_ANDROID_ABIS = arm-v5 arm-v7 arm-v8 x86 x86-64 mips
 MBGL_ANDROID_LOCAL_WORK_DIR = /data/local/tmp/core-tests
 MBGL_ANDROID_LIBDIR = lib$(if $(filter arm-v8 x86-64,$1),64)
 MBGL_ANDROID_DALVIKVM = dalvikvm$(if $(filter arm-v8 x86-64,$1),64,32)
+MBGL_ANDROID_APK_SUFFIX = $(if $(filter Release,$(BUILDTYPE)),release-unsigned,debug)
+MBGL_ANDROID_CORE_TEST_DIR = build/android-$1/$(BUILDTYPE)/core-tests
 
 .PHONY: android-style-code
 android-style-code:
@@ -511,40 +513,50 @@ android-lib-$1: build/android-$1/$(BUILDTYPE)/Makefile
 android-$1: android-lib-$1
 	cd platform/android && ./gradlew --parallel --max-workers=$(JOBS) :MapboxGLAndroidSDKTestApp:assemble$(BUILDTYPE)
 
-run-android-core-test-$1: android-test-lib-$1
+.PHONY: android-core-test-$1
+android-core-test-$1: android-test-lib-$1
+	mkdir -p $(MBGL_ANDROID_CORE_TEST_DIR)
+
 	# Compile main sources and extract the classes (using the test app to get all transitive dependencies in one place)
-	cd platform/android && ./gradlew :MapboxGLAndroidSDKTestApp:assembleDebug
-	unzip -o platform/android/MapboxGLAndroidSDKTestApp/build/outputs/apk/MapboxGLAndroidSDKTestApp-debug.apk classes.dex -d build/android-$1/$(BUILDTYPE)
+	cd platform/android && ./gradlew :MapboxGLAndroidSDKTestApp:assemble$(BUILDTYPE)
+	unzip -o platform/android/MapboxGLAndroidSDKTestApp/build/outputs/apk/MapboxGLAndroidSDKTestApp-$(MBGL_ANDROID_APK_SUFFIX).apk classes.dex -d $(MBGL_ANDROID_CORE_TEST_DIR)
+
+	# Compile Test runner
+	find platform/android/src/test -name "*.java" > $(MBGL_ANDROID_CORE_TEST_DIR)/java-sources.txt
+	javac -sourcepath platform/android/src/test -d $(MBGL_ANDROID_CORE_TEST_DIR) -source 1.7 -target 1.7 @$(MBGL_ANDROID_CORE_TEST_DIR)/java-sources.txt
 
-	#Compile Test runner
-	find platform/android/src/test -name "*.java" > build/android-$1/$(BUILDTYPE)/java-sources.txt
-	javac -sourcepath platform/android/src/test -d build/android-$1/$(BUILDTYPE) -source 1.7 -target 1.7 @build/android-$1/$(BUILDTYPE)/java-sources.txt
-	#Combine and dex
-	cd build/android-$1/$(BUILDTYPE) && $(ANDROID_HOME)/build-tools/25.0.0/dx --dex --output=test.jar *.class classes.dex
+	# Combine and dex
+	cd $(MBGL_ANDROID_CORE_TEST_DIR) && $(ANDROID_HOME)/build-tools/25.0.0/dx --dex --output=test.jar *.class classes.dex
 
-	#Ensure clean state on the device
+run-android-core-test-$1-%: android-core-test-$1
+	# Ensure clean state on the device
 	adb shell "rm -Rf $(MBGL_ANDROID_LOCAL_WORK_DIR) && mkdir -p $(MBGL_ANDROID_LOCAL_WORK_DIR)/test"
 
 	# Generate zipped asset files
 	cd test/fixtures/api && zip -r assets.zip assets && cd -
 	cd test/fixtures/storage && zip -r assets.zip assets && cd -
 
-	#Push all needed files to the device
-	adb push build/android-$1/$(BUILDTYPE)/test.jar $(MBGL_ANDROID_LOCAL_WORK_DIR) > /dev/null 2>&1
+	# Push all needed files to the device
+	adb push $(MBGL_ANDROID_CORE_TEST_DIR)/test.jar $(MBGL_ANDROID_LOCAL_WORK_DIR) > /dev/null 2>&1
 	adb push test/fixtures $(MBGL_ANDROID_LOCAL_WORK_DIR)/test > /dev/null 2>&1
 	adb push build/android-$1/$(BUILDTYPE)/stripped/libmapbox-gl.so $(MBGL_ANDROID_LOCAL_WORK_DIR) > /dev/null 2>&1
 	adb push build/android-$1/$(BUILDTYPE)/stripped/libmbgl-test.so $(MBGL_ANDROID_LOCAL_WORK_DIR) > /dev/null 2>&1
 
-	#Kick off the tests
-	adb shell "export LD_LIBRARY_PATH=/system/$(MBGL_ANDROID_LIBDIR):$(MBGL_ANDROID_LOCAL_WORK_DIR) && cd $(MBGL_ANDROID_LOCAL_WORK_DIR) && $(MBGL_ANDROID_DALVIKVM) -cp $(MBGL_ANDROID_LOCAL_WORK_DIR)/test.jar Main"
+	# Kick off the tests
+	adb shell "export LD_LIBRARY_PATH=/system/$(MBGL_ANDROID_LIBDIR):$(MBGL_ANDROID_LOCAL_WORK_DIR) && cd $(MBGL_ANDROID_LOCAL_WORK_DIR) && $(MBGL_ANDROID_DALVIKVM) -cp $(MBGL_ANDROID_LOCAL_WORK_DIR)/test.jar Main --gtest_filter=$$*"
 
-	#Gather the results
+	# Gather the results and unpack them
 	adb shell "cd $(MBGL_ANDROID_LOCAL_WORK_DIR) && tar -cvzf results.tgz test/fixtures/*  > /dev/null 2>&1"
-	adb pull $(MBGL_ANDROID_LOCAL_WORK_DIR)/results.tgz build/android-$1/$(BUILDTYPE)/ > /dev/null 2>&1
+	adb pull $(MBGL_ANDROID_LOCAL_WORK_DIR)/results.tgz $(MBGL_ANDROID_CORE_TEST_DIR)/ > /dev/null 2>&1
+	rm -rf $(MBGL_ANDROID_CORE_TEST_DIR)/results && mkdir -p $(MBGL_ANDROID_CORE_TEST_DIR)/results
+	tar -xzf $(MBGL_ANDROID_CORE_TEST_DIR)/results.tgz --strip-components=2 -C $(MBGL_ANDROID_CORE_TEST_DIR)/results
+
+.PHONY: run-android-core-test-$1
+run-android-core-test-$1: run-android-core-test-$1-*
 
 .PHONY: run-android-$1
 run-android-$1: android-$1
-	cd platform/android  && ./gradlew :MapboxGLAndroidSDKTestApp:installDebug && adb shell am start -n com.mapbox.mapboxsdk.testapp/.activity.FeatureOverviewActivity
+	cd platform/android && ./gradlew :MapboxGLAndroidSDKTestApp:install$(BUILDTYPE) && adb shell am start -n com.mapbox.mapboxsdk.testapp/.activity.FeatureOverviewActivity
 
 apackage: android-lib-$1
 endef
diff --git a/platform/android/src/test/Main.java b/platform/android/src/test/Main.java
index 2abcf2bfdb..b6f540f666 100644
--- a/platform/android/src/test/Main.java
+++ b/platform/android/src/test/Main.java
@@ -1,13 +1,13 @@
 
 public class Main {
-    public native void runAllTests();
+    public native void runAllTests(String[] args);
 
     public static void main(String[] args) throws Exception {
         //Load the tests
         System.loadLibrary("mbgl-test");
 
         //Run the tests
-        new Main().runAllTests();
+        new Main().runAllTests(args);
 
         //Exit explicitly otherwise dalvikvm won't quit
         System.exit(0);
diff --git a/platform/android/src/test/main.jni.cpp b/platform/android/src/test/main.jni.cpp
index d37b908202..f79d7671cb 100644
--- a/platform/android/src/test/main.jni.cpp
+++ b/platform/android/src/test/main.jni.cpp
@@ -2,25 +2,51 @@
 
 #include <android/log.h>
 #include <jni/jni.hpp>
+#include <jni/jni.hpp>
 
 #include <mbgl/util/logging.hpp>
 #include <mbgl/test.hpp>
 
+#include <vector>
+
 #pragma clang diagnostic ignored "-Wunused-parameter"
 
 #define MAKE_NATIVE_METHOD(name, sig) jni::MakeNativeMethod<decltype(name), name>( #name, sig )
 
 namespace {
 
-/**
- * JNI Bound to Main#runAllTests()
- */
-void runAllTests(JNIEnv *env, jni::jobject* obj) {
-    mbgl::Log::Info(mbgl::Event::JNI, "Starting tests");
-    mbgl::runTests(0, nullptr);
-}
+// Main class (entry point for tests from dalvikvm)
+struct Main {
+    static constexpr auto Name() {
+        return "Main";
+    }
 
-}
+    /**
+     * JNI Bound to Main#runAllTests()
+     */
+    static void runAllTests(jni::JNIEnv& env, jni::Object<Main>, jni::Array<jni::String> args) {
+        mbgl::Log::Warning(mbgl::Event::JNI, "Starting tests");
+
+        // We need to create a copy of the argv data since Java-internals are stored in UTF-16.
+        std::vector<std::string> data;
+        // Add a fake first argument to align indices. Google Test expects the first argument to
+        // start at index 1; index 0 is the name of the executable.
+        data.push_back("main.jar");
+        const int argc = args.Length(env);
+        for (auto i = 0; i < argc; i++) {
+            data.emplace_back(jni::Make<std::string>(env, args.Get(env, i)));
+        }
+
+        // Create an array of char pointers that point back to the data array.
+        std::vector<const char*> argv;
+        for (const auto& arg : data) {
+            argv.push_back(arg.data());
+        }
+        mbgl::runTests(argv.size(), const_cast<char**>(argv.data()));
+    }
+};
+
+} // namespace
 
 // JNI Bindings to stub the android.util.Log implementation
 
@@ -47,20 +73,19 @@ static jint logger_entry_max_payload_native(JNIEnv* env, jni::jobject* clazz) {
 // Main entry point
 
 extern "C" JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) {
-    //Load the main library jni bindings
+    // Load the main library jni bindings
     mbgl::Log::Info(mbgl::Event::JNI, "Registering main JNI Methods");
     mbgl::android::registerNatives(vm);
 
-    //Load the test library jni bindings
+    // Load the test library jni bindings
     mbgl::Log::Info(mbgl::Event::JNI, "Registering test JNI Methods");
 
     jni::JNIEnv& env = jni::GetEnv(*vm, jni::jni_version_1_6);
 
-    //Main class (entry point for tests from dalvikvm)
-    struct Main { static constexpr auto Name() { return "Main"; } };
-    jni::RegisterNatives(env, jni::Class<Main>::Find(env), MAKE_NATIVE_METHOD(runAllTests, "()V"));
+    jni::RegisterNatives(env, jni::Class<Main>::Find(env),
+                         jni::MakeNativeMethod<decltype(Main::runAllTests), &Main::runAllTests>("runAllTests"));
 
-    //Bindings for system classes
+    // Bindings for system classes
     struct Log { static constexpr auto Name() { return "android/util/Log"; } };
     try {
         jni::RegisterNatives(env, jni::Class<Log>::Find(env),
diff --git a/platform/android/tests/README.md b/platform/android/tests/README.md
index 69e75b693b..8c4d29e831 100644
--- a/platform/android/tests/README.md
+++ b/platform/android/tests/README.md
@@ -3,7 +3,8 @@
 ## Testing
 We currently support the following types of testing on the Mapbox Android SDK:
 
- - [Unit tests](https://github.com/mapbox/mapbox-gl-native/blob/3447-Add-test-documentation/platform/android/tests/docs/UNIT_TESTS.md) using [JUnit](http://developer.android.com/tools/testing-support-library/index.html#AndroidJUnitRunner)
- - [UI tests](https://github.com/mapbox/mapbox-gl-native/blob/3447-Add-test-documentation/platform/android/tests/docs/UI_TESTS.md) using [Espresso](http://developer.android.com/tools/testing-support-library/index.html#Espresso)
- - [Exerciser](https://github.com/mapbox/mapbox-gl-native/blob/3447-Add-test-documentation/platform/android/tests/docs/EXERCISER_TESTS.md) tests using [Monkey](http://developer.android.com/tools/help/monkey.html) or [Build-in Fuzz test](http://docs.aws.amazon.com/devicefarm/latest/developerguide/test-types-built-in-fuzz.html)
- - [Performance tests](https://github.com/mapbox/mapbox-gl-native/blob/3447-Add-test-documentation/platform/android/tests/docs/PERFORMANCE_TESTS.md) using [Systrace](https://codelabs.developers.google.com/codelabs/android-perf-testing/index.html?index=..%2F..%2Fbabbq-2015&viewga=UA-68632703-1#0)
+ - [Unit tests](docs/UNIT_TESTS.md) using [JUnit](http://developer.android.com/tools/testing-support-library/index.html#AndroidJUnitRunner)
+ - [UI tests](docs/UI_TESTS.md) using [Espresso](http://developer.android.com/tools/testing-support-library/index.html#Espresso)
+ - [Exerciser](docs/EXERCISER_TESTS.md) tests using [Monkey](http://developer.android.com/tools/help/monkey.html) or [Build-in Fuzz test](http://docs.aws.amazon.com/devicefarm/latest/developerguide/test-types-built-in-fuzz.html)
+ - [Performance tests](docs/PERFORMANCE_TESTS.md) using [Systrace](https://codelabs.developers.google.com/codelabs/android-perf-testing/index.html?index=..%2F..%2Fbabbq-2015&viewga=UA-68632703-1#0)
+ - [Core tests](docs/CORE_TESTS.md) using [Google Test](https://github.com/google/googletest) to run the unit tests written for the core rendering component of Mapbox GL
\ No newline at end of file
diff --git a/platform/android/tests/docs/CORE_TESTS.md b/platform/android/tests/docs/CORE_TESTS.md
new file mode 100644
index 0000000000..b9cd7bcc76
--- /dev/null
+++ b/platform/android/tests/docs/CORE_TESTS.md
@@ -0,0 +1,21 @@
+# Core Tests
+
+Core tests run the [unit tests](../../../../test) written for the core rendering component of Mapbox GL.
+
+## Running the tests
+
+To run the tests, connect a device to your computer, or launch a virtual device. Then run
+
+```
+$ make run-android-core-test-XXX
+```
+
+with `XXX` being the architecture of the device (one of `arm-v5`, `arm-v7`, `arm-v8`, `x86`, `x86-64`,  or `mips`). This command first builds the library, creates a test program, zips up the assets required for the unit tests, deploys it on the device, runs the test and unpacks the results again on your computer.
+
+You can also run individual tests by specifying the name like this:
+
+```
+$ make run-android-core-test-XXX-TileCover.*
+```
+
+Everything after the last dash is passed as the [`--gtest_filter` argument](https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#running-a-subset-of-the-tests) to the Google Test binary.