[android] - Render tests with PixelMatch

20 files changed, 513 insertions, 0 deletions

diff --git a/Makefile b/Makefile
index c2daa617b4..6ff4518603 100644
--- a/Makefile
+++ b/Makefile
@@ -588,6 +588,17 @@ run-android-ui-test-$1-%: platform/android/gradle/configuration.gradle
 android-ndk-stack-$1: platform/android/gradle/configuration.gradle
 	adb logcat | ndk-stack -sym platform/android/MapboxGLAndroidSDK/build/intermediates/cmake/debug/obj/$2/
 
+# Run render tests with pixelmatch
+.PHONY: run-android-render-test-$1
+run-android-render-test-$1: $(BUILD_DEPS) platform/android/gradle/configuration.gradle
+	-adb uninstall com.mapbox.mapboxsdk.testapp 2> /dev/null
+	# run RenderTest.java to generate static map images
+	cd platform/android && $(MBGL_ANDROID_GRADLE) -Pmapbox.abis=$2 :MapboxGLAndroidSDKTestApp:connectedAndroidTest -Pandroid.testInstrumentationRunnerArguments.class="com.mapbox.mapboxsdk.testapp.render.RenderTest"
+	# pull generated images from the device
+	adb pull "`adb shell 'printenv EXTERNAL_STORAGE' | tr -d '\r'`/mapbox" platform/android/build/render-test
+	# copy expected result and run pixelmatch
+	python platform/android/scripts/run-render-test.py
+
 endef
 
 # Explodes the arguments into individual variables
@@ -680,6 +691,11 @@ android-javadoc: platform/android/gradle/configuration.gradle
 .PHONY: android-ndk-stack
 android-ndk-stack: android-ndk-stack-arm-v7
 
+# Update render tests
+,PHONY: update-android-render-test
+update-android-render-test:
+	python platform/android/scripts/update-render-expected.py
+
 # Open Android Studio if machine is macos
 ifeq ($(HOST_PLATFORM), macos)
 .PHONY: aproj
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java
new file mode 100644
index 0000000000..68957aab94
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java
@@ -0,0 +1,78 @@
+package com.mapbox.mapboxsdk.testapp.render;
+
+import android.os.Build;
+import android.support.test.espresso.Espresso;
+import android.support.test.espresso.IdlingPolicies;
+import android.support.test.espresso.IdlingResourceTimeoutException;
+import android.support.test.rule.ActivityTestRule;
+import android.support.test.runner.AndroidJUnit4;
+
+import com.mapbox.mapboxsdk.testapp.activity.render.RenderTestActivity;
+import com.mapbox.mapboxsdk.testapp.utils.SnapshotterIdlingResource;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.util.concurrent.TimeUnit;
+
+import timber.log.Timber;
+
+import static android.support.test.InstrumentationRegistry.getInstrumentation;
+import static android.support.test.InstrumentationRegistry.getTargetContext;
+import static android.support.test.espresso.Espresso.onView;
+import static android.support.test.espresso.assertion.ViewAssertions.matches;
+import static android.support.test.espresso.matcher.ViewMatchers.isDisplayed;
+import static android.support.test.espresso.matcher.ViewMatchers.withId;
+
+/**
+ * Instrumentation render tests
+ */
+@RunWith(AndroidJUnit4.class)
+public class RenderTest {
+
+  private SnapshotterIdlingResource idlingResource;
+
+  @Rule
+  public ActivityTestRule<RenderTestActivity> rule = new ActivityTestRule<>(RenderTestActivity.class);
+
+  @Before
+  public void beforeTest() {
+    grantWriteRuntimePermission();
+    setupIdlingResource();
+  }
+
+  private void grantWriteRuntimePermission() {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+      getInstrumentation().getUiAutomation().executeShellCommand(
+        "pm grant " + getTargetContext().getPackageName()
+          + " android.permission.WRITE_EXTERNAL_STORAGE");
+      getInstrumentation().getUiAutomation().executeShellCommand(
+        "pm grant " + getTargetContext().getPackageName()
+          + " android.permission.READ_EXTERNAL_STORAGE");
+    }
+  }
+
+  private void setupIdlingResource() {
+    try {
+      Timber.e("@Before test: register idle resource");
+      IdlingPolicies.setIdlingResourceTimeout(2, TimeUnit.MINUTES);
+      Espresso.registerIdlingResources(idlingResource = new SnapshotterIdlingResource(rule.getActivity()));
+    } catch (IdlingResourceTimeoutException idlingResourceTimeoutException) {
+      throw new RuntimeException("Idling out!");
+    }
+  }
+
+  @Test
+  public void testRender() {
+    onView(withId(android.R.id.content)).check(matches(isDisplayed()));
+  }
+
+  @After
+  public void afterTest() {
+    Timber.e("@After test: unregister idle resource");
+    Espresso.unregisterIdlingResources(idlingResource);
+  }
+}
\ No newline at end of file
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java
new file mode 100644
index 0000000000..0aae17f26a
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java
@@ -0,0 +1,38 @@
+package com.mapbox.mapboxsdk.testapp.utils;
+
+import android.support.test.espresso.IdlingResource;
+
+import com.mapbox.mapboxsdk.testapp.activity.render.RenderTestActivity;
+
+public class SnapshotterIdlingResource implements IdlingResource, RenderTestActivity.OnSnapshotReadyListener {
+
+  private IdlingResource.ResourceCallback resourceCallback;
+  private boolean isSnapshotReady;
+
+  public SnapshotterIdlingResource(RenderTestActivity activity) {
+    activity.setOnSnapshotReadyListener(this);
+  }
+
+  @Override
+  public String getName() {
+    return "SnapshotterIdlingResource";
+  }
+
+  @Override
+  public boolean isIdleNow() {
+    return isSnapshotReady;
+  }
+
+  @Override
+  public void registerIdleTransitionCallback(ResourceCallback resourceCallback) {
+    this.resourceCallback = resourceCallback;
+  }
+
+  @Override
+  public void onSnapshotReady() {
+    isSnapshotReady = true;
+    if (resourceCallback != null) {
+      resourceCallback.onTransitionToIdle();
+    }
+  }
+}
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml b/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml
index b78fba0aae..7123c151e9 100644
--- a/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml
@@ -791,6 +791,9 @@
         <activity
             android:name=".activity.style.FillExtrusionStyleTestActivity"
             android:screenOrientation="portrait" />
+        <activity
+            android:name=".activity.render.RenderTestActivity"
+            android:screenOrientation="portrait"/>
         <!-- Configuration Settings -->
         <meta-data
             android:name="com.mapbox.TestEventsServer"
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/assets/render-test.json b/platform/android/MapboxGLAndroidSDKTestApp/src/main/assets/render-test.json
new file mode 100644
index 0000000000..9902b7db7c
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/assets/render-test.json
@@ -0,0 +1,145 @@
+[
+  {
+    "name": "World",
+    "width": 512,
+    "height": 512,
+    "center": [
+      0,
+      0
+    ],
+    "zoom": 0,
+    "tilt": 0,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "Brussels",
+    "width": 512,
+    "height": 512,
+    "center": [
+      50.846728,
+      4.352429
+    ],
+    "zoom": 12,
+    "tilt": 0,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "San Francisco",
+    "width": 512,
+    "height": 512,
+    "center": [
+      37.772544,
+      -122.426622
+    ],
+    "zoom": 9,
+    "tilt": 60,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "Washington DC",
+    "width": 512,
+    "height": 512,
+    "center": [
+      38.897717,
+      -77.036035
+    ],
+    "zoom": 18,
+    "tilt": 20,
+    "bearing": 180,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "Ayacucho",
+    "width": 512,
+    "height": 512,
+    "center": [
+      -13.164131,
+      -74.220973
+    ],
+    "zoom": 12,
+    "tilt": 30,
+    "bearing": 270,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "Bangalore",
+    "width": 512,
+    "height": 512,
+    "center": [
+      12.970020,
+      77.590421
+    ],
+    "zoom": 14,
+    "tilt": 0,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "Berlin",
+    "width": 512,
+    "height": 512,
+    "center": [
+      52.519716,
+      13.402068
+    ],
+    "zoom": 12,
+    "tilt": 0,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "Lisbon",
+    "width": 512,
+    "height": 512,
+    "center": [
+      38.722062,
+      -9.138746
+    ],
+    "zoom": 15,
+    "tilt": 45,
+    "bearing": 90,
+    "styleUrl": "mapbox://styles/mapbox/outdoors-v10"
+  },
+  {
+    "name": "Helsinki",
+    "width": 512,
+    "height": 512,
+    "center": [
+      60.171283,
+      24.933533
+    ],
+    "zoom": 12,
+    "tilt": 0,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/streets-v10"
+  },
+  {
+    "name": "Madrid",
+    "width": 512,
+    "height": 512,
+    "center": [
+      40.417025,
+      -3.705287
+    ],
+    "zoom": 16,
+    "tilt": 35,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/light-v9"
+  },
+  {
+    "name": "London",
+    "width": 512,
+    "height": 512,
+    "center": [
+      51.508049,
+      -0.129756
+    ],
+    "zoom": 12,
+    "tilt": 0,
+    "bearing": 0,
+    "styleUrl": "mapbox://styles/mapbox/dark-v9"
+  }
+]
\ No newline at end of file
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java
new file mode 100644
index 0000000000..f6bc318f9c
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java
@@ -0,0 +1,164 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import android.graphics.Bitmap;
+import android.os.Bundle;
+import android.os.Environment;
+import android.support.annotation.Nullable;
+import android.support.v7.app.AppCompatActivity;
+import android.view.Gravity;
+import android.widget.FrameLayout;
+import android.widget.ImageView;
+
+import com.google.gson.Gson;
+import com.mapbox.mapboxsdk.maps.MapboxMap;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshot;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RenderTestActivity extends AppCompatActivity {
+
+  private final Map<RenderTestDefinition, Bitmap> renderResultMap = new HashMap<>();
+  private final List<MapSnapshotter> mapSnapshotterList = new ArrayList<>();
+
+  private ImageView imageView;
+  private OnSnapshotReadyListener onSnapshotReadyListener;
+
+  @Override
+  protected void onCreate(@Nullable Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(imageView = new ImageView(RenderTestActivity.this));
+    imageView.setLayoutParams(new FrameLayout.LayoutParams(512, 512, Gravity.CENTER));
+    try {
+      RenderTestDefinition[] renderTestDefinitions = createRenderTestDefinition();
+      renderTests(renderTestDefinitions);
+    } catch (IOException exception) {
+      throw new RuntimeException(exception);
+    }
+  }
+
+  private RenderTestDefinition[] createRenderTestDefinition() throws IOException {
+    return new Gson().fromJson(
+      new InputStreamReader(getAssets().open("render-test.json")),
+      RenderTestDefinition[].class
+    );
+  }
+
+  private void renderTests(RenderTestDefinition[] renderTestDefinitions) {
+    for (RenderTestDefinition renderTestDefinition : renderTestDefinitions) {
+      renderTest(renderTestDefinition, renderTestDefinitions.length);
+    }
+  }
+
+  private void renderTest(final RenderTestDefinition renderTestDefinition, final int testSize) {
+    MapSnapshotter mapSnapshotter = new MapSnapshotter(this, renderTestDefinition.toOptions());
+    mapSnapshotterList.add(mapSnapshotter);
+    mapSnapshotter.start(new MapSnapshotter.SnapshotReadyCallback() {
+      @Override
+      public void onSnapshotReady(MapSnapshot result) {
+        Bitmap snapshot = result.getBitmap();
+        imageView.setImageBitmap(snapshot);
+        renderResultMap.put(renderTestDefinition, snapshot);
+        if (renderResultMap.size() == testSize) {
+          writeResultsToDisk();
+          onSnapshotReadyListener.onSnapshotReady();
+        }
+      }
+    });
+  }
+
+  private void writeResultsToDisk() {
+    if (isExternalStorageWritable()) {
+      try {
+        File testResultDir = createTestResultRootFolder();
+        String basePath = testResultDir.getAbsolutePath();
+
+        for (Map.Entry<RenderTestDefinition, Bitmap> testResult : renderResultMap.entrySet()) {
+          String testName = testResult.getKey().getName();
+          String testDir = createTestDirectory(basePath, testName);
+          writeTestResultToDisk(testDir, testResult.getValue());
+        }
+      } catch (final Exception exception) {
+        imageView.post(new Runnable() {
+          @Override
+          public void run() {
+            throw new RuntimeException(exception);
+          }
+        });
+      }
+    }
+  }
+
+  private File createTestResultRootFolder() {
+    File testResultDir = new File(Environment.getExternalStorageDirectory() + File.separator + "mapbox");
+    if (testResultDir.exists()) {
+      // cleanup old files
+      deleteRecursive(testResultDir);
+    }
+
+    if (!testResultDir.mkdirs()) {
+      throw new RuntimeException("can't create root test directory");
+    }
+    return testResultDir;
+  }
+
+  private void deleteRecursive(File fileOrDirectory) {
+    if (fileOrDirectory.isDirectory()) {
+      File[] files = fileOrDirectory.listFiles();
+      if (files != null) {
+        for (File file : files) {
+          deleteRecursive(file);
+        }
+      }
+    }
+
+    if (!fileOrDirectory.delete()) {
+      throw new RuntimeException("can't delete directory");
+    }
+  }
+
+  private String createTestDirectory(String basePath, String testName) {
+    File testDir = new File(basePath + "/" + testName);
+    if (!testDir.exists()) {
+      if (!testDir.mkdir()) {
+        throw new RuntimeException("can't create sub directory for " + testName);
+      }
+    }
+    return testDir.getAbsolutePath();
+  }
+
+  private void writeTestResultToDisk(String testPath, Bitmap testResult) throws IOException {
+    String filePath = testPath + "/actual.png";
+    FileOutputStream out = new FileOutputStream(filePath);
+    testResult.compress(Bitmap.CompressFormat.PNG, 100, out);
+    out.flush();
+    out.close();
+  }
+
+  @Override
+  protected void onStop() {
+    super.onStop();
+    for (MapSnapshotter snapshotter : mapSnapshotterList) {
+      snapshotter.cancel();
+    }
+  }
+
+  private boolean isExternalStorageWritable() {
+    return Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState());
+  }
+
+  public void setOnSnapshotReadyListener(OnSnapshotReadyListener listener) {
+    this.onSnapshotReadyListener = listener;
+  }
+
+  public interface OnSnapshotReadyListener {
+    void onSnapshotReady();
+  }
+}
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java
new file mode 100644
index 0000000000..502fb93080
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java
@@ -0,0 +1,35 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import com.mapbox.mapboxsdk.camera.CameraPosition;
+import com.mapbox.mapboxsdk.geometry.LatLng;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
+
+public class RenderTestDefinition {
+
+  private String name;
+  private int width;
+  private int height;
+  private double[] center;
+  private double zoom;
+  private double tilt;
+  private double bearing;
+  private String styleUrl;
+
+  public String getName() {
+    return name.replaceAll(" ", "_").toLowerCase();
+  }
+
+  public MapSnapshotter.Options toOptions() {
+    return new MapSnapshotter
+      .Options(width, height)
+      .withStyle(styleUrl)
+      .withCameraPosition(
+        new CameraPosition.Builder()
+          .target(new LatLng(center[0], center[1]))
+          .zoom(zoom)
+          .tilt(tilt)
+          .bearing(bearing)
+          .build()
+      );
+  }
+}
diff --git a/platform/android/scripts/run-render-test.py b/platform/android/scripts/run-render-test.py
new file mode 100644
index 0000000000..34b145c7fe
--- /dev/null
+++ b/platform/android/scripts/run-render-test.py
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+
+import os
+from shutil import copyfile
+
+path = os.getcwd() + "/platform/android/tests/render/expected/"
+dirs = os.listdir(path)
+testDir = os.getcwd() + "/platform/android/build/render-test/"
+for test in dirs:
+    src = os.getcwd() + "/platform/android/tests/render/expected/" + test + "/expected.png"
+    expected = testDir + test + "/expected.png"
+    actual = testDir + test + "/actual.png"
+    output = testDir + test +"/output.png"
+    copyfile(src, expected)
+    pixelmatch = "node_modules/pixelmatch/bin/pixelmatch " + actual + " " + expected + " " + output + " 0.1"
+    print
+    print "Pixel match "+ test
+    os.system(pixelmatch)
+print
diff --git a/platform/android/scripts/update-render-expected.py b/platform/android/scripts/update-render-expected.py
new file mode 100644
index 0000000000..4f6bffe16e
--- /dev/null
+++ b/platform/android/scripts/update-render-expected.py
@@ -0,0 +1,15 @@
+#!/usr/bin/python
+
+import os
+from shutil import copyfile
+
+print "Update render test with executed output:"
+
+path = os.getcwd() + "/platform/android/tests/render/expected/"
+dirs = os.listdir(path)
+testDir = os.getcwd() + "/platform/android/build/render-test/"
+for test in dirs:
+    dst = os.getcwd() + "/platform/android/tests/render/expected/" + test + "/expected.png"
+    actual = testDir + test + "/actual.png"
+    print "Copy file from " + actual + " to " + dst 
+    copyfile(actual, dst)
diff --git a/platform/android/tests/render/expected/ayacucho/expected.png b/platform/android/tests/render/expected/ayacucho/expected.png
new file mode 100644
index 0000000000..1c7d02960f
--- /dev/null
+++ b/platform/android/tests/render/expected/ayacucho/expected.png
diff --git a/platform/android/tests/render/expected/bangalore/expected.png b/platform/android/tests/render/expected/bangalore/expected.png
new file mode 100644
index 0000000000..8373974cf9
--- /dev/null
+++ b/platform/android/tests/render/expected/bangalore/expected.png
diff --git a/platform/android/tests/render/expected/berlin/expected.png b/platform/android/tests/render/expected/berlin/expected.png
new file mode 100644
index 0000000000..8b4d919492
--- /dev/null
+++ b/platform/android/tests/render/expected/berlin/expected.png
diff --git a/platform/android/tests/render/expected/brussels/expected.png b/platform/android/tests/render/expected/brussels/expected.png
new file mode 100644
index 0000000000..335e526dae
--- /dev/null
+++ b/platform/android/tests/render/expected/brussels/expected.png
diff --git a/platform/android/tests/render/expected/helsinki/expected.png b/platform/android/tests/render/expected/helsinki/expected.png
new file mode 100644
index 0000000000..607a48600c
--- /dev/null
+++ b/platform/android/tests/render/expected/helsinki/expected.png
diff --git a/platform/android/tests/render/expected/lisbon/expected.png b/platform/android/tests/render/expected/lisbon/expected.png
new file mode 100644
index 0000000000..27b0d378db
--- /dev/null
+++ b/platform/android/tests/render/expected/lisbon/expected.png
diff --git a/platform/android/tests/render/expected/london/expected.png b/platform/android/tests/render/expected/london/expected.png
new file mode 100644
index 0000000000..2df27bfe74
--- /dev/null
+++ b/platform/android/tests/render/expected/london/expected.png
diff --git a/platform/android/tests/render/expected/madrid/expected.png b/platform/android/tests/render/expected/madrid/expected.png
new file mode 100644
index 0000000000..129b9889fe
--- /dev/null
+++ b/platform/android/tests/render/expected/madrid/expected.png
diff --git a/platform/android/tests/render/expected/san_francisco/expected.png b/platform/android/tests/render/expected/san_francisco/expected.png
new file mode 100644
index 0000000000..3bbe82d70b
--- /dev/null
+++ b/platform/android/tests/render/expected/san_francisco/expected.png
diff --git a/platform/android/tests/render/expected/washington_dc/expected.png b/platform/android/tests/render/expected/washington_dc/expected.png
new file mode 100644
index 0000000000..f4bdfb9b05
--- /dev/null
+++ b/platform/android/tests/render/expected/washington_dc/expected.png
diff --git a/platform/android/tests/render/expected/world/expected.png b/platform/android/tests/render/expected/world/expected.png
new file mode 100644
index 0000000000..7f6c7ef211
--- /dev/null
+++ b/platform/android/tests/render/expected/world/expected.png