Render test using the node render test suiteupstream/tvn-render-test-wip

23 files changed, 604 insertions, 179 deletions

diff --git a/Makefile b/Makefile
index 6ff4518603..6b9704dd33 100644
--- a/Makefile
+++ b/Makefile
@@ -592,6 +592,11 @@ android-ndk-stack-$1: platform/android/gradle/configuration.gradle
 .PHONY: run-android-render-test-$1
 run-android-render-test-$1: $(BUILD_DEPS) platform/android/gradle/configuration.gradle
 	-adb uninstall com.mapbox.mapboxsdk.testapp 2> /dev/null
+	# delete old test results
+	rm -rf platform/android/build/render-test/mapbox/
+  # copy test definitions to test app assets folder, clear old ones first
+	rm -rf platform/android/MapboxGLAndroidSDKTestApp/src/main/assets/integration
+	cp -r mapbox-gl-js/test/integration/ platform/android/MapboxGLAndroidSDKTestApp/src/main/assets
 	# run RenderTest.java to generate static map images
 	cd platform/android && $(MBGL_ANDROID_GRADLE) -Pmapbox.abis=$2 :MapboxGLAndroidSDKTestApp:connectedAndroidTest -Pandroid.testInstrumentationRunnerArguments.class="com.mapbox.mapboxsdk.testapp.render.RenderTest"
 	# pull generated images from the device
diff --git a/platform/android/.gitignore b/platform/android/.gitignore
index 3d6d48dc02..8cd5f26a89 100644
--- a/platform/android/.gitignore
+++ b/platform/android/.gitignore
@@ -32,3 +32,7 @@ MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/acti
 
 # Generated list files from code generation
 /scripts/generate-style-code.list
+
+# Input files for running render tests
+MapboxGLAndroidSDKTestApp/src/main/assets/integration/
+
diff --git a/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java
index d0e51f941f..0b1b7eec35 100644
--- a/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java
+++ b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java
@@ -2,26 +2,16 @@ package com.mapbox.mapboxsdk.http;
 
 import android.content.Context;
 import android.content.pm.PackageInfo;
+import android.content.res.AssetManager;
+import android.os.AsyncTask;
 import android.os.Build;
 import android.support.annotation.NonNull;
 import android.text.TextUtils;
 import android.util.Log;
-
 import com.mapbox.android.telemetry.TelemetryUtils;
 import com.mapbox.mapboxsdk.BuildConfig;
 import com.mapbox.mapboxsdk.Mapbox;
 import com.mapbox.mapboxsdk.constants.MapboxConstants;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.net.NoRouteToHostException;
-import java.net.ProtocolException;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-import java.util.concurrent.locks.ReentrantLock;
-
-import javax.net.ssl.SSLException;
-
 import okhttp3.Call;
 import okhttp3.Callback;
 import okhttp3.Dispatcher;
@@ -30,8 +20,21 @@ import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Response;
 import okhttp3.ResponseBody;
+import okio.BufferedSource;
+import okio.Okio;
 import timber.log.Timber;
 
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.net.NoRouteToHostException;
+import java.net.ProtocolException;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.util.concurrent.locks.ReentrantLock;
+
 import static android.util.Log.DEBUG;
 import static android.util.Log.ERROR;
 import static android.util.Log.INFO;
@@ -57,43 +60,12 @@ class HTTPRequest implements Callback {
   private HTTPRequest(long nativePtr, String resourceUrl, String etag, String modified) {
     this.nativePtr = nativePtr;
 
-    try {
-      HttpUrl httpUrl = HttpUrl.parse(resourceUrl);
-      if (httpUrl == null) {
-        log(Log.ERROR, String.format("[HTTP] Unable to parse resourceUrl %s", resourceUrl));
-      }
-
-      final String host = httpUrl.host().toLowerCase(MapboxConstants.MAPBOX_LOCALE);
-      // Don't try a request to remote server if we aren't connected
-      if (!Mapbox.isConnected() && !host.equals("127.0.0.1") && !host.equals("localhost")) {
-        throw new NoRouteToHostException("No Internet connection available.");
-      }
-
-      if (host.equals("mapbox.com") || host.endsWith(".mapbox.com") || host.equals("mapbox.cn")
-        || host.endsWith(".mapbox.cn")) {
-        if (httpUrl.querySize() == 0) {
-          resourceUrl = resourceUrl + "?";
-        } else {
-          resourceUrl = resourceUrl + "&";
-        }
-        resourceUrl = resourceUrl + "events=true";
-      }
-
-      Request.Builder builder = new Request.Builder()
-        .url(resourceUrl)
-        .tag(resourceUrl.toLowerCase(MapboxConstants.MAPBOX_LOCALE))
-        .addHeader("User-Agent", getUserAgent());
-      if (etag.length() > 0) {
-        builder = builder.addHeader("If-None-Match", etag);
-      } else if (modified.length() > 0) {
-        builder = builder.addHeader("If-Modified-Since", modified);
-      }
-      Request request = builder.build();
-      call = client.newCall(request);
-      call.enqueue(this);
-    } catch (Exception exception) {
-      handleFailure(call, exception);
+    if (resourceUrl.startsWith("local://")) {
+      // used by render test to serve files from assets
+      executeLocalRequest(resourceUrl);
+      return;
     }
+    executeRequest(resourceUrl, etag, modified);
   }
 
   public void cancel() {
@@ -178,6 +150,57 @@ class HTTPRequest implements Callback {
     return dispatcher;
   }
 
+  private void executeRequest(String resourceUrl, String etag, String modified){
+    try {
+      HttpUrl httpUrl = HttpUrl.parse(resourceUrl);
+      if (httpUrl == null) {
+        log(Log.ERROR, String.format("[HTTP] Unable to parse resourceUrl %s", resourceUrl));
+      }
+
+      final String host = httpUrl.host().toLowerCase(MapboxConstants.MAPBOX_LOCALE);
+      // Don't try a request to remote server if we aren't connected
+      if (!Mapbox.isConnected() && !host.equals("127.0.0.1") && !host.equals("localhost")) {
+        throw new NoRouteToHostException("No Internet connection available.");
+      }
+
+      if (host.equals("mapbox.com") || host.endsWith(".mapbox.com") || host.equals("mapbox.cn")
+        || host.endsWith(".mapbox.cn")) {
+        if (httpUrl.querySize() == 0) {
+          resourceUrl = resourceUrl + "?";
+        } else {
+          resourceUrl = resourceUrl + "&";
+        }
+        resourceUrl = resourceUrl + "events=true";
+      }
+
+      Request.Builder builder = new Request.Builder()
+        .url(resourceUrl)
+        .tag(resourceUrl.toLowerCase(MapboxConstants.MAPBOX_LOCALE))
+        .addHeader("User-Agent", getUserAgent());
+      if (etag.length() > 0) {
+        builder = builder.addHeader("If-None-Match", etag);
+      } else if (modified.length() > 0) {
+        builder = builder.addHeader("If-Modified-Since", modified);
+      }
+      Request request = builder.build();
+      call = client.newCall(request);
+      call.enqueue(this);
+    } catch (Exception exception) {
+      handleFailure(call, exception);
+    }
+  }
+
+  private void executeLocalRequest(String resourceUrl){
+    new LocalRequestTask(new LocalRequestTask.OnLocalRequestResponse() {
+      @Override
+      public void onResponse(byte[] bytes) {
+        if(bytes!=null) {
+          nativeOnResponse(200, null, null, null, null, null, null, bytes);
+        }
+      }
+    }).execute(resourceUrl);
+  }
+
   private void handleFailure(Call call, Exception e) {
     String errorMessage = e.getMessage() != null ? e.getMessage() : "Error processing the request";
     int type = getFailureType(e);
diff --git a/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java
new file mode 100644
index 0000000000..43d5870ace
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java
@@ -0,0 +1,53 @@
+package com.mapbox.mapboxsdk.http;
+
+import android.content.res.AssetManager;
+import android.os.AsyncTask;
+import com.mapbox.mapboxsdk.Mapbox;
+import timber.log.Timber;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+class LocalRequestTask extends AsyncTask<String, Void, byte[]> {
+
+  private OnLocalRequestResponse requestResponse;
+
+  LocalRequestTask(OnLocalRequestResponse requestResponse) {
+    this.requestResponse = requestResponse;
+  }
+
+  @Override
+  protected byte[] doInBackground(String... strings) {
+    try {
+      return loadFile(Mapbox.getApplicationContext().getAssets(),
+        "integration/" + strings[0]
+          .substring(8)
+          .replaceAll("%20", " ")
+          .replaceAll("%2c", ","));
+    } catch (IOException exception) {
+      Timber.e(exception);
+    }
+    return null;
+  }
+
+  @Override
+  protected void onPostExecute(byte[] bytes) {
+    super.onPostExecute(bytes);
+    if (bytes != null && requestResponse != null) {
+      requestResponse.onResponse(bytes);
+    }
+  }
+
+  private static byte[] loadFile(AssetManager assets, String path) throws IOException {
+    InputStream input = assets.open(path);
+    int size = input.available();
+    byte[] buffer = new byte[size];
+    input.read(buffer);
+    input.close();
+    return buffer;
+  }
+
+  public interface OnLocalRequestResponse {
+    void onResponse(byte[] bytes);
+  }
+}
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java
index 68957aab94..21504b8f99 100644
--- a/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java
@@ -40,6 +40,7 @@ public class RenderTest {
 
   @Before
   public void beforeTest() {
+    IdlingPolicies.setMasterPolicyTimeout(30, TimeUnit.MINUTES);
     grantWriteRuntimePermission();
     setupIdlingResource();
   }
@@ -58,7 +59,7 @@ public class RenderTest {
   private void setupIdlingResource() {
     try {
       Timber.e("@Before test: register idle resource");
-      IdlingPolicies.setIdlingResourceTimeout(2, TimeUnit.MINUTES);
+      IdlingPolicies.setIdlingResourceTimeout(30, TimeUnit.MINUTES);
       Espresso.registerIdlingResources(idlingResource = new SnapshotterIdlingResource(rule.getActivity()));
     } catch (IdlingResourceTimeoutException idlingResourceTimeoutException) {
       throw new RuntimeException("Idling out!");
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml b/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml
index 7123c151e9..d72a568a04 100644
--- a/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml
@@ -793,6 +793,7 @@
             android:screenOrientation="portrait" />
         <activity
             android:name=".activity.render.RenderTestActivity"
+            android:label="@string/activity_heatmaplayer"
             android:screenOrientation="portrait"/>
         <!-- Configuration Settings -->
         <meta-data
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java
index f6bc318f9c..411d3382e8 100644
--- a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java
@@ -1,164 +1,305 @@
 package com.mapbox.mapboxsdk.testapp.activity.render;
 
+import android.content.res.AssetManager;
 import android.graphics.Bitmap;
+import android.os.AsyncTask;
 import android.os.Bundle;
 import android.os.Environment;
 import android.support.annotation.Nullable;
 import android.support.v7.app.AppCompatActivity;
 import android.view.Gravity;
+import android.view.ViewGroup;
 import android.widget.FrameLayout;
 import android.widget.ImageView;
-
 import com.google.gson.Gson;
-import com.mapbox.mapboxsdk.maps.MapboxMap;
-import com.mapbox.mapboxsdk.snapshotter.MapSnapshot;
 import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
+import okio.BufferedSource;
+import okio.Okio;
+import timber.log.Timber;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
+import java.io.InputStream;
+import java.lang.ref.WeakReference;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+/**
+ * Activity that generates map snapshots based on the node render test suite.
+ */
 public class RenderTestActivity extends AppCompatActivity {
 
-  private final Map<RenderTestDefinition, Bitmap> renderResultMap = new HashMap<>();
-  private final List<MapSnapshotter> mapSnapshotterList = new ArrayList<>();
+  private static final String RENDER_TEST_BASE_PATH = "integration/render-tests";
 
-  private ImageView imageView;
+  // TODO read out excluded tests from /platform/node/test/ignore.json
+  private static final List<String> EXCLUDED_TESTS = new ArrayList<String>() {{
+    add("overlay,background-opacity");
+    add("collision-lines-pitched,debug");
+    add("1024-circle,extent");
+    add("empty,empty");
+    add("rotation-alignment-map,icon-pitch-scaling");
+    add("rotation-alignment-viewport,icon-pitch-scaling");
+    add("pitch15,line-pitch");
+    add("pitch30,line-pitch");
+    add("line-placement-true-pitched,text-keep-upright");
+    add("180,raster-rotation");
+    add("45,raster-rotation");
+    add("90,raster-rotation");
+    add("mapbox-gl-js#5631,regressions"); // crashes
+    add("overlapping,raster-masking");
+    add("missing,raster-loading");
+    add("pitchAndBearing,line-pitch");
+  }};
+
+  private final Map<RenderTestDefinition, Bitmap> renderResultMap = new HashMap<>();
+  private List<RenderTestDefinition> renderTestDefinitions;
   private OnSnapshotReadyListener onSnapshotReadyListener;
+  private MapSnapshotter mapSnapshotter;
+  private ImageView imageView;
 
   @Override
   protected void onCreate(@Nullable Bundle savedInstanceState) {
     super.onCreate(savedInstanceState);
     setContentView(imageView = new ImageView(RenderTestActivity.this));
-    imageView.setLayoutParams(new FrameLayout.LayoutParams(512, 512, Gravity.CENTER));
-    try {
-      RenderTestDefinition[] renderTestDefinitions = createRenderTestDefinition();
-      renderTests(renderTestDefinitions);
-    } catch (IOException exception) {
-      throw new RuntimeException(exception);
-    }
+    imageView.setLayoutParams(new FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT, Gravity.CENTER));
+    new LoadRenderDefinitionTask(this).execute();
   }
 
-  private RenderTestDefinition[] createRenderTestDefinition() throws IOException {
-    return new Gson().fromJson(
-      new InputStreamReader(getAssets().open("render-test.json")),
-      RenderTestDefinition[].class
-    );
-  }
-
-  private void renderTests(RenderTestDefinition[] renderTestDefinitions) {
-    for (RenderTestDefinition renderTestDefinition : renderTestDefinitions) {
-      renderTest(renderTestDefinition, renderTestDefinitions.length);
+  @Override
+  protected void onStop() {
+    super.onStop();
+    if (mapSnapshotter != null) {
+      mapSnapshotter.cancel();
     }
   }
 
-  private void renderTest(final RenderTestDefinition renderTestDefinition, final int testSize) {
-    MapSnapshotter mapSnapshotter = new MapSnapshotter(this, renderTestDefinition.toOptions());
-    mapSnapshotterList.add(mapSnapshotter);
-    mapSnapshotter.start(new MapSnapshotter.SnapshotReadyCallback() {
-      @Override
-      public void onSnapshotReady(MapSnapshot result) {
-        Bitmap snapshot = result.getBitmap();
-        imageView.setImageBitmap(snapshot);
-        renderResultMap.put(renderTestDefinition, snapshot);
-        if (renderResultMap.size() == testSize) {
-          writeResultsToDisk();
-          onSnapshotReadyListener.onSnapshotReady();
-        }
-      }
-    });
-  }
+  //
+  // Loads the render test definitions from assets folder
+  //
+  private static class LoadRenderDefinitionTask extends AsyncTask<Void, Void, List<RenderTestDefinition>> {
 
-  private void writeResultsToDisk() {
-    if (isExternalStorageWritable()) {
+    private WeakReference<RenderTestActivity> renderTestActivityWeakReference;
+
+    LoadRenderDefinitionTask(RenderTestActivity renderTestActivity) {
+      this.renderTestActivityWeakReference = new WeakReference<>(renderTestActivity);
+    }
+
+    @Override
+    protected List<RenderTestDefinition> doInBackground(Void... voids) {
+      List<RenderTestDefinition> definitions = new ArrayList<>();
+      AssetManager assetManager = renderTestActivityWeakReference.get().getAssets();
+      String[] categories = new String[0];
       try {
-        File testResultDir = createTestResultRootFolder();
-        String basePath = testResultDir.getAbsolutePath();
+        categories = assetManager.list(RENDER_TEST_BASE_PATH);
+      } catch (IOException exception) {
+        Timber.e(exception);
+      }
+      for (int counter = categories.length - 1; counter >= 0; counter--) {
+        try {
+          String[] tests = assetManager.list(String.format("%s/%s", RENDER_TEST_BASE_PATH, categories[counter]));
+          for (String test : tests) {
+            String styleJson = null;
+            try {
+              styleJson = loadStyleJson(assetManager, categories[counter], test);
+            } catch (IOException exception) {
+              Timber.e(exception);
+            }
 
-        for (Map.Entry<RenderTestDefinition, Bitmap> testResult : renderResultMap.entrySet()) {
-          String testName = testResult.getKey().getName();
-          String testDir = createTestDirectory(basePath, testName);
-          writeTestResultToDisk(testDir, testResult.getValue());
-        }
-      } catch (final Exception exception) {
-        imageView.post(new Runnable() {
-          @Override
-          public void run() {
-            throw new RuntimeException(exception);
+            RenderTestStyleDefinition renderTestStyleDefinition = new Gson().fromJson(styleJson, RenderTestStyleDefinition.class);
+            RenderTestDefinition definition = new RenderTestDefinition(categories[counter], test, styleJson, renderTestStyleDefinition);
+            if (!definition.hasOperations()) {
+              if (!definition.getCategory().equals("combinations") && !EXCLUDED_TESTS.contains(definition.getName() + "," + definition.getCategory())) {
+                definitions.add(definition);
+              }
+            } else {
+              Timber.e("could not add test, test requires operations: %s from %s", test, categories[counter]);
+            }
           }
-        });
+        } catch (Exception exception) {
+          Timber.e(exception);
+        }
       }
+      return definitions;
     }
-  }
 
-  private File createTestResultRootFolder() {
-    File testResultDir = new File(Environment.getExternalStorageDirectory() + File.separator + "mapbox");
-    if (testResultDir.exists()) {
-      // cleanup old files
-      deleteRecursive(testResultDir);
+    @Override
+    protected void onPostExecute(List<RenderTestDefinition> renderTestDefinitions) {
+      super.onPostExecute(renderTestDefinitions);
+      RenderTestActivity renderTestActivity = renderTestActivityWeakReference.get();
+      if (renderTestActivity != null) {
+        renderTestActivity.startRenderTests(renderTestDefinitions);
+      }
     }
 
-    if (!testResultDir.mkdirs()) {
-      throw new RuntimeException("can't create root test directory");
+    private static String loadStyleJson(AssetManager assets, String category, String test) throws IOException {
+      InputStream input = assets.open(String.format("%s/%s/%s/style.json", RENDER_TEST_BASE_PATH, category, test));
+      BufferedSource source = Okio.buffer(Okio.source(input));
+      String output = source.readByteString().string(Charset.forName("utf-8"));
+      input.close();
+      return output;
     }
-    return testResultDir;
   }
 
-  private void deleteRecursive(File fileOrDirectory) {
-    if (fileOrDirectory.isDirectory()) {
-      File[] files = fileOrDirectory.listFiles();
-      if (files != null) {
-        for (File file : files) {
-          deleteRecursive(file);
-        }
-      }
-    }
-
-    if (!fileOrDirectory.delete()) {
-      throw new RuntimeException("can't delete directory");
+  private void startRenderTests(List<RenderTestDefinition> renderTestDefinitions) {
+    this.renderTestDefinitions = renderTestDefinitions;
+    if (!renderTestDefinitions.isEmpty()) {
+      render(renderTestDefinitions.get(0), renderTestDefinitions.size());
     }
   }
 
-  private String createTestDirectory(String basePath, String testName) {
-    File testDir = new File(basePath + "/" + testName);
-    if (!testDir.exists()) {
-      if (!testDir.mkdir()) {
-        throw new RuntimeException("can't create sub directory for " + testName);
+  private void render(final RenderTestDefinition renderTestDefinition, final int testSize) {
+    Timber.d("Render test %s,%s", renderTestDefinition.getName(), renderTestDefinition.getCategory());
+    mapSnapshotter = new RenderTestSnapshotter(this, renderTestDefinition.toOptions());
+    mapSnapshotter.start(result -> {
+      Bitmap snapshot = result.getBitmap();
+      imageView.setImageBitmap(snapshot);
+      renderResultMap.put(renderTestDefinition, snapshot);
+      if (renderResultMap.size() != testSize) {
+        continueTesting(renderTestDefinition);
+      } else {
+        finishTesting();
       }
-    }
-    return testDir.getAbsolutePath();
+    }, error -> Timber.e(error));
   }
 
-  private void writeTestResultToDisk(String testPath, Bitmap testResult) throws IOException {
-    String filePath = testPath + "/actual.png";
-    FileOutputStream out = new FileOutputStream(filePath);
-    testResult.compress(Bitmap.CompressFormat.PNG, 100, out);
-    out.flush();
-    out.close();
+  private void continueTesting(RenderTestDefinition renderTestDefinition) {
+    int next = renderTestDefinitions.indexOf(renderTestDefinition) + 1;
+    Timber.d("Next test: %s / %s", next, renderTestDefinitions.size());
+    render(renderTestDefinitions.get(next), renderTestDefinitions.size());
   }
 
-  @Override
-  protected void onStop() {
-    super.onStop();
-    for (MapSnapshotter snapshotter : mapSnapshotterList) {
-      snapshotter.cancel();
+  private void finishTesting() {
+    new SaveResultToDiskTask(onSnapshotReadyListener, renderResultMap).execute();
+  }
+
+  //
+  // Save tests results to disk
+  //
+  private static class SaveResultToDiskTask extends AsyncTask<Void, Void, Void> {
+
+    private OnSnapshotReadyListener onSnapshotReadyListener;
+    private Map<RenderTestDefinition, Bitmap> renderResultMap;
+
+    SaveResultToDiskTask(OnSnapshotReadyListener onSnapshotReadyListener, Map<RenderTestDefinition, Bitmap> renderResultMap) {
+      this.onSnapshotReadyListener = onSnapshotReadyListener;
+      this.renderResultMap = renderResultMap;
+    }
+
+    @Override
+    protected Void doInBackground(Void... voids) {
+      if (isExternalStorageWritable()) {
+        try {
+          File testResultDir = FileUtils.createTestResultRootFolder();
+          String basePath = testResultDir.getAbsolutePath();
+          for (Map.Entry<RenderTestDefinition, Bitmap> testResult : renderResultMap.entrySet()) {
+            writeResultToDisk(basePath, testResult);
+          }
+        } catch (final Exception exception) {
+          Timber.e(exception);
+        }
+      }
+      return null;
+    }
+
+    private void writeResultToDisk(String basePath, Map.Entry<RenderTestDefinition, Bitmap> testResult) throws IOException {
+      RenderTestDefinition definition = testResult.getKey();
+      String categoryName = definition.getCategory();
+      String categoryPath = String.format("%s/%s", basePath, categoryName);
+      FileUtils.createCategoryDirectory(categoryPath);
+      String testName = testResult.getKey().getName();
+      String testDir = FileUtils.createTestDirectory(categoryPath, testName);
+      FileUtils.writeTestResultToDisk(testDir, testResult.getValue());
+    }
+
+    private boolean isExternalStorageWritable() {
+      return Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState());
+    }
+
+    @Override
+    protected void onPostExecute(Void aVoid) {
+      super.onPostExecute(aVoid);
+      if (onSnapshotReadyListener != null) {
+        onSnapshotReadyListener.onSnapshotReady();
+      }
     }
   }
 
-  private boolean isExternalStorageWritable() {
-    return Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState());
+  //
+  // Callback configuration to notify test executor of test finishing
+  //
+  public interface OnSnapshotReadyListener {
+    void onSnapshotReady();
   }
 
   public void setOnSnapshotReadyListener(OnSnapshotReadyListener listener) {
     this.onSnapshotReadyListener = listener;
   }
 
-  public interface OnSnapshotReadyListener {
-    void onSnapshotReady();
+  //
+  // FileUtils
+  //
+
+  private static class FileUtils {
+
+    private static void createCategoryDirectory(String catPath) {
+      File testResultDir = new File(catPath);
+      if (testResultDir.exists()) {
+        return;
+      }
+
+      if (!testResultDir.mkdirs()) {
+        throw new RuntimeException("can't create root test directory");
+      }
+    }
+
+    private static File createTestResultRootFolder() {
+      File testResultDir = new File(Environment.getExternalStorageDirectory() + File.separator + "mapbox");
+      if (testResultDir.exists()) {
+        // cleanup old files
+        deleteRecursive(testResultDir);
+      }
+
+      if (!testResultDir.mkdirs()) {
+        throw new RuntimeException("can't create root test directory");
+      }
+      return testResultDir;
+    }
+
+    private static void deleteRecursive(File fileOrDirectory) {
+      if (fileOrDirectory.isDirectory()) {
+        File[] files = fileOrDirectory.listFiles();
+        if (files != null) {
+          for (File file : files) {
+            deleteRecursive(file);
+          }
+        }
+      }
+
+      if (!fileOrDirectory.delete()) {
+        Timber.e("can't delete directory");
+      }
+    }
+
+    private static String createTestDirectory(String basePath, String testName) {
+      File testDir = new File(basePath + "/" + testName);
+      if (!testDir.exists()) {
+        if (!testDir.mkdir()) {
+          throw new RuntimeException("can't create sub directory for " + testName);
+        }
+      }
+      return testDir.getAbsolutePath();
+    }
+
+    private static void writeTestResultToDisk(String testPath, Bitmap testResult) throws IOException {
+      String filePath = testPath + "/actual.png";
+      FileOutputStream out = new FileOutputStream(filePath);
+      testResult.compress(Bitmap.CompressFormat.PNG, 100, out);
+      out.flush();
+      out.close();
+    }
   }
 }
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java
index 502fb93080..72549f6143 100644
--- a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java
@@ -1,35 +1,79 @@
 package com.mapbox.mapboxsdk.testapp.activity.render;
 
-import com.mapbox.mapboxsdk.camera.CameraPosition;
-import com.mapbox.mapboxsdk.geometry.LatLng;
 import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
 
 public class RenderTestDefinition {
 
-  private String name;
-  private int width;
-  private int height;
-  private double[] center;
-  private double zoom;
-  private double tilt;
-  private double bearing;
-  private String styleUrl;
+  private final static int DEFAULT_WIDTH = 512;
+  private final static int DEFAULT_HEIGHT = 512;
+
+  private String category; // eg. background-color
+  private String name; // eg. colorSpace-hcl
+  private String styleJson;
+  private RenderTestStyleDefinition definition;
+
+  RenderTestDefinition(String category, String name, String styleJson, RenderTestStyleDefinition definition) {
+    this.category = category;
+    this.name = name;
+    this.styleJson = styleJson;
+    this.definition = definition;
+  }
 
   public String getName() {
-    return name.replaceAll(" ", "_").toLowerCase();
+    return name;
+  }
+
+  public String getCategory() {
+    return category;
+  }
+
+  public int getWidth() {
+    RenderTestStyleDefinition.Test test = getTest();
+    if (test != null) {
+      Integer testWidth = test.getWidth();
+      if (testWidth != null && testWidth > 0) {
+        return testWidth;
+      }
+    }
+    return DEFAULT_WIDTH;
+  }
+
+  public int getHeight() {
+    RenderTestStyleDefinition.Test test = getTest();
+    if (test != null) {
+      Integer testHeight = test.getHeight();
+      if (testHeight != null && testHeight > 0) {
+        return testHeight;
+      }
+    }
+    return DEFAULT_HEIGHT;
+  }
+
+  public String getStyleJson() {
+    return styleJson;
+  }
+
+  public boolean hasOperations() {
+    return getTest().getOperations() != null;
+  }
+
+  public RenderTestStyleDefinition.Test getTest() {
+    return definition.getMetadata().getTest();
   }
 
   public MapSnapshotter.Options toOptions() {
     return new MapSnapshotter
-      .Options(width, height)
-      .withStyle(styleUrl)
-      .withCameraPosition(
-        new CameraPosition.Builder()
-          .target(new LatLng(center[0], center[1]))
-          .zoom(zoom)
-          .tilt(tilt)
-          .bearing(bearing)
-          .build()
-      );
+      .Options(getWidth(), getHeight())
+      .withStyleJson(styleJson)
+      .withLogo(false);
+  }
+
+  @Override
+  public String toString() {
+    return "RenderTestDefinition{" +
+      "category='" + category + '\'' +
+      ", name='" + name + '\'' +
+      ", styleJson='" + styleJson + '\'' +
+      '}';
   }
 }
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java
new file mode 100644
index 0000000000..cb971fee70
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java
@@ -0,0 +1,18 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import android.content.Context;
+import android.support.annotation.NonNull;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshot;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
+
+public class RenderTestSnapshotter extends MapSnapshotter {
+
+  RenderTestSnapshotter(@NonNull Context context, @NonNull Options options) {
+    super(context, options);
+  }
+
+  @Override
+  protected void addOverlay(MapSnapshot mapSnapshot) {
+    // don't add an overlay
+  }
+}
diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java
new file mode 100644
index 0000000000..fdd7e9aaf1
--- /dev/null
+++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java
@@ -0,0 +1,127 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RenderTestStyleDefinition {
+
+  private Integer version;
+  private Metadata metadata;
+  private Map<String, Object> additionalProperties = new HashMap<String, Object>();
+
+  public Integer getVersion() {
+    return version;
+  }
+
+  public void setVersion(Integer version) {
+    this.version = version;
+  }
+
+  public Metadata getMetadata() {
+    return metadata;
+  }
+
+  public void setMetadata(Metadata metadata) {
+    this.metadata = metadata;
+  }
+
+  public Map<String, Object> getAdditionalProperties() {
+    return this.additionalProperties;
+  }
+
+  public void setAdditionalProperty(String name, Object value) {
+    this.additionalProperties.put(name, value);
+  }
+
+
+  public class Metadata {
+
+    private Test test;
+    private Map<String, Object> additionalProperties = new HashMap<String, Object>();
+
+    public Test getTest() {
+      return test;
+    }
+
+    public void setTest(Test test) {
+      this.test = test;
+    }
+
+    public Map<String, Object> getAdditionalProperties() {
+      return this.additionalProperties;
+    }
+
+    public void setAdditionalProperty(String name, Object value) {
+      this.additionalProperties.put(name, value);
+    }
+  }
+
+  public class Test {
+
+    private Integer width;
+    private Integer height;
+    private List<Integer> center = null;
+    private Integer zoom;
+    private Double diff;
+    private List<List<String>> operations = null;
+    private Map<String, Object> additionalProperties = new HashMap<String, Object>();
+
+    public Integer getWidth() {
+      return width;
+    }
+
+    public void setWidth(Integer width) {
+      this.width = width;
+    }
+
+    public Integer getHeight() {
+      return height;
+    }
+
+    public void setHeight(Integer height) {
+      this.height = height;
+    }
+
+    public List<Integer> getCenter() {
+      return center;
+    }
+
+    public void setCenter(List<Integer> center) {
+      this.center = center;
+    }
+
+    public Integer getZoom() {
+      return zoom;
+    }
+
+    public void setZoom(Integer zoom) {
+      this.zoom = zoom;
+    }
+
+    public Double getDiff() {
+      return diff;
+    }
+
+    public void setDiff(Double diff) {
+      this.diff = diff;
+    }
+
+    public List<List<String>> getOperations() {
+      return operations;
+    }
+
+    public void setOperations(List<List<String>> operations) {
+      this.operations = operations;
+    }
+
+    public Map<String, Object> getAdditionalProperties() {
+      return this.additionalProperties;
+    }
+
+    public void setAdditionalProperty(String name, Object value) {
+      this.additionalProperties.put(name, value);
+    }
+
+  }
+}
\ No newline at end of file
diff --git a/platform/android/scripts/run-render-test.py b/platform/android/scripts/run-render-test.py
index 34b145c7fe..0027cb362e 100644
--- a/platform/android/scripts/run-render-test.py
+++ b/platform/android/scripts/run-render-test.py
@@ -5,15 +5,23 @@ from shutil import copyfile
 
 path = os.getcwd() + "/platform/android/tests/render/expected/"
 dirs = os.listdir(path)
-testDir = os.getcwd() + "/platform/android/build/render-test/"
-for test in dirs:
-    src = os.getcwd() + "/platform/android/tests/render/expected/" + test + "/expected.png"
-    expected = testDir + test + "/expected.png"
-    actual = testDir + test + "/actual.png"
-    output = testDir + test +"/output.png"
-    copyfile(src, expected)
-    pixelmatch = "node_modules/pixelmatch/bin/pixelmatch " + actual + " " + expected + " " + output + " 0.1"
-    print
-    print "Pixel match "+ test
-    os.system(pixelmatch)
+
+catPath = os.getcwd() + "/platform/android/build/render-test/mapbox/"
+catDir = os.listdir(catPath)
+
+for cat in catDir:
+    testPath = catPath + cat + "/"
+    testDir = os.listdir(testPath)
+    for test in testDir:
+        src = os.getcwd() + "/mapbox-gl-js/test/integration/render-tests/"+cat+"/"+ test + "/expected.png"
+        json = os.getcwd() + "/mapbox-gl-js/test/integration/render-tests/"+cat+"/"+ test + "/style.json"
+        expected = testPath + test + "/expected.png"
+        actual = testPath + test + "/actual.png"
+        output = testPath + test +"/output.png"
+        copyfile(src, expected)
+        copyfile(json, testPath + test + "/style.json")
+        pixelmatch = "node_modules/pixelmatch/bin/pixelmatch " + actual + " " + expected + " " + output + " 0.1"
+        print
+        print "Pixel match "+ cat + " " + test
+        os.system(pixelmatch)
 print
diff --git a/platform/android/tests/render/expected/ayacucho/expected.png b/platform/android/tests/render/expected/ayacucho/expected.png
deleted file mode 100644
index 1c7d02960f..0000000000
--- a/platform/android/tests/render/expected/ayacucho/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/background-color/colorSpace-hcl/expected.png b/platform/android/tests/render/expected/background-color/colorSpace-hcl/expected.png
new file mode 100644
index 0000000000..1e701f28bd
--- /dev/null
+++ b/platform/android/tests/render/expected/background-color/colorSpace-hcl/expected.png
diff --git a/platform/android/tests/render/expected/bangalore/expected.png b/platform/android/tests/render/expected/bangalore/expected.png
deleted file mode 100644
index 8373974cf9..0000000000
--- a/platform/android/tests/render/expected/bangalore/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/berlin/expected.png b/platform/android/tests/render/expected/berlin/expected.png
deleted file mode 100644
index 8b4d919492..0000000000
--- a/platform/android/tests/render/expected/berlin/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/brussels/expected.png b/platform/android/tests/render/expected/brussels/expected.png
deleted file mode 100644
index 335e526dae..0000000000
--- a/platform/android/tests/render/expected/brussels/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/helsinki/expected.png b/platform/android/tests/render/expected/helsinki/expected.png
deleted file mode 100644
index 607a48600c..0000000000
--- a/platform/android/tests/render/expected/helsinki/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/lisbon/expected.png b/platform/android/tests/render/expected/lisbon/expected.png
deleted file mode 100644
index 27b0d378db..0000000000
--- a/platform/android/tests/render/expected/lisbon/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/london/expected.png b/platform/android/tests/render/expected/london/expected.png
deleted file mode 100644
index 2df27bfe74..0000000000
--- a/platform/android/tests/render/expected/london/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/madrid/expected.png b/platform/android/tests/render/expected/madrid/expected.png
deleted file mode 100644
index 129b9889fe..0000000000
--- a/platform/android/tests/render/expected/madrid/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/san_francisco/expected.png b/platform/android/tests/render/expected/san_francisco/expected.png
deleted file mode 100644
index 3bbe82d70b..0000000000
--- a/platform/android/tests/render/expected/san_francisco/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/washington_dc/expected.png b/platform/android/tests/render/expected/washington_dc/expected.png
deleted file mode 100644
index f4bdfb9b05..0000000000
--- a/platform/android/tests/render/expected/washington_dc/expected.png
+++ /dev/null
diff --git a/platform/android/tests/render/expected/world/expected.png b/platform/android/tests/render/expected/world/expected.png
deleted file mode 100644
index 7f6c7ef211..0000000000
--- a/platform/android/tests/render/expected/world/expected.png
+++ /dev/null