From 3f04f13d4c190abd128b9a65ae38545984869e55 Mon Sep 17 00:00:00 2001 From: Tobrun Van Nuland Date: Thu, 21 Sep 2017 07:26:24 +0200 Subject: [android] - Render tests with PixelMatch --- Makefile | 16 ++ platform/android/.gitignore | 4 + platform/android/MapboxGLAndroidSDK/build.gradle | 4 +- .../com/mapbox/mapboxsdk/http/HTTPRequest.java | 113 ++++---- .../mapbox/mapboxsdk/http/LocalRequestTask.java | 51 ++++ .../mapboxsdk/testapp/render/RenderTest.java | 70 +++++ .../testapp/utils/SnapshotterIdlingResource.java | 38 +++ .../src/main/AndroidManifest.xml | 3 + .../testapp/activity/FeatureOverviewActivity.java | 7 +- .../activity/render/RenderTestActivity.java | 316 +++++++++++++++++++++ .../activity/render/RenderTestDefinition.java | 79 ++++++ .../activity/render/RenderTestSnapshotter.java | 18 ++ .../activity/render/RenderTestStyleDefinition.java | 127 +++++++++ platform/android/scripts/exclude-activity-gen.json | 3 +- platform/android/scripts/run-render-test.py | 24 ++ 15 files changed, 819 insertions(+), 54 deletions(-) create mode 100644 platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java create mode 100644 platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java create mode 100644 platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java create mode 100644 platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java create mode 100644 platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java create mode 100644 platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java create mode 100644 platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java create mode 100644 platform/android/scripts/run-render-test.py diff --git a/Makefile b/Makefile index c2daa617b4..8ba8084412 100644 --- a/Makefile +++ b/Makefile @@ -588,6 +588,22 @@ run-android-ui-test-$1-%: platform/android/gradle/configuration.gradle android-ndk-stack-$1: platform/android/gradle/configuration.gradle adb logcat | ndk-stack -sym platform/android/MapboxGLAndroidSDK/build/intermediates/cmake/debug/obj/$2/ +# Run render tests with pixelmatch +.PHONY: run-android-render-test-$1 +run-android-render-test-$1: $(BUILD_DEPS) platform/android/gradle/configuration.gradle + -adb uninstall com.mapbox.mapboxsdk.testapp 2> /dev/null + # delete old test results + rm -rf platform/android/build/render-test/mapbox/ + # copy test definitions to test app assets folder, clear old ones first + rm -rf platform/android/MapboxGLAndroidSDKTestApp/src/main/assets/integration + cp -r mapbox-gl-js/test/integration platform/android/MapboxGLAndroidSDKTestApp/src/main/assets + # run RenderTest.java to generate static map images + cd platform/android && $(MBGL_ANDROID_GRADLE) -Pmapbox.abis=$2 :MapboxGLAndroidSDKTestApp:connectedAndroidTest -Pandroid.testInstrumentationRunnerArguments.class="com.mapbox.mapboxsdk.testapp.render.RenderTest" + # pull generated images from the device + adb pull "`adb shell 'printenv EXTERNAL_STORAGE' | tr -d '\r'`/mapbox/render" platform/android/build/render-test + # copy expected result and run pixelmatch + python platform/android/scripts/run-render-test.py + endef # Explodes the arguments into individual variables diff --git a/platform/android/.gitignore b/platform/android/.gitignore index 3d6d48dc02..8cd5f26a89 100644 --- a/platform/android/.gitignore +++ b/platform/android/.gitignore @@ -32,3 +32,7 @@ MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/acti # Generated list files from code generation /scripts/generate-style-code.list + +# Input files for running render tests +MapboxGLAndroidSDKTestApp/src/main/assets/integration/ + diff --git a/platform/android/MapboxGLAndroidSDK/build.gradle b/platform/android/MapboxGLAndroidSDK/build.gradle index 21ed25e2c2..86919d21bc 100644 --- a/platform/android/MapboxGLAndroidSDK/build.gradle +++ b/platform/android/MapboxGLAndroidSDK/build.gradle @@ -110,8 +110,8 @@ android { } compileOptions { - sourceCompatibility JavaVersion.VERSION_1_7 - targetCompatibility JavaVersion.VERSION_1_7 + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 } lintOptions { diff --git a/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java index d0e51f941f..449e89a586 100644 --- a/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java +++ b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java @@ -6,22 +6,10 @@ import android.os.Build; import android.support.annotation.NonNull; import android.text.TextUtils; import android.util.Log; - import com.mapbox.android.telemetry.TelemetryUtils; import com.mapbox.mapboxsdk.BuildConfig; import com.mapbox.mapboxsdk.Mapbox; import com.mapbox.mapboxsdk.constants.MapboxConstants; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.net.NoRouteToHostException; -import java.net.ProtocolException; -import java.net.SocketException; -import java.net.UnknownHostException; -import java.util.concurrent.locks.ReentrantLock; - -import javax.net.ssl.SSLException; - import okhttp3.Call; import okhttp3.Callback; import okhttp3.Dispatcher; @@ -32,6 +20,15 @@ import okhttp3.Response; import okhttp3.ResponseBody; import timber.log.Timber; +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.NoRouteToHostException; +import java.net.ProtocolException; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.concurrent.locks.ReentrantLock; + import static android.util.Log.DEBUG; import static android.util.Log.ERROR; import static android.util.Log.INFO; @@ -57,43 +54,12 @@ class HTTPRequest implements Callback { private HTTPRequest(long nativePtr, String resourceUrl, String etag, String modified) { this.nativePtr = nativePtr; - try { - HttpUrl httpUrl = HttpUrl.parse(resourceUrl); - if (httpUrl == null) { - log(Log.ERROR, String.format("[HTTP] Unable to parse resourceUrl %s", resourceUrl)); - } - - final String host = httpUrl.host().toLowerCase(MapboxConstants.MAPBOX_LOCALE); - // Don't try a request to remote server if we aren't connected - if (!Mapbox.isConnected() && !host.equals("127.0.0.1") && !host.equals("localhost")) { - throw new NoRouteToHostException("No Internet connection available."); - } - - if (host.equals("mapbox.com") || host.endsWith(".mapbox.com") || host.equals("mapbox.cn") - || host.endsWith(".mapbox.cn")) { - if (httpUrl.querySize() == 0) { - resourceUrl = resourceUrl + "?"; - } else { - resourceUrl = resourceUrl + "&"; - } - resourceUrl = resourceUrl + "events=true"; - } - - Request.Builder builder = new Request.Builder() - .url(resourceUrl) - .tag(resourceUrl.toLowerCase(MapboxConstants.MAPBOX_LOCALE)) - .addHeader("User-Agent", getUserAgent()); - if (etag.length() > 0) { - builder = builder.addHeader("If-None-Match", etag); - } else if (modified.length() > 0) { - builder = builder.addHeader("If-Modified-Since", modified); - } - Request request = builder.build(); - call = client.newCall(request); - call.enqueue(this); - } catch (Exception exception) { - handleFailure(call, exception); + if (resourceUrl.startsWith("local://")) { + // used by render test to serve files from assets + executeLocalRequest(resourceUrl); + return; } + executeRequest(resourceUrl, etag, modified); } public void cancel() { @@ -178,6 +144,57 @@ class HTTPRequest implements Callback { return dispatcher; } + private void executeRequest(String resourceUrl, String etag, String modified) { + try { + HttpUrl httpUrl = HttpUrl.parse(resourceUrl); + if (httpUrl == null) { + log(Log.ERROR, String.format("[HTTP] Unable to parse resourceUrl %s", resourceUrl)); + } + + final String host = httpUrl.host().toLowerCase(MapboxConstants.MAPBOX_LOCALE); + // Don't try a request to remote server if we aren't connected + if (!Mapbox.isConnected() && !host.equals("127.0.0.1") && !host.equals("localhost")) { + throw new NoRouteToHostException("No Internet connection available."); + } + + if (host.equals("mapbox.com") || host.endsWith(".mapbox.com") || host.equals("mapbox.cn") + || host.endsWith(".mapbox.cn")) { + if (httpUrl.querySize() == 0) { + resourceUrl = resourceUrl + "?"; + } else { + resourceUrl = resourceUrl + "&"; + } + resourceUrl = resourceUrl + "events=true"; + } + + Request.Builder builder = new Request.Builder() + .url(resourceUrl) + .tag(resourceUrl.toLowerCase(MapboxConstants.MAPBOX_LOCALE)) + .addHeader("User-Agent", getUserAgent()); + if (etag.length() > 0) { + builder = builder.addHeader("If-None-Match", etag); + } else if (modified.length() > 0) { + builder = builder.addHeader("If-Modified-Since", modified); + } + Request request = builder.build(); + call = client.newCall(request); + call.enqueue(this); + } catch (Exception exception) { + handleFailure(call, exception); + } + } + + private void executeLocalRequest(String resourceUrl) { + new LocalRequestTask(new LocalRequestTask.OnLocalRequestResponse() { + @Override + public void onResponse(byte[] bytes) { + if (bytes != null) { + nativeOnResponse(200, null, null, null, null, null, null, bytes); + } + } + }).execute(resourceUrl); + } + private void handleFailure(Call call, Exception e) { String errorMessage = e.getMessage() != null ? e.getMessage() : "Error processing the request"; int type = getFailureType(e); diff --git a/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java new file mode 100644 index 0000000000..8f31364c39 --- /dev/null +++ b/platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java @@ -0,0 +1,51 @@ +package com.mapbox.mapboxsdk.http; + +import android.content.res.AssetManager; +import android.os.AsyncTask; +import com.mapbox.mapboxsdk.Mapbox; +import timber.log.Timber; + +import java.io.IOException; +import java.io.InputStream; + +class LocalRequestTask extends AsyncTask { + + private OnLocalRequestResponse requestResponse; + + LocalRequestTask(OnLocalRequestResponse requestResponse) { + this.requestResponse = requestResponse; + } + + @Override + protected byte[] doInBackground(String... strings) { + return loadFile(Mapbox.getApplicationContext().getAssets(), + "integration/" + strings[0] + .substring(8) + .replaceAll("%20", " ") + .replaceAll("%2c", ",")); + } + + @Override + protected void onPostExecute(byte[] bytes) { + super.onPostExecute(bytes); + if (bytes != null && requestResponse != null) { + requestResponse.onResponse(bytes); + } + } + + private static byte[] loadFile(AssetManager assets, String path) { + byte[] buffer = null; + try (InputStream input = assets.open(path)) { + int size = input.available(); + buffer = new byte[size]; + input.read(buffer); + } catch (IOException exception) { + Timber.e(exception); + } + return buffer; + } + + public interface OnLocalRequestResponse { + void onResponse(byte[] bytes); + } +} diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java new file mode 100644 index 0000000000..17fc4cfc43 --- /dev/null +++ b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java @@ -0,0 +1,70 @@ +package com.mapbox.mapboxsdk.testapp.render; + +import android.Manifest; +import android.support.test.espresso.IdlingPolicies; +import android.support.test.espresso.IdlingRegistry; +import android.support.test.espresso.IdlingResourceTimeoutException; +import android.support.test.rule.ActivityTestRule; +import android.support.test.rule.GrantPermissionRule; +import android.support.test.runner.AndroidJUnit4; +import com.mapbox.mapboxsdk.testapp.activity.render.RenderTestActivity; +import com.mapbox.mapboxsdk.testapp.utils.SnapshotterIdlingResource; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import timber.log.Timber; + +import java.util.concurrent.TimeUnit; + +import static android.support.test.espresso.Espresso.onView; +import static android.support.test.espresso.assertion.ViewAssertions.matches; +import static android.support.test.espresso.matcher.ViewMatchers.isDisplayed; +import static android.support.test.espresso.matcher.ViewMatchers.withId; + +/** + * Instrumentation render tests + */ +@RunWith(AndroidJUnit4.class) +public class RenderTest { + + private static final int RENDER_TEST_TIMEOUT = 30; + private SnapshotterIdlingResource idlingResource; + + @Rule + public ActivityTestRule activityRule = new ActivityTestRule<>(RenderTestActivity.class); + + @Rule + public GrantPermissionRule writeRule = GrantPermissionRule.grant(Manifest.permission.WRITE_EXTERNAL_STORAGE); + + @Rule + public GrantPermissionRule readRule = GrantPermissionRule.grant(Manifest.permission.READ_EXTERNAL_STORAGE); + + @Before + public void beforeTest() { + IdlingPolicies.setMasterPolicyTimeout(RENDER_TEST_TIMEOUT, TimeUnit.MINUTES); + setupIdlingResource(); + } + + private void setupIdlingResource() { + try { + Timber.e("@Before test: register idle resource"); + IdlingPolicies.setIdlingResourceTimeout(RENDER_TEST_TIMEOUT, TimeUnit.MINUTES); + IdlingRegistry.getInstance().register(idlingResource = new SnapshotterIdlingResource(activityRule.getActivity())); + } catch (IdlingResourceTimeoutException idlingResourceTimeoutException) { + throw new RuntimeException("Idling out!"); + } + } + + @Test + public void testRender() { + onView(withId(android.R.id.content)).check(matches(isDisplayed())); + } + + @After + public void afterTest() { + Timber.e("@After test: unregister idle resource"); + IdlingRegistry.getInstance().unregister(idlingResource); + } +} \ No newline at end of file diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java new file mode 100644 index 0000000000..e0da683e6d --- /dev/null +++ b/platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java @@ -0,0 +1,38 @@ +package com.mapbox.mapboxsdk.testapp.utils; + +import android.support.test.espresso.IdlingResource; + +import com.mapbox.mapboxsdk.testapp.activity.render.RenderTestActivity; + +public class SnapshotterIdlingResource implements IdlingResource, RenderTestActivity.OnRenderTestCompletionListener { + + private IdlingResource.ResourceCallback resourceCallback; + private boolean isSnapshotReady; + + public SnapshotterIdlingResource(RenderTestActivity activity) { + activity.setOnRenderTestCompletionListener(this); + } + + @Override + public String getName() { + return "SnapshotterIdlingResource"; + } + + @Override + public boolean isIdleNow() { + return isSnapshotReady; + } + + @Override + public void registerIdleTransitionCallback(ResourceCallback resourceCallback) { + this.resourceCallback = resourceCallback; + } + + @Override + public void onFinish() { + isSnapshotReady = true; + if (resourceCallback != null) { + resourceCallback.onTransitionToIdle(); + } + } +} diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml b/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml index b78fba0aae..595b87917a 100644 --- a/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml +++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml @@ -791,6 +791,9 @@ + @@ -79,6 +77,9 @@ public class FeatureOverviewActivity extends AppCompatActivity { private void onFeaturesLoaded(List featuresList) { features = featuresList; + if (featuresList == null || featuresList.isEmpty()) { + return; + } List sections = new ArrayList<>(); String currentCat = ""; diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java new file mode 100644 index 0000000000..81a7758d44 --- /dev/null +++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java @@ -0,0 +1,316 @@ +package com.mapbox.mapboxsdk.testapp.activity.render; + +import android.content.res.AssetManager; +import android.graphics.Bitmap; +import android.os.AsyncTask; +import android.os.Bundle; +import android.os.Environment; +import android.support.annotation.Nullable; +import android.support.v7.app.AppCompatActivity; +import android.view.Gravity; +import android.view.ViewGroup; +import android.widget.FrameLayout; +import android.widget.ImageView; +import com.google.gson.Gson; +import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter; +import okio.BufferedSource; +import okio.Okio; +import timber.log.Timber; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.ref.WeakReference; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Activity that generates map snapshots based on the node render test suite. + */ +public class RenderTestActivity extends AppCompatActivity { + + private static final String TEST_BASE_PATH = "integration/render-tests"; + + // TODO read out excluded tests from /platform/node/test/ignore.json + private static final List EXCLUDED_TESTS = new ArrayList() { + { + add("overlay,background-opacity"); + add("collision-lines-pitched,debug"); + add("1024-circle,extent"); + add("empty,empty"); + add("rotation-alignment-map,icon-pitch-scaling"); + add("rotation-alignment-viewport,icon-pitch-scaling"); + add("pitch15,line-pitch"); + add("pitch30,line-pitch"); + add("line-placement-true-pitched,text-keep-upright"); + add("180,raster-rotation"); + add("45,raster-rotation"); + add("90,raster-rotation"); + add("mapbox-gl-js#5631,regressions"); // crashes + add("overlapping,raster-masking"); + add("missing,raster-loading"); + add("pitchAndBearing,line-pitch"); + } + }; + + private final Map renderResultMap = new HashMap<>(); + private List renderTestDefinitions; + private OnRenderTestCompletionListener onRenderTestCompletionListener; + private MapSnapshotter mapSnapshotter; + private ImageView imageView; + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(imageView = new ImageView(RenderTestActivity.this)); + imageView.setLayoutParams(new FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT, Gravity.CENTER) + ); + } + + @Override + protected void onStop() { + super.onStop(); + if (mapSnapshotter != null) { + mapSnapshotter.cancel(); + } + } + + // + // Loads the render test definitions from assets folder + // + private static class LoadRenderDefinitionTask extends AsyncTask> { + + private WeakReference renderTestActivityWeakReference; + + LoadRenderDefinitionTask(RenderTestActivity renderTestActivity) { + this.renderTestActivityWeakReference = new WeakReference<>(renderTestActivity); + } + + @Override + protected List doInBackground(Void... voids) { + List definitions = new ArrayList<>(); + AssetManager assetManager = renderTestActivityWeakReference.get().getAssets(); + String[] categories = new String[0]; + try { + categories = assetManager.list(TEST_BASE_PATH); + } catch (IOException exception) { + Timber.e(exception); + } + for (int counter = categories.length - 1; counter >= 0; counter--) { + try { + String[] tests = assetManager.list(String.format("%s/%s", TEST_BASE_PATH, categories[counter])); + for (String test : tests) { + String styleJson = loadStyleJson(assetManager, categories[counter], test); + RenderTestStyleDefinition renderTestStyleDefinition = new Gson() + .fromJson(styleJson, RenderTestStyleDefinition.class); + RenderTestDefinition definition = new RenderTestDefinition( + categories[counter], test, styleJson, renderTestStyleDefinition); + if (!definition.hasOperations()) { + if (!definition.getCategory().equals("combinations") + && !EXCLUDED_TESTS.contains(definition.getName() + "," + definition.getCategory())) { + definitions.add(definition); + } + } else { + Timber.e("could not add test, test requires operations: %s from %s", test, categories[counter]); + } + } + } catch (Exception exception) { + Timber.e(exception); + } + } + return definitions; + } + + @Override + protected void onPostExecute(List renderTestDefinitions) { + super.onPostExecute(renderTestDefinitions); + RenderTestActivity renderTestActivity = renderTestActivityWeakReference.get(); + if (renderTestActivity != null) { + renderTestActivity.startRenderTests(renderTestDefinitions); + } + } + + private static String loadStyleJson(AssetManager assets, String category, String test) { + String styleJson = null; + try (InputStream input = assets.open(String.format("%s/%s/%s/style.json", TEST_BASE_PATH, category, test))) { + BufferedSource source = Okio.buffer(Okio.source(input)); + styleJson = source.readByteString().string(Charset.forName("utf-8")); + } catch (IOException exception) { + Timber.e(exception); + } + return styleJson; + } + } + + private void startRenderTests(List renderTestDefinitions) { + this.renderTestDefinitions = renderTestDefinitions; + if (!renderTestDefinitions.isEmpty()) { + render(renderTestDefinitions.get(0), renderTestDefinitions.size()); + } else { + // no definitions, finish test without rendering + onRenderTestCompletionListener.onFinish(); + } + } + + private void render(final RenderTestDefinition renderTestDefinition, final int testSize) { + Timber.d("Render test %s,%s", renderTestDefinition.getName(), renderTestDefinition.getCategory()); + mapSnapshotter = new RenderTestSnapshotter(this, renderTestDefinition.toOptions()); + mapSnapshotter.start(result -> { + Bitmap snapshot = result.getBitmap(); + imageView.setImageBitmap(snapshot); + renderResultMap.put(renderTestDefinition, snapshot); + if (renderResultMap.size() != testSize) { + continueTesting(renderTestDefinition); + } else { + finishTesting(); + } + }, error -> Timber.e(error)); + } + + private void continueTesting(RenderTestDefinition renderTestDefinition) { + int next = renderTestDefinitions.indexOf(renderTestDefinition) + 1; + Timber.d("Next test: %s / %s", next, renderTestDefinitions.size()); + render(renderTestDefinitions.get(next), renderTestDefinitions.size()); + } + + private void finishTesting() { + new SaveResultToDiskTask(onRenderTestCompletionListener, renderResultMap).execute(); + } + + // + // Save tests results to disk + // + private static class SaveResultToDiskTask extends AsyncTask { + + private OnRenderTestCompletionListener onRenderTestCompletionListener; + private Map renderResultMap; + + SaveResultToDiskTask(OnRenderTestCompletionListener onRenderTestCompletionListener, + Map renderResultMap) { + this.onRenderTestCompletionListener = onRenderTestCompletionListener; + this.renderResultMap = renderResultMap; + } + + @Override + protected Void doInBackground(Void... voids) { + if (isExternalStorageWritable()) { + try { + File testResultDir = FileUtils.createTestResultRootFolder(); + String basePath = testResultDir.getAbsolutePath(); + for (Map.Entry testResult : renderResultMap.entrySet()) { + writeResultToDisk(basePath, testResult); + } + } catch (final Exception exception) { + Timber.e(exception); + } + } + return null; + } + + private void writeResultToDisk(String path, Map.Entry result) { + RenderTestDefinition definition = result.getKey(); + String categoryName = definition.getCategory(); + String categoryPath = String.format("%s/%s", path, categoryName); + FileUtils.createCategoryDirectory(categoryPath); + String testName = result.getKey().getName(); + String testDir = FileUtils.createTestDirectory(categoryPath, testName); + FileUtils.writeTestResultToDisk(testDir, result.getValue()); + } + + private boolean isExternalStorageWritable() { + return Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState()); + } + + @Override + protected void onPostExecute(Void aVoid) { + super.onPostExecute(aVoid); + if (onRenderTestCompletionListener != null) { + onRenderTestCompletionListener.onFinish(); + } + } + } + + // + // Callback configuration to notify test executor of test finishing + // + public interface OnRenderTestCompletionListener { + void onFinish(); + } + + public void setOnRenderTestCompletionListener(OnRenderTestCompletionListener listener) { + this.onRenderTestCompletionListener = listener; + new LoadRenderDefinitionTask(this).execute(); + } + + // + // FileUtils + // + + private static class FileUtils { + + private static void createCategoryDirectory(String catPath) { + File testResultDir = new File(catPath); + if (testResultDir.exists()) { + return; + } + + if (!testResultDir.mkdirs()) { + throw new RuntimeException("can't create root test directory"); + } + } + + private static File createTestResultRootFolder() { + File testResultDir = new File(Environment.getExternalStorageDirectory() + + File.separator + "mapbox" + File.separator + "render"); + if (testResultDir.exists()) { + // cleanup old files + deleteRecursive(testResultDir); + } + + if (!testResultDir.mkdirs()) { + throw new RuntimeException("can't create root test directory"); + } + return testResultDir; + } + + private static void deleteRecursive(File fileOrDirectory) { + if (fileOrDirectory.isDirectory()) { + File[] files = fileOrDirectory.listFiles(); + if (files != null) { + for (File file : files) { + deleteRecursive(file); + } + } + } + + if (!fileOrDirectory.delete()) { + Timber.e("can't delete directory"); + } + } + + private static String createTestDirectory(String basePath, String testName) { + File testDir = new File(basePath + "/" + testName); + if (!testDir.exists()) { + if (!testDir.mkdir()) { + throw new RuntimeException("can't create sub directory for " + testName); + } + } + return testDir.getAbsolutePath(); + } + + private static void writeTestResultToDisk(String testPath, Bitmap testResult) { + String filePath = testPath + "/actual.png"; + try (FileOutputStream out = new FileOutputStream(filePath)) { + testResult.compress(Bitmap.CompressFormat.PNG, 100, out); + } catch (IOException exception) { + Timber.e(exception); + } + } + } +} \ No newline at end of file diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java new file mode 100644 index 0000000000..fa8c816203 --- /dev/null +++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java @@ -0,0 +1,79 @@ +package com.mapbox.mapboxsdk.testapp.activity.render; + +import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter; + +public class RenderTestDefinition { + + private static final int DEFAULT_WIDTH = 512; + private static final int DEFAULT_HEIGHT = 512; + + private String category; // eg. background-color + private String name; // eg. colorSpace-hcl + private String styleJson; + private RenderTestStyleDefinition definition; + + RenderTestDefinition(String category, String name, String styleJson, RenderTestStyleDefinition definition) { + this.category = category; + this.name = name; + this.styleJson = styleJson; + this.definition = definition; + } + + public String getName() { + return name; + } + + public String getCategory() { + return category; + } + + public int getWidth() { + RenderTestStyleDefinition.Test test = getTest(); + if (test != null) { + Integer testWidth = test.getWidth(); + if (testWidth != null && testWidth > 0) { + return testWidth; + } + } + return DEFAULT_WIDTH; + } + + public int getHeight() { + RenderTestStyleDefinition.Test test = getTest(); + if (test != null) { + Integer testHeight = test.getHeight(); + if (testHeight != null && testHeight > 0) { + return testHeight; + } + } + return DEFAULT_HEIGHT; + } + + public String getStyleJson() { + return styleJson; + } + + public boolean hasOperations() { + return getTest().getOperations() != null; + } + + public RenderTestStyleDefinition.Test getTest() { + return definition.getMetadata().getTest(); + } + + public MapSnapshotter.Options toOptions() { + return new MapSnapshotter + .Options(getWidth(), getHeight()) + .withStyleJson(styleJson) + .withLogo(false); + } + + @Override + public String toString() { + return "RenderTestDefinition{" + + "category='" + category + '\'' + + ", name='" + name + '\'' + + ", styleJson='" + styleJson + '\'' + + '}'; + } +} diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java new file mode 100644 index 0000000000..cb971fee70 --- /dev/null +++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java @@ -0,0 +1,18 @@ +package com.mapbox.mapboxsdk.testapp.activity.render; + +import android.content.Context; +import android.support.annotation.NonNull; +import com.mapbox.mapboxsdk.snapshotter.MapSnapshot; +import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter; + +public class RenderTestSnapshotter extends MapSnapshotter { + + RenderTestSnapshotter(@NonNull Context context, @NonNull Options options) { + super(context, options); + } + + @Override + protected void addOverlay(MapSnapshot mapSnapshot) { + // don't add an overlay + } +} diff --git a/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java new file mode 100644 index 0000000000..fdd7e9aaf1 --- /dev/null +++ b/platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java @@ -0,0 +1,127 @@ +package com.mapbox.mapboxsdk.testapp.activity.render; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RenderTestStyleDefinition { + + private Integer version; + private Metadata metadata; + private Map additionalProperties = new HashMap(); + + public Integer getVersion() { + return version; + } + + public void setVersion(Integer version) { + this.version = version; + } + + public Metadata getMetadata() { + return metadata; + } + + public void setMetadata(Metadata metadata) { + this.metadata = metadata; + } + + public Map getAdditionalProperties() { + return this.additionalProperties; + } + + public void setAdditionalProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } + + + public class Metadata { + + private Test test; + private Map additionalProperties = new HashMap(); + + public Test getTest() { + return test; + } + + public void setTest(Test test) { + this.test = test; + } + + public Map getAdditionalProperties() { + return this.additionalProperties; + } + + public void setAdditionalProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } + } + + public class Test { + + private Integer width; + private Integer height; + private List center = null; + private Integer zoom; + private Double diff; + private List> operations = null; + private Map additionalProperties = new HashMap(); + + public Integer getWidth() { + return width; + } + + public void setWidth(Integer width) { + this.width = width; + } + + public Integer getHeight() { + return height; + } + + public void setHeight(Integer height) { + this.height = height; + } + + public List getCenter() { + return center; + } + + public void setCenter(List center) { + this.center = center; + } + + public Integer getZoom() { + return zoom; + } + + public void setZoom(Integer zoom) { + this.zoom = zoom; + } + + public Double getDiff() { + return diff; + } + + public void setDiff(Double diff) { + this.diff = diff; + } + + public List> getOperations() { + return operations; + } + + public void setOperations(List> operations) { + this.operations = operations; + } + + public Map getAdditionalProperties() { + return this.additionalProperties; + } + + public void setAdditionalProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } + + } +} \ No newline at end of file diff --git a/platform/android/scripts/exclude-activity-gen.json b/platform/android/scripts/exclude-activity-gen.json index d7d9c16550..36d8d36e68 100644 --- a/platform/android/scripts/exclude-activity-gen.json +++ b/platform/android/scripts/exclude-activity-gen.json @@ -34,5 +34,6 @@ "ZoomFunctionSymbolLayerActivity", "SymbolGeneratorActivity", "TextureViewTransparentBackgroundActivity", - "SimpleMapActivity" + "SimpleMapActivity", + "RenderTestActivity" ] diff --git a/platform/android/scripts/run-render-test.py b/platform/android/scripts/run-render-test.py new file mode 100644 index 0000000000..83376256a4 --- /dev/null +++ b/platform/android/scripts/run-render-test.py @@ -0,0 +1,24 @@ +#!/usr/bin/python + +import os +from shutil import copyfile + +catPath = os.getcwd() + "/platform/android/build/render-test/render/" +for cat in os.listdir(catPath): + testPath = catPath + cat + "/" + for test in os.listdir(testPath): + inputPath = os.getcwd() + "/mapbox-gl-js/test/integration/render-tests/" + cat + "/" + test + outputPath = testPath + test + + expected = outputPath + "/expected.png" + actual = outputPath + "/actual.png" + output = outputPath + "/output.png" + + copyfile(inputPath + "/expected.png", expected) + copyfile(inputPath + "/style.json", outputPath + "/style.json") + pixelmatch = "node_modules/pixelmatch/bin/pixelmatch " + actual + " " + expected + " " + output + " 0.1" + + print + print "Pixel match "+ cat + " " + test + os.system(pixelmatch) +print -- cgit v1.2.1