rewrite storage layer to be independent of the Map's event loop

23 files changed, 710 insertions, 285 deletions

diff --git a/include/mbgl/map/map.hpp b/include/mbgl/map/map.hpp
index 62f0d62014..876b7b7207 100644
--- a/include/mbgl/map/map.hpp
+++ b/include/mbgl/map/map.hpp
@@ -123,6 +123,10 @@ public:
     void startRotating();
     void stopRotating();
 
+    // API
+    void setAccessToken(const std::string &token);
+    const std::string &getAccessToken() const;
+
     // Debug
     void setDebug(bool value);
     void toggleDebug();
@@ -160,7 +164,10 @@ private:
 
     Mode mode = Mode::None;
 
+public: // TODO: make private again
     std::unique_ptr<uv::loop> loop;
+
+private:
     std::unique_ptr<uv::worker> workers;
     std::thread thread;
     std::unique_ptr<uv::async> asyncTerminate;
@@ -214,6 +221,7 @@ private:
     std::string styleURL;
     std::string styleJSON = "";
     std::vector<std::string> classes;
+    std::string accessToken;
 
     std::chrono::steady_clock::duration defaultTransitionDuration;
 
diff --git a/include/mbgl/storage/asset_request.hpp b/include/mbgl/storage/asset_request.hpp
deleted file mode 100644
index 3114d41ad2..0000000000
--- a/include/mbgl/storage/asset_request.hpp
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef MBGL_STORAGE_ASSET_REQUEST
-#define MBGL_STORAGE_ASSET_REQUEST
-
-#include <mbgl/storage/base_request.hpp>
-
-namespace mbgl {
-
-typedef struct uv_loop_s uv_loop_t;
-
-struct AssetRequestBaton;
-
-class AssetRequest : public BaseRequest {
-public:
-    AssetRequest(const std::string &path, uv_loop_t *loop);
-    ~AssetRequest();
-
-    void cancel();
-
-private:
-    AssetRequestBaton *ptr = nullptr;
-
-    friend struct AssetRequestBaton;
-};
-
-}
-
-#endif
diff --git a/include/mbgl/storage/caching_http_file_source.hpp b/include/mbgl/storage/caching_http_file_source.hpp
deleted file mode 100644
index 655afa6396..0000000000
--- a/include/mbgl/storage/caching_http_file_source.hpp
+++ /dev/null
@@ -1,62 +0,0 @@
-#ifndef MBGL_STORAGE_CACHING_HTTP_FILE_SOURCE
-#define MBGL_STORAGE_CACHING_HTTP_FILE_SOURCE
-
-#include <mbgl/storage/file_source.hpp>
-
-#include <thread>
-#include <unordered_map>
-
-typedef struct uv_messenger_s uv_messenger_t;
-
-namespace mbgl {
-
-class BaseRequest;
-class SQLiteStore;
-
-class CachingHTTPFileSource : public FileSource {
-public:
-    CachingHTTPFileSource(const std::string &path_);
-    virtual ~CachingHTTPFileSource();
-
-    // Stores and checks the libuv loop for requests
-    void setLoop(uv_loop_t*);
-    bool hasLoop();
-    void clearLoop();
-
-    // Set the base path/URL for relative requests
-    void setBase(std::string);
-
-    // Set the Mapbox API access token
-    void setAccessToken(std::string);
-
-    // Get the Mapbox API access token
-    std::string getAccessToken() const;
-
-    std::unique_ptr<Request> request(ResourceType type, const std::string &url);
-
-    void prepare(std::function<void()> fn);
-
-    // Call this when the network status reachability changed.
-    void setReachability(bool reachable);
-
-private:
-    std::thread::id threadId;
-
-    // Mapbox API access token.
-    std::string accessToken;
-
-    // Path to the cache database.
-    std::string path;
-
-    // Stores a URL that is used as a base for loading resources with relative path.
-    std::string base;
-
-    std::unordered_map<std::string, std::weak_ptr<BaseRequest>> pending;
-    util::ptr<SQLiteStore> store;
-    uv_loop_t *loop = nullptr;
-    uv_messenger_t *queue = nullptr;
-};
-
-}
-
-#endif
diff --git a/include/mbgl/storage/default/asset_request.hpp b/include/mbgl/storage/default/asset_request.hpp
new file mode 100644
index 0000000000..c582c025fb
--- /dev/null
+++ b/include/mbgl/storage/default/asset_request.hpp
@@ -0,0 +1,24 @@
+#ifndef MBGL_STORAGE_DEFAULT_ASSET_REQUEST
+#define MBGL_STORAGE_DEFAULT_ASSET_REQUEST
+
+#include "shared_request_base.hpp"
+
+namespace mbgl {
+
+class AssetRequest : public SharedRequestBase {
+public:
+    AssetRequest(DefaultFileSource *source, const Resource &resource);
+
+    void start(uv_loop_t *loop, std::unique_ptr<Response> response = nullptr);
+    void cancel();
+
+private:
+    ~AssetRequest();
+    void *ptr = nullptr;
+
+    friend class AssetRequestImpl;
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/default/default_file_source.hpp b/include/mbgl/storage/default/default_file_source.hpp
new file mode 100644
index 0000000000..189ff7b9ee
--- /dev/null
+++ b/include/mbgl/storage/default/default_file_source.hpp
@@ -0,0 +1,60 @@
+#ifndef MBGL_STORAGE_DEFAULT_DEFAULT_FILE_SOURCE
+#define MBGL_STORAGE_DEFAULT_DEFAULT_FILE_SOURCE
+
+#include <mbgl/storage/file_source.hpp>
+#include <mbgl/storage/file_cache.hpp>
+
+#include <set>
+#include <unordered_map>
+#include <thread>
+
+namespace mapbox { namespace util { template<typename... Types> class variant; } }
+
+namespace mbgl {
+
+namespace util { template <typename T> class AsyncQueue; }
+
+class SharedRequestBase;
+
+class DefaultFileSource : public FileSource {
+public:
+    DefaultFileSource(FileCache *cache);
+    DefaultFileSource(FileCache *cache, uv_loop_t *loop);
+    ~DefaultFileSource();
+
+    Request *request(const Resource &resource, uv_loop_t *loop, Callback callback);
+    void cancel(Request *request);
+    void request(const Resource &resource, Callback callback);
+
+    enum class CacheHint : uint8_t { Full, Refresh, No };
+    void notify(SharedRequestBase *sharedRequest, const std::set<Request *> &observers,
+                std::shared_ptr<const Response> response, FileCache::Hint hint);
+
+private:
+    struct ActionDispatcher;
+    struct AddRequestAction;
+    struct RemoveRequestAction;
+    struct ResultAction;
+    struct StopAction;
+    using Action =
+        mapbox::util::variant<AddRequestAction, RemoveRequestAction, ResultAction, StopAction>;
+    using Queue = util::AsyncQueue<Action>;
+
+    void process(AddRequestAction &action);
+    void process(RemoveRequestAction &action);
+    void process(ResultAction &action);
+    void process(StopAction &action);
+
+    SharedRequestBase *find(const Resource &resource);
+
+    std::unordered_map<Resource, SharedRequestBase *> pending;
+
+    uv_loop_t *loop = nullptr;
+    FileCache *cache = nullptr;
+    Queue *queue = nullptr;
+    std::thread thread;
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/default/http_context.hpp b/include/mbgl/storage/default/http_context.hpp
new file mode 100644
index 0000000000..18f17ef0cb
--- /dev/null
+++ b/include/mbgl/storage/default/http_context.hpp
@@ -0,0 +1,117 @@
+#ifndef MBGL_STORAGE_DEFAULT_HTTP_CONTEXT
+#define MBGL_STORAGE_DEFAULT_HTTP_CONTEXT
+
+#include <mbgl/storage/network_status.hpp>
+#include <mbgl/util/noncopyable.hpp>
+#include <mbgl/util/std.hpp>
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/uv.hpp>
+
+#include <uv.h>
+#include <pthread.h>
+
+#include <map>
+#include <cassert>
+#include <set>
+
+namespace mbgl {
+
+class HTTPRequest;
+
+// This is a template class that provides a per-thread Context object. It can be used by HTTP
+// implementations to store global state. It also implements the NetworkStatus mechanism and
+// triggers immediate retries on all requests waiting for network status changes.
+
+template <typename Context>
+class HTTPContext : private util::noncopyable {
+protected:
+    MBGL_STORE_THREAD(tid)
+    using Map = std::map<uv_loop_t *, std::unique_ptr<Context>>;
+
+public:
+    static Context *Get(uv_loop_t *loop);
+
+private:
+    static pthread_key_t key;
+    static pthread_once_t once;
+
+public:
+    HTTPContext(uv_loop_t *loop);
+    ~HTTPContext();
+
+    void addRequest(HTTPRequest *request);
+    void removeRequest(HTTPRequest *baton);
+
+public:
+    uv_loop_t *loop;
+
+    // Will be fired when the network status becomes reachable.
+    uv_async_t *reachability = nullptr;
+
+    // A list of all pending HTTPRequestImpls that we need to notify when the network status
+    // changes.
+    std::set<HTTPRequest *> requests;
+};
+
+template <typename Context>
+Context *HTTPContext<Context>::Get(uv_loop_t *loop) {
+    pthread_once(&once, []() {
+        pthread_key_create(&key, [](void *ptr) {
+            assert(ptr);
+            delete reinterpret_cast<Map *>(ptr);
+        });
+    });
+    auto contexts = reinterpret_cast<Map *>(pthread_getspecific(key));
+    if (!contexts) {
+        contexts = new Map();
+        pthread_setspecific(key, contexts);
+    }
+
+    // Now find a HTTPContext that matches the requested loop.
+    auto it = contexts->find(loop);
+    if (it == contexts->end()) {
+        auto result = contexts->emplace(loop, util::make_unique<Context>(loop));
+        assert(result.second); // Make sure it was actually inserted.
+        return result.first->second.get();
+    } else {
+        return it->second.get();
+    }
+}
+
+template <typename Context>
+HTTPContext<Context>::HTTPContext(uv_loop_t *loop_) : loop(loop_) {
+    reachability = new uv_async_t;
+    reachability->data = this;
+    uv_async_init(loop, reachability, [](uv_async_t *async, int) {
+        for (auto request : reinterpret_cast<Context *>(async->data)->requests) {
+            request->retryImmediately();
+        }
+    });
+    // Allow the loop to quit even though this handle is still active.
+    uv_unref(reinterpret_cast<uv_handle_t *>(reachability));
+    NetworkStatus::Subscribe(reachability);
+}
+
+template <typename Context>
+HTTPContext<Context>::~HTTPContext() {
+    MBGL_VERIFY_THREAD(tid);
+
+    assert(requests.empty());
+
+    NetworkStatus::Unsubscribe(reachability);
+    uv::close(reachability);
+}
+
+template <typename Context>
+void HTTPContext<Context>::addRequest(HTTPRequest *request) {
+    requests.insert(request);
+}
+
+template <typename Context>
+void HTTPContext<Context>::removeRequest(HTTPRequest *request) {
+    requests.erase(request);
+}
+
+}
+
+#endif
diff --git a/include/mbgl/storage/default/http_request.hpp b/include/mbgl/storage/default/http_request.hpp
new file mode 100644
index 0000000000..914e622f13
--- /dev/null
+++ b/include/mbgl/storage/default/http_request.hpp
@@ -0,0 +1,26 @@
+#ifndef MBGL_STORAGE_DEFAULT_HTTP_REQUEST
+#define MBGL_STORAGE_DEFAULT_HTTP_REQUEST
+
+#include "shared_request_base.hpp"
+
+namespace mbgl {
+
+class HTTPRequest : public SharedRequestBase {
+public:
+    HTTPRequest(DefaultFileSource *source, const Resource &resource);
+
+    void start(uv_loop_t *loop, std::unique_ptr<Response> response = nullptr);
+    void cancel();
+
+    void retryImmediately();
+
+private:
+    ~HTTPRequest();
+    void *ptr = nullptr;
+
+    friend class HTTPRequestImpl;
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/default/request.hpp b/include/mbgl/storage/default/request.hpp
new file mode 100644
index 0000000000..0ed544093c
--- /dev/null
+++ b/include/mbgl/storage/default/request.hpp
@@ -0,0 +1,46 @@
+#ifndef MBGL_STORAGE_DEFAULT_REQUEST
+#define MBGL_STORAGE_DEFAULT_REQUEST
+
+#include <mbgl/storage/resource.hpp>
+
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/noncopyable.hpp>
+
+#include <functional>
+
+typedef struct uv_async_s uv_async_t;
+typedef struct uv_loop_s uv_loop_t;
+
+namespace mbgl {
+
+class Response;
+
+class Request : private util::noncopyable {
+    MBGL_STORE_THREAD(tid)
+
+public:
+    using Callback = std::function<void(const Response &)>;
+    Request(const Resource &resource, uv_loop_t *loop, Callback callback);
+    ~Request();
+
+public:
+    // May be called from any thread.
+    void notify(const std::shared_ptr<const Response> &response);
+    void destruct();
+
+    // May be called only from the thread the Request was created in.
+    void cancel();
+
+private:
+    uv_async_t *notify_async = nullptr;
+    uv_async_t *destruct_async = nullptr;
+    Callback callback;
+    std::shared_ptr<const Response> response;
+
+public:
+    const Resource resource;
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/default/shared_request_base.hpp b/include/mbgl/storage/default/shared_request_base.hpp
new file mode 100644
index 0000000000..8591d66ba3
--- /dev/null
+++ b/include/mbgl/storage/default/shared_request_base.hpp
@@ -0,0 +1,89 @@
+#ifndef MBGL_STORAGE_DEFAULT_SHARED_REQUEST_BASE
+#define MBGL_STORAGE_DEFAULT_SHARED_REQUEST_BASE
+
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/storage/file_cache.hpp>
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/noncopyable.hpp>
+
+#include "default_file_source.hpp"
+
+#include <string>
+#include <set>
+#include <cassert>
+
+typedef struct uv_loop_s uv_loop_t;
+
+namespace mbgl {
+
+class Request;
+class Response;
+class DefaultFileSource;
+
+class SharedRequestBase : private util::noncopyable {
+protected:
+    MBGL_STORE_THREAD(tid)
+
+public:
+    SharedRequestBase(DefaultFileSource *source_, const Resource &resource_)
+        : resource(resource_), source(source_) {}
+
+    virtual void start(uv_loop_t *loop, std::unique_ptr<Response> response = nullptr) = 0;
+    virtual void cancel() = 0;
+
+    void notify(std::unique_ptr<Response> response, FileCache::Hint hint) {
+        MBGL_VERIFY_THREAD(tid);
+
+        if (source) {
+            source->notify(this, observers, std::shared_ptr<const Response>(response.release()),
+                           hint);
+        }
+    }
+
+    void subscribe(Request *request) {
+        MBGL_VERIFY_THREAD(tid);
+
+        observers.insert(request);
+    }
+
+    void unsubscribeAll() {
+        MBGL_VERIFY_THREAD(tid);
+
+        source = nullptr;
+        observers.clear();
+        cancel();
+    }
+
+    void unsubscribe(Request *request) {
+        MBGL_VERIFY_THREAD(tid);
+
+        observers.erase(request);
+
+        if (observers.empty()) {
+            // There are no observers anymore. We are initiating cancelation.
+            if (source) {
+                // First, remove this SharedRequestBase from the source.
+                source->notify(this, observers, nullptr, FileCache::Hint::No);
+            }
+
+            // Then, initiate cancelation of this request
+            cancel();
+        }
+    }
+
+protected:
+    virtual ~SharedRequestBase() {
+        MBGL_VERIFY_THREAD(tid);
+    }
+
+public:
+    const Resource resource;
+
+private:
+    DefaultFileSource *source = nullptr;
+    std::set<Request *> observers;
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/default/sqlite_cache.hpp b/include/mbgl/storage/default/sqlite_cache.hpp
new file mode 100644
index 0000000000..8f2746561c
--- /dev/null
+++ b/include/mbgl/storage/default/sqlite_cache.hpp
@@ -0,0 +1,52 @@
+#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE
+#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE
+
+#include <mbgl/storage/file_cache.hpp>
+
+#include <string>
+#include <thread>
+
+typedef struct uv_loop_s uv_loop_t;
+
+namespace mapbox { namespace util { template<typename... Types> class variant; } }
+namespace mapbox { namespace sqlite { class Database; class Statement; } }
+
+namespace mbgl {
+
+namespace util { template <typename T> class AsyncQueue; }
+
+class SQLiteCache : public FileCache {
+    struct GetAction;
+    struct PutAction;
+    struct RefreshAction;
+    struct StopAction;
+    using Action = mapbox::util::variant<GetAction, PutAction, RefreshAction, StopAction>;
+    using Queue = util::AsyncQueue<Action>;
+
+public:
+    SQLiteCache(const std::string &path = ":memory:");
+    ~SQLiteCache();
+
+    void get(const Resource &resource, std::function<void(std::unique_ptr<Response>)> callback);
+    void put(const Resource &resource, std::shared_ptr<const Response> response, Hint hint);
+
+private:
+    struct ActionDispatcher;
+    void process(GetAction &action);
+    void process(PutAction &action);
+    void process(RefreshAction &action);
+    void process(StopAction &action);
+
+    void createDatabase();
+
+    const std::string path;
+    uv_loop_t *loop = nullptr;
+    Queue *queue = nullptr;
+    std::thread thread;
+    std::unique_ptr<::mapbox::sqlite::Database> db;
+    std::unique_ptr<::mapbox::sqlite::Statement> getStmt, putStmt, refreshStmt;
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/file_cache.hpp b/include/mbgl/storage/file_cache.hpp
new file mode 100644
index 0000000000..38a82cc237
--- /dev/null
+++ b/include/mbgl/storage/file_cache.hpp
@@ -0,0 +1,26 @@
+#ifndef MBGL_STORAGE_FILE_CACHE
+#define MBGL_STORAGE_FILE_CACHE
+
+#include <mbgl/util/noncopyable.hpp>
+
+#include <functional>
+
+namespace mbgl {
+
+struct Resource;
+class Response;
+
+class FileCache : private util::noncopyable {
+public:
+    virtual ~FileCache() = default;
+
+    enum class Hint : uint8_t { Full, Refresh, No };
+
+    virtual void get(const Resource &resource,
+                     std::function<void(std::unique_ptr<Response>)> callback) = 0;
+    virtual void put(const Resource &resource, std::shared_ptr<const Response> response, Hint hint) = 0;
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/file_source.hpp b/include/mbgl/storage/file_source.hpp
index 23a1462ae8..8517d6e4a6 100644
--- a/include/mbgl/storage/file_source.hpp
+++ b/include/mbgl/storage/file_source.hpp
@@ -1,29 +1,38 @@
 #ifndef MBGL_STORAGE_FILE_SOURCE
 #define MBGL_STORAGE_FILE_SOURCE
 
+#include "response.hpp"
+#include "resource.hpp"
+
 #include <mbgl/util/noncopyable.hpp>
-#include <mbgl/storage/resource_type.hpp>
-#include <mbgl/storage/request.hpp>
+#include <mbgl/util/std.hpp>
+#include <mbgl/util/util.hpp>
 
-#include <string>
 #include <functional>
 
 typedef struct uv_loop_s uv_loop_t;
 
-
 namespace mbgl {
 
-class FileSource : public util::noncopyable {
+class Request;
+
+class FileSource : private util::noncopyable {
+protected:
+    MBGL_STORE_THREAD(tid)
+
 public:
     virtual ~FileSource() = default;
 
-    virtual void setLoop(uv_loop_t*) = 0;
-    virtual bool hasLoop() = 0;
-    virtual void clearLoop() = 0;
+    using Callback = std::function<void(const Response &)>;
+
+    // These can be called from any thread. The callback will be invoked in the loop.
+    // You can only cancel a request from the same thread it was created in.
+    virtual Request *request(const Resource &resource, uv_loop_t *loop, Callback callback) = 0;
+    virtual void cancel(Request *request) = 0;
 
-    virtual void setBase(std::string) = 0;
-    virtual std::unique_ptr<Request> request(ResourceType type, const std::string &url) = 0;
-    virtual void prepare(std::function<void()> fn) = 0;
+    // These can be called from any thread. The callback will be invoked in an arbitrary other thread.
+    // You cannot cancel these requests.
+    virtual void request(const Resource &resource, Callback callback) = 0;
 };
 
 }
diff --git a/include/mbgl/storage/http_request_baton.hpp b/include/mbgl/storage/http_request_baton.hpp
deleted file mode 100644
index 11abfb71d4..0000000000
--- a/include/mbgl/storage/http_request_baton.hpp
+++ /dev/null
@@ -1,74 +0,0 @@
-#ifndef MBGL_STORAGE_HTTP_REQUEST_BATON
-#define MBGL_STORAGE_HTTP_REQUEST_BATON
-
-#include <mbgl/storage/response.hpp>
-#include <mbgl/util/ptr.hpp>
-
-#include <string>
-#include <thread>
-
-typedef struct uv_async_s uv_async_t;
-
-namespace mbgl {
-
-class HTTPRequest;
-
-enum class HTTPResponseType : int8_t {
-    // This error probably won't be resolved by retrying anytime soon. We are giving up.
-    PermanentError = -5,
-
-    // This error might be resolved by waiting some time (e.g. server issues).
-    // We are going to do an exponential back-off and will try again in a few seconds.
-    TemporaryError = -4,
-
-    // This error was caused by a temporary error and it is likely that it will be resolved
-    // immediately. We are going to try again right away. This is like the TemporaryError, except
-    // that we will not perform exponential back-off.
-    SingularError = -3,
-
-    // This error might be resolved once the network reachability status changes.
-    // We are going to watch the network status for changes and will retry as soon as the operating
-    // system notifies us of a network status change.
-    ConnectionError = -2,
-
-    // The request was canceled programatically.
-    Canceled = -1,
-
-    // The request is still in progress.
-    Unknown = 0,
-
-    // The request returned data successfully. We retrieved and decoded the data successfully.
-    Successful = 1,
-
-    // The request confirmed that the data wasn't changed. We already have the data.
-    NotModified = 2,
-};
-
-struct HTTPRequestBaton {
-    HTTPRequestBaton(const std::string &path);
-    ~HTTPRequestBaton();
-
-    const std::thread::id threadId;
-    const std::string path;
-
-    HTTPRequest *request = nullptr;
-    uv_async_t *async = nullptr;
-
-    HTTPResponseType type = HTTPResponseType::Unknown;
-    std::unique_ptr<Response> response;
-
-    // Implementation specific use.
-    void *ptr = nullptr;
-
-    // IMPLEMENT THESE 3 PLATFORM SPECIFIC FUNCTIONS:
-
-    // Begin the HTTP request. Platform-specific implementation.
-    static void start(const util::ptr<HTTPRequestBaton> &ptr);
-
-    // This will be called to stop/cancel the HTTP request (if possible). Platform-specific implementation.
-    static void stop(const util::ptr<HTTPRequestBaton> &ptr);
-};
-
-}
-
-#endif
diff --git a/include/mbgl/storage/network_status.hpp b/include/mbgl/storage/network_status.hpp
new file mode 100644
index 0000000000..cac2ae193b
--- /dev/null
+++ b/include/mbgl/storage/network_status.hpp
@@ -0,0 +1,25 @@
+#ifndef MBGL_STORAGE_NETWORK_STATUS
+#define MBGL_STORAGE_NETWORK_STATUS
+
+#include <mutex>
+#include <set>
+
+typedef struct uv_async_s uv_async_t;
+
+namespace mbgl {
+
+class NetworkStatus {
+public:
+    static void Reachable();
+
+    static void Subscribe(uv_async_t *async);
+    static void Unsubscribe(uv_async_t *async);
+
+private:
+    static std::mutex mtx;
+    static std::set<uv_async_t *> observers;
+};
+
+}
+
+#endif
\ No newline at end of file
diff --git a/include/mbgl/storage/request.hpp b/include/mbgl/storage/request.hpp
deleted file mode 100644
index 845c9a6dad..0000000000
--- a/include/mbgl/storage/request.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-#ifndef MBGL_STORAGE_REQUEST
-#define MBGL_STORAGE_REQUEST
-
-#include <mbgl/storage/request_callback.hpp>
-#include <mbgl/storage/response.hpp>
-
-#include <mbgl/util/ptr.hpp>
-
-#include <thread>
-#include <forward_list>
-
-typedef struct uv_loop_s uv_loop_t;
-
-namespace mbgl {
-
-class BaseRequest;
-
-class Request {
-private:
-    Request(const Request &) = delete;
-    Request(Request &&) = delete;
-    Request& operator=(const Request &) = delete;
-    Request& operator=(Request &&) = delete;
-
-public:
-    Request(const util::ptr<BaseRequest> &base);
-    ~Request();
-
-    void onload(CompletedCallback cb);
-    void oncancel(AbortedCallback cb);
-    void cancel();
-
-private:
-    const std::thread::id thread_id;
-    util::ptr<BaseRequest> base;
-    std::forward_list<Callback *> callbacks;
-};
-
-}
-
-#endif
\ No newline at end of file
diff --git a/include/mbgl/storage/request_callback.hpp b/include/mbgl/storage/request_callback.hpp
deleted file mode 100644
index 01427bd96d..0000000000
--- a/include/mbgl/storage/request_callback.hpp
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef MBGL_STORAGE_REQUEST_CALLBACK
-#define MBGL_STORAGE_REQUEST_CALLBACK
-
-#include <mbgl/util/variant.hpp>
-
-#include <functional>
-
-namespace mbgl {
-
-class Response;
-
-using CompletedCallback = std::function<void(const Response &)>;
-using AbortedCallback = std::function<void()>;
-
-using Callback = mapbox::util::variant<
-    CompletedCallback,
-    AbortedCallback
->;
-
-}
-
-#endif
diff --git a/include/mbgl/storage/resource.hpp b/include/mbgl/storage/resource.hpp
new file mode 100644
index 0000000000..e499f84a28
--- /dev/null
+++ b/include/mbgl/storage/resource.hpp
@@ -0,0 +1,39 @@
+#ifndef MBGL_STORAGE_RESOURCE
+#define MBGL_STORAGE_RESOURCE
+
+#include <string>
+#include <functional>
+
+namespace mbgl {
+
+struct Resource {
+    enum Kind : uint8_t {
+        Unknown = 0,
+        Tile = 1,
+        Glyphs = 2,
+        Image = 3,
+        JSON = 4,
+    };
+
+    const Kind kind;
+    const std::string url;
+
+    inline bool operator==(const Resource &res) const {
+        return kind == res.kind && url == res.url;
+    }
+};
+
+}
+
+// Specialize std::hash for use in std::unordered_map
+namespace std {
+
+template<> struct hash<mbgl::Resource> {
+    std::size_t operator()(mbgl::Resource const& r) const {
+        return std::hash<std::string>()(r.url) ^ (std::hash<uint8_t>()(r.kind) << 1);
+    }
+};
+
+}
+
+#endif
diff --git a/include/mbgl/storage/resource_type.hpp b/include/mbgl/storage/resource_type.hpp
deleted file mode 100644
index b7204a9fa1..0000000000
--- a/include/mbgl/storage/resource_type.hpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef MBGL_STORAGE_RESOURCE_TYPE
-#define MBGL_STORAGE_RESOURCE_TYPE
-
-#include <cstdint>
-
-namespace mbgl {
-
-enum class ResourceType : uint8_t {
-    Unknown = 0,
-    Tile = 1,
-    Glyphs = 2,
-    Image = 3,
-    JSON = 4
-};
-
-}
-
-#endif
diff --git a/include/mbgl/storage/response.hpp b/include/mbgl/storage/response.hpp
index 9357ad3c63..cf22d9002b 100644
--- a/include/mbgl/storage/response.hpp
+++ b/include/mbgl/storage/response.hpp
@@ -2,25 +2,21 @@
 #define MBGL_STORAGE_RESPONSE
 
 #include <string>
-#include <ctime>
 
 namespace mbgl {
 
-
-
 class Response {
 public:
-    long code = 0;
+    enum Status : bool { Error, Successful };
+
+    Status status = Error;
+    std::string message;
     int64_t modified = 0;
     int64_t expires = 0;
     std::string etag;
     std::string data;
-
-    std::string message;
-
-    static int64_t parseCacheControl(const char *value);
 };
 
 }
 
-#endif
\ No newline at end of file
+#endif
diff --git a/include/mbgl/util/async_queue.hpp b/include/mbgl/util/async_queue.hpp
new file mode 100644
index 0000000000..b3eaabc319
--- /dev/null
+++ b/include/mbgl/util/async_queue.hpp
@@ -0,0 +1,95 @@
+#ifndef MBGL_UTIL_ASYNC_QUEUE
+#define MBGL_UTIL_ASYNC_QUEUE
+
+#include "std.hpp"
+
+#include <uv.h>
+
+#include <thread>
+#include <mutex>
+#include <functional>
+#include <queue>
+#include <string>
+
+
+#if UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR <= 10
+#define UV_ASYNC_PARAMS(handle) uv_async_t *handle, int
+#else
+#define UV_ASYNC_PARAMS(handle) uv_async_t *handle
+#endif
+
+namespace mbgl {
+namespace util {
+
+template <typename T>
+class AsyncQueue {
+public:
+    AsyncQueue(uv_loop_t *loop, std::function<void(T &)> fn) :
+          callback(fn) {
+        async.data = this;
+        uv_async_init(loop, &async, [](UV_ASYNC_PARAMS(handle)) {
+            auto q = reinterpret_cast<AsyncQueue *>(handle->data);
+            q->process();
+        });
+    }
+
+    void send(T &&data) {
+        {
+            std::lock_guard<std::mutex> lock(mutex);
+            queue.push(util::make_unique<T>(std::move(data)));
+        }
+        uv_async_send(&async);
+    }
+
+    void send(std::unique_ptr<T> data) {
+        {
+            std::lock_guard<std::mutex> lock(mutex);
+            queue.push(std::move(data));
+        }
+        uv_async_send(&async);
+    }
+
+    void stop() {
+        uv_close((uv_handle_t *)&async, [](uv_handle_t *handle) {
+            delete reinterpret_cast<AsyncQueue *>(handle->data);
+        });
+    }
+
+    void ref() {
+        uv_ref((uv_handle_t *)&async);
+    }
+
+    void unref() {
+        uv_unref((uv_handle_t *)&async);
+    }
+
+private:
+    ~AsyncQueue() {
+    }
+
+    void process() {
+        std::unique_ptr<T> item;
+        while (true) {
+            mutex.lock();
+            if (queue.empty()) {
+                mutex.unlock();
+                break;
+            }
+            item = std::move(queue.front());
+            queue.pop();
+            mutex.unlock();
+            callback(*item);
+        }
+    }
+
+private:
+    std::mutex mutex;
+    uv_async_t async;
+    std::queue<std::unique_ptr<T>> queue;
+    std::function<void(T &)> callback;
+};
+
+}
+}
+
+#endif
diff --git a/include/mbgl/util/util.hpp b/include/mbgl/util/util.hpp
new file mode 100644
index 0000000000..bf5dad3c01
--- /dev/null
+++ b/include/mbgl/util/util.hpp
@@ -0,0 +1,15 @@
+#ifndef MBGL_UTIL_UTIL
+#define MBGL_UTIL_UTIL
+
+#include <thread>
+
+#ifndef NDEBUG
+#include <thread>
+#define MBGL_STORE_THREAD(tid) const std::thread::id tid = std::this_thread::get_id();
+#define MBGL_VERIFY_THREAD(tid) assert(tid == std::this_thread::get_id());
+#else
+#define MBGL_STORE_THREAD(tid)
+#define MBGL_VERIFY_THREAD(tid)
+#endif
+
+#endif
diff --git a/include/mbgl/util/uv.hpp b/include/mbgl/util/uv.hpp
index f59037c1d8..85f93e78bd 100644
--- a/include/mbgl/util/uv.hpp
+++ b/include/mbgl/util/uv.hpp
@@ -3,10 +3,11 @@
 
 #include <string>
 
+typedef struct uv_handle_s uv_handle_t;
 typedef struct uv_async_s uv_async_t;
 typedef struct uv_timer_s uv_timer_t;
-typedef struct uv_handle_s uv_handle_t;
 typedef struct uv_loop_s uv_loop_t;
+typedef struct uv_fs_s uv_fs_t;
 
 namespace uv {
 
@@ -19,6 +20,15 @@ class worker;
 class mutex;
 class cond;
 
+const char *getFileRequestError(uv_fs_t *req);
+
+template <typename T>
+void close(T *specific) {
+    uv_close(reinterpret_cast<uv_handle_t *>(specific), [](uv_handle_t *generic) {
+        delete reinterpret_cast<T *>(generic);
+    });
+}
+
 }
 
 #endif
diff --git a/include/mbgl/util/variant.hpp b/include/mbgl/util/variant.hpp
index 2de195cd69..411f1918d5 100644
--- a/include/mbgl/util/variant.hpp
+++ b/include/mbgl/util/variant.hpp
@@ -10,7 +10,7 @@
 #include <iosfwd>
 #include <string>
 
-#include <mbgl/util/recursive_wrapper.hpp>
+#include "recursive_wrapper.hpp"
 
 #ifdef _MSC_VER
  // http://msdn.microsoft.com/en-us/library/z8y1yy88.aspx
@@ -34,7 +34,19 @@
 // translates to 100
 #define VARIANT_VERSION (VARIANT_MAJOR_VERSION*100000) + (VARIANT_MINOR_VERSION*100) + (VARIANT_PATCH_VERSION)
 
-namespace mapbox { namespace util { namespace detail {
+namespace mapbox { namespace util {
+
+// static visitor
+template <typename R = void>
+struct static_visitor
+{
+    using result_type = R;
+protected:
+    static_visitor() {}
+    ~static_visitor() {}
+};
+
+namespace detail {
 
 static constexpr std::size_t invalid_value = std::size_t(-1);
 
@@ -109,18 +121,38 @@ struct select_type<0, T, Types...>
     using type = T;
 };
 
-} // namespace detail
 
-// static visitor
-template <typename R = void>
-struct static_visitor
+template <typename T, typename R = void>
+struct enable_if_type { using type = R; };
+
+template <typename F, typename V, typename Enable = void>
+struct result_of_unary_visit
 {
-    using result_type = R;
-protected:
-    static_visitor() {}
-    ~static_visitor() {}
+    using type = typename std::result_of<F(V&)>::type;
 };
 
+template <typename F, typename V>
+struct result_of_unary_visit<F, V, typename enable_if_type<typename F::result_type>::type >
+{
+    using type = typename F::result_type;
+};
+
+template <typename F, typename V, class Enable = void>
+struct result_of_binary_visit
+{
+    using type = typename std::result_of<F(V&,V&)>::type;
+};
+
+
+template <typename F, typename V>
+struct result_of_binary_visit<F, V, typename enable_if_type<typename F::result_type>::type >
+{
+    using type = typename F::result_type;
+};
+
+
+} // namespace detail
+
 
 template <std::size_t arg1, std::size_t ... others>
 struct static_max;
@@ -225,7 +257,7 @@ struct dispatcher;
 template <typename F, typename V, typename T, typename...Types>
 struct dispatcher<F, V, T, Types...>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_unary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const& v, F f)
     {
         if (v.get_type_index() == sizeof...(Types))
@@ -254,7 +286,7 @@ struct dispatcher<F, V, T, Types...>
 template<typename F, typename V>
 struct dispatcher<F, V>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_unary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const&, F)
     {
         throw std::runtime_error(std::string("unary dispatch: FAIL ") + typeid(V).name());
@@ -273,7 +305,7 @@ struct binary_dispatcher_rhs;
 template <typename F, typename V, typename T0, typename T1, typename...Types>
 struct binary_dispatcher_rhs<F, V, T0, T1, Types...>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_binary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const& lhs, V const& rhs, F f)
     {
         if (rhs.get_type_index() == sizeof...(Types)) // call binary functor
@@ -305,7 +337,7 @@ struct binary_dispatcher_rhs<F, V, T0, T1, Types...>
 template<typename F, typename V, typename T>
 struct binary_dispatcher_rhs<F, V, T>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_binary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const&, V const&, F)
     {
         throw std::runtime_error("binary dispatch: FAIL");
@@ -323,7 +355,7 @@ struct binary_dispatcher_lhs;
 template <typename F, typename V, typename T0, typename T1, typename...Types>
 struct binary_dispatcher_lhs<F, V, T0, T1, Types...>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_binary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const& lhs, V const& rhs, F f)
     {
         if (lhs.get_type_index() == sizeof...(Types)) // call binary functor
@@ -353,7 +385,7 @@ struct binary_dispatcher_lhs<F, V, T0, T1, Types...>
 template<typename F, typename V, typename T>
 struct binary_dispatcher_lhs<F, V, T>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_binary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const&, V const&, F)
     {
         throw std::runtime_error("binary dispatch: FAIL");
@@ -371,7 +403,7 @@ struct binary_dispatcher;
 template <typename F, typename V, typename T, typename...Types>
 struct binary_dispatcher<F, V, T, Types...>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_binary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const& v0, V const& v1, F f)
     {
         if (v0.get_type_index() == sizeof...(Types))
@@ -416,7 +448,7 @@ struct binary_dispatcher<F, V, T, Types...>
 template<typename F, typename V>
 struct binary_dispatcher<F, V>
 {
-    using result_type = typename F::result_type;
+    using result_type = typename detail::result_of_binary_visit<F, V>::type;
     VARIANT_INLINE static result_type apply_const(V const&, V const&, F)
     {
         throw std::runtime_error("binary dispatch: FAIL");
@@ -448,7 +480,7 @@ struct less_comp
 };
 
 template <typename Variant, typename Comp>
-class comparer : public static_visitor<bool>
+class comparer
 {
 public:
     explicit comparer(Variant const& lhs) noexcept
@@ -467,7 +499,7 @@ private:
 
 // operator<< helper
 template <typename Out>
-class printer : public static_visitor<>
+class printer
 {
 public:
     explicit printer(Out & out)