summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cmake/core-files.cmake2
-rw-r--r--cmake/test-files.cmake1
-rw-r--r--include/mbgl/storage/response.hpp6
-rw-r--r--include/mbgl/util/chrono.hpp2
-rw-r--r--include/mbgl/util/constants.hpp2
-rw-r--r--platform/default/online_file_source.cpp32
-rw-r--r--src/mbgl/storage/response.cpp6
-rw-r--r--src/mbgl/util/chrono.cpp4
-rw-r--r--src/mbgl/util/http_header.cpp20
-rw-r--r--src/mbgl/util/http_header.hpp3
-rw-r--r--src/mbgl/util/http_timeout.cpp40
-rw-r--r--src/mbgl/util/http_timeout.hpp15
-rw-r--r--test/storage/online_file_source.cpp44
-rwxr-xr-xtest/storage/server.js11
-rw-r--r--test/util/http_timeout.cpp54
15 files changed, 213 insertions, 29 deletions
diff --git a/cmake/core-files.cmake b/cmake/core-files.cmake
index adc37dc71b..eafe513d22 100644
--- a/cmake/core-files.cmake
+++ b/cmake/core-files.cmake
@@ -421,6 +421,8 @@ set(MBGL_CORE_FILES
src/mbgl/util/grid_index.hpp
src/mbgl/util/http_header.cpp
src/mbgl/util/http_header.hpp
+ src/mbgl/util/http_timeout.cpp
+ src/mbgl/util/http_timeout.hpp
src/mbgl/util/interpolate.hpp
src/mbgl/util/intersection_tests.cpp
src/mbgl/util/intersection_tests.hpp
diff --git a/cmake/test-files.cmake b/cmake/test-files.cmake
index 8b27e3dac1..0a4a6e3c41 100644
--- a/cmake/test-files.cmake
+++ b/cmake/test-files.cmake
@@ -87,6 +87,7 @@ set(MBGL_TEST_FILES
# util
test/util/async_task.cpp
test/util/geo.cpp
+ test/util/http_timeout.cpp
test/util/image.cpp
test/util/mapbox.cpp
test/util/memory.cpp
diff --git a/include/mbgl/storage/response.hpp b/include/mbgl/storage/response.hpp
index 4dc3f36681..448423c1cb 100644
--- a/include/mbgl/storage/response.hpp
+++ b/include/mbgl/storage/response.hpp
@@ -2,6 +2,7 @@
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/optional.hpp>
+#include <mbgl/util/variant.hpp>
#include <string>
#include <memory>
@@ -45,15 +46,18 @@ public:
NotFound = 2,
Server = 3,
Connection = 4,
+ RateLimit = 5,
Other = 6,
} reason = Reason::Other;
// An error message from the request handler, e.g. a server message or a system message
// informing the user about the reason for the failure.
std::string message;
+
+ optional<Timestamp> retryAfter;
public:
- Error(Reason, std::string = "");
+ Error(Reason, std::string = "", optional<Timestamp> = {});
};
std::ostream& operator<<(std::ostream&, Response::Error::Reason);
diff --git a/include/mbgl/util/chrono.hpp b/include/mbgl/util/chrono.hpp
index 48460d3377..4adf030331 100644
--- a/include/mbgl/util/chrono.hpp
+++ b/include/mbgl/util/chrono.hpp
@@ -29,6 +29,8 @@ std::string rfc1123(Timestamp);
std::string iso8601(Timestamp);
Timestamp parseTimestamp(const char *);
+
+Timestamp parseTimestamp(const int32_t timestamp);
// C++17 polyfill
template <class Rep, class Period, class = std::enable_if_t<
diff --git a/include/mbgl/util/constants.hpp b/include/mbgl/util/constants.hpp
index 3c0b3eb93e..75a1ace5c5 100644
--- a/include/mbgl/util/constants.hpp
+++ b/include/mbgl/util/constants.hpp
@@ -42,6 +42,8 @@ constexpr Duration DEFAULT_FADE_DURATION = Milliseconds(300);
constexpr Seconds CLOCK_SKEW_RETRY_TIMEOUT { 30 };
constexpr UnitBezier DEFAULT_TRANSITION_EASE = { 0, 0, 0.25, 1 };
+
+constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5;
} // namespace util
diff --git a/platform/default/online_file_source.cpp b/platform/default/online_file_source.cpp
index b7d658694c..44245b9c13 100644
--- a/platform/default/online_file_source.cpp
+++ b/platform/default/online_file_source.cpp
@@ -13,6 +13,7 @@
#include <mbgl/util/async_task.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/timer.hpp>
+#include <mbgl/util/http_timeout.hpp>
#include <algorithm>
#include <cassert>
@@ -48,6 +49,7 @@ public:
// backoff when retrying requests.
uint32_t failedRequests = 0;
Response::Error::Reason failedRequestReason = Response::Error::Reason::Success;
+ optional<Timestamp> retryAfter;
};
class OnlineFileSource::Impl {
@@ -200,30 +202,6 @@ OnlineFileRequest::~OnlineFileRequest() {
impl.remove(this);
}
-static Duration errorRetryTimeout(Response::Error::Reason failedRequestReason, uint32_t failedRequests) {
- if (failedRequestReason == Response::Error::Reason::Server) {
- // Retry after one second three times, then start exponential backoff.
- return Seconds(failedRequests <= 3 ? 1 : 1 << std::min(failedRequests - 3, 31u));
- } else if (failedRequestReason == Response::Error::Reason::Connection) {
- // Immediate exponential backoff.
- assert(failedRequests > 0);
- return Seconds(1 << std::min(failedRequests - 1, 31u));
- } else {
- // No error, or not an error that triggers retries.
- return Duration::max();
- }
-}
-
-static Duration expirationTimeout(optional<Timestamp> expires, uint32_t expiredRequests) {
- if (expiredRequests) {
- return Seconds(1 << std::min(expiredRequests - 1, 31u));
- } else if (expires) {
- return std::max(Seconds::zero(), *expires - util::now());
- } else {
- return Duration::max();
- }
-}
-
Timestamp interpolateExpiration(const Timestamp& current,
optional<Timestamp> prior,
bool& expired) {
@@ -267,8 +245,9 @@ void OnlineFileRequest::schedule(optional<Timestamp> expires) {
// If we're not being asked for a forced refresh, calculate a timeout that depends on how many
// consecutive errors we've encountered, and on the expiration time, if present.
- Duration timeout = std::min(errorRetryTimeout(failedRequestReason, failedRequests),
- expirationTimeout(expires, expiredRequests));
+ Duration timeout = std::min(
+ http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter),
+ http::expirationTimeout(expires, expiredRequests));
if (timeout == Duration::max()) {
return;
@@ -321,6 +300,7 @@ void OnlineFileRequest::completed(Response response) {
if (response.error) {
failedRequests++;
failedRequestReason = response.error->reason;
+ retryAfter = response.error->retryAfter;
} else {
failedRequests = 0;
failedRequestReason = Response::Error::Reason::Success;
diff --git a/src/mbgl/storage/response.cpp b/src/mbgl/storage/response.cpp
index 6809220166..a174339074 100644
--- a/src/mbgl/storage/response.cpp
+++ b/src/mbgl/storage/response.cpp
@@ -21,8 +21,8 @@ Response& Response::operator=(const Response& res) {
return *this;
}
-Response::Error::Error(Reason reason_, std::string message_)
- : reason(reason_), message(std::move(message_)) {
+Response::Error::Error(Reason reason_, std::string message_, optional<Timestamp> retryAfter_)
+ : reason(reason_), message(std::move(message_)), retryAfter(std::move(retryAfter_)) {
}
std::ostream& operator<<(std::ostream& os, Response::Error::Reason r) {
@@ -35,6 +35,8 @@ std::ostream& operator<<(std::ostream& os, Response::Error::Reason r) {
return os << "Response::Error::Reason::Server";
case Response::Error::Reason::Connection:
return os << "Response::Error::Reason::Connection";
+ case Response::Error::Reason::RateLimit:
+ return os << "Response::Error::Reason::RateLimit";
case Response::Error::Reason::Other:
return os << "Response::Error::Reason::Other";
}
diff --git a/src/mbgl/util/chrono.cpp b/src/mbgl/util/chrono.cpp
index 55b8a86e39..f338f524b9 100644
--- a/src/mbgl/util/chrono.cpp
+++ b/src/mbgl/util/chrono.cpp
@@ -33,6 +33,10 @@ std::string iso8601(Timestamp timestamp) {
Timestamp parseTimestamp(const char* timestamp) {
return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::from_time_t(parse_date(timestamp)));
}
+
+Timestamp parseTimestamp(const int32_t timestamp) {
+ return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::from_time_t(timestamp));
+}
} // namespace util
diff --git a/src/mbgl/util/http_header.cpp b/src/mbgl/util/http_header.cpp
index dceb331a3f..e337d4c8ab 100644
--- a/src/mbgl/util/http_header.cpp
+++ b/src/mbgl/util/http_header.cpp
@@ -28,6 +28,26 @@ CacheControl CacheControl::parse(const std::string& value) {
optional<Timestamp> CacheControl::toTimePoint() const {
return maxAge ? util::now() + Seconds(*maxAge) : optional<Timestamp>{};
}
+
+optional<Timestamp> parseRetryHeaders(const optional<std::string>& retryAfter,
+ const optional<std::string>& xRateLimitReset) {
+ if (retryAfter) {
+ try {
+ auto secs = std::chrono::seconds(std::stoi(*retryAfter));
+ return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::now() + secs);
+ } catch (...) {
+ return util::parseTimestamp((*retryAfter).c_str());
+ }
+ } else if (xRateLimitReset) {
+ try {
+ return util::parseTimestamp(std::stoi(*xRateLimitReset));
+ } catch (...) {
+ return {};
+ }
+ }
+
+ return {};
+}
} // namespace http
} // namespace mbgl
diff --git a/src/mbgl/util/http_header.hpp b/src/mbgl/util/http_header.hpp
index 8a83356977..fa76cb724e 100644
--- a/src/mbgl/util/http_header.hpp
+++ b/src/mbgl/util/http_header.hpp
@@ -17,6 +17,9 @@ public:
optional<Timestamp> toTimePoint() const;
};
+
+optional<Timestamp> parseRetryHeaders(const optional<std::string>& retryAfter,
+ const optional<std::string>& xRateLimitReset);
} // namespace http
} // namespace mbgl
diff --git a/src/mbgl/util/http_timeout.cpp b/src/mbgl/util/http_timeout.cpp
new file mode 100644
index 0000000000..ded0128ac9
--- /dev/null
+++ b/src/mbgl/util/http_timeout.cpp
@@ -0,0 +1,40 @@
+#include <mbgl/util/http_timeout.hpp>
+#include <mbgl/util/constants.hpp>
+
+namespace mbgl {
+namespace http {
+
+Duration errorRetryTimeout(Response::Error::Reason failedRequestReason, uint32_t failedRequests, optional<Timestamp> retryAfter) {
+
+ if (failedRequestReason == Response::Error::Reason::Server) {
+ // Retry after one second three times, then start exponential backoff.
+ return Seconds(failedRequests <= 3 ? 1 : 1u << std::min(failedRequests - 3, 31u));
+ } else if (failedRequestReason == Response::Error::Reason::Connection) {
+ // Immediate exponential backoff.
+ assert(failedRequests > 0);
+ return Seconds(1u << std::min(failedRequests - 1, 31u));
+ } else if (failedRequestReason == Response::Error::Reason::RateLimit) {
+ if (retryAfter) {
+ return *retryAfter - util::now();
+ } else {
+ //Default
+ return Seconds(util::DEFAULT_RATE_LIMIT_TIMEOUT);
+ }
+ } else {
+ // No error, or not an error that triggers retries.
+ return Duration::max();
+ }
+}
+
+Duration expirationTimeout(optional<Timestamp> expires, uint32_t expiredRequests) {
+ if (expiredRequests) {
+ return Seconds(1u << std::min(expiredRequests - 1, 31u));
+ } else if (expires) {
+ return std::max(Seconds::zero(), *expires - util::now());
+ } else {
+ return Duration::max();
+ }
+}
+
+} // namespace http
+} // namespace mbgl
diff --git a/src/mbgl/util/http_timeout.hpp b/src/mbgl/util/http_timeout.hpp
new file mode 100644
index 0000000000..b694794997
--- /dev/null
+++ b/src/mbgl/util/http_timeout.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/chrono.hpp>
+
+namespace mbgl {
+namespace http {
+
+Duration errorRetryTimeout(Response::Error::Reason failedRequestReason, uint32_t failedRequests, optional<Timestamp> retryAfter = {});
+
+Duration expirationTimeout(optional<Timestamp> expires, uint32_t expiredRequests);
+
+} // namespace http
+} // namespace mbgl
diff --git a/test/storage/online_file_source.cpp b/test/storage/online_file_source.cpp
index 18179c8448..f67d96f257 100644
--- a/test/storage/online_file_source.cpp
+++ b/test/storage/online_file_source.cpp
@@ -359,3 +359,47 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusOnlineOffline)) {
loop.run();
}
+
+TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitStandard)) {
+ util::RunLoop loop;
+ OnlineFileSource fs;
+
+ auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/rate-limit?std=true" }, [&](Response res) {
+ ASSERT_NE(nullptr, res.error);
+ EXPECT_EQ(Response::Error::Reason::RateLimit, res.error->reason);
+ ASSERT_EQ(true, bool(res.error->retryAfter));
+ ASSERT_LT(util::now(), res.error->retryAfter);
+ loop.stop();
+ });
+
+ loop.run();
+}
+
+TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitMBX)) {
+ util::RunLoop loop;
+ OnlineFileSource fs;
+
+ auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/rate-limit?mbx=true" }, [&](Response res) {
+ ASSERT_NE(nullptr, res.error);
+ EXPECT_EQ(Response::Error::Reason::RateLimit, res.error->reason);
+ ASSERT_EQ(true, bool(res.error->retryAfter));
+ ASSERT_LT(util::now(), res.error->retryAfter);
+ loop.stop();
+ });
+
+ loop.run();
+}
+
+TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitDefault)) {
+ util::RunLoop loop;
+ OnlineFileSource fs;
+
+ auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/rate-limit" }, [&](Response res) {
+ ASSERT_NE(nullptr, res.error);
+ EXPECT_EQ(Response::Error::Reason::RateLimit, res.error->reason);
+ ASSERT_EQ(false, bool(res.error->retryAfter));
+ loop.stop();
+ });
+
+ loop.run();
+}
diff --git a/test/storage/server.js b/test/storage/server.js
index c2f30abc49..a7538b55f1 100755
--- a/test/storage/server.js
+++ b/test/storage/server.js
@@ -116,6 +116,17 @@ app.get('/temporary-error', function(req, res) {
temporaryErrorCounter++;
});
+app.get('/rate-limit', function(req, res) {
+
+ if (req.query.std) {
+ res.setHeader('Retry-After', 1);
+ } else if (req.query.mbx) {
+ res.setHeader('x-rate-limit-reset', Math.round(Date.now() / 1000) + 1);
+ }
+
+ res.status(429).end();
+});
+
app.get('/delayed', function(req, res) {
setTimeout(function() {
res.status(200).send('Response');
diff --git a/test/util/http_timeout.cpp b/test/util/http_timeout.cpp
new file mode 100644
index 0000000000..365c40523f
--- /dev/null
+++ b/test/util/http_timeout.cpp
@@ -0,0 +1,54 @@
+#include <mbgl/test/util.hpp>
+
+#include <mbgl/platform/log.hpp>
+#include <mbgl/util/http_timeout.hpp>
+#include <regex>
+#include <iostream>
+
+using namespace mbgl;
+using namespace mbgl::http;
+
+TEST(HttpRetry, OtherError) {
+ //Non-retryable
+ ASSERT_EQ(Duration::max(), errorRetryTimeout(Response::Error::Reason::Other, 1));
+}
+
+TEST(HttpRetry, ServerError) {
+ // 1-3 failures -> 1 sec
+ ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 1));
+ ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 3));
+
+ // After 3, exponential backoff
+ ASSERT_EQ(Seconds(2), errorRetryTimeout(Response::Error::Reason::Server, 4));
+ ASSERT_EQ(Seconds(1u << 31), errorRetryTimeout(Response::Error::Reason::Server, 50));
+}
+
+TEST(HttpRetry, ConnectionError) {
+ // Exponential backoff
+ ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 1));
+ ASSERT_EQ(Seconds(1u << 31), errorRetryTimeout(Response::Error::Reason::Connection, 50));
+}
+
+TEST(HttpRetry, RateLimit) {
+ // Pre-set value from header
+ ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 1, { util::now() + Seconds(1) }));
+
+ //Default
+ ASSERT_EQ(Seconds(5), errorRetryTimeout(Response::Error::Reason::RateLimit, 1, {}));
+}
+
+TEST(HttpRetry, ExpiredInitial) {
+ // 1 sec timeout
+ ASSERT_EQ(Seconds(1), expirationTimeout({ util::now() + Seconds(1) }, 0));
+}
+
+TEST(HttpRetry, ExpiredSubsequent) {
+ // exponential backoff
+ ASSERT_EQ(Seconds(1), expirationTimeout({}, 1));
+ ASSERT_EQ(Seconds(1u << 31), expirationTimeout({}, 50));
+}
+
+TEST(HttpRetry, ExpiredNotSet) {
+ // No expires header set
+ ASSERT_EQ(Duration::max(), expirationTimeout({}, 0));
+}
View Lorry Controller Status