summaryrefslogtreecommitdiff
path: root/Source/WebKit2/NetworkProcess/cache
diff options
context:
space:
mode:
authorLorry Tar Creator <lorry-tar-importer@lorry>2017-06-27 06:07:23 +0000
committerLorry Tar Creator <lorry-tar-importer@lorry>2017-06-27 06:07:23 +0000
commit1bf1084f2b10c3b47fd1a588d85d21ed0eb41d0c (patch)
tree46dcd36c86e7fbc6e5df36deb463b33e9967a6f7 /Source/WebKit2/NetworkProcess/cache
parent32761a6cee1d0dee366b885b7b9c777e67885688 (diff)
downloadWebKitGtk-tarball-master.tar.gz
Diffstat (limited to 'Source/WebKit2/NetworkProcess/cache')
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp612
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCache.h160
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp153
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h75
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp65
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h57
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp194
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp94
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp163
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h169
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp142
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp246
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h99
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp170
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h57
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h87
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp285
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp187
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h141
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp162
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h87
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp589
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h93
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp450
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h95
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp1048
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h193
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp175
-rw-r--r--Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h106
29 files changed, 6154 insertions, 0 deletions
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp
new file mode 100644
index 000000000..bbedce3f1
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp
@@ -0,0 +1,612 @@
+/*
+ * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCache.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "Logging.h"
+#include "NetworkCacheSpeculativeLoadManager.h"
+#include "NetworkCacheStatistics.h"
+#include "NetworkCacheStorage.h"
+#include <WebCore/CacheValidation.h>
+#include <WebCore/FileSystem.h>
+#include <WebCore/HTTPHeaderNames.h>
+#include <WebCore/NetworkStorageSession.h>
+#include <WebCore/PlatformCookieJar.h>
+#include <WebCore/ResourceRequest.h>
+#include <WebCore/ResourceResponse.h>
+#include <WebCore/SharedBuffer.h>
+#include <wtf/MainThread.h>
+#include <wtf/NeverDestroyed.h>
+#include <wtf/RunLoop.h>
+#include <wtf/text/StringBuilder.h>
+
+#if PLATFORM(COCOA)
+#include <notify.h>
+#endif
+
+namespace WebKit {
+namespace NetworkCache {
+
+static const AtomicString& resourceType()
+{
+ ASSERT(WTF::isMainThread());
+ static NeverDestroyed<const AtomicString> resource("Resource", AtomicString::ConstructFromLiteral);
+ return resource;
+}
+
+Cache& singleton()
+{
+ static NeverDestroyed<Cache> instance;
+ return instance;
+}
+
+#if PLATFORM(GTK)
+static void dumpFileChanged(Cache* cache)
+{
+ cache->dumpContentsToFile();
+}
+#endif
+
+bool Cache::initialize(const String& cachePath, const Parameters& parameters)
+{
+ m_storage = Storage::open(cachePath);
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+ if (parameters.enableNetworkCacheSpeculativeRevalidation)
+ m_speculativeLoadManager = std::make_unique<SpeculativeLoadManager>(*m_storage);
+#endif
+
+ if (parameters.enableEfficacyLogging)
+ m_statistics = Statistics::open(cachePath);
+
+#if PLATFORM(COCOA)
+ // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump".
+ if (m_storage) {
+ int token;
+ notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) {
+ dumpContentsToFile();
+ });
+ }
+#endif
+#if PLATFORM(GTK)
+ // Triggers with "touch $cachePath/dump".
+ if (m_storage) {
+ CString dumpFilePath = WebCore::fileSystemRepresentation(WebCore::pathByAppendingComponent(m_storage->basePath(), "dump"));
+ GRefPtr<GFile> dumpFile = adoptGRef(g_file_new_for_path(dumpFilePath.data()));
+ GFileMonitor* monitor = g_file_monitor_file(dumpFile.get(), G_FILE_MONITOR_NONE, nullptr, nullptr);
+ g_signal_connect_swapped(monitor, "changed", G_CALLBACK(dumpFileChanged), this);
+ }
+#endif
+
+ LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage);
+ return !!m_storage;
+}
+
+void Cache::setCapacity(size_t maximumSize)
+{
+ if (!m_storage)
+ return;
+ m_storage->setCapacity(maximumSize);
+}
+
+Key Cache::makeCacheKey(const WebCore::ResourceRequest& request)
+{
+ // FIXME: This implements minimal Range header disk cache support. We don't parse
+ // ranges so only the same exact range request will be served from the cache.
+ String range = request.httpHeaderField(WebCore::HTTPHeaderName::Range);
+ return { request.cachePartition(), resourceType(), range, request.url().string(), m_storage->salt() };
+}
+
+static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy)
+{
+ switch (policy) {
+ case WebCore::ReturnCacheDataElseLoad:
+ case WebCore::ReturnCacheDataDontLoad:
+ return true;
+ case WebCore::UseProtocolCachePolicy:
+ case WebCore::ReloadIgnoringCacheData:
+ case WebCore::RefreshAnyCacheData:
+ return false;
+ case WebCore::DoNotUseAnyCache:
+ ASSERT_NOT_REACHED();
+ return false;
+ }
+ return false;
+}
+
+static bool responseHasExpired(const WebCore::ResourceResponse& response, std::chrono::system_clock::time_point timestamp, std::optional<std::chrono::microseconds> maxStale)
+{
+ if (response.cacheControlContainsNoCache())
+ return true;
+
+ auto age = WebCore::computeCurrentAge(response, timestamp);
+ auto lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(response, timestamp);
+
+ auto maximumStaleness = maxStale ? maxStale.value() : 0ms;
+ bool hasExpired = age - lifetime > maximumStaleness;
+
+#ifndef LOG_DISABLED
+ if (hasExpired)
+ LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasExpired age=%f lifetime=%f max-stale=%g", age, lifetime, maxStale);
+#endif
+
+ return hasExpired;
+}
+
+static bool responseNeedsRevalidation(const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& request, std::chrono::system_clock::time_point timestamp)
+{
+ auto requestDirectives = WebCore::parseCacheControlDirectives(request.httpHeaderFields());
+ if (requestDirectives.noCache)
+ return true;
+ // For requests we ignore max-age values other than zero.
+ if (requestDirectives.maxAge && requestDirectives.maxAge.value() == 0ms)
+ return true;
+
+ return responseHasExpired(response, timestamp, requestDirectives.maxStale);
+}
+
+static UseDecision makeUseDecision(const Entry& entry, const WebCore::ResourceRequest& request)
+{
+ // The request is conditional so we force revalidation from the network. We merely check the disk cache
+ // so we can update the cache entry.
+ if (request.isConditional() && !entry.redirectRequest())
+ return UseDecision::Validate;
+
+ if (!WebCore::verifyVaryingRequestHeaders(entry.varyingRequestHeaders(), request))
+ return UseDecision::NoDueToVaryingHeaderMismatch;
+
+ // We never revalidate in the case of a history navigation.
+ if (cachePolicyAllowsExpired(request.cachePolicy()))
+ return UseDecision::Use;
+
+ if (!responseNeedsRevalidation(entry.response(), request, entry.timeStamp()))
+ return UseDecision::Use;
+
+ if (!entry.response().hasCacheValidatorFields())
+ return UseDecision::NoDueToMissingValidatorFields;
+
+ return entry.redirectRequest() ? UseDecision::NoDueToExpiredRedirect : UseDecision::Validate;
+}
+
+static RetrieveDecision makeRetrieveDecision(const WebCore::ResourceRequest& request)
+{
+ ASSERT(request.cachePolicy() != WebCore::DoNotUseAnyCache);
+
+ // FIXME: Support HEAD requests.
+ if (request.httpMethod() != "GET")
+ return RetrieveDecision::NoDueToHTTPMethod;
+ if (request.requester() == WebCore::ResourceRequest::Requester::Media)
+ return RetrieveDecision::NoDueToStreamingMedia;
+ if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData && !request.isConditional())
+ return RetrieveDecision::NoDueToReloadIgnoringCache;
+
+ return RetrieveDecision::Yes;
+}
+
+static bool isMediaMIMEType(const String& mimeType)
+{
+ if (mimeType.startsWith("video/", /*caseSensitive*/ false))
+ return true;
+ if (mimeType.startsWith("audio/", /*caseSensitive*/ false))
+ return true;
+ return false;
+}
+
+static StoreDecision makeStoreDecision(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response)
+{
+ if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP())
+ return StoreDecision::NoDueToProtocol;
+
+ if (originalRequest.httpMethod() != "GET")
+ return StoreDecision::NoDueToHTTPMethod;
+
+ auto requestDirectives = WebCore::parseCacheControlDirectives(originalRequest.httpHeaderFields());
+ if (requestDirectives.noStore)
+ return StoreDecision::NoDueToNoStoreRequest;
+
+ if (response.cacheControlContainsNoStore())
+ return StoreDecision::NoDueToNoStoreResponse;
+
+ if (!WebCore::isStatusCodeCacheableByDefault(response.httpStatusCode())) {
+ // http://tools.ietf.org/html/rfc7234#section-4.3.2
+ bool hasExpirationHeaders = response.expires() || response.cacheControlMaxAge();
+ bool expirationHeadersAllowCaching = WebCore::isStatusCodePotentiallyCacheable(response.httpStatusCode()) && hasExpirationHeaders;
+ if (!expirationHeadersAllowCaching)
+ return StoreDecision::NoDueToHTTPStatusCode;
+ }
+
+ bool isMainResource = originalRequest.requester() == WebCore::ResourceRequest::Requester::Main;
+ bool storeUnconditionallyForHistoryNavigation = isMainResource || originalRequest.priority() == WebCore::ResourceLoadPriority::VeryHigh;
+ if (!storeUnconditionallyForHistoryNavigation) {
+ auto now = std::chrono::system_clock::now();
+ bool hasNonZeroLifetime = !response.cacheControlContainsNoCache() && WebCore::computeFreshnessLifetimeForHTTPFamily(response, now) > 0ms;
+
+ bool possiblyReusable = response.hasCacheValidatorFields() || hasNonZeroLifetime;
+ if (!possiblyReusable)
+ return StoreDecision::NoDueToUnlikelyToReuse;
+ }
+
+ // Media loaded via XHR is likely being used for MSE streaming (YouTube and Netflix for example).
+ // Streaming media fills the cache quickly and is unlikely to be reused.
+ // FIXME: We should introduce a separate media cache partition that doesn't affect other resources.
+ // FIXME: We should also make sure make the MSE paths are copy-free so we can use mapped buffers from disk effectively.
+ auto requester = originalRequest.requester();
+ bool isDefinitelyStreamingMedia = requester == WebCore::ResourceRequest::Requester::Media;
+ bool isLikelyStreamingMedia = requester == WebCore::ResourceRequest::Requester::XHR && isMediaMIMEType(response.mimeType());
+ if (isLikelyStreamingMedia || isDefinitelyStreamingMedia)
+ return StoreDecision::NoDueToStreamingMedia;
+
+ return StoreDecision::Yes;
+}
+
+void Cache::retrieve(const WebCore::ResourceRequest& request, const GlobalFrameID& frameID, Function<void (std::unique_ptr<Entry>)>&& completionHandler)
+{
+ ASSERT(isEnabled());
+ ASSERT(request.url().protocolIsInHTTPFamily());
+
+ LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %d", request.url().string().ascii().data(), static_cast<int>(request.priority()));
+
+ if (m_statistics)
+ m_statistics->recordRetrievalRequest(frameID.first);
+
+ Key storageKey = makeCacheKey(request);
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+ bool canUseSpeculativeRevalidation = m_speculativeLoadManager && !request.isConditional();
+ if (canUseSpeculativeRevalidation)
+ m_speculativeLoadManager->registerLoad(frameID, request, storageKey);
+#endif
+
+ auto retrieveDecision = makeRetrieveDecision(request);
+ if (retrieveDecision != RetrieveDecision::Yes) {
+ if (m_statistics)
+ m_statistics->recordNotUsingCacheForRequest(frameID.first, storageKey, request, retrieveDecision);
+
+ completionHandler(nullptr);
+ return;
+ }
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+ if (canUseSpeculativeRevalidation && m_speculativeLoadManager->canRetrieve(storageKey, request, frameID)) {
+ m_speculativeLoadManager->retrieve(storageKey, [request, completionHandler = WTFMove(completionHandler)](std::unique_ptr<Entry> entry) {
+ if (entry && WebCore::verifyVaryingRequestHeaders(entry->varyingRequestHeaders(), request))
+ completionHandler(WTFMove(entry));
+ else
+ completionHandler(nullptr);
+ });
+ return;
+ }
+#endif
+
+ auto startTime = std::chrono::system_clock::now();
+ auto priority = static_cast<unsigned>(request.priority());
+
+ m_storage->retrieve(storageKey, priority, [this, request, completionHandler = WTFMove(completionHandler), startTime, storageKey, frameID](auto record) {
+ if (!record) {
+ LOG(NetworkCache, "(NetworkProcess) not found in storage");
+
+ if (m_statistics)
+ m_statistics->recordRetrievalFailure(frameID.first, storageKey, request);
+
+ completionHandler(nullptr);
+ return false;
+ }
+
+ ASSERT(record->key == storageKey);
+
+ auto entry = Entry::decodeStorageRecord(*record);
+
+ auto useDecision = entry ? makeUseDecision(*entry, request) : UseDecision::NoDueToDecodeFailure;
+ switch (useDecision) {
+ case UseDecision::Use:
+ break;
+ case UseDecision::Validate:
+ entry->setNeedsValidation(true);
+ break;
+ default:
+ entry = nullptr;
+ };
+
+#if !LOG_DISABLED
+ auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count());
+ LOG(NetworkCache, "(NetworkProcess) retrieve complete useDecision=%d priority=%d time=%" PRIi64 "ms", static_cast<int>(useDecision), static_cast<int>(request.priority()), elapsedMS);
+#endif
+ completionHandler(WTFMove(entry));
+
+ if (m_statistics)
+ m_statistics->recordRetrievedCachedEntry(frameID.first, storageKey, request, useDecision);
+ return useDecision != UseDecision::NoDueToDecodeFailure;
+ });
+}
+
+
+std::unique_ptr<Entry> Cache::makeEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData)
+{
+ return std::make_unique<Entry>(makeCacheKey(request), response, WTFMove(responseData), WebCore::collectVaryingRequestHeaders(request, response));
+}
+
+std::unique_ptr<Entry> Cache::makeRedirectEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest)
+{
+ return std::make_unique<Entry>(makeCacheKey(request), response, redirectRequest, WebCore::collectVaryingRequestHeaders(request, response));
+}
+
+std::unique_ptr<Entry> Cache::store(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, Function<void (MappedBody&)>&& completionHandler)
+{
+ ASSERT(isEnabled());
+ ASSERT(responseData);
+
+ LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", request.url().string().latin1().data(), makeCacheKey(request).partition().latin1().data());
+
+ StoreDecision storeDecision = makeStoreDecision(request, response);
+ if (storeDecision != StoreDecision::Yes) {
+ LOG(NetworkCache, "(NetworkProcess) didn't store, storeDecision=%d", static_cast<int>(storeDecision));
+ auto key = makeCacheKey(request);
+
+ auto isSuccessfulRevalidation = response.httpStatusCode() == 304;
+ if (!isSuccessfulRevalidation) {
+ // Make sure we don't keep a stale entry in the cache.
+ remove(key);
+ }
+
+ if (m_statistics)
+ m_statistics->recordNotCachingResponse(key, storeDecision);
+
+ return nullptr;
+ }
+
+ auto cacheEntry = makeEntry(request, response, WTFMove(responseData));
+ auto record = cacheEntry->encodeAsStorageRecord();
+
+ m_storage->store(record, [this, completionHandler = WTFMove(completionHandler)](const Data& bodyData) {
+ MappedBody mappedBody;
+#if ENABLE(SHAREABLE_RESOURCE)
+ if (canUseSharedMemoryForBodyData()) {
+ if (auto sharedMemory = bodyData.tryCreateSharedMemory()) {
+ mappedBody.shareableResource = ShareableResource::create(sharedMemory.releaseNonNull(), 0, bodyData.size());
+ ASSERT(mappedBody.shareableResource);
+ mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle);
+ }
+ }
+#endif
+ completionHandler(mappedBody);
+ LOG(NetworkCache, "(NetworkProcess) stored");
+ });
+
+ return cacheEntry;
+}
+
+std::unique_ptr<Entry> Cache::storeRedirect(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest)
+{
+ ASSERT(isEnabled());
+
+ LOG(NetworkCache, "(NetworkProcess) storing redirect %s -> %s", request.url().string().latin1().data(), redirectRequest.url().string().latin1().data());
+
+ StoreDecision storeDecision = makeStoreDecision(request, response);
+ if (storeDecision != StoreDecision::Yes) {
+ LOG(NetworkCache, "(NetworkProcess) didn't store redirect, storeDecision=%d", static_cast<int>(storeDecision));
+ auto key = makeCacheKey(request);
+ if (m_statistics)
+ m_statistics->recordNotCachingResponse(key, storeDecision);
+
+ return nullptr;
+ }
+
+ auto cacheEntry = makeRedirectEntry(request, response, redirectRequest);
+ auto record = cacheEntry->encodeAsStorageRecord();
+
+ m_storage->store(record, nullptr);
+
+ return cacheEntry;
+}
+
+std::unique_ptr<Entry> Cache::update(const WebCore::ResourceRequest& originalRequest, const GlobalFrameID& frameID, const Entry& existingEntry, const WebCore::ResourceResponse& validatingResponse)
+{
+ LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data());
+
+ WebCore::ResourceResponse response = existingEntry.response();
+ WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse);
+
+ auto updateEntry = std::make_unique<Entry>(existingEntry.key(), response, existingEntry.buffer(), WebCore::collectVaryingRequestHeaders(originalRequest, response));
+ auto updateRecord = updateEntry->encodeAsStorageRecord();
+
+ m_storage->store(updateRecord, { });
+
+ if (m_statistics)
+ m_statistics->recordRevalidationSuccess(frameID.first, existingEntry.key(), originalRequest);
+
+ return updateEntry;
+}
+
+void Cache::remove(const Key& key)
+{
+ ASSERT(isEnabled());
+
+ m_storage->remove(key);
+}
+
+void Cache::remove(const WebCore::ResourceRequest& request)
+{
+ remove(makeCacheKey(request));
+}
+
+void Cache::traverse(Function<void (const TraversalEntry*)>&& traverseHandler)
+{
+ ASSERT(isEnabled());
+
+ // Protect against clients making excessive traversal requests.
+ const unsigned maximumTraverseCount = 3;
+ if (m_traverseCount >= maximumTraverseCount) {
+ WTFLogAlways("Maximum parallel cache traverse count exceeded. Ignoring traversal request.");
+
+ RunLoop::main().dispatch([traverseHandler = WTFMove(traverseHandler)] {
+ traverseHandler(nullptr);
+ });
+ return;
+ }
+
+ ++m_traverseCount;
+
+ m_storage->traverse(resourceType(), 0, [this, traverseHandler = WTFMove(traverseHandler)](const Storage::Record* record, const Storage::RecordInfo& recordInfo) {
+ if (!record) {
+ --m_traverseCount;
+ traverseHandler(nullptr);
+ return;
+ }
+
+ auto entry = Entry::decodeStorageRecord(*record);
+ if (!entry)
+ return;
+
+ TraversalEntry traversalEntry { *entry, recordInfo };
+ traverseHandler(&traversalEntry);
+ });
+}
+
+String Cache::dumpFilePath() const
+{
+ return WebCore::pathByAppendingComponent(m_storage->versionPath(), "dump.json");
+}
+
+void Cache::dumpContentsToFile()
+{
+ if (!m_storage)
+ return;
+ auto fd = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite);
+ if (!WebCore::isHandleValid(fd))
+ return;
+ auto prologue = String("{\n\"entries\": [\n").utf8();
+ WebCore::writeToFile(fd, prologue.data(), prologue.length());
+
+ struct Totals {
+ unsigned count { 0 };
+ double worth { 0 };
+ size_t bodySize { 0 };
+ };
+ Totals totals;
+ auto flags = Storage::TraverseFlag::ComputeWorth | Storage::TraverseFlag::ShareCount;
+ size_t capacity = m_storage->capacity();
+ m_storage->traverse(resourceType(), flags, [fd, totals, capacity](const Storage::Record* record, const Storage::RecordInfo& info) mutable {
+ if (!record) {
+ StringBuilder epilogue;
+ epilogue.appendLiteral("{}\n],\n");
+ epilogue.appendLiteral("\"totals\": {\n");
+ epilogue.appendLiteral("\"capacity\": ");
+ epilogue.appendNumber(capacity);
+ epilogue.appendLiteral(",\n");
+ epilogue.appendLiteral("\"count\": ");
+ epilogue.appendNumber(totals.count);
+ epilogue.appendLiteral(",\n");
+ epilogue.appendLiteral("\"bodySize\": ");
+ epilogue.appendNumber(totals.bodySize);
+ epilogue.appendLiteral(",\n");
+ epilogue.appendLiteral("\"averageWorth\": ");
+ epilogue.appendNumber(totals.count ? totals.worth / totals.count : 0);
+ epilogue.appendLiteral("\n");
+ epilogue.appendLiteral("}\n}\n");
+ auto writeData = epilogue.toString().utf8();
+ WebCore::writeToFile(fd, writeData.data(), writeData.length());
+ WebCore::closeFile(fd);
+ return;
+ }
+ auto entry = Entry::decodeStorageRecord(*record);
+ if (!entry)
+ return;
+ ++totals.count;
+ totals.worth += info.worth;
+ totals.bodySize += info.bodySize;
+
+ StringBuilder json;
+ entry->asJSON(json, info);
+ json.appendLiteral(",\n");
+ auto writeData = json.toString().utf8();
+ WebCore::writeToFile(fd, writeData.data(), writeData.length());
+ });
+}
+
+void Cache::deleteDumpFile()
+{
+ WorkQueue::create("com.apple.WebKit.Cache.delete")->dispatch([path = dumpFilePath().isolatedCopy()] {
+ WebCore::deleteFile(path);
+ });
+}
+
+void Cache::clear(std::chrono::system_clock::time_point modifiedSince, Function<void ()>&& completionHandler)
+{
+ LOG(NetworkCache, "(NetworkProcess) clearing cache");
+
+ if (m_statistics)
+ m_statistics->clear();
+
+ if (!m_storage) {
+ RunLoop::main().dispatch(WTFMove(completionHandler));
+ return;
+ }
+ String anyType;
+ m_storage->clear(anyType, modifiedSince, WTFMove(completionHandler));
+
+ deleteDumpFile();
+}
+
+void Cache::clear()
+{
+ clear(std::chrono::system_clock::time_point::min(), nullptr);
+}
+
+String Cache::recordsPath() const
+{
+ return m_storage ? m_storage->recordsPath() : String();
+}
+
+void Cache::retrieveData(const DataKey& dataKey, Function<void (const uint8_t* data, size_t size)> completionHandler)
+{
+ ASSERT(isEnabled());
+
+ Key key { dataKey, m_storage->salt() };
+ m_storage->retrieve(key, 4, [completionHandler = WTFMove(completionHandler)] (auto record) {
+ if (!record || !record->body.size()) {
+ completionHandler(nullptr, 0);
+ return true;
+ }
+ completionHandler(record->body.data(), record->body.size());
+ return true;
+ });
+}
+
+void Cache::storeData(const DataKey& dataKey, const uint8_t* data, size_t size)
+{
+ if (!m_storage)
+ return;
+ Key key { dataKey, m_storage->salt() };
+ Storage::Record record { key, std::chrono::system_clock::now(), { }, Data { data, size }, { } };
+ m_storage->store(record, { });
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.h b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h
new file mode 100644
index 000000000..ca2bbc2dc
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCache_h
+#define NetworkCache_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheEntry.h"
+#include "NetworkCacheStorage.h"
+#include "ShareableResource.h"
+#include <WebCore/ResourceResponse.h>
+#include <wtf/Function.h>
+#include <wtf/text/WTFString.h>
+
+namespace WebCore {
+class ResourceRequest;
+class SharedBuffer;
+class URL;
+}
+
+namespace WebKit {
+namespace NetworkCache {
+
+class Cache;
+class SpeculativeLoadManager;
+class Statistics;
+
+Cache& singleton();
+
+struct MappedBody {
+#if ENABLE(SHAREABLE_RESOURCE)
+ RefPtr<ShareableResource> shareableResource;
+ ShareableResource::Handle shareableResourceHandle;
+#endif
+};
+
+enum class RetrieveDecision {
+ Yes,
+ NoDueToHTTPMethod,
+ NoDueToConditionalRequest,
+ NoDueToReloadIgnoringCache,
+ NoDueToStreamingMedia,
+};
+
+// FIXME: This enum is used in the Statistics code in a way that prevents removing or reordering anything.
+enum class StoreDecision {
+ Yes,
+ NoDueToProtocol,
+ NoDueToHTTPMethod,
+ NoDueToAttachmentResponse, // Unused.
+ NoDueToNoStoreResponse,
+ NoDueToHTTPStatusCode,
+ NoDueToNoStoreRequest,
+ NoDueToUnlikelyToReuse,
+ NoDueToStreamingMedia,
+};
+
+enum class UseDecision {
+ Use,
+ Validate,
+ NoDueToVaryingHeaderMismatch,
+ NoDueToMissingValidatorFields,
+ NoDueToDecodeFailure,
+ NoDueToExpiredRedirect,
+};
+
+using GlobalFrameID = std::pair<uint64_t /*webPageID*/, uint64_t /*webFrameID*/>;
+
+class Cache {
+ WTF_MAKE_NONCOPYABLE(Cache);
+ friend class WTF::NeverDestroyed<Cache>;
+public:
+ struct Parameters {
+ bool enableEfficacyLogging;
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+ bool enableNetworkCacheSpeculativeRevalidation;
+#endif
+ };
+ bool initialize(const String& cachePath, const Parameters&);
+ void setCapacity(size_t);
+
+ bool isEnabled() const { return !!m_storage; }
+
+ // Completion handler may get called back synchronously on failure.
+ void retrieve(const WebCore::ResourceRequest&, const GlobalFrameID&, Function<void (std::unique_ptr<Entry>)>&&);
+ std::unique_ptr<Entry> store(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, Function<void (MappedBody&)>&&);
+ std::unique_ptr<Entry> storeRedirect(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest);
+ std::unique_ptr<Entry> update(const WebCore::ResourceRequest&, const GlobalFrameID&, const Entry&, const WebCore::ResourceResponse& validatingResponse);
+
+ struct TraversalEntry {
+ const Entry& entry;
+ const Storage::RecordInfo& recordInfo;
+ };
+ void traverse(Function<void (const TraversalEntry*)>&&);
+ void remove(const Key&);
+ void remove(const WebCore::ResourceRequest&);
+
+ void clear();
+ void clear(std::chrono::system_clock::time_point modifiedSince, Function<void ()>&& completionHandler);
+
+ void retrieveData(const DataKey&, Function<void (const uint8_t* data, size_t size)>);
+ void storeData(const DataKey&, const uint8_t* data, size_t);
+
+ std::unique_ptr<Entry> makeEntry(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&);
+ std::unique_ptr<Entry> makeRedirectEntry(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest);
+
+ void dumpContentsToFile();
+
+ String recordsPath() const;
+ bool canUseSharedMemoryForBodyData() const { return m_storage && m_storage->canUseSharedMemoryForBodyData(); }
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+ SpeculativeLoadManager* speculativeLoadManager() { return m_speculativeLoadManager.get(); }
+#endif
+
+private:
+ Cache() = default;
+ ~Cache() = delete;
+
+ Key makeCacheKey(const WebCore::ResourceRequest&);
+
+ String dumpFilePath() const;
+ void deleteDumpFile();
+
+ std::unique_ptr<Storage> m_storage;
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+ std::unique_ptr<SpeculativeLoadManager> m_speculativeLoadManager;
+#endif
+ std::unique_ptr<Statistics> m_statistics;
+
+ unsigned m_traverseCount { 0 };
+};
+
+}
+}
+#endif
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp
new file mode 100644
index 000000000..267caf6a0
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp
@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheBlobStorage.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "Logging.h"
+#include "NetworkCacheFileSystem.h"
+#include <WebCore/FileSystem.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <wtf/RunLoop.h>
+#include <wtf/SHA1.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+BlobStorage::BlobStorage(const String& blobDirectoryPath, Salt salt)
+ : m_blobDirectoryPath(blobDirectoryPath)
+ , m_salt(salt)
+{
+}
+
+String BlobStorage::blobDirectoryPath() const
+{
+ return m_blobDirectoryPath.isolatedCopy();
+}
+
+void BlobStorage::synchronize()
+{
+ ASSERT(!RunLoop::isMain());
+
+ WebCore::makeAllDirectories(blobDirectoryPath());
+
+ m_approximateSize = 0;
+ auto blobDirectory = blobDirectoryPath();
+ traverseDirectory(blobDirectory, [this, &blobDirectory](const String& name, DirectoryEntryType type) {
+ if (type != DirectoryEntryType::File)
+ return;
+ auto path = WebCore::pathByAppendingComponent(blobDirectory, name);
+ auto filePath = WebCore::fileSystemRepresentation(path);
+ struct stat stat;
+ ::stat(filePath.data(), &stat);
+ // No clients left for this blob.
+ if (stat.st_nlink == 1)
+ unlink(filePath.data());
+ else
+ m_approximateSize += stat.st_size;
+ });
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) blob synchronization completed approximateSize=%zu", approximateSize());
+}
+
+String BlobStorage::blobPathForHash(const SHA1::Digest& hash) const
+{
+ auto hashAsString = SHA1::hexDigest(hash);
+ return WebCore::pathByAppendingComponent(blobDirectoryPath(), String::fromUTF8(hashAsString));
+}
+
+BlobStorage::Blob BlobStorage::add(const String& path, const Data& data)
+{
+ ASSERT(!RunLoop::isMain());
+
+ auto hash = computeSHA1(data, m_salt);
+ if (data.isEmpty())
+ return { data, hash };
+
+ auto blobPath = WebCore::fileSystemRepresentation(blobPathForHash(hash));
+ auto linkPath = WebCore::fileSystemRepresentation(path);
+ unlink(linkPath.data());
+
+ bool blobExists = access(blobPath.data(), F_OK) != -1;
+ if (blobExists) {
+ auto existingData = mapFile(blobPath.data());
+ if (bytesEqual(existingData, data)) {
+ if (link(blobPath.data(), linkPath.data()) == -1)
+ WTFLogAlways("Failed to create hard link from %s to %s", blobPath.data(), linkPath.data());
+ return { existingData, hash };
+ }
+ unlink(blobPath.data());
+ }
+
+ auto mappedData = data.mapToFile(blobPath.data());
+ if (mappedData.isNull())
+ return { };
+
+ if (link(blobPath.data(), linkPath.data()) == -1)
+ WTFLogAlways("Failed to create hard link from %s to %s", blobPath.data(), linkPath.data());
+
+ m_approximateSize += mappedData.size();
+
+ return { mappedData, hash };
+}
+
+BlobStorage::Blob BlobStorage::get(const String& path)
+{
+ ASSERT(!RunLoop::isMain());
+
+ auto linkPath = WebCore::fileSystemRepresentation(path);
+ auto data = mapFile(linkPath.data());
+
+ return { data, computeSHA1(data, m_salt) };
+}
+
+void BlobStorage::remove(const String& path)
+{
+ ASSERT(!RunLoop::isMain());
+
+ auto linkPath = WebCore::fileSystemRepresentation(path);
+ unlink(linkPath.data());
+}
+
+unsigned BlobStorage::shareCount(const String& path)
+{
+ ASSERT(!RunLoop::isMain());
+
+ auto linkPath = WebCore::fileSystemRepresentation(path);
+ struct stat stat;
+ if (::stat(linkPath.data(), &stat) < 0)
+ return 0;
+ // Link count is 2 in the single client case (the blob file and a link).
+ return stat.st_nlink - 1;
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h
new file mode 100644
index 000000000..adf0b085e
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheBlobStorage_h
+#define NetworkCacheBlobStorage_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheData.h"
+#include "NetworkCacheKey.h"
+#include <wtf/SHA1.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+// BlobStorage deduplicates the data using SHA1 hash computed over the blob bytes.
+class BlobStorage {
+ WTF_MAKE_NONCOPYABLE(BlobStorage);
+public:
+ BlobStorage(const String& blobDirectoryPath, Salt);
+
+ struct Blob {
+ Data data;
+ SHA1::Digest hash;
+ };
+ // These are all synchronous and should not be used from the main thread.
+ Blob add(const String& path, const Data&);
+ Blob get(const String& path);
+
+ // Blob won't be removed until synchronization.
+ void remove(const String& path);
+
+ unsigned shareCount(const String& path);
+
+ size_t approximateSize() const { return m_approximateSize; }
+
+ void synchronize();
+
+private:
+ String blobDirectoryPath() const;
+ String blobPathForHash(const SHA1::Digest&) const;
+
+ const String m_blobDirectoryPath;
+ const Salt m_salt;
+
+ std::atomic<size_t> m_approximateSize { 0 };
+};
+
+}
+}
+
+#endif
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp
new file mode 100644
index 000000000..8423b31e6
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheCoders.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+namespace WTF {
+namespace Persistence {
+
+// Store common HTTP headers as strings instead of using their value in the HTTPHeaderName enumeration
+// so that the headers stored in the cache stays valid even after HTTPHeaderName.in gets updated.
+void Coder<WebCore::HTTPHeaderMap>::encode(Encoder& encoder, const WebCore::HTTPHeaderMap& headers)
+{
+ encoder << static_cast<uint64_t>(headers.size());
+ for (auto& keyValue : headers) {
+ encoder << keyValue.key;
+ encoder << keyValue.value;
+ }
+}
+
+bool Coder<WebCore::HTTPHeaderMap>::decode(Decoder& decoder, WebCore::HTTPHeaderMap& headers)
+{
+ uint64_t headersSize;
+ if (!decoder.decode(headersSize))
+ return false;
+ for (uint64_t i = 0; i < headersSize; ++i) {
+ String name;
+ if (!decoder.decode(name))
+ return false;
+ String value;
+ if (!decoder.decode(value))
+ return false;
+ headers.add(name, value);
+ }
+ return true;
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h
new file mode 100644
index 000000000..8cac8c448
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2010, 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(NETWORK_CACHE)
+
+#include <WebCore/CertificateInfo.h>
+#include <WebCore/HTTPHeaderMap.h>
+#include <utility>
+#include <wtf/Forward.h>
+#include <wtf/HashMap.h>
+#include <wtf/HashSet.h>
+#include <wtf/SHA1.h>
+#include <wtf/Vector.h>
+#include <wtf/persistence/Coders.h>
+#include <wtf/persistence/Decoder.h>
+#include <wtf/persistence/Encoder.h>
+
+namespace WTF {
+namespace Persistence {
+
+template<> struct Coder<WebCore::CertificateInfo> {
+ static void encode(Encoder&, const WebCore::CertificateInfo&);
+ static bool decode(Decoder&, WebCore::CertificateInfo&);
+};
+
+template<> struct Coder<WebCore::HTTPHeaderMap> {
+ static void encode(Encoder&, const WebCore::HTTPHeaderMap&);
+ static bool decode(Decoder&, WebCore::HTTPHeaderMap&);
+};
+
+}
+}
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp
new file mode 100644
index 000000000..5de5095d3
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp
@@ -0,0 +1,194 @@
+/*
+ * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheCoders.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#if PLATFORM(COCOA)
+#include <Security/SecCertificate.h>
+#include <Security/SecTrust.h>
+#include <wtf/spi/cocoa/SecuritySPI.h>
+#endif
+
+namespace WTF {
+namespace Persistence {
+
+static void encodeCFData(Encoder& encoder, CFDataRef data)
+{
+ uint64_t length = CFDataGetLength(data);
+ const uint8_t* bytePtr = CFDataGetBytePtr(data);
+
+ encoder << length;
+ encoder.encodeFixedLengthData(bytePtr, length);
+}
+
+static bool decodeCFData(Decoder& decoder, RetainPtr<CFDataRef>& data)
+{
+ uint64_t size = 0;
+ if (!decoder.decode(size))
+ return false;
+
+ Vector<uint8_t> vector(size);
+ if (!decoder.decodeFixedLengthData(vector.data(), vector.size()))
+ return false;
+
+ data = adoptCF(CFDataCreate(nullptr, vector.data(), vector.size()));
+ return true;
+}
+
+
+#if HAVE(SEC_TRUST_SERIALIZATION)
+static void encodeSecTrustRef(Encoder& encoder, SecTrustRef trust)
+{
+ auto data = adoptCF(SecTrustSerialize(trust, nullptr));
+ if (!data) {
+ encoder << false;
+ return;
+ }
+
+ encoder << true;
+ encodeCFData(encoder, data.get());
+}
+
+static bool decodeSecTrustRef(Decoder& decoder, RetainPtr<SecTrustRef>& result)
+{
+ bool hasTrust;
+ if (!decoder.decode(hasTrust))
+ return false;
+
+ if (!hasTrust)
+ return true;
+
+ RetainPtr<CFDataRef> trustData;
+ if (!decodeCFData(decoder, trustData))
+ return false;
+
+ auto trust = adoptCF(SecTrustDeserialize(trustData.get(), nullptr));
+ if (!trust)
+ return false;
+
+ result = WTFMove(trust);
+ return true;
+}
+#endif
+
+static void encodeCertificateChain(Encoder& encoder, CFArrayRef certificateChain)
+{
+ CFIndex size = CFArrayGetCount(certificateChain);
+ Vector<CFTypeRef, 32> values(size);
+
+ CFArrayGetValues(certificateChain, CFRangeMake(0, size), values.data());
+
+ encoder << static_cast<uint64_t>(size);
+
+ for (CFIndex i = 0; i < size; ++i) {
+ ASSERT(values[i]);
+ ASSERT(CFGetTypeID(values[i]) == SecCertificateGetTypeID());
+
+ auto data = adoptCF(SecCertificateCopyData((SecCertificateRef)values[i]));
+ encodeCFData(encoder, data.get());
+ }
+}
+
+static bool decodeCertificateChain(Decoder& decoder, RetainPtr<CFArrayRef>& certificateChain)
+{
+ uint64_t size;
+ if (!decoder.decode(size))
+ return false;
+
+ auto array = adoptCF(CFArrayCreateMutable(0, 0, &kCFTypeArrayCallBacks));
+
+ for (size_t i = 0; i < size; ++i) {
+ RetainPtr<CFDataRef> data;
+ if (!decodeCFData(decoder, data))
+ return false;
+
+ auto certificate = adoptCF(SecCertificateCreateWithData(0, data.get()));
+ CFArrayAppendValue(array.get(), certificate.get());
+ }
+
+ certificateChain = WTFMove(array);
+ return true;
+}
+
+void Coder<WebCore::CertificateInfo>::encode(Encoder& encoder, const WebCore::CertificateInfo& certificateInfo)
+{
+ encoder.encodeEnum(certificateInfo.type());
+
+ switch (certificateInfo.type()) {
+#if HAVE(SEC_TRUST_SERIALIZATION)
+ case WebCore::CertificateInfo::Type::Trust:
+ encodeSecTrustRef(encoder, certificateInfo.trust());
+ break;
+#endif
+ case WebCore::CertificateInfo::Type::CertificateChain: {
+ encodeCertificateChain(encoder, certificateInfo.certificateChain());
+ break;
+ }
+ case WebCore::CertificateInfo::Type::None:
+ // Do nothing.
+ break;
+ }
+}
+
+bool Coder<WebCore::CertificateInfo>::decode(Decoder& decoder, WebCore::CertificateInfo& certificateInfo)
+{
+ WebCore::CertificateInfo::Type certificateInfoType;
+ if (!decoder.decodeEnum(certificateInfoType))
+ return false;
+
+ switch (certificateInfoType) {
+#if HAVE(SEC_TRUST_SERIALIZATION)
+ case WebCore::CertificateInfo::Type::Trust: {
+ RetainPtr<SecTrustRef> trust;
+ if (!decodeSecTrustRef(decoder, trust))
+ return false;
+
+ certificateInfo = WebCore::CertificateInfo(WTFMove(trust));
+ return true;
+ }
+#endif
+ case WebCore::CertificateInfo::Type::CertificateChain: {
+ RetainPtr<CFArrayRef> certificateChain;
+ if (!decodeCertificateChain(decoder, certificateChain))
+ return false;
+
+ certificateInfo = WebCore::CertificateInfo(WTFMove(certificateChain));
+ return true;
+ }
+ case WebCore::CertificateInfo::Type::None:
+ // Do nothing.
+ break;
+ }
+
+ return true;
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp
new file mode 100644
index 000000000..0cef6ae03
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheCoders.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+namespace WTF {
+namespace Persistence {
+
+void Coder<WebCore::CertificateInfo>::encode(Encoder& encoder, const WebCore::CertificateInfo& certificateInfo)
+{
+ if (!certificateInfo.certificate()) {
+ encoder << false;
+ return;
+ }
+
+ GByteArray* certificateData = 0;
+ g_object_get(G_OBJECT(certificateInfo.certificate()), "certificate", &certificateData, NULL);
+ if (!certificateData) {
+ encoder << false;
+ return;
+ }
+
+ encoder << true;
+
+ GRefPtr<GByteArray> certificate = adoptGRef(certificateData);
+ encoder << static_cast<uint64_t>(certificate->len);
+ encoder.encodeFixedLengthData(certificate->data, certificate->len);
+
+ encoder << static_cast<uint32_t>(certificateInfo.tlsErrors());
+}
+
+bool Coder<WebCore::CertificateInfo>::decode(Decoder& decoder, WebCore::CertificateInfo& certificateInfo)
+{
+ bool hasCertificate;
+ if (!decoder.decode(hasCertificate))
+ return false;
+
+ if (!hasCertificate)
+ return true;
+
+ uint64_t size = 0;
+ if (!decoder.decode(size))
+ return false;
+
+ Vector<uint8_t> vector(size);
+ if (!decoder.decodeFixedLengthData(vector.data(), vector.size()))
+ return false;
+
+ GByteArray* certificateData = g_byte_array_sized_new(vector.size());
+ certificateData = g_byte_array_append(certificateData, vector.data(), vector.size());
+ GRefPtr<GByteArray> certificateBytes = adoptGRef(certificateData);
+
+ GTlsBackend* backend = g_tls_backend_get_default();
+ GRefPtr<GTlsCertificate> certificate = adoptGRef(G_TLS_CERTIFICATE(g_initable_new(
+ g_tls_backend_get_certificate_type(backend), 0, 0, "certificate", certificateBytes.get(), nullptr)));
+ certificateInfo.setCertificate(certificate.get());
+
+ uint32_t tlsErrors;
+ if (!decoder.decode(tlsErrors))
+ return false;
+ certificateInfo.setTLSErrors(static_cast<GTlsCertificateFlags>(tlsErrors));
+
+ return true;
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp
new file mode 100644
index 000000000..31ae48fe5
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp
@@ -0,0 +1,163 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheData.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include <WebCore/FileSystem.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <wtf/CryptographicallyRandomNumber.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+Data Data::mapToFile(const char* path) const
+{
+ int fd = open(path, O_CREAT | O_EXCL | O_RDWR , S_IRUSR | S_IWUSR);
+ if (fd < 0)
+ return { };
+
+ if (ftruncate(fd, m_size) < 0) {
+ close(fd);
+ return { };
+ }
+
+ void* map = mmap(nullptr, m_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (map == MAP_FAILED) {
+ close(fd);
+ return { };
+ }
+
+ uint8_t* mapData = static_cast<uint8_t*>(map);
+ apply([&mapData](const uint8_t* bytes, size_t bytesSize) {
+ memcpy(mapData, bytes, bytesSize);
+ mapData += bytesSize;
+ return true;
+ });
+
+ // Drop the write permission.
+ mprotect(map, m_size, PROT_READ);
+
+ // Flush (asynchronously) to file, turning this into clean memory.
+ msync(map, m_size, MS_ASYNC);
+
+ return Data::adoptMap(map, m_size, fd);
+}
+
+Data mapFile(const char* path)
+{
+ int fd = open(path, O_RDONLY, 0);
+ if (fd < 0)
+ return { };
+ struct stat stat;
+ if (fstat(fd, &stat) < 0) {
+ close(fd);
+ return { };
+ }
+ size_t size = stat.st_size;
+ if (!size) {
+ close(fd);
+ return Data::empty();
+ }
+
+ return adoptAndMapFile(fd, 0, size);
+}
+
+Data adoptAndMapFile(int fd, size_t offset, size_t size)
+{
+ if (!size) {
+ close(fd);
+ return Data::empty();
+ }
+
+ void* map = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, offset);
+ if (map == MAP_FAILED) {
+ close(fd);
+ return { };
+ }
+
+ return Data::adoptMap(map, size, fd);
+}
+
+SHA1::Digest computeSHA1(const Data& data, const Salt& salt)
+{
+ SHA1 sha1;
+ sha1.addBytes(salt.data(), salt.size());
+ data.apply([&sha1](const uint8_t* data, size_t size) {
+ sha1.addBytes(data, size);
+ return true;
+ });
+
+ SHA1::Digest digest;
+ sha1.computeHash(digest);
+ return digest;
+}
+
+bool bytesEqual(const Data& a, const Data& b)
+{
+ if (a.isNull() || b.isNull())
+ return false;
+ if (a.size() != b.size())
+ return false;
+ return !memcmp(a.data(), b.data(), a.size());
+}
+
+static Salt makeSalt()
+{
+ Salt salt;
+ static_assert(salt.size() == 8, "Salt size");
+ *reinterpret_cast<uint32_t*>(&salt[0]) = cryptographicallyRandomNumber();
+ *reinterpret_cast<uint32_t*>(&salt[4]) = cryptographicallyRandomNumber();
+ return salt;
+}
+
+std::optional<Salt> readOrMakeSalt(const String& path)
+{
+ auto cpath = WebCore::fileSystemRepresentation(path);
+ auto fd = open(cpath.data(), O_RDONLY, 0);
+ Salt salt;
+ auto bytesRead = read(fd, salt.data(), salt.size());
+ close(fd);
+ if (bytesRead != salt.size()) {
+ salt = makeSalt();
+
+ unlink(cpath.data());
+ fd = open(cpath.data(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+ bool success = write(fd, salt.data(), salt.size()) == salt.size();
+ close(fd);
+ if (!success)
+ return { };
+ }
+ return salt;
+}
+
+} // namespace NetworkCache
+} // namespace WebKit
+
+#endif // #if ENABLE(NETWORK_CACHE)
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h
new file mode 100644
index 000000000..29c264b4b
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h
@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheData_h
+#define NetworkCacheData_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include <functional>
+#include <wtf/FunctionDispatcher.h>
+#include <wtf/SHA1.h>
+#include <wtf/ThreadSafeRefCounted.h>
+#include <wtf/text/WTFString.h>
+
+#if USE(SOUP)
+#include <WebCore/GRefPtrSoup.h>
+#endif
+
+namespace WebKit {
+
+class SharedMemory;
+
+namespace NetworkCache {
+
+#if PLATFORM(COCOA)
+template <typename T> class DispatchPtr;
+template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject);
+
+// FIXME: Use OSObjectPtr instead when it works with dispatch_data_t on all platforms.
+template<typename T> class DispatchPtr {
+public:
+ DispatchPtr()
+ : m_ptr(nullptr)
+ {
+ }
+ explicit DispatchPtr(T ptr)
+ : m_ptr(ptr)
+ {
+ if (m_ptr)
+ dispatch_retain(m_ptr);
+ }
+ DispatchPtr(const DispatchPtr& other)
+ : m_ptr(other.m_ptr)
+ {
+ if (m_ptr)
+ dispatch_retain(m_ptr);
+ }
+ ~DispatchPtr()
+ {
+ if (m_ptr)
+ dispatch_release(m_ptr);
+ }
+
+ DispatchPtr& operator=(const DispatchPtr& other)
+ {
+ auto copy = other;
+ std::swap(m_ptr, copy.m_ptr);
+ return *this;
+ }
+
+ T get() const { return m_ptr; }
+ explicit operator bool() const { return m_ptr; }
+
+ friend DispatchPtr adoptDispatch<T>(T);
+
+private:
+ struct Adopt { };
+ DispatchPtr(Adopt, T data)
+ : m_ptr(data)
+ {
+ }
+
+ T m_ptr;
+};
+
+template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject)
+{
+ return DispatchPtr<T>(typename DispatchPtr<T>::Adopt { }, dispatchObject);
+}
+#endif
+
+class Data {
+public:
+ Data() { }
+ Data(const uint8_t*, size_t);
+
+ ~Data() { }
+
+ static Data empty();
+ static Data adoptMap(void* map, size_t, int fd);
+
+#if PLATFORM(COCOA)
+ enum class Backing { Buffer, Map };
+ Data(DispatchPtr<dispatch_data_t>, Backing = Backing::Buffer);
+#endif
+#if USE(SOUP)
+ Data(GRefPtr<SoupBuffer>&&, int fd = -1);
+#endif
+ bool isNull() const;
+ bool isEmpty() const { return !m_size; }
+
+ const uint8_t* data() const;
+ size_t size() const { return m_size; }
+ bool isMap() const { return m_isMap; }
+ RefPtr<SharedMemory> tryCreateSharedMemory() const;
+
+ Data subrange(size_t offset, size_t) const;
+
+ bool apply(const Function<bool (const uint8_t*, size_t)>&) const;
+
+ Data mapToFile(const char* path) const;
+
+#if PLATFORM(COCOA)
+ dispatch_data_t dispatchData() const { return m_dispatchData.get(); }
+#endif
+
+#if USE(SOUP)
+ SoupBuffer* soupBuffer() const { return m_buffer.get(); }
+#endif
+private:
+#if PLATFORM(COCOA)
+ mutable DispatchPtr<dispatch_data_t> m_dispatchData;
+#endif
+#if USE(SOUP)
+ mutable GRefPtr<SoupBuffer> m_buffer;
+ int m_fileDescriptor { -1 };
+#endif
+ mutable const uint8_t* m_data { nullptr };
+ size_t m_size { 0 };
+ bool m_isMap { false };
+};
+
+Data concatenate(const Data&, const Data&);
+bool bytesEqual(const Data&, const Data&);
+Data adoptAndMapFile(int fd, size_t offset, size_t);
+Data mapFile(const char* path);
+
+using Salt = std::array<uint8_t, 8>;
+
+std::optional<Salt> readOrMakeSalt(const String& path);
+SHA1::Digest computeSHA1(const Data&, const Salt&);
+
+}
+}
+
+#endif
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp
new file mode 100644
index 000000000..21b58fa7e
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp
@@ -0,0 +1,142 @@
+/*
+ * Copyright (C) 2015 Igalia S.L
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheData.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "SharedMemory.h"
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+Data::Data(const uint8_t* data, size_t size)
+ : m_size(size)
+{
+ uint8_t* copiedData = static_cast<uint8_t*>(fastMalloc(size));
+ memcpy(copiedData, data, size);
+ m_buffer = adoptGRef(soup_buffer_new_with_owner(copiedData, size, copiedData, fastFree));
+}
+
+Data::Data(GRefPtr<SoupBuffer>&& buffer, int fd)
+ : m_buffer(buffer)
+ , m_fileDescriptor(fd)
+ , m_size(buffer ? buffer->length : 0)
+ , m_isMap(m_size && fd != -1)
+{
+}
+
+Data Data::empty()
+{
+ GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new(SOUP_MEMORY_TAKE, nullptr, 0));
+ return { WTFMove(buffer) };
+}
+
+const uint8_t* Data::data() const
+{
+ return m_buffer ? reinterpret_cast<const uint8_t*>(m_buffer->data) : nullptr;
+}
+
+bool Data::isNull() const
+{
+ return !m_buffer;
+}
+
+bool Data::apply(const Function<bool (const uint8_t*, size_t)>& applier) const
+{
+ if (!m_size)
+ return false;
+
+ return applier(reinterpret_cast<const uint8_t*>(m_buffer->data), m_buffer->length);
+}
+
+Data Data::subrange(size_t offset, size_t size) const
+{
+ if (!m_buffer)
+ return { };
+
+ GRefPtr<SoupBuffer> subBuffer = adoptGRef(soup_buffer_new_subbuffer(m_buffer.get(), offset, size));
+ return { WTFMove(subBuffer) };
+}
+
+Data concatenate(const Data& a, const Data& b)
+{
+ if (a.isNull())
+ return b;
+ if (b.isNull())
+ return a;
+
+ size_t size = a.size() + b.size();
+ uint8_t* data = static_cast<uint8_t*>(fastMalloc(size));
+ memcpy(data, a.soupBuffer()->data, a.soupBuffer()->length);
+ memcpy(data + a.soupBuffer()->length, b.soupBuffer()->data, b.soupBuffer()->length);
+ GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(data, size, data, fastFree));
+ return { WTFMove(buffer) };
+}
+
+struct MapWrapper {
+ ~MapWrapper()
+ {
+ munmap(map, size);
+ close(fileDescriptor);
+ }
+
+ void* map;
+ size_t size;
+ int fileDescriptor;
+};
+
+static void deleteMapWrapper(MapWrapper* wrapper)
+{
+ delete wrapper;
+}
+
+Data Data::adoptMap(void* map, size_t size, int fd)
+{
+ ASSERT(map);
+ ASSERT(map != MAP_FAILED);
+ MapWrapper* wrapper = new MapWrapper { map, size, fd };
+ GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(map, size, wrapper, reinterpret_cast<GDestroyNotify>(deleteMapWrapper)));
+ return { WTFMove(buffer), fd };
+}
+
+RefPtr<SharedMemory> Data::tryCreateSharedMemory() const
+{
+ if (isNull() || !isMap())
+ return nullptr;
+
+ return SharedMemory::wrapMap(const_cast<char*>(m_buffer->data), m_buffer->length, m_fileDescriptor);
+}
+
+} // namespace NetworkCache
+} // namespace WebKit
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp
new file mode 100644
index 000000000..b96bbcd5a
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp
@@ -0,0 +1,246 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCache.h"
+
+#include "Logging.h"
+#include "NetworkCacheCoders.h"
+#include <WebCore/ResourceRequest.h>
+#include <WebCore/SharedBuffer.h>
+#include <wtf/text/StringBuilder.h>
+
+#if ENABLE(NETWORK_CACHE)
+
+namespace WebKit {
+namespace NetworkCache {
+
+Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& buffer, const Vector<std::pair<String, String>>& varyingRequestHeaders)
+ : m_key(key)
+ , m_timeStamp(std::chrono::system_clock::now())
+ , m_response(response)
+ , m_varyingRequestHeaders(varyingRequestHeaders)
+ , m_buffer(WTFMove(buffer))
+{
+ ASSERT(m_key.type() == "Resource");
+}
+
+Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders)
+ : m_key(key)
+ , m_timeStamp(std::chrono::system_clock::now())
+ , m_response(response)
+ , m_varyingRequestHeaders(varyingRequestHeaders)
+{
+ ASSERT(m_key.type() == "Resource");
+
+ m_redirectRequest.emplace();
+ m_redirectRequest->setAsIsolatedCopy(redirectRequest);
+ // Redirect body is not needed even if exists.
+ m_redirectRequest->setHTTPBody(nullptr);
+}
+
+Entry::Entry(const Entry& other)
+ : m_key(other.m_key)
+ , m_timeStamp(other.m_timeStamp)
+ , m_response(other.m_response)
+ , m_varyingRequestHeaders(other.m_varyingRequestHeaders)
+ , m_redirectRequest(other.m_redirectRequest)
+ , m_buffer(other.m_buffer)
+ , m_sourceStorageRecord(other.m_sourceStorageRecord)
+{
+}
+
+Entry::Entry(const Storage::Record& storageEntry)
+ : m_key(storageEntry.key)
+ , m_timeStamp(storageEntry.timeStamp)
+ , m_sourceStorageRecord(storageEntry)
+{
+ ASSERT(m_key.type() == "Resource");
+}
+
+Storage::Record Entry::encodeAsStorageRecord() const
+{
+ WTF::Persistence::Encoder encoder;
+ encoder << m_response;
+
+ bool hasVaryingRequestHeaders = !m_varyingRequestHeaders.isEmpty();
+ encoder << hasVaryingRequestHeaders;
+ if (hasVaryingRequestHeaders)
+ encoder << m_varyingRequestHeaders;
+
+ bool isRedirect = !!m_redirectRequest;
+ encoder << isRedirect;
+ if (isRedirect)
+ m_redirectRequest->encodeWithoutPlatformData(encoder);
+
+ encoder.encodeChecksum();
+
+ Data header(encoder.buffer(), encoder.bufferSize());
+ Data body;
+ if (m_buffer)
+ body = { reinterpret_cast<const uint8_t*>(m_buffer->data()), m_buffer->size() };
+
+ return { m_key, m_timeStamp, header, body, { } };
+}
+
+std::unique_ptr<Entry> Entry::decodeStorageRecord(const Storage::Record& storageEntry)
+{
+ auto entry = std::make_unique<Entry>(storageEntry);
+
+ WTF::Persistence::Decoder decoder(storageEntry.header.data(), storageEntry.header.size());
+ if (!decoder.decode(entry->m_response))
+ return nullptr;
+ entry->m_response.setSource(WebCore::ResourceResponse::Source::DiskCache);
+ if (storageEntry.bodyHash)
+ entry->m_response.setCacheBodyKey(*storageEntry.bodyHash);
+
+ bool hasVaryingRequestHeaders;
+ if (!decoder.decode(hasVaryingRequestHeaders))
+ return nullptr;
+
+ if (hasVaryingRequestHeaders) {
+ if (!decoder.decode(entry->m_varyingRequestHeaders))
+ return nullptr;
+ }
+
+ bool isRedirect;
+ if (!decoder.decode(isRedirect))
+ return nullptr;
+
+ if (isRedirect) {
+ entry->m_redirectRequest.emplace();
+ if (!entry->m_redirectRequest->decodeWithoutPlatformData(decoder))
+ return nullptr;
+ }
+
+ if (!decoder.verifyChecksum()) {
+ LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n");
+ return nullptr;
+ }
+
+ return entry;
+}
+
+#if ENABLE(SHAREABLE_RESOURCE)
+void Entry::initializeShareableResourceHandleFromStorageRecord() const
+{
+ if (!NetworkCache::singleton().canUseSharedMemoryForBodyData())
+ return;
+
+ auto sharedMemory = m_sourceStorageRecord.body.tryCreateSharedMemory();
+ if (!sharedMemory)
+ return;
+
+ auto shareableResource = ShareableResource::create(sharedMemory.releaseNonNull(), 0, m_sourceStorageRecord.body.size());
+ shareableResource->createHandle(m_shareableResourceHandle);
+}
+#endif
+
+void Entry::initializeBufferFromStorageRecord() const
+{
+#if ENABLE(SHAREABLE_RESOURCE)
+ if (!shareableResourceHandle().isNull()) {
+ m_buffer = m_shareableResourceHandle.tryWrapInSharedBuffer();
+ if (m_buffer)
+ return;
+ }
+#endif
+ m_buffer = WebCore::SharedBuffer::create(m_sourceStorageRecord.body.data(), m_sourceStorageRecord.body.size());
+}
+
+WebCore::SharedBuffer* Entry::buffer() const
+{
+ if (!m_buffer)
+ initializeBufferFromStorageRecord();
+
+ return m_buffer.get();
+}
+
+#if ENABLE(SHAREABLE_RESOURCE)
+ShareableResource::Handle& Entry::shareableResourceHandle() const
+{
+ if (m_shareableResourceHandle.isNull())
+ initializeShareableResourceHandleFromStorageRecord();
+
+ return m_shareableResourceHandle;
+}
+#endif
+
+bool Entry::needsValidation() const
+{
+ return m_response.source() == WebCore::ResourceResponse::Source::DiskCacheAfterValidation;
+}
+
+void Entry::setNeedsValidation(bool value)
+{
+ m_response.setSource(value ? WebCore::ResourceResponse::Source::DiskCacheAfterValidation : WebCore::ResourceResponse::Source::DiskCache);
+}
+
+void Entry::asJSON(StringBuilder& json, const Storage::RecordInfo& info) const
+{
+ json.appendLiteral("{\n");
+ json.appendLiteral("\"hash\": ");
+ json.appendQuotedJSONString(m_key.hashAsString());
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"bodySize\": ");
+ json.appendNumber(info.bodySize);
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"worth\": ");
+ json.appendNumber(info.worth);
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"partition\": ");
+ json.appendQuotedJSONString(m_key.partition());
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"timestamp\": ");
+ json.appendNumber(std::chrono::duration_cast<std::chrono::milliseconds>(m_timeStamp.time_since_epoch()).count());
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"URL\": ");
+ json.appendQuotedJSONString(m_response.url().string());
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"bodyHash\": ");
+ json.appendQuotedJSONString(info.bodyHash);
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"bodyShareCount\": ");
+ json.appendNumber(info.bodyShareCount);
+ json.appendLiteral(",\n");
+ json.appendLiteral("\"headers\": {\n");
+ bool firstHeader = true;
+ for (auto& header : m_response.httpHeaderFields()) {
+ if (!firstHeader)
+ json.appendLiteral(",\n");
+ firstHeader = false;
+ json.appendLiteral(" ");
+ json.appendQuotedJSONString(header.key);
+ json.appendLiteral(": ");
+ json.appendQuotedJSONString(header.value);
+ }
+ json.appendLiteral("\n}\n");
+ json.appendLiteral("}");
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h
new file mode 100644
index 000000000..c03910894
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheEntry_h
+#define NetworkCacheEntry_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheStorage.h"
+#include "ShareableResource.h"
+#include <WebCore/ResourceRequest.h>
+#include <WebCore/ResourceResponse.h>
+#include <wtf/Noncopyable.h>
+#include <wtf/text/WTFString.h>
+
+namespace WebCore {
+class SharedBuffer;
+}
+
+namespace WebKit {
+namespace NetworkCache {
+
+class Entry {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ Entry(const Key&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, const Vector<std::pair<String, String>>& varyingRequestHeaders);
+ Entry(const Key&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders);
+ explicit Entry(const Storage::Record&);
+ Entry(const Entry&);
+
+ Storage::Record encodeAsStorageRecord() const;
+ static std::unique_ptr<Entry> decodeStorageRecord(const Storage::Record&);
+
+ const Key& key() const { return m_key; }
+ std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; }
+ const WebCore::ResourceResponse& response() const { return m_response; }
+ const Vector<std::pair<String, String>>& varyingRequestHeaders() const { return m_varyingRequestHeaders; }
+
+ WebCore::SharedBuffer* buffer() const;
+ const std::optional<WebCore::ResourceRequest>& redirectRequest() const { return m_redirectRequest; }
+
+#if ENABLE(SHAREABLE_RESOURCE)
+ ShareableResource::Handle& shareableResourceHandle() const;
+#endif
+
+ bool needsValidation() const;
+ void setNeedsValidation(bool);
+
+ const Storage::Record& sourceStorageRecord() const { return m_sourceStorageRecord; }
+
+ void asJSON(StringBuilder&, const Storage::RecordInfo&) const;
+
+private:
+ void initializeBufferFromStorageRecord() const;
+#if ENABLE(SHAREABLE_RESOURCE)
+ void initializeShareableResourceHandleFromStorageRecord() const;
+#endif
+
+ Key m_key;
+ std::chrono::system_clock::time_point m_timeStamp;
+ WebCore::ResourceResponse m_response;
+ Vector<std::pair<String, String>> m_varyingRequestHeaders;
+
+ std::optional<WebCore::ResourceRequest> m_redirectRequest;
+ mutable RefPtr<WebCore::SharedBuffer> m_buffer;
+#if ENABLE(SHAREABLE_RESOURCE)
+ mutable ShareableResource::Handle m_shareableResourceHandle;
+#endif
+
+ Storage::Record m_sourceStorageRecord { };
+};
+
+}
+}
+
+#endif
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp
new file mode 100644
index 000000000..52ac56696
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp
@@ -0,0 +1,170 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheFileSystem.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "Logging.h"
+#include <WebCore/FileSystem.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <wtf/Assertions.h>
+#include <wtf/Function.h>
+#include <wtf/text/CString.h>
+
+#if PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR)
+#include <sys/attr.h>
+#include <unistd.h>
+#endif
+
+#if USE(SOUP)
+#include <gio/gio.h>
+#include <wtf/glib/GRefPtr.h>
+#endif
+
+namespace WebKit {
+namespace NetworkCache {
+
+static DirectoryEntryType directoryEntryType(uint8_t dtype)
+{
+ switch (dtype) {
+ case DT_DIR:
+ return DirectoryEntryType::Directory;
+ case DT_REG:
+ return DirectoryEntryType::File;
+ default:
+ ASSERT_NOT_REACHED();
+ return DirectoryEntryType::File;
+ }
+}
+
+void traverseDirectory(const String& path, const Function<void (const String&, DirectoryEntryType)>& function)
+{
+ DIR* dir = opendir(WebCore::fileSystemRepresentation(path).data());
+ if (!dir)
+ return;
+ dirent* dp;
+ while ((dp = readdir(dir))) {
+ if (dp->d_type != DT_DIR && dp->d_type != DT_REG)
+ continue;
+ const char* name = dp->d_name;
+ if (!strcmp(name, ".") || !strcmp(name, ".."))
+ continue;
+ auto nameString = String::fromUTF8(name);
+ if (nameString.isNull())
+ continue;
+ function(nameString, directoryEntryType(dp->d_type));
+ }
+ closedir(dir);
+}
+
+void deleteDirectoryRecursively(const String& path)
+{
+ traverseDirectory(path, [&path](const String& name, DirectoryEntryType type) {
+ String entryPath = WebCore::pathByAppendingComponent(path, name);
+ switch (type) {
+ case DirectoryEntryType::File:
+ WebCore::deleteFile(entryPath);
+ break;
+ case DirectoryEntryType::Directory:
+ deleteDirectoryRecursively(entryPath);
+ break;
+ // This doesn't follow symlinks.
+ }
+ });
+ WebCore::deleteEmptyDirectory(path);
+}
+
+FileTimes fileTimes(const String& path)
+{
+#if HAVE(STAT_BIRTHTIME)
+ struct stat fileInfo;
+ if (stat(WebCore::fileSystemRepresentation(path).data(), &fileInfo))
+ return { };
+ return { std::chrono::system_clock::from_time_t(fileInfo.st_birthtime), std::chrono::system_clock::from_time_t(fileInfo.st_mtime) };
+#elif USE(SOUP)
+ // There's no st_birthtime in some operating systems like Linux, so we use xattrs to set/get the creation time.
+ GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(WebCore::fileSystemRepresentation(path).data()));
+ GRefPtr<GFileInfo> fileInfo = adoptGRef(g_file_query_info(file.get(), "xattr::birthtime,time::modified", G_FILE_QUERY_INFO_NONE, nullptr, nullptr));
+ if (!fileInfo)
+ return { };
+ const char* birthtimeString = g_file_info_get_attribute_string(fileInfo.get(), "xattr::birthtime");
+ if (!birthtimeString)
+ return { };
+ return { std::chrono::system_clock::from_time_t(g_ascii_strtoull(birthtimeString, nullptr, 10)),
+ std::chrono::system_clock::from_time_t(g_file_info_get_attribute_uint64(fileInfo.get(), "time::modified")) };
+#endif
+}
+
+void updateFileModificationTimeIfNeeded(const String& path)
+{
+ auto times = fileTimes(path);
+ if (times.creation != times.modification) {
+ // Don't update more than once per hour.
+ if (std::chrono::system_clock::now() - times.modification < std::chrono::hours(1))
+ return;
+ }
+ // This really updates both the access time and the modification time.
+ utimes(WebCore::fileSystemRepresentation(path).data(), nullptr);
+}
+
+bool canUseSharedMemoryForPath(const String& path)
+{
+#if PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR)
+ struct {
+ uint32_t length;
+ uint32_t protectionClass;
+ } attrBuffer;
+
+ attrlist attrList = { };
+ attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
+ attrList.commonattr = ATTR_CMN_DATA_PROTECT_FLAGS;
+ int32_t error = getattrlist(WebCore::fileSystemRepresentation(path).data(), &attrList, &attrBuffer, sizeof(attrBuffer), FSOPT_NOFOLLOW);
+ if (error) {
+ RELEASE_LOG_ERROR(Network, "Unable to get cache directory protection class, disabling use of shared mapped memory");
+ return false;
+ }
+
+ // For stricter protection classes shared maps could disappear when device is locked.
+ const uint32_t fileProtectionCompleteUntilFirstUserAuthentication = 3;
+ bool isSafe = attrBuffer.protectionClass >= fileProtectionCompleteUntilFirstUserAuthentication;
+
+ if (!isSafe)
+ RELEASE_LOG(Network, "Disallowing use of shared mapped memory due to container protection class %u", attrBuffer.protectionClass);
+
+ return isSafe;
+#else
+ UNUSED_PARAM(path);
+ return true;
+#endif
+}
+
+}
+}
+
+#endif // ENABLE(NETWORK_CACHE)
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h
new file mode 100644
index 000000000..22e716f52
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheFileSystem_h
+#define NetworkCacheFileSystem_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include <WebCore/FileSystem.h>
+#include <functional>
+
+namespace WebKit {
+namespace NetworkCache {
+
+enum class DirectoryEntryType { Directory, File };
+void traverseDirectory(const String& path, const Function<void (const String& fileName, DirectoryEntryType)>&);
+
+void deleteDirectoryRecursively(const String& path);
+
+struct FileTimes {
+ std::chrono::system_clock::time_point creation;
+ std::chrono::system_clock::time_point modification;
+};
+FileTimes fileTimes(const String& path);
+void updateFileModificationTimeIfNeeded(const String& path);
+
+bool canUseSharedMemoryForPath(const String& path);
+
+}
+}
+
+#endif
+
+#endif
+
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h
new file mode 100644
index 000000000..f19db6fa1
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheIOChannel_h
+#define NetworkCacheIOChannel_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheData.h"
+#include <functional>
+#include <wtf/ThreadSafeRefCounted.h>
+#include <wtf/WorkQueue.h>
+#include <wtf/text/WTFString.h>
+
+#if USE(SOUP)
+#include <wtf/glib/GRefPtr.h>
+#endif
+
+namespace WebKit {
+namespace NetworkCache {
+
+class IOChannel : public ThreadSafeRefCounted<IOChannel> {
+public:
+ enum class Type { Read, Write, Create };
+ static Ref<IOChannel> open(const String& file, Type);
+
+ // Using nullptr as queue submits the result to the main queue.
+ // FIXME: We should add WorkQueue::main() instead.
+ void read(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>);
+ void write(size_t offset, const Data&, WorkQueue*, std::function<void (int error)>);
+
+ const String& path() const { return m_path; }
+ Type type() const { return m_type; }
+
+ int fileDescriptor() const { return m_fileDescriptor; }
+
+ ~IOChannel();
+
+private:
+ IOChannel(const String& filePath, IOChannel::Type);
+
+#if USE(SOUP)
+ void readSyncInThread(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>);
+#endif
+
+ String m_path;
+ Type m_type;
+
+ int m_fileDescriptor { 0 };
+ std::atomic<bool> m_wasDeleted { false }; // Try to narrow down a crash, https://bugs.webkit.org/show_bug.cgi?id=165659
+#if PLATFORM(COCOA)
+ DispatchPtr<dispatch_io_t> m_dispatchIO;
+#endif
+#if USE(SOUP)
+ GRefPtr<GInputStream> m_inputStream;
+ GRefPtr<GOutputStream> m_outputStream;
+ GRefPtr<GFileIOStream> m_ioStream;
+#endif
+};
+
+}
+}
+
+#endif
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp
new file mode 100644
index 000000000..ef7ba8998
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp
@@ -0,0 +1,285 @@
+/*
+ * Copyright (C) 2015 Igalia S.L.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheIOChannel.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheFileSystem.h"
+#include <wtf/MainThread.h>
+#include <wtf/RunLoop.h>
+#include <wtf/glib/GUniquePtr.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+static const size_t gDefaultReadBufferSize = 4096;
+
+IOChannel::IOChannel(const String& filePath, Type type)
+ : m_path(filePath)
+ , m_type(type)
+{
+ auto path = WebCore::fileSystemRepresentation(filePath);
+ GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(path.data()));
+ switch (m_type) {
+ case Type::Create: {
+ g_file_delete(file.get(), nullptr, nullptr);
+ m_outputStream = adoptGRef(G_OUTPUT_STREAM(g_file_create(file.get(), static_cast<GFileCreateFlags>(G_FILE_CREATE_PRIVATE), nullptr, nullptr)));
+#if !HAVE(STAT_BIRTHTIME)
+ GUniquePtr<char> birthtimeString(g_strdup_printf("%" G_GUINT64_FORMAT, std::chrono::system_clock::to_time_t(std::chrono::system_clock::now())));
+ g_file_set_attribute_string(file.get(), "xattr::birthtime", birthtimeString.get(), G_FILE_QUERY_INFO_NONE, nullptr, nullptr);
+#endif
+ break;
+ }
+ case Type::Write: {
+ m_ioStream = adoptGRef(g_file_open_readwrite(file.get(), nullptr, nullptr));
+ break;
+ }
+ case Type::Read:
+ m_inputStream = adoptGRef(G_INPUT_STREAM(g_file_read(file.get(), nullptr, nullptr)));
+ break;
+ }
+}
+
+IOChannel::~IOChannel()
+{
+ RELEASE_ASSERT(!m_wasDeleted.exchange(true));
+}
+
+Ref<IOChannel> IOChannel::open(const String& filePath, IOChannel::Type type)
+{
+ return adoptRef(*new IOChannel(filePath, type));
+}
+
+static inline void runTaskInQueue(Function<void ()>&& task, WorkQueue* queue)
+{
+ if (queue) {
+ queue->dispatch(WTFMove(task));
+ return;
+ }
+
+ // Using nullptr as queue submits the result to the main context.
+ RunLoop::main().dispatch(WTFMove(task));
+}
+
+static void fillDataFromReadBuffer(SoupBuffer* readBuffer, size_t size, Data& data)
+{
+ GRefPtr<SoupBuffer> buffer;
+ if (size != readBuffer->length) {
+ // The subbuffer does not copy the data.
+ buffer = adoptGRef(soup_buffer_new_subbuffer(readBuffer, 0, size));
+ } else
+ buffer = readBuffer;
+
+ if (data.isNull()) {
+ // First chunk, we need to force the data to be copied.
+ data = { reinterpret_cast<const uint8_t*>(buffer->data), size };
+ } else {
+ Data dataRead(WTFMove(buffer));
+ // Concatenate will copy the data.
+ data = concatenate(data, dataRead);
+ }
+}
+
+struct ReadAsyncData {
+ RefPtr<IOChannel> channel;
+ GRefPtr<SoupBuffer> buffer;
+ RefPtr<WorkQueue> queue;
+ size_t bytesToRead;
+ std::function<void (Data&, int error)> completionHandler;
+ Data data;
+};
+
+static void inputStreamReadReadyCallback(GInputStream* stream, GAsyncResult* result, gpointer userData)
+{
+ std::unique_ptr<ReadAsyncData> asyncData(static_cast<ReadAsyncData*>(userData));
+ gssize bytesRead = g_input_stream_read_finish(stream, result, nullptr);
+ if (bytesRead == -1) {
+ WorkQueue* queue = asyncData->queue.get();
+ runTaskInQueue([asyncData = WTFMove(asyncData)] {
+ asyncData->completionHandler(asyncData->data, -1);
+ }, queue);
+ return;
+ }
+
+ if (!bytesRead) {
+ WorkQueue* queue = asyncData->queue.get();
+ runTaskInQueue([asyncData = WTFMove(asyncData)] {
+ asyncData->completionHandler(asyncData->data, 0);
+ }, queue);
+ return;
+ }
+
+ ASSERT(bytesRead > 0);
+ fillDataFromReadBuffer(asyncData->buffer.get(), static_cast<size_t>(bytesRead), asyncData->data);
+
+ size_t pendingBytesToRead = asyncData->bytesToRead - asyncData->data.size();
+ if (!pendingBytesToRead) {
+ WorkQueue* queue = asyncData->queue.get();
+ runTaskInQueue([asyncData = WTFMove(asyncData)] {
+ asyncData->completionHandler(asyncData->data, 0);
+ }, queue);
+ return;
+ }
+
+ size_t bytesToRead = std::min(pendingBytesToRead, asyncData->buffer->length);
+ // Use a local variable for the data buffer to pass it to g_input_stream_read_async(), because ReadAsyncData is released.
+ auto data = const_cast<char*>(asyncData->buffer->data);
+ g_input_stream_read_async(stream, data, bytesToRead, G_PRIORITY_DEFAULT, nullptr,
+ reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData.release());
+}
+
+void IOChannel::read(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler)
+{
+ RefPtr<IOChannel> channel(this);
+ if (!m_inputStream) {
+ runTaskInQueue([channel, completionHandler] {
+ Data data;
+ completionHandler(data, -1);
+ }, queue);
+ return;
+ }
+
+ if (!isMainThread()) {
+ readSyncInThread(offset, size, queue, completionHandler);
+ return;
+ }
+
+ size_t bufferSize = std::min(size, gDefaultReadBufferSize);
+ uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize));
+ GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree));
+ ReadAsyncData* asyncData = new ReadAsyncData { this, buffer.get(), queue, size, completionHandler, { } };
+
+ // FIXME: implement offset.
+ g_input_stream_read_async(m_inputStream.get(), const_cast<char*>(buffer->data), bufferSize, G_PRIORITY_DEFAULT, nullptr,
+ reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData);
+}
+
+void IOChannel::readSyncInThread(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler)
+{
+ ASSERT(!isMainThread());
+
+ RefPtr<IOChannel> channel(this);
+ detachThread(createThread("IOChannel::readSync", [channel, size, queue, completionHandler] {
+ size_t bufferSize = std::min(size, gDefaultReadBufferSize);
+ uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize));
+ GRefPtr<SoupBuffer> readBuffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree));
+ Data data;
+ size_t pendingBytesToRead = size;
+ size_t bytesToRead = bufferSize;
+ do {
+ // FIXME: implement offset.
+ gssize bytesRead = g_input_stream_read(channel->m_inputStream.get(), const_cast<char*>(readBuffer->data), bytesToRead, nullptr, nullptr);
+ if (bytesRead == -1) {
+ runTaskInQueue([channel, completionHandler] {
+ Data data;
+ completionHandler(data, -1);
+ }, queue);
+ return;
+ }
+
+ if (!bytesRead)
+ break;
+
+ ASSERT(bytesRead > 0);
+ fillDataFromReadBuffer(readBuffer.get(), static_cast<size_t>(bytesRead), data);
+
+ pendingBytesToRead = size - data.size();
+ bytesToRead = std::min(pendingBytesToRead, readBuffer->length);
+ } while (pendingBytesToRead);
+
+ GRefPtr<SoupBuffer> bufferCapture = data.soupBuffer();
+ runTaskInQueue([channel, bufferCapture, completionHandler] {
+ GRefPtr<SoupBuffer> buffer = bufferCapture;
+ Data data = { WTFMove(buffer) };
+ completionHandler(data, 0);
+ }, queue);
+ }));
+}
+
+struct WriteAsyncData {
+ RefPtr<IOChannel> channel;
+ GRefPtr<SoupBuffer> buffer;
+ RefPtr<WorkQueue> queue;
+ std::function<void (int error)> completionHandler;
+};
+
+static void outputStreamWriteReadyCallback(GOutputStream* stream, GAsyncResult* result, gpointer userData)
+{
+ std::unique_ptr<WriteAsyncData> asyncData(static_cast<WriteAsyncData*>(userData));
+ gssize bytesWritten = g_output_stream_write_finish(stream, result, nullptr);
+ if (bytesWritten == -1) {
+ WorkQueue* queue = asyncData->queue.get();
+ runTaskInQueue([asyncData = WTFMove(asyncData)] {
+ asyncData->completionHandler(-1);
+ }, queue);
+ return;
+ }
+
+ gssize pendingBytesToWrite = asyncData->buffer->length - bytesWritten;
+ if (!pendingBytesToWrite) {
+ WorkQueue* queue = asyncData->queue.get();
+ runTaskInQueue([asyncData = WTFMove(asyncData)] {
+ asyncData->completionHandler(0);
+ }, queue);
+ return;
+ }
+
+ asyncData->buffer = adoptGRef(soup_buffer_new_subbuffer(asyncData->buffer.get(), bytesWritten, pendingBytesToWrite));
+ // Use a local variable for the data buffer to pass it to g_output_stream_write_async(), because WriteAsyncData is released.
+ auto data = asyncData->buffer->data;
+ g_output_stream_write_async(stream, data, pendingBytesToWrite, G_PRIORITY_DEFAULT_IDLE, nullptr,
+ reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData.release());
+}
+
+void IOChannel::write(size_t offset, const Data& data, WorkQueue* queue, std::function<void (int error)> completionHandler)
+{
+ RefPtr<IOChannel> channel(this);
+ if (!m_outputStream && !m_ioStream) {
+ runTaskInQueue([channel, completionHandler] {
+ completionHandler(-1);
+ }, queue);
+ return;
+ }
+
+ GOutputStream* stream = m_outputStream ? m_outputStream.get() : g_io_stream_get_output_stream(G_IO_STREAM(m_ioStream.get()));
+ if (!stream) {
+ runTaskInQueue([channel, completionHandler] {
+ completionHandler(-1);
+ }, queue);
+ return;
+ }
+
+ WriteAsyncData* asyncData = new WriteAsyncData { this, data.soupBuffer(), queue, completionHandler };
+ // FIXME: implement offset.
+ g_output_stream_write_async(stream, asyncData->buffer->data, data.size(), G_PRIORITY_DEFAULT_IDLE, nullptr,
+ reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData);
+}
+
+} // namespace NetworkCache
+} // namespace WebKit
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp
new file mode 100644
index 000000000..cce50a7d0
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp
@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheKey.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheCoders.h"
+#include <wtf/ASCIICType.h>
+#include <wtf/NeverDestroyed.h>
+#include <wtf/persistence/Decoder.h>
+#include <wtf/persistence/Encoder.h>
+#include <wtf/text/CString.h>
+#include <wtf/text/StringBuilder.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+Key::Key(const Key& o)
+ : m_partition(o.m_partition.isolatedCopy())
+ , m_type(o.m_type.isolatedCopy())
+ , m_identifier(o.m_identifier.isolatedCopy())
+ , m_range(o.m_range.isolatedCopy())
+ , m_hash(o.m_hash)
+ , m_partitionHash(o.m_partitionHash)
+{
+}
+
+Key::Key(const String& partition, const String& type, const String& range, const String& identifier, const Salt& salt)
+ : m_partition(partition)
+ , m_type(type)
+ , m_identifier(identifier)
+ , m_range(range)
+ , m_hash(computeHash(salt))
+ , m_partitionHash(computePartitionHash(salt))
+{
+}
+
+Key::Key(WTF::HashTableDeletedValueType)
+ : m_identifier(WTF::HashTableDeletedValue)
+{
+}
+
+Key::Key(const DataKey& dataKey, const Salt& salt)
+ : m_partition(dataKey.partition)
+ , m_type(dataKey.type)
+ , m_identifier(hashAsString(dataKey.identifier))
+ , m_hash(computeHash(salt))
+ , m_partitionHash(computePartitionHash(salt))
+{
+}
+
+Key& Key::operator=(const Key& other)
+{
+ m_partition = other.m_partition.isolatedCopy();
+ m_type = other.m_type.isolatedCopy();
+ m_identifier = other.m_identifier.isolatedCopy();
+ m_range = other.m_range.isolatedCopy();
+ m_hash = other.m_hash;
+ m_partitionHash = other.m_partitionHash;
+ return *this;
+}
+
+static void hashString(SHA1& sha1, const String& string)
+{
+ if (string.isNull())
+ return;
+
+ if (string.is8Bit() && string.containsOnlyASCII()) {
+ const uint8_t nullByte = 0;
+ sha1.addBytes(string.characters8(), string.length());
+ sha1.addBytes(&nullByte, 1);
+ return;
+ }
+ auto cString = string.utf8();
+ // Include terminating null byte.
+ sha1.addBytes(reinterpret_cast<const uint8_t*>(cString.data()), cString.length() + 1);
+}
+
+Key::HashType Key::computeHash(const Salt& salt) const
+{
+ // We don't really need a cryptographic hash. The key is always verified against the entry header.
+ // SHA1 just happens to be suitably sized, fast and available.
+ SHA1 sha1;
+ sha1.addBytes(salt.data(), salt.size());
+
+ hashString(sha1, m_partition);
+ hashString(sha1, m_type);
+ hashString(sha1, m_identifier);
+ hashString(sha1, m_range);
+
+ SHA1::Digest hash;
+ sha1.computeHash(hash);
+ return hash;
+}
+
+Key::HashType Key::computePartitionHash(const Salt& salt) const
+{
+ SHA1 sha1;
+ sha1.addBytes(salt.data(), salt.size());
+
+ hashString(sha1, m_partition);
+
+ SHA1::Digest hash;
+ sha1.computeHash(hash);
+ return hash;
+}
+
+String Key::hashAsString(const HashType& hash)
+{
+ StringBuilder builder;
+ builder.reserveCapacity(hashStringLength());
+ for (auto byte : hash) {
+ builder.append(upperNibbleToASCIIHexDigit(byte));
+ builder.append(lowerNibbleToASCIIHexDigit(byte));
+ }
+ return builder.toString();
+}
+
+template <typename CharType> bool hexDigitsToHash(CharType* characters, Key::HashType& hash)
+{
+ for (unsigned i = 0; i < sizeof(hash); ++i) {
+ auto high = characters[2 * i];
+ auto low = characters[2 * i + 1];
+ if (!isASCIIHexDigit(high) || !isASCIIHexDigit(low))
+ return false;
+ hash[i] = toASCIIHexValue(high, low);
+ }
+ return true;
+}
+
+bool Key::stringToHash(const String& string, HashType& hash)
+{
+ if (string.length() != hashStringLength())
+ return false;
+ if (string.is8Bit())
+ return hexDigitsToHash(string.characters8(), hash);
+ return hexDigitsToHash(string.characters16(), hash);
+}
+
+bool Key::operator==(const Key& other) const
+{
+ return m_hash == other.m_hash && m_partition == other.m_partition && m_type == other.m_type && m_identifier == other.m_identifier && m_range == other.m_range;
+}
+
+void Key::encode(WTF::Persistence::Encoder& encoder) const
+{
+ encoder << m_partition;
+ encoder << m_type;
+ encoder << m_identifier;
+ encoder << m_range;
+ encoder << m_hash;
+ encoder << m_partitionHash;
+}
+
+bool Key::decode(WTF::Persistence::Decoder& decoder, Key& key)
+{
+ return decoder.decode(key.m_partition) && decoder.decode(key.m_type) && decoder.decode(key.m_identifier) && decoder.decode(key.m_range) && decoder.decode(key.m_hash) && decoder.decode(key.m_partitionHash);
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h
new file mode 100644
index 000000000..439ebf0f7
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h
@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheKey_h
+#define NetworkCacheKey_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheData.h"
+#include <wtf/SHA1.h>
+#include <wtf/persistence/Coder.h>
+#include <wtf/text/WTFString.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+struct DataKey {
+ String partition;
+ String type;
+ SHA1::Digest identifier;
+
+ template <class Encoder> void encode(Encoder& encoder) const
+ {
+ encoder << partition << type << identifier;
+ }
+
+ template <class Decoder> static bool decode(Decoder& decoder, DataKey& dataKey)
+ {
+ return decoder.decode(dataKey.partition) && decoder.decode(dataKey.type) && decoder.decode(dataKey.identifier);
+ }
+};
+
+class Key {
+public:
+ typedef SHA1::Digest HashType;
+
+ Key() { }
+ Key(const Key&);
+ Key(Key&&) = default;
+ Key(const String& partition, const String& type, const String& range, const String& identifier, const Salt&);
+ Key(const DataKey&, const Salt&);
+
+ Key& operator=(const Key&);
+ Key& operator=(Key&&) = default;
+
+ Key(WTF::HashTableDeletedValueType);
+ bool isHashTableDeletedValue() const { return m_identifier.isHashTableDeletedValue(); }
+
+ bool isNull() const { return m_identifier.isNull(); }
+
+ const String& partition() const { return m_partition; }
+ const String& identifier() const { return m_identifier; }
+ const String& type() const { return m_type; }
+ const String& range() const { return m_range; }
+
+ const HashType& hash() const { return m_hash; }
+ const HashType& partitionHash() const { return m_partitionHash; }
+
+ static bool stringToHash(const String&, HashType&);
+
+ static size_t hashStringLength() { return 2 * sizeof(m_hash); }
+ String hashAsString() const { return hashAsString(m_hash); }
+ String partitionHashAsString() const { return hashAsString(m_partitionHash); }
+
+ void encode(WTF::Persistence::Encoder&) const;
+ static bool decode(WTF::Persistence::Decoder&, Key&);
+
+ bool operator==(const Key&) const;
+ bool operator!=(const Key& other) const { return !(*this == other); }
+
+private:
+ static String hashAsString(const HashType&);
+ HashType computeHash(const Salt&) const;
+ HashType computePartitionHash(const Salt&) const;
+
+ String m_partition;
+ String m_type;
+ String m_identifier;
+ String m_range;
+ HashType m_hash;
+ HashType m_partitionHash;
+};
+
+}
+}
+
+namespace WTF {
+
+struct NetworkCacheKeyHash {
+ static unsigned hash(const WebKit::NetworkCache::Key& key)
+ {
+ static_assert(SHA1::hashSize >= sizeof(unsigned), "Hash size must be greater than sizeof(unsigned)");
+ return *reinterpret_cast<const unsigned*>(key.hash().data());
+ }
+
+ static bool equal(const WebKit::NetworkCache::Key& a, const WebKit::NetworkCache::Key& b)
+ {
+ return a == b;
+ }
+
+ static const bool safeToCompareToEmptyOrDeleted = false;
+};
+
+template<typename T> struct DefaultHash;
+template<> struct DefaultHash<WebKit::NetworkCache::Key> {
+ typedef NetworkCacheKeyHash Hash;
+};
+
+template<> struct HashTraits<WebKit::NetworkCache::Key> : SimpleClassHashTraits<WebKit::NetworkCache::Key> {
+ static const bool emptyValueIsZero = false;
+
+ static const bool hasIsEmptyValueFunction = true;
+ static bool isEmptyValue(const WebKit::NetworkCache::Key& key) { return key.isNull(); }
+};
+
+} // namespace WTF
+
+#endif
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp
new file mode 100644
index 000000000..6db85d2de
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp
@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+#include "NetworkCacheSpeculativeLoad.h"
+
+#include "Logging.h"
+#include "NetworkCache.h"
+#include "NetworkLoad.h"
+#include "NetworkSession.h"
+#include <WebCore/SessionID.h>
+#include <wtf/CurrentTime.h>
+#include <wtf/RunLoop.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+using namespace WebCore;
+
+SpeculativeLoad::SpeculativeLoad(const GlobalFrameID& frameID, const ResourceRequest& request, std::unique_ptr<NetworkCache::Entry> cacheEntryForValidation, RevalidationCompletionHandler&& completionHandler)
+ : m_frameID(frameID)
+ , m_completionHandler(WTFMove(completionHandler))
+ , m_originalRequest(request)
+ , m_bufferedDataForCache(SharedBuffer::create())
+ , m_cacheEntry(WTFMove(cacheEntryForValidation))
+{
+ ASSERT(!m_cacheEntry || m_cacheEntry->needsValidation());
+
+ NetworkLoadParameters parameters;
+ parameters.sessionID = SessionID::defaultSessionID();
+ parameters.allowStoredCredentials = AllowStoredCredentials;
+ parameters.contentSniffingPolicy = DoNotSniffContent;
+ parameters.request = m_originalRequest;
+#if USE(NETWORK_SESSION)
+ m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters), NetworkSession::defaultSession());
+#else
+ m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters));
+#endif
+}
+
+SpeculativeLoad::~SpeculativeLoad()
+{
+ ASSERT(!m_networkLoad);
+}
+
+void SpeculativeLoad::willSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse)
+{
+ LOG(NetworkCacheSpeculativePreloading, "Speculative redirect %s -> %s", request.url().string().utf8().data(), redirectRequest.url().string().utf8().data());
+
+ m_cacheEntry = NetworkCache::singleton().storeRedirect(request, redirectResponse, redirectRequest);
+ // Create a synthetic cache entry if we can't store.
+ if (!m_cacheEntry)
+ m_cacheEntry = NetworkCache::singleton().makeRedirectEntry(request, redirectResponse, redirectRequest);
+
+ // Don't follow the redirect. The redirect target will be registered for speculative load when it is loaded.
+ didComplete();
+}
+
+auto SpeculativeLoad::didReceiveResponse(ResourceResponse&& receivedResponse) -> ShouldContinueDidReceiveResponse
+{
+ m_response = receivedResponse;
+
+ if (m_response.isMultipart())
+ m_bufferedDataForCache = nullptr;
+
+ bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified
+ if (validationSucceeded && m_cacheEntry)
+ m_cacheEntry = NetworkCache::singleton().update(m_originalRequest, m_frameID, *m_cacheEntry, m_response);
+ else
+ m_cacheEntry = nullptr;
+
+ return ShouldContinueDidReceiveResponse::Yes;
+}
+
+void SpeculativeLoad::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength)
+{
+ ASSERT(!m_cacheEntry);
+
+ if (m_bufferedDataForCache) {
+ // Prevent memory growth in case of streaming data.
+ const size_t maximumCacheBufferSize = 10 * 1024 * 1024;
+ if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize)
+ m_bufferedDataForCache->append(buffer.get());
+ else
+ m_bufferedDataForCache = nullptr;
+ }
+}
+
+void SpeculativeLoad::didFinishLoading(double finishTime)
+{
+ if (m_didComplete)
+ return;
+ if (!m_cacheEntry && m_bufferedDataForCache) {
+ m_cacheEntry = NetworkCache::singleton().store(m_originalRequest, m_response, m_bufferedDataForCache.copyRef(), [](auto& mappedBody) { });
+ // Create a synthetic cache entry if we can't store.
+ if (!m_cacheEntry && isStatusCodeCacheableByDefault(m_response.httpStatusCode()))
+ m_cacheEntry = NetworkCache::singleton().makeEntry(m_originalRequest, m_response, WTFMove(m_bufferedDataForCache));
+ }
+
+ didComplete();
+}
+
+#if USE(PROTECTION_SPACE_AUTH_CALLBACK)
+void SpeculativeLoad::canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&)
+{
+ m_networkLoad->continueCanAuthenticateAgainstProtectionSpace(false);
+}
+#endif
+
+void SpeculativeLoad::didFailLoading(const ResourceError&)
+{
+ if (m_didComplete)
+ return;
+ m_cacheEntry = nullptr;
+
+ didComplete();
+}
+
+void SpeculativeLoad::didComplete()
+{
+ RELEASE_ASSERT(RunLoop::isMain());
+
+ if (m_didComplete)
+ return;
+ m_didComplete = true;
+ m_networkLoad = nullptr;
+
+ // Make sure speculatively revalidated resources do not get validated by the NetworkResourceLoader again.
+ if (m_cacheEntry)
+ m_cacheEntry->setNeedsValidation(false);
+
+ m_completionHandler(WTFMove(m_cacheEntry));
+}
+
+} // namespace NetworkCache
+} // namespace WebKit
+
+#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h
new file mode 100644
index 000000000..3bc2d2e65
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheSpeculativeLoad_h
+#define NetworkCacheSpeculativeLoad_h
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+
+#include "NetworkCache.h"
+#include "NetworkCacheEntry.h"
+#include "NetworkLoadClient.h"
+#include <WebCore/ResourceRequest.h>
+#include <WebCore/ResourceResponse.h>
+#include <WebCore/SharedBuffer.h>
+
+namespace WebKit {
+
+class NetworkLoad;
+
+namespace NetworkCache {
+
+class SpeculativeLoad final : public NetworkLoadClient {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ typedef std::function<void (std::unique_ptr<NetworkCache::Entry>)> RevalidationCompletionHandler;
+ SpeculativeLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, std::unique_ptr<NetworkCache::Entry>, RevalidationCompletionHandler&&);
+
+ virtual ~SpeculativeLoad();
+
+ const WebCore::ResourceRequest& originalRequest() const { return m_originalRequest; }
+
+private:
+ // NetworkLoadClient.
+ void didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override { }
+#if USE(PROTECTION_SPACE_AUTH_CALLBACK)
+ void canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) override;
+#endif
+ bool isSynchronous() const override { return false; }
+ void willSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceRequest&& redirectRequest, WebCore::ResourceResponse&& redirectResponse) override;
+ ShouldContinueDidReceiveResponse didReceiveResponse(WebCore::ResourceResponse&&) override;
+ void didReceiveBuffer(Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) override;
+ void didFinishLoading(double finishTime) override;
+ void didFailLoading(const WebCore::ResourceError&) override;
+
+ void didComplete();
+
+ GlobalFrameID m_frameID;
+ RevalidationCompletionHandler m_completionHandler;
+ WebCore::ResourceRequest m_originalRequest;
+
+ std::unique_ptr<NetworkLoad> m_networkLoad;
+
+ WebCore::ResourceResponse m_response;
+
+ RefPtr<WebCore::SharedBuffer> m_bufferedDataForCache;
+ std::unique_ptr<NetworkCache::Entry> m_cacheEntry;
+ bool m_didComplete { false };
+};
+
+} // namespace NetworkCache
+} // namespace WebKit
+
+#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+
+#endif // NetworkCacheSpeculativeLoad_h
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp
new file mode 100644
index 000000000..db36bb8eb
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp
@@ -0,0 +1,589 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+#include "NetworkCacheSpeculativeLoadManager.h"
+
+#include "Logging.h"
+#include "NetworkCacheEntry.h"
+#include "NetworkCacheSpeculativeLoad.h"
+#include "NetworkCacheSubresourcesEntry.h"
+#include "NetworkProcess.h"
+#include <WebCore/DiagnosticLoggingKeys.h>
+#include <WebCore/HysteresisActivity.h>
+#include <wtf/HashCountedSet.h>
+#include <wtf/NeverDestroyed.h>
+#include <wtf/RefCounted.h>
+#include <wtf/RunLoop.h>
+
+namespace WebKit {
+
+namespace NetworkCache {
+
+using namespace WebCore;
+
+static const auto preloadedEntryLifetime = 10s;
+
+#if !LOG_DISABLED
+static HashCountedSet<String>& allSpeculativeLoadingDiagnosticMessages()
+{
+ static NeverDestroyed<HashCountedSet<String>> messages;
+ return messages;
+}
+
+static void printSpeculativeLoadingDiagnosticMessageCounts()
+{
+ LOG(NetworkCacheSpeculativePreloading, "-- Speculative loading statistics --");
+ for (auto& pair : allSpeculativeLoadingDiagnosticMessages())
+ LOG(NetworkCacheSpeculativePreloading, "%s: %u", pair.key.utf8().data(), pair.value);
+}
+#endif
+
+static void logSpeculativeLoadingDiagnosticMessage(const GlobalFrameID& frameID, const String& message)
+{
+#if !LOG_DISABLED
+ if (WebKit2LogNetworkCacheSpeculativePreloading.state == WTFLogChannelOn)
+ allSpeculativeLoadingDiagnosticMessages().add(message);
+#endif
+ NetworkProcess::singleton().logDiagnosticMessage(frameID.first, WebCore::DiagnosticLoggingKeys::networkCacheKey(), message, WebCore::ShouldSample::Yes);
+}
+
+static const AtomicString& subresourcesType()
+{
+ ASSERT(RunLoop::isMain());
+ static NeverDestroyed<const AtomicString> resource("SubResources", AtomicString::ConstructFromLiteral);
+ return resource;
+}
+
+static inline Key makeSubresourcesKey(const Key& resourceKey, const Salt& salt)
+{
+ return Key(resourceKey.partition(), subresourcesType(), resourceKey.range(), resourceKey.identifier(), salt);
+}
+
+static inline ResourceRequest constructRevalidationRequest(const Key& key, const SubresourceInfo& subResourceInfo, const Entry* entry)
+{
+ ResourceRequest revalidationRequest(key.identifier());
+ revalidationRequest.setHTTPHeaderFields(subResourceInfo.requestHeaders());
+ revalidationRequest.setFirstPartyForCookies(subResourceInfo.firstPartyForCookies());
+ if (!key.partition().isEmpty())
+ revalidationRequest.setCachePartition(key.partition());
+ ASSERT_WITH_MESSAGE(key.range().isEmpty(), "range is not supported");
+
+ revalidationRequest.makeUnconditional();
+ if (entry) {
+ String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag);
+ if (!eTag.isEmpty())
+ revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag);
+
+ String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified);
+ if (!lastModified.isEmpty())
+ revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified);
+ }
+
+ revalidationRequest.setPriority(subResourceInfo.priority());
+
+ return revalidationRequest;
+}
+
+static bool responseNeedsRevalidation(const ResourceResponse& response, std::chrono::system_clock::time_point timestamp)
+{
+ if (response.cacheControlContainsNoCache())
+ return true;
+
+ auto age = computeCurrentAge(response, timestamp);
+ auto lifetime = computeFreshnessLifetimeForHTTPFamily(response, timestamp);
+ return age - lifetime > 0ms;
+}
+
+class SpeculativeLoadManager::ExpiringEntry {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ explicit ExpiringEntry(std::function<void()>&& expirationHandler)
+ : m_lifetimeTimer(WTFMove(expirationHandler))
+ {
+ m_lifetimeTimer.startOneShot(preloadedEntryLifetime);
+ }
+
+private:
+ Timer m_lifetimeTimer;
+};
+
+class SpeculativeLoadManager::PreloadedEntry : private ExpiringEntry {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ PreloadedEntry(std::unique_ptr<Entry> entry, std::optional<ResourceRequest>&& speculativeValidationRequest, std::function<void()>&& lifetimeReachedHandler)
+ : ExpiringEntry(WTFMove(lifetimeReachedHandler))
+ , m_entry(WTFMove(entry))
+ , m_speculativeValidationRequest(WTFMove(speculativeValidationRequest))
+ { }
+
+ std::unique_ptr<Entry> takeCacheEntry()
+ {
+ ASSERT(m_entry);
+ return WTFMove(m_entry);
+ }
+
+ const std::optional<ResourceRequest>& revalidationRequest() const { return m_speculativeValidationRequest; }
+ bool wasRevalidated() const { return !!m_speculativeValidationRequest; }
+
+private:
+ std::unique_ptr<Entry> m_entry;
+ std::optional<ResourceRequest> m_speculativeValidationRequest;
+};
+
+class SpeculativeLoadManager::PendingFrameLoad : public RefCounted<PendingFrameLoad> {
+public:
+ static Ref<PendingFrameLoad> create(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler)
+ {
+ return adoptRef(*new PendingFrameLoad(storage, mainResourceKey, WTFMove(loadCompletionHandler)));
+ }
+
+ ~PendingFrameLoad()
+ {
+ ASSERT(m_didFinishLoad);
+ ASSERT(m_didRetrieveExistingEntry);
+ }
+
+ void registerSubresourceLoad(const ResourceRequest& request, const Key& subresourceKey)
+ {
+ ASSERT(RunLoop::isMain());
+ m_subresourceLoads.append(std::make_unique<SubresourceLoad>(request, subresourceKey));
+ m_loadHysteresisActivity.impulse();
+ }
+
+ void markLoadAsCompleted()
+ {
+ ASSERT(RunLoop::isMain());
+ if (m_didFinishLoad)
+ return;
+
+#if !LOG_DISABLED
+ printSpeculativeLoadingDiagnosticMessageCounts();
+#endif
+
+ m_didFinishLoad = true;
+ saveToDiskIfReady();
+ m_loadCompletionHandler();
+ }
+
+ void setExistingSubresourcesEntry(std::unique_ptr<SubresourcesEntry> entry)
+ {
+ ASSERT(!m_existingEntry);
+ ASSERT(!m_didRetrieveExistingEntry);
+
+ m_existingEntry = WTFMove(entry);
+ m_didRetrieveExistingEntry = true;
+ saveToDiskIfReady();
+ }
+
+private:
+ PendingFrameLoad(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler)
+ : m_storage(storage)
+ , m_mainResourceKey(mainResourceKey)
+ , m_loadCompletionHandler(WTFMove(loadCompletionHandler))
+ , m_loadHysteresisActivity([this](HysteresisState state) { if (state == HysteresisState::Stopped) markLoadAsCompleted(); })
+ {
+ m_loadHysteresisActivity.impulse();
+ }
+
+ void saveToDiskIfReady()
+ {
+ if (!m_didFinishLoad || !m_didRetrieveExistingEntry)
+ return;
+
+ if (m_subresourceLoads.isEmpty())
+ return;
+
+#if !LOG_DISABLED
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Saving to disk list of subresources for '%s':", m_mainResourceKey.identifier().utf8().data());
+ for (auto& subresourceLoad : m_subresourceLoads)
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) * Subresource: '%s'.", subresourceLoad->key.identifier().utf8().data());
+#endif
+
+ if (m_existingEntry) {
+ m_existingEntry->updateSubresourceLoads(m_subresourceLoads);
+ m_storage.store(m_existingEntry->encodeAsStorageRecord(), [](const Data&) { });
+ } else {
+ SubresourcesEntry entry(makeSubresourcesKey(m_mainResourceKey, m_storage.salt()), m_subresourceLoads);
+ m_storage.store(entry.encodeAsStorageRecord(), [](const Data&) { });
+ }
+ }
+
+ Storage& m_storage;
+ Key m_mainResourceKey;
+ Vector<std::unique_ptr<SubresourceLoad>> m_subresourceLoads;
+ std::function<void()> m_loadCompletionHandler;
+ HysteresisActivity m_loadHysteresisActivity;
+ std::unique_ptr<SubresourcesEntry> m_existingEntry;
+ bool m_didFinishLoad { false };
+ bool m_didRetrieveExistingEntry { false };
+};
+
+SpeculativeLoadManager::SpeculativeLoadManager(Storage& storage)
+ : m_storage(storage)
+{
+}
+
+SpeculativeLoadManager::~SpeculativeLoadManager()
+{
+}
+
+#if !LOG_DISABLED
+
+static void dumpHTTPHeadersDiff(const HTTPHeaderMap& headersA, const HTTPHeaderMap& headersB)
+{
+ auto aEnd = headersA.end();
+ for (auto it = headersA.begin(); it != aEnd; ++it) {
+ String valueB = headersB.get(it->key);
+ if (valueB.isNull())
+ LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header is only in first request (value: %s)", it->key.utf8().data(), it->value.utf8().data());
+ else if (it->value != valueB)
+ LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header differs in both requests: %s != %s", it->key.utf8().data(), it->value.utf8().data(), valueB.utf8().data());
+ }
+ auto bEnd = headersB.end();
+ for (auto it = headersB.begin(); it != bEnd; ++it) {
+ if (!headersA.contains(it->key))
+ LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header is only in second request (value: %s)", it->key.utf8().data(), it->value.utf8().data());
+ }
+}
+
+#endif
+
+static bool requestsHeadersMatch(const ResourceRequest& speculativeValidationRequest, const ResourceRequest& actualRequest)
+{
+ ASSERT(!actualRequest.isConditional());
+ ResourceRequest speculativeRequest = speculativeValidationRequest;
+ speculativeRequest.makeUnconditional();
+
+ if (speculativeRequest.httpHeaderFields() != actualRequest.httpHeaderFields()) {
+ LOG(NetworkCacheSpeculativePreloading, "Cannot reuse speculatively validated entry because HTTP headers used for validation do not match");
+#if !LOG_DISABLED
+ dumpHTTPHeadersDiff(speculativeRequest.httpHeaderFields(), actualRequest.httpHeaderFields());
+#endif
+ return false;
+ }
+ return true;
+}
+
+bool SpeculativeLoadManager::canUsePreloadedEntry(const PreloadedEntry& entry, const ResourceRequest& actualRequest)
+{
+ if (!entry.wasRevalidated())
+ return true;
+
+ ASSERT(entry.revalidationRequest());
+ return requestsHeadersMatch(*entry.revalidationRequest(), actualRequest);
+}
+
+bool SpeculativeLoadManager::canUsePendingPreload(const SpeculativeLoad& load, const ResourceRequest& actualRequest)
+{
+ return requestsHeadersMatch(load.originalRequest(), actualRequest);
+}
+
+bool SpeculativeLoadManager::canRetrieve(const Key& storageKey, const WebCore::ResourceRequest& request, const GlobalFrameID& frameID) const
+{
+ // Check already preloaded entries.
+ if (auto preloadedEntry = m_preloadedEntries.get(storageKey)) {
+ if (!canUsePreloadedEntry(*preloadedEntry, request)) {
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: Could not use preloaded entry to satisfy request for '%s' due to HTTP headers mismatch:", storageKey.identifier().utf8().data());
+ logSpeculativeLoadingDiagnosticMessage(frameID, preloadedEntry->wasRevalidated() ? DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey() : DiagnosticLoggingKeys::wastedSpeculativeWarmupWithoutRevalidationKey());
+ return false;
+ }
+
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: Using preloaded entry to satisfy request for '%s':", storageKey.identifier().utf8().data());
+ logSpeculativeLoadingDiagnosticMessage(frameID, preloadedEntry->wasRevalidated() ? DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey() : DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey());
+ return true;
+ }
+
+ // Check pending speculative revalidations.
+ auto* pendingPreload = m_pendingPreloads.get(storageKey);
+ if (!pendingPreload) {
+ if (m_notPreloadedEntries.get(storageKey))
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryWronglyNotWarmedUpKey());
+ else
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::unknownEntryRequestKey());
+
+ return false;
+ }
+
+ if (!canUsePendingPreload(*pendingPreload, request)) {
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: revalidation already in progress for '%s' but unusable due to HTTP headers mismatch:", storageKey.identifier().utf8().data());
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey());
+ return false;
+ }
+
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: revalidation already in progress for '%s':", storageKey.identifier().utf8().data());
+
+ return true;
+}
+
+void SpeculativeLoadManager::retrieve(const Key& storageKey, RetrieveCompletionHandler&& completionHandler)
+{
+ if (auto preloadedEntry = m_preloadedEntries.take(storageKey)) {
+ RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), cacheEntry = preloadedEntry->takeCacheEntry()] () mutable {
+ completionHandler(WTFMove(cacheEntry));
+ });
+ return;
+ }
+ ASSERT(m_pendingPreloads.contains(storageKey));
+ // FIXME: This breaks incremental loading when the revalidation is not successful.
+ auto addResult = m_pendingRetrieveRequests.ensure(storageKey, [] {
+ return std::make_unique<Vector<RetrieveCompletionHandler>>();
+ });
+ addResult.iterator->value->append(WTFMove(completionHandler));
+}
+
+void SpeculativeLoadManager::registerLoad(const GlobalFrameID& frameID, const ResourceRequest& request, const Key& resourceKey)
+{
+ ASSERT(RunLoop::isMain());
+ ASSERT(request.url().protocolIsInHTTPFamily());
+
+ if (request.httpMethod() != "GET")
+ return;
+
+ auto isMainResource = request.requester() == ResourceRequest::Requester::Main;
+ if (isMainResource) {
+ // Mark previous load in this frame as completed if necessary.
+ if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID))
+ pendingFrameLoad->markLoadAsCompleted();
+
+ ASSERT(!m_pendingFrameLoads.contains(frameID));
+
+ // Start tracking loads in this frame.
+ RefPtr<PendingFrameLoad> pendingFrameLoad = PendingFrameLoad::create(m_storage, resourceKey, [this, frameID] {
+ bool wasRemoved = m_pendingFrameLoads.remove(frameID);
+ ASSERT_UNUSED(wasRemoved, wasRemoved);
+ });
+ m_pendingFrameLoads.add(frameID, pendingFrameLoad);
+
+ // Retrieve the subresources entry if it exists to start speculative revalidation and to update it.
+ retrieveSubresourcesEntry(resourceKey, [this, frameID, pendingFrameLoad](std::unique_ptr<SubresourcesEntry> entry) {
+ if (entry)
+ startSpeculativeRevalidation(frameID, *entry);
+
+ pendingFrameLoad->setExistingSubresourcesEntry(WTFMove(entry));
+ });
+ return;
+ }
+
+ if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID))
+ pendingFrameLoad->registerSubresourceLoad(request, resourceKey);
+}
+
+void SpeculativeLoadManager::addPreloadedEntry(std::unique_ptr<Entry> entry, const GlobalFrameID& frameID, std::optional<ResourceRequest>&& revalidationRequest)
+{
+ ASSERT(entry);
+ ASSERT(!entry->needsValidation());
+ auto key = entry->key();
+ m_preloadedEntries.add(key, std::make_unique<PreloadedEntry>(WTFMove(entry), WTFMove(revalidationRequest), [this, key, frameID] {
+ auto preloadedEntry = m_preloadedEntries.take(key);
+ ASSERT(preloadedEntry);
+ if (preloadedEntry->wasRevalidated())
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey());
+ else
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithoutRevalidationKey());
+ }));
+}
+
+void SpeculativeLoadManager::retrieveEntryFromStorage(const SubresourceInfo& info, RetrieveCompletionHandler&& completionHandler)
+{
+ m_storage.retrieve(info.key(), static_cast<unsigned>(info.priority()), [completionHandler = WTFMove(completionHandler)](auto record) {
+ if (!record) {
+ completionHandler(nullptr);
+ return false;
+ }
+ auto entry = Entry::decodeStorageRecord(*record);
+ if (!entry) {
+ completionHandler(nullptr);
+ return false;
+ }
+
+ auto& response = entry->response();
+ if (responseNeedsRevalidation(response, entry->timeStamp())) {
+ // Do not use cached redirects that have expired.
+ if (entry->redirectRequest()) {
+ completionHandler(nullptr);
+ return true;
+ }
+ entry->setNeedsValidation(true);
+ }
+
+ completionHandler(WTFMove(entry));
+ return true;
+ });
+}
+
+bool SpeculativeLoadManager::satisfyPendingRequests(const Key& key, Entry* entry)
+{
+ auto completionHandlers = m_pendingRetrieveRequests.take(key);
+ if (!completionHandlers)
+ return false;
+
+ for (auto& completionHandler : *completionHandlers)
+ completionHandler(entry ? std::make_unique<Entry>(*entry) : nullptr);
+
+ return true;
+}
+
+void SpeculativeLoadManager::revalidateSubresource(const SubresourceInfo& subresourceInfo, std::unique_ptr<Entry> entry, const GlobalFrameID& frameID)
+{
+ ASSERT(!entry || entry->needsValidation());
+
+ auto& key = subresourceInfo.key();
+
+ // Range is not supported.
+ if (!key.range().isEmpty())
+ return;
+
+ ResourceRequest revalidationRequest = constructRevalidationRequest(key, subresourceInfo, entry.get());
+
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculatively revalidating '%s':", key.identifier().utf8().data());
+
+ auto revalidator = std::make_unique<SpeculativeLoad>(frameID, revalidationRequest, WTFMove(entry), [this, key, revalidationRequest, frameID](std::unique_ptr<Entry> revalidatedEntry) {
+ ASSERT(!revalidatedEntry || !revalidatedEntry->needsValidation());
+ ASSERT(!revalidatedEntry || revalidatedEntry->key() == key);
+
+ auto protectRevalidator = m_pendingPreloads.take(key);
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculative revalidation completed for '%s':", key.identifier().utf8().data());
+
+ if (satisfyPendingRequests(key, revalidatedEntry.get())) {
+ if (revalidatedEntry)
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey());
+ return;
+ }
+
+ if (revalidatedEntry)
+ addPreloadedEntry(WTFMove(revalidatedEntry), frameID, revalidationRequest);
+ });
+ m_pendingPreloads.add(key, WTFMove(revalidator));
+}
+
+static bool canRevalidate(const SubresourceInfo& subresourceInfo, const Entry* entry)
+{
+ ASSERT(!subresourceInfo.isTransient());
+ ASSERT(!entry || entry->needsValidation());
+
+ if (entry && entry->response().hasCacheValidatorFields())
+ return true;
+
+ auto seenAge = subresourceInfo.lastSeen() - subresourceInfo.firstSeen();
+ if (seenAge == 0ms) {
+ LOG(NetworkCacheSpeculativePreloading, "Speculative load: Seen only once");
+ return false;
+ }
+
+ auto now = std::chrono::system_clock::now();
+ auto firstSeenAge = now - subresourceInfo.firstSeen();
+ auto lastSeenAge = now - subresourceInfo.lastSeen();
+ // Sanity check.
+ if (seenAge <= 0ms || firstSeenAge <= 0ms || lastSeenAge <= 0ms)
+ return false;
+
+ // Load full resources speculatively if they seem to stay the same.
+ const auto minimumAgeRatioToLoad = 2. / 3;
+ const auto recentMinimumAgeRatioToLoad = 1. / 3;
+ const auto recentThreshold = 5min;
+
+ auto ageRatio = std::chrono::duration_cast<std::chrono::duration<double>>(seenAge) / firstSeenAge;
+ auto minimumAgeRatio = lastSeenAge > recentThreshold ? minimumAgeRatioToLoad : recentMinimumAgeRatioToLoad;
+
+ LOG(NetworkCacheSpeculativePreloading, "Speculative load: ok=%d ageRatio=%f entry=%d", ageRatio > minimumAgeRatio, ageRatio, !!entry);
+
+ if (ageRatio > minimumAgeRatio)
+ return true;
+
+ return false;
+}
+
+void SpeculativeLoadManager::preloadEntry(const Key& key, const SubresourceInfo& subresourceInfo, const GlobalFrameID& frameID)
+{
+ if (m_pendingPreloads.contains(key))
+ return;
+ m_pendingPreloads.add(key, nullptr);
+
+ retrieveEntryFromStorage(subresourceInfo, [this, key, subresourceInfo, frameID](std::unique_ptr<Entry> entry) {
+ ASSERT(!m_pendingPreloads.get(key));
+ bool removed = m_pendingPreloads.remove(key);
+ ASSERT_UNUSED(removed, removed);
+
+ if (satisfyPendingRequests(key, entry.get())) {
+ if (entry)
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey());
+ return;
+ }
+
+ if (!entry || entry->needsValidation()) {
+ if (canRevalidate(subresourceInfo, entry.get()))
+ revalidateSubresource(subresourceInfo, WTFMove(entry), frameID);
+ return;
+ }
+
+ addPreloadedEntry(WTFMove(entry), frameID);
+ });
+}
+
+void SpeculativeLoadManager::startSpeculativeRevalidation(const GlobalFrameID& frameID, SubresourcesEntry& entry)
+{
+ for (auto& subresourceInfo : entry.subresources()) {
+ auto& key = subresourceInfo.key();
+ if (!subresourceInfo.isTransient())
+ preloadEntry(key, subresourceInfo, frameID);
+ else {
+ LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Not preloading '%s' because it is marked as transient", key.identifier().utf8().data());
+ m_notPreloadedEntries.add(key, std::make_unique<ExpiringEntry>([this, key, frameID] {
+ logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryRightlyNotWarmedUpKey());
+ m_notPreloadedEntries.remove(key);
+ }));
+ }
+ }
+}
+
+void SpeculativeLoadManager::retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)>&& completionHandler)
+{
+ ASSERT(storageKey.type() == "Resource");
+ auto subresourcesStorageKey = makeSubresourcesKey(storageKey, m_storage.salt());
+ m_storage.retrieve(subresourcesStorageKey, static_cast<unsigned>(ResourceLoadPriority::Medium), [completionHandler = WTFMove(completionHandler)](auto record) {
+ if (!record) {
+ completionHandler(nullptr);
+ return false;
+ }
+
+ auto subresourcesEntry = SubresourcesEntry::decodeStorageRecord(*record);
+ if (!subresourcesEntry) {
+ completionHandler(nullptr);
+ return false;
+ }
+
+ completionHandler(WTFMove(subresourcesEntry));
+ return true;
+ });
+}
+
+} // namespace NetworkCache
+
+} // namespace WebKit
+
+#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h
new file mode 100644
index 000000000..e9c621a91
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheSpeculativeLoadManager_h
+#define NetworkCacheSpeculativeLoadManager_h
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+
+#include "NetworkCache.h"
+#include "NetworkCacheStorage.h"
+#include <WebCore/ResourceRequest.h>
+#include <wtf/HashMap.h>
+#include <wtf/Vector.h>
+
+namespace WebKit {
+
+namespace NetworkCache {
+
+class Entry;
+class SpeculativeLoad;
+class SubresourceInfo;
+class SubresourcesEntry;
+
+class SpeculativeLoadManager {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ explicit SpeculativeLoadManager(Storage&);
+ ~SpeculativeLoadManager();
+
+ void registerLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, const Key& resourceKey);
+
+ typedef Function<void (std::unique_ptr<Entry>)> RetrieveCompletionHandler;
+
+ bool canRetrieve(const Key& storageKey, const WebCore::ResourceRequest&, const GlobalFrameID&) const;
+ void retrieve(const Key& storageKey, RetrieveCompletionHandler&&);
+
+private:
+ class PreloadedEntry;
+
+ void addPreloadedEntry(std::unique_ptr<Entry>, const GlobalFrameID&, std::optional<WebCore::ResourceRequest>&& revalidationRequest = std::nullopt);
+ void preloadEntry(const Key&, const SubresourceInfo&, const GlobalFrameID&);
+ void retrieveEntryFromStorage(const SubresourceInfo&, RetrieveCompletionHandler&&);
+ void revalidateSubresource(const SubresourceInfo&, std::unique_ptr<Entry>, const GlobalFrameID&);
+ bool satisfyPendingRequests(const Key&, Entry*);
+ void retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)>&&);
+ void startSpeculativeRevalidation(const GlobalFrameID&, SubresourcesEntry&);
+
+ static bool canUsePreloadedEntry(const PreloadedEntry&, const WebCore::ResourceRequest& actualRequest);
+ static bool canUsePendingPreload(const SpeculativeLoad&, const WebCore::ResourceRequest& actualRequest);
+
+ Storage& m_storage;
+
+ class PendingFrameLoad;
+ HashMap<GlobalFrameID, RefPtr<PendingFrameLoad>> m_pendingFrameLoads;
+
+ HashMap<Key, std::unique_ptr<SpeculativeLoad>> m_pendingPreloads;
+ HashMap<Key, std::unique_ptr<Vector<RetrieveCompletionHandler>>> m_pendingRetrieveRequests;
+
+ HashMap<Key, std::unique_ptr<PreloadedEntry>> m_preloadedEntries;
+
+ class ExpiringEntry;
+ HashMap<Key, std::unique_ptr<ExpiringEntry>> m_notPreloadedEntries; // For logging.
+};
+
+} // namespace NetworkCache
+
+} // namespace WebKit
+
+#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+
+#endif // NetworkCacheSpeculativeLoadManager_h
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp
new file mode 100644
index 000000000..0fc8d1d3d
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp
@@ -0,0 +1,450 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#if ENABLE(NETWORK_CACHE)
+#include "NetworkCacheStatistics.h"
+
+#include "Logging.h"
+#include "NetworkCache.h"
+#include "NetworkCacheFileSystem.h"
+#include "NetworkProcess.h"
+#include <WebCore/DiagnosticLoggingKeys.h>
+#include <WebCore/DiagnosticLoggingResultType.h>
+#include <WebCore/ResourceRequest.h>
+#include <WebCore/SQLiteDatabaseTracker.h>
+#include <WebCore/SQLiteStatement.h>
+#include <WebCore/SQLiteTransaction.h>
+#include <wtf/RunLoop.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+static const char* StatisticsDatabaseName = "WebKitCacheStatistics.db";
+static const std::chrono::milliseconds mininumWriteInterval = std::chrono::milliseconds(10000);
+
+static bool executeSQLCommand(WebCore::SQLiteDatabase& database, const String& sql)
+{
+ ASSERT(!RunLoop::isMain());
+ ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress());
+ ASSERT(database.isOpen());
+
+ bool result = database.executeCommand(sql);
+ if (!result)
+ LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", sql.utf8().data(), database.lastErrorMsg());
+
+ return result;
+}
+
+static bool executeSQLStatement(WebCore::SQLiteStatement& statement)
+{
+ ASSERT(!RunLoop::isMain());
+ ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress());
+ ASSERT(statement.database().isOpen());
+
+ if (statement.step() != SQLITE_DONE) {
+ LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", statement.query().utf8().data(), statement.database().lastErrorMsg());
+ return false;
+ }
+
+ return true;
+}
+
+std::unique_ptr<Statistics> Statistics::open(const String& cachePath)
+{
+ ASSERT(RunLoop::isMain());
+
+ String databasePath = WebCore::pathByAppendingComponent(cachePath, StatisticsDatabaseName);
+ return std::make_unique<Statistics>(databasePath);
+}
+
+Statistics::Statistics(const String& databasePath)
+ : m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Statistics.Background", WorkQueue::Type::Serial, WorkQueue::QOS::Background))
+ , m_writeTimer(*this, &Statistics::writeTimerFired)
+{
+ initialize(databasePath);
+}
+
+void Statistics::initialize(const String& databasePath)
+{
+ ASSERT(RunLoop::isMain());
+
+ auto startTime = std::chrono::system_clock::now();
+
+ serialBackgroundIOQueue().dispatch([this, databasePath = databasePath.isolatedCopy(), networkCachePath = singleton().recordsPath().isolatedCopy(), startTime] {
+ WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter;
+
+ if (!WebCore::makeAllDirectories(WebCore::directoryName(databasePath)))
+ return;
+
+ LOG(NetworkCache, "(NetworkProcess) Opening network cache statistics database at %s...", databasePath.utf8().data());
+ m_database.open(databasePath);
+ m_database.disableThreadingChecks();
+ if (!m_database.isOpen()) {
+ LOG_ERROR("Network cache statistics: Failed to open / create the network cache statistics database");
+ return;
+ }
+
+ executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS AlreadyRequested (hash TEXT PRIMARY KEY)"));
+ executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS UncachedReason (hash TEXT PRIMARY KEY, reason INTEGER)"));
+
+ WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT count(*) FROM AlreadyRequested"));
+ if (statement.prepareAndStep() != SQLITE_ROW) {
+ LOG_ERROR("Network cache statistics: Failed to count the number of rows in AlreadyRequested table");
+ return;
+ }
+
+ m_approximateEntryCount = statement.getColumnInt(0);
+
+#if !LOG_DISABLED
+ auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count());
+#endif
+ LOG(NetworkCache, "(NetworkProcess) Network cache statistics database load complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS);
+
+ if (!m_approximateEntryCount) {
+ bootstrapFromNetworkCache(networkCachePath);
+#if !LOG_DISABLED
+ elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count());
+#endif
+ LOG(NetworkCache, "(NetworkProcess) Network cache statistics database bootstrapping complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS);
+ }
+ });
+}
+
+void Statistics::bootstrapFromNetworkCache(const String& networkCachePath)
+{
+ ASSERT(!RunLoop::isMain());
+
+ LOG(NetworkCache, "(NetworkProcess) Bootstrapping the network cache statistics database from the network cache...");
+
+ HashSet<String> hashes;
+ traverseRecordsFiles(networkCachePath, ASCIILiteral("Resource"), [&hashes](const String& fileName, const String& hashString, const String& type, bool isBodyBlob, const String& recordDirectoryPath) {
+ if (isBodyBlob)
+ return;
+
+ Key::HashType hash;
+ if (!Key::stringToHash(hashString, hash))
+ return;
+
+ hashes.add(hashString);
+ });
+
+ WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter;
+ WebCore::SQLiteTransaction writeTransaction(m_database);
+ writeTransaction.begin();
+
+ addHashesToDatabase(hashes);
+
+ writeTransaction.commit();
+}
+
+void Statistics::shrinkIfNeeded()
+{
+ ASSERT(RunLoop::isMain());
+ const size_t maxEntries = 100000;
+
+ if (m_approximateEntryCount < maxEntries)
+ return;
+
+ LOG(NetworkCache, "(NetworkProcess) shrinking statistics cache m_approximateEntryCount=%lu, maxEntries=%lu", static_cast<size_t>(m_approximateEntryCount), maxEntries);
+
+ clear();
+
+ serialBackgroundIOQueue().dispatch([this, networkCachePath = singleton().recordsPath().isolatedCopy()] {
+ bootstrapFromNetworkCache(networkCachePath);
+ LOG(NetworkCache, "(NetworkProcess) statistics cache shrink completed m_approximateEntryCount=%lu", static_cast<size_t>(m_approximateEntryCount));
+ });
+}
+
+void Statistics::recordRetrievalRequest(uint64_t webPageID)
+{
+ NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalRequestKey(), WebCore::ShouldSample::Yes);
+}
+
+void Statistics::recordNotCachingResponse(const Key& key, StoreDecision storeDecision)
+{
+ ASSERT(storeDecision != StoreDecision::Yes);
+
+ m_storeDecisionsToAdd.set(key.hashAsString(), storeDecision);
+ if (!m_writeTimer.isActive())
+ m_writeTimer.startOneShot(mininumWriteInterval);
+}
+
+static String retrieveDecisionToDiagnosticKey(RetrieveDecision retrieveDecision)
+{
+ switch (retrieveDecision) {
+ case RetrieveDecision::NoDueToHTTPMethod:
+ return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey();
+ case RetrieveDecision::NoDueToConditionalRequest:
+ return WebCore::DiagnosticLoggingKeys::isConditionalRequestKey();
+ case RetrieveDecision::NoDueToReloadIgnoringCache:
+ return WebCore::DiagnosticLoggingKeys::isReloadIgnoringCacheDataKey();
+ case RetrieveDecision::NoDueToStreamingMedia:
+ return WebCore::DiagnosticLoggingKeys::streamingMedia();
+ case RetrieveDecision::Yes:
+ ASSERT_NOT_REACHED();
+ break;
+ }
+ return emptyString();
+}
+
+void Statistics::recordNotUsingCacheForRequest(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, RetrieveDecision retrieveDecision)
+{
+ ASSERT(retrieveDecision != RetrieveDecision::Yes);
+
+ auto hash = key.hashAsString();
+ queryWasEverRequested(hash, NeedUncachedReason::No, [this, hash, requestURL = request.url(), webPageID, retrieveDecision](bool wasEverRequested, const std::optional<StoreDecision>&) {
+ if (wasEverRequested) {
+ String diagnosticKey = retrieveDecisionToDiagnosticKey(retrieveDecision);
+ LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously requested but we are not using the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data());
+ NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheUnusedReasonKey(), diagnosticKey, WebCore::ShouldSample::Yes);
+ } else {
+ NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheUnusedReasonKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes);
+ markAsRequested(hash);
+ }
+ });
+}
+
+static String storeDecisionToDiagnosticKey(StoreDecision storeDecision)
+{
+ switch (storeDecision) {
+ case StoreDecision::NoDueToProtocol:
+ return WebCore::DiagnosticLoggingKeys::notHTTPFamilyKey();
+ case StoreDecision::NoDueToHTTPMethod:
+ return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey();
+ case StoreDecision::NoDueToAttachmentResponse:
+ return WebCore::DiagnosticLoggingKeys::isAttachmentKey();
+ case StoreDecision::NoDueToNoStoreResponse:
+ case StoreDecision::NoDueToNoStoreRequest:
+ return WebCore::DiagnosticLoggingKeys::cacheControlNoStoreKey();
+ case StoreDecision::NoDueToHTTPStatusCode:
+ return WebCore::DiagnosticLoggingKeys::uncacheableStatusCodeKey();
+ case StoreDecision::NoDueToUnlikelyToReuse:
+ return WebCore::DiagnosticLoggingKeys::unlikelyToReuseKey();
+ case StoreDecision::NoDueToStreamingMedia:
+ return WebCore::DiagnosticLoggingKeys::streamingMedia();
+ case StoreDecision::Yes:
+ // It was stored but could not be retrieved so it must have been pruned from the cache.
+ return WebCore::DiagnosticLoggingKeys::noLongerInCacheKey();
+ }
+ return String();
+}
+
+void Statistics::recordRetrievalFailure(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request)
+{
+ auto hash = key.hashAsString();
+ queryWasEverRequested(hash, NeedUncachedReason::Yes, [this, hash, requestURL = request.url(), webPageID](bool wasPreviouslyRequested, const std::optional<StoreDecision>& storeDecision) {
+ if (wasPreviouslyRequested) {
+ String diagnosticKey = storeDecisionToDiagnosticKey(storeDecision.value());
+ LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously request but is not in the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data());
+ NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheFailureReasonKey(), diagnosticKey, WebCore::ShouldSample::Yes);
+ } else {
+ NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheFailureReasonKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes);
+ markAsRequested(hash);
+ }
+ });
+}
+
+static String cachedEntryReuseFailureToDiagnosticKey(UseDecision decision)
+{
+ switch (decision) {
+ case UseDecision::NoDueToVaryingHeaderMismatch:
+ return WebCore::DiagnosticLoggingKeys::varyingHeaderMismatchKey();
+ case UseDecision::NoDueToMissingValidatorFields:
+ return WebCore::DiagnosticLoggingKeys::missingValidatorFieldsKey();
+ case UseDecision::NoDueToDecodeFailure:
+ case UseDecision::NoDueToExpiredRedirect:
+ return WebCore::DiagnosticLoggingKeys::otherKey();
+ case UseDecision::Use:
+ case UseDecision::Validate:
+ ASSERT_NOT_REACHED();
+ break;
+ }
+ return emptyString();
+}
+
+void Statistics::recordRetrievedCachedEntry(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, UseDecision decision)
+{
+ WebCore::URL requestURL = request.url();
+ if (decision == UseDecision::Use) {
+ LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache and is used", webPageID, requestURL.string().ascii().data());
+ NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes);
+ return;
+ }
+
+ if (decision == UseDecision::Validate) {
+ LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but needs revalidation", webPageID, requestURL.string().ascii().data());
+ NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::needsRevalidationKey(), WebCore::ShouldSample::Yes);
+ return;
+ }
+
+ String diagnosticKey = cachedEntryReuseFailureToDiagnosticKey(decision);
+ LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but wasn't used, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data());
+ NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheReuseFailureKey(), diagnosticKey, WebCore::ShouldSample::Yes);
+}
+
+void Statistics::recordRevalidationSuccess(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request)
+{
+ WebCore::URL requestURL = request.url();
+ LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was successfully revalidated", webPageID, requestURL.string().ascii().data());
+
+ NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::revalidatingKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes);
+}
+
+void Statistics::markAsRequested(const String& hash)
+{
+ ASSERT(RunLoop::isMain());
+
+ m_hashesToAdd.add(hash);
+ if (!m_writeTimer.isActive())
+ m_writeTimer.startOneShot(mininumWriteInterval);
+}
+
+void Statistics::writeTimerFired()
+{
+ ASSERT(RunLoop::isMain());
+
+ serialBackgroundIOQueue().dispatch([this, hashesToAdd = WTFMove(m_hashesToAdd), storeDecisionsToAdd = WTFMove(m_storeDecisionsToAdd)] {
+ if (!m_database.isOpen())
+ return;
+
+ WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter;
+ WebCore::SQLiteTransaction writeTransaction(m_database);
+ writeTransaction.begin();
+
+ addHashesToDatabase(hashesToAdd);
+ addStoreDecisionsToDatabase(storeDecisionsToAdd);
+
+ writeTransaction.commit();
+ });
+
+ shrinkIfNeeded();
+}
+
+void Statistics::queryWasEverRequested(const String& hash, NeedUncachedReason needUncachedReason, RequestedCompletionHandler&& completionHandler)
+{
+ ASSERT(RunLoop::isMain());
+
+ // Query pending writes first.
+ bool wasAlreadyRequested = m_hashesToAdd.contains(hash);
+ if (wasAlreadyRequested && needUncachedReason == NeedUncachedReason::No) {
+ completionHandler(true, std::nullopt);
+ return;
+ }
+ if (needUncachedReason == NeedUncachedReason::Yes && m_storeDecisionsToAdd.contains(hash)) {
+ completionHandler(true, m_storeDecisionsToAdd.get(hash));
+ return;
+ }
+
+ // Query the database.
+ auto everRequestedQuery = std::make_unique<EverRequestedQuery>(EverRequestedQuery { hash, needUncachedReason == NeedUncachedReason::Yes, WTFMove(completionHandler) });
+ auto& query = *everRequestedQuery;
+ m_activeQueries.add(WTFMove(everRequestedQuery));
+ serialBackgroundIOQueue().dispatch([this, wasAlreadyRequested, &query] () mutable {
+ WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter;
+ std::optional<StoreDecision> storeDecision;
+ if (m_database.isOpen()) {
+ if (!wasAlreadyRequested) {
+ WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT hash FROM AlreadyRequested WHERE hash=?"));
+ if (statement.prepare() == SQLITE_OK) {
+ statement.bindText(1, query.hash);
+ wasAlreadyRequested = (statement.step() == SQLITE_ROW);
+ }
+ }
+ if (wasAlreadyRequested && query.needUncachedReason) {
+ WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT reason FROM UncachedReason WHERE hash=?"));
+ storeDecision = StoreDecision::Yes;
+ if (statement.prepare() == SQLITE_OK) {
+ statement.bindText(1, query.hash);
+ if (statement.step() == SQLITE_ROW)
+ storeDecision = static_cast<StoreDecision>(statement.getColumnInt(0));
+ }
+ }
+ }
+ RunLoop::main().dispatch([this, &query, wasAlreadyRequested, storeDecision] {
+ query.completionHandler(wasAlreadyRequested, storeDecision);
+ m_activeQueries.remove(&query);
+ });
+ });
+}
+
+void Statistics::clear()
+{
+ ASSERT(RunLoop::isMain());
+
+ serialBackgroundIOQueue().dispatch([this] {
+ if (m_database.isOpen()) {
+ WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter;
+ WebCore::SQLiteTransaction deleteTransaction(m_database);
+ deleteTransaction.begin();
+ executeSQLCommand(m_database, ASCIILiteral("DELETE FROM AlreadyRequested"));
+ executeSQLCommand(m_database, ASCIILiteral("DELETE FROM UncachedReason"));
+ deleteTransaction.commit();
+ m_approximateEntryCount = 0;
+ }
+ });
+}
+
+void Statistics::addHashesToDatabase(const HashSet<String>& hashes)
+{
+ ASSERT(!RunLoop::isMain());
+ ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress());
+ ASSERT(m_database.isOpen());
+
+ WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR IGNORE INTO AlreadyRequested (hash) VALUES (?)"));
+ if (statement.prepare() != SQLITE_OK)
+ return;
+
+ for (auto& hash : hashes) {
+ statement.bindText(1, hash);
+ if (executeSQLStatement(statement))
+ ++m_approximateEntryCount;
+ statement.reset();
+ }
+}
+
+void Statistics::addStoreDecisionsToDatabase(const HashMap<String, NetworkCache::StoreDecision>& storeDecisions)
+{
+ ASSERT(!RunLoop::isMain());
+ ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress());
+ ASSERT(m_database.isOpen());
+
+ WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR REPLACE INTO UncachedReason (hash, reason) VALUES (?, ?)"));
+ if (statement.prepare() != SQLITE_OK)
+ return;
+
+ for (auto& pair : storeDecisions) {
+ statement.bindText(1, pair.key);
+ statement.bindInt(2, static_cast<int>(pair.value));
+ executeSQLStatement(statement);
+ statement.reset();
+ }
+}
+
+}
+}
+
+#endif // ENABLE(NETWORK_CACHE)
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h
new file mode 100644
index 000000000..b67c260e1
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheStatistics_h
+#define NetworkCacheStatistics_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCache.h"
+#include "NetworkCacheKey.h"
+#include <WebCore/SQLiteDatabase.h>
+#include <WebCore/Timer.h>
+#include <wtf/WorkQueue.h>
+
+namespace WebCore {
+class ResourceRequest;
+}
+
+namespace WebKit {
+namespace NetworkCache {
+
+class Statistics {
+public:
+ static std::unique_ptr<Statistics> open(const String& cachePath);
+ explicit Statistics(const String& databasePath);
+
+ void clear();
+
+ void recordRetrievalRequest(uint64_t webPageID);
+ void recordNotCachingResponse(const Key&, StoreDecision);
+ void recordNotUsingCacheForRequest(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, RetrieveDecision);
+ void recordRetrievalFailure(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&);
+ void recordRetrievedCachedEntry(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, UseDecision);
+ void recordRevalidationSuccess(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&);
+
+private:
+ WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); }
+
+ void initialize(const String& databasePath);
+ void bootstrapFromNetworkCache(const String& networkCachePath);
+ void shrinkIfNeeded();
+
+ void addHashesToDatabase(const HashSet<String>& hashes);
+ void addStoreDecisionsToDatabase(const HashMap<String, NetworkCache::StoreDecision>&);
+ void writeTimerFired();
+
+ typedef std::function<void (bool wasEverRequested, const std::optional<StoreDecision>&)> RequestedCompletionHandler;
+ enum class NeedUncachedReason { No, Yes };
+ void queryWasEverRequested(const String&, NeedUncachedReason, RequestedCompletionHandler&&);
+ void markAsRequested(const String& hash);
+
+ struct EverRequestedQuery {
+ String hash;
+ bool needUncachedReason;
+ RequestedCompletionHandler completionHandler;
+ };
+
+ std::atomic<size_t> m_approximateEntryCount { 0 };
+
+ mutable Ref<WorkQueue> m_serialBackgroundIOQueue;
+ mutable HashSet<std::unique_ptr<const EverRequestedQuery>> m_activeQueries;
+ WebCore::SQLiteDatabase m_database;
+ HashSet<String> m_hashesToAdd;
+ HashMap<String, NetworkCache::StoreDecision> m_storeDecisionsToAdd;
+ WebCore::Timer m_writeTimer;
+};
+
+}
+}
+
+#endif // ENABLE(NETWORK_CACHE)
+
+#endif // NetworkCacheStatistics_h
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp
new file mode 100644
index 000000000..4f272449f
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp
@@ -0,0 +1,1048 @@
+/*
+ * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "NetworkCacheStorage.h"
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "Logging.h"
+#include "NetworkCacheCoders.h"
+#include "NetworkCacheFileSystem.h"
+#include "NetworkCacheIOChannel.h"
+#include <mutex>
+#include <wtf/Condition.h>
+#include <wtf/Lock.h>
+#include <wtf/RandomNumber.h>
+#include <wtf/RunLoop.h>
+#include <wtf/text/CString.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+static const char saltFileName[] = "salt";
+static const char versionDirectoryPrefix[] = "Version ";
+static const char recordsDirectoryName[] = "Records";
+static const char blobsDirectoryName[] = "Blobs";
+static const char blobSuffix[] = "-blob";
+
+static double computeRecordWorth(FileTimes);
+
+struct Storage::ReadOperation {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ ReadOperation(const Key& key, RetrieveCompletionHandler&& completionHandler)
+ : key(key)
+ , completionHandler(WTFMove(completionHandler))
+ { }
+
+ void cancel();
+ bool finish();
+
+ const Key key;
+ const RetrieveCompletionHandler completionHandler;
+
+ std::unique_ptr<Record> resultRecord;
+ SHA1::Digest expectedBodyHash;
+ BlobStorage::Blob resultBodyBlob;
+ std::atomic<unsigned> activeCount { 0 };
+ bool isCanceled { false };
+};
+
+void Storage::ReadOperation::cancel()
+{
+ ASSERT(RunLoop::isMain());
+
+ if (isCanceled)
+ return;
+ isCanceled = true;
+ completionHandler(nullptr);
+}
+
+bool Storage::ReadOperation::finish()
+{
+ ASSERT(RunLoop::isMain());
+
+ if (isCanceled)
+ return false;
+ if (resultRecord && resultRecord->body.isNull()) {
+ if (resultBodyBlob.hash == expectedBodyHash)
+ resultRecord->body = resultBodyBlob.data;
+ else
+ resultRecord = nullptr;
+ }
+ return completionHandler(WTFMove(resultRecord));
+}
+
+struct Storage::WriteOperation {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ WriteOperation(const Record& record, MappedBodyHandler&& mappedBodyHandler)
+ : record(record)
+ , mappedBodyHandler(WTFMove(mappedBodyHandler))
+ { }
+
+ const Record record;
+ const MappedBodyHandler mappedBodyHandler;
+
+ std::atomic<unsigned> activeCount { 0 };
+};
+
+struct Storage::TraverseOperation {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ TraverseOperation(const String& type, TraverseFlags flags, TraverseHandler&& handler)
+ : type(type)
+ , flags(flags)
+ , handler(WTFMove(handler))
+ { }
+
+ const String type;
+ const TraverseFlags flags;
+ const TraverseHandler handler;
+
+ Lock activeMutex;
+ Condition activeCondition;
+ unsigned activeCount { 0 };
+};
+
+static String makeVersionedDirectoryPath(const String& baseDirectoryPath)
+{
+ String versionSubdirectory = versionDirectoryPrefix + String::number(Storage::version);
+ return WebCore::pathByAppendingComponent(baseDirectoryPath, versionSubdirectory);
+}
+
+static String makeRecordsDirectoryPath(const String& baseDirectoryPath)
+{
+ return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), recordsDirectoryName);
+}
+
+static String makeBlobDirectoryPath(const String& baseDirectoryPath)
+{
+ return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), blobsDirectoryName);
+}
+
+static String makeSaltFilePath(const String& baseDirectoryPath)
+{
+ return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), saltFileName);
+}
+
+std::unique_ptr<Storage> Storage::open(const String& cachePath)
+{
+ ASSERT(RunLoop::isMain());
+
+ if (!WebCore::makeAllDirectories(makeVersionedDirectoryPath(cachePath)))
+ return nullptr;
+ auto salt = readOrMakeSalt(makeSaltFilePath(cachePath));
+ if (!salt)
+ return nullptr;
+ return std::unique_ptr<Storage>(new Storage(cachePath, *salt));
+}
+
+void traverseRecordsFiles(const String& recordsPath, const String& expectedType, const RecordFileTraverseFunction& function)
+{
+ traverseDirectory(recordsPath, [&](const String& partitionName, DirectoryEntryType entryType) {
+ if (entryType != DirectoryEntryType::Directory)
+ return;
+ String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName);
+ traverseDirectory(partitionPath, [&](const String& actualType, DirectoryEntryType entryType) {
+ if (entryType != DirectoryEntryType::Directory)
+ return;
+ if (!expectedType.isEmpty() && expectedType != actualType)
+ return;
+ String recordDirectoryPath = WebCore::pathByAppendingComponent(partitionPath, actualType);
+ traverseDirectory(recordDirectoryPath, [&function, &recordDirectoryPath, &actualType](const String& fileName, DirectoryEntryType entryType) {
+ if (entryType != DirectoryEntryType::File || fileName.length() < Key::hashStringLength())
+ return;
+
+ String hashString = fileName.substring(0, Key::hashStringLength());
+ auto isBlob = fileName.length() > Key::hashStringLength() && fileName.endsWith(blobSuffix);
+ function(fileName, hashString, actualType, isBlob, recordDirectoryPath);
+ });
+ });
+ });
+}
+
+static void deleteEmptyRecordsDirectories(const String& recordsPath)
+{
+ traverseDirectory(recordsPath, [&recordsPath](const String& partitionName, DirectoryEntryType type) {
+ if (type != DirectoryEntryType::Directory)
+ return;
+
+ // Delete [type] sub-folders.
+ String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName);
+ traverseDirectory(partitionPath, [&partitionPath](const String& subdirName, DirectoryEntryType entryType) {
+ if (entryType != DirectoryEntryType::Directory)
+ return;
+
+ // Let system figure out if it is really empty.
+ WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(partitionPath, subdirName));
+ });
+
+ // Delete [Partition] folders.
+ // Let system figure out if it is really empty.
+ WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(recordsPath, partitionName));
+ });
+}
+
+Storage::Storage(const String& baseDirectoryPath, Salt salt)
+ : m_basePath(baseDirectoryPath)
+ , m_recordsPath(makeRecordsDirectoryPath(baseDirectoryPath))
+ , m_salt(salt)
+ , m_canUseSharedMemoryForBodyData(canUseSharedMemoryForPath(baseDirectoryPath))
+ , m_readOperationTimeoutTimer(*this, &Storage::cancelAllReadOperations)
+ , m_writeOperationDispatchTimer(*this, &Storage::dispatchPendingWriteOperations)
+ , m_ioQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage", WorkQueue::Type::Concurrent))
+ , m_backgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.background", WorkQueue::Type::Concurrent, WorkQueue::QOS::Background))
+ , m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.serialBackground", WorkQueue::Type::Serial, WorkQueue::QOS::Background))
+ , m_blobStorage(makeBlobDirectoryPath(baseDirectoryPath), m_salt)
+{
+ deleteOldVersions();
+ synchronize();
+}
+
+Storage::~Storage()
+{
+}
+
+String Storage::basePath() const
+{
+ return m_basePath.isolatedCopy();
+}
+
+String Storage::versionPath() const
+{
+ return makeVersionedDirectoryPath(basePath());
+}
+
+String Storage::recordsPath() const
+{
+ return m_recordsPath.isolatedCopy();
+}
+
+size_t Storage::approximateSize() const
+{
+ return m_approximateRecordsSize + m_blobStorage.approximateSize();
+}
+
+void Storage::synchronize()
+{
+ ASSERT(RunLoop::isMain());
+
+ if (m_synchronizationInProgress || m_shrinkInProgress)
+ return;
+ m_synchronizationInProgress = true;
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) synchronizing cache");
+
+ backgroundIOQueue().dispatch([this] {
+ auto recordFilter = std::make_unique<ContentsFilter>();
+ auto blobFilter = std::make_unique<ContentsFilter>();
+ size_t recordsSize = 0;
+ unsigned count = 0;
+ String anyType;
+ traverseRecordsFiles(recordsPath(), anyType, [&recordFilter, &blobFilter, &recordsSize, &count](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) {
+ auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName);
+
+ Key::HashType hash;
+ if (!Key::stringToHash(hashString, hash)) {
+ WebCore::deleteFile(filePath);
+ return;
+ }
+ long long fileSize = 0;
+ WebCore::getFileSize(filePath, fileSize);
+ if (!fileSize) {
+ WebCore::deleteFile(filePath);
+ return;
+ }
+
+ if (isBlob) {
+ blobFilter->add(hash);
+ return;
+ }
+
+ recordFilter->add(hash);
+ recordsSize += fileSize;
+ ++count;
+ });
+
+ RunLoop::main().dispatch([this, recordFilter = WTFMove(recordFilter), blobFilter = WTFMove(blobFilter), recordsSize]() mutable {
+ for (auto& recordFilterKey : m_recordFilterHashesAddedDuringSynchronization)
+ recordFilter->add(recordFilterKey);
+ m_recordFilterHashesAddedDuringSynchronization.clear();
+
+ for (auto& hash : m_blobFilterHashesAddedDuringSynchronization)
+ blobFilter->add(hash);
+ m_blobFilterHashesAddedDuringSynchronization.clear();
+
+ m_recordFilter = WTFMove(recordFilter);
+ m_blobFilter = WTFMove(blobFilter);
+ m_approximateRecordsSize = recordsSize;
+ m_synchronizationInProgress = false;
+ });
+
+ m_blobStorage.synchronize();
+
+ deleteEmptyRecordsDirectories(recordsPath());
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) cache synchronization completed size=%zu count=%u", recordsSize, count);
+ });
+}
+
+void Storage::addToRecordFilter(const Key& key)
+{
+ ASSERT(RunLoop::isMain());
+
+ if (m_recordFilter)
+ m_recordFilter->add(key.hash());
+
+ // If we get new entries during filter synchronization take care to add them to the new filter as well.
+ if (m_synchronizationInProgress)
+ m_recordFilterHashesAddedDuringSynchronization.append(key.hash());
+}
+
+bool Storage::mayContain(const Key& key) const
+{
+ ASSERT(RunLoop::isMain());
+ return !m_recordFilter || m_recordFilter->mayContain(key.hash());
+}
+
+bool Storage::mayContainBlob(const Key& key) const
+{
+ ASSERT(RunLoop::isMain());
+ return !m_blobFilter || m_blobFilter->mayContain(key.hash());
+}
+
+String Storage::recordDirectoryPathForKey(const Key& key) const
+{
+ ASSERT(!key.type().isEmpty());
+ return WebCore::pathByAppendingComponent(WebCore::pathByAppendingComponent(recordsPath(), key.partitionHashAsString()), key.type());
+}
+
+String Storage::recordPathForKey(const Key& key) const
+{
+ return WebCore::pathByAppendingComponent(recordDirectoryPathForKey(key), key.hashAsString());
+}
+
+static String blobPathForRecordPath(const String& recordPath)
+{
+ return recordPath + blobSuffix;
+}
+
+String Storage::blobPathForKey(const Key& key) const
+{
+ return blobPathForRecordPath(recordPathForKey(key));
+}
+
+struct RecordMetaData {
+ RecordMetaData() { }
+ explicit RecordMetaData(const Key& key)
+ : cacheStorageVersion(Storage::version)
+ , key(key)
+ { }
+
+ unsigned cacheStorageVersion;
+ Key key;
+ std::chrono::system_clock::time_point timeStamp;
+ SHA1::Digest headerHash;
+ uint64_t headerSize;
+ SHA1::Digest bodyHash;
+ uint64_t bodySize;
+ bool isBodyInline;
+
+ // Not encoded as a field. Header starts immediately after meta data.
+ uint64_t headerOffset;
+};
+
+static bool decodeRecordMetaData(RecordMetaData& metaData, const Data& fileData)
+{
+ bool success = false;
+ fileData.apply([&metaData, &success](const uint8_t* data, size_t size) {
+ WTF::Persistence::Decoder decoder(data, size);
+ if (!decoder.decode(metaData.cacheStorageVersion))
+ return false;
+ if (!decoder.decode(metaData.key))
+ return false;
+ if (!decoder.decode(metaData.timeStamp))
+ return false;
+ if (!decoder.decode(metaData.headerHash))
+ return false;
+ if (!decoder.decode(metaData.headerSize))
+ return false;
+ if (!decoder.decode(metaData.bodyHash))
+ return false;
+ if (!decoder.decode(metaData.bodySize))
+ return false;
+ if (!decoder.decode(metaData.isBodyInline))
+ return false;
+ if (!decoder.verifyChecksum())
+ return false;
+ metaData.headerOffset = decoder.currentOffset();
+ success = true;
+ return false;
+ });
+ return success;
+}
+
+static bool decodeRecordHeader(const Data& fileData, RecordMetaData& metaData, Data& headerData, const Salt& salt)
+{
+ if (!decodeRecordMetaData(metaData, fileData)) {
+ LOG(NetworkCacheStorage, "(NetworkProcess) meta data decode failure");
+ return false;
+ }
+
+ if (metaData.cacheStorageVersion != Storage::version) {
+ LOG(NetworkCacheStorage, "(NetworkProcess) version mismatch");
+ return false;
+ }
+
+ headerData = fileData.subrange(metaData.headerOffset, metaData.headerSize);
+ if (metaData.headerHash != computeSHA1(headerData, salt)) {
+ LOG(NetworkCacheStorage, "(NetworkProcess) header checksum mismatch");
+ return false;
+ }
+ return true;
+}
+
+void Storage::readRecord(ReadOperation& readOperation, const Data& recordData)
+{
+ ASSERT(!RunLoop::isMain());
+
+ RecordMetaData metaData;
+ Data headerData;
+ if (!decodeRecordHeader(recordData, metaData, headerData, m_salt))
+ return;
+
+ if (metaData.key != readOperation.key)
+ return;
+
+ // Sanity check against time stamps in future.
+ if (metaData.timeStamp > std::chrono::system_clock::now())
+ return;
+
+ Data bodyData;
+ if (metaData.isBodyInline) {
+ size_t bodyOffset = metaData.headerOffset + headerData.size();
+ if (bodyOffset + metaData.bodySize != recordData.size())
+ return;
+ bodyData = recordData.subrange(bodyOffset, metaData.bodySize);
+ if (metaData.bodyHash != computeSHA1(bodyData, m_salt))
+ return;
+ }
+
+ readOperation.expectedBodyHash = metaData.bodyHash;
+ readOperation.resultRecord = std::make_unique<Storage::Record>(Storage::Record {
+ metaData.key,
+ metaData.timeStamp,
+ headerData,
+ bodyData,
+ metaData.bodyHash
+ });
+}
+
+static Data encodeRecordMetaData(const RecordMetaData& metaData)
+{
+ WTF::Persistence::Encoder encoder;
+
+ encoder << metaData.cacheStorageVersion;
+ encoder << metaData.key;
+ encoder << metaData.timeStamp;
+ encoder << metaData.headerHash;
+ encoder << metaData.headerSize;
+ encoder << metaData.bodyHash;
+ encoder << metaData.bodySize;
+ encoder << metaData.isBodyInline;
+
+ encoder.encodeChecksum();
+
+ return Data(encoder.buffer(), encoder.bufferSize());
+}
+
+std::optional<BlobStorage::Blob> Storage::storeBodyAsBlob(WriteOperation& writeOperation)
+{
+ auto blobPath = blobPathForKey(writeOperation.record.key);
+
+ // Store the body.
+ auto blob = m_blobStorage.add(blobPath, writeOperation.record.body);
+ if (blob.data.isNull())
+ return { };
+
+ ++writeOperation.activeCount;
+
+ RunLoop::main().dispatch([this, blob, &writeOperation] {
+ if (m_blobFilter)
+ m_blobFilter->add(writeOperation.record.key.hash());
+ if (m_synchronizationInProgress)
+ m_blobFilterHashesAddedDuringSynchronization.append(writeOperation.record.key.hash());
+
+ if (writeOperation.mappedBodyHandler)
+ writeOperation.mappedBodyHandler(blob.data);
+
+ finishWriteOperation(writeOperation);
+ });
+ return blob;
+}
+
+Data Storage::encodeRecord(const Record& record, std::optional<BlobStorage::Blob> blob)
+{
+ ASSERT(!blob || bytesEqual(blob.value().data, record.body));
+
+ RecordMetaData metaData(record.key);
+ metaData.timeStamp = record.timeStamp;
+ metaData.headerHash = computeSHA1(record.header, m_salt);
+ metaData.headerSize = record.header.size();
+ metaData.bodyHash = blob ? blob.value().hash : computeSHA1(record.body, m_salt);
+ metaData.bodySize = record.body.size();
+ metaData.isBodyInline = !blob;
+
+ auto encodedMetaData = encodeRecordMetaData(metaData);
+ auto headerData = concatenate(encodedMetaData, record.header);
+
+ if (metaData.isBodyInline)
+ return concatenate(headerData, record.body);
+
+ return { headerData };
+}
+
+void Storage::removeFromPendingWriteOperations(const Key& key)
+{
+ while (true) {
+ auto found = m_pendingWriteOperations.findIf([&key](auto& operation) {
+ return operation->record.key == key;
+ });
+
+ if (found == m_pendingWriteOperations.end())
+ break;
+
+ m_pendingWriteOperations.remove(found);
+ }
+}
+
+void Storage::remove(const Key& key)
+{
+ ASSERT(RunLoop::isMain());
+
+ if (!mayContain(key))
+ return;
+
+ // We can't remove the key from the Bloom filter (but some false positives are expected anyway).
+ // For simplicity we also don't reduce m_approximateSize on removals.
+ // The next synchronization will update everything.
+
+ removeFromPendingWriteOperations(key);
+
+ serialBackgroundIOQueue().dispatch([this, key] {
+ WebCore::deleteFile(recordPathForKey(key));
+ m_blobStorage.remove(blobPathForKey(key));
+ });
+}
+
+void Storage::updateFileModificationTime(const String& path)
+{
+ serialBackgroundIOQueue().dispatch([path = path.isolatedCopy()] {
+ updateFileModificationTimeIfNeeded(path);
+ });
+}
+
+void Storage::dispatchReadOperation(std::unique_ptr<ReadOperation> readOperationPtr)
+{
+ ASSERT(RunLoop::isMain());
+
+ auto& readOperation = *readOperationPtr;
+ m_activeReadOperations.add(WTFMove(readOperationPtr));
+
+ // I/O pressure may make disk operations slow. If they start taking very long time we rather go to network.
+ const auto readTimeout = 1500ms;
+ m_readOperationTimeoutTimer.startOneShot(readTimeout);
+
+ bool shouldGetBodyBlob = mayContainBlob(readOperation.key);
+
+ ioQueue().dispatch([this, &readOperation, shouldGetBodyBlob] {
+ auto recordPath = recordPathForKey(readOperation.key);
+
+ ++readOperation.activeCount;
+ if (shouldGetBodyBlob)
+ ++readOperation.activeCount;
+
+ auto channel = IOChannel::open(recordPath, IOChannel::Type::Read);
+ channel->read(0, std::numeric_limits<size_t>::max(), &ioQueue(), [this, &readOperation](const Data& fileData, int error) {
+ if (!error)
+ readRecord(readOperation, fileData);
+ finishReadOperation(readOperation);
+ });
+
+ if (shouldGetBodyBlob) {
+ // Read the blob in parallel with the record read.
+ auto blobPath = blobPathForKey(readOperation.key);
+ readOperation.resultBodyBlob = m_blobStorage.get(blobPath);
+ finishReadOperation(readOperation);
+ }
+ });
+}
+
+void Storage::finishReadOperation(ReadOperation& readOperation)
+{
+ ASSERT(readOperation.activeCount);
+ // Record and blob reads must finish.
+ if (--readOperation.activeCount)
+ return;
+
+ RunLoop::main().dispatch([this, &readOperation] {
+ bool success = readOperation.finish();
+ if (success)
+ updateFileModificationTime(recordPathForKey(readOperation.key));
+ else if (!readOperation.isCanceled)
+ remove(readOperation.key);
+
+ ASSERT(m_activeReadOperations.contains(&readOperation));
+ m_activeReadOperations.remove(&readOperation);
+
+ if (m_activeReadOperations.isEmpty())
+ m_readOperationTimeoutTimer.stop();
+
+ dispatchPendingReadOperations();
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) read complete success=%d", success);
+ });
+}
+
+void Storage::cancelAllReadOperations()
+{
+ ASSERT(RunLoop::isMain());
+
+ for (auto& readOperation : m_activeReadOperations)
+ readOperation->cancel();
+
+ size_t pendingCount = 0;
+ for (int priority = maximumRetrievePriority; priority >= 0; --priority) {
+ auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority];
+ pendingCount += pendingRetrieveQueue.size();
+ for (auto it = pendingRetrieveQueue.rbegin(), end = pendingRetrieveQueue.rend(); it != end; ++it)
+ (*it)->cancel();
+ pendingRetrieveQueue.clear();
+ }
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) retrieve timeout, canceled %u active and %zu pending", m_activeReadOperations.size(), pendingCount);
+}
+
+void Storage::dispatchPendingReadOperations()
+{
+ ASSERT(RunLoop::isMain());
+
+ const int maximumActiveReadOperationCount = 5;
+
+ for (int priority = maximumRetrievePriority; priority >= 0; --priority) {
+ if (m_activeReadOperations.size() > maximumActiveReadOperationCount) {
+ LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel retrieves");
+ return;
+ }
+ auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority];
+ if (pendingRetrieveQueue.isEmpty())
+ continue;
+ dispatchReadOperation(pendingRetrieveQueue.takeLast());
+ }
+}
+
+template <class T> bool retrieveFromMemory(const T& operations, const Key& key, Storage::RetrieveCompletionHandler& completionHandler)
+{
+ for (auto& operation : operations) {
+ if (operation->record.key == key) {
+ LOG(NetworkCacheStorage, "(NetworkProcess) found write operation in progress");
+ RunLoop::main().dispatch([record = operation->record, completionHandler = WTFMove(completionHandler)] {
+ completionHandler(std::make_unique<Storage::Record>(record));
+ });
+ return true;
+ }
+ }
+ return false;
+}
+
+void Storage::dispatchPendingWriteOperations()
+{
+ ASSERT(RunLoop::isMain());
+
+ const int maximumActiveWriteOperationCount { 1 };
+
+ while (!m_pendingWriteOperations.isEmpty()) {
+ if (m_activeWriteOperations.size() >= maximumActiveWriteOperationCount) {
+ LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel writes");
+ return;
+ }
+ dispatchWriteOperation(m_pendingWriteOperations.takeLast());
+ }
+}
+
+static bool shouldStoreBodyAsBlob(const Data& bodyData)
+{
+ const size_t maximumInlineBodySize { 16 * 1024 };
+ return bodyData.size() > maximumInlineBodySize;
+}
+
+void Storage::dispatchWriteOperation(std::unique_ptr<WriteOperation> writeOperationPtr)
+{
+ ASSERT(RunLoop::isMain());
+
+ auto& writeOperation = *writeOperationPtr;
+ m_activeWriteOperations.add(WTFMove(writeOperationPtr));
+
+ // This was added already when starting the store but filter might have been wiped.
+ addToRecordFilter(writeOperation.record.key);
+
+ backgroundIOQueue().dispatch([this, &writeOperation] {
+ auto recordDirectorPath = recordDirectoryPathForKey(writeOperation.record.key);
+ auto recordPath = recordPathForKey(writeOperation.record.key);
+
+ WebCore::makeAllDirectories(recordDirectorPath);
+
+ ++writeOperation.activeCount;
+
+ bool shouldStoreAsBlob = shouldStoreBodyAsBlob(writeOperation.record.body);
+ auto blob = shouldStoreAsBlob ? storeBodyAsBlob(writeOperation) : std::nullopt;
+
+ auto recordData = encodeRecord(writeOperation.record, blob);
+
+ auto channel = IOChannel::open(recordPath, IOChannel::Type::Create);
+ size_t recordSize = recordData.size();
+ channel->write(0, recordData, nullptr, [this, &writeOperation, recordSize](int error) {
+ // On error the entry still stays in the contents filter until next synchronization.
+ m_approximateRecordsSize += recordSize;
+ finishWriteOperation(writeOperation);
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) write complete error=%d", error);
+ });
+ });
+}
+
+void Storage::finishWriteOperation(WriteOperation& writeOperation)
+{
+ ASSERT(RunLoop::isMain());
+ ASSERT(writeOperation.activeCount);
+ ASSERT(m_activeWriteOperations.contains(&writeOperation));
+
+ if (--writeOperation.activeCount)
+ return;
+
+ m_activeWriteOperations.remove(&writeOperation);
+ dispatchPendingWriteOperations();
+
+ shrinkIfNeeded();
+}
+
+void Storage::retrieve(const Key& key, unsigned priority, RetrieveCompletionHandler&& completionHandler)
+{
+ ASSERT(RunLoop::isMain());
+ ASSERT(priority <= maximumRetrievePriority);
+ ASSERT(!key.isNull());
+
+ if (!m_capacity) {
+ completionHandler(nullptr);
+ return;
+ }
+
+ if (!mayContain(key)) {
+ completionHandler(nullptr);
+ return;
+ }
+
+ if (retrieveFromMemory(m_pendingWriteOperations, key, completionHandler))
+ return;
+ if (retrieveFromMemory(m_activeWriteOperations, key, completionHandler))
+ return;
+
+ auto readOperation = std::make_unique<ReadOperation>(key, WTFMove(completionHandler));
+ m_pendingReadOperationsByPriority[priority].prepend(WTFMove(readOperation));
+ dispatchPendingReadOperations();
+}
+
+void Storage::store(const Record& record, MappedBodyHandler&& mappedBodyHandler)
+{
+ ASSERT(RunLoop::isMain());
+ ASSERT(!record.key.isNull());
+
+ if (!m_capacity)
+ return;
+
+ auto writeOperation = std::make_unique<WriteOperation>(record, WTFMove(mappedBodyHandler));
+ m_pendingWriteOperations.prepend(WTFMove(writeOperation));
+
+ // Add key to the filter already here as we do lookups from the pending operations too.
+ addToRecordFilter(record.key);
+
+ bool isInitialWrite = m_pendingWriteOperations.size() == 1;
+ if (!isInitialWrite)
+ return;
+
+ // Delay the start of writes a bit to avoid affecting early page load.
+ // Completing writes will dispatch more writes without delay.
+ static const auto initialWriteDelay = 1s;
+ m_writeOperationDispatchTimer.startOneShot(initialWriteDelay);
+}
+
+void Storage::traverse(const String& type, TraverseFlags flags, TraverseHandler&& traverseHandler)
+{
+ ASSERT(RunLoop::isMain());
+ ASSERT(traverseHandler);
+ // Avoid non-thread safe Function copies.
+
+ auto traverseOperationPtr = std::make_unique<TraverseOperation>(type, flags, WTFMove(traverseHandler));
+ auto& traverseOperation = *traverseOperationPtr;
+ m_activeTraverseOperations.add(WTFMove(traverseOperationPtr));
+
+ ioQueue().dispatch([this, &traverseOperation] {
+ traverseRecordsFiles(recordsPath(), traverseOperation.type, [this, &traverseOperation](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) {
+ ASSERT(type == traverseOperation.type);
+ if (isBlob)
+ return;
+
+ auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName);
+
+ double worth = -1;
+ if (traverseOperation.flags & TraverseFlag::ComputeWorth)
+ worth = computeRecordWorth(fileTimes(recordPath));
+ unsigned bodyShareCount = 0;
+ if (traverseOperation.flags & TraverseFlag::ShareCount)
+ bodyShareCount = m_blobStorage.shareCount(blobPathForRecordPath(recordPath));
+
+ std::unique_lock<Lock> lock(traverseOperation.activeMutex);
+ ++traverseOperation.activeCount;
+
+ auto channel = IOChannel::open(recordPath, IOChannel::Type::Read);
+ channel->read(0, std::numeric_limits<size_t>::max(), nullptr, [this, &traverseOperation, worth, bodyShareCount](Data& fileData, int) {
+ RecordMetaData metaData;
+ Data headerData;
+ if (decodeRecordHeader(fileData, metaData, headerData, m_salt)) {
+ Record record {
+ metaData.key,
+ metaData.timeStamp,
+ headerData,
+ { },
+ metaData.bodyHash
+ };
+ RecordInfo info {
+ static_cast<size_t>(metaData.bodySize),
+ worth,
+ bodyShareCount,
+ String::fromUTF8(SHA1::hexDigest(metaData.bodyHash))
+ };
+ traverseOperation.handler(&record, info);
+ }
+
+ std::lock_guard<Lock> lock(traverseOperation.activeMutex);
+ --traverseOperation.activeCount;
+ traverseOperation.activeCondition.notifyOne();
+ });
+
+ const unsigned maximumParallelReadCount = 5;
+ traverseOperation.activeCondition.wait(lock, [&traverseOperation] {
+ return traverseOperation.activeCount <= maximumParallelReadCount;
+ });
+ });
+ {
+ // Wait for all reads to finish.
+ std::unique_lock<Lock> lock(traverseOperation.activeMutex);
+ traverseOperation.activeCondition.wait(lock, [&traverseOperation] {
+ return !traverseOperation.activeCount;
+ });
+ }
+ RunLoop::main().dispatch([this, &traverseOperation] {
+ traverseOperation.handler(nullptr, { });
+ m_activeTraverseOperations.remove(&traverseOperation);
+ });
+ });
+}
+
+void Storage::setCapacity(size_t capacity)
+{
+ ASSERT(RunLoop::isMain());
+
+#if !ASSERT_DISABLED
+ const size_t assumedAverageRecordSize = 50 << 10;
+ size_t maximumRecordCount = capacity / assumedAverageRecordSize;
+ // ~10 bits per element are required for <1% false positive rate.
+ size_t effectiveBloomFilterCapacity = ContentsFilter::tableSize / 10;
+ // If this gets hit it might be time to increase the filter size.
+ ASSERT(maximumRecordCount < effectiveBloomFilterCapacity);
+#endif
+
+ m_capacity = capacity;
+
+ shrinkIfNeeded();
+}
+
+void Storage::clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, Function<void ()>&& completionHandler)
+{
+ ASSERT(RunLoop::isMain());
+ LOG(NetworkCacheStorage, "(NetworkProcess) clearing cache");
+
+ if (m_recordFilter)
+ m_recordFilter->clear();
+ if (m_blobFilter)
+ m_blobFilter->clear();
+ m_approximateRecordsSize = 0;
+
+ ioQueue().dispatch([this, modifiedSinceTime, completionHandler = WTFMove(completionHandler), type = type.isolatedCopy()] () mutable {
+ auto recordsPath = this->recordsPath();
+ traverseRecordsFiles(recordsPath, type, [modifiedSinceTime](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) {
+ auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName);
+ if (modifiedSinceTime > std::chrono::system_clock::time_point::min()) {
+ auto times = fileTimes(filePath);
+ if (times.modification < modifiedSinceTime)
+ return;
+ }
+ WebCore::deleteFile(filePath);
+ });
+
+ deleteEmptyRecordsDirectories(recordsPath);
+
+ // This cleans unreferenced blobs.
+ m_blobStorage.synchronize();
+
+ if (completionHandler) {
+ RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler)] {
+ completionHandler();
+ });
+ }
+ });
+}
+
+static double computeRecordWorth(FileTimes times)
+{
+ using namespace std::chrono;
+ auto age = system_clock::now() - times.creation;
+ // File modification time is updated manually on cache read. We don't use access time since OS may update it automatically.
+ auto accessAge = times.modification - times.creation;
+
+ // For sanity.
+ if (age <= 0s || accessAge < 0s || accessAge > age)
+ return 0;
+
+ // We like old entries that have been accessed recently.
+ return duration<double>(accessAge) / age;
+}
+
+static double deletionProbability(FileTimes times, unsigned bodyShareCount)
+{
+ static const double maximumProbability { 0.33 };
+ static const unsigned maximumEffectiveShareCount { 5 };
+
+ auto worth = computeRecordWorth(times);
+
+ // Adjust a bit so the most valuable entries don't get deleted at all.
+ auto effectiveWorth = std::min(1.1 * worth, 1.);
+
+ auto probability = (1 - effectiveWorth) * maximumProbability;
+
+ // It is less useful to remove an entry that shares its body data.
+ if (bodyShareCount)
+ probability /= std::min(bodyShareCount, maximumEffectiveShareCount);
+
+ return probability;
+}
+
+void Storage::shrinkIfNeeded()
+{
+ ASSERT(RunLoop::isMain());
+
+ if (approximateSize() > m_capacity)
+ shrink();
+}
+
+void Storage::shrink()
+{
+ ASSERT(RunLoop::isMain());
+
+ if (m_shrinkInProgress || m_synchronizationInProgress)
+ return;
+ m_shrinkInProgress = true;
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) shrinking cache approximateSize=%zu capacity=%zu", approximateSize(), m_capacity);
+
+ backgroundIOQueue().dispatch([this] {
+ auto recordsPath = this->recordsPath();
+ String anyType;
+ traverseRecordsFiles(recordsPath, anyType, [this](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) {
+ if (isBlob)
+ return;
+
+ auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName);
+ auto blobPath = blobPathForRecordPath(recordPath);
+
+ auto times = fileTimes(recordPath);
+ unsigned bodyShareCount = m_blobStorage.shareCount(blobPath);
+ auto probability = deletionProbability(times, bodyShareCount);
+
+ bool shouldDelete = randomNumber() < probability;
+
+ LOG(NetworkCacheStorage, "Deletion probability=%f bodyLinkCount=%d shouldDelete=%d", probability, bodyShareCount, shouldDelete);
+
+ if (shouldDelete) {
+ WebCore::deleteFile(recordPath);
+ m_blobStorage.remove(blobPath);
+ }
+ });
+
+ RunLoop::main().dispatch([this] {
+ m_shrinkInProgress = false;
+ // We could synchronize during the shrink traversal. However this is fast and it is better to have just one code path.
+ synchronize();
+ });
+
+ LOG(NetworkCacheStorage, "(NetworkProcess) cache shrink completed");
+ });
+}
+
+void Storage::deleteOldVersions()
+{
+ backgroundIOQueue().dispatch([this] {
+ auto cachePath = basePath();
+ traverseDirectory(cachePath, [&cachePath](const String& subdirName, DirectoryEntryType type) {
+ if (type != DirectoryEntryType::Directory)
+ return;
+ if (!subdirName.startsWith(versionDirectoryPrefix))
+ return;
+ auto versionString = subdirName.substring(strlen(versionDirectoryPrefix));
+ bool success;
+ unsigned directoryVersion = versionString.toUIntStrict(&success);
+ if (!success)
+ return;
+ if (directoryVersion >= version)
+ return;
+#if PLATFORM(MAC)
+ if (directoryVersion == lastStableVersion)
+ return;
+#endif
+
+ auto oldVersionPath = WebCore::pathByAppendingComponent(cachePath, subdirName);
+ LOG(NetworkCacheStorage, "(NetworkProcess) deleting old cache version, path %s", oldVersionPath.utf8().data());
+
+ deleteDirectoryRecursively(oldVersionPath);
+ });
+ });
+}
+
+}
+}
+
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h
new file mode 100644
index 000000000..30b5f1368
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h
@@ -0,0 +1,193 @@
+/*
+ * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheStorage_h
+#define NetworkCacheStorage_h
+
+#if ENABLE(NETWORK_CACHE)
+
+#include "NetworkCacheBlobStorage.h"
+#include "NetworkCacheData.h"
+#include "NetworkCacheKey.h"
+#include <WebCore/Timer.h>
+#include <wtf/BloomFilter.h>
+#include <wtf/Deque.h>
+#include <wtf/Function.h>
+#include <wtf/HashSet.h>
+#include <wtf/Optional.h>
+#include <wtf/WorkQueue.h>
+#include <wtf/text/WTFString.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+class IOChannel;
+
+class Storage {
+ WTF_MAKE_NONCOPYABLE(Storage);
+public:
+ static std::unique_ptr<Storage> open(const String& cachePath);
+
+ struct Record {
+ WTF_MAKE_FAST_ALLOCATED;
+ public:
+ Key key;
+ std::chrono::system_clock::time_point timeStamp;
+ Data header;
+ Data body;
+ std::optional<SHA1::Digest> bodyHash;
+ };
+ // This may call completion handler synchronously on failure.
+ typedef Function<bool (std::unique_ptr<Record>)> RetrieveCompletionHandler;
+ void retrieve(const Key&, unsigned priority, RetrieveCompletionHandler&&);
+
+ typedef Function<void (const Data& mappedBody)> MappedBodyHandler;
+ void store(const Record&, MappedBodyHandler&&);
+
+ void remove(const Key&);
+ void clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, Function<void ()>&& completionHandler);
+
+ struct RecordInfo {
+ size_t bodySize;
+ double worth; // 0-1 where 1 is the most valuable.
+ unsigned bodyShareCount;
+ String bodyHash;
+ };
+ enum TraverseFlag {
+ ComputeWorth = 1 << 0,
+ ShareCount = 1 << 1,
+ };
+ typedef unsigned TraverseFlags;
+ typedef Function<void (const Record*, const RecordInfo&)> TraverseHandler;
+ // Null record signals end.
+ void traverse(const String& type, TraverseFlags, TraverseHandler&&);
+
+ void setCapacity(size_t);
+ size_t capacity() const { return m_capacity; }
+ size_t approximateSize() const;
+
+ static const unsigned version = 11;
+#if PLATFORM(MAC)
+ /// Allow the last stable version of the cache to co-exist with the latest development one.
+ static const unsigned lastStableVersion = 9;
+#endif
+
+ String basePath() const;
+ String versionPath() const;
+ String recordsPath() const;
+
+ const Salt& salt() const { return m_salt; }
+
+ bool canUseSharedMemoryForBodyData() const { return m_canUseSharedMemoryForBodyData; }
+
+ ~Storage();
+
+private:
+ Storage(const String& directoryPath, Salt);
+
+ String recordDirectoryPathForKey(const Key&) const;
+ String recordPathForKey(const Key&) const;
+ String blobPathForKey(const Key&) const;
+
+ void synchronize();
+ void deleteOldVersions();
+ void shrinkIfNeeded();
+ void shrink();
+
+ struct ReadOperation;
+ void dispatchReadOperation(std::unique_ptr<ReadOperation>);
+ void dispatchPendingReadOperations();
+ void finishReadOperation(ReadOperation&);
+ void cancelAllReadOperations();
+
+ struct WriteOperation;
+ void dispatchWriteOperation(std::unique_ptr<WriteOperation>);
+ void dispatchPendingWriteOperations();
+ void finishWriteOperation(WriteOperation&);
+
+ std::optional<BlobStorage::Blob> storeBodyAsBlob(WriteOperation&);
+ Data encodeRecord(const Record&, std::optional<BlobStorage::Blob>);
+ void readRecord(ReadOperation&, const Data&);
+
+ void updateFileModificationTime(const String& path);
+ void removeFromPendingWriteOperations(const Key&);
+
+ WorkQueue& ioQueue() { return m_ioQueue.get(); }
+ WorkQueue& backgroundIOQueue() { return m_backgroundIOQueue.get(); }
+ WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); }
+
+ bool mayContain(const Key&) const;
+ bool mayContainBlob(const Key&) const;
+
+ void addToRecordFilter(const Key&);
+
+ const String m_basePath;
+ const String m_recordsPath;
+
+ const Salt m_salt;
+
+ const bool m_canUseSharedMemoryForBodyData;
+
+ size_t m_capacity { std::numeric_limits<size_t>::max() };
+ size_t m_approximateRecordsSize { 0 };
+
+ // 2^18 bit filter can support up to 26000 entries with false positive rate < 1%.
+ using ContentsFilter = BloomFilter<18>;
+ std::unique_ptr<ContentsFilter> m_recordFilter;
+ std::unique_ptr<ContentsFilter> m_blobFilter;
+
+ bool m_synchronizationInProgress { false };
+ bool m_shrinkInProgress { false };
+
+ Vector<Key::HashType> m_recordFilterHashesAddedDuringSynchronization;
+ Vector<Key::HashType> m_blobFilterHashesAddedDuringSynchronization;
+
+ static const int maximumRetrievePriority = 4;
+ Deque<std::unique_ptr<ReadOperation>> m_pendingReadOperationsByPriority[maximumRetrievePriority + 1];
+ HashSet<std::unique_ptr<ReadOperation>> m_activeReadOperations;
+ WebCore::Timer m_readOperationTimeoutTimer;
+
+ Deque<std::unique_ptr<WriteOperation>> m_pendingWriteOperations;
+ HashSet<std::unique_ptr<WriteOperation>> m_activeWriteOperations;
+ WebCore::Timer m_writeOperationDispatchTimer;
+
+ struct TraverseOperation;
+ HashSet<std::unique_ptr<TraverseOperation>> m_activeTraverseOperations;
+
+ Ref<WorkQueue> m_ioQueue;
+ Ref<WorkQueue> m_backgroundIOQueue;
+ Ref<WorkQueue> m_serialBackgroundIOQueue;
+
+ BlobStorage m_blobStorage;
+};
+
+// FIXME: Remove, used by NetworkCacheStatistics only.
+using RecordFileTraverseFunction = Function<void (const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath)>;
+void traverseRecordsFiles(const String& recordsPath, const String& type, const RecordFileTraverseFunction&);
+
+}
+}
+#endif
+#endif
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp
new file mode 100644
index 000000000..561e4e6b8
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp
@@ -0,0 +1,175 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+#include "NetworkCacheSubresourcesEntry.h"
+
+#include "Logging.h"
+#include "NetworkCacheCoders.h"
+
+using namespace std::chrono;
+
+namespace WebKit {
+namespace NetworkCache {
+
+void SubresourceInfo::encode(WTF::Persistence::Encoder& encoder) const
+{
+ encoder << m_key;
+ encoder << m_lastSeen;
+ encoder << m_firstSeen;
+ encoder << m_isTransient;
+
+ // Do not bother serializing other data members of transient resources as they are empty.
+ if (m_isTransient)
+ return;
+
+ encoder << m_firstPartyForCookies;
+ encoder << m_requestHeaders;
+ encoder.encodeEnum(m_priority);
+}
+
+bool SubresourceInfo::decode(WTF::Persistence::Decoder& decoder, SubresourceInfo& info)
+{
+ if (!decoder.decode(info.m_key))
+ return false;
+ if (!decoder.decode(info.m_lastSeen))
+ return false;
+ if (!decoder.decode(info.m_firstSeen))
+ return false;
+
+ if (!decoder.decode(info.m_isTransient))
+ return false;
+
+ if (info.m_isTransient)
+ return true;
+
+ if (!decoder.decode(info.m_firstPartyForCookies))
+ return false;
+
+ if (!decoder.decode(info.m_requestHeaders))
+ return false;
+
+ if (!decoder.decodeEnum(info.m_priority))
+ return false;
+
+ return true;
+}
+
+Storage::Record SubresourcesEntry::encodeAsStorageRecord() const
+{
+ WTF::Persistence::Encoder encoder;
+ encoder << m_subresources;
+
+ encoder.encodeChecksum();
+
+ return { m_key, m_timeStamp, { encoder.buffer(), encoder.bufferSize() }, { }, { }};
+}
+
+std::unique_ptr<SubresourcesEntry> SubresourcesEntry::decodeStorageRecord(const Storage::Record& storageEntry)
+{
+ auto entry = std::make_unique<SubresourcesEntry>(storageEntry);
+
+ WTF::Persistence::Decoder decoder(storageEntry.header.data(), storageEntry.header.size());
+ if (!decoder.decode(entry->m_subresources))
+ return nullptr;
+
+ if (!decoder.verifyChecksum()) {
+ LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n");
+ return nullptr;
+ }
+
+ return entry;
+}
+
+SubresourcesEntry::SubresourcesEntry(const Storage::Record& storageEntry)
+ : m_key(storageEntry.key)
+ , m_timeStamp(storageEntry.timeStamp)
+{
+ ASSERT(m_key.type() == "SubResources");
+}
+
+SubresourceInfo::SubresourceInfo(const Key& key, const WebCore::ResourceRequest& request, const SubresourceInfo* previousInfo)
+ : m_key(key)
+ , m_lastSeen(std::chrono::system_clock::now())
+ , m_firstSeen(previousInfo ? previousInfo->firstSeen() : m_lastSeen)
+ , m_isTransient(!previousInfo)
+ , m_firstPartyForCookies(request.firstPartyForCookies())
+ , m_requestHeaders(request.httpHeaderFields())
+ , m_priority(request.priority())
+{
+}
+
+static Vector<SubresourceInfo> makeSubresourceInfoVector(const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads, Vector<SubresourceInfo>* previousSubresources)
+{
+ Vector<SubresourceInfo> result;
+ result.reserveInitialCapacity(subresourceLoads.size());
+
+ HashMap<Key, unsigned> previousMap;
+ if (previousSubresources) {
+ for (unsigned i = 0; i < previousSubresources->size(); ++i)
+ previousMap.add(previousSubresources->at(i).key(), i);
+ }
+
+ HashSet<Key> deduplicationSet;
+ for (auto& load : subresourceLoads) {
+ if (!deduplicationSet.add(load->key).isNewEntry)
+ continue;
+
+ SubresourceInfo* previousInfo = nullptr;
+ if (previousSubresources) {
+ auto it = previousMap.find(load->key);
+ if (it != previousMap.end())
+ previousInfo = &(*previousSubresources)[it->value];
+ }
+
+ result.uncheckedAppend({ load->key, load->request, previousInfo });
+
+ // FIXME: We should really consider all resources seen for the first time transient.
+ if (!previousSubresources)
+ result.last().setNonTransient();
+ }
+
+ return result;
+}
+
+SubresourcesEntry::SubresourcesEntry(Key&& key, const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads)
+ : m_key(WTFMove(key))
+ , m_timeStamp(std::chrono::system_clock::now())
+ , m_subresources(makeSubresourceInfoVector(subresourceLoads, nullptr))
+{
+ ASSERT(m_key.type() == "SubResources");
+}
+
+void SubresourcesEntry::updateSubresourceLoads(const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads)
+{
+ m_subresources = makeSubresourceInfoVector(subresourceLoads, &m_subresources);
+}
+
+} // namespace WebKit
+} // namespace NetworkCache
+
+#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h
new file mode 100644
index 000000000..c2d5da8b1
--- /dev/null
+++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2015 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef NetworkCacheSubresourcesEntry_h
+#define NetworkCacheSubresourcesEntry_h
+
+#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+
+#include "NetworkCacheStorage.h"
+#include <WebCore/ResourceRequest.h>
+#include <WebCore/URL.h>
+#include <wtf/HashMap.h>
+
+namespace WebKit {
+namespace NetworkCache {
+
+class SubresourceInfo {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ void encode(WTF::Persistence::Encoder&) const;
+ static bool decode(WTF::Persistence::Decoder&, SubresourceInfo&);
+
+ SubresourceInfo() = default;
+ SubresourceInfo(const Key&, const WebCore::ResourceRequest&, const SubresourceInfo* previousInfo);
+
+ const Key& key() const { return m_key; }
+ std::chrono::system_clock::time_point lastSeen() const { return m_lastSeen; }
+ std::chrono::system_clock::time_point firstSeen() const { return m_firstSeen; }
+
+ bool isTransient() const { return m_isTransient; }
+ const WebCore::URL& firstPartyForCookies() const { ASSERT(!m_isTransient); return m_firstPartyForCookies; }
+ const WebCore::HTTPHeaderMap& requestHeaders() const { ASSERT(!m_isTransient); return m_requestHeaders; }
+ WebCore::ResourceLoadPriority priority() const { ASSERT(!m_isTransient); return m_priority; }
+
+ void setNonTransient() { m_isTransient = false; }
+
+private:
+ Key m_key;
+ std::chrono::system_clock::time_point m_lastSeen;
+ std::chrono::system_clock::time_point m_firstSeen;
+ bool m_isTransient { false };
+ WebCore::URL m_firstPartyForCookies;
+ WebCore::HTTPHeaderMap m_requestHeaders;
+ WebCore::ResourceLoadPriority m_priority;
+};
+
+struct SubresourceLoad {
+ WTF_MAKE_NONCOPYABLE(SubresourceLoad); WTF_MAKE_FAST_ALLOCATED;
+public:
+ SubresourceLoad(const WebCore::ResourceRequest& request, const Key& key)
+ : request(request)
+ , key(key)
+ { }
+
+ WebCore::ResourceRequest request;
+ Key key;
+};
+
+class SubresourcesEntry {
+ WTF_MAKE_NONCOPYABLE(SubresourcesEntry); WTF_MAKE_FAST_ALLOCATED;
+public:
+ SubresourcesEntry(Key&&, const Vector<std::unique_ptr<SubresourceLoad>>&);
+ explicit SubresourcesEntry(const Storage::Record&);
+
+ Storage::Record encodeAsStorageRecord() const;
+ static std::unique_ptr<SubresourcesEntry> decodeStorageRecord(const Storage::Record&);
+
+ const Key& key() const { return m_key; }
+ std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; }
+ const Vector<SubresourceInfo>& subresources() const { return m_subresources; }
+
+ void updateSubresourceLoads(const Vector<std::unique_ptr<SubresourceLoad>>&);
+
+private:
+ Key m_key;
+ std::chrono::system_clock::time_point m_timeStamp;
+ Vector<SubresourceInfo> m_subresources;
+};
+
+} // namespace WebKit
+} // namespace NetworkCache
+
+#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION)
+#endif // NetworkCacheSubresourcesEntry_h