diff options
author | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-06-27 06:07:23 +0000 |
---|---|---|
committer | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-06-27 06:07:23 +0000 |
commit | 1bf1084f2b10c3b47fd1a588d85d21ed0eb41d0c (patch) | |
tree | 46dcd36c86e7fbc6e5df36deb463b33e9967a6f7 /Source/WebKit2/NetworkProcess/cache | |
parent | 32761a6cee1d0dee366b885b7b9c777e67885688 (diff) | |
download | WebKitGtk-tarball-master.tar.gz |
webkitgtk-2.16.5HEADwebkitgtk-2.16.5master
Diffstat (limited to 'Source/WebKit2/NetworkProcess/cache')
29 files changed, 6154 insertions, 0 deletions
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp new file mode 100644 index 000000000..bbedce3f1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp @@ -0,0 +1,612 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCache.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheSpeculativeLoadManager.h" +#include "NetworkCacheStatistics.h" +#include "NetworkCacheStorage.h" +#include <WebCore/CacheValidation.h> +#include <WebCore/FileSystem.h> +#include <WebCore/HTTPHeaderNames.h> +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/PlatformCookieJar.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> +#include <wtf/MainThread.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/RunLoop.h> +#include <wtf/text/StringBuilder.h> + +#if PLATFORM(COCOA) +#include <notify.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +static const AtomicString& resourceType() +{ + ASSERT(WTF::isMainThread()); + static NeverDestroyed<const AtomicString> resource("Resource", AtomicString::ConstructFromLiteral); + return resource; +} + +Cache& singleton() +{ + static NeverDestroyed<Cache> instance; + return instance; +} + +#if PLATFORM(GTK) +static void dumpFileChanged(Cache* cache) +{ + cache->dumpContentsToFile(); +} +#endif + +bool Cache::initialize(const String& cachePath, const Parameters& parameters) +{ + m_storage = Storage::open(cachePath); + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (parameters.enableNetworkCacheSpeculativeRevalidation) + m_speculativeLoadManager = std::make_unique<SpeculativeLoadManager>(*m_storage); +#endif + + if (parameters.enableEfficacyLogging) + m_statistics = Statistics::open(cachePath); + +#if PLATFORM(COCOA) + // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump". + if (m_storage) { + int token; + notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) { + dumpContentsToFile(); + }); + } +#endif +#if PLATFORM(GTK) + // Triggers with "touch $cachePath/dump". + if (m_storage) { + CString dumpFilePath = WebCore::fileSystemRepresentation(WebCore::pathByAppendingComponent(m_storage->basePath(), "dump")); + GRefPtr<GFile> dumpFile = adoptGRef(g_file_new_for_path(dumpFilePath.data())); + GFileMonitor* monitor = g_file_monitor_file(dumpFile.get(), G_FILE_MONITOR_NONE, nullptr, nullptr); + g_signal_connect_swapped(monitor, "changed", G_CALLBACK(dumpFileChanged), this); + } +#endif + + LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage); + return !!m_storage; +} + +void Cache::setCapacity(size_t maximumSize) +{ + if (!m_storage) + return; + m_storage->setCapacity(maximumSize); +} + +Key Cache::makeCacheKey(const WebCore::ResourceRequest& request) +{ + // FIXME: This implements minimal Range header disk cache support. We don't parse + // ranges so only the same exact range request will be served from the cache. + String range = request.httpHeaderField(WebCore::HTTPHeaderName::Range); + return { request.cachePartition(), resourceType(), range, request.url().string(), m_storage->salt() }; +} + +static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy) +{ + switch (policy) { + case WebCore::ReturnCacheDataElseLoad: + case WebCore::ReturnCacheDataDontLoad: + return true; + case WebCore::UseProtocolCachePolicy: + case WebCore::ReloadIgnoringCacheData: + case WebCore::RefreshAnyCacheData: + return false; + case WebCore::DoNotUseAnyCache: + ASSERT_NOT_REACHED(); + return false; + } + return false; +} + +static bool responseHasExpired(const WebCore::ResourceResponse& response, std::chrono::system_clock::time_point timestamp, std::optional<std::chrono::microseconds> maxStale) +{ + if (response.cacheControlContainsNoCache()) + return true; + + auto age = WebCore::computeCurrentAge(response, timestamp); + auto lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(response, timestamp); + + auto maximumStaleness = maxStale ? maxStale.value() : 0ms; + bool hasExpired = age - lifetime > maximumStaleness; + +#ifndef LOG_DISABLED + if (hasExpired) + LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasExpired age=%f lifetime=%f max-stale=%g", age, lifetime, maxStale); +#endif + + return hasExpired; +} + +static bool responseNeedsRevalidation(const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& request, std::chrono::system_clock::time_point timestamp) +{ + auto requestDirectives = WebCore::parseCacheControlDirectives(request.httpHeaderFields()); + if (requestDirectives.noCache) + return true; + // For requests we ignore max-age values other than zero. + if (requestDirectives.maxAge && requestDirectives.maxAge.value() == 0ms) + return true; + + return responseHasExpired(response, timestamp, requestDirectives.maxStale); +} + +static UseDecision makeUseDecision(const Entry& entry, const WebCore::ResourceRequest& request) +{ + // The request is conditional so we force revalidation from the network. We merely check the disk cache + // so we can update the cache entry. + if (request.isConditional() && !entry.redirectRequest()) + return UseDecision::Validate; + + if (!WebCore::verifyVaryingRequestHeaders(entry.varyingRequestHeaders(), request)) + return UseDecision::NoDueToVaryingHeaderMismatch; + + // We never revalidate in the case of a history navigation. + if (cachePolicyAllowsExpired(request.cachePolicy())) + return UseDecision::Use; + + if (!responseNeedsRevalidation(entry.response(), request, entry.timeStamp())) + return UseDecision::Use; + + if (!entry.response().hasCacheValidatorFields()) + return UseDecision::NoDueToMissingValidatorFields; + + return entry.redirectRequest() ? UseDecision::NoDueToExpiredRedirect : UseDecision::Validate; +} + +static RetrieveDecision makeRetrieveDecision(const WebCore::ResourceRequest& request) +{ + ASSERT(request.cachePolicy() != WebCore::DoNotUseAnyCache); + + // FIXME: Support HEAD requests. + if (request.httpMethod() != "GET") + return RetrieveDecision::NoDueToHTTPMethod; + if (request.requester() == WebCore::ResourceRequest::Requester::Media) + return RetrieveDecision::NoDueToStreamingMedia; + if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData && !request.isConditional()) + return RetrieveDecision::NoDueToReloadIgnoringCache; + + return RetrieveDecision::Yes; +} + +static bool isMediaMIMEType(const String& mimeType) +{ + if (mimeType.startsWith("video/", /*caseSensitive*/ false)) + return true; + if (mimeType.startsWith("audio/", /*caseSensitive*/ false)) + return true; + return false; +} + +static StoreDecision makeStoreDecision(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response) +{ + if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP()) + return StoreDecision::NoDueToProtocol; + + if (originalRequest.httpMethod() != "GET") + return StoreDecision::NoDueToHTTPMethod; + + auto requestDirectives = WebCore::parseCacheControlDirectives(originalRequest.httpHeaderFields()); + if (requestDirectives.noStore) + return StoreDecision::NoDueToNoStoreRequest; + + if (response.cacheControlContainsNoStore()) + return StoreDecision::NoDueToNoStoreResponse; + + if (!WebCore::isStatusCodeCacheableByDefault(response.httpStatusCode())) { + // http://tools.ietf.org/html/rfc7234#section-4.3.2 + bool hasExpirationHeaders = response.expires() || response.cacheControlMaxAge(); + bool expirationHeadersAllowCaching = WebCore::isStatusCodePotentiallyCacheable(response.httpStatusCode()) && hasExpirationHeaders; + if (!expirationHeadersAllowCaching) + return StoreDecision::NoDueToHTTPStatusCode; + } + + bool isMainResource = originalRequest.requester() == WebCore::ResourceRequest::Requester::Main; + bool storeUnconditionallyForHistoryNavigation = isMainResource || originalRequest.priority() == WebCore::ResourceLoadPriority::VeryHigh; + if (!storeUnconditionallyForHistoryNavigation) { + auto now = std::chrono::system_clock::now(); + bool hasNonZeroLifetime = !response.cacheControlContainsNoCache() && WebCore::computeFreshnessLifetimeForHTTPFamily(response, now) > 0ms; + + bool possiblyReusable = response.hasCacheValidatorFields() || hasNonZeroLifetime; + if (!possiblyReusable) + return StoreDecision::NoDueToUnlikelyToReuse; + } + + // Media loaded via XHR is likely being used for MSE streaming (YouTube and Netflix for example). + // Streaming media fills the cache quickly and is unlikely to be reused. + // FIXME: We should introduce a separate media cache partition that doesn't affect other resources. + // FIXME: We should also make sure make the MSE paths are copy-free so we can use mapped buffers from disk effectively. + auto requester = originalRequest.requester(); + bool isDefinitelyStreamingMedia = requester == WebCore::ResourceRequest::Requester::Media; + bool isLikelyStreamingMedia = requester == WebCore::ResourceRequest::Requester::XHR && isMediaMIMEType(response.mimeType()); + if (isLikelyStreamingMedia || isDefinitelyStreamingMedia) + return StoreDecision::NoDueToStreamingMedia; + + return StoreDecision::Yes; +} + +void Cache::retrieve(const WebCore::ResourceRequest& request, const GlobalFrameID& frameID, Function<void (std::unique_ptr<Entry>)>&& completionHandler) +{ + ASSERT(isEnabled()); + ASSERT(request.url().protocolIsInHTTPFamily()); + + LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %d", request.url().string().ascii().data(), static_cast<int>(request.priority())); + + if (m_statistics) + m_statistics->recordRetrievalRequest(frameID.first); + + Key storageKey = makeCacheKey(request); + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + bool canUseSpeculativeRevalidation = m_speculativeLoadManager && !request.isConditional(); + if (canUseSpeculativeRevalidation) + m_speculativeLoadManager->registerLoad(frameID, request, storageKey); +#endif + + auto retrieveDecision = makeRetrieveDecision(request); + if (retrieveDecision != RetrieveDecision::Yes) { + if (m_statistics) + m_statistics->recordNotUsingCacheForRequest(frameID.first, storageKey, request, retrieveDecision); + + completionHandler(nullptr); + return; + } + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (canUseSpeculativeRevalidation && m_speculativeLoadManager->canRetrieve(storageKey, request, frameID)) { + m_speculativeLoadManager->retrieve(storageKey, [request, completionHandler = WTFMove(completionHandler)](std::unique_ptr<Entry> entry) { + if (entry && WebCore::verifyVaryingRequestHeaders(entry->varyingRequestHeaders(), request)) + completionHandler(WTFMove(entry)); + else + completionHandler(nullptr); + }); + return; + } +#endif + + auto startTime = std::chrono::system_clock::now(); + auto priority = static_cast<unsigned>(request.priority()); + + m_storage->retrieve(storageKey, priority, [this, request, completionHandler = WTFMove(completionHandler), startTime, storageKey, frameID](auto record) { + if (!record) { + LOG(NetworkCache, "(NetworkProcess) not found in storage"); + + if (m_statistics) + m_statistics->recordRetrievalFailure(frameID.first, storageKey, request); + + completionHandler(nullptr); + return false; + } + + ASSERT(record->key == storageKey); + + auto entry = Entry::decodeStorageRecord(*record); + + auto useDecision = entry ? makeUseDecision(*entry, request) : UseDecision::NoDueToDecodeFailure; + switch (useDecision) { + case UseDecision::Use: + break; + case UseDecision::Validate: + entry->setNeedsValidation(true); + break; + default: + entry = nullptr; + }; + +#if !LOG_DISABLED + auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); + LOG(NetworkCache, "(NetworkProcess) retrieve complete useDecision=%d priority=%d time=%" PRIi64 "ms", static_cast<int>(useDecision), static_cast<int>(request.priority()), elapsedMS); +#endif + completionHandler(WTFMove(entry)); + + if (m_statistics) + m_statistics->recordRetrievedCachedEntry(frameID.first, storageKey, request, useDecision); + return useDecision != UseDecision::NoDueToDecodeFailure; + }); +} + + +std::unique_ptr<Entry> Cache::makeEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData) +{ + return std::make_unique<Entry>(makeCacheKey(request), response, WTFMove(responseData), WebCore::collectVaryingRequestHeaders(request, response)); +} + +std::unique_ptr<Entry> Cache::makeRedirectEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest) +{ + return std::make_unique<Entry>(makeCacheKey(request), response, redirectRequest, WebCore::collectVaryingRequestHeaders(request, response)); +} + +std::unique_ptr<Entry> Cache::store(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, Function<void (MappedBody&)>&& completionHandler) +{ + ASSERT(isEnabled()); + ASSERT(responseData); + + LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", request.url().string().latin1().data(), makeCacheKey(request).partition().latin1().data()); + + StoreDecision storeDecision = makeStoreDecision(request, response); + if (storeDecision != StoreDecision::Yes) { + LOG(NetworkCache, "(NetworkProcess) didn't store, storeDecision=%d", static_cast<int>(storeDecision)); + auto key = makeCacheKey(request); + + auto isSuccessfulRevalidation = response.httpStatusCode() == 304; + if (!isSuccessfulRevalidation) { + // Make sure we don't keep a stale entry in the cache. + remove(key); + } + + if (m_statistics) + m_statistics->recordNotCachingResponse(key, storeDecision); + + return nullptr; + } + + auto cacheEntry = makeEntry(request, response, WTFMove(responseData)); + auto record = cacheEntry->encodeAsStorageRecord(); + + m_storage->store(record, [this, completionHandler = WTFMove(completionHandler)](const Data& bodyData) { + MappedBody mappedBody; +#if ENABLE(SHAREABLE_RESOURCE) + if (canUseSharedMemoryForBodyData()) { + if (auto sharedMemory = bodyData.tryCreateSharedMemory()) { + mappedBody.shareableResource = ShareableResource::create(sharedMemory.releaseNonNull(), 0, bodyData.size()); + ASSERT(mappedBody.shareableResource); + mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle); + } + } +#endif + completionHandler(mappedBody); + LOG(NetworkCache, "(NetworkProcess) stored"); + }); + + return cacheEntry; +} + +std::unique_ptr<Entry> Cache::storeRedirect(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest) +{ + ASSERT(isEnabled()); + + LOG(NetworkCache, "(NetworkProcess) storing redirect %s -> %s", request.url().string().latin1().data(), redirectRequest.url().string().latin1().data()); + + StoreDecision storeDecision = makeStoreDecision(request, response); + if (storeDecision != StoreDecision::Yes) { + LOG(NetworkCache, "(NetworkProcess) didn't store redirect, storeDecision=%d", static_cast<int>(storeDecision)); + auto key = makeCacheKey(request); + if (m_statistics) + m_statistics->recordNotCachingResponse(key, storeDecision); + + return nullptr; + } + + auto cacheEntry = makeRedirectEntry(request, response, redirectRequest); + auto record = cacheEntry->encodeAsStorageRecord(); + + m_storage->store(record, nullptr); + + return cacheEntry; +} + +std::unique_ptr<Entry> Cache::update(const WebCore::ResourceRequest& originalRequest, const GlobalFrameID& frameID, const Entry& existingEntry, const WebCore::ResourceResponse& validatingResponse) +{ + LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data()); + + WebCore::ResourceResponse response = existingEntry.response(); + WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse); + + auto updateEntry = std::make_unique<Entry>(existingEntry.key(), response, existingEntry.buffer(), WebCore::collectVaryingRequestHeaders(originalRequest, response)); + auto updateRecord = updateEntry->encodeAsStorageRecord(); + + m_storage->store(updateRecord, { }); + + if (m_statistics) + m_statistics->recordRevalidationSuccess(frameID.first, existingEntry.key(), originalRequest); + + return updateEntry; +} + +void Cache::remove(const Key& key) +{ + ASSERT(isEnabled()); + + m_storage->remove(key); +} + +void Cache::remove(const WebCore::ResourceRequest& request) +{ + remove(makeCacheKey(request)); +} + +void Cache::traverse(Function<void (const TraversalEntry*)>&& traverseHandler) +{ + ASSERT(isEnabled()); + + // Protect against clients making excessive traversal requests. + const unsigned maximumTraverseCount = 3; + if (m_traverseCount >= maximumTraverseCount) { + WTFLogAlways("Maximum parallel cache traverse count exceeded. Ignoring traversal request."); + + RunLoop::main().dispatch([traverseHandler = WTFMove(traverseHandler)] { + traverseHandler(nullptr); + }); + return; + } + + ++m_traverseCount; + + m_storage->traverse(resourceType(), 0, [this, traverseHandler = WTFMove(traverseHandler)](const Storage::Record* record, const Storage::RecordInfo& recordInfo) { + if (!record) { + --m_traverseCount; + traverseHandler(nullptr); + return; + } + + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) + return; + + TraversalEntry traversalEntry { *entry, recordInfo }; + traverseHandler(&traversalEntry); + }); +} + +String Cache::dumpFilePath() const +{ + return WebCore::pathByAppendingComponent(m_storage->versionPath(), "dump.json"); +} + +void Cache::dumpContentsToFile() +{ + if (!m_storage) + return; + auto fd = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite); + if (!WebCore::isHandleValid(fd)) + return; + auto prologue = String("{\n\"entries\": [\n").utf8(); + WebCore::writeToFile(fd, prologue.data(), prologue.length()); + + struct Totals { + unsigned count { 0 }; + double worth { 0 }; + size_t bodySize { 0 }; + }; + Totals totals; + auto flags = Storage::TraverseFlag::ComputeWorth | Storage::TraverseFlag::ShareCount; + size_t capacity = m_storage->capacity(); + m_storage->traverse(resourceType(), flags, [fd, totals, capacity](const Storage::Record* record, const Storage::RecordInfo& info) mutable { + if (!record) { + StringBuilder epilogue; + epilogue.appendLiteral("{}\n],\n"); + epilogue.appendLiteral("\"totals\": {\n"); + epilogue.appendLiteral("\"capacity\": "); + epilogue.appendNumber(capacity); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"count\": "); + epilogue.appendNumber(totals.count); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"bodySize\": "); + epilogue.appendNumber(totals.bodySize); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"averageWorth\": "); + epilogue.appendNumber(totals.count ? totals.worth / totals.count : 0); + epilogue.appendLiteral("\n"); + epilogue.appendLiteral("}\n}\n"); + auto writeData = epilogue.toString().utf8(); + WebCore::writeToFile(fd, writeData.data(), writeData.length()); + WebCore::closeFile(fd); + return; + } + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) + return; + ++totals.count; + totals.worth += info.worth; + totals.bodySize += info.bodySize; + + StringBuilder json; + entry->asJSON(json, info); + json.appendLiteral(",\n"); + auto writeData = json.toString().utf8(); + WebCore::writeToFile(fd, writeData.data(), writeData.length()); + }); +} + +void Cache::deleteDumpFile() +{ + WorkQueue::create("com.apple.WebKit.Cache.delete")->dispatch([path = dumpFilePath().isolatedCopy()] { + WebCore::deleteFile(path); + }); +} + +void Cache::clear(std::chrono::system_clock::time_point modifiedSince, Function<void ()>&& completionHandler) +{ + LOG(NetworkCache, "(NetworkProcess) clearing cache"); + + if (m_statistics) + m_statistics->clear(); + + if (!m_storage) { + RunLoop::main().dispatch(WTFMove(completionHandler)); + return; + } + String anyType; + m_storage->clear(anyType, modifiedSince, WTFMove(completionHandler)); + + deleteDumpFile(); +} + +void Cache::clear() +{ + clear(std::chrono::system_clock::time_point::min(), nullptr); +} + +String Cache::recordsPath() const +{ + return m_storage ? m_storage->recordsPath() : String(); +} + +void Cache::retrieveData(const DataKey& dataKey, Function<void (const uint8_t* data, size_t size)> completionHandler) +{ + ASSERT(isEnabled()); + + Key key { dataKey, m_storage->salt() }; + m_storage->retrieve(key, 4, [completionHandler = WTFMove(completionHandler)] (auto record) { + if (!record || !record->body.size()) { + completionHandler(nullptr, 0); + return true; + } + completionHandler(record->body.data(), record->body.size()); + return true; + }); +} + +void Cache::storeData(const DataKey& dataKey, const uint8_t* data, size_t size) +{ + if (!m_storage) + return; + Key key { dataKey, m_storage->salt() }; + Storage::Record record { key, std::chrono::system_clock::now(), { }, Data { data, size }, { } }; + m_storage->store(record, { }); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.h b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h new file mode 100644 index 000000000..ca2bbc2dc --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCache_h +#define NetworkCache_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheEntry.h" +#include "NetworkCacheStorage.h" +#include "ShareableResource.h" +#include <WebCore/ResourceResponse.h> +#include <wtf/Function.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class ResourceRequest; +class SharedBuffer; +class URL; +} + +namespace WebKit { +namespace NetworkCache { + +class Cache; +class SpeculativeLoadManager; +class Statistics; + +Cache& singleton(); + +struct MappedBody { +#if ENABLE(SHAREABLE_RESOURCE) + RefPtr<ShareableResource> shareableResource; + ShareableResource::Handle shareableResourceHandle; +#endif +}; + +enum class RetrieveDecision { + Yes, + NoDueToHTTPMethod, + NoDueToConditionalRequest, + NoDueToReloadIgnoringCache, + NoDueToStreamingMedia, +}; + +// FIXME: This enum is used in the Statistics code in a way that prevents removing or reordering anything. +enum class StoreDecision { + Yes, + NoDueToProtocol, + NoDueToHTTPMethod, + NoDueToAttachmentResponse, // Unused. + NoDueToNoStoreResponse, + NoDueToHTTPStatusCode, + NoDueToNoStoreRequest, + NoDueToUnlikelyToReuse, + NoDueToStreamingMedia, +}; + +enum class UseDecision { + Use, + Validate, + NoDueToVaryingHeaderMismatch, + NoDueToMissingValidatorFields, + NoDueToDecodeFailure, + NoDueToExpiredRedirect, +}; + +using GlobalFrameID = std::pair<uint64_t /*webPageID*/, uint64_t /*webFrameID*/>; + +class Cache { + WTF_MAKE_NONCOPYABLE(Cache); + friend class WTF::NeverDestroyed<Cache>; +public: + struct Parameters { + bool enableEfficacyLogging; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + bool enableNetworkCacheSpeculativeRevalidation; +#endif + }; + bool initialize(const String& cachePath, const Parameters&); + void setCapacity(size_t); + + bool isEnabled() const { return !!m_storage; } + + // Completion handler may get called back synchronously on failure. + void retrieve(const WebCore::ResourceRequest&, const GlobalFrameID&, Function<void (std::unique_ptr<Entry>)>&&); + std::unique_ptr<Entry> store(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, Function<void (MappedBody&)>&&); + std::unique_ptr<Entry> storeRedirect(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest); + std::unique_ptr<Entry> update(const WebCore::ResourceRequest&, const GlobalFrameID&, const Entry&, const WebCore::ResourceResponse& validatingResponse); + + struct TraversalEntry { + const Entry& entry; + const Storage::RecordInfo& recordInfo; + }; + void traverse(Function<void (const TraversalEntry*)>&&); + void remove(const Key&); + void remove(const WebCore::ResourceRequest&); + + void clear(); + void clear(std::chrono::system_clock::time_point modifiedSince, Function<void ()>&& completionHandler); + + void retrieveData(const DataKey&, Function<void (const uint8_t* data, size_t size)>); + void storeData(const DataKey&, const uint8_t* data, size_t); + + std::unique_ptr<Entry> makeEntry(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&); + std::unique_ptr<Entry> makeRedirectEntry(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest); + + void dumpContentsToFile(); + + String recordsPath() const; + bool canUseSharedMemoryForBodyData() const { return m_storage && m_storage->canUseSharedMemoryForBodyData(); } + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + SpeculativeLoadManager* speculativeLoadManager() { return m_speculativeLoadManager.get(); } +#endif + +private: + Cache() = default; + ~Cache() = delete; + + Key makeCacheKey(const WebCore::ResourceRequest&); + + String dumpFilePath() const; + void deleteDumpFile(); + + std::unique_ptr<Storage> m_storage; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + std::unique_ptr<SpeculativeLoadManager> m_speculativeLoadManager; +#endif + std::unique_ptr<Statistics> m_statistics; + + unsigned m_traverseCount { 0 }; +}; + +} +} +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp new file mode 100644 index 000000000..267caf6a0 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheBlobStorage.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheFileSystem.h" +#include <WebCore/FileSystem.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <wtf/RunLoop.h> +#include <wtf/SHA1.h> + +namespace WebKit { +namespace NetworkCache { + +BlobStorage::BlobStorage(const String& blobDirectoryPath, Salt salt) + : m_blobDirectoryPath(blobDirectoryPath) + , m_salt(salt) +{ +} + +String BlobStorage::blobDirectoryPath() const +{ + return m_blobDirectoryPath.isolatedCopy(); +} + +void BlobStorage::synchronize() +{ + ASSERT(!RunLoop::isMain()); + + WebCore::makeAllDirectories(blobDirectoryPath()); + + m_approximateSize = 0; + auto blobDirectory = blobDirectoryPath(); + traverseDirectory(blobDirectory, [this, &blobDirectory](const String& name, DirectoryEntryType type) { + if (type != DirectoryEntryType::File) + return; + auto path = WebCore::pathByAppendingComponent(blobDirectory, name); + auto filePath = WebCore::fileSystemRepresentation(path); + struct stat stat; + ::stat(filePath.data(), &stat); + // No clients left for this blob. + if (stat.st_nlink == 1) + unlink(filePath.data()); + else + m_approximateSize += stat.st_size; + }); + + LOG(NetworkCacheStorage, "(NetworkProcess) blob synchronization completed approximateSize=%zu", approximateSize()); +} + +String BlobStorage::blobPathForHash(const SHA1::Digest& hash) const +{ + auto hashAsString = SHA1::hexDigest(hash); + return WebCore::pathByAppendingComponent(blobDirectoryPath(), String::fromUTF8(hashAsString)); +} + +BlobStorage::Blob BlobStorage::add(const String& path, const Data& data) +{ + ASSERT(!RunLoop::isMain()); + + auto hash = computeSHA1(data, m_salt); + if (data.isEmpty()) + return { data, hash }; + + auto blobPath = WebCore::fileSystemRepresentation(blobPathForHash(hash)); + auto linkPath = WebCore::fileSystemRepresentation(path); + unlink(linkPath.data()); + + bool blobExists = access(blobPath.data(), F_OK) != -1; + if (blobExists) { + auto existingData = mapFile(blobPath.data()); + if (bytesEqual(existingData, data)) { + if (link(blobPath.data(), linkPath.data()) == -1) + WTFLogAlways("Failed to create hard link from %s to %s", blobPath.data(), linkPath.data()); + return { existingData, hash }; + } + unlink(blobPath.data()); + } + + auto mappedData = data.mapToFile(blobPath.data()); + if (mappedData.isNull()) + return { }; + + if (link(blobPath.data(), linkPath.data()) == -1) + WTFLogAlways("Failed to create hard link from %s to %s", blobPath.data(), linkPath.data()); + + m_approximateSize += mappedData.size(); + + return { mappedData, hash }; +} + +BlobStorage::Blob BlobStorage::get(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + auto data = mapFile(linkPath.data()); + + return { data, computeSHA1(data, m_salt) }; +} + +void BlobStorage::remove(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + unlink(linkPath.data()); +} + +unsigned BlobStorage::shareCount(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + struct stat stat; + if (::stat(linkPath.data(), &stat) < 0) + return 0; + // Link count is 2 in the single client case (the blob file and a link). + return stat.st_nlink - 1; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h new file mode 100644 index 000000000..adf0b085e --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheBlobStorage_h +#define NetworkCacheBlobStorage_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include "NetworkCacheKey.h" +#include <wtf/SHA1.h> + +namespace WebKit { +namespace NetworkCache { + +// BlobStorage deduplicates the data using SHA1 hash computed over the blob bytes. +class BlobStorage { + WTF_MAKE_NONCOPYABLE(BlobStorage); +public: + BlobStorage(const String& blobDirectoryPath, Salt); + + struct Blob { + Data data; + SHA1::Digest hash; + }; + // These are all synchronous and should not be used from the main thread. + Blob add(const String& path, const Data&); + Blob get(const String& path); + + // Blob won't be removed until synchronization. + void remove(const String& path); + + unsigned shareCount(const String& path); + + size_t approximateSize() const { return m_approximateSize; } + + void synchronize(); + +private: + String blobDirectoryPath() const; + String blobPathForHash(const SHA1::Digest&) const; + + const String m_blobDirectoryPath; + const Salt m_salt; + + std::atomic<size_t> m_approximateSize { 0 }; +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp new file mode 100644 index 000000000..8423b31e6 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheCoders.h" + +#if ENABLE(NETWORK_CACHE) + +namespace WTF { +namespace Persistence { + +// Store common HTTP headers as strings instead of using their value in the HTTPHeaderName enumeration +// so that the headers stored in the cache stays valid even after HTTPHeaderName.in gets updated. +void Coder<WebCore::HTTPHeaderMap>::encode(Encoder& encoder, const WebCore::HTTPHeaderMap& headers) +{ + encoder << static_cast<uint64_t>(headers.size()); + for (auto& keyValue : headers) { + encoder << keyValue.key; + encoder << keyValue.value; + } +} + +bool Coder<WebCore::HTTPHeaderMap>::decode(Decoder& decoder, WebCore::HTTPHeaderMap& headers) +{ + uint64_t headersSize; + if (!decoder.decode(headersSize)) + return false; + for (uint64_t i = 0; i < headersSize; ++i) { + String name; + if (!decoder.decode(name)) + return false; + String value; + if (!decoder.decode(value)) + return false; + headers.add(name, value); + } + return true; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h new file mode 100644 index 000000000..8cac8c448 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2010, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/CertificateInfo.h> +#include <WebCore/HTTPHeaderMap.h> +#include <utility> +#include <wtf/Forward.h> +#include <wtf/HashMap.h> +#include <wtf/HashSet.h> +#include <wtf/SHA1.h> +#include <wtf/Vector.h> +#include <wtf/persistence/Coders.h> +#include <wtf/persistence/Decoder.h> +#include <wtf/persistence/Encoder.h> + +namespace WTF { +namespace Persistence { + +template<> struct Coder<WebCore::CertificateInfo> { + static void encode(Encoder&, const WebCore::CertificateInfo&); + static bool decode(Decoder&, WebCore::CertificateInfo&); +}; + +template<> struct Coder<WebCore::HTTPHeaderMap> { + static void encode(Encoder&, const WebCore::HTTPHeaderMap&); + static bool decode(Decoder&, WebCore::HTTPHeaderMap&); +}; + +} +} +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp new file mode 100644 index 000000000..5de5095d3 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheCoders.h" + +#if ENABLE(NETWORK_CACHE) + +#if PLATFORM(COCOA) +#include <Security/SecCertificate.h> +#include <Security/SecTrust.h> +#include <wtf/spi/cocoa/SecuritySPI.h> +#endif + +namespace WTF { +namespace Persistence { + +static void encodeCFData(Encoder& encoder, CFDataRef data) +{ + uint64_t length = CFDataGetLength(data); + const uint8_t* bytePtr = CFDataGetBytePtr(data); + + encoder << length; + encoder.encodeFixedLengthData(bytePtr, length); +} + +static bool decodeCFData(Decoder& decoder, RetainPtr<CFDataRef>& data) +{ + uint64_t size = 0; + if (!decoder.decode(size)) + return false; + + Vector<uint8_t> vector(size); + if (!decoder.decodeFixedLengthData(vector.data(), vector.size())) + return false; + + data = adoptCF(CFDataCreate(nullptr, vector.data(), vector.size())); + return true; +} + + +#if HAVE(SEC_TRUST_SERIALIZATION) +static void encodeSecTrustRef(Encoder& encoder, SecTrustRef trust) +{ + auto data = adoptCF(SecTrustSerialize(trust, nullptr)); + if (!data) { + encoder << false; + return; + } + + encoder << true; + encodeCFData(encoder, data.get()); +} + +static bool decodeSecTrustRef(Decoder& decoder, RetainPtr<SecTrustRef>& result) +{ + bool hasTrust; + if (!decoder.decode(hasTrust)) + return false; + + if (!hasTrust) + return true; + + RetainPtr<CFDataRef> trustData; + if (!decodeCFData(decoder, trustData)) + return false; + + auto trust = adoptCF(SecTrustDeserialize(trustData.get(), nullptr)); + if (!trust) + return false; + + result = WTFMove(trust); + return true; +} +#endif + +static void encodeCertificateChain(Encoder& encoder, CFArrayRef certificateChain) +{ + CFIndex size = CFArrayGetCount(certificateChain); + Vector<CFTypeRef, 32> values(size); + + CFArrayGetValues(certificateChain, CFRangeMake(0, size), values.data()); + + encoder << static_cast<uint64_t>(size); + + for (CFIndex i = 0; i < size; ++i) { + ASSERT(values[i]); + ASSERT(CFGetTypeID(values[i]) == SecCertificateGetTypeID()); + + auto data = adoptCF(SecCertificateCopyData((SecCertificateRef)values[i])); + encodeCFData(encoder, data.get()); + } +} + +static bool decodeCertificateChain(Decoder& decoder, RetainPtr<CFArrayRef>& certificateChain) +{ + uint64_t size; + if (!decoder.decode(size)) + return false; + + auto array = adoptCF(CFArrayCreateMutable(0, 0, &kCFTypeArrayCallBacks)); + + for (size_t i = 0; i < size; ++i) { + RetainPtr<CFDataRef> data; + if (!decodeCFData(decoder, data)) + return false; + + auto certificate = adoptCF(SecCertificateCreateWithData(0, data.get())); + CFArrayAppendValue(array.get(), certificate.get()); + } + + certificateChain = WTFMove(array); + return true; +} + +void Coder<WebCore::CertificateInfo>::encode(Encoder& encoder, const WebCore::CertificateInfo& certificateInfo) +{ + encoder.encodeEnum(certificateInfo.type()); + + switch (certificateInfo.type()) { +#if HAVE(SEC_TRUST_SERIALIZATION) + case WebCore::CertificateInfo::Type::Trust: + encodeSecTrustRef(encoder, certificateInfo.trust()); + break; +#endif + case WebCore::CertificateInfo::Type::CertificateChain: { + encodeCertificateChain(encoder, certificateInfo.certificateChain()); + break; + } + case WebCore::CertificateInfo::Type::None: + // Do nothing. + break; + } +} + +bool Coder<WebCore::CertificateInfo>::decode(Decoder& decoder, WebCore::CertificateInfo& certificateInfo) +{ + WebCore::CertificateInfo::Type certificateInfoType; + if (!decoder.decodeEnum(certificateInfoType)) + return false; + + switch (certificateInfoType) { +#if HAVE(SEC_TRUST_SERIALIZATION) + case WebCore::CertificateInfo::Type::Trust: { + RetainPtr<SecTrustRef> trust; + if (!decodeSecTrustRef(decoder, trust)) + return false; + + certificateInfo = WebCore::CertificateInfo(WTFMove(trust)); + return true; + } +#endif + case WebCore::CertificateInfo::Type::CertificateChain: { + RetainPtr<CFArrayRef> certificateChain; + if (!decodeCertificateChain(decoder, certificateChain)) + return false; + + certificateInfo = WebCore::CertificateInfo(WTFMove(certificateChain)); + return true; + } + case WebCore::CertificateInfo::Type::None: + // Do nothing. + break; + } + + return true; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp new file mode 100644 index 000000000..0cef6ae03 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheCoders.h" + +#if ENABLE(NETWORK_CACHE) + +namespace WTF { +namespace Persistence { + +void Coder<WebCore::CertificateInfo>::encode(Encoder& encoder, const WebCore::CertificateInfo& certificateInfo) +{ + if (!certificateInfo.certificate()) { + encoder << false; + return; + } + + GByteArray* certificateData = 0; + g_object_get(G_OBJECT(certificateInfo.certificate()), "certificate", &certificateData, NULL); + if (!certificateData) { + encoder << false; + return; + } + + encoder << true; + + GRefPtr<GByteArray> certificate = adoptGRef(certificateData); + encoder << static_cast<uint64_t>(certificate->len); + encoder.encodeFixedLengthData(certificate->data, certificate->len); + + encoder << static_cast<uint32_t>(certificateInfo.tlsErrors()); +} + +bool Coder<WebCore::CertificateInfo>::decode(Decoder& decoder, WebCore::CertificateInfo& certificateInfo) +{ + bool hasCertificate; + if (!decoder.decode(hasCertificate)) + return false; + + if (!hasCertificate) + return true; + + uint64_t size = 0; + if (!decoder.decode(size)) + return false; + + Vector<uint8_t> vector(size); + if (!decoder.decodeFixedLengthData(vector.data(), vector.size())) + return false; + + GByteArray* certificateData = g_byte_array_sized_new(vector.size()); + certificateData = g_byte_array_append(certificateData, vector.data(), vector.size()); + GRefPtr<GByteArray> certificateBytes = adoptGRef(certificateData); + + GTlsBackend* backend = g_tls_backend_get_default(); + GRefPtr<GTlsCertificate> certificate = adoptGRef(G_TLS_CERTIFICATE(g_initable_new( + g_tls_backend_get_certificate_type(backend), 0, 0, "certificate", certificateBytes.get(), nullptr))); + certificateInfo.setCertificate(certificate.get()); + + uint32_t tlsErrors; + if (!decoder.decode(tlsErrors)) + return false; + certificateInfo.setTLSErrors(static_cast<GTlsCertificateFlags>(tlsErrors)); + + return true; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp new file mode 100644 index 000000000..31ae48fe5 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheData.h" + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/FileSystem.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <wtf/CryptographicallyRandomNumber.h> + +namespace WebKit { +namespace NetworkCache { + +Data Data::mapToFile(const char* path) const +{ + int fd = open(path, O_CREAT | O_EXCL | O_RDWR , S_IRUSR | S_IWUSR); + if (fd < 0) + return { }; + + if (ftruncate(fd, m_size) < 0) { + close(fd); + return { }; + } + + void* map = mmap(nullptr, m_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + close(fd); + return { }; + } + + uint8_t* mapData = static_cast<uint8_t*>(map); + apply([&mapData](const uint8_t* bytes, size_t bytesSize) { + memcpy(mapData, bytes, bytesSize); + mapData += bytesSize; + return true; + }); + + // Drop the write permission. + mprotect(map, m_size, PROT_READ); + + // Flush (asynchronously) to file, turning this into clean memory. + msync(map, m_size, MS_ASYNC); + + return Data::adoptMap(map, m_size, fd); +} + +Data mapFile(const char* path) +{ + int fd = open(path, O_RDONLY, 0); + if (fd < 0) + return { }; + struct stat stat; + if (fstat(fd, &stat) < 0) { + close(fd); + return { }; + } + size_t size = stat.st_size; + if (!size) { + close(fd); + return Data::empty(); + } + + return adoptAndMapFile(fd, 0, size); +} + +Data adoptAndMapFile(int fd, size_t offset, size_t size) +{ + if (!size) { + close(fd); + return Data::empty(); + } + + void* map = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, offset); + if (map == MAP_FAILED) { + close(fd); + return { }; + } + + return Data::adoptMap(map, size, fd); +} + +SHA1::Digest computeSHA1(const Data& data, const Salt& salt) +{ + SHA1 sha1; + sha1.addBytes(salt.data(), salt.size()); + data.apply([&sha1](const uint8_t* data, size_t size) { + sha1.addBytes(data, size); + return true; + }); + + SHA1::Digest digest; + sha1.computeHash(digest); + return digest; +} + +bool bytesEqual(const Data& a, const Data& b) +{ + if (a.isNull() || b.isNull()) + return false; + if (a.size() != b.size()) + return false; + return !memcmp(a.data(), b.data(), a.size()); +} + +static Salt makeSalt() +{ + Salt salt; + static_assert(salt.size() == 8, "Salt size"); + *reinterpret_cast<uint32_t*>(&salt[0]) = cryptographicallyRandomNumber(); + *reinterpret_cast<uint32_t*>(&salt[4]) = cryptographicallyRandomNumber(); + return salt; +} + +std::optional<Salt> readOrMakeSalt(const String& path) +{ + auto cpath = WebCore::fileSystemRepresentation(path); + auto fd = open(cpath.data(), O_RDONLY, 0); + Salt salt; + auto bytesRead = read(fd, salt.data(), salt.size()); + close(fd); + if (bytesRead != salt.size()) { + salt = makeSalt(); + + unlink(cpath.data()); + fd = open(cpath.data(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + bool success = write(fd, salt.data(), salt.size()) == salt.size(); + close(fd); + if (!success) + return { }; + } + return salt; +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif // #if ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h new file mode 100644 index 000000000..29c264b4b --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheData_h +#define NetworkCacheData_h + +#if ENABLE(NETWORK_CACHE) + +#include <functional> +#include <wtf/FunctionDispatcher.h> +#include <wtf/SHA1.h> +#include <wtf/ThreadSafeRefCounted.h> +#include <wtf/text/WTFString.h> + +#if USE(SOUP) +#include <WebCore/GRefPtrSoup.h> +#endif + +namespace WebKit { + +class SharedMemory; + +namespace NetworkCache { + +#if PLATFORM(COCOA) +template <typename T> class DispatchPtr; +template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject); + +// FIXME: Use OSObjectPtr instead when it works with dispatch_data_t on all platforms. +template<typename T> class DispatchPtr { +public: + DispatchPtr() + : m_ptr(nullptr) + { + } + explicit DispatchPtr(T ptr) + : m_ptr(ptr) + { + if (m_ptr) + dispatch_retain(m_ptr); + } + DispatchPtr(const DispatchPtr& other) + : m_ptr(other.m_ptr) + { + if (m_ptr) + dispatch_retain(m_ptr); + } + ~DispatchPtr() + { + if (m_ptr) + dispatch_release(m_ptr); + } + + DispatchPtr& operator=(const DispatchPtr& other) + { + auto copy = other; + std::swap(m_ptr, copy.m_ptr); + return *this; + } + + T get() const { return m_ptr; } + explicit operator bool() const { return m_ptr; } + + friend DispatchPtr adoptDispatch<T>(T); + +private: + struct Adopt { }; + DispatchPtr(Adopt, T data) + : m_ptr(data) + { + } + + T m_ptr; +}; + +template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject) +{ + return DispatchPtr<T>(typename DispatchPtr<T>::Adopt { }, dispatchObject); +} +#endif + +class Data { +public: + Data() { } + Data(const uint8_t*, size_t); + + ~Data() { } + + static Data empty(); + static Data adoptMap(void* map, size_t, int fd); + +#if PLATFORM(COCOA) + enum class Backing { Buffer, Map }; + Data(DispatchPtr<dispatch_data_t>, Backing = Backing::Buffer); +#endif +#if USE(SOUP) + Data(GRefPtr<SoupBuffer>&&, int fd = -1); +#endif + bool isNull() const; + bool isEmpty() const { return !m_size; } + + const uint8_t* data() const; + size_t size() const { return m_size; } + bool isMap() const { return m_isMap; } + RefPtr<SharedMemory> tryCreateSharedMemory() const; + + Data subrange(size_t offset, size_t) const; + + bool apply(const Function<bool (const uint8_t*, size_t)>&) const; + + Data mapToFile(const char* path) const; + +#if PLATFORM(COCOA) + dispatch_data_t dispatchData() const { return m_dispatchData.get(); } +#endif + +#if USE(SOUP) + SoupBuffer* soupBuffer() const { return m_buffer.get(); } +#endif +private: +#if PLATFORM(COCOA) + mutable DispatchPtr<dispatch_data_t> m_dispatchData; +#endif +#if USE(SOUP) + mutable GRefPtr<SoupBuffer> m_buffer; + int m_fileDescriptor { -1 }; +#endif + mutable const uint8_t* m_data { nullptr }; + size_t m_size { 0 }; + bool m_isMap { false }; +}; + +Data concatenate(const Data&, const Data&); +bool bytesEqual(const Data&, const Data&); +Data adoptAndMapFile(int fd, size_t offset, size_t); +Data mapFile(const char* path); + +using Salt = std::array<uint8_t, 8>; + +std::optional<Salt> readOrMakeSalt(const String& path); +SHA1::Digest computeSHA1(const Data&, const Salt&); + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp new file mode 100644 index 000000000..21b58fa7e --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2015 Igalia S.L + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheData.h" + +#if ENABLE(NETWORK_CACHE) + +#include "SharedMemory.h" +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +namespace WebKit { +namespace NetworkCache { + +Data::Data(const uint8_t* data, size_t size) + : m_size(size) +{ + uint8_t* copiedData = static_cast<uint8_t*>(fastMalloc(size)); + memcpy(copiedData, data, size); + m_buffer = adoptGRef(soup_buffer_new_with_owner(copiedData, size, copiedData, fastFree)); +} + +Data::Data(GRefPtr<SoupBuffer>&& buffer, int fd) + : m_buffer(buffer) + , m_fileDescriptor(fd) + , m_size(buffer ? buffer->length : 0) + , m_isMap(m_size && fd != -1) +{ +} + +Data Data::empty() +{ + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new(SOUP_MEMORY_TAKE, nullptr, 0)); + return { WTFMove(buffer) }; +} + +const uint8_t* Data::data() const +{ + return m_buffer ? reinterpret_cast<const uint8_t*>(m_buffer->data) : nullptr; +} + +bool Data::isNull() const +{ + return !m_buffer; +} + +bool Data::apply(const Function<bool (const uint8_t*, size_t)>& applier) const +{ + if (!m_size) + return false; + + return applier(reinterpret_cast<const uint8_t*>(m_buffer->data), m_buffer->length); +} + +Data Data::subrange(size_t offset, size_t size) const +{ + if (!m_buffer) + return { }; + + GRefPtr<SoupBuffer> subBuffer = adoptGRef(soup_buffer_new_subbuffer(m_buffer.get(), offset, size)); + return { WTFMove(subBuffer) }; +} + +Data concatenate(const Data& a, const Data& b) +{ + if (a.isNull()) + return b; + if (b.isNull()) + return a; + + size_t size = a.size() + b.size(); + uint8_t* data = static_cast<uint8_t*>(fastMalloc(size)); + memcpy(data, a.soupBuffer()->data, a.soupBuffer()->length); + memcpy(data + a.soupBuffer()->length, b.soupBuffer()->data, b.soupBuffer()->length); + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(data, size, data, fastFree)); + return { WTFMove(buffer) }; +} + +struct MapWrapper { + ~MapWrapper() + { + munmap(map, size); + close(fileDescriptor); + } + + void* map; + size_t size; + int fileDescriptor; +}; + +static void deleteMapWrapper(MapWrapper* wrapper) +{ + delete wrapper; +} + +Data Data::adoptMap(void* map, size_t size, int fd) +{ + ASSERT(map); + ASSERT(map != MAP_FAILED); + MapWrapper* wrapper = new MapWrapper { map, size, fd }; + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(map, size, wrapper, reinterpret_cast<GDestroyNotify>(deleteMapWrapper))); + return { WTFMove(buffer), fd }; +} + +RefPtr<SharedMemory> Data::tryCreateSharedMemory() const +{ + if (isNull() || !isMap()) + return nullptr; + + return SharedMemory::wrapMap(const_cast<char*>(m_buffer->data), m_buffer->length, m_fileDescriptor); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp new file mode 100644 index 000000000..b96bbcd5a --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCache.h" + +#include "Logging.h" +#include "NetworkCacheCoders.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/SharedBuffer.h> +#include <wtf/text/StringBuilder.h> + +#if ENABLE(NETWORK_CACHE) + +namespace WebKit { +namespace NetworkCache { + +Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& buffer, const Vector<std::pair<String, String>>& varyingRequestHeaders) + : m_key(key) + , m_timeStamp(std::chrono::system_clock::now()) + , m_response(response) + , m_varyingRequestHeaders(varyingRequestHeaders) + , m_buffer(WTFMove(buffer)) +{ + ASSERT(m_key.type() == "Resource"); +} + +Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders) + : m_key(key) + , m_timeStamp(std::chrono::system_clock::now()) + , m_response(response) + , m_varyingRequestHeaders(varyingRequestHeaders) +{ + ASSERT(m_key.type() == "Resource"); + + m_redirectRequest.emplace(); + m_redirectRequest->setAsIsolatedCopy(redirectRequest); + // Redirect body is not needed even if exists. + m_redirectRequest->setHTTPBody(nullptr); +} + +Entry::Entry(const Entry& other) + : m_key(other.m_key) + , m_timeStamp(other.m_timeStamp) + , m_response(other.m_response) + , m_varyingRequestHeaders(other.m_varyingRequestHeaders) + , m_redirectRequest(other.m_redirectRequest) + , m_buffer(other.m_buffer) + , m_sourceStorageRecord(other.m_sourceStorageRecord) +{ +} + +Entry::Entry(const Storage::Record& storageEntry) + : m_key(storageEntry.key) + , m_timeStamp(storageEntry.timeStamp) + , m_sourceStorageRecord(storageEntry) +{ + ASSERT(m_key.type() == "Resource"); +} + +Storage::Record Entry::encodeAsStorageRecord() const +{ + WTF::Persistence::Encoder encoder; + encoder << m_response; + + bool hasVaryingRequestHeaders = !m_varyingRequestHeaders.isEmpty(); + encoder << hasVaryingRequestHeaders; + if (hasVaryingRequestHeaders) + encoder << m_varyingRequestHeaders; + + bool isRedirect = !!m_redirectRequest; + encoder << isRedirect; + if (isRedirect) + m_redirectRequest->encodeWithoutPlatformData(encoder); + + encoder.encodeChecksum(); + + Data header(encoder.buffer(), encoder.bufferSize()); + Data body; + if (m_buffer) + body = { reinterpret_cast<const uint8_t*>(m_buffer->data()), m_buffer->size() }; + + return { m_key, m_timeStamp, header, body, { } }; +} + +std::unique_ptr<Entry> Entry::decodeStorageRecord(const Storage::Record& storageEntry) +{ + auto entry = std::make_unique<Entry>(storageEntry); + + WTF::Persistence::Decoder decoder(storageEntry.header.data(), storageEntry.header.size()); + if (!decoder.decode(entry->m_response)) + return nullptr; + entry->m_response.setSource(WebCore::ResourceResponse::Source::DiskCache); + if (storageEntry.bodyHash) + entry->m_response.setCacheBodyKey(*storageEntry.bodyHash); + + bool hasVaryingRequestHeaders; + if (!decoder.decode(hasVaryingRequestHeaders)) + return nullptr; + + if (hasVaryingRequestHeaders) { + if (!decoder.decode(entry->m_varyingRequestHeaders)) + return nullptr; + } + + bool isRedirect; + if (!decoder.decode(isRedirect)) + return nullptr; + + if (isRedirect) { + entry->m_redirectRequest.emplace(); + if (!entry->m_redirectRequest->decodeWithoutPlatformData(decoder)) + return nullptr; + } + + if (!decoder.verifyChecksum()) { + LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n"); + return nullptr; + } + + return entry; +} + +#if ENABLE(SHAREABLE_RESOURCE) +void Entry::initializeShareableResourceHandleFromStorageRecord() const +{ + if (!NetworkCache::singleton().canUseSharedMemoryForBodyData()) + return; + + auto sharedMemory = m_sourceStorageRecord.body.tryCreateSharedMemory(); + if (!sharedMemory) + return; + + auto shareableResource = ShareableResource::create(sharedMemory.releaseNonNull(), 0, m_sourceStorageRecord.body.size()); + shareableResource->createHandle(m_shareableResourceHandle); +} +#endif + +void Entry::initializeBufferFromStorageRecord() const +{ +#if ENABLE(SHAREABLE_RESOURCE) + if (!shareableResourceHandle().isNull()) { + m_buffer = m_shareableResourceHandle.tryWrapInSharedBuffer(); + if (m_buffer) + return; + } +#endif + m_buffer = WebCore::SharedBuffer::create(m_sourceStorageRecord.body.data(), m_sourceStorageRecord.body.size()); +} + +WebCore::SharedBuffer* Entry::buffer() const +{ + if (!m_buffer) + initializeBufferFromStorageRecord(); + + return m_buffer.get(); +} + +#if ENABLE(SHAREABLE_RESOURCE) +ShareableResource::Handle& Entry::shareableResourceHandle() const +{ + if (m_shareableResourceHandle.isNull()) + initializeShareableResourceHandleFromStorageRecord(); + + return m_shareableResourceHandle; +} +#endif + +bool Entry::needsValidation() const +{ + return m_response.source() == WebCore::ResourceResponse::Source::DiskCacheAfterValidation; +} + +void Entry::setNeedsValidation(bool value) +{ + m_response.setSource(value ? WebCore::ResourceResponse::Source::DiskCacheAfterValidation : WebCore::ResourceResponse::Source::DiskCache); +} + +void Entry::asJSON(StringBuilder& json, const Storage::RecordInfo& info) const +{ + json.appendLiteral("{\n"); + json.appendLiteral("\"hash\": "); + json.appendQuotedJSONString(m_key.hashAsString()); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodySize\": "); + json.appendNumber(info.bodySize); + json.appendLiteral(",\n"); + json.appendLiteral("\"worth\": "); + json.appendNumber(info.worth); + json.appendLiteral(",\n"); + json.appendLiteral("\"partition\": "); + json.appendQuotedJSONString(m_key.partition()); + json.appendLiteral(",\n"); + json.appendLiteral("\"timestamp\": "); + json.appendNumber(std::chrono::duration_cast<std::chrono::milliseconds>(m_timeStamp.time_since_epoch()).count()); + json.appendLiteral(",\n"); + json.appendLiteral("\"URL\": "); + json.appendQuotedJSONString(m_response.url().string()); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodyHash\": "); + json.appendQuotedJSONString(info.bodyHash); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodyShareCount\": "); + json.appendNumber(info.bodyShareCount); + json.appendLiteral(",\n"); + json.appendLiteral("\"headers\": {\n"); + bool firstHeader = true; + for (auto& header : m_response.httpHeaderFields()) { + if (!firstHeader) + json.appendLiteral(",\n"); + firstHeader = false; + json.appendLiteral(" "); + json.appendQuotedJSONString(header.key); + json.appendLiteral(": "); + json.appendQuotedJSONString(header.value); + } + json.appendLiteral("\n}\n"); + json.appendLiteral("}"); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h new file mode 100644 index 000000000..c03910894 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheEntry_h +#define NetworkCacheEntry_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheStorage.h" +#include "ShareableResource.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <wtf/Noncopyable.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class SharedBuffer; +} + +namespace WebKit { +namespace NetworkCache { + +class Entry { + WTF_MAKE_FAST_ALLOCATED; +public: + Entry(const Key&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, const Vector<std::pair<String, String>>& varyingRequestHeaders); + Entry(const Key&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders); + explicit Entry(const Storage::Record&); + Entry(const Entry&); + + Storage::Record encodeAsStorageRecord() const; + static std::unique_ptr<Entry> decodeStorageRecord(const Storage::Record&); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; } + const WebCore::ResourceResponse& response() const { return m_response; } + const Vector<std::pair<String, String>>& varyingRequestHeaders() const { return m_varyingRequestHeaders; } + + WebCore::SharedBuffer* buffer() const; + const std::optional<WebCore::ResourceRequest>& redirectRequest() const { return m_redirectRequest; } + +#if ENABLE(SHAREABLE_RESOURCE) + ShareableResource::Handle& shareableResourceHandle() const; +#endif + + bool needsValidation() const; + void setNeedsValidation(bool); + + const Storage::Record& sourceStorageRecord() const { return m_sourceStorageRecord; } + + void asJSON(StringBuilder&, const Storage::RecordInfo&) const; + +private: + void initializeBufferFromStorageRecord() const; +#if ENABLE(SHAREABLE_RESOURCE) + void initializeShareableResourceHandleFromStorageRecord() const; +#endif + + Key m_key; + std::chrono::system_clock::time_point m_timeStamp; + WebCore::ResourceResponse m_response; + Vector<std::pair<String, String>> m_varyingRequestHeaders; + + std::optional<WebCore::ResourceRequest> m_redirectRequest; + mutable RefPtr<WebCore::SharedBuffer> m_buffer; +#if ENABLE(SHAREABLE_RESOURCE) + mutable ShareableResource::Handle m_shareableResourceHandle; +#endif + + Storage::Record m_sourceStorageRecord { }; +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp new file mode 100644 index 000000000..52ac56696 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheFileSystem.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include <WebCore/FileSystem.h> +#include <dirent.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <wtf/Assertions.h> +#include <wtf/Function.h> +#include <wtf/text/CString.h> + +#if PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR) +#include <sys/attr.h> +#include <unistd.h> +#endif + +#if USE(SOUP) +#include <gio/gio.h> +#include <wtf/glib/GRefPtr.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +static DirectoryEntryType directoryEntryType(uint8_t dtype) +{ + switch (dtype) { + case DT_DIR: + return DirectoryEntryType::Directory; + case DT_REG: + return DirectoryEntryType::File; + default: + ASSERT_NOT_REACHED(); + return DirectoryEntryType::File; + } +} + +void traverseDirectory(const String& path, const Function<void (const String&, DirectoryEntryType)>& function) +{ + DIR* dir = opendir(WebCore::fileSystemRepresentation(path).data()); + if (!dir) + return; + dirent* dp; + while ((dp = readdir(dir))) { + if (dp->d_type != DT_DIR && dp->d_type != DT_REG) + continue; + const char* name = dp->d_name; + if (!strcmp(name, ".") || !strcmp(name, "..")) + continue; + auto nameString = String::fromUTF8(name); + if (nameString.isNull()) + continue; + function(nameString, directoryEntryType(dp->d_type)); + } + closedir(dir); +} + +void deleteDirectoryRecursively(const String& path) +{ + traverseDirectory(path, [&path](const String& name, DirectoryEntryType type) { + String entryPath = WebCore::pathByAppendingComponent(path, name); + switch (type) { + case DirectoryEntryType::File: + WebCore::deleteFile(entryPath); + break; + case DirectoryEntryType::Directory: + deleteDirectoryRecursively(entryPath); + break; + // This doesn't follow symlinks. + } + }); + WebCore::deleteEmptyDirectory(path); +} + +FileTimes fileTimes(const String& path) +{ +#if HAVE(STAT_BIRTHTIME) + struct stat fileInfo; + if (stat(WebCore::fileSystemRepresentation(path).data(), &fileInfo)) + return { }; + return { std::chrono::system_clock::from_time_t(fileInfo.st_birthtime), std::chrono::system_clock::from_time_t(fileInfo.st_mtime) }; +#elif USE(SOUP) + // There's no st_birthtime in some operating systems like Linux, so we use xattrs to set/get the creation time. + GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(WebCore::fileSystemRepresentation(path).data())); + GRefPtr<GFileInfo> fileInfo = adoptGRef(g_file_query_info(file.get(), "xattr::birthtime,time::modified", G_FILE_QUERY_INFO_NONE, nullptr, nullptr)); + if (!fileInfo) + return { }; + const char* birthtimeString = g_file_info_get_attribute_string(fileInfo.get(), "xattr::birthtime"); + if (!birthtimeString) + return { }; + return { std::chrono::system_clock::from_time_t(g_ascii_strtoull(birthtimeString, nullptr, 10)), + std::chrono::system_clock::from_time_t(g_file_info_get_attribute_uint64(fileInfo.get(), "time::modified")) }; +#endif +} + +void updateFileModificationTimeIfNeeded(const String& path) +{ + auto times = fileTimes(path); + if (times.creation != times.modification) { + // Don't update more than once per hour. + if (std::chrono::system_clock::now() - times.modification < std::chrono::hours(1)) + return; + } + // This really updates both the access time and the modification time. + utimes(WebCore::fileSystemRepresentation(path).data(), nullptr); +} + +bool canUseSharedMemoryForPath(const String& path) +{ +#if PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR) + struct { + uint32_t length; + uint32_t protectionClass; + } attrBuffer; + + attrlist attrList = { }; + attrList.bitmapcount = ATTR_BIT_MAP_COUNT; + attrList.commonattr = ATTR_CMN_DATA_PROTECT_FLAGS; + int32_t error = getattrlist(WebCore::fileSystemRepresentation(path).data(), &attrList, &attrBuffer, sizeof(attrBuffer), FSOPT_NOFOLLOW); + if (error) { + RELEASE_LOG_ERROR(Network, "Unable to get cache directory protection class, disabling use of shared mapped memory"); + return false; + } + + // For stricter protection classes shared maps could disappear when device is locked. + const uint32_t fileProtectionCompleteUntilFirstUserAuthentication = 3; + bool isSafe = attrBuffer.protectionClass >= fileProtectionCompleteUntilFirstUserAuthentication; + + if (!isSafe) + RELEASE_LOG(Network, "Disallowing use of shared mapped memory due to container protection class %u", attrBuffer.protectionClass); + + return isSafe; +#else + UNUSED_PARAM(path); + return true; +#endif +} + +} +} + +#endif // ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h new file mode 100644 index 000000000..22e716f52 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheFileSystem_h +#define NetworkCacheFileSystem_h + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/FileSystem.h> +#include <functional> + +namespace WebKit { +namespace NetworkCache { + +enum class DirectoryEntryType { Directory, File }; +void traverseDirectory(const String& path, const Function<void (const String& fileName, DirectoryEntryType)>&); + +void deleteDirectoryRecursively(const String& path); + +struct FileTimes { + std::chrono::system_clock::time_point creation; + std::chrono::system_clock::time_point modification; +}; +FileTimes fileTimes(const String& path); +void updateFileModificationTimeIfNeeded(const String& path); + +bool canUseSharedMemoryForPath(const String& path); + +} +} + +#endif + +#endif + diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h new file mode 100644 index 000000000..f19db6fa1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheIOChannel_h +#define NetworkCacheIOChannel_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include <functional> +#include <wtf/ThreadSafeRefCounted.h> +#include <wtf/WorkQueue.h> +#include <wtf/text/WTFString.h> + +#if USE(SOUP) +#include <wtf/glib/GRefPtr.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +class IOChannel : public ThreadSafeRefCounted<IOChannel> { +public: + enum class Type { Read, Write, Create }; + static Ref<IOChannel> open(const String& file, Type); + + // Using nullptr as queue submits the result to the main queue. + // FIXME: We should add WorkQueue::main() instead. + void read(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>); + void write(size_t offset, const Data&, WorkQueue*, std::function<void (int error)>); + + const String& path() const { return m_path; } + Type type() const { return m_type; } + + int fileDescriptor() const { return m_fileDescriptor; } + + ~IOChannel(); + +private: + IOChannel(const String& filePath, IOChannel::Type); + +#if USE(SOUP) + void readSyncInThread(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>); +#endif + + String m_path; + Type m_type; + + int m_fileDescriptor { 0 }; + std::atomic<bool> m_wasDeleted { false }; // Try to narrow down a crash, https://bugs.webkit.org/show_bug.cgi?id=165659 +#if PLATFORM(COCOA) + DispatchPtr<dispatch_io_t> m_dispatchIO; +#endif +#if USE(SOUP) + GRefPtr<GInputStream> m_inputStream; + GRefPtr<GOutputStream> m_outputStream; + GRefPtr<GFileIOStream> m_ioStream; +#endif +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp new file mode 100644 index 000000000..ef7ba8998 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp @@ -0,0 +1,285 @@ +/* + * Copyright (C) 2015 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheIOChannel.h" + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheFileSystem.h" +#include <wtf/MainThread.h> +#include <wtf/RunLoop.h> +#include <wtf/glib/GUniquePtr.h> + +namespace WebKit { +namespace NetworkCache { + +static const size_t gDefaultReadBufferSize = 4096; + +IOChannel::IOChannel(const String& filePath, Type type) + : m_path(filePath) + , m_type(type) +{ + auto path = WebCore::fileSystemRepresentation(filePath); + GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(path.data())); + switch (m_type) { + case Type::Create: { + g_file_delete(file.get(), nullptr, nullptr); + m_outputStream = adoptGRef(G_OUTPUT_STREAM(g_file_create(file.get(), static_cast<GFileCreateFlags>(G_FILE_CREATE_PRIVATE), nullptr, nullptr))); +#if !HAVE(STAT_BIRTHTIME) + GUniquePtr<char> birthtimeString(g_strdup_printf("%" G_GUINT64_FORMAT, std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()))); + g_file_set_attribute_string(file.get(), "xattr::birthtime", birthtimeString.get(), G_FILE_QUERY_INFO_NONE, nullptr, nullptr); +#endif + break; + } + case Type::Write: { + m_ioStream = adoptGRef(g_file_open_readwrite(file.get(), nullptr, nullptr)); + break; + } + case Type::Read: + m_inputStream = adoptGRef(G_INPUT_STREAM(g_file_read(file.get(), nullptr, nullptr))); + break; + } +} + +IOChannel::~IOChannel() +{ + RELEASE_ASSERT(!m_wasDeleted.exchange(true)); +} + +Ref<IOChannel> IOChannel::open(const String& filePath, IOChannel::Type type) +{ + return adoptRef(*new IOChannel(filePath, type)); +} + +static inline void runTaskInQueue(Function<void ()>&& task, WorkQueue* queue) +{ + if (queue) { + queue->dispatch(WTFMove(task)); + return; + } + + // Using nullptr as queue submits the result to the main context. + RunLoop::main().dispatch(WTFMove(task)); +} + +static void fillDataFromReadBuffer(SoupBuffer* readBuffer, size_t size, Data& data) +{ + GRefPtr<SoupBuffer> buffer; + if (size != readBuffer->length) { + // The subbuffer does not copy the data. + buffer = adoptGRef(soup_buffer_new_subbuffer(readBuffer, 0, size)); + } else + buffer = readBuffer; + + if (data.isNull()) { + // First chunk, we need to force the data to be copied. + data = { reinterpret_cast<const uint8_t*>(buffer->data), size }; + } else { + Data dataRead(WTFMove(buffer)); + // Concatenate will copy the data. + data = concatenate(data, dataRead); + } +} + +struct ReadAsyncData { + RefPtr<IOChannel> channel; + GRefPtr<SoupBuffer> buffer; + RefPtr<WorkQueue> queue; + size_t bytesToRead; + std::function<void (Data&, int error)> completionHandler; + Data data; +}; + +static void inputStreamReadReadyCallback(GInputStream* stream, GAsyncResult* result, gpointer userData) +{ + std::unique_ptr<ReadAsyncData> asyncData(static_cast<ReadAsyncData*>(userData)); + gssize bytesRead = g_input_stream_read_finish(stream, result, nullptr); + if (bytesRead == -1) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(asyncData->data, -1); + }, queue); + return; + } + + if (!bytesRead) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(asyncData->data, 0); + }, queue); + return; + } + + ASSERT(bytesRead > 0); + fillDataFromReadBuffer(asyncData->buffer.get(), static_cast<size_t>(bytesRead), asyncData->data); + + size_t pendingBytesToRead = asyncData->bytesToRead - asyncData->data.size(); + if (!pendingBytesToRead) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(asyncData->data, 0); + }, queue); + return; + } + + size_t bytesToRead = std::min(pendingBytesToRead, asyncData->buffer->length); + // Use a local variable for the data buffer to pass it to g_input_stream_read_async(), because ReadAsyncData is released. + auto data = const_cast<char*>(asyncData->buffer->data); + g_input_stream_read_async(stream, data, bytesToRead, G_PRIORITY_DEFAULT, nullptr, + reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData.release()); +} + +void IOChannel::read(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler) +{ + RefPtr<IOChannel> channel(this); + if (!m_inputStream) { + runTaskInQueue([channel, completionHandler] { + Data data; + completionHandler(data, -1); + }, queue); + return; + } + + if (!isMainThread()) { + readSyncInThread(offset, size, queue, completionHandler); + return; + } + + size_t bufferSize = std::min(size, gDefaultReadBufferSize); + uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize)); + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree)); + ReadAsyncData* asyncData = new ReadAsyncData { this, buffer.get(), queue, size, completionHandler, { } }; + + // FIXME: implement offset. + g_input_stream_read_async(m_inputStream.get(), const_cast<char*>(buffer->data), bufferSize, G_PRIORITY_DEFAULT, nullptr, + reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData); +} + +void IOChannel::readSyncInThread(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler) +{ + ASSERT(!isMainThread()); + + RefPtr<IOChannel> channel(this); + detachThread(createThread("IOChannel::readSync", [channel, size, queue, completionHandler] { + size_t bufferSize = std::min(size, gDefaultReadBufferSize); + uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize)); + GRefPtr<SoupBuffer> readBuffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree)); + Data data; + size_t pendingBytesToRead = size; + size_t bytesToRead = bufferSize; + do { + // FIXME: implement offset. + gssize bytesRead = g_input_stream_read(channel->m_inputStream.get(), const_cast<char*>(readBuffer->data), bytesToRead, nullptr, nullptr); + if (bytesRead == -1) { + runTaskInQueue([channel, completionHandler] { + Data data; + completionHandler(data, -1); + }, queue); + return; + } + + if (!bytesRead) + break; + + ASSERT(bytesRead > 0); + fillDataFromReadBuffer(readBuffer.get(), static_cast<size_t>(bytesRead), data); + + pendingBytesToRead = size - data.size(); + bytesToRead = std::min(pendingBytesToRead, readBuffer->length); + } while (pendingBytesToRead); + + GRefPtr<SoupBuffer> bufferCapture = data.soupBuffer(); + runTaskInQueue([channel, bufferCapture, completionHandler] { + GRefPtr<SoupBuffer> buffer = bufferCapture; + Data data = { WTFMove(buffer) }; + completionHandler(data, 0); + }, queue); + })); +} + +struct WriteAsyncData { + RefPtr<IOChannel> channel; + GRefPtr<SoupBuffer> buffer; + RefPtr<WorkQueue> queue; + std::function<void (int error)> completionHandler; +}; + +static void outputStreamWriteReadyCallback(GOutputStream* stream, GAsyncResult* result, gpointer userData) +{ + std::unique_ptr<WriteAsyncData> asyncData(static_cast<WriteAsyncData*>(userData)); + gssize bytesWritten = g_output_stream_write_finish(stream, result, nullptr); + if (bytesWritten == -1) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(-1); + }, queue); + return; + } + + gssize pendingBytesToWrite = asyncData->buffer->length - bytesWritten; + if (!pendingBytesToWrite) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(0); + }, queue); + return; + } + + asyncData->buffer = adoptGRef(soup_buffer_new_subbuffer(asyncData->buffer.get(), bytesWritten, pendingBytesToWrite)); + // Use a local variable for the data buffer to pass it to g_output_stream_write_async(), because WriteAsyncData is released. + auto data = asyncData->buffer->data; + g_output_stream_write_async(stream, data, pendingBytesToWrite, G_PRIORITY_DEFAULT_IDLE, nullptr, + reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData.release()); +} + +void IOChannel::write(size_t offset, const Data& data, WorkQueue* queue, std::function<void (int error)> completionHandler) +{ + RefPtr<IOChannel> channel(this); + if (!m_outputStream && !m_ioStream) { + runTaskInQueue([channel, completionHandler] { + completionHandler(-1); + }, queue); + return; + } + + GOutputStream* stream = m_outputStream ? m_outputStream.get() : g_io_stream_get_output_stream(G_IO_STREAM(m_ioStream.get())); + if (!stream) { + runTaskInQueue([channel, completionHandler] { + completionHandler(-1); + }, queue); + return; + } + + WriteAsyncData* asyncData = new WriteAsyncData { this, data.soupBuffer(), queue, completionHandler }; + // FIXME: implement offset. + g_output_stream_write_async(stream, asyncData->buffer->data, data.size(), G_PRIORITY_DEFAULT_IDLE, nullptr, + reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp new file mode 100644 index 000000000..cce50a7d0 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheKey.h" + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheCoders.h" +#include <wtf/ASCIICType.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/persistence/Decoder.h> +#include <wtf/persistence/Encoder.h> +#include <wtf/text/CString.h> +#include <wtf/text/StringBuilder.h> + +namespace WebKit { +namespace NetworkCache { + +Key::Key(const Key& o) + : m_partition(o.m_partition.isolatedCopy()) + , m_type(o.m_type.isolatedCopy()) + , m_identifier(o.m_identifier.isolatedCopy()) + , m_range(o.m_range.isolatedCopy()) + , m_hash(o.m_hash) + , m_partitionHash(o.m_partitionHash) +{ +} + +Key::Key(const String& partition, const String& type, const String& range, const String& identifier, const Salt& salt) + : m_partition(partition) + , m_type(type) + , m_identifier(identifier) + , m_range(range) + , m_hash(computeHash(salt)) + , m_partitionHash(computePartitionHash(salt)) +{ +} + +Key::Key(WTF::HashTableDeletedValueType) + : m_identifier(WTF::HashTableDeletedValue) +{ +} + +Key::Key(const DataKey& dataKey, const Salt& salt) + : m_partition(dataKey.partition) + , m_type(dataKey.type) + , m_identifier(hashAsString(dataKey.identifier)) + , m_hash(computeHash(salt)) + , m_partitionHash(computePartitionHash(salt)) +{ +} + +Key& Key::operator=(const Key& other) +{ + m_partition = other.m_partition.isolatedCopy(); + m_type = other.m_type.isolatedCopy(); + m_identifier = other.m_identifier.isolatedCopy(); + m_range = other.m_range.isolatedCopy(); + m_hash = other.m_hash; + m_partitionHash = other.m_partitionHash; + return *this; +} + +static void hashString(SHA1& sha1, const String& string) +{ + if (string.isNull()) + return; + + if (string.is8Bit() && string.containsOnlyASCII()) { + const uint8_t nullByte = 0; + sha1.addBytes(string.characters8(), string.length()); + sha1.addBytes(&nullByte, 1); + return; + } + auto cString = string.utf8(); + // Include terminating null byte. + sha1.addBytes(reinterpret_cast<const uint8_t*>(cString.data()), cString.length() + 1); +} + +Key::HashType Key::computeHash(const Salt& salt) const +{ + // We don't really need a cryptographic hash. The key is always verified against the entry header. + // SHA1 just happens to be suitably sized, fast and available. + SHA1 sha1; + sha1.addBytes(salt.data(), salt.size()); + + hashString(sha1, m_partition); + hashString(sha1, m_type); + hashString(sha1, m_identifier); + hashString(sha1, m_range); + + SHA1::Digest hash; + sha1.computeHash(hash); + return hash; +} + +Key::HashType Key::computePartitionHash(const Salt& salt) const +{ + SHA1 sha1; + sha1.addBytes(salt.data(), salt.size()); + + hashString(sha1, m_partition); + + SHA1::Digest hash; + sha1.computeHash(hash); + return hash; +} + +String Key::hashAsString(const HashType& hash) +{ + StringBuilder builder; + builder.reserveCapacity(hashStringLength()); + for (auto byte : hash) { + builder.append(upperNibbleToASCIIHexDigit(byte)); + builder.append(lowerNibbleToASCIIHexDigit(byte)); + } + return builder.toString(); +} + +template <typename CharType> bool hexDigitsToHash(CharType* characters, Key::HashType& hash) +{ + for (unsigned i = 0; i < sizeof(hash); ++i) { + auto high = characters[2 * i]; + auto low = characters[2 * i + 1]; + if (!isASCIIHexDigit(high) || !isASCIIHexDigit(low)) + return false; + hash[i] = toASCIIHexValue(high, low); + } + return true; +} + +bool Key::stringToHash(const String& string, HashType& hash) +{ + if (string.length() != hashStringLength()) + return false; + if (string.is8Bit()) + return hexDigitsToHash(string.characters8(), hash); + return hexDigitsToHash(string.characters16(), hash); +} + +bool Key::operator==(const Key& other) const +{ + return m_hash == other.m_hash && m_partition == other.m_partition && m_type == other.m_type && m_identifier == other.m_identifier && m_range == other.m_range; +} + +void Key::encode(WTF::Persistence::Encoder& encoder) const +{ + encoder << m_partition; + encoder << m_type; + encoder << m_identifier; + encoder << m_range; + encoder << m_hash; + encoder << m_partitionHash; +} + +bool Key::decode(WTF::Persistence::Decoder& decoder, Key& key) +{ + return decoder.decode(key.m_partition) && decoder.decode(key.m_type) && decoder.decode(key.m_identifier) && decoder.decode(key.m_range) && decoder.decode(key.m_hash) && decoder.decode(key.m_partitionHash); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h new file mode 100644 index 000000000..439ebf0f7 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheKey_h +#define NetworkCacheKey_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include <wtf/SHA1.h> +#include <wtf/persistence/Coder.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { +namespace NetworkCache { + +struct DataKey { + String partition; + String type; + SHA1::Digest identifier; + + template <class Encoder> void encode(Encoder& encoder) const + { + encoder << partition << type << identifier; + } + + template <class Decoder> static bool decode(Decoder& decoder, DataKey& dataKey) + { + return decoder.decode(dataKey.partition) && decoder.decode(dataKey.type) && decoder.decode(dataKey.identifier); + } +}; + +class Key { +public: + typedef SHA1::Digest HashType; + + Key() { } + Key(const Key&); + Key(Key&&) = default; + Key(const String& partition, const String& type, const String& range, const String& identifier, const Salt&); + Key(const DataKey&, const Salt&); + + Key& operator=(const Key&); + Key& operator=(Key&&) = default; + + Key(WTF::HashTableDeletedValueType); + bool isHashTableDeletedValue() const { return m_identifier.isHashTableDeletedValue(); } + + bool isNull() const { return m_identifier.isNull(); } + + const String& partition() const { return m_partition; } + const String& identifier() const { return m_identifier; } + const String& type() const { return m_type; } + const String& range() const { return m_range; } + + const HashType& hash() const { return m_hash; } + const HashType& partitionHash() const { return m_partitionHash; } + + static bool stringToHash(const String&, HashType&); + + static size_t hashStringLength() { return 2 * sizeof(m_hash); } + String hashAsString() const { return hashAsString(m_hash); } + String partitionHashAsString() const { return hashAsString(m_partitionHash); } + + void encode(WTF::Persistence::Encoder&) const; + static bool decode(WTF::Persistence::Decoder&, Key&); + + bool operator==(const Key&) const; + bool operator!=(const Key& other) const { return !(*this == other); } + +private: + static String hashAsString(const HashType&); + HashType computeHash(const Salt&) const; + HashType computePartitionHash(const Salt&) const; + + String m_partition; + String m_type; + String m_identifier; + String m_range; + HashType m_hash; + HashType m_partitionHash; +}; + +} +} + +namespace WTF { + +struct NetworkCacheKeyHash { + static unsigned hash(const WebKit::NetworkCache::Key& key) + { + static_assert(SHA1::hashSize >= sizeof(unsigned), "Hash size must be greater than sizeof(unsigned)"); + return *reinterpret_cast<const unsigned*>(key.hash().data()); + } + + static bool equal(const WebKit::NetworkCache::Key& a, const WebKit::NetworkCache::Key& b) + { + return a == b; + } + + static const bool safeToCompareToEmptyOrDeleted = false; +}; + +template<typename T> struct DefaultHash; +template<> struct DefaultHash<WebKit::NetworkCache::Key> { + typedef NetworkCacheKeyHash Hash; +}; + +template<> struct HashTraits<WebKit::NetworkCache::Key> : SimpleClassHashTraits<WebKit::NetworkCache::Key> { + static const bool emptyValueIsZero = false; + + static const bool hasIsEmptyValueFunction = true; + static bool isEmptyValue(const WebKit::NetworkCache::Key& key) { return key.isNull(); } +}; + +} // namespace WTF + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp new file mode 100644 index 000000000..6db85d2de --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSpeculativeLoad.h" + +#include "Logging.h" +#include "NetworkCache.h" +#include "NetworkLoad.h" +#include "NetworkSession.h" +#include <WebCore/SessionID.h> +#include <wtf/CurrentTime.h> +#include <wtf/RunLoop.h> + +namespace WebKit { +namespace NetworkCache { + +using namespace WebCore; + +SpeculativeLoad::SpeculativeLoad(const GlobalFrameID& frameID, const ResourceRequest& request, std::unique_ptr<NetworkCache::Entry> cacheEntryForValidation, RevalidationCompletionHandler&& completionHandler) + : m_frameID(frameID) + , m_completionHandler(WTFMove(completionHandler)) + , m_originalRequest(request) + , m_bufferedDataForCache(SharedBuffer::create()) + , m_cacheEntry(WTFMove(cacheEntryForValidation)) +{ + ASSERT(!m_cacheEntry || m_cacheEntry->needsValidation()); + + NetworkLoadParameters parameters; + parameters.sessionID = SessionID::defaultSessionID(); + parameters.allowStoredCredentials = AllowStoredCredentials; + parameters.contentSniffingPolicy = DoNotSniffContent; + parameters.request = m_originalRequest; +#if USE(NETWORK_SESSION) + m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters), NetworkSession::defaultSession()); +#else + m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters)); +#endif +} + +SpeculativeLoad::~SpeculativeLoad() +{ + ASSERT(!m_networkLoad); +} + +void SpeculativeLoad::willSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse) +{ + LOG(NetworkCacheSpeculativePreloading, "Speculative redirect %s -> %s", request.url().string().utf8().data(), redirectRequest.url().string().utf8().data()); + + m_cacheEntry = NetworkCache::singleton().storeRedirect(request, redirectResponse, redirectRequest); + // Create a synthetic cache entry if we can't store. + if (!m_cacheEntry) + m_cacheEntry = NetworkCache::singleton().makeRedirectEntry(request, redirectResponse, redirectRequest); + + // Don't follow the redirect. The redirect target will be registered for speculative load when it is loaded. + didComplete(); +} + +auto SpeculativeLoad::didReceiveResponse(ResourceResponse&& receivedResponse) -> ShouldContinueDidReceiveResponse +{ + m_response = receivedResponse; + + if (m_response.isMultipart()) + m_bufferedDataForCache = nullptr; + + bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified + if (validationSucceeded && m_cacheEntry) + m_cacheEntry = NetworkCache::singleton().update(m_originalRequest, m_frameID, *m_cacheEntry, m_response); + else + m_cacheEntry = nullptr; + + return ShouldContinueDidReceiveResponse::Yes; +} + +void SpeculativeLoad::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength) +{ + ASSERT(!m_cacheEntry); + + if (m_bufferedDataForCache) { + // Prevent memory growth in case of streaming data. + const size_t maximumCacheBufferSize = 10 * 1024 * 1024; + if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize) + m_bufferedDataForCache->append(buffer.get()); + else + m_bufferedDataForCache = nullptr; + } +} + +void SpeculativeLoad::didFinishLoading(double finishTime) +{ + if (m_didComplete) + return; + if (!m_cacheEntry && m_bufferedDataForCache) { + m_cacheEntry = NetworkCache::singleton().store(m_originalRequest, m_response, m_bufferedDataForCache.copyRef(), [](auto& mappedBody) { }); + // Create a synthetic cache entry if we can't store. + if (!m_cacheEntry && isStatusCodeCacheableByDefault(m_response.httpStatusCode())) + m_cacheEntry = NetworkCache::singleton().makeEntry(m_originalRequest, m_response, WTFMove(m_bufferedDataForCache)); + } + + didComplete(); +} + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void SpeculativeLoad::canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) +{ + m_networkLoad->continueCanAuthenticateAgainstProtectionSpace(false); +} +#endif + +void SpeculativeLoad::didFailLoading(const ResourceError&) +{ + if (m_didComplete) + return; + m_cacheEntry = nullptr; + + didComplete(); +} + +void SpeculativeLoad::didComplete() +{ + RELEASE_ASSERT(RunLoop::isMain()); + + if (m_didComplete) + return; + m_didComplete = true; + m_networkLoad = nullptr; + + // Make sure speculatively revalidated resources do not get validated by the NetworkResourceLoader again. + if (m_cacheEntry) + m_cacheEntry->setNeedsValidation(false); + + m_completionHandler(WTFMove(m_cacheEntry)); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h new file mode 100644 index 000000000..3bc2d2e65 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSpeculativeLoad_h +#define NetworkCacheSpeculativeLoad_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCache.h" +#include "NetworkCacheEntry.h" +#include "NetworkLoadClient.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> + +namespace WebKit { + +class NetworkLoad; + +namespace NetworkCache { + +class SpeculativeLoad final : public NetworkLoadClient { + WTF_MAKE_FAST_ALLOCATED; +public: + typedef std::function<void (std::unique_ptr<NetworkCache::Entry>)> RevalidationCompletionHandler; + SpeculativeLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, std::unique_ptr<NetworkCache::Entry>, RevalidationCompletionHandler&&); + + virtual ~SpeculativeLoad(); + + const WebCore::ResourceRequest& originalRequest() const { return m_originalRequest; } + +private: + // NetworkLoadClient. + void didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override { } +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) override; +#endif + bool isSynchronous() const override { return false; } + void willSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceRequest&& redirectRequest, WebCore::ResourceResponse&& redirectResponse) override; + ShouldContinueDidReceiveResponse didReceiveResponse(WebCore::ResourceResponse&&) override; + void didReceiveBuffer(Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) override; + void didFinishLoading(double finishTime) override; + void didFailLoading(const WebCore::ResourceError&) override; + + void didComplete(); + + GlobalFrameID m_frameID; + RevalidationCompletionHandler m_completionHandler; + WebCore::ResourceRequest m_originalRequest; + + std::unique_ptr<NetworkLoad> m_networkLoad; + + WebCore::ResourceResponse m_response; + + RefPtr<WebCore::SharedBuffer> m_bufferedDataForCache; + std::unique_ptr<NetworkCache::Entry> m_cacheEntry; + bool m_didComplete { false }; +}; + +} // namespace NetworkCache +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#endif // NetworkCacheSpeculativeLoad_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp new file mode 100644 index 000000000..db36bb8eb --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp @@ -0,0 +1,589 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSpeculativeLoadManager.h" + +#include "Logging.h" +#include "NetworkCacheEntry.h" +#include "NetworkCacheSpeculativeLoad.h" +#include "NetworkCacheSubresourcesEntry.h" +#include "NetworkProcess.h" +#include <WebCore/DiagnosticLoggingKeys.h> +#include <WebCore/HysteresisActivity.h> +#include <wtf/HashCountedSet.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/RefCounted.h> +#include <wtf/RunLoop.h> + +namespace WebKit { + +namespace NetworkCache { + +using namespace WebCore; + +static const auto preloadedEntryLifetime = 10s; + +#if !LOG_DISABLED +static HashCountedSet<String>& allSpeculativeLoadingDiagnosticMessages() +{ + static NeverDestroyed<HashCountedSet<String>> messages; + return messages; +} + +static void printSpeculativeLoadingDiagnosticMessageCounts() +{ + LOG(NetworkCacheSpeculativePreloading, "-- Speculative loading statistics --"); + for (auto& pair : allSpeculativeLoadingDiagnosticMessages()) + LOG(NetworkCacheSpeculativePreloading, "%s: %u", pair.key.utf8().data(), pair.value); +} +#endif + +static void logSpeculativeLoadingDiagnosticMessage(const GlobalFrameID& frameID, const String& message) +{ +#if !LOG_DISABLED + if (WebKit2LogNetworkCacheSpeculativePreloading.state == WTFLogChannelOn) + allSpeculativeLoadingDiagnosticMessages().add(message); +#endif + NetworkProcess::singleton().logDiagnosticMessage(frameID.first, WebCore::DiagnosticLoggingKeys::networkCacheKey(), message, WebCore::ShouldSample::Yes); +} + +static const AtomicString& subresourcesType() +{ + ASSERT(RunLoop::isMain()); + static NeverDestroyed<const AtomicString> resource("SubResources", AtomicString::ConstructFromLiteral); + return resource; +} + +static inline Key makeSubresourcesKey(const Key& resourceKey, const Salt& salt) +{ + return Key(resourceKey.partition(), subresourcesType(), resourceKey.range(), resourceKey.identifier(), salt); +} + +static inline ResourceRequest constructRevalidationRequest(const Key& key, const SubresourceInfo& subResourceInfo, const Entry* entry) +{ + ResourceRequest revalidationRequest(key.identifier()); + revalidationRequest.setHTTPHeaderFields(subResourceInfo.requestHeaders()); + revalidationRequest.setFirstPartyForCookies(subResourceInfo.firstPartyForCookies()); + if (!key.partition().isEmpty()) + revalidationRequest.setCachePartition(key.partition()); + ASSERT_WITH_MESSAGE(key.range().isEmpty(), "range is not supported"); + + revalidationRequest.makeUnconditional(); + if (entry) { + String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag); + if (!eTag.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag); + + String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified); + if (!lastModified.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified); + } + + revalidationRequest.setPriority(subResourceInfo.priority()); + + return revalidationRequest; +} + +static bool responseNeedsRevalidation(const ResourceResponse& response, std::chrono::system_clock::time_point timestamp) +{ + if (response.cacheControlContainsNoCache()) + return true; + + auto age = computeCurrentAge(response, timestamp); + auto lifetime = computeFreshnessLifetimeForHTTPFamily(response, timestamp); + return age - lifetime > 0ms; +} + +class SpeculativeLoadManager::ExpiringEntry { + WTF_MAKE_FAST_ALLOCATED; +public: + explicit ExpiringEntry(std::function<void()>&& expirationHandler) + : m_lifetimeTimer(WTFMove(expirationHandler)) + { + m_lifetimeTimer.startOneShot(preloadedEntryLifetime); + } + +private: + Timer m_lifetimeTimer; +}; + +class SpeculativeLoadManager::PreloadedEntry : private ExpiringEntry { + WTF_MAKE_FAST_ALLOCATED; +public: + PreloadedEntry(std::unique_ptr<Entry> entry, std::optional<ResourceRequest>&& speculativeValidationRequest, std::function<void()>&& lifetimeReachedHandler) + : ExpiringEntry(WTFMove(lifetimeReachedHandler)) + , m_entry(WTFMove(entry)) + , m_speculativeValidationRequest(WTFMove(speculativeValidationRequest)) + { } + + std::unique_ptr<Entry> takeCacheEntry() + { + ASSERT(m_entry); + return WTFMove(m_entry); + } + + const std::optional<ResourceRequest>& revalidationRequest() const { return m_speculativeValidationRequest; } + bool wasRevalidated() const { return !!m_speculativeValidationRequest; } + +private: + std::unique_ptr<Entry> m_entry; + std::optional<ResourceRequest> m_speculativeValidationRequest; +}; + +class SpeculativeLoadManager::PendingFrameLoad : public RefCounted<PendingFrameLoad> { +public: + static Ref<PendingFrameLoad> create(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler) + { + return adoptRef(*new PendingFrameLoad(storage, mainResourceKey, WTFMove(loadCompletionHandler))); + } + + ~PendingFrameLoad() + { + ASSERT(m_didFinishLoad); + ASSERT(m_didRetrieveExistingEntry); + } + + void registerSubresourceLoad(const ResourceRequest& request, const Key& subresourceKey) + { + ASSERT(RunLoop::isMain()); + m_subresourceLoads.append(std::make_unique<SubresourceLoad>(request, subresourceKey)); + m_loadHysteresisActivity.impulse(); + } + + void markLoadAsCompleted() + { + ASSERT(RunLoop::isMain()); + if (m_didFinishLoad) + return; + +#if !LOG_DISABLED + printSpeculativeLoadingDiagnosticMessageCounts(); +#endif + + m_didFinishLoad = true; + saveToDiskIfReady(); + m_loadCompletionHandler(); + } + + void setExistingSubresourcesEntry(std::unique_ptr<SubresourcesEntry> entry) + { + ASSERT(!m_existingEntry); + ASSERT(!m_didRetrieveExistingEntry); + + m_existingEntry = WTFMove(entry); + m_didRetrieveExistingEntry = true; + saveToDiskIfReady(); + } + +private: + PendingFrameLoad(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler) + : m_storage(storage) + , m_mainResourceKey(mainResourceKey) + , m_loadCompletionHandler(WTFMove(loadCompletionHandler)) + , m_loadHysteresisActivity([this](HysteresisState state) { if (state == HysteresisState::Stopped) markLoadAsCompleted(); }) + { + m_loadHysteresisActivity.impulse(); + } + + void saveToDiskIfReady() + { + if (!m_didFinishLoad || !m_didRetrieveExistingEntry) + return; + + if (m_subresourceLoads.isEmpty()) + return; + +#if !LOG_DISABLED + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Saving to disk list of subresources for '%s':", m_mainResourceKey.identifier().utf8().data()); + for (auto& subresourceLoad : m_subresourceLoads) + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) * Subresource: '%s'.", subresourceLoad->key.identifier().utf8().data()); +#endif + + if (m_existingEntry) { + m_existingEntry->updateSubresourceLoads(m_subresourceLoads); + m_storage.store(m_existingEntry->encodeAsStorageRecord(), [](const Data&) { }); + } else { + SubresourcesEntry entry(makeSubresourcesKey(m_mainResourceKey, m_storage.salt()), m_subresourceLoads); + m_storage.store(entry.encodeAsStorageRecord(), [](const Data&) { }); + } + } + + Storage& m_storage; + Key m_mainResourceKey; + Vector<std::unique_ptr<SubresourceLoad>> m_subresourceLoads; + std::function<void()> m_loadCompletionHandler; + HysteresisActivity m_loadHysteresisActivity; + std::unique_ptr<SubresourcesEntry> m_existingEntry; + bool m_didFinishLoad { false }; + bool m_didRetrieveExistingEntry { false }; +}; + +SpeculativeLoadManager::SpeculativeLoadManager(Storage& storage) + : m_storage(storage) +{ +} + +SpeculativeLoadManager::~SpeculativeLoadManager() +{ +} + +#if !LOG_DISABLED + +static void dumpHTTPHeadersDiff(const HTTPHeaderMap& headersA, const HTTPHeaderMap& headersB) +{ + auto aEnd = headersA.end(); + for (auto it = headersA.begin(); it != aEnd; ++it) { + String valueB = headersB.get(it->key); + if (valueB.isNull()) + LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header is only in first request (value: %s)", it->key.utf8().data(), it->value.utf8().data()); + else if (it->value != valueB) + LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header differs in both requests: %s != %s", it->key.utf8().data(), it->value.utf8().data(), valueB.utf8().data()); + } + auto bEnd = headersB.end(); + for (auto it = headersB.begin(); it != bEnd; ++it) { + if (!headersA.contains(it->key)) + LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header is only in second request (value: %s)", it->key.utf8().data(), it->value.utf8().data()); + } +} + +#endif + +static bool requestsHeadersMatch(const ResourceRequest& speculativeValidationRequest, const ResourceRequest& actualRequest) +{ + ASSERT(!actualRequest.isConditional()); + ResourceRequest speculativeRequest = speculativeValidationRequest; + speculativeRequest.makeUnconditional(); + + if (speculativeRequest.httpHeaderFields() != actualRequest.httpHeaderFields()) { + LOG(NetworkCacheSpeculativePreloading, "Cannot reuse speculatively validated entry because HTTP headers used for validation do not match"); +#if !LOG_DISABLED + dumpHTTPHeadersDiff(speculativeRequest.httpHeaderFields(), actualRequest.httpHeaderFields()); +#endif + return false; + } + return true; +} + +bool SpeculativeLoadManager::canUsePreloadedEntry(const PreloadedEntry& entry, const ResourceRequest& actualRequest) +{ + if (!entry.wasRevalidated()) + return true; + + ASSERT(entry.revalidationRequest()); + return requestsHeadersMatch(*entry.revalidationRequest(), actualRequest); +} + +bool SpeculativeLoadManager::canUsePendingPreload(const SpeculativeLoad& load, const ResourceRequest& actualRequest) +{ + return requestsHeadersMatch(load.originalRequest(), actualRequest); +} + +bool SpeculativeLoadManager::canRetrieve(const Key& storageKey, const WebCore::ResourceRequest& request, const GlobalFrameID& frameID) const +{ + // Check already preloaded entries. + if (auto preloadedEntry = m_preloadedEntries.get(storageKey)) { + if (!canUsePreloadedEntry(*preloadedEntry, request)) { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: Could not use preloaded entry to satisfy request for '%s' due to HTTP headers mismatch:", storageKey.identifier().utf8().data()); + logSpeculativeLoadingDiagnosticMessage(frameID, preloadedEntry->wasRevalidated() ? DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey() : DiagnosticLoggingKeys::wastedSpeculativeWarmupWithoutRevalidationKey()); + return false; + } + + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: Using preloaded entry to satisfy request for '%s':", storageKey.identifier().utf8().data()); + logSpeculativeLoadingDiagnosticMessage(frameID, preloadedEntry->wasRevalidated() ? DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey() : DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey()); + return true; + } + + // Check pending speculative revalidations. + auto* pendingPreload = m_pendingPreloads.get(storageKey); + if (!pendingPreload) { + if (m_notPreloadedEntries.get(storageKey)) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryWronglyNotWarmedUpKey()); + else + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::unknownEntryRequestKey()); + + return false; + } + + if (!canUsePendingPreload(*pendingPreload, request)) { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: revalidation already in progress for '%s' but unusable due to HTTP headers mismatch:", storageKey.identifier().utf8().data()); + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey()); + return false; + } + + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: revalidation already in progress for '%s':", storageKey.identifier().utf8().data()); + + return true; +} + +void SpeculativeLoadManager::retrieve(const Key& storageKey, RetrieveCompletionHandler&& completionHandler) +{ + if (auto preloadedEntry = m_preloadedEntries.take(storageKey)) { + RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), cacheEntry = preloadedEntry->takeCacheEntry()] () mutable { + completionHandler(WTFMove(cacheEntry)); + }); + return; + } + ASSERT(m_pendingPreloads.contains(storageKey)); + // FIXME: This breaks incremental loading when the revalidation is not successful. + auto addResult = m_pendingRetrieveRequests.ensure(storageKey, [] { + return std::make_unique<Vector<RetrieveCompletionHandler>>(); + }); + addResult.iterator->value->append(WTFMove(completionHandler)); +} + +void SpeculativeLoadManager::registerLoad(const GlobalFrameID& frameID, const ResourceRequest& request, const Key& resourceKey) +{ + ASSERT(RunLoop::isMain()); + ASSERT(request.url().protocolIsInHTTPFamily()); + + if (request.httpMethod() != "GET") + return; + + auto isMainResource = request.requester() == ResourceRequest::Requester::Main; + if (isMainResource) { + // Mark previous load in this frame as completed if necessary. + if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID)) + pendingFrameLoad->markLoadAsCompleted(); + + ASSERT(!m_pendingFrameLoads.contains(frameID)); + + // Start tracking loads in this frame. + RefPtr<PendingFrameLoad> pendingFrameLoad = PendingFrameLoad::create(m_storage, resourceKey, [this, frameID] { + bool wasRemoved = m_pendingFrameLoads.remove(frameID); + ASSERT_UNUSED(wasRemoved, wasRemoved); + }); + m_pendingFrameLoads.add(frameID, pendingFrameLoad); + + // Retrieve the subresources entry if it exists to start speculative revalidation and to update it. + retrieveSubresourcesEntry(resourceKey, [this, frameID, pendingFrameLoad](std::unique_ptr<SubresourcesEntry> entry) { + if (entry) + startSpeculativeRevalidation(frameID, *entry); + + pendingFrameLoad->setExistingSubresourcesEntry(WTFMove(entry)); + }); + return; + } + + if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID)) + pendingFrameLoad->registerSubresourceLoad(request, resourceKey); +} + +void SpeculativeLoadManager::addPreloadedEntry(std::unique_ptr<Entry> entry, const GlobalFrameID& frameID, std::optional<ResourceRequest>&& revalidationRequest) +{ + ASSERT(entry); + ASSERT(!entry->needsValidation()); + auto key = entry->key(); + m_preloadedEntries.add(key, std::make_unique<PreloadedEntry>(WTFMove(entry), WTFMove(revalidationRequest), [this, key, frameID] { + auto preloadedEntry = m_preloadedEntries.take(key); + ASSERT(preloadedEntry); + if (preloadedEntry->wasRevalidated()) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey()); + else + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithoutRevalidationKey()); + })); +} + +void SpeculativeLoadManager::retrieveEntryFromStorage(const SubresourceInfo& info, RetrieveCompletionHandler&& completionHandler) +{ + m_storage.retrieve(info.key(), static_cast<unsigned>(info.priority()), [completionHandler = WTFMove(completionHandler)](auto record) { + if (!record) { + completionHandler(nullptr); + return false; + } + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) { + completionHandler(nullptr); + return false; + } + + auto& response = entry->response(); + if (responseNeedsRevalidation(response, entry->timeStamp())) { + // Do not use cached redirects that have expired. + if (entry->redirectRequest()) { + completionHandler(nullptr); + return true; + } + entry->setNeedsValidation(true); + } + + completionHandler(WTFMove(entry)); + return true; + }); +} + +bool SpeculativeLoadManager::satisfyPendingRequests(const Key& key, Entry* entry) +{ + auto completionHandlers = m_pendingRetrieveRequests.take(key); + if (!completionHandlers) + return false; + + for (auto& completionHandler : *completionHandlers) + completionHandler(entry ? std::make_unique<Entry>(*entry) : nullptr); + + return true; +} + +void SpeculativeLoadManager::revalidateSubresource(const SubresourceInfo& subresourceInfo, std::unique_ptr<Entry> entry, const GlobalFrameID& frameID) +{ + ASSERT(!entry || entry->needsValidation()); + + auto& key = subresourceInfo.key(); + + // Range is not supported. + if (!key.range().isEmpty()) + return; + + ResourceRequest revalidationRequest = constructRevalidationRequest(key, subresourceInfo, entry.get()); + + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculatively revalidating '%s':", key.identifier().utf8().data()); + + auto revalidator = std::make_unique<SpeculativeLoad>(frameID, revalidationRequest, WTFMove(entry), [this, key, revalidationRequest, frameID](std::unique_ptr<Entry> revalidatedEntry) { + ASSERT(!revalidatedEntry || !revalidatedEntry->needsValidation()); + ASSERT(!revalidatedEntry || revalidatedEntry->key() == key); + + auto protectRevalidator = m_pendingPreloads.take(key); + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculative revalidation completed for '%s':", key.identifier().utf8().data()); + + if (satisfyPendingRequests(key, revalidatedEntry.get())) { + if (revalidatedEntry) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey()); + return; + } + + if (revalidatedEntry) + addPreloadedEntry(WTFMove(revalidatedEntry), frameID, revalidationRequest); + }); + m_pendingPreloads.add(key, WTFMove(revalidator)); +} + +static bool canRevalidate(const SubresourceInfo& subresourceInfo, const Entry* entry) +{ + ASSERT(!subresourceInfo.isTransient()); + ASSERT(!entry || entry->needsValidation()); + + if (entry && entry->response().hasCacheValidatorFields()) + return true; + + auto seenAge = subresourceInfo.lastSeen() - subresourceInfo.firstSeen(); + if (seenAge == 0ms) { + LOG(NetworkCacheSpeculativePreloading, "Speculative load: Seen only once"); + return false; + } + + auto now = std::chrono::system_clock::now(); + auto firstSeenAge = now - subresourceInfo.firstSeen(); + auto lastSeenAge = now - subresourceInfo.lastSeen(); + // Sanity check. + if (seenAge <= 0ms || firstSeenAge <= 0ms || lastSeenAge <= 0ms) + return false; + + // Load full resources speculatively if they seem to stay the same. + const auto minimumAgeRatioToLoad = 2. / 3; + const auto recentMinimumAgeRatioToLoad = 1. / 3; + const auto recentThreshold = 5min; + + auto ageRatio = std::chrono::duration_cast<std::chrono::duration<double>>(seenAge) / firstSeenAge; + auto minimumAgeRatio = lastSeenAge > recentThreshold ? minimumAgeRatioToLoad : recentMinimumAgeRatioToLoad; + + LOG(NetworkCacheSpeculativePreloading, "Speculative load: ok=%d ageRatio=%f entry=%d", ageRatio > minimumAgeRatio, ageRatio, !!entry); + + if (ageRatio > minimumAgeRatio) + return true; + + return false; +} + +void SpeculativeLoadManager::preloadEntry(const Key& key, const SubresourceInfo& subresourceInfo, const GlobalFrameID& frameID) +{ + if (m_pendingPreloads.contains(key)) + return; + m_pendingPreloads.add(key, nullptr); + + retrieveEntryFromStorage(subresourceInfo, [this, key, subresourceInfo, frameID](std::unique_ptr<Entry> entry) { + ASSERT(!m_pendingPreloads.get(key)); + bool removed = m_pendingPreloads.remove(key); + ASSERT_UNUSED(removed, removed); + + if (satisfyPendingRequests(key, entry.get())) { + if (entry) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey()); + return; + } + + if (!entry || entry->needsValidation()) { + if (canRevalidate(subresourceInfo, entry.get())) + revalidateSubresource(subresourceInfo, WTFMove(entry), frameID); + return; + } + + addPreloadedEntry(WTFMove(entry), frameID); + }); +} + +void SpeculativeLoadManager::startSpeculativeRevalidation(const GlobalFrameID& frameID, SubresourcesEntry& entry) +{ + for (auto& subresourceInfo : entry.subresources()) { + auto& key = subresourceInfo.key(); + if (!subresourceInfo.isTransient()) + preloadEntry(key, subresourceInfo, frameID); + else { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Not preloading '%s' because it is marked as transient", key.identifier().utf8().data()); + m_notPreloadedEntries.add(key, std::make_unique<ExpiringEntry>([this, key, frameID] { + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryRightlyNotWarmedUpKey()); + m_notPreloadedEntries.remove(key); + })); + } + } +} + +void SpeculativeLoadManager::retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)>&& completionHandler) +{ + ASSERT(storageKey.type() == "Resource"); + auto subresourcesStorageKey = makeSubresourcesKey(storageKey, m_storage.salt()); + m_storage.retrieve(subresourcesStorageKey, static_cast<unsigned>(ResourceLoadPriority::Medium), [completionHandler = WTFMove(completionHandler)](auto record) { + if (!record) { + completionHandler(nullptr); + return false; + } + + auto subresourcesEntry = SubresourcesEntry::decodeStorageRecord(*record); + if (!subresourcesEntry) { + completionHandler(nullptr); + return false; + } + + completionHandler(WTFMove(subresourcesEntry)); + return true; + }); +} + +} // namespace NetworkCache + +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h new file mode 100644 index 000000000..e9c621a91 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSpeculativeLoadManager_h +#define NetworkCacheSpeculativeLoadManager_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCache.h" +#include "NetworkCacheStorage.h" +#include <WebCore/ResourceRequest.h> +#include <wtf/HashMap.h> +#include <wtf/Vector.h> + +namespace WebKit { + +namespace NetworkCache { + +class Entry; +class SpeculativeLoad; +class SubresourceInfo; +class SubresourcesEntry; + +class SpeculativeLoadManager { + WTF_MAKE_FAST_ALLOCATED; +public: + explicit SpeculativeLoadManager(Storage&); + ~SpeculativeLoadManager(); + + void registerLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, const Key& resourceKey); + + typedef Function<void (std::unique_ptr<Entry>)> RetrieveCompletionHandler; + + bool canRetrieve(const Key& storageKey, const WebCore::ResourceRequest&, const GlobalFrameID&) const; + void retrieve(const Key& storageKey, RetrieveCompletionHandler&&); + +private: + class PreloadedEntry; + + void addPreloadedEntry(std::unique_ptr<Entry>, const GlobalFrameID&, std::optional<WebCore::ResourceRequest>&& revalidationRequest = std::nullopt); + void preloadEntry(const Key&, const SubresourceInfo&, const GlobalFrameID&); + void retrieveEntryFromStorage(const SubresourceInfo&, RetrieveCompletionHandler&&); + void revalidateSubresource(const SubresourceInfo&, std::unique_ptr<Entry>, const GlobalFrameID&); + bool satisfyPendingRequests(const Key&, Entry*); + void retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)>&&); + void startSpeculativeRevalidation(const GlobalFrameID&, SubresourcesEntry&); + + static bool canUsePreloadedEntry(const PreloadedEntry&, const WebCore::ResourceRequest& actualRequest); + static bool canUsePendingPreload(const SpeculativeLoad&, const WebCore::ResourceRequest& actualRequest); + + Storage& m_storage; + + class PendingFrameLoad; + HashMap<GlobalFrameID, RefPtr<PendingFrameLoad>> m_pendingFrameLoads; + + HashMap<Key, std::unique_ptr<SpeculativeLoad>> m_pendingPreloads; + HashMap<Key, std::unique_ptr<Vector<RetrieveCompletionHandler>>> m_pendingRetrieveRequests; + + HashMap<Key, std::unique_ptr<PreloadedEntry>> m_preloadedEntries; + + class ExpiringEntry; + HashMap<Key, std::unique_ptr<ExpiringEntry>> m_notPreloadedEntries; // For logging. +}; + +} // namespace NetworkCache + +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#endif // NetworkCacheSpeculativeLoadManager_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp new file mode 100644 index 000000000..0fc8d1d3d --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp @@ -0,0 +1,450 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE) +#include "NetworkCacheStatistics.h" + +#include "Logging.h" +#include "NetworkCache.h" +#include "NetworkCacheFileSystem.h" +#include "NetworkProcess.h" +#include <WebCore/DiagnosticLoggingKeys.h> +#include <WebCore/DiagnosticLoggingResultType.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/SQLiteDatabaseTracker.h> +#include <WebCore/SQLiteStatement.h> +#include <WebCore/SQLiteTransaction.h> +#include <wtf/RunLoop.h> + +namespace WebKit { +namespace NetworkCache { + +static const char* StatisticsDatabaseName = "WebKitCacheStatistics.db"; +static const std::chrono::milliseconds mininumWriteInterval = std::chrono::milliseconds(10000); + +static bool executeSQLCommand(WebCore::SQLiteDatabase& database, const String& sql) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(database.isOpen()); + + bool result = database.executeCommand(sql); + if (!result) + LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", sql.utf8().data(), database.lastErrorMsg()); + + return result; +} + +static bool executeSQLStatement(WebCore::SQLiteStatement& statement) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(statement.database().isOpen()); + + if (statement.step() != SQLITE_DONE) { + LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", statement.query().utf8().data(), statement.database().lastErrorMsg()); + return false; + } + + return true; +} + +std::unique_ptr<Statistics> Statistics::open(const String& cachePath) +{ + ASSERT(RunLoop::isMain()); + + String databasePath = WebCore::pathByAppendingComponent(cachePath, StatisticsDatabaseName); + return std::make_unique<Statistics>(databasePath); +} + +Statistics::Statistics(const String& databasePath) + : m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Statistics.Background", WorkQueue::Type::Serial, WorkQueue::QOS::Background)) + , m_writeTimer(*this, &Statistics::writeTimerFired) +{ + initialize(databasePath); +} + +void Statistics::initialize(const String& databasePath) +{ + ASSERT(RunLoop::isMain()); + + auto startTime = std::chrono::system_clock::now(); + + serialBackgroundIOQueue().dispatch([this, databasePath = databasePath.isolatedCopy(), networkCachePath = singleton().recordsPath().isolatedCopy(), startTime] { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + + if (!WebCore::makeAllDirectories(WebCore::directoryName(databasePath))) + return; + + LOG(NetworkCache, "(NetworkProcess) Opening network cache statistics database at %s...", databasePath.utf8().data()); + m_database.open(databasePath); + m_database.disableThreadingChecks(); + if (!m_database.isOpen()) { + LOG_ERROR("Network cache statistics: Failed to open / create the network cache statistics database"); + return; + } + + executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS AlreadyRequested (hash TEXT PRIMARY KEY)")); + executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS UncachedReason (hash TEXT PRIMARY KEY, reason INTEGER)")); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT count(*) FROM AlreadyRequested")); + if (statement.prepareAndStep() != SQLITE_ROW) { + LOG_ERROR("Network cache statistics: Failed to count the number of rows in AlreadyRequested table"); + return; + } + + m_approximateEntryCount = statement.getColumnInt(0); + +#if !LOG_DISABLED + auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); +#endif + LOG(NetworkCache, "(NetworkProcess) Network cache statistics database load complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS); + + if (!m_approximateEntryCount) { + bootstrapFromNetworkCache(networkCachePath); +#if !LOG_DISABLED + elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); +#endif + LOG(NetworkCache, "(NetworkProcess) Network cache statistics database bootstrapping complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS); + } + }); +} + +void Statistics::bootstrapFromNetworkCache(const String& networkCachePath) +{ + ASSERT(!RunLoop::isMain()); + + LOG(NetworkCache, "(NetworkProcess) Bootstrapping the network cache statistics database from the network cache..."); + + HashSet<String> hashes; + traverseRecordsFiles(networkCachePath, ASCIILiteral("Resource"), [&hashes](const String& fileName, const String& hashString, const String& type, bool isBodyBlob, const String& recordDirectoryPath) { + if (isBodyBlob) + return; + + Key::HashType hash; + if (!Key::stringToHash(hashString, hash)) + return; + + hashes.add(hashString); + }); + + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction writeTransaction(m_database); + writeTransaction.begin(); + + addHashesToDatabase(hashes); + + writeTransaction.commit(); +} + +void Statistics::shrinkIfNeeded() +{ + ASSERT(RunLoop::isMain()); + const size_t maxEntries = 100000; + + if (m_approximateEntryCount < maxEntries) + return; + + LOG(NetworkCache, "(NetworkProcess) shrinking statistics cache m_approximateEntryCount=%lu, maxEntries=%lu", static_cast<size_t>(m_approximateEntryCount), maxEntries); + + clear(); + + serialBackgroundIOQueue().dispatch([this, networkCachePath = singleton().recordsPath().isolatedCopy()] { + bootstrapFromNetworkCache(networkCachePath); + LOG(NetworkCache, "(NetworkProcess) statistics cache shrink completed m_approximateEntryCount=%lu", static_cast<size_t>(m_approximateEntryCount)); + }); +} + +void Statistics::recordRetrievalRequest(uint64_t webPageID) +{ + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalRequestKey(), WebCore::ShouldSample::Yes); +} + +void Statistics::recordNotCachingResponse(const Key& key, StoreDecision storeDecision) +{ + ASSERT(storeDecision != StoreDecision::Yes); + + m_storeDecisionsToAdd.set(key.hashAsString(), storeDecision); + if (!m_writeTimer.isActive()) + m_writeTimer.startOneShot(mininumWriteInterval); +} + +static String retrieveDecisionToDiagnosticKey(RetrieveDecision retrieveDecision) +{ + switch (retrieveDecision) { + case RetrieveDecision::NoDueToHTTPMethod: + return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey(); + case RetrieveDecision::NoDueToConditionalRequest: + return WebCore::DiagnosticLoggingKeys::isConditionalRequestKey(); + case RetrieveDecision::NoDueToReloadIgnoringCache: + return WebCore::DiagnosticLoggingKeys::isReloadIgnoringCacheDataKey(); + case RetrieveDecision::NoDueToStreamingMedia: + return WebCore::DiagnosticLoggingKeys::streamingMedia(); + case RetrieveDecision::Yes: + ASSERT_NOT_REACHED(); + break; + } + return emptyString(); +} + +void Statistics::recordNotUsingCacheForRequest(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, RetrieveDecision retrieveDecision) +{ + ASSERT(retrieveDecision != RetrieveDecision::Yes); + + auto hash = key.hashAsString(); + queryWasEverRequested(hash, NeedUncachedReason::No, [this, hash, requestURL = request.url(), webPageID, retrieveDecision](bool wasEverRequested, const std::optional<StoreDecision>&) { + if (wasEverRequested) { + String diagnosticKey = retrieveDecisionToDiagnosticKey(retrieveDecision); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously requested but we are not using the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheUnusedReasonKey(), diagnosticKey, WebCore::ShouldSample::Yes); + } else { + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheUnusedReasonKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes); + markAsRequested(hash); + } + }); +} + +static String storeDecisionToDiagnosticKey(StoreDecision storeDecision) +{ + switch (storeDecision) { + case StoreDecision::NoDueToProtocol: + return WebCore::DiagnosticLoggingKeys::notHTTPFamilyKey(); + case StoreDecision::NoDueToHTTPMethod: + return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey(); + case StoreDecision::NoDueToAttachmentResponse: + return WebCore::DiagnosticLoggingKeys::isAttachmentKey(); + case StoreDecision::NoDueToNoStoreResponse: + case StoreDecision::NoDueToNoStoreRequest: + return WebCore::DiagnosticLoggingKeys::cacheControlNoStoreKey(); + case StoreDecision::NoDueToHTTPStatusCode: + return WebCore::DiagnosticLoggingKeys::uncacheableStatusCodeKey(); + case StoreDecision::NoDueToUnlikelyToReuse: + return WebCore::DiagnosticLoggingKeys::unlikelyToReuseKey(); + case StoreDecision::NoDueToStreamingMedia: + return WebCore::DiagnosticLoggingKeys::streamingMedia(); + case StoreDecision::Yes: + // It was stored but could not be retrieved so it must have been pruned from the cache. + return WebCore::DiagnosticLoggingKeys::noLongerInCacheKey(); + } + return String(); +} + +void Statistics::recordRetrievalFailure(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request) +{ + auto hash = key.hashAsString(); + queryWasEverRequested(hash, NeedUncachedReason::Yes, [this, hash, requestURL = request.url(), webPageID](bool wasPreviouslyRequested, const std::optional<StoreDecision>& storeDecision) { + if (wasPreviouslyRequested) { + String diagnosticKey = storeDecisionToDiagnosticKey(storeDecision.value()); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously request but is not in the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheFailureReasonKey(), diagnosticKey, WebCore::ShouldSample::Yes); + } else { + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheFailureReasonKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes); + markAsRequested(hash); + } + }); +} + +static String cachedEntryReuseFailureToDiagnosticKey(UseDecision decision) +{ + switch (decision) { + case UseDecision::NoDueToVaryingHeaderMismatch: + return WebCore::DiagnosticLoggingKeys::varyingHeaderMismatchKey(); + case UseDecision::NoDueToMissingValidatorFields: + return WebCore::DiagnosticLoggingKeys::missingValidatorFieldsKey(); + case UseDecision::NoDueToDecodeFailure: + case UseDecision::NoDueToExpiredRedirect: + return WebCore::DiagnosticLoggingKeys::otherKey(); + case UseDecision::Use: + case UseDecision::Validate: + ASSERT_NOT_REACHED(); + break; + } + return emptyString(); +} + +void Statistics::recordRetrievedCachedEntry(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, UseDecision decision) +{ + WebCore::URL requestURL = request.url(); + if (decision == UseDecision::Use) { + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache and is used", webPageID, requestURL.string().ascii().data()); + NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes); + return; + } + + if (decision == UseDecision::Validate) { + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but needs revalidation", webPageID, requestURL.string().ascii().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::needsRevalidationKey(), WebCore::ShouldSample::Yes); + return; + } + + String diagnosticKey = cachedEntryReuseFailureToDiagnosticKey(decision); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but wasn't used, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheReuseFailureKey(), diagnosticKey, WebCore::ShouldSample::Yes); +} + +void Statistics::recordRevalidationSuccess(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request) +{ + WebCore::URL requestURL = request.url(); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was successfully revalidated", webPageID, requestURL.string().ascii().data()); + + NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::revalidatingKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes); +} + +void Statistics::markAsRequested(const String& hash) +{ + ASSERT(RunLoop::isMain()); + + m_hashesToAdd.add(hash); + if (!m_writeTimer.isActive()) + m_writeTimer.startOneShot(mininumWriteInterval); +} + +void Statistics::writeTimerFired() +{ + ASSERT(RunLoop::isMain()); + + serialBackgroundIOQueue().dispatch([this, hashesToAdd = WTFMove(m_hashesToAdd), storeDecisionsToAdd = WTFMove(m_storeDecisionsToAdd)] { + if (!m_database.isOpen()) + return; + + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction writeTransaction(m_database); + writeTransaction.begin(); + + addHashesToDatabase(hashesToAdd); + addStoreDecisionsToDatabase(storeDecisionsToAdd); + + writeTransaction.commit(); + }); + + shrinkIfNeeded(); +} + +void Statistics::queryWasEverRequested(const String& hash, NeedUncachedReason needUncachedReason, RequestedCompletionHandler&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + + // Query pending writes first. + bool wasAlreadyRequested = m_hashesToAdd.contains(hash); + if (wasAlreadyRequested && needUncachedReason == NeedUncachedReason::No) { + completionHandler(true, std::nullopt); + return; + } + if (needUncachedReason == NeedUncachedReason::Yes && m_storeDecisionsToAdd.contains(hash)) { + completionHandler(true, m_storeDecisionsToAdd.get(hash)); + return; + } + + // Query the database. + auto everRequestedQuery = std::make_unique<EverRequestedQuery>(EverRequestedQuery { hash, needUncachedReason == NeedUncachedReason::Yes, WTFMove(completionHandler) }); + auto& query = *everRequestedQuery; + m_activeQueries.add(WTFMove(everRequestedQuery)); + serialBackgroundIOQueue().dispatch([this, wasAlreadyRequested, &query] () mutable { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + std::optional<StoreDecision> storeDecision; + if (m_database.isOpen()) { + if (!wasAlreadyRequested) { + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT hash FROM AlreadyRequested WHERE hash=?")); + if (statement.prepare() == SQLITE_OK) { + statement.bindText(1, query.hash); + wasAlreadyRequested = (statement.step() == SQLITE_ROW); + } + } + if (wasAlreadyRequested && query.needUncachedReason) { + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT reason FROM UncachedReason WHERE hash=?")); + storeDecision = StoreDecision::Yes; + if (statement.prepare() == SQLITE_OK) { + statement.bindText(1, query.hash); + if (statement.step() == SQLITE_ROW) + storeDecision = static_cast<StoreDecision>(statement.getColumnInt(0)); + } + } + } + RunLoop::main().dispatch([this, &query, wasAlreadyRequested, storeDecision] { + query.completionHandler(wasAlreadyRequested, storeDecision); + m_activeQueries.remove(&query); + }); + }); +} + +void Statistics::clear() +{ + ASSERT(RunLoop::isMain()); + + serialBackgroundIOQueue().dispatch([this] { + if (m_database.isOpen()) { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction deleteTransaction(m_database); + deleteTransaction.begin(); + executeSQLCommand(m_database, ASCIILiteral("DELETE FROM AlreadyRequested")); + executeSQLCommand(m_database, ASCIILiteral("DELETE FROM UncachedReason")); + deleteTransaction.commit(); + m_approximateEntryCount = 0; + } + }); +} + +void Statistics::addHashesToDatabase(const HashSet<String>& hashes) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(m_database.isOpen()); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR IGNORE INTO AlreadyRequested (hash) VALUES (?)")); + if (statement.prepare() != SQLITE_OK) + return; + + for (auto& hash : hashes) { + statement.bindText(1, hash); + if (executeSQLStatement(statement)) + ++m_approximateEntryCount; + statement.reset(); + } +} + +void Statistics::addStoreDecisionsToDatabase(const HashMap<String, NetworkCache::StoreDecision>& storeDecisions) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(m_database.isOpen()); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR REPLACE INTO UncachedReason (hash, reason) VALUES (?, ?)")); + if (statement.prepare() != SQLITE_OK) + return; + + for (auto& pair : storeDecisions) { + statement.bindText(1, pair.key); + statement.bindInt(2, static_cast<int>(pair.value)); + executeSQLStatement(statement); + statement.reset(); + } +} + +} +} + +#endif // ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h new file mode 100644 index 000000000..b67c260e1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheStatistics_h +#define NetworkCacheStatistics_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCache.h" +#include "NetworkCacheKey.h" +#include <WebCore/SQLiteDatabase.h> +#include <WebCore/Timer.h> +#include <wtf/WorkQueue.h> + +namespace WebCore { +class ResourceRequest; +} + +namespace WebKit { +namespace NetworkCache { + +class Statistics { +public: + static std::unique_ptr<Statistics> open(const String& cachePath); + explicit Statistics(const String& databasePath); + + void clear(); + + void recordRetrievalRequest(uint64_t webPageID); + void recordNotCachingResponse(const Key&, StoreDecision); + void recordNotUsingCacheForRequest(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, RetrieveDecision); + void recordRetrievalFailure(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&); + void recordRetrievedCachedEntry(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, UseDecision); + void recordRevalidationSuccess(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&); + +private: + WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); } + + void initialize(const String& databasePath); + void bootstrapFromNetworkCache(const String& networkCachePath); + void shrinkIfNeeded(); + + void addHashesToDatabase(const HashSet<String>& hashes); + void addStoreDecisionsToDatabase(const HashMap<String, NetworkCache::StoreDecision>&); + void writeTimerFired(); + + typedef std::function<void (bool wasEverRequested, const std::optional<StoreDecision>&)> RequestedCompletionHandler; + enum class NeedUncachedReason { No, Yes }; + void queryWasEverRequested(const String&, NeedUncachedReason, RequestedCompletionHandler&&); + void markAsRequested(const String& hash); + + struct EverRequestedQuery { + String hash; + bool needUncachedReason; + RequestedCompletionHandler completionHandler; + }; + + std::atomic<size_t> m_approximateEntryCount { 0 }; + + mutable Ref<WorkQueue> m_serialBackgroundIOQueue; + mutable HashSet<std::unique_ptr<const EverRequestedQuery>> m_activeQueries; + WebCore::SQLiteDatabase m_database; + HashSet<String> m_hashesToAdd; + HashMap<String, NetworkCache::StoreDecision> m_storeDecisionsToAdd; + WebCore::Timer m_writeTimer; +}; + +} +} + +#endif // ENABLE(NETWORK_CACHE) + +#endif // NetworkCacheStatistics_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp new file mode 100644 index 000000000..4f272449f --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp @@ -0,0 +1,1048 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheStorage.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheCoders.h" +#include "NetworkCacheFileSystem.h" +#include "NetworkCacheIOChannel.h" +#include <mutex> +#include <wtf/Condition.h> +#include <wtf/Lock.h> +#include <wtf/RandomNumber.h> +#include <wtf/RunLoop.h> +#include <wtf/text/CString.h> + +namespace WebKit { +namespace NetworkCache { + +static const char saltFileName[] = "salt"; +static const char versionDirectoryPrefix[] = "Version "; +static const char recordsDirectoryName[] = "Records"; +static const char blobsDirectoryName[] = "Blobs"; +static const char blobSuffix[] = "-blob"; + +static double computeRecordWorth(FileTimes); + +struct Storage::ReadOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + ReadOperation(const Key& key, RetrieveCompletionHandler&& completionHandler) + : key(key) + , completionHandler(WTFMove(completionHandler)) + { } + + void cancel(); + bool finish(); + + const Key key; + const RetrieveCompletionHandler completionHandler; + + std::unique_ptr<Record> resultRecord; + SHA1::Digest expectedBodyHash; + BlobStorage::Blob resultBodyBlob; + std::atomic<unsigned> activeCount { 0 }; + bool isCanceled { false }; +}; + +void Storage::ReadOperation::cancel() +{ + ASSERT(RunLoop::isMain()); + + if (isCanceled) + return; + isCanceled = true; + completionHandler(nullptr); +} + +bool Storage::ReadOperation::finish() +{ + ASSERT(RunLoop::isMain()); + + if (isCanceled) + return false; + if (resultRecord && resultRecord->body.isNull()) { + if (resultBodyBlob.hash == expectedBodyHash) + resultRecord->body = resultBodyBlob.data; + else + resultRecord = nullptr; + } + return completionHandler(WTFMove(resultRecord)); +} + +struct Storage::WriteOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + WriteOperation(const Record& record, MappedBodyHandler&& mappedBodyHandler) + : record(record) + , mappedBodyHandler(WTFMove(mappedBodyHandler)) + { } + + const Record record; + const MappedBodyHandler mappedBodyHandler; + + std::atomic<unsigned> activeCount { 0 }; +}; + +struct Storage::TraverseOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + TraverseOperation(const String& type, TraverseFlags flags, TraverseHandler&& handler) + : type(type) + , flags(flags) + , handler(WTFMove(handler)) + { } + + const String type; + const TraverseFlags flags; + const TraverseHandler handler; + + Lock activeMutex; + Condition activeCondition; + unsigned activeCount { 0 }; +}; + +static String makeVersionedDirectoryPath(const String& baseDirectoryPath) +{ + String versionSubdirectory = versionDirectoryPrefix + String::number(Storage::version); + return WebCore::pathByAppendingComponent(baseDirectoryPath, versionSubdirectory); +} + +static String makeRecordsDirectoryPath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), recordsDirectoryName); +} + +static String makeBlobDirectoryPath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), blobsDirectoryName); +} + +static String makeSaltFilePath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), saltFileName); +} + +std::unique_ptr<Storage> Storage::open(const String& cachePath) +{ + ASSERT(RunLoop::isMain()); + + if (!WebCore::makeAllDirectories(makeVersionedDirectoryPath(cachePath))) + return nullptr; + auto salt = readOrMakeSalt(makeSaltFilePath(cachePath)); + if (!salt) + return nullptr; + return std::unique_ptr<Storage>(new Storage(cachePath, *salt)); +} + +void traverseRecordsFiles(const String& recordsPath, const String& expectedType, const RecordFileTraverseFunction& function) +{ + traverseDirectory(recordsPath, [&](const String& partitionName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName); + traverseDirectory(partitionPath, [&](const String& actualType, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + if (!expectedType.isEmpty() && expectedType != actualType) + return; + String recordDirectoryPath = WebCore::pathByAppendingComponent(partitionPath, actualType); + traverseDirectory(recordDirectoryPath, [&function, &recordDirectoryPath, &actualType](const String& fileName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::File || fileName.length() < Key::hashStringLength()) + return; + + String hashString = fileName.substring(0, Key::hashStringLength()); + auto isBlob = fileName.length() > Key::hashStringLength() && fileName.endsWith(blobSuffix); + function(fileName, hashString, actualType, isBlob, recordDirectoryPath); + }); + }); + }); +} + +static void deleteEmptyRecordsDirectories(const String& recordsPath) +{ + traverseDirectory(recordsPath, [&recordsPath](const String& partitionName, DirectoryEntryType type) { + if (type != DirectoryEntryType::Directory) + return; + + // Delete [type] sub-folders. + String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName); + traverseDirectory(partitionPath, [&partitionPath](const String& subdirName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + + // Let system figure out if it is really empty. + WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(partitionPath, subdirName)); + }); + + // Delete [Partition] folders. + // Let system figure out if it is really empty. + WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(recordsPath, partitionName)); + }); +} + +Storage::Storage(const String& baseDirectoryPath, Salt salt) + : m_basePath(baseDirectoryPath) + , m_recordsPath(makeRecordsDirectoryPath(baseDirectoryPath)) + , m_salt(salt) + , m_canUseSharedMemoryForBodyData(canUseSharedMemoryForPath(baseDirectoryPath)) + , m_readOperationTimeoutTimer(*this, &Storage::cancelAllReadOperations) + , m_writeOperationDispatchTimer(*this, &Storage::dispatchPendingWriteOperations) + , m_ioQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage", WorkQueue::Type::Concurrent)) + , m_backgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.background", WorkQueue::Type::Concurrent, WorkQueue::QOS::Background)) + , m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.serialBackground", WorkQueue::Type::Serial, WorkQueue::QOS::Background)) + , m_blobStorage(makeBlobDirectoryPath(baseDirectoryPath), m_salt) +{ + deleteOldVersions(); + synchronize(); +} + +Storage::~Storage() +{ +} + +String Storage::basePath() const +{ + return m_basePath.isolatedCopy(); +} + +String Storage::versionPath() const +{ + return makeVersionedDirectoryPath(basePath()); +} + +String Storage::recordsPath() const +{ + return m_recordsPath.isolatedCopy(); +} + +size_t Storage::approximateSize() const +{ + return m_approximateRecordsSize + m_blobStorage.approximateSize(); +} + +void Storage::synchronize() +{ + ASSERT(RunLoop::isMain()); + + if (m_synchronizationInProgress || m_shrinkInProgress) + return; + m_synchronizationInProgress = true; + + LOG(NetworkCacheStorage, "(NetworkProcess) synchronizing cache"); + + backgroundIOQueue().dispatch([this] { + auto recordFilter = std::make_unique<ContentsFilter>(); + auto blobFilter = std::make_unique<ContentsFilter>(); + size_t recordsSize = 0; + unsigned count = 0; + String anyType; + traverseRecordsFiles(recordsPath(), anyType, [&recordFilter, &blobFilter, &recordsSize, &count](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + + Key::HashType hash; + if (!Key::stringToHash(hashString, hash)) { + WebCore::deleteFile(filePath); + return; + } + long long fileSize = 0; + WebCore::getFileSize(filePath, fileSize); + if (!fileSize) { + WebCore::deleteFile(filePath); + return; + } + + if (isBlob) { + blobFilter->add(hash); + return; + } + + recordFilter->add(hash); + recordsSize += fileSize; + ++count; + }); + + RunLoop::main().dispatch([this, recordFilter = WTFMove(recordFilter), blobFilter = WTFMove(blobFilter), recordsSize]() mutable { + for (auto& recordFilterKey : m_recordFilterHashesAddedDuringSynchronization) + recordFilter->add(recordFilterKey); + m_recordFilterHashesAddedDuringSynchronization.clear(); + + for (auto& hash : m_blobFilterHashesAddedDuringSynchronization) + blobFilter->add(hash); + m_blobFilterHashesAddedDuringSynchronization.clear(); + + m_recordFilter = WTFMove(recordFilter); + m_blobFilter = WTFMove(blobFilter); + m_approximateRecordsSize = recordsSize; + m_synchronizationInProgress = false; + }); + + m_blobStorage.synchronize(); + + deleteEmptyRecordsDirectories(recordsPath()); + + LOG(NetworkCacheStorage, "(NetworkProcess) cache synchronization completed size=%zu count=%u", recordsSize, count); + }); +} + +void Storage::addToRecordFilter(const Key& key) +{ + ASSERT(RunLoop::isMain()); + + if (m_recordFilter) + m_recordFilter->add(key.hash()); + + // If we get new entries during filter synchronization take care to add them to the new filter as well. + if (m_synchronizationInProgress) + m_recordFilterHashesAddedDuringSynchronization.append(key.hash()); +} + +bool Storage::mayContain(const Key& key) const +{ + ASSERT(RunLoop::isMain()); + return !m_recordFilter || m_recordFilter->mayContain(key.hash()); +} + +bool Storage::mayContainBlob(const Key& key) const +{ + ASSERT(RunLoop::isMain()); + return !m_blobFilter || m_blobFilter->mayContain(key.hash()); +} + +String Storage::recordDirectoryPathForKey(const Key& key) const +{ + ASSERT(!key.type().isEmpty()); + return WebCore::pathByAppendingComponent(WebCore::pathByAppendingComponent(recordsPath(), key.partitionHashAsString()), key.type()); +} + +String Storage::recordPathForKey(const Key& key) const +{ + return WebCore::pathByAppendingComponent(recordDirectoryPathForKey(key), key.hashAsString()); +} + +static String blobPathForRecordPath(const String& recordPath) +{ + return recordPath + blobSuffix; +} + +String Storage::blobPathForKey(const Key& key) const +{ + return blobPathForRecordPath(recordPathForKey(key)); +} + +struct RecordMetaData { + RecordMetaData() { } + explicit RecordMetaData(const Key& key) + : cacheStorageVersion(Storage::version) + , key(key) + { } + + unsigned cacheStorageVersion; + Key key; + std::chrono::system_clock::time_point timeStamp; + SHA1::Digest headerHash; + uint64_t headerSize; + SHA1::Digest bodyHash; + uint64_t bodySize; + bool isBodyInline; + + // Not encoded as a field. Header starts immediately after meta data. + uint64_t headerOffset; +}; + +static bool decodeRecordMetaData(RecordMetaData& metaData, const Data& fileData) +{ + bool success = false; + fileData.apply([&metaData, &success](const uint8_t* data, size_t size) { + WTF::Persistence::Decoder decoder(data, size); + if (!decoder.decode(metaData.cacheStorageVersion)) + return false; + if (!decoder.decode(metaData.key)) + return false; + if (!decoder.decode(metaData.timeStamp)) + return false; + if (!decoder.decode(metaData.headerHash)) + return false; + if (!decoder.decode(metaData.headerSize)) + return false; + if (!decoder.decode(metaData.bodyHash)) + return false; + if (!decoder.decode(metaData.bodySize)) + return false; + if (!decoder.decode(metaData.isBodyInline)) + return false; + if (!decoder.verifyChecksum()) + return false; + metaData.headerOffset = decoder.currentOffset(); + success = true; + return false; + }); + return success; +} + +static bool decodeRecordHeader(const Data& fileData, RecordMetaData& metaData, Data& headerData, const Salt& salt) +{ + if (!decodeRecordMetaData(metaData, fileData)) { + LOG(NetworkCacheStorage, "(NetworkProcess) meta data decode failure"); + return false; + } + + if (metaData.cacheStorageVersion != Storage::version) { + LOG(NetworkCacheStorage, "(NetworkProcess) version mismatch"); + return false; + } + + headerData = fileData.subrange(metaData.headerOffset, metaData.headerSize); + if (metaData.headerHash != computeSHA1(headerData, salt)) { + LOG(NetworkCacheStorage, "(NetworkProcess) header checksum mismatch"); + return false; + } + return true; +} + +void Storage::readRecord(ReadOperation& readOperation, const Data& recordData) +{ + ASSERT(!RunLoop::isMain()); + + RecordMetaData metaData; + Data headerData; + if (!decodeRecordHeader(recordData, metaData, headerData, m_salt)) + return; + + if (metaData.key != readOperation.key) + return; + + // Sanity check against time stamps in future. + if (metaData.timeStamp > std::chrono::system_clock::now()) + return; + + Data bodyData; + if (metaData.isBodyInline) { + size_t bodyOffset = metaData.headerOffset + headerData.size(); + if (bodyOffset + metaData.bodySize != recordData.size()) + return; + bodyData = recordData.subrange(bodyOffset, metaData.bodySize); + if (metaData.bodyHash != computeSHA1(bodyData, m_salt)) + return; + } + + readOperation.expectedBodyHash = metaData.bodyHash; + readOperation.resultRecord = std::make_unique<Storage::Record>(Storage::Record { + metaData.key, + metaData.timeStamp, + headerData, + bodyData, + metaData.bodyHash + }); +} + +static Data encodeRecordMetaData(const RecordMetaData& metaData) +{ + WTF::Persistence::Encoder encoder; + + encoder << metaData.cacheStorageVersion; + encoder << metaData.key; + encoder << metaData.timeStamp; + encoder << metaData.headerHash; + encoder << metaData.headerSize; + encoder << metaData.bodyHash; + encoder << metaData.bodySize; + encoder << metaData.isBodyInline; + + encoder.encodeChecksum(); + + return Data(encoder.buffer(), encoder.bufferSize()); +} + +std::optional<BlobStorage::Blob> Storage::storeBodyAsBlob(WriteOperation& writeOperation) +{ + auto blobPath = blobPathForKey(writeOperation.record.key); + + // Store the body. + auto blob = m_blobStorage.add(blobPath, writeOperation.record.body); + if (blob.data.isNull()) + return { }; + + ++writeOperation.activeCount; + + RunLoop::main().dispatch([this, blob, &writeOperation] { + if (m_blobFilter) + m_blobFilter->add(writeOperation.record.key.hash()); + if (m_synchronizationInProgress) + m_blobFilterHashesAddedDuringSynchronization.append(writeOperation.record.key.hash()); + + if (writeOperation.mappedBodyHandler) + writeOperation.mappedBodyHandler(blob.data); + + finishWriteOperation(writeOperation); + }); + return blob; +} + +Data Storage::encodeRecord(const Record& record, std::optional<BlobStorage::Blob> blob) +{ + ASSERT(!blob || bytesEqual(blob.value().data, record.body)); + + RecordMetaData metaData(record.key); + metaData.timeStamp = record.timeStamp; + metaData.headerHash = computeSHA1(record.header, m_salt); + metaData.headerSize = record.header.size(); + metaData.bodyHash = blob ? blob.value().hash : computeSHA1(record.body, m_salt); + metaData.bodySize = record.body.size(); + metaData.isBodyInline = !blob; + + auto encodedMetaData = encodeRecordMetaData(metaData); + auto headerData = concatenate(encodedMetaData, record.header); + + if (metaData.isBodyInline) + return concatenate(headerData, record.body); + + return { headerData }; +} + +void Storage::removeFromPendingWriteOperations(const Key& key) +{ + while (true) { + auto found = m_pendingWriteOperations.findIf([&key](auto& operation) { + return operation->record.key == key; + }); + + if (found == m_pendingWriteOperations.end()) + break; + + m_pendingWriteOperations.remove(found); + } +} + +void Storage::remove(const Key& key) +{ + ASSERT(RunLoop::isMain()); + + if (!mayContain(key)) + return; + + // We can't remove the key from the Bloom filter (but some false positives are expected anyway). + // For simplicity we also don't reduce m_approximateSize on removals. + // The next synchronization will update everything. + + removeFromPendingWriteOperations(key); + + serialBackgroundIOQueue().dispatch([this, key] { + WebCore::deleteFile(recordPathForKey(key)); + m_blobStorage.remove(blobPathForKey(key)); + }); +} + +void Storage::updateFileModificationTime(const String& path) +{ + serialBackgroundIOQueue().dispatch([path = path.isolatedCopy()] { + updateFileModificationTimeIfNeeded(path); + }); +} + +void Storage::dispatchReadOperation(std::unique_ptr<ReadOperation> readOperationPtr) +{ + ASSERT(RunLoop::isMain()); + + auto& readOperation = *readOperationPtr; + m_activeReadOperations.add(WTFMove(readOperationPtr)); + + // I/O pressure may make disk operations slow. If they start taking very long time we rather go to network. + const auto readTimeout = 1500ms; + m_readOperationTimeoutTimer.startOneShot(readTimeout); + + bool shouldGetBodyBlob = mayContainBlob(readOperation.key); + + ioQueue().dispatch([this, &readOperation, shouldGetBodyBlob] { + auto recordPath = recordPathForKey(readOperation.key); + + ++readOperation.activeCount; + if (shouldGetBodyBlob) + ++readOperation.activeCount; + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Read); + channel->read(0, std::numeric_limits<size_t>::max(), &ioQueue(), [this, &readOperation](const Data& fileData, int error) { + if (!error) + readRecord(readOperation, fileData); + finishReadOperation(readOperation); + }); + + if (shouldGetBodyBlob) { + // Read the blob in parallel with the record read. + auto blobPath = blobPathForKey(readOperation.key); + readOperation.resultBodyBlob = m_blobStorage.get(blobPath); + finishReadOperation(readOperation); + } + }); +} + +void Storage::finishReadOperation(ReadOperation& readOperation) +{ + ASSERT(readOperation.activeCount); + // Record and blob reads must finish. + if (--readOperation.activeCount) + return; + + RunLoop::main().dispatch([this, &readOperation] { + bool success = readOperation.finish(); + if (success) + updateFileModificationTime(recordPathForKey(readOperation.key)); + else if (!readOperation.isCanceled) + remove(readOperation.key); + + ASSERT(m_activeReadOperations.contains(&readOperation)); + m_activeReadOperations.remove(&readOperation); + + if (m_activeReadOperations.isEmpty()) + m_readOperationTimeoutTimer.stop(); + + dispatchPendingReadOperations(); + + LOG(NetworkCacheStorage, "(NetworkProcess) read complete success=%d", success); + }); +} + +void Storage::cancelAllReadOperations() +{ + ASSERT(RunLoop::isMain()); + + for (auto& readOperation : m_activeReadOperations) + readOperation->cancel(); + + size_t pendingCount = 0; + for (int priority = maximumRetrievePriority; priority >= 0; --priority) { + auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority]; + pendingCount += pendingRetrieveQueue.size(); + for (auto it = pendingRetrieveQueue.rbegin(), end = pendingRetrieveQueue.rend(); it != end; ++it) + (*it)->cancel(); + pendingRetrieveQueue.clear(); + } + + LOG(NetworkCacheStorage, "(NetworkProcess) retrieve timeout, canceled %u active and %zu pending", m_activeReadOperations.size(), pendingCount); +} + +void Storage::dispatchPendingReadOperations() +{ + ASSERT(RunLoop::isMain()); + + const int maximumActiveReadOperationCount = 5; + + for (int priority = maximumRetrievePriority; priority >= 0; --priority) { + if (m_activeReadOperations.size() > maximumActiveReadOperationCount) { + LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel retrieves"); + return; + } + auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority]; + if (pendingRetrieveQueue.isEmpty()) + continue; + dispatchReadOperation(pendingRetrieveQueue.takeLast()); + } +} + +template <class T> bool retrieveFromMemory(const T& operations, const Key& key, Storage::RetrieveCompletionHandler& completionHandler) +{ + for (auto& operation : operations) { + if (operation->record.key == key) { + LOG(NetworkCacheStorage, "(NetworkProcess) found write operation in progress"); + RunLoop::main().dispatch([record = operation->record, completionHandler = WTFMove(completionHandler)] { + completionHandler(std::make_unique<Storage::Record>(record)); + }); + return true; + } + } + return false; +} + +void Storage::dispatchPendingWriteOperations() +{ + ASSERT(RunLoop::isMain()); + + const int maximumActiveWriteOperationCount { 1 }; + + while (!m_pendingWriteOperations.isEmpty()) { + if (m_activeWriteOperations.size() >= maximumActiveWriteOperationCount) { + LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel writes"); + return; + } + dispatchWriteOperation(m_pendingWriteOperations.takeLast()); + } +} + +static bool shouldStoreBodyAsBlob(const Data& bodyData) +{ + const size_t maximumInlineBodySize { 16 * 1024 }; + return bodyData.size() > maximumInlineBodySize; +} + +void Storage::dispatchWriteOperation(std::unique_ptr<WriteOperation> writeOperationPtr) +{ + ASSERT(RunLoop::isMain()); + + auto& writeOperation = *writeOperationPtr; + m_activeWriteOperations.add(WTFMove(writeOperationPtr)); + + // This was added already when starting the store but filter might have been wiped. + addToRecordFilter(writeOperation.record.key); + + backgroundIOQueue().dispatch([this, &writeOperation] { + auto recordDirectorPath = recordDirectoryPathForKey(writeOperation.record.key); + auto recordPath = recordPathForKey(writeOperation.record.key); + + WebCore::makeAllDirectories(recordDirectorPath); + + ++writeOperation.activeCount; + + bool shouldStoreAsBlob = shouldStoreBodyAsBlob(writeOperation.record.body); + auto blob = shouldStoreAsBlob ? storeBodyAsBlob(writeOperation) : std::nullopt; + + auto recordData = encodeRecord(writeOperation.record, blob); + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Create); + size_t recordSize = recordData.size(); + channel->write(0, recordData, nullptr, [this, &writeOperation, recordSize](int error) { + // On error the entry still stays in the contents filter until next synchronization. + m_approximateRecordsSize += recordSize; + finishWriteOperation(writeOperation); + + LOG(NetworkCacheStorage, "(NetworkProcess) write complete error=%d", error); + }); + }); +} + +void Storage::finishWriteOperation(WriteOperation& writeOperation) +{ + ASSERT(RunLoop::isMain()); + ASSERT(writeOperation.activeCount); + ASSERT(m_activeWriteOperations.contains(&writeOperation)); + + if (--writeOperation.activeCount) + return; + + m_activeWriteOperations.remove(&writeOperation); + dispatchPendingWriteOperations(); + + shrinkIfNeeded(); +} + +void Storage::retrieve(const Key& key, unsigned priority, RetrieveCompletionHandler&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(priority <= maximumRetrievePriority); + ASSERT(!key.isNull()); + + if (!m_capacity) { + completionHandler(nullptr); + return; + } + + if (!mayContain(key)) { + completionHandler(nullptr); + return; + } + + if (retrieveFromMemory(m_pendingWriteOperations, key, completionHandler)) + return; + if (retrieveFromMemory(m_activeWriteOperations, key, completionHandler)) + return; + + auto readOperation = std::make_unique<ReadOperation>(key, WTFMove(completionHandler)); + m_pendingReadOperationsByPriority[priority].prepend(WTFMove(readOperation)); + dispatchPendingReadOperations(); +} + +void Storage::store(const Record& record, MappedBodyHandler&& mappedBodyHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(!record.key.isNull()); + + if (!m_capacity) + return; + + auto writeOperation = std::make_unique<WriteOperation>(record, WTFMove(mappedBodyHandler)); + m_pendingWriteOperations.prepend(WTFMove(writeOperation)); + + // Add key to the filter already here as we do lookups from the pending operations too. + addToRecordFilter(record.key); + + bool isInitialWrite = m_pendingWriteOperations.size() == 1; + if (!isInitialWrite) + return; + + // Delay the start of writes a bit to avoid affecting early page load. + // Completing writes will dispatch more writes without delay. + static const auto initialWriteDelay = 1s; + m_writeOperationDispatchTimer.startOneShot(initialWriteDelay); +} + +void Storage::traverse(const String& type, TraverseFlags flags, TraverseHandler&& traverseHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(traverseHandler); + // Avoid non-thread safe Function copies. + + auto traverseOperationPtr = std::make_unique<TraverseOperation>(type, flags, WTFMove(traverseHandler)); + auto& traverseOperation = *traverseOperationPtr; + m_activeTraverseOperations.add(WTFMove(traverseOperationPtr)); + + ioQueue().dispatch([this, &traverseOperation] { + traverseRecordsFiles(recordsPath(), traverseOperation.type, [this, &traverseOperation](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + ASSERT(type == traverseOperation.type); + if (isBlob) + return; + + auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + + double worth = -1; + if (traverseOperation.flags & TraverseFlag::ComputeWorth) + worth = computeRecordWorth(fileTimes(recordPath)); + unsigned bodyShareCount = 0; + if (traverseOperation.flags & TraverseFlag::ShareCount) + bodyShareCount = m_blobStorage.shareCount(blobPathForRecordPath(recordPath)); + + std::unique_lock<Lock> lock(traverseOperation.activeMutex); + ++traverseOperation.activeCount; + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Read); + channel->read(0, std::numeric_limits<size_t>::max(), nullptr, [this, &traverseOperation, worth, bodyShareCount](Data& fileData, int) { + RecordMetaData metaData; + Data headerData; + if (decodeRecordHeader(fileData, metaData, headerData, m_salt)) { + Record record { + metaData.key, + metaData.timeStamp, + headerData, + { }, + metaData.bodyHash + }; + RecordInfo info { + static_cast<size_t>(metaData.bodySize), + worth, + bodyShareCount, + String::fromUTF8(SHA1::hexDigest(metaData.bodyHash)) + }; + traverseOperation.handler(&record, info); + } + + std::lock_guard<Lock> lock(traverseOperation.activeMutex); + --traverseOperation.activeCount; + traverseOperation.activeCondition.notifyOne(); + }); + + const unsigned maximumParallelReadCount = 5; + traverseOperation.activeCondition.wait(lock, [&traverseOperation] { + return traverseOperation.activeCount <= maximumParallelReadCount; + }); + }); + { + // Wait for all reads to finish. + std::unique_lock<Lock> lock(traverseOperation.activeMutex); + traverseOperation.activeCondition.wait(lock, [&traverseOperation] { + return !traverseOperation.activeCount; + }); + } + RunLoop::main().dispatch([this, &traverseOperation] { + traverseOperation.handler(nullptr, { }); + m_activeTraverseOperations.remove(&traverseOperation); + }); + }); +} + +void Storage::setCapacity(size_t capacity) +{ + ASSERT(RunLoop::isMain()); + +#if !ASSERT_DISABLED + const size_t assumedAverageRecordSize = 50 << 10; + size_t maximumRecordCount = capacity / assumedAverageRecordSize; + // ~10 bits per element are required for <1% false positive rate. + size_t effectiveBloomFilterCapacity = ContentsFilter::tableSize / 10; + // If this gets hit it might be time to increase the filter size. + ASSERT(maximumRecordCount < effectiveBloomFilterCapacity); +#endif + + m_capacity = capacity; + + shrinkIfNeeded(); +} + +void Storage::clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, Function<void ()>&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + LOG(NetworkCacheStorage, "(NetworkProcess) clearing cache"); + + if (m_recordFilter) + m_recordFilter->clear(); + if (m_blobFilter) + m_blobFilter->clear(); + m_approximateRecordsSize = 0; + + ioQueue().dispatch([this, modifiedSinceTime, completionHandler = WTFMove(completionHandler), type = type.isolatedCopy()] () mutable { + auto recordsPath = this->recordsPath(); + traverseRecordsFiles(recordsPath, type, [modifiedSinceTime](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + if (modifiedSinceTime > std::chrono::system_clock::time_point::min()) { + auto times = fileTimes(filePath); + if (times.modification < modifiedSinceTime) + return; + } + WebCore::deleteFile(filePath); + }); + + deleteEmptyRecordsDirectories(recordsPath); + + // This cleans unreferenced blobs. + m_blobStorage.synchronize(); + + if (completionHandler) { + RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler)] { + completionHandler(); + }); + } + }); +} + +static double computeRecordWorth(FileTimes times) +{ + using namespace std::chrono; + auto age = system_clock::now() - times.creation; + // File modification time is updated manually on cache read. We don't use access time since OS may update it automatically. + auto accessAge = times.modification - times.creation; + + // For sanity. + if (age <= 0s || accessAge < 0s || accessAge > age) + return 0; + + // We like old entries that have been accessed recently. + return duration<double>(accessAge) / age; +} + +static double deletionProbability(FileTimes times, unsigned bodyShareCount) +{ + static const double maximumProbability { 0.33 }; + static const unsigned maximumEffectiveShareCount { 5 }; + + auto worth = computeRecordWorth(times); + + // Adjust a bit so the most valuable entries don't get deleted at all. + auto effectiveWorth = std::min(1.1 * worth, 1.); + + auto probability = (1 - effectiveWorth) * maximumProbability; + + // It is less useful to remove an entry that shares its body data. + if (bodyShareCount) + probability /= std::min(bodyShareCount, maximumEffectiveShareCount); + + return probability; +} + +void Storage::shrinkIfNeeded() +{ + ASSERT(RunLoop::isMain()); + + if (approximateSize() > m_capacity) + shrink(); +} + +void Storage::shrink() +{ + ASSERT(RunLoop::isMain()); + + if (m_shrinkInProgress || m_synchronizationInProgress) + return; + m_shrinkInProgress = true; + + LOG(NetworkCacheStorage, "(NetworkProcess) shrinking cache approximateSize=%zu capacity=%zu", approximateSize(), m_capacity); + + backgroundIOQueue().dispatch([this] { + auto recordsPath = this->recordsPath(); + String anyType; + traverseRecordsFiles(recordsPath, anyType, [this](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + if (isBlob) + return; + + auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + auto blobPath = blobPathForRecordPath(recordPath); + + auto times = fileTimes(recordPath); + unsigned bodyShareCount = m_blobStorage.shareCount(blobPath); + auto probability = deletionProbability(times, bodyShareCount); + + bool shouldDelete = randomNumber() < probability; + + LOG(NetworkCacheStorage, "Deletion probability=%f bodyLinkCount=%d shouldDelete=%d", probability, bodyShareCount, shouldDelete); + + if (shouldDelete) { + WebCore::deleteFile(recordPath); + m_blobStorage.remove(blobPath); + } + }); + + RunLoop::main().dispatch([this] { + m_shrinkInProgress = false; + // We could synchronize during the shrink traversal. However this is fast and it is better to have just one code path. + synchronize(); + }); + + LOG(NetworkCacheStorage, "(NetworkProcess) cache shrink completed"); + }); +} + +void Storage::deleteOldVersions() +{ + backgroundIOQueue().dispatch([this] { + auto cachePath = basePath(); + traverseDirectory(cachePath, [&cachePath](const String& subdirName, DirectoryEntryType type) { + if (type != DirectoryEntryType::Directory) + return; + if (!subdirName.startsWith(versionDirectoryPrefix)) + return; + auto versionString = subdirName.substring(strlen(versionDirectoryPrefix)); + bool success; + unsigned directoryVersion = versionString.toUIntStrict(&success); + if (!success) + return; + if (directoryVersion >= version) + return; +#if PLATFORM(MAC) + if (directoryVersion == lastStableVersion) + return; +#endif + + auto oldVersionPath = WebCore::pathByAppendingComponent(cachePath, subdirName); + LOG(NetworkCacheStorage, "(NetworkProcess) deleting old cache version, path %s", oldVersionPath.utf8().data()); + + deleteDirectoryRecursively(oldVersionPath); + }); + }); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h new file mode 100644 index 000000000..30b5f1368 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheStorage_h +#define NetworkCacheStorage_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheBlobStorage.h" +#include "NetworkCacheData.h" +#include "NetworkCacheKey.h" +#include <WebCore/Timer.h> +#include <wtf/BloomFilter.h> +#include <wtf/Deque.h> +#include <wtf/Function.h> +#include <wtf/HashSet.h> +#include <wtf/Optional.h> +#include <wtf/WorkQueue.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { +namespace NetworkCache { + +class IOChannel; + +class Storage { + WTF_MAKE_NONCOPYABLE(Storage); +public: + static std::unique_ptr<Storage> open(const String& cachePath); + + struct Record { + WTF_MAKE_FAST_ALLOCATED; + public: + Key key; + std::chrono::system_clock::time_point timeStamp; + Data header; + Data body; + std::optional<SHA1::Digest> bodyHash; + }; + // This may call completion handler synchronously on failure. + typedef Function<bool (std::unique_ptr<Record>)> RetrieveCompletionHandler; + void retrieve(const Key&, unsigned priority, RetrieveCompletionHandler&&); + + typedef Function<void (const Data& mappedBody)> MappedBodyHandler; + void store(const Record&, MappedBodyHandler&&); + + void remove(const Key&); + void clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, Function<void ()>&& completionHandler); + + struct RecordInfo { + size_t bodySize; + double worth; // 0-1 where 1 is the most valuable. + unsigned bodyShareCount; + String bodyHash; + }; + enum TraverseFlag { + ComputeWorth = 1 << 0, + ShareCount = 1 << 1, + }; + typedef unsigned TraverseFlags; + typedef Function<void (const Record*, const RecordInfo&)> TraverseHandler; + // Null record signals end. + void traverse(const String& type, TraverseFlags, TraverseHandler&&); + + void setCapacity(size_t); + size_t capacity() const { return m_capacity; } + size_t approximateSize() const; + + static const unsigned version = 11; +#if PLATFORM(MAC) + /// Allow the last stable version of the cache to co-exist with the latest development one. + static const unsigned lastStableVersion = 9; +#endif + + String basePath() const; + String versionPath() const; + String recordsPath() const; + + const Salt& salt() const { return m_salt; } + + bool canUseSharedMemoryForBodyData() const { return m_canUseSharedMemoryForBodyData; } + + ~Storage(); + +private: + Storage(const String& directoryPath, Salt); + + String recordDirectoryPathForKey(const Key&) const; + String recordPathForKey(const Key&) const; + String blobPathForKey(const Key&) const; + + void synchronize(); + void deleteOldVersions(); + void shrinkIfNeeded(); + void shrink(); + + struct ReadOperation; + void dispatchReadOperation(std::unique_ptr<ReadOperation>); + void dispatchPendingReadOperations(); + void finishReadOperation(ReadOperation&); + void cancelAllReadOperations(); + + struct WriteOperation; + void dispatchWriteOperation(std::unique_ptr<WriteOperation>); + void dispatchPendingWriteOperations(); + void finishWriteOperation(WriteOperation&); + + std::optional<BlobStorage::Blob> storeBodyAsBlob(WriteOperation&); + Data encodeRecord(const Record&, std::optional<BlobStorage::Blob>); + void readRecord(ReadOperation&, const Data&); + + void updateFileModificationTime(const String& path); + void removeFromPendingWriteOperations(const Key&); + + WorkQueue& ioQueue() { return m_ioQueue.get(); } + WorkQueue& backgroundIOQueue() { return m_backgroundIOQueue.get(); } + WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); } + + bool mayContain(const Key&) const; + bool mayContainBlob(const Key&) const; + + void addToRecordFilter(const Key&); + + const String m_basePath; + const String m_recordsPath; + + const Salt m_salt; + + const bool m_canUseSharedMemoryForBodyData; + + size_t m_capacity { std::numeric_limits<size_t>::max() }; + size_t m_approximateRecordsSize { 0 }; + + // 2^18 bit filter can support up to 26000 entries with false positive rate < 1%. + using ContentsFilter = BloomFilter<18>; + std::unique_ptr<ContentsFilter> m_recordFilter; + std::unique_ptr<ContentsFilter> m_blobFilter; + + bool m_synchronizationInProgress { false }; + bool m_shrinkInProgress { false }; + + Vector<Key::HashType> m_recordFilterHashesAddedDuringSynchronization; + Vector<Key::HashType> m_blobFilterHashesAddedDuringSynchronization; + + static const int maximumRetrievePriority = 4; + Deque<std::unique_ptr<ReadOperation>> m_pendingReadOperationsByPriority[maximumRetrievePriority + 1]; + HashSet<std::unique_ptr<ReadOperation>> m_activeReadOperations; + WebCore::Timer m_readOperationTimeoutTimer; + + Deque<std::unique_ptr<WriteOperation>> m_pendingWriteOperations; + HashSet<std::unique_ptr<WriteOperation>> m_activeWriteOperations; + WebCore::Timer m_writeOperationDispatchTimer; + + struct TraverseOperation; + HashSet<std::unique_ptr<TraverseOperation>> m_activeTraverseOperations; + + Ref<WorkQueue> m_ioQueue; + Ref<WorkQueue> m_backgroundIOQueue; + Ref<WorkQueue> m_serialBackgroundIOQueue; + + BlobStorage m_blobStorage; +}; + +// FIXME: Remove, used by NetworkCacheStatistics only. +using RecordFileTraverseFunction = Function<void (const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath)>; +void traverseRecordsFiles(const String& recordsPath, const String& type, const RecordFileTraverseFunction&); + +} +} +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp new file mode 100644 index 000000000..561e4e6b8 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSubresourcesEntry.h" + +#include "Logging.h" +#include "NetworkCacheCoders.h" + +using namespace std::chrono; + +namespace WebKit { +namespace NetworkCache { + +void SubresourceInfo::encode(WTF::Persistence::Encoder& encoder) const +{ + encoder << m_key; + encoder << m_lastSeen; + encoder << m_firstSeen; + encoder << m_isTransient; + + // Do not bother serializing other data members of transient resources as they are empty. + if (m_isTransient) + return; + + encoder << m_firstPartyForCookies; + encoder << m_requestHeaders; + encoder.encodeEnum(m_priority); +} + +bool SubresourceInfo::decode(WTF::Persistence::Decoder& decoder, SubresourceInfo& info) +{ + if (!decoder.decode(info.m_key)) + return false; + if (!decoder.decode(info.m_lastSeen)) + return false; + if (!decoder.decode(info.m_firstSeen)) + return false; + + if (!decoder.decode(info.m_isTransient)) + return false; + + if (info.m_isTransient) + return true; + + if (!decoder.decode(info.m_firstPartyForCookies)) + return false; + + if (!decoder.decode(info.m_requestHeaders)) + return false; + + if (!decoder.decodeEnum(info.m_priority)) + return false; + + return true; +} + +Storage::Record SubresourcesEntry::encodeAsStorageRecord() const +{ + WTF::Persistence::Encoder encoder; + encoder << m_subresources; + + encoder.encodeChecksum(); + + return { m_key, m_timeStamp, { encoder.buffer(), encoder.bufferSize() }, { }, { }}; +} + +std::unique_ptr<SubresourcesEntry> SubresourcesEntry::decodeStorageRecord(const Storage::Record& storageEntry) +{ + auto entry = std::make_unique<SubresourcesEntry>(storageEntry); + + WTF::Persistence::Decoder decoder(storageEntry.header.data(), storageEntry.header.size()); + if (!decoder.decode(entry->m_subresources)) + return nullptr; + + if (!decoder.verifyChecksum()) { + LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n"); + return nullptr; + } + + return entry; +} + +SubresourcesEntry::SubresourcesEntry(const Storage::Record& storageEntry) + : m_key(storageEntry.key) + , m_timeStamp(storageEntry.timeStamp) +{ + ASSERT(m_key.type() == "SubResources"); +} + +SubresourceInfo::SubresourceInfo(const Key& key, const WebCore::ResourceRequest& request, const SubresourceInfo* previousInfo) + : m_key(key) + , m_lastSeen(std::chrono::system_clock::now()) + , m_firstSeen(previousInfo ? previousInfo->firstSeen() : m_lastSeen) + , m_isTransient(!previousInfo) + , m_firstPartyForCookies(request.firstPartyForCookies()) + , m_requestHeaders(request.httpHeaderFields()) + , m_priority(request.priority()) +{ +} + +static Vector<SubresourceInfo> makeSubresourceInfoVector(const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads, Vector<SubresourceInfo>* previousSubresources) +{ + Vector<SubresourceInfo> result; + result.reserveInitialCapacity(subresourceLoads.size()); + + HashMap<Key, unsigned> previousMap; + if (previousSubresources) { + for (unsigned i = 0; i < previousSubresources->size(); ++i) + previousMap.add(previousSubresources->at(i).key(), i); + } + + HashSet<Key> deduplicationSet; + for (auto& load : subresourceLoads) { + if (!deduplicationSet.add(load->key).isNewEntry) + continue; + + SubresourceInfo* previousInfo = nullptr; + if (previousSubresources) { + auto it = previousMap.find(load->key); + if (it != previousMap.end()) + previousInfo = &(*previousSubresources)[it->value]; + } + + result.uncheckedAppend({ load->key, load->request, previousInfo }); + + // FIXME: We should really consider all resources seen for the first time transient. + if (!previousSubresources) + result.last().setNonTransient(); + } + + return result; +} + +SubresourcesEntry::SubresourcesEntry(Key&& key, const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads) + : m_key(WTFMove(key)) + , m_timeStamp(std::chrono::system_clock::now()) + , m_subresources(makeSubresourceInfoVector(subresourceLoads, nullptr)) +{ + ASSERT(m_key.type() == "SubResources"); +} + +void SubresourcesEntry::updateSubresourceLoads(const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads) +{ + m_subresources = makeSubresourceInfoVector(subresourceLoads, &m_subresources); +} + +} // namespace WebKit +} // namespace NetworkCache + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h new file mode 100644 index 000000000..c2d5da8b1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSubresourcesEntry_h +#define NetworkCacheSubresourcesEntry_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCacheStorage.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/URL.h> +#include <wtf/HashMap.h> + +namespace WebKit { +namespace NetworkCache { + +class SubresourceInfo { + WTF_MAKE_FAST_ALLOCATED; +public: + void encode(WTF::Persistence::Encoder&) const; + static bool decode(WTF::Persistence::Decoder&, SubresourceInfo&); + + SubresourceInfo() = default; + SubresourceInfo(const Key&, const WebCore::ResourceRequest&, const SubresourceInfo* previousInfo); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point lastSeen() const { return m_lastSeen; } + std::chrono::system_clock::time_point firstSeen() const { return m_firstSeen; } + + bool isTransient() const { return m_isTransient; } + const WebCore::URL& firstPartyForCookies() const { ASSERT(!m_isTransient); return m_firstPartyForCookies; } + const WebCore::HTTPHeaderMap& requestHeaders() const { ASSERT(!m_isTransient); return m_requestHeaders; } + WebCore::ResourceLoadPriority priority() const { ASSERT(!m_isTransient); return m_priority; } + + void setNonTransient() { m_isTransient = false; } + +private: + Key m_key; + std::chrono::system_clock::time_point m_lastSeen; + std::chrono::system_clock::time_point m_firstSeen; + bool m_isTransient { false }; + WebCore::URL m_firstPartyForCookies; + WebCore::HTTPHeaderMap m_requestHeaders; + WebCore::ResourceLoadPriority m_priority; +}; + +struct SubresourceLoad { + WTF_MAKE_NONCOPYABLE(SubresourceLoad); WTF_MAKE_FAST_ALLOCATED; +public: + SubresourceLoad(const WebCore::ResourceRequest& request, const Key& key) + : request(request) + , key(key) + { } + + WebCore::ResourceRequest request; + Key key; +}; + +class SubresourcesEntry { + WTF_MAKE_NONCOPYABLE(SubresourcesEntry); WTF_MAKE_FAST_ALLOCATED; +public: + SubresourcesEntry(Key&&, const Vector<std::unique_ptr<SubresourceLoad>>&); + explicit SubresourcesEntry(const Storage::Record&); + + Storage::Record encodeAsStorageRecord() const; + static std::unique_ptr<SubresourcesEntry> decodeStorageRecord(const Storage::Record&); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; } + const Vector<SubresourceInfo>& subresources() const { return m_subresources; } + + void updateSubresourceLoads(const Vector<std::unique_ptr<SubresourceLoad>>&); + +private: + Key m_key; + std::chrono::system_clock::time_point m_timeStamp; + Vector<SubresourceInfo> m_subresources; +}; + +} // namespace WebKit +} // namespace NetworkCache + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#endif // NetworkCacheSubresourcesEntry_h |