diff options
author | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-06-27 06:07:23 +0000 |
---|---|---|
committer | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-06-27 06:07:23 +0000 |
commit | 1bf1084f2b10c3b47fd1a588d85d21ed0eb41d0c (patch) | |
tree | 46dcd36c86e7fbc6e5df36deb463b33e9967a6f7 /Source/WebKit2/NetworkProcess | |
parent | 32761a6cee1d0dee366b885b7b9c777e67885688 (diff) | |
download | WebKitGtk-tarball-master.tar.gz |
webkitgtk-2.16.5HEADwebkitgtk-2.16.5master
Diffstat (limited to 'Source/WebKit2/NetworkProcess')
116 files changed, 27897 insertions, 1904 deletions
diff --git a/Source/WebKit2/NetworkProcess/AsynchronousNetworkLoaderClient.cpp b/Source/WebKit2/NetworkProcess/AsynchronousNetworkLoaderClient.cpp deleted file mode 100644 index 3cb5cb435..000000000 --- a/Source/WebKit2/NetworkProcess/AsynchronousNetworkLoaderClient.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2013 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "AsynchronousNetworkLoaderClient.h" - -#include "DataReference.h" -#include "NetworkResourceLoader.h" -#include "WebCoreArgumentCoders.h" -#include "WebResourceLoaderMessages.h" -#include <WebCore/CertificateInfo.h> -#include <WebCore/ResourceError.h> -#include <WebCore/SharedBuffer.h> -#include <wtf/CurrentTime.h> - -#if ENABLE(NETWORK_PROCESS) - -using namespace WebCore; - -namespace WebKit { - -AsynchronousNetworkLoaderClient::AsynchronousNetworkLoaderClient() -{ -} - -void AsynchronousNetworkLoaderClient::willSendRequest(NetworkResourceLoader* loader, ResourceRequest& request, const ResourceResponse& redirectResponse) -{ - // This message is DispatchMessageEvenWhenWaitingForSyncReply to avoid a situation where the NetworkProcess is deadlocked - // waiting for 6 connections to complete while the WebProcess is waiting for a 7th (Synchronous XHR) to complete. - loader->sendAbortingOnFailure(Messages::WebResourceLoader::WillSendRequest(request, redirectResponse), IPC::DispatchMessageEvenWhenWaitingForSyncReply); -} - -#if USE(PROTECTION_SPACE_AUTH_CALLBACK) -void AsynchronousNetworkLoaderClient::canAuthenticateAgainstProtectionSpace(NetworkResourceLoader* loader, const ProtectionSpace& protectionSpace) -{ - // This message is DispatchMessageEvenWhenWaitingForSyncReply to avoid a situation where the NetworkProcess is deadlocked - // waiting for 6 connections to complete while the WebProcess is waiting for a 7th (Synchronous XHR) to complete. - loader->sendAbortingOnFailure(Messages::WebResourceLoader::CanAuthenticateAgainstProtectionSpace(protectionSpace), IPC::DispatchMessageEvenWhenWaitingForSyncReply); -} -#endif - -void AsynchronousNetworkLoaderClient::didReceiveResponse(NetworkResourceLoader* loader, const ResourceResponse& response) -{ - loader->sendAbortingOnFailure(Messages::WebResourceLoader::DidReceiveResponseWithCertificateInfo(response, CertificateInfo(response), loader->isLoadingMainResource())); -} - -void AsynchronousNetworkLoaderClient::didReceiveBuffer(NetworkResourceLoader* loader, SharedBuffer* buffer, int encodedDataLength) -{ -#if PLATFORM(IOS) || (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) - ShareableResource::Handle shareableResourceHandle; - NetworkResourceLoader::tryGetShareableHandleFromSharedBuffer(shareableResourceHandle, buffer); - if (!shareableResourceHandle.isNull()) { - // Since we're delivering this resource by ourselves all at once and don't need anymore data or callbacks from the network layer, abort the loader. - loader->abort(); - loader->send(Messages::WebResourceLoader::DidReceiveResource(shareableResourceHandle, currentTime())); - return; - } -#endif // __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090 - - IPC::DataReference dataReference(reinterpret_cast<const uint8_t*>(buffer->data()), buffer->size()); - loader->sendAbortingOnFailure(Messages::WebResourceLoader::DidReceiveData(dataReference, encodedDataLength)); -} - -void AsynchronousNetworkLoaderClient::didSendData(NetworkResourceLoader* loader, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) -{ - loader->send(Messages::WebResourceLoader::DidSendData(bytesSent, totalBytesToBeSent)); -} - -void AsynchronousNetworkLoaderClient::didFinishLoading(NetworkResourceLoader* loader, double finishTime) -{ - loader->send(Messages::WebResourceLoader::DidFinishResourceLoad(finishTime)); -} - -void AsynchronousNetworkLoaderClient::didFail(NetworkResourceLoader* loader, const ResourceError& error) -{ - loader->send(Messages::WebResourceLoader::DidFailResourceLoad(error)); -} - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.cpp b/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.cpp new file mode 100644 index 000000000..df33db9fb --- /dev/null +++ b/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2012, 2013 Apple Inc. All rights reserved. + * Copyright (C) 2017 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "CustomProtocolManager.h" + +#include "ChildProcess.h" +#include "CustomProtocolManagerMessages.h" +#include "CustomProtocolManagerProxyMessages.h" +#include "NetworkProcessCreationParameters.h" +#include "WebCoreArgumentCoders.h" +#include <WebCore/ResourceRequest.h> + +namespace WebKit { + +static uint64_t generateCustomProtocolID() +{ + static uint64_t uniqueCustomProtocolID = 0; + return ++uniqueCustomProtocolID; +} + +const char* CustomProtocolManager::supplementName() +{ + return "CustomProtocolManager"; +} + +CustomProtocolManager::CustomProtocolManager(ChildProcess* childProcess) + : m_childProcess(childProcess) +{ + m_childProcess->addMessageReceiver(Messages::CustomProtocolManager::messageReceiverName(), *this); +} + +void CustomProtocolManager::initialize(const NetworkProcessCreationParameters& parameters) +{ + registerProtocolClass(); + + for (const auto& scheme : parameters.urlSchemesRegisteredForCustomProtocols) + registerScheme(scheme); +} + +uint64_t CustomProtocolManager::addCustomProtocol(CustomProtocol&& customProtocol) +{ + LockHolder locker(m_customProtocolMapMutex); + auto customProtocolID = generateCustomProtocolID(); + m_customProtocolMap.add(customProtocolID, WTFMove(customProtocol)); + return customProtocolID; +} + +void CustomProtocolManager::removeCustomProtocol(uint64_t customProtocolID) +{ + LockHolder locker(m_customProtocolMapMutex); + m_customProtocolMap.remove(customProtocolID); +} + +void CustomProtocolManager::startLoading(uint64_t customProtocolID, const WebCore::ResourceRequest& request) +{ + m_childProcess->send(Messages::CustomProtocolManagerProxy::StartLoading(customProtocolID, request)); +} + +void CustomProtocolManager::stopLoading(uint64_t customProtocolID) +{ + m_childProcess->send(Messages::CustomProtocolManagerProxy::StopLoading(customProtocolID), 0); +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.h b/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.h new file mode 100644 index 000000000..95937ba10 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.h @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2012, 2013 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "MessageReceiver.h" +#include "NetworkProcessSupplement.h" +#include <wtf/HashMap.h> +#include <wtf/HashSet.h> +#include <wtf/text/StringHash.h> +#include <wtf/text/WTFString.h> + +#if PLATFORM(COCOA) +#include <wtf/RetainPtr.h> +OBJC_CLASS NSURLSessionConfiguration; +OBJC_CLASS WKCustomProtocol; +#endif + +#if USE(SOUP) +#include <wtf/glib/GRefPtr.h> + +typedef struct _GCancellable GCancellable; +typedef struct _GInputStream GInputStream; +typedef struct _GTask GTask; +typedef struct _WebKitSoupRequestGeneric WebKitSoupRequestGeneric; +#endif + +namespace IPC { +class DataReference; +} // namespace IPC + +namespace WebCore { +class ResourceError; +class ResourceRequest; +class ResourceResponse; +} // namespace WebCore + +namespace WebKit { + +class ChildProcess; +struct NetworkProcessCreationParameters; + +class CustomProtocolManager : public NetworkProcessSupplement, public IPC::MessageReceiver { + WTF_MAKE_NONCOPYABLE(CustomProtocolManager); +public: + explicit CustomProtocolManager(ChildProcess*); + + static const char* supplementName(); + + void registerScheme(const String&); + void unregisterScheme(const String&); + bool supportsScheme(const String&); + +#if PLATFORM(COCOA) + typedef RetainPtr<WKCustomProtocol> CustomProtocol; +#endif +#if USE(SOUP) + struct WebSoupRequestAsyncData { + WebSoupRequestAsyncData(GRefPtr<GTask>&&, WebKitSoupRequestGeneric*); + ~WebSoupRequestAsyncData(); + + GRefPtr<GTask> task; + WebKitSoupRequestGeneric* request; + GRefPtr<GCancellable> cancellable; + GRefPtr<GInputStream> stream; + }; + typedef std::unique_ptr<WebSoupRequestAsyncData> CustomProtocol; +#endif + + uint64_t addCustomProtocol(CustomProtocol&&); + void removeCustomProtocol(uint64_t customProtocolID); + void startLoading(uint64_t customProtocolID, const WebCore::ResourceRequest&); + void stopLoading(uint64_t customProtocolID); + +#if PLATFORM(COCOA) && USE(NETWORK_SESSION) + void registerProtocolClass(NSURLSessionConfiguration*); +#endif + +private: + // NetworkProcessSupplement + void initialize(const NetworkProcessCreationParameters&) override; + + // IPC::MessageReceiver + void didReceiveMessage(IPC::Connection&, IPC::Decoder&) override; + + void didFailWithError(uint64_t customProtocolID, const WebCore::ResourceError&); + void didLoadData(uint64_t customProtocolID, const IPC::DataReference&); + void didReceiveResponse(uint64_t customProtocolID, const WebCore::ResourceResponse&, uint32_t cacheStoragePolicy); + void didFinishLoading(uint64_t customProtocolID); + void wasRedirectedToRequest(uint64_t customProtocolID, const WebCore::ResourceRequest&, const WebCore::ResourceResponse& redirectResponse); + + void registerProtocolClass(); + + ChildProcess* m_childProcess; + + typedef HashMap<uint64_t, CustomProtocol> CustomProtocolMap; + CustomProtocolMap m_customProtocolMap; + Lock m_customProtocolMapMutex; + +#if PLATFORM(COCOA) + HashSet<String, ASCIICaseInsensitiveHash> m_registeredSchemes; + Lock m_registeredSchemesMutex; + + // WKCustomProtocol objects can be removed from the m_customProtocolMap from multiple threads. + // We return a RetainPtr here because it is unsafe to return a raw pointer since the object might immediately be destroyed from a different thread. + RetainPtr<WKCustomProtocol> protocolForID(uint64_t customProtocolID); +#endif + +#if USE(SOUP) + GRefPtr<GPtrArray> m_registeredSchemes; +#endif +}; + +} // namespace WebKit + diff --git a/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.messages.in b/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.messages.in new file mode 100644 index 000000000..0f55eceb1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/CustomProtocols/CustomProtocolManager.messages.in @@ -0,0 +1,32 @@ +# Copyright (C) 2012 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +messages -> CustomProtocolManager { + DidFailWithError(uint64_t customProtocolID, WebCore::ResourceError error) + DidLoadData(uint64_t customProtocolID, IPC::DataReference data) + DidReceiveResponse(uint64_t customProtocolID, WebCore::ResourceResponse response, uint32_t cacheStoragePolicy) + DidFinishLoading(uint64_t customProtocolID) + WasRedirectedToRequest(uint64_t customProtocolID, WebCore::ResourceRequest request, WebCore::ResourceResponse redirectResponse); + + RegisterScheme(String name) + UnregisterScheme(String name) +} diff --git a/Source/WebKit2/NetworkProcess/CustomProtocols/soup/CustomProtocolManagerSoup.cpp b/Source/WebKit2/NetworkProcess/CustomProtocols/soup/CustomProtocolManagerSoup.cpp new file mode 100644 index 000000000..43a03b8cb --- /dev/null +++ b/Source/WebKit2/NetworkProcess/CustomProtocols/soup/CustomProtocolManagerSoup.cpp @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2013 Igalia S.L. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" +#include "CustomProtocolManager.h" + +#include "CustomProtocolManagerMessages.h" +#include "DataReference.h" +#include "NetworkProcess.h" +#include "WebKitSoupRequestInputStream.h" +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/NotImplemented.h> +#include <WebCore/ResourceError.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SoupNetworkSession.h> +#include <WebCore/WebKitSoupRequestGeneric.h> +#include <wtf/NeverDestroyed.h> + +using namespace WebCore; + +namespace WebKit { + + +CustomProtocolManager::WebSoupRequestAsyncData::WebSoupRequestAsyncData(GRefPtr<GTask>&& requestTask, WebKitSoupRequestGeneric* requestGeneric) + : task(WTFMove(requestTask)) + , request(requestGeneric) + , cancellable(g_task_get_cancellable(task.get())) +{ + // If the struct contains a null request, it is because the request failed. + g_object_add_weak_pointer(G_OBJECT(request), reinterpret_cast<void**>(&request)); +} + +CustomProtocolManager::WebSoupRequestAsyncData::~WebSoupRequestAsyncData() +{ + if (request) + g_object_remove_weak_pointer(G_OBJECT(request), reinterpret_cast<void**>(&request)); +} + +class CustomProtocolRequestClient final : public WebKitSoupRequestGenericClient { +public: + static CustomProtocolRequestClient& singleton() + { + static NeverDestroyed<CustomProtocolRequestClient> client; + return client; + } + +private: + void startRequest(GRefPtr<GTask>&& task) override + { + WebKitSoupRequestGeneric* request = WEBKIT_SOUP_REQUEST_GENERIC(g_task_get_source_object(task.get())); + auto* customProtocolManager = NetworkProcess::singleton().supplement<CustomProtocolManager>(); + if (!customProtocolManager) + return; + + auto customProtocolID = customProtocolManager->addCustomProtocol(std::make_unique<CustomProtocolManager::WebSoupRequestAsyncData>(WTFMove(task), request)); + customProtocolManager->startLoading(customProtocolID, webkitSoupRequestGenericGetRequest(request)); + } +}; + +void CustomProtocolManager::registerProtocolClass() +{ + static_cast<WebKitSoupRequestGenericClass*>(g_type_class_ref(WEBKIT_TYPE_SOUP_REQUEST_GENERIC))->client = &CustomProtocolRequestClient::singleton(); + SoupNetworkSession::setCustomProtocolRequestType(WEBKIT_TYPE_SOUP_REQUEST_GENERIC); +} + +void CustomProtocolManager::registerScheme(const String& scheme) +{ + if (!m_registeredSchemes) + m_registeredSchemes = adoptGRef(g_ptr_array_new_with_free_func(g_free)); + + if (m_registeredSchemes->len) + g_ptr_array_remove_index_fast(m_registeredSchemes.get(), m_registeredSchemes->len - 1); + g_ptr_array_add(m_registeredSchemes.get(), g_strdup(scheme.utf8().data())); + g_ptr_array_add(m_registeredSchemes.get(), nullptr); + + auto* genericRequestClass = static_cast<SoupRequestClass*>(g_type_class_peek(WEBKIT_TYPE_SOUP_REQUEST_GENERIC)); + ASSERT(genericRequestClass); + genericRequestClass->schemes = const_cast<const char**>(reinterpret_cast<char**>(m_registeredSchemes->pdata)); + NetworkStorageSession::forEach([](const WebCore::NetworkStorageSession& session) { + if (auto* soupSession = session.soupNetworkSession()) + soupSession->setupCustomProtocols(); + }); +} + +void CustomProtocolManager::unregisterScheme(const String&) +{ + notImplemented(); +} + +bool CustomProtocolManager::supportsScheme(const String& scheme) +{ + if (scheme.isNull()) + return false; + + CString cScheme = scheme.utf8(); + for (unsigned i = 0; i < m_registeredSchemes->len; ++i) { + if (cScheme == static_cast<char*>(g_ptr_array_index(m_registeredSchemes.get(), i))) + return true; + } + + return false; +} + +void CustomProtocolManager::didFailWithError(uint64_t customProtocolID, const ResourceError& error) +{ + auto* data = m_customProtocolMap.get(customProtocolID); + ASSERT(data); + + // Either we haven't started reading the stream yet, in which case we need to complete the + // task first, or we failed reading it and the task was already completed by didLoadData(). + ASSERT(!data->stream || !data->task); + + if (!data->stream) { + GRefPtr<GTask> task = std::exchange(data->task, nullptr); + ASSERT(task.get()); + g_task_return_new_error(task.get(), g_quark_from_string(error.domain().utf8().data()), + error.errorCode(), "%s", error.localizedDescription().utf8().data()); + } else + webkitSoupRequestInputStreamDidFailWithError(WEBKIT_SOUP_REQUEST_INPUT_STREAM(data->stream.get()), error); + + removeCustomProtocol(customProtocolID); +} + +void CustomProtocolManager::didLoadData(uint64_t customProtocolID, const IPC::DataReference& dataReference) +{ + auto* data = m_customProtocolMap.get(customProtocolID); + // The data might have been removed from the request map if a previous chunk failed + // and a new message was sent by the UI process before being notified about the failure. + if (!data) + return; + + if (!data->stream) { + GRefPtr<GTask> task = std::exchange(data->task, nullptr); + ASSERT(task.get()); + + goffset soupContentLength = soup_request_get_content_length(SOUP_REQUEST(g_task_get_source_object(task.get()))); + uint64_t contentLength = soupContentLength == -1 ? 0 : static_cast<uint64_t>(soupContentLength); + if (!dataReference.size()) { + // Empty reply, just create and empty GMemoryInputStream. + data->stream = g_memory_input_stream_new(); + } else if (dataReference.size() == contentLength) { + // We don't expect more data, so we can just create a GMemoryInputStream with all the data. + data->stream = g_memory_input_stream_new_from_data(g_memdup(dataReference.data(), dataReference.size()), contentLength, g_free); + } else { + // We expect more data chunks from the UI process. + data->stream = webkitSoupRequestInputStreamNew(contentLength); + webkitSoupRequestInputStreamAddData(WEBKIT_SOUP_REQUEST_INPUT_STREAM(data->stream.get()), dataReference.data(), dataReference.size()); + } + g_task_return_pointer(task.get(), data->stream.get(), g_object_unref); + return; + } + + if (g_cancellable_is_cancelled(data->cancellable.get()) || !data->request) { + // ResourceRequest failed or it was cancelled. It doesn't matter here the error or if it was cancelled, + // because that's already handled by the resource handle client, we just want to notify the UI process + // to stop reading data from the user input stream. If UI process already sent all the data we simply + // finish silently. + if (!webkitSoupRequestInputStreamFinished(WEBKIT_SOUP_REQUEST_INPUT_STREAM(data->stream.get()))) + stopLoading(customProtocolID); + + return; + } + + webkitSoupRequestInputStreamAddData(WEBKIT_SOUP_REQUEST_INPUT_STREAM(data->stream.get()), dataReference.data(), dataReference.size()); +} + +void CustomProtocolManager::didReceiveResponse(uint64_t customProtocolID, const ResourceResponse& response, uint32_t) +{ + auto* data = m_customProtocolMap.get(customProtocolID); + // The data might have been removed from the request map if an error happened even before this point. + if (!data) + return; + + ASSERT(data->task); + + WebKitSoupRequestGeneric* request = WEBKIT_SOUP_REQUEST_GENERIC(g_task_get_source_object(data->task.get())); + webkitSoupRequestGenericSetContentLength(request, response.expectedContentLength() ? response.expectedContentLength() : -1); + webkitSoupRequestGenericSetContentType(request, !response.mimeType().isEmpty() ? response.mimeType().utf8().data() : 0); +} + +void CustomProtocolManager::didFinishLoading(uint64_t customProtocolID) +{ + ASSERT(m_customProtocolMap.contains(customProtocolID)); + removeCustomProtocol(customProtocolID); +} + +void CustomProtocolManager::wasRedirectedToRequest(uint64_t, const ResourceRequest&, const ResourceResponse&) +{ + notImplemented(); +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/Downloads/BlobDownloadClient.cpp b/Source/WebKit2/NetworkProcess/Downloads/BlobDownloadClient.cpp new file mode 100644 index 000000000..cf9acce9b --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/BlobDownloadClient.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "BlobDownloadClient.h" + +#if !USE(NETWORK_SESSION) + +#include "DataReference.h" +#include "Download.h" +#include "WebErrors.h" +#include <WebCore/MIMETypeRegistry.h> +#include <WebCore/ResourceError.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> + +namespace WebKit { + +using namespace WebCore; + +BlobDownloadClient::BlobDownloadClient(Download& download) + : m_download(download) +{ +} + +void BlobDownloadClient::didReceiveResponseAsync(ResourceHandle*, ResourceResponse&& response) +{ + m_download.didReceiveResponse(WTFMove(response)); + String suggestedFilename = MIMETypeRegistry::appendFileExtensionIfNecessary(m_download.suggestedName().isEmpty() ? ASCIILiteral("unknown") : m_download.suggestedName(), response.mimeType()); + m_download.decideDestinationWithSuggestedFilenameAsync(suggestedFilename); +} + +void BlobDownloadClient::didDecideDownloadDestination(const String& destinationPath, bool allowOverwrite) +{ + ASSERT(!destinationPath.isEmpty()); + + if (fileExists(destinationPath)) { + if (!allowOverwrite) { + didFail(nullptr, cancelledError(m_download.request())); + return; + } + deleteFile(destinationPath); + } + + m_destinationPath = destinationPath; + m_destinationFile = openFile(m_destinationPath, OpenForWrite); + m_download.didCreateDestination(m_destinationPath); + + m_download.continueDidReceiveResponse(); +} + +void BlobDownloadClient::didReceiveBuffer(ResourceHandle*, Ref<SharedBuffer>&& buffer, int) +{ + writeToFile(m_destinationFile, buffer->data(), buffer->size()); + m_download.didReceiveData(buffer->size()); +} + +void BlobDownloadClient::didFinishLoading(ResourceHandle*, double) +{ + closeFile(m_destinationFile); + m_download.didFinish(); +} + +void BlobDownloadClient::didCancel() +{ + closeFile(m_destinationFile); + if (!m_destinationPath.isEmpty()) + deleteFile(m_destinationPath); + + m_download.didCancel(IPC::DataReference()); +} + +void BlobDownloadClient::didFail(ResourceHandle*, const ResourceError& error) +{ + closeFile(m_destinationFile); + if (!m_destinationPath.isEmpty()) + deleteFile(m_destinationPath); + + m_download.didFail(error, IPC::DataReference()); +} + +} + +#endif // !USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/Downloads/BlobDownloadClient.h b/Source/WebKit2/NetworkProcess/Downloads/BlobDownloadClient.h new file mode 100644 index 000000000..4a3e63ee3 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/BlobDownloadClient.h @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if !USE(NETWORK_SESSION) + +#include <WebCore/FileSystem.h> +#include <WebCore/ResourceHandleClient.h> +#include <wtf/RefPtr.h> + +namespace WebKit { + +class Download; +class NetworkLoadParameters; + +class BlobDownloadClient final : public WebCore::ResourceHandleClient { +public: + explicit BlobDownloadClient(Download&); + + void didCancel(); + void didDecideDownloadDestination(const String& destinationPath, bool allowOverwrite); + +private: + // ResourceHandleClient + void didReceiveResponseAsync(WebCore::ResourceHandle*, WebCore::ResourceResponse&&) final; + void didReceiveBuffer(WebCore::ResourceHandle*, Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) final; + void didFinishLoading(WebCore::ResourceHandle*, double finishTime) final; + void didFail(WebCore::ResourceHandle*, const WebCore::ResourceError&) final; + bool usesAsyncCallbacks() final { return true; } + + Download& m_download; + String m_destinationPath; + WebCore::PlatformFileHandle m_destinationFile { WebCore::invalidPlatformFileHandle }; +}; + +} + +#endif // !USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/Downloads/Download.cpp b/Source/WebKit2/NetworkProcess/Downloads/Download.cpp new file mode 100644 index 000000000..bf8443b57 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/Download.cpp @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2010-2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "Download.h" + +#include "AuthenticationManager.h" +#include "BlobDownloadClient.h" +#include "Connection.h" +#include "DataReference.h" +#include "DownloadManager.h" +#include "DownloadProxyMessages.h" +#include "Logging.h" +#include "NetworkDataTask.h" +#include "SandboxExtension.h" +#include "WebCoreArgumentCoders.h" +#include <WebCore/NotImplemented.h> + +using namespace WebCore; + +#define RELEASE_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(isAlwaysOnLoggingAllowed(), Network, "%p - Download::" fmt, this, ##__VA_ARGS__) + +namespace WebKit { + +#if USE(NETWORK_SESSION) +Download::Download(DownloadManager& downloadManager, DownloadID downloadID, NetworkDataTask& download, const SessionID& sessionID, const String& suggestedName) + : m_downloadManager(downloadManager) + , m_downloadID(downloadID) + , m_download(&download) + , m_sessionID(sessionID) + , m_suggestedName(suggestedName) +{ + ASSERT(m_downloadID.downloadID()); + + m_downloadManager.didCreateDownload(); +} +#if PLATFORM(COCOA) +Download::Download(DownloadManager& downloadManager, DownloadID downloadID, NSURLSessionDownloadTask* download, const SessionID& sessionID, const String& suggestedName) + : m_downloadManager(downloadManager) + , m_downloadID(downloadID) + , m_downloadTask(download) + , m_sessionID(sessionID) + , m_suggestedName(suggestedName) +{ + ASSERT(m_downloadID.downloadID()); + + m_downloadManager.didCreateDownload(); +} +#endif +#else +Download::Download(DownloadManager& downloadManager, DownloadID downloadID, const ResourceRequest& request, const String& suggestedName) + : m_downloadManager(downloadManager) + , m_downloadID(downloadID) + , m_request(request) + , m_suggestedName(suggestedName) +{ + ASSERT(m_downloadID.downloadID()); + + m_downloadManager.didCreateDownload(); +} +#endif // USE(NETWORK_SESSION) + +Download::~Download() +{ +#if !USE(NETWORK_SESSION) + for (auto& fileReference : m_blobFileReferences) + fileReference->revokeFileAccess(); + + if (m_resourceHandle) { + m_resourceHandle->clearClient(); + m_resourceHandle->cancel(); + m_resourceHandle = nullptr; + } + m_downloadClient = nullptr; + + platformInvalidate(); +#endif + + m_downloadManager.didDestroyDownload(); +} + +#if !USE(NETWORK_SESSION) +void Download::start() +{ + if (m_request.url().protocolIsBlob()) { + m_downloadClient = std::make_unique<BlobDownloadClient>(*this); + m_resourceHandle = ResourceHandle::create(nullptr, m_request, m_downloadClient.get(), false, false); + didStart(); + return; + } + + startNetworkLoad(); +} + +void Download::startWithHandle(ResourceHandle* handle, const ResourceResponse& response) +{ + if (m_request.url().protocolIsBlob()) { + m_downloadClient = std::make_unique<BlobDownloadClient>(*this); + m_resourceHandle = ResourceHandle::create(nullptr, m_request, m_downloadClient.get(), false, false); + didStart(); + return; + } + + startNetworkLoadWithHandle(handle, response); +} +#endif + +void Download::cancel() +{ +#if USE(NETWORK_SESSION) + if (m_download) { + m_download->cancel(); + didCancel({ }); + return; + } +#else + if (m_request.url().protocolIsBlob()) { + auto resourceHandle = WTFMove(m_resourceHandle); + resourceHandle->cancel(); + static_cast<BlobDownloadClient*>(m_downloadClient.get())->didCancel(); + return; + } +#endif + platformCancelNetworkLoad(); +} + +#if !USE(NETWORK_SESSION) +void Download::didStart() +{ + send(Messages::DownloadProxy::DidStart(m_request, m_suggestedName)); +} + +void Download::didReceiveAuthenticationChallenge(const AuthenticationChallenge& authenticationChallenge) +{ + m_downloadManager.downloadsAuthenticationManager().didReceiveAuthenticationChallenge(*this, authenticationChallenge); +} + +void Download::didReceiveResponse(const ResourceResponse& response) +{ + RELEASE_LOG_IF_ALLOWED("didReceiveResponse: Created (id = %" PRIu64 ")", downloadID().downloadID()); + + m_responseMIMEType = response.mimeType(); + send(Messages::DownloadProxy::DidReceiveResponse(response)); +} + +bool Download::shouldDecodeSourceDataOfMIMEType(const String& mimeType) +{ + bool result; + if (!sendSync(Messages::DownloadProxy::ShouldDecodeSourceDataOfMIMEType(mimeType), Messages::DownloadProxy::ShouldDecodeSourceDataOfMIMEType::Reply(result))) + return true; + + return result; +} + +String Download::decideDestinationWithSuggestedFilename(const String& filename, bool& allowOverwrite) +{ + String destination; + SandboxExtension::Handle sandboxExtensionHandle; + if (!sendSync(Messages::DownloadProxy::DecideDestinationWithSuggestedFilename(filename, m_responseMIMEType), Messages::DownloadProxy::DecideDestinationWithSuggestedFilename::Reply(destination, allowOverwrite, sandboxExtensionHandle))) + return String(); + + m_sandboxExtension = SandboxExtension::create(sandboxExtensionHandle); + if (m_sandboxExtension) + m_sandboxExtension->consume(); + + return destination; +} + +void Download::decideDestinationWithSuggestedFilenameAsync(const String& suggestedFilename) +{ + send(Messages::DownloadProxy::DecideDestinationWithSuggestedFilenameAsync(downloadID(), suggestedFilename)); +} + +void Download::didDecideDownloadDestination(const String& destinationPath, const SandboxExtension::Handle& sandboxExtensionHandle, bool allowOverwrite) +{ + ASSERT(!m_sandboxExtension); + m_sandboxExtension = SandboxExtension::create(sandboxExtensionHandle); + if (m_sandboxExtension) + m_sandboxExtension->consume(); + + if (m_request.url().protocolIsBlob()) { + static_cast<BlobDownloadClient*>(m_downloadClient.get())->didDecideDownloadDestination(destinationPath, allowOverwrite); + return; + } + + // For now, only Blob URL downloads go through this code path. + ASSERT_NOT_REACHED(); +} + +void Download::continueDidReceiveResponse() +{ + m_resourceHandle->continueDidReceiveResponse(); +} +#endif + +void Download::didCreateDestination(const String& path) +{ + send(Messages::DownloadProxy::DidCreateDestination(path)); +} + +void Download::didReceiveData(uint64_t length) +{ + if (!m_hasReceivedData) { + RELEASE_LOG_IF_ALLOWED("didReceiveData: Started receiving data (id = %" PRIu64 ")", downloadID().downloadID()); + m_hasReceivedData = true; + } + + send(Messages::DownloadProxy::DidReceiveData(length)); +} + +void Download::didFinish() +{ + RELEASE_LOG_IF_ALLOWED("didFinish: (id = %" PRIu64 ")", downloadID().downloadID()); + +#if !USE(NETWORK_SESSION) + platformDidFinish(); +#endif + + send(Messages::DownloadProxy::DidFinish()); + + if (m_sandboxExtension) { + m_sandboxExtension->revoke(); + m_sandboxExtension = nullptr; + } + + m_downloadManager.downloadFinished(this); +} + +void Download::didFail(const ResourceError& error, const IPC::DataReference& resumeData) +{ + RELEASE_LOG_IF_ALLOWED("didFail: (id = %" PRIu64 ", isTimeout = %d, isCancellation = %d, errCode = %d)", + downloadID().downloadID(), error.isTimeout(), error.isCancellation(), error.errorCode()); + + send(Messages::DownloadProxy::DidFail(error, resumeData)); + + if (m_sandboxExtension) { + m_sandboxExtension->revoke(); + m_sandboxExtension = nullptr; + } + m_downloadManager.downloadFinished(this); +} + +void Download::didCancel(const IPC::DataReference& resumeData) +{ + RELEASE_LOG_IF_ALLOWED("didCancel: (id = %" PRIu64 ")", downloadID().downloadID()); + + send(Messages::DownloadProxy::DidCancel(resumeData)); + + if (m_sandboxExtension) { + m_sandboxExtension->revoke(); + m_sandboxExtension = nullptr; + } + m_downloadManager.downloadFinished(this); +} + +IPC::Connection* Download::messageSenderConnection() +{ + return m_downloadManager.downloadProxyConnection(); +} + +uint64_t Download::messageSenderDestinationID() +{ + return m_downloadID.downloadID(); +} + +bool Download::isAlwaysOnLoggingAllowed() const +{ +#if USE(NETWORK_SESSION) && PLATFORM(COCOA) + return m_sessionID.isAlwaysOnLoggingAllowed(); +#else + return false; +#endif +} + +#if !PLATFORM(COCOA) +void Download::platformCancelNetworkLoad() +{ +} +#endif + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/Downloads/Download.h b/Source/WebKit2/NetworkProcess/Downloads/Download.h new file mode 100644 index 000000000..ae4d7cff6 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/Download.h @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2010-2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef Download_h +#define Download_h + +#include "DownloadID.h" +#include "MessageSender.h" +#include "SandboxExtension.h" +#include <WebCore/ResourceHandle.h> +#include <WebCore/ResourceHandleClient.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/SessionID.h> +#include <memory> +#include <wtf/Noncopyable.h> +#include <wtf/RetainPtr.h> + +#if USE(NETWORK_SESSION) +#if PLATFORM(COCOA) +OBJC_CLASS NSURLSessionDownloadTask; +#endif +#else // USE(NETWORK_SESSION) +#if PLATFORM(COCOA) +OBJC_CLASS NSURLDownload; +OBJC_CLASS WKDownloadAsDelegate; +#endif +#endif // USE(NETWORK_SESSION) + +#if USE(CFURLCONNECTION) +#include <CFNetwork/CFURLDownloadPriv.h> +#endif + +namespace IPC { +class DataReference; +} + +namespace WebCore { +class AuthenticationChallenge; +class BlobDataFileReference; +class Credential; +class ResourceError; +class ResourceHandle; +class ResourceResponse; +} + +namespace WebKit { + +class DownloadManager; +class NetworkDataTask; +class NetworkSession; +class WebPage; + +class Download : public IPC::MessageSender { + WTF_MAKE_NONCOPYABLE(Download); WTF_MAKE_FAST_ALLOCATED; +public: +#if USE(NETWORK_SESSION) + Download(DownloadManager&, DownloadID, NetworkDataTask&, const WebCore::SessionID& sessionID, const String& suggestedFilename = { }); +#if PLATFORM(COCOA) + Download(DownloadManager&, DownloadID, NSURLSessionDownloadTask*, const WebCore::SessionID& sessionID, const String& suggestedFilename = { }); +#endif +#else + Download(DownloadManager&, DownloadID, const WebCore::ResourceRequest&, const String& suggestedFilename = { }); +#endif + + ~Download(); + +#if !USE(NETWORK_SESSION) + void setBlobFileReferences(Vector<RefPtr<WebCore::BlobDataFileReference>>&& fileReferences) { m_blobFileReferences = WTFMove(fileReferences); } + + void start(); + void startWithHandle(WebCore::ResourceHandle*, const WebCore::ResourceResponse&); +#endif + void resume(const IPC::DataReference& resumeData, const String& path, const SandboxExtension::Handle&); + void cancel(); + + DownloadID downloadID() const { return m_downloadID; } + const String& suggestedName() const { return m_suggestedName; } + +#if USE(NETWORK_SESSION) + void setSandboxExtension(RefPtr<SandboxExtension>&& sandboxExtension) { m_sandboxExtension = WTFMove(sandboxExtension); } +#else + const WebCore::ResourceRequest& request() const { return m_request; } + void didReceiveAuthenticationChallenge(const WebCore::AuthenticationChallenge&); + void didStart(); + void didReceiveResponse(const WebCore::ResourceResponse&); + bool shouldDecodeSourceDataOfMIMEType(const String& mimeType); + String decideDestinationWithSuggestedFilename(const String& filename, bool& allowOverwrite); + void decideDestinationWithSuggestedFilenameAsync(const String&); + void didDecideDownloadDestination(const String& destinationPath, const SandboxExtension::Handle&, bool allowOverwrite); + void continueDidReceiveResponse(); + void platformDidFinish(); +#endif + void didCreateDestination(const String& path); + void didReceiveData(uint64_t length); + void didFinish(); + void didFail(const WebCore::ResourceError&, const IPC::DataReference& resumeData); + void didCancel(const IPC::DataReference& resumeData); + +private: + // IPC::MessageSender + IPC::Connection* messageSenderConnection() override; + uint64_t messageSenderDestinationID() override; + +#if !USE(NETWORK_SESSION) + void startNetworkLoad(); + void startNetworkLoadWithHandle(WebCore::ResourceHandle*, const WebCore::ResourceResponse&); + void platformInvalidate(); +#endif + void platformCancelNetworkLoad(); + + bool isAlwaysOnLoggingAllowed() const; + + DownloadManager& m_downloadManager; + DownloadID m_downloadID; + + Vector<RefPtr<WebCore::BlobDataFileReference>> m_blobFileReferences; + RefPtr<SandboxExtension> m_sandboxExtension; + +#if USE(NETWORK_SESSION) + RefPtr<NetworkDataTask> m_download; +#if PLATFORM(COCOA) + RetainPtr<NSURLSessionDownloadTask> m_downloadTask; +#endif + WebCore::SessionID m_sessionID; +#else // USE(NETWORK_SESSION) + WebCore::ResourceRequest m_request; + String m_responseMIMEType; +#if PLATFORM(COCOA) + RetainPtr<NSURLDownload> m_nsURLDownload; + RetainPtr<WKDownloadAsDelegate> m_delegate; +#endif +#if USE(CFURLCONNECTION) + RetainPtr<CFURLDownloadRef> m_download; +#endif + std::unique_ptr<WebCore::ResourceHandleClient> m_downloadClient; + RefPtr<WebCore::ResourceHandle> m_resourceHandle; +#endif // USE(NETWORK_SESSION) + String m_suggestedName; + bool m_hasReceivedData { false }; +}; + +} // namespace WebKit + +#endif // Download_h diff --git a/Source/WebKit2/NetworkProcess/Downloads/DownloadID.h b/Source/WebKit2/NetworkProcess/Downloads/DownloadID.h new file mode 100644 index 000000000..04df707f8 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/DownloadID.h @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DownloadID_h +#define DownloadID_h + +#include "ArgumentCoder.h" +#include "Decoder.h" +#include "Encoder.h" +#include <wtf/HashTraits.h> + +namespace WebKit { + +class DownloadID { +public: + DownloadID() + { + } + + explicit DownloadID(uint64_t downloadID) + : m_downloadID(downloadID) + { + } + + bool operator==(DownloadID other) const { return m_downloadID == other.m_downloadID; } + bool operator!=(DownloadID other) const { return m_downloadID != other.m_downloadID; } + + uint64_t downloadID() const { return m_downloadID; } +private: + uint64_t m_downloadID { 0 }; +}; + +} + +namespace IPC { + +template<> struct ArgumentCoder<WebKit::DownloadID> { + static void encode(Encoder& encoder, const WebKit::DownloadID& downloadID) + { + encoder << downloadID.downloadID(); + } + static bool decode(Decoder& decoder, WebKit::DownloadID& downloadID) + { + uint64_t id; + if (!decoder.decode(id)) + return false; + + downloadID = WebKit::DownloadID(id); + + return true; + } +}; + +} + +namespace WTF { + +struct DownloadIDHash { + static unsigned hash(const WebKit::DownloadID& d) { return intHash(d.downloadID()); } + static bool equal(const WebKit::DownloadID& a, const WebKit::DownloadID& b) { return a.downloadID() == b.downloadID(); } + static const bool safeToCompareToEmptyOrDeleted = true; +}; +template<> struct HashTraits<WebKit::DownloadID> : GenericHashTraits<WebKit::DownloadID> { + static WebKit::DownloadID emptyValue() { return { }; } + + static void constructDeletedValue(WebKit::DownloadID& slot) { slot = WebKit::DownloadID(std::numeric_limits<uint64_t>::max()); } + static bool isDeletedValue(const WebKit::DownloadID& slot) { return slot.downloadID() == std::numeric_limits<uint64_t>::max(); } +}; +template<> struct DefaultHash<WebKit::DownloadID> { + typedef DownloadIDHash Hash; +}; + +} +#endif /* DownloadID_h */ diff --git a/Source/WebKit2/NetworkProcess/Downloads/DownloadManager.cpp b/Source/WebKit2/NetworkProcess/Downloads/DownloadManager.cpp new file mode 100644 index 000000000..b76ba47bd --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/DownloadManager.cpp @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2010-2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "DownloadManager.h" + +#include "Download.h" +#include "NetworkBlobRegistry.h" +#include "NetworkLoad.h" +#include "NetworkSession.h" +#include "PendingDownload.h" +#include "SessionTracker.h" +#include <WebCore/NotImplemented.h> +#include <WebCore/SessionID.h> +#include <wtf/StdLibExtras.h> + +using namespace WebCore; + +namespace WebKit { + +DownloadManager::DownloadManager(Client& client) + : m_client(client) +{ +} + +void DownloadManager::startDownload(NetworkConnectionToWebProcess* connection, SessionID sessionID, DownloadID downloadID, const ResourceRequest& request, const String& suggestedName) +{ +#if USE(NETWORK_SESSION) + auto* networkSession = SessionTracker::networkSession(sessionID); + if (!networkSession) + return; + + NetworkLoadParameters parameters; + parameters.sessionID = sessionID; + parameters.request = request; + parameters.clientCredentialPolicy = ClientCredentialPolicy::MayAskClientForCredentials; + if (request.url().protocolIsBlob() && connection) + parameters.blobFileReferences = NetworkBlobRegistry::singleton().filesInBlob(*connection, request.url()); + parameters.allowStoredCredentials = sessionID.isEphemeral() ? DoNotAllowStoredCredentials : AllowStoredCredentials; + + m_pendingDownloads.add(downloadID, std::make_unique<PendingDownload>(WTFMove(parameters), downloadID, *networkSession, suggestedName)); +#else + auto download = std::make_unique<Download>(*this, downloadID, request, suggestedName); + if (request.url().protocolIsBlob() && connection) { + auto blobFileReferences = NetworkBlobRegistry::singleton().filesInBlob(*connection, request.url()); + for (auto& fileReference : blobFileReferences) + fileReference->prepareForFileAccess(); + download->setBlobFileReferences(WTFMove(blobFileReferences)); + } + download->start(); + + ASSERT(!m_downloads.contains(downloadID)); + m_downloads.add(downloadID, WTFMove(download)); +#endif +} + +#if USE(NETWORK_SESSION) +void DownloadManager::dataTaskBecameDownloadTask(DownloadID downloadID, std::unique_ptr<Download>&& download) +{ + ASSERT(m_pendingDownloads.contains(downloadID)); + m_pendingDownloads.remove(downloadID); + ASSERT(!m_downloads.contains(downloadID)); + m_downloadsAfterDestinationDecided.remove(downloadID); + m_downloads.add(downloadID, WTFMove(download)); +} + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void DownloadManager::continueCanAuthenticateAgainstProtectionSpace(DownloadID downloadID, bool canAuthenticate) +{ + auto* pendingDownload = m_pendingDownloads.get(downloadID); + ASSERT(pendingDownload); + if (pendingDownload) + pendingDownload->continueCanAuthenticateAgainstProtectionSpace(canAuthenticate); +} +#endif + +void DownloadManager::continueWillSendRequest(DownloadID downloadID, WebCore::ResourceRequest&& request) +{ + auto* pendingDownload = m_pendingDownloads.get(downloadID); + ASSERT(pendingDownload); + if (pendingDownload) + pendingDownload->continueWillSendRequest(WTFMove(request)); +} + +void DownloadManager::willDecidePendingDownloadDestination(NetworkDataTask& networkDataTask, ResponseCompletionHandler&& completionHandler) +{ + auto downloadID = networkDataTask.pendingDownloadID(); + auto addResult = m_downloadsWaitingForDestination.set(downloadID, std::make_pair<RefPtr<NetworkDataTask>, ResponseCompletionHandler>(&networkDataTask, WTFMove(completionHandler))); + ASSERT_UNUSED(addResult, addResult.isNewEntry); +} +#endif // USE(NETWORK_SESSION) + +void DownloadManager::convertNetworkLoadToDownload(DownloadID downloadID, std::unique_ptr<NetworkLoad>&& networkLoad, Vector<RefPtr<WebCore::BlobDataFileReference>>&& blobFileReferences, const ResourceRequest& request, const ResourceResponse& response) +{ +#if USE(NETWORK_SESSION) + ASSERT(!m_pendingDownloads.contains(downloadID)); + m_pendingDownloads.add(downloadID, std::make_unique<PendingDownload>(WTFMove(networkLoad), downloadID, request, response)); +#else + auto download = std::make_unique<Download>(*this, downloadID, request); + download->setBlobFileReferences(WTFMove(blobFileReferences)); + + auto* handle = networkLoad->handle(); + ASSERT(handle); + download->startWithHandle(handle, response); + ASSERT(!m_downloads.contains(downloadID)); + m_downloads.add(downloadID, WTFMove(download)); + + // Unblock the URL connection operation queue. + handle->continueDidReceiveResponse(); +#endif +} + +void DownloadManager::continueDecidePendingDownloadDestination(DownloadID downloadID, String destination, const SandboxExtension::Handle& sandboxExtensionHandle, bool allowOverwrite) +{ +#if USE(NETWORK_SESSION) + if (m_downloadsWaitingForDestination.contains(downloadID)) { + auto pair = m_downloadsWaitingForDestination.take(downloadID); + auto networkDataTask = WTFMove(pair.first); + auto completionHandler = WTFMove(pair.second); + ASSERT(networkDataTask); + ASSERT(completionHandler); + ASSERT(m_pendingDownloads.contains(downloadID)); + + networkDataTask->setPendingDownloadLocation(destination, sandboxExtensionHandle, allowOverwrite); + completionHandler(PolicyDownload); + if (networkDataTask->state() == NetworkDataTask::State::Canceling || networkDataTask->state() == NetworkDataTask::State::Completed) + return; + + if (m_downloads.contains(downloadID)) { + // The completion handler already called dataTaskBecameDownloadTask(). + return; + } + + ASSERT(!m_downloadsAfterDestinationDecided.contains(downloadID)); + m_downloadsAfterDestinationDecided.set(downloadID, networkDataTask); + } +#else + if (auto* waitingDownload = download(downloadID)) + waitingDownload->didDecideDownloadDestination(destination, sandboxExtensionHandle, allowOverwrite); +#endif +} + +void DownloadManager::resumeDownload(SessionID, DownloadID downloadID, const IPC::DataReference& resumeData, const String& path, const SandboxExtension::Handle& sandboxExtensionHandle) +{ +#if USE(NETWORK_SESSION) + notImplemented(); +#else + // Download::resume() is responsible for setting the Download's resource request. + auto download = std::make_unique<Download>(*this, downloadID, ResourceRequest()); + + download->resume(resumeData, path, sandboxExtensionHandle); + ASSERT(!m_downloads.contains(downloadID)); + m_downloads.add(downloadID, WTFMove(download)); +#endif +} + +void DownloadManager::cancelDownload(DownloadID downloadID) +{ + if (Download* download = m_downloads.get(downloadID)) { +#if USE(NETWORK_SESSION) + ASSERT(!m_downloadsWaitingForDestination.contains(downloadID)); + ASSERT(!m_pendingDownloads.contains(downloadID)); +#endif + download->cancel(); + return; + } +#if USE(NETWORK_SESSION) + auto pendingDownload = m_pendingDownloads.take(downloadID); + if (m_downloadsWaitingForDestination.contains(downloadID)) { + auto pair = m_downloadsWaitingForDestination.take(downloadID); + auto networkDataTask = WTFMove(pair.first); + auto completionHandler = WTFMove(pair.second); + ASSERT(networkDataTask); + ASSERT(completionHandler); + + networkDataTask->cancel(); + completionHandler(PolicyIgnore); + m_client.pendingDownloadCanceled(downloadID); + return; + } + + if (pendingDownload) + pendingDownload->cancel(); +#endif +} + +void DownloadManager::downloadFinished(Download* download) +{ + ASSERT(m_downloads.contains(download->downloadID())); + m_downloads.remove(download->downloadID()); +} + +void DownloadManager::didCreateDownload() +{ + m_client.didCreateDownload(); +} + +void DownloadManager::didDestroyDownload() +{ + m_client.didDestroyDownload(); +} + +IPC::Connection* DownloadManager::downloadProxyConnection() +{ + return m_client.downloadProxyConnection(); +} + +AuthenticationManager& DownloadManager::downloadsAuthenticationManager() +{ + return m_client.downloadsAuthenticationManager(); +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/Downloads/DownloadManager.h b/Source/WebKit2/NetworkProcess/Downloads/DownloadManager.h new file mode 100644 index 000000000..b4d525b72 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/DownloadManager.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2010-2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DownloadManager_h +#define DownloadManager_h + +#include "DownloadID.h" +#include "NetworkDataTask.h" +#include "PendingDownload.h" +#include "SandboxExtension.h" +#include <WebCore/NotImplemented.h> +#include <wtf/Forward.h> +#include <wtf/HashMap.h> +#include <wtf/Noncopyable.h> + +namespace WebCore { +class BlobDataFileReference; +class ResourceHandle; +class ResourceRequest; +class ResourceResponse; +class SessionID; +} + +namespace IPC { +class Connection; +class DataReference; +} + +namespace WebKit { + +class AuthenticationManager; +class Download; +class NetworkConnectionToWebProcess; +class NetworkLoad; +class PendingDownload; + +class DownloadManager { + WTF_MAKE_NONCOPYABLE(DownloadManager); + +public: + class Client { + public: + virtual ~Client() { } + + virtual void didCreateDownload() = 0; + virtual void didDestroyDownload() = 0; + virtual IPC::Connection* downloadProxyConnection() = 0; + virtual AuthenticationManager& downloadsAuthenticationManager() = 0; +#if USE(NETWORK_SESSION) + virtual void pendingDownloadCanceled(DownloadID) = 0; +#endif + }; + + explicit DownloadManager(Client&); + + void startDownload(NetworkConnectionToWebProcess*, WebCore::SessionID, DownloadID, const WebCore::ResourceRequest&, const String& suggestedName = { }); +#if USE(NETWORK_SESSION) + void dataTaskBecameDownloadTask(DownloadID, std::unique_ptr<Download>&&); +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void continueCanAuthenticateAgainstProtectionSpace(DownloadID, bool canAuthenticate); +#endif + void continueWillSendRequest(DownloadID, WebCore::ResourceRequest&&); + void willDecidePendingDownloadDestination(NetworkDataTask&, ResponseCompletionHandler&&); +#endif + void convertNetworkLoadToDownload(DownloadID, std::unique_ptr<NetworkLoad>&&, Vector<RefPtr<WebCore::BlobDataFileReference>>&&, const WebCore::ResourceRequest&, const WebCore::ResourceResponse&); + void continueDecidePendingDownloadDestination(DownloadID, String destination, const SandboxExtension::Handle&, bool allowOverwrite); + + void resumeDownload(WebCore::SessionID, DownloadID, const IPC::DataReference& resumeData, const String& path, const SandboxExtension::Handle&); + + void cancelDownload(DownloadID); + + Download* download(DownloadID downloadID) { return m_downloads.get(downloadID); } + + void downloadFinished(Download*); + bool isDownloading() const { return !m_downloads.isEmpty(); } + uint64_t activeDownloadCount() const { return m_downloads.size(); } + + void didCreateDownload(); + void didDestroyDownload(); + + IPC::Connection* downloadProxyConnection(); + AuthenticationManager& downloadsAuthenticationManager(); + +private: + Client& m_client; +#if USE(NETWORK_SESSION) + HashMap<DownloadID, std::unique_ptr<PendingDownload>> m_pendingDownloads; + HashMap<DownloadID, std::pair<RefPtr<NetworkDataTask>, ResponseCompletionHandler>> m_downloadsWaitingForDestination; + HashMap<DownloadID, RefPtr<NetworkDataTask>> m_downloadsAfterDestinationDecided; +#endif + HashMap<DownloadID, std::unique_ptr<Download>> m_downloads; +}; + +} // namespace WebKit + +#endif // DownloadManager_h diff --git a/Source/WebKit2/NetworkProcess/Downloads/PendingDownload.cpp b/Source/WebKit2/NetworkProcess/Downloads/PendingDownload.cpp new file mode 100644 index 000000000..876775b8f --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/PendingDownload.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "PendingDownload.h" + +#if USE(NETWORK_SESSION) + +#include "DataReference.h" +#include "DownloadProxyMessages.h" +#include "NetworkLoad.h" +#include "NetworkProcess.h" +#include "WebCoreArgumentCoders.h" + +using namespace WebCore; + +namespace WebKit { + +PendingDownload::PendingDownload(NetworkLoadParameters&& parameters, DownloadID downloadID, NetworkSession& networkSession, const String& suggestedName) + : m_networkLoad(std::make_unique<NetworkLoad>(*this, WTFMove(parameters), networkSession)) +{ + m_networkLoad->setPendingDownloadID(downloadID); + m_networkLoad->setPendingDownload(*this); + m_networkLoad->setSuggestedFilename(suggestedName); + + send(Messages::DownloadProxy::DidStart(m_networkLoad->currentRequest(), suggestedName)); +} + +PendingDownload::PendingDownload(std::unique_ptr<NetworkLoad>&& networkLoad, DownloadID downloadID, const ResourceRequest& request, const ResourceResponse& response) + : m_networkLoad(WTFMove(networkLoad)) +{ + m_networkLoad->setPendingDownloadID(downloadID); + send(Messages::DownloadProxy::DidStart(request, String())); + + m_networkLoad->convertTaskToDownload(*this, request, response); +} + +void PendingDownload::willSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceRequest&& redirectRequest, WebCore::ResourceResponse&& redirectResponse) +{ + send(Messages::DownloadProxy::WillSendRequest(WTFMove(redirectRequest), WTFMove(redirectResponse))); +}; + +void PendingDownload::continueWillSendRequest(WebCore::ResourceRequest&& newRequest) +{ + m_networkLoad->continueWillSendRequest(WTFMove(newRequest)); +} + +void PendingDownload::cancel() +{ + ASSERT(m_networkLoad); + m_networkLoad->cancel(); + send(Messages::DownloadProxy::DidCancel({ })); +} + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void PendingDownload::canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace& protectionSpace) +{ + send(Messages::DownloadProxy::CanAuthenticateAgainstProtectionSpace(protectionSpace)); +} + +void PendingDownload::continueCanAuthenticateAgainstProtectionSpace(bool canAuthenticate) +{ + m_networkLoad->continueCanAuthenticateAgainstProtectionSpace(canAuthenticate); +} +#endif + +void PendingDownload::didFailLoading(const WebCore::ResourceError& error) +{ + send(Messages::DownloadProxy::DidFail(error, { })); +} + +IPC::Connection* PendingDownload::messageSenderConnection() +{ + return NetworkProcess::singleton().parentProcessConnection(); +} + +uint64_t PendingDownload::messageSenderDestinationID() +{ + return m_networkLoad->pendingDownloadID().downloadID(); +} + +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/Downloads/PendingDownload.h b/Source/WebKit2/NetworkProcess/Downloads/PendingDownload.h new file mode 100644 index 000000000..208d73dae --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/PendingDownload.h @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PendingDownload_h +#define PendingDownload_h + +#if USE(NETWORK_SESSION) + +#include "MessageSender.h" +#include "NetworkLoadClient.h" + +namespace WebCore { +class ResourceResponse; +} + +namespace WebKit { + +class DownloadID; +class NetworkLoad; +class NetworkLoadParameters; +class NetworkSession; + +class PendingDownload : public NetworkLoadClient, public IPC::MessageSender { + WTF_MAKE_FAST_ALLOCATED; +public: + PendingDownload(NetworkLoadParameters&&, DownloadID, NetworkSession&, const String& suggestedName); + PendingDownload(std::unique_ptr<NetworkLoad>&&, DownloadID, const WebCore::ResourceRequest&, const WebCore::ResourceResponse&); + + void continueWillSendRequest(WebCore::ResourceRequest&&); +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void continueCanAuthenticateAgainstProtectionSpace(bool canAuthenticate); +#endif + void cancel(); + +private: + // NetworkLoadClient. + void didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override { } +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) override; +#endif + bool isSynchronous() const override { return false; } + void willSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceRequest&& redirectRequest, WebCore::ResourceResponse&& redirectResponse) override; + ShouldContinueDidReceiveResponse didReceiveResponse(WebCore::ResourceResponse&&) override { return ShouldContinueDidReceiveResponse::No; }; + void didReceiveBuffer(Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) override { }; + void didFinishLoading(double finishTime) override { }; + void didFailLoading(const WebCore::ResourceError&) override; + + // MessageSender. + IPC::Connection* messageSenderConnection() override; + uint64_t messageSenderDestinationID() override; + +private: + std::unique_ptr<NetworkLoad> m_networkLoad; +}; + +} + +#endif + +#endif diff --git a/Source/WebKit2/NetworkProcess/soup/NetworkResourceLoadSchedulerSoup.cpp b/Source/WebKit2/NetworkProcess/Downloads/gtk/DownloadSoupErrorsGtk.cpp index 9d03979e3..ba2e7b456 100644 --- a/Source/WebKit2/NetworkProcess/soup/NetworkResourceLoadSchedulerSoup.cpp +++ b/Source/WebKit2/NetworkProcess/Downloads/gtk/DownloadSoupErrorsGtk.cpp @@ -1,6 +1,5 @@ /* - * Copyright (C) 2013 Apple Inc. All rights reserved. - * Copyright (C) 2013 Company 100 Inc. + * Copyright (C) 2012 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -25,25 +24,23 @@ */ #include "config.h" -#if ENABLE(NETWORK_PROCESS) +#include "DownloadSoupErrors.h" -#include "NetworkResourceLoadScheduler.h" +#include <WebCore/ErrorsGtk.h> +#include <WebCore/ResourceError.h> + +using namespace WebCore; namespace WebKit { -void NetworkResourceLoadScheduler::platformInitializeMaximumHTTPConnectionCountPerHost() +ResourceError platformDownloadNetworkError(int errorCode, const URL& failingURL, const String& localizedDescription) { - // Soup has its own queue control; it wants to have all requests given to - // it, so that it is able to look ahead, and schedule them in a good way. - // See the comment in ResourceRequestSoup.cpp - static const unsigned unlimitedConnectionCount = 10000; + return downloadNetworkError(ResourceError(errorDomainDownload, errorCode, failingURL, localizedDescription)); +} - // FIXME: Take advantage of Web-platform specific knowledge that can help - // prioritization better than libsoup alone can do. - // See https://bugs.webkit.org/show_bug.cgi?id=110115#c13 - m_maxRequestsInFlightPerHost = unlimitedConnectionCount; +ResourceError platformDownloadDestinationError(const ResourceResponse& response, const String& message) +{ + return downloadDestinationError(response, message); } } // namespace WebKit - -#endif diff --git a/Source/WebKit2/NetworkProcess/Downloads/soup/DownloadSoupErrors.h b/Source/WebKit2/NetworkProcess/Downloads/soup/DownloadSoupErrors.h new file mode 100644 index 000000000..ff397ec22 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/Downloads/soup/DownloadSoupErrors.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2012 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DownloadSoupErrors_h +#define DownloadSoupErrors_h + +#include <WebCore/ResourceHandle.h> +#include <WebCore/URL.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { + +WebCore::ResourceError platformDownloadNetworkError(int errorCode, const WebCore::URL& failingURL, const String& localizedDescription); +WebCore::ResourceError platformDownloadDestinationError(const WebCore::ResourceResponse&, const String& message); + +} // namespace WebKit + +#endif // DownloadSoupErrors_h diff --git a/Source/WebKit2/NetworkProcess/EntryPoint/unix/NetworkProcessMain.cpp b/Source/WebKit2/NetworkProcess/EntryPoint/unix/NetworkProcessMain.cpp new file mode 100644 index 000000000..6c0db1e74 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/EntryPoint/unix/NetworkProcessMain.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2014 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "NetworkProcessMainUnix.h" + +#include <cstdlib> + +using namespace WebKit; + +int main(int argc, char** argv) +{ + // Disable SSLv3 very early because it is practically impossible to safely + // use setenv() when multiple threads are running, as another thread calling + // getenv() could cause a crash, and many functions use getenv() internally. + // This workaround will stop working if glib-networking switches away from + // GnuTLS or simply stops parsing this variable. We intentionally do not + // overwrite this priority string if it's already set by the user. + // https://bugzilla.gnome.org/show_bug.cgi?id=738633 + // WARNING: This needs to be KEPT IN SYNC with WebProcessMain.cpp. + setenv("G_TLS_GNUTLS_PRIORITY", "NORMAL:%COMPAT:!VERS-SSL3.0:!ARCFOUR-128", 0); + + return NetworkProcessMainUnix(argc, argv); +} diff --git a/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.cpp b/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.cpp index a2ac7d124..172558d92 100644 --- a/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.cpp +++ b/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Apple Inc. All rights reserved. + * Copyright (C) 2013-2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,20 +26,21 @@ #include "config.h" #include "NetworkBlobRegistry.h" -#if ENABLE(BLOB) && ENABLE(NETWORK_PROCESS) - +#include "BlobDataFileReferenceWithSandboxExtension.h" +#include "NetworkConnectionToWebProcess.h" #include "SandboxExtension.h" +#include <WebCore/BlobPart.h> #include <WebCore/BlobRegistryImpl.h> -#include <wtf/MainThread.h> #include <wtf/NeverDestroyed.h> +#include <wtf/RunLoop.h> using namespace WebCore; namespace WebKit { -NetworkBlobRegistry& NetworkBlobRegistry::shared() +NetworkBlobRegistry& NetworkBlobRegistry::singleton() { - ASSERT(isMainThread()); + ASSERT(RunLoop::isMain()); static NeverDestroyed<NetworkBlobRegistry> registry; return registry; } @@ -48,22 +49,20 @@ NetworkBlobRegistry::NetworkBlobRegistry() { } -void NetworkBlobRegistry::registerBlobURL(NetworkConnectionToWebProcess* connection, const URL& url, std::unique_ptr<BlobData> data, const Vector<RefPtr<SandboxExtension>>& newSandboxExtensions) +void NetworkBlobRegistry::registerFileBlobURL(NetworkConnectionToWebProcess* connection, const URL& url, const String& path, RefPtr<SandboxExtension>&& sandboxExtension, const String& contentType) { - ASSERT(!m_sandboxExtensions.contains(url.string())); - - // Combine new extensions for File items and existing extensions for inner Blob items. - Vector<RefPtr<SandboxExtension>> sandboxExtensions = newSandboxExtensions; - const BlobDataItemList& items = data->items(); - for (size_t i = 0, count = items.size(); i < count; ++i) { - if (items[i].type == BlobDataItem::Blob) - sandboxExtensions.appendVector(m_sandboxExtensions.get(items[i].url.string())); - } + blobRegistry().registerFileBlobURL(url, BlobDataFileReferenceWithSandboxExtension::create(path, sandboxExtension), contentType); - blobRegistry().registerBlobURL(url, std::move(data)); + ASSERT(!m_blobsForConnection.get(connection).contains(url)); + BlobForConnectionMap::iterator mapIterator = m_blobsForConnection.find(connection); + if (mapIterator == m_blobsForConnection.end()) + mapIterator = m_blobsForConnection.add(connection, HashSet<URL>()).iterator; + mapIterator->value.add(url); +} - if (!sandboxExtensions.isEmpty()) - m_sandboxExtensions.add(url.string(), sandboxExtensions); +void NetworkBlobRegistry::registerBlobURL(NetworkConnectionToWebProcess* connection, const URL& url, Vector<WebCore::BlobPart>&& blobParts, const String& contentType) +{ + blobRegistry().registerBlobURL(url, WTFMove(blobParts), contentType); ASSERT(!m_blobsForConnection.get(connection).contains(url)); BlobForConnectionMap::iterator mapIterator = m_blobsForConnection.find(connection); @@ -74,24 +73,68 @@ void NetworkBlobRegistry::registerBlobURL(NetworkConnectionToWebProcess* connect void NetworkBlobRegistry::registerBlobURL(NetworkConnectionToWebProcess* connection, const WebCore::URL& url, const WebCore::URL& srcURL) { + // The connection may not be registered if NetworkProcess prevously crashed for any reason. + BlobForConnectionMap::iterator mapIterator = m_blobsForConnection.find(connection); + if (mapIterator == m_blobsForConnection.end()) + return; + blobRegistry().registerBlobURL(url, srcURL); - SandboxExtensionMap::iterator iter = m_sandboxExtensions.find(srcURL.string()); - if (iter != m_sandboxExtensions.end()) - m_sandboxExtensions.add(url.string(), iter->value); - ASSERT(m_blobsForConnection.contains(connection)); - ASSERT(m_blobsForConnection.find(connection)->value.contains(srcURL)); - m_blobsForConnection.find(connection)->value.add(url); + ASSERT(mapIterator->value.contains(srcURL)); + mapIterator->value.add(url); +} + +void NetworkBlobRegistry::registerBlobURLOptionallyFileBacked(NetworkConnectionToWebProcess* connection, const URL& url, const URL& srcURL, const String& fileBackedPath, const String& contentType) +{ + auto fileReference = connection->getBlobDataFileReferenceForPath(fileBackedPath); + ASSERT(fileReference); + + blobRegistry().registerBlobURLOptionallyFileBacked(url, srcURL, WTFMove(fileReference), contentType); + + ASSERT(!m_blobsForConnection.get(connection).contains(url)); + BlobForConnectionMap::iterator mapIterator = m_blobsForConnection.find(connection); + if (mapIterator == m_blobsForConnection.end()) + mapIterator = m_blobsForConnection.add(connection, HashSet<URL>()).iterator; + mapIterator->value.add(url); +} + +void NetworkBlobRegistry::registerBlobURLForSlice(NetworkConnectionToWebProcess* connection, const WebCore::URL& url, const WebCore::URL& srcURL, int64_t start, int64_t end) +{ + // The connection may not be registered if NetworkProcess prevously crashed for any reason. + BlobForConnectionMap::iterator mapIterator = m_blobsForConnection.find(connection); + if (mapIterator == m_blobsForConnection.end()) + return; + + blobRegistry().registerBlobURLForSlice(url, srcURL, start, end); + + ASSERT(mapIterator->value.contains(srcURL)); + mapIterator->value.add(url); } void NetworkBlobRegistry::unregisterBlobURL(NetworkConnectionToWebProcess* connection, const WebCore::URL& url) { + // The connection may not be registered if NetworkProcess prevously crashed for any reason. + BlobForConnectionMap::iterator mapIterator = m_blobsForConnection.find(connection); + if (mapIterator == m_blobsForConnection.end()) + return; + blobRegistry().unregisterBlobURL(url); - m_sandboxExtensions.remove(url.string()); - ASSERT(m_blobsForConnection.contains(connection)); - ASSERT(m_blobsForConnection.find(connection)->value.contains(url)); - m_blobsForConnection.find(connection)->value.remove(url); + ASSERT(mapIterator->value.contains(url)); + mapIterator->value.remove(url); +} + +uint64_t NetworkBlobRegistry::blobSize(NetworkConnectionToWebProcess* connection, const WebCore::URL& url) +{ + if (!m_blobsForConnection.contains(connection) || !m_blobsForConnection.find(connection)->value.contains(url)) + return 0; + + return blobRegistry().blobSize(url); +} + +void NetworkBlobRegistry::writeBlobsToTemporaryFiles(const Vector<String>& blobURLs, Function<void(const Vector<String>&)>&& completionHandler) +{ + blobRegistry().writeBlobsToTemporaryFiles(blobURLs, WTFMove(completionHandler)); } void NetworkBlobRegistry::connectionToWebProcessDidClose(NetworkConnectionToWebProcess* connection) @@ -100,19 +143,29 @@ void NetworkBlobRegistry::connectionToWebProcessDidClose(NetworkConnectionToWebP return; HashSet<URL>& blobsForConnection = m_blobsForConnection.find(connection)->value; - for (HashSet<URL>::iterator iter = blobsForConnection.begin(), end = blobsForConnection.end(); iter != end; ++iter) { + for (HashSet<URL>::iterator iter = blobsForConnection.begin(), end = blobsForConnection.end(); iter != end; ++iter) blobRegistry().unregisterBlobURL(*iter); - m_sandboxExtensions.remove(*iter); - } m_blobsForConnection.remove(connection); } -const Vector<RefPtr<SandboxExtension>> NetworkBlobRegistry::sandboxExtensions(const WebCore::URL& url) +Vector<RefPtr<BlobDataFileReference>> NetworkBlobRegistry::filesInBlob(NetworkConnectionToWebProcess& connection, const WebCore::URL& url) { - return m_sandboxExtensions.get(url.string()); -} + if (!m_blobsForConnection.contains(&connection) || !m_blobsForConnection.find(&connection)->value.contains(url)) + return Vector<RefPtr<BlobDataFileReference>>(); + + ASSERT(blobRegistry().isBlobRegistryImpl()); + BlobData* blobData = static_cast<BlobRegistryImpl&>(blobRegistry()).getBlobDataFromURL(url); + if (!blobData) + return Vector<RefPtr<BlobDataFileReference>>(); + + Vector<RefPtr<BlobDataFileReference>> result; + for (const BlobDataItem& item : blobData->items()) { + if (item.type() == BlobDataItem::Type::File) + result.append(item.file()); + } + return result; } -#endif +} diff --git a/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.h b/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.h index 16c6adb09..40378b0bf 100644 --- a/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.h +++ b/Source/WebKit2/NetworkProcess/FileAPI/NetworkBlobRegistry.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Apple Inc. All rights reserved. + * Copyright (C) 2013-2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -23,17 +23,16 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef NetworkBlobRegistry_h -#define NetworkBlobRegistry_h - -#if ENABLE(BLOB) && ENABLE(NETWORK_PROCESS) +#pragma once #include <WebCore/URLHash.h> +#include <wtf/Function.h> #include <wtf/HashMap.h> #include <wtf/HashSet.h> namespace WebCore { -class BlobData; +class BlobDataFileReference; +class BlobPart; } namespace WebKit { @@ -45,28 +44,26 @@ class NetworkBlobRegistry { WTF_MAKE_NONCOPYABLE(NetworkBlobRegistry); public: NetworkBlobRegistry(); - static NetworkBlobRegistry& shared(); + static NetworkBlobRegistry& singleton(); - void registerBlobURL(NetworkConnectionToWebProcess*, const WebCore::URL&, std::unique_ptr<WebCore::BlobData>, const Vector<RefPtr<SandboxExtension>>&); + void registerFileBlobURL(NetworkConnectionToWebProcess*, const WebCore::URL&, const String& path, RefPtr<SandboxExtension>&&, const String& contentType); + void registerBlobURL(NetworkConnectionToWebProcess*, const WebCore::URL&, Vector<WebCore::BlobPart>&&, const String& contentType); void registerBlobURL(NetworkConnectionToWebProcess*, const WebCore::URL&, const WebCore::URL& srcURL); + void registerBlobURLOptionallyFileBacked(NetworkConnectionToWebProcess*, const WebCore::URL&, const WebCore::URL& srcURL, const String& fileBackedPath, const String& contentType); + void registerBlobURLForSlice(NetworkConnectionToWebProcess*, const WebCore::URL&, const WebCore::URL& srcURL, int64_t start, int64_t end); void unregisterBlobURL(NetworkConnectionToWebProcess*, const WebCore::URL&); + uint64_t blobSize(NetworkConnectionToWebProcess*, const WebCore::URL&); + void writeBlobsToTemporaryFiles(const Vector<String>& blobURLs, Function<void (const Vector<String>&)>&& completionHandler); void connectionToWebProcessDidClose(NetworkConnectionToWebProcess*); - const Vector<RefPtr<SandboxExtension>> sandboxExtensions(const WebCore::URL&); + Vector<RefPtr<WebCore::BlobDataFileReference>> filesInBlob(NetworkConnectionToWebProcess&, const WebCore::URL&); private: ~NetworkBlobRegistry(); - typedef HashMap<String, Vector<RefPtr<SandboxExtension>>> SandboxExtensionMap; - SandboxExtensionMap m_sandboxExtensions; - typedef HashMap<NetworkConnectionToWebProcess*, HashSet<WebCore::URL>> BlobForConnectionMap; BlobForConnectionMap m_blobsForConnection; }; } - -#endif // ENABLE(BLOB) && ENABLE(NETWORK_PROCESS) - -#endif // NetworkBlobRegistry_h diff --git a/Source/WebKit2/NetworkProcess/HostRecord.cpp b/Source/WebKit2/NetworkProcess/HostRecord.cpp deleted file mode 100644 index 35248f296..000000000 --- a/Source/WebKit2/NetworkProcess/HostRecord.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (C) 2012 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "HostRecord.h" - -#include "Logging.h" -#include "NetworkConnectionToWebProcess.h" -#include "NetworkProcess.h" -#include "NetworkResourceLoadParameters.h" -#include "NetworkResourceLoadScheduler.h" -#include "NetworkResourceLoader.h" -#include <wtf/MainThread.h> - -#if ENABLE(NETWORK_PROCESS) - -using namespace WebCore; - -namespace WebKit { - -HostRecord::HostRecord(const String& name, int maxRequestsInFlight) - : m_name(name) - , m_maxRequestsInFlight(maxRequestsInFlight) -{ -} - -HostRecord::~HostRecord() -{ -#ifndef NDEBUG - ASSERT(m_loadersInProgress.isEmpty()); - for (unsigned p = 0; p <= ResourceLoadPriorityHighest; p++) - ASSERT(m_loadersPending[p].isEmpty()); -#endif -} - -void HostRecord::scheduleResourceLoader(PassRefPtr<NetworkResourceLoader> loader) -{ - ASSERT(isMainThread()); - - loader->setHostRecord(this); - - if (loader->isSynchronous()) - m_syncLoadersPending.append(loader); - else - m_loadersPending[loader->priority()].append(loader); -} - -void HostRecord::addLoaderInProgress(NetworkResourceLoader* loader) -{ - ASSERT(isMainThread()); - - m_loadersInProgress.add(loader); - loader->setHostRecord(this); -} - -inline bool removeLoaderFromQueue(NetworkResourceLoader* loader, LoaderQueue& queue) -{ - LoaderQueue::iterator end = queue.end(); - for (LoaderQueue::iterator it = queue.begin(); it != end; ++it) { - if (it->get() == loader) { - loader->setHostRecord(0); - queue.remove(it); - return true; - } - } - return false; -} - -void HostRecord::removeLoader(NetworkResourceLoader* loader) -{ - ASSERT(isMainThread()); - - // FIXME (NetworkProcess): Due to IPC race conditions, it's possible this HostRecord will be asked to remove the same loader twice. - // It would be nice to know the loader has already been removed and treat it as a no-op. - - NetworkResourceLoaderSet::iterator i = m_loadersInProgress.find(loader); - if (i != m_loadersInProgress.end()) { - i->get()->setHostRecord(0); - m_loadersInProgress.remove(i); - return; - } - - if (removeLoaderFromQueue(loader, m_syncLoadersPending)) - return; - - for (int priority = ResourceLoadPriorityHighest; priority >= ResourceLoadPriorityLowest; --priority) { - if (removeLoaderFromQueue(loader, m_loadersPending[priority])) - return; - } -} - -bool HostRecord::hasRequests() const -{ - if (!m_loadersInProgress.isEmpty()) - return true; - - for (unsigned p = 0; p <= ResourceLoadPriorityHighest; p++) { - if (!m_loadersPending[p].isEmpty()) - return true; - } - - return false; -} - -uint64_t HostRecord::pendingRequestCount() const -{ - uint64_t count = 0; - - for (unsigned p = 0; p <= ResourceLoadPriorityHighest; p++) - count += m_loadersPending[p].size(); - - return count; -} - -uint64_t HostRecord::activeLoadCount() const -{ - return m_loadersInProgress.size(); -} - -void HostRecord::servePendingRequestsForQueue(LoaderQueue& queue, ResourceLoadPriority priority) -{ - // We only enforce the connection limit for http(s) hosts, which are the only ones with names. - bool shouldLimitRequests = !name().isNull(); - - // For non-named hosts - everything but http(s) - we should only enforce the limit if the document - // isn't done parsing and we don't know all stylesheets yet. - - // FIXME (NetworkProcess): The above comment about document parsing and stylesheets is a holdover - // from the WebCore::ResourceLoadScheduler. - // The behavior described was at one time important for WebCore's single threadedness. - // It's possible that we don't care about it with the NetworkProcess. - // We should either decide it's not important and change the above comment, or decide it is - // still important and somehow account for it. - - // Very low priority loaders are only handled when no other loaders are in progress. - if (shouldLimitRequests && priority == ResourceLoadPriorityVeryLow && !m_loadersInProgress.isEmpty()) - return; - - while (!queue.isEmpty()) { - RefPtr<NetworkResourceLoader> loader = queue.first(); - ASSERT(loader->hostRecord() == this); - - // This request might be from WebProcess we've lost our connection to. - // If so we should just skip it. - if (!loader->connectionToWebProcess()) { - removeLoader(loader.get()); - continue; - } - - if (shouldLimitRequests && limitsRequests(priority, loader.get())) - return; - - m_loadersInProgress.add(loader); - queue.removeFirst(); - - LOG(NetworkScheduling, "(NetworkProcess) HostRecord::servePendingRequestsForQueue - Starting load of %s\n", loader->request().url().string().utf8().data()); - loader->start(); - } -} - -void HostRecord::servePendingRequests(ResourceLoadPriority minimumPriority) -{ - LOG(NetworkScheduling, "(NetworkProcess) HostRecord::servePendingRequests Host name='%s'", name().utf8().data()); - - // We serve synchronous requests before any other requests to improve responsiveness in any - // WebProcess that is waiting on a synchronous load. - servePendingRequestsForQueue(m_syncLoadersPending, ResourceLoadPriorityHighest); - - for (int priority = ResourceLoadPriorityHighest; priority >= minimumPriority; --priority) - servePendingRequestsForQueue(m_loadersPending[priority], (ResourceLoadPriority)priority); -} - -bool HostRecord::limitsRequests(ResourceLoadPriority priority, NetworkResourceLoader* loader) const -{ - ASSERT(loader); - ASSERT(loader->connectionToWebProcess()); - - if (priority == ResourceLoadPriorityVeryLow && !m_loadersInProgress.isEmpty()) - return true; - - if (loader->connectionToWebProcess()->isSerialLoadingEnabled() && m_loadersInProgress.size() >= 1) - return true; - - // If we're exactly at the limit for requests in flight, and this loader is asynchronous, then we're done serving new requests. - // The synchronous loader exception handles the case where a sync XHR is made while 6 other requests are already in flight. - if (m_loadersInProgress.size() == m_maxRequestsInFlight && !loader->isSynchronous()) - return true; - - // If we're already past the limit of the number of loaders in flight, we won't even serve new synchronous requests right now. - if (m_loadersInProgress.size() > m_maxRequestsInFlight) { -#ifndef NDEBUG - // If we have more loaders in progress than we should, at least one of them had better be synchronous. - NetworkResourceLoaderSet::iterator i = m_loadersInProgress.begin(); - NetworkResourceLoaderSet::iterator end = m_loadersInProgress.end(); - for (; i != end; ++i) { - if (i->get()->isSynchronous()) - break; - } - ASSERT(i != end); -#endif - return true; - } - return false; -} - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/HostRecord.h b/Source/WebKit2/NetworkProcess/HostRecord.h deleted file mode 100644 index 7e18c3a7e..000000000 --- a/Source/WebKit2/NetworkProcess/HostRecord.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2012 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef HostRecord_h -#define HostRecord_h - -#if ENABLE(NETWORK_PROCESS) - -#include <WebCore/ResourceLoadPriority.h> -#include <wtf/Deque.h> -#include <wtf/HashSet.h> -#include <wtf/RefCounted.h> -#include <wtf/text/WTFString.h> - -namespace WebKit { - -class NetworkResourceLoader; - -typedef Deque<RefPtr<NetworkResourceLoader>> LoaderQueue; -typedef uint64_t ResourceLoadIdentifier; - -class HostRecord : public RefCounted<HostRecord> { -public: - static PassRefPtr<HostRecord> create(const String& name, int maxRequestsInFlight) - { - return adoptRef(new HostRecord(name, maxRequestsInFlight)); - } - - ~HostRecord(); - - const String& name() const { return m_name; } - - void scheduleResourceLoader(PassRefPtr<NetworkResourceLoader>); - void addLoaderInProgress(NetworkResourceLoader*); - void removeLoader(NetworkResourceLoader*); - bool hasRequests() const; - void servePendingRequests(WebCore::ResourceLoadPriority); - - uint64_t pendingRequestCount() const; - uint64_t activeLoadCount() const; - -private: - HostRecord(const String& name, int maxRequestsInFlight); - - void servePendingRequestsForQueue(LoaderQueue&, WebCore::ResourceLoadPriority); - bool limitsRequests(WebCore::ResourceLoadPriority, NetworkResourceLoader*) const; - - LoaderQueue m_loadersPending[WebCore::ResourceLoadPriorityHighest + 1]; - LoaderQueue m_syncLoadersPending; - - typedef HashSet<RefPtr<NetworkResourceLoader>> NetworkResourceLoaderSet; - NetworkResourceLoaderSet m_loadersInProgress; - - const String m_name; - int m_maxRequestsInFlight; -}; - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) - -#endif // #ifndef HostRecord_h diff --git a/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.cpp b/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.cpp index 1fee498a9..1416242f1 100644 --- a/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.cpp +++ b/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 Apple Inc. All rights reserved. + * Copyright (C) 2012-2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,46 +26,65 @@ #include "config.h" #include "NetworkConnectionToWebProcess.h" -#if ENABLE(NETWORK_PROCESS) - -#include "BlobRegistrationData.h" -#include "ConnectionStack.h" +#include "BlobDataFileReferenceWithSandboxExtension.h" +#include "DataReference.h" #include "NetworkBlobRegistry.h" +#include "NetworkCache.h" #include "NetworkConnectionToWebProcessMessages.h" +#include "NetworkLoad.h" #include "NetworkProcess.h" +#include "NetworkProcessConnectionMessages.h" +#include "NetworkRTCMonitorMessages.h" +#include "NetworkRTCProviderMessages.h" +#include "NetworkRTCSocketMessages.h" #include "NetworkResourceLoadParameters.h" #include "NetworkResourceLoader.h" #include "NetworkResourceLoaderMessages.h" #include "RemoteNetworkingContext.h" #include "SessionTracker.h" -#include <WebCore/BlobData.h> +#include "WebCoreArgumentCoders.h" +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/PingHandle.h> #include <WebCore/PlatformCookieJar.h> #include <WebCore/ResourceLoaderOptions.h> #include <WebCore/ResourceRequest.h> -#include <wtf/RunLoop.h> +#include <WebCore/SessionID.h> + +#if USE(NETWORK_SESSION) +#include "PingLoad.h" +#endif using namespace WebCore; namespace WebKit { -PassRefPtr<NetworkConnectionToWebProcess> NetworkConnectionToWebProcess::create(IPC::Connection::Identifier connectionIdentifier) +Ref<NetworkConnectionToWebProcess> NetworkConnectionToWebProcess::create(IPC::Connection::Identifier connectionIdentifier) { - return adoptRef(new NetworkConnectionToWebProcess(connectionIdentifier)); + return adoptRef(*new NetworkConnectionToWebProcess(connectionIdentifier)); } NetworkConnectionToWebProcess::NetworkConnectionToWebProcess(IPC::Connection::Identifier connectionIdentifier) - : m_serialLoadingEnabled(false) + : m_connection(IPC::Connection::createServerConnection(connectionIdentifier, *this)) { - m_connection = IPC::Connection::createServerConnection(connectionIdentifier, this, RunLoop::main()); - m_connection->setOnlySendMessagesAsDispatchWhenWaitingForSyncReplyWhenProcessingSuchAMessage(true); m_connection->open(); } NetworkConnectionToWebProcess::~NetworkConnectionToWebProcess() { +#if USE(LIBWEBRTC) + if (m_rtcProvider) + m_rtcProvider->close(); +#endif +} + +void NetworkConnectionToWebProcess::didCleanupResourceLoader(NetworkResourceLoader& loader) +{ + ASSERT(m_networkResourceLoaders.get(loader.identifier()) == &loader); + + m_networkResourceLoaders.remove(loader.identifier()); } - -void NetworkConnectionToWebProcess::didReceiveMessage(IPC::Connection* connection, IPC::MessageDecoder& decoder) + +void NetworkConnectionToWebProcess::didReceiveMessage(IPC::Connection& connection, IPC::Decoder& decoder) { if (decoder.messageReceiverName() == Messages::NetworkConnectionToWebProcess::messageReceiverName()) { didReceiveNetworkConnectionToWebProcessMessage(connection, decoder); @@ -73,16 +92,40 @@ void NetworkConnectionToWebProcess::didReceiveMessage(IPC::Connection* connectio } if (decoder.messageReceiverName() == Messages::NetworkResourceLoader::messageReceiverName()) { - HashMap<ResourceLoadIdentifier, RefPtr<NetworkResourceLoader>>::iterator loaderIterator = m_networkResourceLoaders.find(decoder.destinationID()); + auto loaderIterator = m_networkResourceLoaders.find(decoder.destinationID()); if (loaderIterator != m_networkResourceLoaders.end()) loaderIterator->value->didReceiveNetworkResourceLoaderMessage(connection, decoder); return; } - + +#if USE(LIBWEBRTC) + if (decoder.messageReceiverName() == Messages::NetworkRTCSocket::messageReceiverName()) { + rtcProvider().didReceiveNetworkRTCSocketMessage(connection, decoder); + return; + } + if (decoder.messageReceiverName() == Messages::NetworkRTCMonitor::messageReceiverName()) { + rtcProvider().didReceiveNetworkRTCMonitorMessage(connection, decoder); + return; + } + if (decoder.messageReceiverName() == Messages::NetworkRTCProvider::messageReceiverName()) { + rtcProvider().didReceiveMessage(connection, decoder); + return; + } +#endif + ASSERT_NOT_REACHED(); } -void NetworkConnectionToWebProcess::didReceiveSyncMessage(IPC::Connection* connection, IPC::MessageDecoder& decoder, std::unique_ptr<IPC::MessageEncoder>& reply) +#if USE(LIBWEBRTC) +NetworkRTCProvider& NetworkConnectionToWebProcess::rtcProvider() +{ + if (!m_rtcProvider) + m_rtcProvider = NetworkRTCProvider::create(*this); + return *m_rtcProvider; +} +#endif + +void NetworkConnectionToWebProcess::didReceiveSyncMessage(IPC::Connection& connection, IPC::Decoder& decoder, std::unique_ptr<IPC::Encoder>& reply) { if (decoder.messageReceiverName() == Messages::NetworkConnectionToWebProcess::messageReceiverName()) { didReceiveSyncNetworkConnectionToWebProcessMessage(connection, decoder, reply); @@ -91,43 +134,62 @@ void NetworkConnectionToWebProcess::didReceiveSyncMessage(IPC::Connection* conne ASSERT_NOT_REACHED(); } -void NetworkConnectionToWebProcess::didClose(IPC::Connection*) +void NetworkConnectionToWebProcess::didClose(IPC::Connection&) { // Protect ourself as we might be otherwise be deleted during this function. Ref<NetworkConnectionToWebProcess> protector(*this); - HashMap<ResourceLoadIdentifier, RefPtr<NetworkResourceLoader>>::iterator end = m_networkResourceLoaders.end(); - for (HashMap<ResourceLoadIdentifier, RefPtr<NetworkResourceLoader>>::iterator i = m_networkResourceLoaders.begin(); i != end; ++i) - i->value->abort(); + Vector<RefPtr<NetworkResourceLoader>> loaders; + copyValuesToVector(m_networkResourceLoaders, loaders); + for (auto& loader : loaders) + loader->abort(); + ASSERT(m_networkResourceLoaders.isEmpty()); - NetworkBlobRegistry::shared().connectionToWebProcessDidClose(this); + NetworkBlobRegistry::singleton().connectionToWebProcessDidClose(this); + NetworkProcess::singleton().removeNetworkConnectionToWebProcess(this); - m_networkResourceLoaders.clear(); - - NetworkProcess::shared().removeNetworkConnectionToWebProcess(this); +#if USE(LIBWEBRTC) + if (m_rtcProvider) { + m_rtcProvider->close(); + m_rtcProvider = nullptr; + } +#endif } -void NetworkConnectionToWebProcess::didReceiveInvalidMessage(IPC::Connection*, IPC::StringReference, IPC::StringReference) +void NetworkConnectionToWebProcess::didReceiveInvalidMessage(IPC::Connection&, IPC::StringReference, IPC::StringReference) { } void NetworkConnectionToWebProcess::scheduleResourceLoad(const NetworkResourceLoadParameters& loadParameters) { - RefPtr<NetworkResourceLoader> loader = NetworkResourceLoader::create(loadParameters, this); - m_networkResourceLoaders.add(loadParameters.identifier, loader); - NetworkProcess::shared().networkResourceLoadScheduler().scheduleLoader(loader.get()); + auto loader = NetworkResourceLoader::create(loadParameters, *this); + m_networkResourceLoaders.add(loadParameters.identifier, loader.ptr()); + loader->start(); +} + +void NetworkConnectionToWebProcess::performSynchronousLoad(const NetworkResourceLoadParameters& loadParameters, Ref<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>&& reply) +{ + auto loader = NetworkResourceLoader::create(loadParameters, *this, WTFMove(reply)); + m_networkResourceLoaders.add(loadParameters.identifier, loader.ptr()); + loader->start(); } -void NetworkConnectionToWebProcess::performSynchronousLoad(const NetworkResourceLoadParameters& loadParameters, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> reply) +void NetworkConnectionToWebProcess::loadPing(const NetworkResourceLoadParameters& loadParameters) { - RefPtr<NetworkResourceLoader> loader = NetworkResourceLoader::create(loadParameters, this, reply); - m_networkResourceLoaders.add(loadParameters.identifier, loader); - NetworkProcess::shared().networkResourceLoadScheduler().scheduleLoader(loader.get()); +#if USE(NETWORK_SESSION) + // PingLoad manages its own lifetime, deleting itself when its purpose has been fulfilled. + new PingLoad(loadParameters); +#else + RefPtr<NetworkingContext> context = RemoteNetworkingContext::create(loadParameters.sessionID, loadParameters.shouldClearReferrerOnHTTPSToHTTPRedirect); + + // PingHandle manages its own lifetime, deleting itself when its purpose has been fulfilled. + new PingHandle(context.get(), loadParameters.request, loadParameters.allowStoredCredentials == AllowStoredCredentials, PingHandle::UsesAsyncCallbacks::Yes, loadParameters.shouldFollowRedirects); +#endif } void NetworkConnectionToWebProcess::removeLoadIdentifier(ResourceLoadIdentifier identifier) { - RefPtr<NetworkResourceLoader> loader = m_networkResourceLoaders.take(identifier); + RefPtr<NetworkResourceLoader> loader = m_networkResourceLoaders.get(identifier); // It's possible we have no loader for this identifier if the NetworkProcess crashed and this was a respawned NetworkProcess. if (!loader) @@ -136,24 +198,28 @@ void NetworkConnectionToWebProcess::removeLoadIdentifier(ResourceLoadIdentifier // Abort the load now, as the WebProcess won't be able to respond to messages any more which might lead // to leaked loader resources (connections, threads, etc). loader->abort(); + ASSERT(!m_networkResourceLoaders.contains(identifier)); } -void NetworkConnectionToWebProcess::servePendingRequests(uint32_t resourceLoadPriority) +void NetworkConnectionToWebProcess::setDefersLoading(ResourceLoadIdentifier identifier, bool defers) { - NetworkProcess::shared().networkResourceLoadScheduler().servePendingRequests(static_cast<ResourceLoadPriority>(resourceLoadPriority)); + RefPtr<NetworkResourceLoader> loader = m_networkResourceLoaders.get(identifier); + if (!loader) + return; + + loader->setDefersLoading(defers); } -void NetworkConnectionToWebProcess::setSerialLoadingEnabled(bool enabled) +void NetworkConnectionToWebProcess::prefetchDNS(const String& hostname) { - m_serialLoadingEnabled = enabled; + NetworkProcess::singleton().prefetchDNS(hostname); } -static NetworkStorageSession& storageSession(uint64_t sessionID) +static NetworkStorageSession& storageSession(SessionID sessionID) { - if (SessionTracker::isEphemeralID(sessionID)) { - NetworkStorageSession* privateSession = SessionTracker::session(sessionID); - if (privateSession) - return *privateSession; + if (sessionID.isEphemeral()) { + if (auto* privateStorageSession = NetworkStorageSession::storageSession(sessionID)) + return *privateStorageSession; // Some requests with private browsing mode requested may still be coming shortly after NetworkProcess was told to destroy its session. // FIXME: Find a way to track private browsing sessions more rigorously. LOG_ERROR("Private browsing was requested, but there was no session for it. Please file a bug unless you just disabled private browsing, in which case it's an expected race."); @@ -161,79 +227,149 @@ static NetworkStorageSession& storageSession(uint64_t sessionID) return NetworkStorageSession::defaultStorageSession(); } -void NetworkConnectionToWebProcess::startDownload(uint64_t sessionID, uint64_t downloadID, const ResourceRequest& request) +void NetworkConnectionToWebProcess::startDownload(SessionID sessionID, DownloadID downloadID, const ResourceRequest& request, const String& suggestedName) { - // FIXME: Do something with the session ID. - NetworkProcess::shared().downloadManager().startDownload(downloadID, request); + NetworkProcess::singleton().downloadManager().startDownload(this, sessionID, downloadID, request, suggestedName); } -void NetworkConnectionToWebProcess::convertMainResourceLoadToDownload(uint64_t mainResourceLoadIdentifier, uint64_t downloadID, const ResourceRequest& request, const ResourceResponse& response) +void NetworkConnectionToWebProcess::convertMainResourceLoadToDownload(SessionID sessionID, uint64_t mainResourceLoadIdentifier, DownloadID downloadID, const ResourceRequest& request, const ResourceResponse& response) { + auto& networkProcess = NetworkProcess::singleton(); if (!mainResourceLoadIdentifier) { - NetworkProcess::shared().downloadManager().startDownload(downloadID, request); + networkProcess.downloadManager().startDownload(this, sessionID, downloadID, request); return; } NetworkResourceLoader* loader = m_networkResourceLoaders.get(mainResourceLoadIdentifier); - NetworkProcess::shared().downloadManager().convertHandleToDownload(downloadID, loader->handle(), request, response); + if (!loader) { + // If we're trying to download a blob here loader can be null. + return; + } - // Unblock the URL connection operation queue. - loader->handle()->continueDidReceiveResponse(); - - loader->didConvertHandleToDownload(); + loader->convertToDownload(downloadID, request, response); } -void NetworkConnectionToWebProcess::cookiesForDOM(uint64_t sessionID, const URL& firstParty, const URL& url, String& result) +void NetworkConnectionToWebProcess::cookiesForDOM(SessionID sessionID, const URL& firstParty, const URL& url, String& result) { result = WebCore::cookiesForDOM(storageSession(sessionID), firstParty, url); } -void NetworkConnectionToWebProcess::setCookiesFromDOM(uint64_t sessionID, const URL& firstParty, const URL& url, const String& cookieString) +void NetworkConnectionToWebProcess::setCookiesFromDOM(SessionID sessionID, const URL& firstParty, const URL& url, const String& cookieString) { WebCore::setCookiesFromDOM(storageSession(sessionID), firstParty, url, cookieString); } -void NetworkConnectionToWebProcess::cookiesEnabled(uint64_t sessionID, const URL& firstParty, const URL& url, bool& result) +void NetworkConnectionToWebProcess::cookiesEnabled(SessionID sessionID, const URL& firstParty, const URL& url, bool& result) { result = WebCore::cookiesEnabled(storageSession(sessionID), firstParty, url); } -void NetworkConnectionToWebProcess::cookieRequestHeaderFieldValue(uint64_t sessionID, const URL& firstParty, const URL& url, String& result) +void NetworkConnectionToWebProcess::cookieRequestHeaderFieldValue(SessionID sessionID, const URL& firstParty, const URL& url, String& result) { result = WebCore::cookieRequestHeaderFieldValue(storageSession(sessionID), firstParty, url); } -void NetworkConnectionToWebProcess::getRawCookies(uint64_t sessionID, const URL& firstParty, const URL& url, Vector<Cookie>& result) +void NetworkConnectionToWebProcess::getRawCookies(SessionID sessionID, const URL& firstParty, const URL& url, Vector<Cookie>& result) { WebCore::getRawCookies(storageSession(sessionID), firstParty, url, result); } -void NetworkConnectionToWebProcess::deleteCookie(uint64_t sessionID, const URL& url, const String& cookieName) +void NetworkConnectionToWebProcess::deleteCookie(SessionID sessionID, const URL& url, const String& cookieName) { WebCore::deleteCookie(storageSession(sessionID), url, cookieName); } -void NetworkConnectionToWebProcess::registerBlobURL(const URL& url, const BlobRegistrationData& data) +void NetworkConnectionToWebProcess::addCookie(SessionID sessionID, const URL& url, const Cookie& cookie) { - Vector<RefPtr<SandboxExtension>> extensions; - for (size_t i = 0, count = data.sandboxExtensions().size(); i < count; ++i) { - if (RefPtr<SandboxExtension> extension = SandboxExtension::create(data.sandboxExtensions()[i])) - extensions.append(extension); - } + WebCore::addCookie(storageSession(sessionID), url, cookie); +} + +void NetworkConnectionToWebProcess::registerFileBlobURL(const URL& url, const String& path, const SandboxExtension::Handle& extensionHandle, const String& contentType) +{ + RefPtr<SandboxExtension> extension = SandboxExtension::create(extensionHandle); - NetworkBlobRegistry::shared().registerBlobURL(this, url, data.releaseData(), extensions); + NetworkBlobRegistry::singleton().registerFileBlobURL(this, url, path, WTFMove(extension), contentType); +} + +void NetworkConnectionToWebProcess::registerBlobURL(const URL& url, Vector<BlobPart>&& blobParts, const String& contentType) +{ + NetworkBlobRegistry::singleton().registerBlobURL(this, url, WTFMove(blobParts), contentType); } void NetworkConnectionToWebProcess::registerBlobURLFromURL(const URL& url, const URL& srcURL) { - NetworkBlobRegistry::shared().registerBlobURL(this, url, srcURL); + NetworkBlobRegistry::singleton().registerBlobURL(this, url, srcURL); +} + +void NetworkConnectionToWebProcess::preregisterSandboxExtensionsForOptionallyFileBackedBlob(const Vector<String>& filePaths, const SandboxExtension::HandleArray& handles) +{ +#if ENABLE(SANDBOX_EXTENSIONS) + ASSERT(filePaths.size() == handles.size()); + + for (size_t i = 0; i < filePaths.size(); ++i) + m_blobDataFileReferences.add(filePaths[i], BlobDataFileReferenceWithSandboxExtension::create(filePaths[i], SandboxExtension::create(handles[i]))); +#else + for (size_t i = 0; i < filePaths.size(); ++i) + m_blobDataFileReferences.add(filePaths[i], BlobDataFileReferenceWithSandboxExtension::create(filePaths[i], nullptr)); +#endif +} + +RefPtr<WebCore::BlobDataFileReference> NetworkConnectionToWebProcess::getBlobDataFileReferenceForPath(const String& path) +{ + ASSERT(m_blobDataFileReferences.contains(path)); + return m_blobDataFileReferences.get(path); +} + +void NetworkConnectionToWebProcess::registerBlobURLOptionallyFileBacked(const URL& url, const URL& srcURL, const String& fileBackedPath, const String& contentType) +{ + NetworkBlobRegistry::singleton().registerBlobURLOptionallyFileBacked(this, url, srcURL, fileBackedPath, contentType); +} + +void NetworkConnectionToWebProcess::registerBlobURLForSlice(const URL& url, const URL& srcURL, int64_t start, int64_t end) +{ + NetworkBlobRegistry::singleton().registerBlobURLForSlice(this, url, srcURL, start, end); } void NetworkConnectionToWebProcess::unregisterBlobURL(const URL& url) { - NetworkBlobRegistry::shared().unregisterBlobURL(this, url); + NetworkBlobRegistry::singleton().unregisterBlobURL(this, url); } -} // namespace WebKit +void NetworkConnectionToWebProcess::blobSize(const URL& url, uint64_t& resultSize) +{ + resultSize = NetworkBlobRegistry::singleton().blobSize(this, url); +} + +void NetworkConnectionToWebProcess::writeBlobsToTemporaryFiles(const Vector<String>& blobURLs, uint64_t requestIdentifier) +{ + Vector<RefPtr<BlobDataFileReference>> fileReferences; + for (auto& url : blobURLs) + fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(*this, { ParsedURLString, url })); + + for (auto& file : fileReferences) + file->prepareForFileAccess(); + + NetworkBlobRegistry::singleton().writeBlobsToTemporaryFiles(blobURLs, [this, protectedThis = makeRef(*this), requestIdentifier, fileReferences = WTFMove(fileReferences)](auto& fileNames) mutable { + for (auto& file : fileReferences) + file->revokeFileAccess(); + + NetworkProcess::singleton().grantSandboxExtensionsToDatabaseProcessForBlobs(fileNames, [this, protectedThis = WTFMove(protectedThis), requestIdentifier, fileNames]() { + if (!m_connection->isValid()) + return; + + m_connection->send(Messages::NetworkProcessConnection::DidWriteBlobsToTemporaryFiles(requestIdentifier, fileNames), 0); + }); + }); +} -#endif // ENABLE(NETWORK_PROCESS) +void NetworkConnectionToWebProcess::storeDerivedDataToCache(const WebKit::NetworkCache::DataKey& dataKey, const IPC::DataReference& data) +{ + NetworkCache::singleton().storeData(dataKey, data.data(), data.size()); +} + +void NetworkConnectionToWebProcess::ensureLegacyPrivateBrowsingSession() +{ + NetworkProcess::singleton().ensurePrivateBrowsingSession(SessionID::legacyPrivateSessionID()); +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.h b/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.h index 3981fa5eb..9729314d6 100644 --- a/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.h +++ b/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 Apple Inc. All rights reserved. + * Copyright (C) 2012-2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -23,82 +23,102 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef NetworkConnectionToWebProcess_h -#define NetworkConnectionToWebProcess_h - -#if ENABLE(NETWORK_PROCESS) +#pragma once #include "BlockingResponseMap.h" #include "Connection.h" +#include "DownloadID.h" #include "NetworkConnectionToWebProcessMessages.h" +#include "NetworkRTCProvider.h" + #include <WebCore/ResourceLoadPriority.h> -#include <wtf/HashSet.h> #include <wtf/RefCounted.h> namespace WebCore { +class BlobDataFileReference; class ResourceRequest; } namespace WebKit { -class BlobRegistrationData; class NetworkConnectionToWebProcess; class NetworkResourceLoader; class SyncNetworkResourceLoader; typedef uint64_t ResourceLoadIdentifier; +namespace NetworkCache { +struct DataKey; +} + class NetworkConnectionToWebProcess : public RefCounted<NetworkConnectionToWebProcess>, IPC::Connection::Client { public: - static PassRefPtr<NetworkConnectionToWebProcess> create(IPC::Connection::Identifier); + static Ref<NetworkConnectionToWebProcess> create(IPC::Connection::Identifier); virtual ~NetworkConnectionToWebProcess(); - IPC::Connection* connection() const { return m_connection.get(); } + IPC::Connection& connection() { return m_connection.get(); } + + void didCleanupResourceLoader(NetworkResourceLoader&); - bool isSerialLoadingEnabled() const { return m_serialLoadingEnabled; } + RefPtr<WebCore::BlobDataFileReference> getBlobDataFileReferenceForPath(const String& path); private: NetworkConnectionToWebProcess(IPC::Connection::Identifier); // IPC::Connection::Client - virtual void didReceiveMessage(IPC::Connection*, IPC::MessageDecoder&); - virtual void didReceiveSyncMessage(IPC::Connection*, IPC::MessageDecoder&, std::unique_ptr<IPC::MessageEncoder>&); - virtual void didClose(IPC::Connection*); - virtual void didReceiveInvalidMessage(IPC::Connection*, IPC::StringReference messageReceiverName, IPC::StringReference messageName); + void didReceiveMessage(IPC::Connection&, IPC::Decoder&) override; + void didReceiveSyncMessage(IPC::Connection&, IPC::Decoder&, std::unique_ptr<IPC::Encoder>&) override; + void didClose(IPC::Connection&) override; + void didReceiveInvalidMessage(IPC::Connection&, IPC::StringReference messageReceiverName, IPC::StringReference messageName) override; // Message handlers. - void didReceiveNetworkConnectionToWebProcessMessage(IPC::Connection*, IPC::MessageDecoder&); - void didReceiveSyncNetworkConnectionToWebProcessMessage(IPC::Connection*, IPC::MessageDecoder&, std::unique_ptr<IPC::MessageEncoder>&); - + void didReceiveNetworkConnectionToWebProcessMessage(IPC::Connection&, IPC::Decoder&); + void didReceiveSyncNetworkConnectionToWebProcessMessage(IPC::Connection&, IPC::Decoder&, std::unique_ptr<IPC::Encoder>&); + void scheduleResourceLoad(const NetworkResourceLoadParameters&); - void performSynchronousLoad(const NetworkResourceLoadParameters&, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>); + void performSynchronousLoad(const NetworkResourceLoadParameters&, Ref<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>&&); + void loadPing(const NetworkResourceLoadParameters&); + void prefetchDNS(const String&); void removeLoadIdentifier(ResourceLoadIdentifier); + void setDefersLoading(ResourceLoadIdentifier, bool); void crossOriginRedirectReceived(ResourceLoadIdentifier, const WebCore::URL& redirectURL); - void servePendingRequests(uint32_t resourceLoadPriority); - void setSerialLoadingEnabled(bool); - void startDownload(uint64_t sessionID, uint64_t downloadID, const WebCore::ResourceRequest&); - void convertMainResourceLoadToDownload(uint64_t mainResourceLoadIdentifier, uint64_t downloadID, const WebCore::ResourceRequest&, const WebCore::ResourceResponse&); - - void cookiesForDOM(uint64_t sessionID, const WebCore::URL& firstParty, const WebCore::URL&, String& result); - void setCookiesFromDOM(uint64_t sessionID, const WebCore::URL& firstParty, const WebCore::URL&, const String&); - void cookiesEnabled(uint64_t sessionID, const WebCore::URL& firstParty, const WebCore::URL&, bool& result); - void cookieRequestHeaderFieldValue(uint64_t sessionID, const WebCore::URL& firstParty, const WebCore::URL&, String& result); - void getRawCookies(uint64_t sessionID, const WebCore::URL& firstParty, const WebCore::URL&, Vector<WebCore::Cookie>&); - void deleteCookie(uint64_t sessionID, const WebCore::URL&, const String& cookieName); - - void registerBlobURL(const WebCore::URL&, const BlobRegistrationData&); + void startDownload(WebCore::SessionID, DownloadID, const WebCore::ResourceRequest&, const String& suggestedName = { }); + void convertMainResourceLoadToDownload(WebCore::SessionID, uint64_t mainResourceLoadIdentifier, DownloadID, const WebCore::ResourceRequest&, const WebCore::ResourceResponse&); + + void cookiesForDOM(WebCore::SessionID, const WebCore::URL& firstParty, const WebCore::URL&, String& result); + void setCookiesFromDOM(WebCore::SessionID, const WebCore::URL& firstParty, const WebCore::URL&, const String&); + void cookiesEnabled(WebCore::SessionID, const WebCore::URL& firstParty, const WebCore::URL&, bool& result); + void cookieRequestHeaderFieldValue(WebCore::SessionID, const WebCore::URL& firstParty, const WebCore::URL&, String& result); + void getRawCookies(WebCore::SessionID, const WebCore::URL& firstParty, const WebCore::URL&, Vector<WebCore::Cookie>&); + void deleteCookie(WebCore::SessionID, const WebCore::URL&, const String& cookieName); + void addCookie(WebCore::SessionID, const WebCore::URL&, const WebCore::Cookie&); + + void registerFileBlobURL(const WebCore::URL&, const String& path, const SandboxExtension::Handle&, const String& contentType); + void registerBlobURL(const WebCore::URL&, Vector<WebCore::BlobPart>&&, const String& contentType); void registerBlobURLFromURL(const WebCore::URL&, const WebCore::URL& srcURL); + void preregisterSandboxExtensionsForOptionallyFileBackedBlob(const Vector<String>& fileBackedPath, const SandboxExtension::HandleArray&); + void registerBlobURLOptionallyFileBacked(const WebCore::URL&, const WebCore::URL& srcURL, const String& fileBackedPath, const String& contentType); + void registerBlobURLForSlice(const WebCore::URL&, const WebCore::URL& srcURL, int64_t start, int64_t end); + void blobSize(const WebCore::URL&, uint64_t& resultSize); void unregisterBlobURL(const WebCore::URL&); + void writeBlobsToTemporaryFiles(const Vector<String>& blobURLs, uint64_t requestIdentifier); + + void storeDerivedDataToCache(const WebKit::NetworkCache::DataKey&, const IPC::DataReference&); + + void ensureLegacyPrivateBrowsingSession(); - RefPtr<IPC::Connection> m_connection; +#if USE(LIBWEBRTC) + NetworkRTCProvider& rtcProvider(); +#endif + + Ref<IPC::Connection> m_connection; HashMap<ResourceLoadIdentifier, RefPtr<NetworkResourceLoader>> m_networkResourceLoaders; + HashMap<String, RefPtr<WebCore::BlobDataFileReference>> m_blobDataFileReferences; - bool m_serialLoadingEnabled; +#if USE(LIBWEBRTC) + RefPtr<NetworkRTCProvider> m_rtcProvider; +#endif }; } // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) - -#endif // NetworkConnectionToWebProcess_h diff --git a/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.messages.in b/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.messages.in index a4597fcd2..a263a4ede 100644 --- a/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.messages.in +++ b/Source/WebKit2/NetworkProcess/NetworkConnectionToWebProcess.messages.in @@ -20,31 +20,37 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#if ENABLE(NETWORK_PROCESS) - messages -> NetworkConnectionToWebProcess LegacyReceiver { ScheduleResourceLoad(WebKit::NetworkResourceLoadParameters resourceLoadParameters) PerformSynchronousLoad(WebKit::NetworkResourceLoadParameters resourceLoadParameters) -> (WebCore::ResourceError error, WebCore::ResourceResponse response, Vector<char> data) Delayed + LoadPing(WebKit::NetworkResourceLoadParameters resourceLoadParameters) RemoveLoadIdentifier(uint64_t resourceLoadIdentifier) - - ServePendingRequests(uint32_t resourceLoadPriority) - - SetSerialLoadingEnabled(bool enabled) -> () + SetDefersLoading(uint64_t resourceLoadIdentifier, bool defers) + PrefetchDNS(String hostname) - StartDownload(uint64_t sessionID, uint64_t downloadID, WebCore::ResourceRequest request) - ConvertMainResourceLoadToDownload(uint64_t mainResourceLoadIdentifier, uint64_t downloadID, WebCore::ResourceRequest request, WebCore::ResourceResponse response) + StartDownload(WebCore::SessionID sessionID, WebKit::DownloadID downloadID, WebCore::ResourceRequest request, String suggestedName) + ConvertMainResourceLoadToDownload(WebCore::SessionID sessionID, uint64_t mainResourceLoadIdentifier, WebKit::DownloadID downloadID, WebCore::ResourceRequest request, WebCore::ResourceResponse response) - CookiesForDOM(uint64_t sessionID, WebCore::URL firstParty, WebCore::URL url) -> (String result) - SetCookiesFromDOM(uint64_t sessionID, WebCore::URL firstParty, WebCore::URL url, String cookieString) - CookiesEnabled(uint64_t sessionID, WebCore::URL firstParty, WebCore::URL url) -> (bool enabled) - CookieRequestHeaderFieldValue(uint64_t sessionID, WebCore::URL firstParty, WebCore::URL url) -> (String result) - GetRawCookies(uint64_t sessionID, WebCore::URL firstParty, WebCore::URL url) -> (Vector<WebCore::Cookie> cookies) - DeleteCookie(uint64_t sessionID, WebCore::URL url, String cookieName) + CookiesForDOM(WebCore::SessionID sessionID, WebCore::URL firstParty, WebCore::URL url) -> (String result) + SetCookiesFromDOM(WebCore::SessionID sessionID, WebCore::URL firstParty, WebCore::URL url, String cookieString) + CookiesEnabled(WebCore::SessionID sessionID, WebCore::URL firstParty, WebCore::URL url) -> (bool enabled) + CookieRequestHeaderFieldValue(WebCore::SessionID sessionID, WebCore::URL firstParty, WebCore::URL url) -> (String result) + GetRawCookies(WebCore::SessionID sessionID, WebCore::URL firstParty, WebCore::URL url) -> (Vector<WebCore::Cookie> cookies) + DeleteCookie(WebCore::SessionID sessionID, WebCore::URL url, String cookieName) + AddCookie(WebCore::SessionID sessionID, WebCore::URL url, struct WebCore::Cookie cookie) - RegisterBlobURL(WebCore::URL url, WebKit::BlobRegistrationData data) + RegisterFileBlobURL(WebCore::URL url, String path, WebKit::SandboxExtension::Handle extensionHandle, String contentType) + RegisterBlobURL(WebCore::URL url, Vector<WebCore::BlobPart> blobParts, String contentType) RegisterBlobURLFromURL(WebCore::URL url, WebCore::URL srcURL) + PreregisterSandboxExtensionsForOptionallyFileBackedBlob(Vector<String> filePaths, WebKit::SandboxExtension::HandleArray extensionHandles) + RegisterBlobURLOptionallyFileBacked(WebCore::URL url, WebCore::URL srcURL, String fileBackedPath, String contentType) + RegisterBlobURLForSlice(WebCore::URL url, WebCore::URL srcURL, int64_t start, int64_t end) UnregisterBlobURL(WebCore::URL url) -} + BlobSize(WebCore::URL url) -> (uint64_t resultSize) + WriteBlobsToTemporaryFiles(Vector<String> blobURLs, uint64_t requestIdentifier) -#endif // ENABLE(NETWORK_PROCESS) + StoreDerivedDataToCache(WebKit::NetworkCache::DataKey key, IPC::DataReference data) + + EnsureLegacyPrivateBrowsingSession() +} diff --git a/Source/WebKit2/NetworkProcess/NetworkDataTask.cpp b/Source/WebKit2/NetworkProcess/NetworkDataTask.cpp new file mode 100644 index 000000000..aeee92df7 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkDataTask.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkDataTask.h" + +#if USE(NETWORK_SESSION) + +#include "NetworkDataTaskBlob.h" +#include "NetworkLoadParameters.h" +#include "NetworkSession.h" +#include <WebCore/ResourceError.h> +#include <WebCore/ResourceResponse.h> +#include <wtf/MainThread.h> + +#if PLATFORM(COCOA) +#include "NetworkDataTaskCocoa.h" +#endif +#if USE(SOUP) +#include "NetworkDataTaskSoup.h" +#endif + +using namespace WebCore; + +namespace WebKit { + +Ref<NetworkDataTask> NetworkDataTask::create(NetworkSession& session, NetworkDataTaskClient& client, const NetworkLoadParameters& parameters) +{ + if (parameters.request.url().protocolIsBlob()) + return NetworkDataTaskBlob::create(session, client, parameters.request, parameters.contentSniffingPolicy, parameters.blobFileReferences); + +#if PLATFORM(COCOA) + return NetworkDataTaskCocoa::create(session, client, parameters.request, parameters.allowStoredCredentials, parameters.contentSniffingPolicy, parameters.shouldClearReferrerOnHTTPSToHTTPRedirect); +#endif +#if USE(SOUP) + return NetworkDataTaskSoup::create(session, client, parameters.request, parameters.allowStoredCredentials, parameters.contentSniffingPolicy, parameters.shouldClearReferrerOnHTTPSToHTTPRedirect); +#endif +} + +NetworkDataTask::NetworkDataTask(NetworkSession& session, NetworkDataTaskClient& client, const ResourceRequest& requestWithCredentials, StoredCredentials storedCredentials, bool shouldClearReferrerOnHTTPSToHTTPRedirect) + : m_failureTimer(*this, &NetworkDataTask::failureTimerFired) + , m_session(session) + , m_client(&client) + , m_partition(requestWithCredentials.cachePartition()) + , m_storedCredentials(storedCredentials) + , m_lastHTTPMethod(requestWithCredentials.httpMethod()) + , m_firstRequest(requestWithCredentials) + , m_shouldClearReferrerOnHTTPSToHTTPRedirect(shouldClearReferrerOnHTTPSToHTTPRedirect) +{ + ASSERT(isMainThread()); + + if (!requestWithCredentials.url().isValid()) { + scheduleFailure(InvalidURLFailure); + return; + } + + if (!portAllowed(requestWithCredentials.url())) { + scheduleFailure(BlockedFailure); + return; + } +} + +NetworkDataTask::~NetworkDataTask() +{ + ASSERT(isMainThread()); + ASSERT(!m_client); +} + +void NetworkDataTask::scheduleFailure(FailureType type) +{ + ASSERT(type != NoFailure); + m_scheduledFailureType = type; + m_failureTimer.startOneShot(0); +} + +void NetworkDataTask::didReceiveResponse(ResourceResponse&& response, ResponseCompletionHandler&& completionHandler) +{ + ASSERT(m_client); + if (response.isHTTP09()) { + auto url = response.url(); + std::optional<uint16_t> port = url.port(); + if (port && !isDefaultPortForProtocol(port.value(), url.protocol())) { + cancel(); + m_client->didCompleteWithError({ String(), 0, url, "Cancelled load from '" + url.stringCenterEllipsizedToLength() + "' because it is using HTTP/0.9." }); + return; + } + } + m_client->didReceiveResponseNetworkSession(WTFMove(response), WTFMove(completionHandler)); +} + +void NetworkDataTask::failureTimerFired() +{ + RefPtr<NetworkDataTask> protectedThis(this); + + switch (m_scheduledFailureType) { + case BlockedFailure: + m_scheduledFailureType = NoFailure; + if (m_client) + m_client->wasBlocked(); + return; + case InvalidURLFailure: + m_scheduledFailureType = NoFailure; + if (m_client) + m_client->cannotShowURL(); + return; + case NoFailure: + ASSERT_NOT_REACHED(); + break; + } + ASSERT_NOT_REACHED(); +} + +} // namespace WebKit + +#endif // USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/NetworkDataTask.h b/Source/WebKit2/NetworkProcess/NetworkDataTask.h new file mode 100644 index 000000000..cc47307d8 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkDataTask.h @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(NETWORK_SESSION) + +#include "DownloadID.h" +#include "SandboxExtension.h" +#include <WebCore/Credential.h> +#include <WebCore/FrameLoaderTypes.h> +#include <WebCore/ResourceHandleTypes.h> +#include <WebCore/ResourceLoaderOptions.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/Timer.h> +#include <wtf/Function.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class AuthenticationChallenge; +class ResourceError; +class ResourceResponse; +class SharedBuffer; +} + +namespace WebKit { + +class Download; +class NetworkLoadParameters; +class NetworkSession; +class PendingDownload; +enum class AuthenticationChallengeDisposition; + +typedef Function<void(const WebCore::ResourceRequest&)> RedirectCompletionHandler; +typedef Function<void(AuthenticationChallengeDisposition, const WebCore::Credential&)> ChallengeCompletionHandler; +typedef Function<void(WebCore::PolicyAction)> ResponseCompletionHandler; + +class NetworkDataTaskClient { +public: + virtual void willPerformHTTPRedirection(WebCore::ResourceResponse&&, WebCore::ResourceRequest&&, RedirectCompletionHandler&&) = 0; + virtual void didReceiveChallenge(const WebCore::AuthenticationChallenge&, ChallengeCompletionHandler&&) = 0; + virtual void didReceiveResponseNetworkSession(WebCore::ResourceResponse&&, ResponseCompletionHandler&&) = 0; + virtual void didReceiveData(Ref<WebCore::SharedBuffer>&&) = 0; + virtual void didCompleteWithError(const WebCore::ResourceError&) = 0; + virtual void didSendData(uint64_t totalBytesSent, uint64_t totalBytesExpectedToSend) = 0; + virtual void wasBlocked() = 0; + virtual void cannotShowURL() = 0; + + virtual ~NetworkDataTaskClient() { } +}; + +class NetworkDataTask : public RefCounted<NetworkDataTask> { +public: + static Ref<NetworkDataTask> create(NetworkSession&, NetworkDataTaskClient&, const NetworkLoadParameters&); + + virtual ~NetworkDataTask(); + + virtual void suspend() = 0; + virtual void cancel() = 0; + virtual void resume() = 0; + virtual void invalidateAndCancel() = 0; + + void didReceiveResponse(WebCore::ResourceResponse&&, ResponseCompletionHandler&&); + + enum class State { + Running, + Suspended, + Canceling, + Completed + }; + virtual State state() const = 0; + + NetworkDataTaskClient* client() const { return m_client; } + void clearClient() { m_client = nullptr; } + + DownloadID pendingDownloadID() const { return m_pendingDownloadID; } + PendingDownload* pendingDownload() const { return m_pendingDownload; } + void setPendingDownloadID(DownloadID downloadID) + { + ASSERT(!m_pendingDownloadID.downloadID()); + ASSERT(downloadID.downloadID()); + m_pendingDownloadID = downloadID; + } + void setPendingDownload(PendingDownload& pendingDownload) + { + ASSERT(!m_pendingDownload); + m_pendingDownload = &pendingDownload; + } + + virtual void setPendingDownloadLocation(const String& filename, const SandboxExtension::Handle&, bool /*allowOverwrite*/) { m_pendingDownloadLocation = filename; } + const String& pendingDownloadLocation() const { return m_pendingDownloadLocation; } + bool isDownload() const { return !!m_pendingDownloadID.downloadID(); } + + const WebCore::ResourceRequest& firstRequest() const { return m_firstRequest; } + virtual String suggestedFilename() const { return String(); } + void setSuggestedFilename(const String& suggestedName) { m_suggestedFilename = suggestedName; } + virtual bool allowsSpecificHTTPSCertificateForHost(const WebCore::AuthenticationChallenge&) { return false; } + const String& partition() { return m_partition; } + +protected: + NetworkDataTask(NetworkSession&, NetworkDataTaskClient&, const WebCore::ResourceRequest&, WebCore::StoredCredentials, bool shouldClearReferrerOnHTTPSToHTTPRedirect); + + enum FailureType { + NoFailure, + BlockedFailure, + InvalidURLFailure + }; + void failureTimerFired(); + void scheduleFailure(FailureType); + + FailureType m_scheduledFailureType { NoFailure }; + WebCore::Timer m_failureTimer; + Ref<NetworkSession> m_session; + NetworkDataTaskClient* m_client { nullptr }; + PendingDownload* m_pendingDownload { nullptr }; + DownloadID m_pendingDownloadID; + String m_user; + String m_password; + String m_partition; +#if USE(CREDENTIAL_STORAGE_WITH_NETWORK_SESSION) + WebCore::Credential m_initialCredential; +#endif + WebCore::StoredCredentials m_storedCredentials { WebCore::DoNotAllowStoredCredentials }; + String m_lastHTTPMethod; + String m_pendingDownloadLocation; + WebCore::ResourceRequest m_firstRequest; + bool m_shouldClearReferrerOnHTTPSToHTTPRedirect { true }; + String m_suggestedFilename; +}; + +} // namespace WebKit + +#endif // USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/NetworkDataTaskBlob.cpp b/Source/WebKit2/NetworkProcess/NetworkDataTaskBlob.cpp new file mode 100644 index 000000000..6f4bbf59a --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkDataTaskBlob.cpp @@ -0,0 +1,588 @@ +/* + * Copyright (C) 2010 Google Inc. All rights reserved. + * Copyright (C) 2016 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkDataTaskBlob.h" + +#if USE(NETWORK_SESSION) + +#include "DataReference.h" +#include "Download.h" +#include "Logging.h" +#include "NetworkProcess.h" +#include "NetworkSession.h" +#include "WebErrors.h" +#include <WebCore/AsyncFileStream.h> +#include <WebCore/BlobData.h> +#include <WebCore/BlobRegistryImpl.h> +#include <WebCore/FileStream.h> +#include <WebCore/HTTPHeaderNames.h> +#include <WebCore/HTTPParsers.h> +#include <WebCore/ParsedContentRange.h> +#include <WebCore/ResourceError.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> +#include <WebCore/URL.h> +#include <wtf/MainThread.h> +#include <wtf/RunLoop.h> + +using namespace WebCore; + +namespace WebKit { + +static const unsigned bufferSize = 512 * 1024; + +static const int httpOK = 200; +static const int httpPartialContent = 206; +static const int httpNotAllowed = 403; +static const int httpRequestedRangeNotSatisfiable = 416; +static const int httpInternalError = 500; +static const char* httpOKText = "OK"; +static const char* httpPartialContentText = "Partial Content"; +static const char* httpNotAllowedText = "Not Allowed"; +static const char* httpRequestedRangeNotSatisfiableText = "Requested Range Not Satisfiable"; +static const char* httpInternalErrorText = "Internal Server Error"; + +static const char* const webKitBlobResourceDomain = "WebKitBlobResource"; + +NetworkDataTaskBlob::NetworkDataTaskBlob(NetworkSession& session, NetworkDataTaskClient& client, const ResourceRequest& request, ContentSniffingPolicy shouldContentSniff, const Vector<RefPtr<WebCore::BlobDataFileReference>>& fileReferences) + : NetworkDataTask(session, client, request, DoNotAllowStoredCredentials, false) + , m_stream(std::make_unique<AsyncFileStream>(*this)) + , m_fileReferences(fileReferences) +{ + for (auto& fileReference : m_fileReferences) + fileReference->prepareForFileAccess(); + + m_blobData = static_cast<BlobRegistryImpl&>(blobRegistry()).getBlobDataFromURL(request.url()); + + m_session->registerNetworkDataTask(*this); + LOG(NetworkSession, "%p - Created NetworkDataTaskBlob for %s", this, request.url().string().utf8().data()); +} + +NetworkDataTaskBlob::~NetworkDataTaskBlob() +{ + for (auto& fileReference : m_fileReferences) + fileReference->revokeFileAccess(); + + clearStream(); + m_session->unregisterNetworkDataTask(*this); +} + +void NetworkDataTaskBlob::clearStream() +{ + if (m_state == State::Completed) + return; + + m_state = State::Completed; + + if (m_fileOpened) { + m_fileOpened = false; + m_stream->close(); + } + m_stream = nullptr; +} + +void NetworkDataTaskBlob::resume() +{ + ASSERT(m_state != State::Running); + if (m_state == State::Canceling || m_state == State::Completed) + return; + + m_state = State::Running; + + if (m_scheduledFailureType != NoFailure) { + ASSERT(m_failureTimer.isActive()); + return; + } + + RunLoop::main().dispatch([this, protectedThis = makeRef(*this)] { + if (m_state == State::Canceling || m_state == State::Completed || !m_client) { + clearStream(); + return; + } + + if (!equalLettersIgnoringASCIICase(m_firstRequest.httpMethod(), "get")) { + didFail(Error::MethodNotAllowed); + return; + } + + // If the blob data is not found, fail now. + if (!m_blobData) { + didFail(Error::NotFoundError); + return; + } + + // Parse the "Range" header we care about. + String range = m_firstRequest.httpHeaderField(HTTPHeaderName::Range); + if (!range.isEmpty() && !parseRange(range, m_rangeOffset, m_rangeEnd, m_rangeSuffixLength)) { + dispatchDidReceiveResponse(Error::RangeError); + return; + } + + getSizeForNext(); + }); +} + +void NetworkDataTaskBlob::suspend() +{ + // FIXME: can this happen? +} + +void NetworkDataTaskBlob::cancel() +{ + if (m_state == State::Canceling || m_state == State::Completed) + return; + + m_state = State::Canceling; + + if (m_fileOpened) { + m_fileOpened = false; + m_stream->close(); + } + + if (isDownload()) + cleanDownloadFiles(); +} + +void NetworkDataTaskBlob::invalidateAndCancel() +{ + cancel(); + clearStream(); +} + +void NetworkDataTaskBlob::getSizeForNext() +{ + ASSERT(isMainThread()); + + // Do we finish validating and counting size for all items? + if (m_sizeItemCount >= m_blobData->items().size()) { + seek(); + dispatchDidReceiveResponse(); + return; + } + + const BlobDataItem& item = m_blobData->items().at(m_sizeItemCount); + switch (item.type()) { + case BlobDataItem::Type::Data: + didGetSize(item.length()); + break; + case BlobDataItem::Type::File: + // Files know their sizes, but asking the stream to verify that the file wasn't modified. + m_stream->getSize(item.file()->path(), item.file()->expectedModificationTime()); + break; + default: + ASSERT_NOT_REACHED(); + } +} + +void NetworkDataTaskBlob::didGetSize(long long size) +{ + ASSERT(isMainThread()); + + if (m_state == State::Canceling || m_state == State::Completed || (!m_client && !isDownload())) { + clearStream(); + return; + } + + // If the size is -1, it means the file has been moved or changed. Fail now. + if (size == -1) { + didFail(Error::NotFoundError); + return; + } + + // The size passed back is the size of the whole file. If the underlying item is a sliced file, we need to use the slice length. + const BlobDataItem& item = m_blobData->items().at(m_sizeItemCount); + size = item.length(); + + // Cache the size. + m_itemLengthList.append(size); + + // Count the size. + m_totalSize += size; + m_totalRemainingSize += size; + m_sizeItemCount++; + + // Continue with the next item. + getSizeForNext(); +} + +void NetworkDataTaskBlob::seek() +{ + ASSERT(isMainThread()); + + // Convert from the suffix length to the range. + if (m_rangeSuffixLength != kPositionNotSpecified) { + m_rangeOffset = m_totalRemainingSize - m_rangeSuffixLength; + m_rangeEnd = m_rangeOffset + m_rangeSuffixLength - 1; + } + + // Bail out if the range is not provided. + if (m_rangeOffset == kPositionNotSpecified) + return; + + // Skip the initial items that are not in the range. + long long offset = m_rangeOffset; + for (m_readItemCount = 0; m_readItemCount < m_blobData->items().size() && offset >= m_itemLengthList[m_readItemCount]; ++m_readItemCount) + offset -= m_itemLengthList[m_readItemCount]; + + // Set the offset that need to jump to for the first item in the range. + m_currentItemReadSize = offset; + + // Adjust the total remaining size in order not to go beyond the range. + if (m_rangeEnd != kPositionNotSpecified) { + long long rangeSize = m_rangeEnd - m_rangeOffset + 1; + if (m_totalRemainingSize > rangeSize) + m_totalRemainingSize = rangeSize; + } else + m_totalRemainingSize -= m_rangeOffset; +} + +void NetworkDataTaskBlob::dispatchDidReceiveResponse(Error errorCode) +{ + LOG(NetworkSession, "%p - NetworkDataTaskBlob::dispatchDidReceiveResponse(%u)", this, static_cast<unsigned>(errorCode)); + + Ref<NetworkDataTaskBlob> protectedThis(*this); + ResourceResponse response(m_firstRequest.url(), errorCode != Error::NoError ? "text/plain" : m_blobData->contentType(), errorCode != Error::NoError ? 0 : m_totalRemainingSize, String()); + switch (errorCode) { + case Error::NoError: { + bool isRangeRequest = m_rangeOffset != kPositionNotSpecified; + response.setHTTPStatusCode(isRangeRequest ? httpPartialContent : httpOK); + response.setHTTPStatusText(isRangeRequest ? httpPartialContentText : httpOKText); + + response.setHTTPHeaderField(HTTPHeaderName::ContentType, m_blobData->contentType()); + response.setHTTPHeaderField(HTTPHeaderName::ContentLength, String::number(m_totalRemainingSize)); + + if (isRangeRequest) + response.setHTTPHeaderField(HTTPHeaderName::ContentRange, ParsedContentRange(m_rangeOffset, m_rangeEnd, m_totalSize).headerValue()); + // FIXME: If a resource identified with a blob: URL is a File object, user agents must use that file's name attribute, + // as if the response had a Content-Disposition header with the filename parameter set to the File's name attribute. + // Notably, this will affect a name suggested in "File Save As". + break; + } + case Error::RangeError: + response.setHTTPStatusCode(httpRequestedRangeNotSatisfiable); + response.setHTTPStatusText(httpRequestedRangeNotSatisfiableText); + break; + case Error::SecurityError: + response.setHTTPStatusCode(httpNotAllowed); + response.setHTTPStatusText(httpNotAllowedText); + break; + default: + response.setHTTPStatusCode(httpInternalError); + response.setHTTPStatusText(httpInternalErrorText); + break; + } + + didReceiveResponse(WTFMove(response), [this, protectedThis = WTFMove(protectedThis), errorCode](PolicyAction policyAction) { + LOG(NetworkSession, "%p - NetworkDataTaskBlob::didReceiveResponse completionHandler (%u)", this, static_cast<unsigned>(policyAction)); + + if (m_state == State::Canceling || m_state == State::Completed) { + clearStream(); + return; + } + + if (errorCode != Error::NoError) { + didFinish(); + return; + } + + switch (policyAction) { + case PolicyAction::PolicyUse: + m_buffer.resize(bufferSize); + read(); + break; + case PolicyAction::PolicyIgnore: + break; + case PolicyAction::PolicyDownload: + download(); + break; + } + }); +} + +void NetworkDataTaskBlob::read() +{ + ASSERT(isMainThread()); + + // If there is no more remaining data to read, we are done. + if (!m_totalRemainingSize || m_readItemCount >= m_blobData->items().size()) { + didFinish(); + return; + } + + const BlobDataItem& item = m_blobData->items().at(m_readItemCount); + if (item.type() == BlobDataItem::Type::Data) + readData(item); + else if (item.type() == BlobDataItem::Type::File) + readFile(item); + else + ASSERT_NOT_REACHED(); +} + +void NetworkDataTaskBlob::readData(const BlobDataItem& item) +{ + ASSERT(item.data().data()); + + long long bytesToRead = item.length() - m_currentItemReadSize; + if (bytesToRead > m_totalRemainingSize) + bytesToRead = m_totalRemainingSize; + consumeData(reinterpret_cast<const char*>(item.data().data()->data()) + item.offset() + m_currentItemReadSize, static_cast<int>(bytesToRead)); + m_currentItemReadSize = 0; +} + +void NetworkDataTaskBlob::readFile(const BlobDataItem& item) +{ + ASSERT(m_stream); + + if (m_fileOpened) { + m_stream->read(m_buffer.data(), m_buffer.size()); + return; + } + + long long bytesToRead = m_itemLengthList[m_readItemCount] - m_currentItemReadSize; + if (bytesToRead > m_totalRemainingSize) + bytesToRead = static_cast<int>(m_totalRemainingSize); + m_stream->openForRead(item.file()->path(), item.offset() + m_currentItemReadSize, bytesToRead); + m_fileOpened = true; + m_currentItemReadSize = 0; +} + +void NetworkDataTaskBlob::didOpen(bool success) +{ + if (m_state == State::Canceling || m_state == State::Completed || (!m_client && !isDownload())) { + clearStream(); + return; + } + + if (!success) { + didFail(Error::NotReadableError); + return; + } + + Ref<NetworkDataTaskBlob> protectedThis(*this); + read(); +} + +void NetworkDataTaskBlob::didRead(int bytesRead) +{ + if (m_state == State::Canceling || m_state == State::Completed || (!m_client && !isDownload())) { + clearStream(); + return; + } + + if (bytesRead < 0) { + didFail(Error::NotReadableError); + return; + } + + Ref<NetworkDataTaskBlob> protectedThis(*this); + consumeData(m_buffer.data(), bytesRead); +} + +void NetworkDataTaskBlob::consumeData(const char* data, int bytesRead) +{ + m_totalRemainingSize -= bytesRead; + + if (bytesRead) { + if (m_downloadFile != invalidPlatformFileHandle) { + if (!writeDownload(data, bytesRead)) + return; + } else { + ASSERT(m_client); + m_client->didReceiveData(SharedBuffer::create(data, bytesRead)); + } + } + + if (m_fileOpened) { + // When the current item is a file item, the reading is completed only if bytesRead is 0. + if (!bytesRead) { + // Close the file. + m_fileOpened = false; + m_stream->close(); + + // Move to the next item. + m_readItemCount++; + } + } else { + // Otherwise, we read the current text item as a whole and move to the next item. + m_readItemCount++; + } + + read(); +} + +void NetworkDataTaskBlob::setPendingDownloadLocation(const String& filename, const SandboxExtension::Handle& sandboxExtensionHandle, bool allowOverwrite) +{ + NetworkDataTask::setPendingDownloadLocation(filename, sandboxExtensionHandle, allowOverwrite); + + ASSERT(!m_sandboxExtension); + m_sandboxExtension = SandboxExtension::create(sandboxExtensionHandle); + if (m_sandboxExtension) + m_sandboxExtension->consume(); + + if (allowOverwrite && fileExists(m_pendingDownloadLocation)) + deleteFile(m_pendingDownloadLocation); +} + +String NetworkDataTaskBlob::suggestedFilename() const +{ + if (!m_suggestedFilename.isEmpty()) + return m_suggestedFilename; + + return ASCIILiteral("unknown"); +} + +void NetworkDataTaskBlob::download() +{ + ASSERT(isDownload()); + ASSERT(m_pendingDownloadLocation); + + LOG(NetworkSession, "%p - NetworkDataTaskBlob::download to %s", this, m_pendingDownloadLocation.utf8().data()); + + m_downloadFile = openFile(m_pendingDownloadLocation, OpenForWrite); + if (m_downloadFile == invalidPlatformFileHandle) { + didFailDownload(cancelledError(m_firstRequest)); + return; + } + + auto& downloadManager = NetworkProcess::singleton().downloadManager(); + auto download = std::make_unique<Download>(downloadManager, m_pendingDownloadID, *this, m_session->sessionID(), suggestedFilename()); + auto* downloadPtr = download.get(); + downloadManager.dataTaskBecameDownloadTask(m_pendingDownloadID, WTFMove(download)); + downloadPtr->didCreateDestination(m_pendingDownloadLocation); + + ASSERT(!m_client); + + m_buffer.resize(bufferSize); + read(); +} + +bool NetworkDataTaskBlob::writeDownload(const char* data, int bytesRead) +{ + ASSERT(isDownload()); + int bytesWritten = writeToFile(m_downloadFile, data, bytesRead); + if (bytesWritten == -1) { + didFailDownload(cancelledError(m_firstRequest)); + return false; + } + + ASSERT(bytesWritten == bytesRead); + auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID); + ASSERT(download); + download->didReceiveData(bytesWritten); + return true; +} + +void NetworkDataTaskBlob::cleanDownloadFiles() +{ + if (m_downloadFile != invalidPlatformFileHandle) { + closeFile(m_downloadFile); + m_downloadFile = invalidPlatformFileHandle; + } + deleteFile(m_pendingDownloadLocation); +} + +void NetworkDataTaskBlob::didFailDownload(const ResourceError& error) +{ + LOG(NetworkSession, "%p - NetworkDataTaskBlob::didFailDownload", this); + + clearStream(); + cleanDownloadFiles(); + + if (m_sandboxExtension) { + m_sandboxExtension->revoke(); + m_sandboxExtension = nullptr; + } + + if (m_client) + m_client->didCompleteWithError(error); + else { + auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID); + ASSERT(download); + download->didFail(error, IPC::DataReference()); + } +} + +void NetworkDataTaskBlob::didFinishDownload() +{ + LOG(NetworkSession, "%p - NetworkDataTaskBlob::didFinishDownload", this); + + ASSERT(isDownload()); + closeFile(m_downloadFile); + m_downloadFile = invalidPlatformFileHandle; + + if (m_sandboxExtension) { + m_sandboxExtension->revoke(); + m_sandboxExtension = nullptr; + } + + clearStream(); + auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID); + ASSERT(download); + download->didFinish(); +} + +void NetworkDataTaskBlob::didFail(Error errorCode) +{ + ASSERT(!m_sandboxExtension); + + Ref<NetworkDataTaskBlob> protectedThis(*this); + if (isDownload()) { + didFailDownload(ResourceError(webKitBlobResourceDomain, static_cast<int>(errorCode), m_firstRequest.url(), String())); + return; + } + + LOG(NetworkSession, "%p - NetworkDataTaskBlob::didFail", this); + + clearStream(); + ASSERT(m_client); + m_client->didCompleteWithError(ResourceError(webKitBlobResourceDomain, static_cast<int>(errorCode), m_firstRequest.url(), String())); +} + +void NetworkDataTaskBlob::didFinish() +{ + if (m_downloadFile != invalidPlatformFileHandle) { + didFinishDownload(); + return; + } + + ASSERT(!m_sandboxExtension); + + LOG(NetworkSession, "%p - NetworkDataTaskBlob::didFinish", this); + + clearStream(); + ASSERT(m_client); + m_client->didCompleteWithError({ }); +} + +} // namespace WebKit + +#endif // USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/NetworkDataTaskBlob.h b/Source/WebKit2/NetworkProcess/NetworkDataTaskBlob.h new file mode 100644 index 000000000..519359449 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkDataTaskBlob.h @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2010 Google Inc. All rights reserved. + * Copyright (C) 2016 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(NETWORK_SESSION) + +#include "NetworkDataTask.h" +#include <WebCore/FileStreamClient.h> +#include <WebCore/FileSystem.h> + +namespace WebCore { +class AsyncFileStream; +class BlobDataFileReference; +class BlobData; +class BlobDataItem; +} + +namespace WebKit { + +class NetworkDataTaskBlob final : public NetworkDataTask, public WebCore::FileStreamClient { +public: + static Ref<NetworkDataTask> create(NetworkSession& session, NetworkDataTaskClient& client, const WebCore::ResourceRequest& request, WebCore::ContentSniffingPolicy shouldContentSniff, const Vector<RefPtr<WebCore::BlobDataFileReference>>& fileReferences) + { + return adoptRef(*new NetworkDataTaskBlob(session, client, request, shouldContentSniff, fileReferences)); + } + + ~NetworkDataTaskBlob(); + +private: + NetworkDataTaskBlob(NetworkSession&, NetworkDataTaskClient&, const WebCore::ResourceRequest&, WebCore::ContentSniffingPolicy, const Vector<RefPtr<WebCore::BlobDataFileReference>>&); + + void suspend() override; + void cancel() override; + void resume() override; + void invalidateAndCancel() override; + NetworkDataTask::State state() const override { return m_state; } + + void setPendingDownloadLocation(const String&, const SandboxExtension::Handle&, bool /*allowOverwrite*/) override; + String suggestedFilename() const override; + + // FileStreamClient methods. + void didGetSize(long long) override; + void didOpen(bool) override; + void didRead(int) override; + + enum class Error { + NoError = 0, + NotFoundError = 1, + SecurityError = 2, + RangeError = 3, + NotReadableError = 4, + MethodNotAllowed = 5 + }; + + void clearStream(); + void getSizeForNext(); + void dispatchDidReceiveResponse(Error = Error::NoError); + void seek(); + void consumeData(const char* data, int bytesRead); + void read(); + void readData(const WebCore::BlobDataItem&); + void readFile(const WebCore::BlobDataItem&); + void download(); + bool writeDownload(const char* data, int bytesRead); + void cleanDownloadFiles(); + void didFailDownload(const WebCore::ResourceError&); + void didFinishDownload(); + void didFail(Error); + void didFinish(); + + enum { kPositionNotSpecified = -1 }; + + RefPtr<WebCore::BlobData> m_blobData; + std::unique_ptr<WebCore::AsyncFileStream> m_stream; // For asynchronous loading. + Vector<char> m_buffer; + Vector<long long> m_itemLengthList; + State m_state { State::Suspended }; + long long m_rangeOffset { kPositionNotSpecified }; + long long m_rangeEnd { kPositionNotSpecified }; + long long m_rangeSuffixLength { kPositionNotSpecified }; + long long m_totalSize { 0 }; + long long m_totalRemainingSize { 0 }; + long long m_currentItemReadSize { 0 }; + unsigned m_sizeItemCount { 0 }; + unsigned m_readItemCount { 0 }; + bool m_fileOpened { false }; + WebCore::PlatformFileHandle m_downloadFile { WebCore::invalidPlatformFileHandle }; + + Vector<RefPtr<WebCore::BlobDataFileReference>> m_fileReferences; + RefPtr<SandboxExtension> m_sandboxExtension; +}; + +} // namespace WebKit + +#endif // USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/NetworkLoad.cpp b/Source/WebKit2/NetworkProcess/NetworkLoad.cpp new file mode 100644 index 000000000..1c3e2813d --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkLoad.cpp @@ -0,0 +1,585 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkLoad.h" + +#include "AuthenticationManager.h" +#include "DownloadProxyMessages.h" +#include "NetworkProcess.h" +#include "SessionTracker.h" +#include "WebCoreArgumentCoders.h" +#include "WebErrors.h" +#include <WebCore/NotImplemented.h> +#include <WebCore/ResourceHandle.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/SessionID.h> +#include <WebCore/SharedBuffer.h> +#include <wtf/MainThread.h> + +#if PLATFORM(COCOA) +#include "NetworkDataTaskCocoa.h" +#endif + +#if ENABLE(NETWORK_CAPTURE) +#include "NetworkCaptureManager.h" +#endif + +namespace WebKit { + +using namespace WebCore; + +#if USE(NETWORK_SESSION) + +struct NetworkLoad::Throttle { + Throttle(NetworkLoad& load, std::chrono::milliseconds delay, ResourceResponse&& response, ResponseCompletionHandler&& handler) + : timer(load, &NetworkLoad::throttleDelayCompleted) + , response(WTFMove(response)) + , responseCompletionHandler(WTFMove(handler)) + { + timer.startOneShot(delay); + } + Timer timer; + ResourceResponse response; + ResponseCompletionHandler responseCompletionHandler; +}; + +NetworkLoad::NetworkLoad(NetworkLoadClient& client, NetworkLoadParameters&& parameters, NetworkSession& networkSession) + : m_client(client) + , m_parameters(WTFMove(parameters)) + , m_currentRequest(m_parameters.request) +{ +#if ENABLE(NETWORK_CAPTURE) + switch (NetworkCapture::Manager::singleton().mode()) { + case NetworkCapture::Manager::RecordReplayMode::Record: + initializeForRecord(networkSession); + break; + case NetworkCapture::Manager::RecordReplayMode::Replay: + initializeForReplay(networkSession); + break; + case NetworkCapture::Manager::RecordReplayMode::Disabled: + initialize(networkSession); + break; + } +#else + initialize(networkSession); +#endif +} + +#if ENABLE(NETWORK_CAPTURE) +void NetworkLoad::initializeForRecord(NetworkSession& networkSession) +{ + m_recorder = std::make_unique<NetworkCapture::Recorder>(); + m_task = NetworkDataTask::create(networkSession, *this, m_parameters); + if (!m_parameters.defersLoading) { + m_task->resume(); + m_recorder->recordRequestSent(m_parameters.request); + } +} + +void NetworkLoad::initializeForReplay(NetworkSession& networkSession) +{ + m_replayer = std::make_unique<NetworkCapture::Replayer>(); + m_task = m_replayer->replayResource(networkSession, *this, m_parameters); + if (!m_parameters.defersLoading) + m_task->resume(); +} +#endif + +void NetworkLoad::initialize(NetworkSession& networkSession) +{ + m_task = NetworkDataTask::create(networkSession, *this, m_parameters); + if (!m_parameters.defersLoading) + m_task->resume(); +} + +#else + +NetworkLoad::NetworkLoad(NetworkLoadClient& client, NetworkLoadParameters&& parameters) + : m_client(client) + , m_parameters(WTFMove(parameters)) + , m_networkingContext(RemoteNetworkingContext::create(m_parameters.sessionID, m_parameters.shouldClearReferrerOnHTTPSToHTTPRedirect)) + , m_currentRequest(m_parameters.request) +{ + m_handle = ResourceHandle::create(m_networkingContext.get(), m_parameters.request, this, m_parameters.defersLoading, m_parameters.contentSniffingPolicy == SniffContent); +} + +#endif + +NetworkLoad::~NetworkLoad() +{ + ASSERT(RunLoop::isMain()); +#if USE(NETWORK_SESSION) + if (m_responseCompletionHandler) + m_responseCompletionHandler(PolicyIgnore); +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + if (m_challengeCompletionHandler) + m_challengeCompletionHandler(AuthenticationChallengeDisposition::Cancel, { }); +#endif + if (m_task) + m_task->clearClient(); +#else +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + if (m_handle && m_waitingForContinueCanAuthenticateAgainstProtectionSpace) + m_handle->continueCanAuthenticateAgainstProtectionSpace(false); +#endif + if (m_handle) + m_handle->clearClient(); +#endif +} + +void NetworkLoad::setDefersLoading(bool defers) +{ +#if USE(NETWORK_SESSION) + if (m_task) { + if (defers) + m_task->suspend(); + else { + m_task->resume(); +#if ENABLE(NETWORK_CAPTURE) + if (m_recorder) + m_recorder->recordRequestSent(m_parameters.request); +#endif + } + } +#else + if (m_handle) + m_handle->setDefersLoading(defers); +#endif +} + +void NetworkLoad::cancel() +{ +#if USE(NETWORK_SESSION) + if (m_task) + m_task->cancel(); +#else + if (m_handle) + m_handle->cancel(); +#endif +} + +void NetworkLoad::continueWillSendRequest(WebCore::ResourceRequest&& newRequest) +{ +#if PLATFORM(COCOA) + m_currentRequest.updateFromDelegatePreservingOldProperties(newRequest.nsURLRequest(DoNotUpdateHTTPBody)); +#elif USE(SOUP) + // FIXME: Implement ResourceRequest::updateFromDelegatePreservingOldProperties. See https://bugs.webkit.org/show_bug.cgi?id=126127. + m_currentRequest.updateFromDelegatePreservingOldProperties(newRequest); +#endif + +#if ENABLE(NETWORK_CAPTURE) + if (m_recorder) + m_recorder->recordRedirectSent(newRequest); +#endif + +#if USE(NETWORK_SESSION) + auto redirectCompletionHandler = std::exchange(m_redirectCompletionHandler, nullptr); + ASSERT(redirectCompletionHandler); + if (m_currentRequest.isNull()) { + didCompleteWithError(cancelledError(m_currentRequest)); + if (redirectCompletionHandler) + redirectCompletionHandler({ }); + return; + } + + if (redirectCompletionHandler) + redirectCompletionHandler(m_currentRequest); +#else + if (m_currentRequest.isNull()) { + if (m_handle) + m_handle->cancel(); + didFail(m_handle.get(), cancelledError(m_currentRequest)); + } else if (m_handle) { + auto currentRequestCopy = m_currentRequest; + m_handle->continueWillSendRequest(WTFMove(currentRequestCopy)); + } +#endif +} + +void NetworkLoad::continueDidReceiveResponse() +{ +#if USE(NETWORK_SESSION) + if (m_responseCompletionHandler) { + auto responseCompletionHandler = std::exchange(m_responseCompletionHandler, nullptr); + responseCompletionHandler(PolicyUse); + } +#else + if (m_handle) + m_handle->continueDidReceiveResponse(); +#endif +} + +NetworkLoadClient::ShouldContinueDidReceiveResponse NetworkLoad::sharedDidReceiveResponse(ResourceResponse&& response) +{ + response.setSource(ResourceResponse::Source::Network); + if (m_parameters.needsCertificateInfo) + response.includeCertificateInfo(); + + return m_client.get().didReceiveResponse(WTFMove(response)); +} + +void NetworkLoad::sharedWillSendRedirectedRequest(ResourceRequest&& request, ResourceResponse&& redirectResponse) +{ + // We only expect to get the willSendRequest callback from ResourceHandle as the result of a redirect. + ASSERT(!redirectResponse.isNull()); + ASSERT(RunLoop::isMain()); + +#if ENABLE(NETWORK_CAPTURE) + if (m_recorder) + m_recorder->recordRedirectReceived(request, redirectResponse); +#endif + + auto oldRequest = WTFMove(m_currentRequest); + m_currentRequest = request; + m_client.get().willSendRedirectedRequest(WTFMove(oldRequest), WTFMove(request), WTFMove(redirectResponse)); +} + +#if USE(NETWORK_SESSION) + +void NetworkLoad::convertTaskToDownload(PendingDownload& pendingDownload, const ResourceRequest& updatedRequest, const ResourceResponse& response) +{ + if (!m_task) + return; + + m_client = pendingDownload; + m_currentRequest = updatedRequest; + m_task->setPendingDownload(pendingDownload); + + if (m_responseCompletionHandler) + NetworkProcess::singleton().findPendingDownloadLocation(*m_task.get(), std::exchange(m_responseCompletionHandler, nullptr), response); +} + +void NetworkLoad::setPendingDownloadID(DownloadID downloadID) +{ + if (!m_task) + return; + + m_task->setPendingDownloadID(downloadID); +} + +void NetworkLoad::setSuggestedFilename(const String& suggestedName) +{ + if (!m_task) + return; + + m_task->setSuggestedFilename(suggestedName); +} + +void NetworkLoad::setPendingDownload(PendingDownload& pendingDownload) +{ + if (!m_task) + return; + + m_task->setPendingDownload(pendingDownload); +} + +void NetworkLoad::willPerformHTTPRedirection(ResourceResponse&& response, ResourceRequest&& request, RedirectCompletionHandler&& completionHandler) +{ + ASSERT(!m_redirectCompletionHandler); + m_redirectCompletionHandler = WTFMove(completionHandler); + sharedWillSendRedirectedRequest(WTFMove(request), WTFMove(response)); +} + +void NetworkLoad::didReceiveChallenge(const AuthenticationChallenge& challenge, ChallengeCompletionHandler&& completionHandler) +{ + // Handle server trust evaluation at platform-level if requested, for performance reasons. +#if PLATFORM(COCOA) + if (challenge.protectionSpace().authenticationScheme() == ProtectionSpaceAuthenticationSchemeServerTrustEvaluationRequested + && !NetworkProcess::singleton().canHandleHTTPSServerTrustEvaluation()) { + if (m_task && m_task->allowsSpecificHTTPSCertificateForHost(challenge)) + completionHandler(AuthenticationChallengeDisposition::UseCredential, serverTrustCredential(challenge)); + else + completionHandler(AuthenticationChallengeDisposition::RejectProtectionSpace, { }); + return; + } +#endif + + m_challenge = challenge; +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + m_challengeCompletionHandler = WTFMove(completionHandler); + m_client.get().canAuthenticateAgainstProtectionSpaceAsync(challenge.protectionSpace()); +#else + completeAuthenticationChallenge(WTFMove(completionHandler)); +#endif +} + +void NetworkLoad::completeAuthenticationChallenge(ChallengeCompletionHandler&& completionHandler) +{ + if (m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials) { + completionHandler(AuthenticationChallengeDisposition::UseCredential, { }); + return; + } + + if (!m_task) + return; + + if (auto* pendingDownload = m_task->pendingDownload()) + NetworkProcess::singleton().authenticationManager().didReceiveAuthenticationChallenge(*pendingDownload, *m_challenge, WTFMove(completionHandler)); + else + NetworkProcess::singleton().authenticationManager().didReceiveAuthenticationChallenge(m_parameters.webPageID, m_parameters.webFrameID, *m_challenge, WTFMove(completionHandler)); +} + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void NetworkLoad::continueCanAuthenticateAgainstProtectionSpace(bool result) +{ + ASSERT(m_challengeCompletionHandler); + auto completionHandler = std::exchange(m_challengeCompletionHandler, nullptr); + if (!result) { + if (m_task && m_task->allowsSpecificHTTPSCertificateForHost(*m_challenge)) + completionHandler(AuthenticationChallengeDisposition::UseCredential, serverTrustCredential(*m_challenge)); + else + completionHandler(AuthenticationChallengeDisposition::RejectProtectionSpace, { }); + return; + } + + completeAuthenticationChallenge(WTFMove(completionHandler)); +} +#endif + +void NetworkLoad::didReceiveResponseNetworkSession(ResourceResponse&& response, ResponseCompletionHandler&& completionHandler) +{ + ASSERT(isMainThread()); + ASSERT(!m_throttle); + + if (m_task && m_task->isDownload()) { + NetworkProcess::singleton().findPendingDownloadLocation(*m_task.get(), WTFMove(completionHandler), response); + return; + } + + auto delay = NetworkProcess::singleton().loadThrottleLatency(); + if (delay > 0ms) { + m_throttle = std::make_unique<Throttle>(*this, delay, WTFMove(response), WTFMove(completionHandler)); + return; + } + + notifyDidReceiveResponse(WTFMove(response), WTFMove(completionHandler)); +} + +void NetworkLoad::notifyDidReceiveResponse(ResourceResponse&& response, ResponseCompletionHandler&& completionHandler) +{ + ASSERT(isMainThread()); + +#if ENABLE(NETWORK_CAPTURE) + if (m_recorder) + m_recorder->recordResponseReceived(response); +#endif + + if (sharedDidReceiveResponse(WTFMove(response)) == NetworkLoadClient::ShouldContinueDidReceiveResponse::No) { + m_responseCompletionHandler = WTFMove(completionHandler); + return; + } + completionHandler(PolicyUse); +} + +void NetworkLoad::didReceiveData(Ref<SharedBuffer>&& buffer) +{ + ASSERT(!m_throttle); + +#if ENABLE(NETWORK_CAPTURE) + if (m_recorder) + m_recorder->recordDataReceived(buffer.get()); +#endif + + // FIXME: This should be the encoded data length, not the decoded data length. + auto size = buffer->size(); + m_client.get().didReceiveBuffer(WTFMove(buffer), size); +} + +void NetworkLoad::didCompleteWithError(const ResourceError& error) +{ + ASSERT(!m_throttle); + +#if ENABLE(NETWORK_CAPTURE) + if (m_recorder) + m_recorder->recordFinish(error); +#endif + + if (error.isNull()) + m_client.get().didFinishLoading(WTF::monotonicallyIncreasingTime()); + else + m_client.get().didFailLoading(error); +} + +void NetworkLoad::throttleDelayCompleted() +{ + ASSERT(m_throttle); + + auto throttle = WTFMove(m_throttle); + + notifyDidReceiveResponse(WTFMove(throttle->response), WTFMove(throttle->responseCompletionHandler)); +} + +void NetworkLoad::didSendData(uint64_t totalBytesSent, uint64_t totalBytesExpectedToSend) +{ + m_client.get().didSendData(totalBytesSent, totalBytesExpectedToSend); +} + +void NetworkLoad::wasBlocked() +{ + m_client.get().didFailLoading(blockedError(m_currentRequest)); +} + +void NetworkLoad::cannotShowURL() +{ + m_client.get().didFailLoading(cannotShowURLError(m_currentRequest)); +} + +#else + +void NetworkLoad::didReceiveResponseAsync(ResourceHandle* handle, ResourceResponse&& receivedResponse) +{ + ASSERT_UNUSED(handle, handle == m_handle); + if (sharedDidReceiveResponse(WTFMove(receivedResponse)) == NetworkLoadClient::ShouldContinueDidReceiveResponse::Yes) + m_handle->continueDidReceiveResponse(); +} + +void NetworkLoad::didReceiveData(ResourceHandle*, const char* /* data */, unsigned /* length */, int /* encodedDataLength */) +{ + // The NetworkProcess should never get a didReceiveData callback. + // We should always be using didReceiveBuffer. + ASSERT_NOT_REACHED(); +} + +void NetworkLoad::didReceiveBuffer(ResourceHandle* handle, Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength) +{ + ASSERT_UNUSED(handle, handle == m_handle); + m_client.get().didReceiveBuffer(WTFMove(buffer), reportedEncodedDataLength); +} + +void NetworkLoad::didFinishLoading(ResourceHandle* handle, double finishTime) +{ + ASSERT_UNUSED(handle, handle == m_handle); + m_client.get().didFinishLoading(finishTime); +} + +void NetworkLoad::didFail(ResourceHandle* handle, const ResourceError& error) +{ + ASSERT_UNUSED(handle, !handle || handle == m_handle); + ASSERT(!error.isNull()); + + m_client.get().didFailLoading(error); +} + +void NetworkLoad::willSendRequestAsync(ResourceHandle* handle, ResourceRequest&& request, ResourceResponse&& redirectResponse) +{ + ASSERT_UNUSED(handle, handle == m_handle); + sharedWillSendRedirectedRequest(WTFMove(request), WTFMove(redirectResponse)); +} + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void NetworkLoad::canAuthenticateAgainstProtectionSpaceAsync(ResourceHandle* handle, const ProtectionSpace& protectionSpace) +{ + ASSERT(RunLoop::isMain()); + ASSERT_UNUSED(handle, handle == m_handle); + + // Handle server trust evaluation at platform-level if requested, for performance reasons. + if (protectionSpace.authenticationScheme() == ProtectionSpaceAuthenticationSchemeServerTrustEvaluationRequested + && !NetworkProcess::singleton().canHandleHTTPSServerTrustEvaluation()) { + continueCanAuthenticateAgainstProtectionSpace(false); + return; + } + + m_waitingForContinueCanAuthenticateAgainstProtectionSpace = true; + m_client.get().canAuthenticateAgainstProtectionSpaceAsync(protectionSpace); +} + +void NetworkLoad::continueCanAuthenticateAgainstProtectionSpace(bool result) +{ + m_waitingForContinueCanAuthenticateAgainstProtectionSpace = false; + if (m_handle) + m_handle->continueCanAuthenticateAgainstProtectionSpace(result); +} +#endif + +#if USE(NETWORK_CFDATA_ARRAY_CALLBACK) +bool NetworkLoad::supportsDataArray() +{ + notImplemented(); + return false; +} + +void NetworkLoad::didReceiveDataArray(ResourceHandle*, CFArrayRef) +{ + ASSERT_NOT_REACHED(); + notImplemented(); +} +#endif + +void NetworkLoad::didSendData(ResourceHandle* handle, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) +{ + ASSERT_UNUSED(handle, handle == m_handle); + + m_client.get().didSendData(bytesSent, totalBytesToBeSent); +} + +void NetworkLoad::wasBlocked(ResourceHandle* handle) +{ + ASSERT_UNUSED(handle, handle == m_handle); + + didFail(handle, WebKit::blockedError(m_currentRequest)); +} + +void NetworkLoad::cannotShowURL(ResourceHandle* handle) +{ + ASSERT_UNUSED(handle, handle == m_handle); + + didFail(handle, WebKit::cannotShowURLError(m_currentRequest)); +} + +bool NetworkLoad::shouldUseCredentialStorage(ResourceHandle* handle) +{ + ASSERT_UNUSED(handle, handle == m_handle || !m_handle); // m_handle will be 0 if called from ResourceHandle::start(). + + // When the WebProcess is handling loading a client is consulted each time this shouldUseCredentialStorage question is asked. + // In NetworkProcess mode we ask the WebProcess client up front once and then reuse the cached answer. + + // We still need this sync version, because ResourceHandle itself uses it internally, even when the delegate uses an async one. + + return m_parameters.allowStoredCredentials == AllowStoredCredentials; +} + +void NetworkLoad::didReceiveAuthenticationChallenge(ResourceHandle* handle, const AuthenticationChallenge& challenge) +{ + ASSERT_UNUSED(handle, handle == m_handle); + + if (m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials) { + challenge.authenticationClient()->receivedRequestToContinueWithoutCredential(challenge); + return; + } + + NetworkProcess::singleton().authenticationManager().didReceiveAuthenticationChallenge(m_parameters.webPageID, m_parameters.webFrameID, challenge); +} + +void NetworkLoad::receivedCancellation(ResourceHandle* handle, const AuthenticationChallenge&) +{ + ASSERT_UNUSED(handle, handle == m_handle); + + m_handle->cancel(); + didFail(m_handle.get(), cancelledError(m_currentRequest)); +} +#endif // USE(NETWORK_SESSION) + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/NetworkLoad.h b/Source/WebKit2/NetworkProcess/NetworkLoad.h new file mode 100644 index 000000000..b252a0c01 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkLoad.h @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkLoad_h +#define NetworkLoad_h + +#include "NetworkLoadClient.h" +#include "NetworkLoadParameters.h" +#include "RemoteNetworkingContext.h" +#include <WebCore/ResourceHandleClient.h> +#include <wtf/Optional.h> + +#if USE(NETWORK_SESSION) +#include "DownloadID.h" +#include "NetworkDataTask.h" +#include <WebCore/AuthenticationChallenge.h> +#endif + +#if ENABLE(NETWORK_CAPTURE) +#include "NetworkCaptureRecorder.h" +#include "NetworkCaptureReplayer.h" +#endif + +namespace WebKit { + +class NetworkLoad final : +#if USE(NETWORK_SESSION) + private NetworkDataTaskClient +#else + private WebCore::ResourceHandleClient +#endif +{ + WTF_MAKE_FAST_ALLOCATED; +public: +#if USE(NETWORK_SESSION) + NetworkLoad(NetworkLoadClient&, NetworkLoadParameters&&, NetworkSession&); +#else + NetworkLoad(NetworkLoadClient&, NetworkLoadParameters&&); +#endif + ~NetworkLoad(); + + void setDefersLoading(bool); + void cancel(); + + const WebCore::ResourceRequest& currentRequest() const { return m_currentRequest; } + void clearCurrentRequest() { m_currentRequest = WebCore::ResourceRequest(); } + + void continueWillSendRequest(WebCore::ResourceRequest&&); + void continueDidReceiveResponse(); + +#if USE(NETWORK_SESSION) + void convertTaskToDownload(PendingDownload&, const WebCore::ResourceRequest&, const WebCore::ResourceResponse&); + void setPendingDownloadID(DownloadID); + void setSuggestedFilename(const String&); + void setPendingDownload(PendingDownload&); + DownloadID pendingDownloadID() { return m_task->pendingDownloadID(); } +#else + WebCore::ResourceHandle* handle() const { return m_handle.get(); } + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void canAuthenticateAgainstProtectionSpaceAsync(WebCore::ResourceHandle*, const WebCore::ProtectionSpace&) override; +#endif +#if USE(NETWORK_CFDATA_ARRAY_CALLBACK) + bool supportsDataArray() override; + void didReceiveDataArray(WebCore::ResourceHandle*, CFArrayRef) override; +#endif +#if PLATFORM(COCOA) +#if USE(CFURLCONNECTION) + void willCacheResponseAsync(WebCore::ResourceHandle*, CFCachedURLResponseRef) override; +#else + void willCacheResponseAsync(WebCore::ResourceHandle*, NSCachedURLResponse *) override; +#endif +#endif +#endif // USE(NETWORK_SESSION) + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void continueCanAuthenticateAgainstProtectionSpace(bool); +#endif + +private: +#if USE(NETWORK_SESSION) +#if ENABLE(NETWORK_CAPTURE) + void initializeForRecord(NetworkSession&); + void initializeForReplay(NetworkSession&); +#endif + void initialize(NetworkSession&); +#endif + + NetworkLoadClient::ShouldContinueDidReceiveResponse sharedDidReceiveResponse(WebCore::ResourceResponse&&); + void sharedWillSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceResponse&&); + +#if !USE(NETWORK_SESSION) + // ResourceHandleClient + void willSendRequestAsync(WebCore::ResourceHandle*, WebCore::ResourceRequest&&, WebCore::ResourceResponse&& redirectResponse) final; + void didSendData(WebCore::ResourceHandle*, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) final; + void didReceiveResponseAsync(WebCore::ResourceHandle*, WebCore::ResourceResponse&&) final; + void didReceiveData(WebCore::ResourceHandle*, const char*, unsigned, int encodedDataLength) final; + void didReceiveBuffer(WebCore::ResourceHandle*, Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) final; + void didFinishLoading(WebCore::ResourceHandle*, double finishTime) final; + void didFail(WebCore::ResourceHandle*, const WebCore::ResourceError&) final; + void wasBlocked(WebCore::ResourceHandle*) final; + void cannotShowURL(WebCore::ResourceHandle*) final; + bool shouldUseCredentialStorage(WebCore::ResourceHandle*) final; + void didReceiveAuthenticationChallenge(WebCore::ResourceHandle*, const WebCore::AuthenticationChallenge&) final; + void receivedCancellation(WebCore::ResourceHandle*, const WebCore::AuthenticationChallenge&) final; + bool usesAsyncCallbacks() final { return true; } + bool loadingSynchronousXHR() final { return m_client.get().isSynchronous(); } +#else + // NetworkDataTaskClient + void willPerformHTTPRedirection(WebCore::ResourceResponse&&, WebCore::ResourceRequest&&, RedirectCompletionHandler&&) final; + void didReceiveChallenge(const WebCore::AuthenticationChallenge&, ChallengeCompletionHandler&&) final; + void didReceiveResponseNetworkSession(WebCore::ResourceResponse&&, ResponseCompletionHandler&&) final; + void didReceiveData(Ref<WebCore::SharedBuffer>&&) final; + void didCompleteWithError(const WebCore::ResourceError&) final; + void didSendData(uint64_t totalBytesSent, uint64_t totalBytesExpectedToSend) final; + void wasBlocked() final; + void cannotShowURL() final; + + void notifyDidReceiveResponse(WebCore::ResourceResponse&&, ResponseCompletionHandler&&); + void throttleDelayCompleted(); + + void completeAuthenticationChallenge(ChallengeCompletionHandler&&); +#endif + + std::reference_wrapper<NetworkLoadClient> m_client; + const NetworkLoadParameters m_parameters; +#if USE(NETWORK_SESSION) + RefPtr<NetworkDataTask> m_task; + std::optional<WebCore::AuthenticationChallenge> m_challenge; +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + ChallengeCompletionHandler m_challengeCompletionHandler; +#endif + ResponseCompletionHandler m_responseCompletionHandler; + RedirectCompletionHandler m_redirectCompletionHandler; + + struct Throttle; + std::unique_ptr<Throttle> m_throttle; +#else + bool m_waitingForContinueCanAuthenticateAgainstProtectionSpace { false }; + RefPtr<RemoteNetworkingContext> m_networkingContext; + RefPtr<WebCore::ResourceHandle> m_handle; +#endif + + WebCore::ResourceRequest m_currentRequest; // Updated on redirects. + +#if ENABLE(NETWORK_CAPTURE) + std::unique_ptr<NetworkCapture::Recorder> m_recorder; + std::unique_ptr<NetworkCapture::Replayer> m_replayer; +#endif +}; + +} // namespace WebKit + +#endif // NetworkLoad_h diff --git a/Source/WebKit2/NetworkProcess/NetworkLoaderClient.h b/Source/WebKit2/NetworkProcess/NetworkLoadClient.h index e6d6b889c..87a52d2f4 100644 --- a/Source/WebKit2/NetworkProcess/NetworkLoaderClient.h +++ b/Source/WebKit2/NetworkProcess/NetworkLoadClient.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Apple Inc. All rights reserved. + * Copyright (C) 2015 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -23,47 +23,40 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef NetworkLoaderClient_h -#define NetworkLoaderClient_h +#pragma once -#include <wtf/RefCounted.h> +#include <WebCore/ResourceError.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <wtf/Forward.h> -#if ENABLE(NETWORK_PROCESS) +#if PLATFORM(COCOA) +typedef const struct _CFCachedURLResponse* CFCachedURLResponseRef; +#endif namespace WebCore { class ProtectionSpace; -class ResourceError; -class ResourceRequest; -class ResourceResponse; class SharedBuffer; } namespace WebKit { -class NetworkResourceLoader; - -class NetworkLoaderClient { +class NetworkLoadClient { public: - virtual ~NetworkLoaderClient() { } + virtual ~NetworkLoadClient() { } - virtual void willSendRequest(NetworkResourceLoader*, WebCore::ResourceRequest& newRequest, const WebCore::ResourceResponse& redirectResponse) = 0; + virtual bool isSynchronous() const = 0; + + virtual void didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) = 0; #if USE(PROTECTION_SPACE_AUTH_CALLBACK) - virtual void canAuthenticateAgainstProtectionSpace(NetworkResourceLoader*, const WebCore::ProtectionSpace&) = 0; + virtual void canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) = 0; #endif - virtual void didReceiveResponse(NetworkResourceLoader*, const WebCore::ResourceResponse&) = 0; - virtual void didReceiveBuffer(NetworkResourceLoader*, WebCore::SharedBuffer*, int encodedDataLength) = 0; - virtual void didSendData(NetworkResourceLoader*, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) = 0; - virtual void didFinishLoading(NetworkResourceLoader*, double finishTime) = 0; - virtual void didFail(NetworkResourceLoader*, const WebCore::ResourceError&) = 0; - - virtual bool isSynchronous() { return false; } - -protected: - NetworkLoaderClient() { } + virtual void willSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceRequest&& redirectRequest, WebCore::ResourceResponse&& redirectResponse) = 0; + enum class ShouldContinueDidReceiveResponse { No, Yes }; + virtual ShouldContinueDidReceiveResponse didReceiveResponse(WebCore::ResourceResponse&&) = 0; + virtual void didReceiveBuffer(Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) = 0; + virtual void didFinishLoading(double finishTime) = 0; + virtual void didFailLoading(const WebCore::ResourceError&) = 0; }; } // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) - -#endif // NetworkLoaderClient_h diff --git a/Source/WebKit2/NetworkProcess/NetworkLoadParameters.h b/Source/WebKit2/NetworkProcess/NetworkLoadParameters.h new file mode 100644 index 000000000..d064bebec --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkLoadParameters.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkLoadParameters_h +#define NetworkLoadParameters_h + +#include <WebCore/BlobDataFileReference.h> +#include <WebCore/ResourceLoaderOptions.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/SessionID.h> + +namespace WebKit { + +class NetworkLoadParameters { +public: + uint64_t webPageID { 0 }; + uint64_t webFrameID { 0 }; + WebCore::SessionID sessionID { WebCore::SessionID::emptySessionID() }; + WebCore::ResourceRequest request; + WebCore::ContentSniffingPolicy contentSniffingPolicy { WebCore::SniffContent }; + WebCore::StoredCredentials allowStoredCredentials { WebCore::DoNotAllowStoredCredentials }; + WebCore::ClientCredentialPolicy clientCredentialPolicy { WebCore::ClientCredentialPolicy::CannotAskClientForCredentials }; + bool shouldFollowRedirects { true }; + bool shouldClearReferrerOnHTTPSToHTTPRedirect { true }; + bool defersLoading { false }; + bool needsCertificateInfo { false }; +#if USE(NETWORK_SESSION) + Vector<RefPtr<WebCore::BlobDataFileReference>> blobFileReferences; +#endif +}; + +} // namespace WebKit + +#endif // NetworkLoadParameters_h diff --git a/Source/WebKit2/NetworkProcess/NetworkProcess.cpp b/Source/WebKit2/NetworkProcess/NetworkProcess.cpp index 9c1c1cc1e..a685bdb77 100644 --- a/Source/WebKit2/NetworkProcess/NetworkProcess.cpp +++ b/Source/WebKit2/NetworkProcess/NetworkProcess.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 Apple Inc. All rights reserved. + * Copyright (C) 2012-2015 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,25 +26,43 @@ #include "config.h" #include "NetworkProcess.h" -#if ENABLE(NETWORK_PROCESS) - #include "ArgumentCoders.h" #include "Attachment.h" #include "AuthenticationManager.h" +#include "ChildProcessMessages.h" #include "CustomProtocolManager.h" +#include "DataReference.h" +#include "DownloadProxyMessages.h" #include "Logging.h" #include "NetworkConnectionToWebProcess.h" #include "NetworkProcessCreationParameters.h" #include "NetworkProcessPlatformStrategies.h" #include "NetworkProcessProxyMessages.h" #include "NetworkResourceLoader.h" +#include "NetworkSession.h" #include "RemoteNetworkingContext.h" #include "SessionTracker.h" #include "StatisticsData.h" -#include "WebContextMessages.h" #include "WebCookieManager.h" -#include <WebCore/Logging.h> +#include "WebCoreArgumentCoders.h" +#include "WebPageProxyMessages.h" +#include "WebProcessPoolMessages.h" +#include "WebsiteData.h" +#include "WebsiteDataFetchOption.h" +#include "WebsiteDataType.h" +#include <WebCore/DNS.h> +#include <WebCore/DiagnosticLoggingClient.h> +#include <WebCore/LogInitialization.h> +#include <WebCore/MIMETypeRegistry.h> +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/PlatformCookieJar.h> #include <WebCore/ResourceRequest.h> +#include <WebCore/RuntimeApplicationChecks.h> +#include <WebCore/SecurityOriginData.h> +#include <WebCore/SecurityOriginHash.h> +#include <WebCore/SessionID.h> +#include <WebCore/URLParser.h> +#include <wtf/OptionSet.h> #include <wtf/RunLoop.h> #include <wtf/text/CString.h> @@ -52,11 +70,24 @@ #include "SecItemShim.h" #endif +#if ENABLE(NETWORK_CACHE) +#include "NetworkCache.h" +#include "NetworkCacheCoders.h" +#endif + +#if ENABLE(NETWORK_CAPTURE) +#include "NetworkCaptureManager.h" +#endif + +#if PLATFORM(COCOA) +#include "NetworkSessionCocoa.h" +#endif + using namespace WebCore; namespace WebKit { -NetworkProcess& NetworkProcess::shared() +NetworkProcess& NetworkProcess::singleton() { static NeverDestroyed<NetworkProcess> networkProcess; return networkProcess; @@ -65,16 +96,22 @@ NetworkProcess& NetworkProcess::shared() NetworkProcess::NetworkProcess() : m_hasSetCacheModel(false) , m_cacheModel(CacheModelDocumentViewer) -#if PLATFORM(MAC) + , m_diskCacheIsDisabledForTesting(false) + , m_canHandleHTTPSServerTrustEvaluation(true) +#if PLATFORM(COCOA) , m_clearCacheDispatchGroup(0) #endif +#if PLATFORM(IOS) + , m_webSQLiteDatabaseTracker(*this) +#endif { NetworkProcessPlatformStrategies::initialize(); addSupplement<AuthenticationManager>(); addSupplement<WebCookieManager>(); -#if ENABLE(CUSTOM_PROTOCOLS) addSupplement<CustomProtocolManager>(); +#if USE(NETWORK_SESSION) && PLATFORM(COCOA) + NetworkSessionCocoa::setCustomProtocolManager(supplement<CustomProtocolManager>()); #endif } @@ -89,7 +126,7 @@ AuthenticationManager& NetworkProcess::authenticationManager() DownloadManager& NetworkProcess::downloadManager() { - static NeverDestroyed<DownloadManager> downloadManager(this); + static NeverDestroyed<DownloadManager> downloadManager(*this); return downloadManager; } @@ -107,28 +144,31 @@ bool NetworkProcess::shouldTerminate() return false; } -void NetworkProcess::didReceiveMessage(IPC::Connection* connection, IPC::MessageDecoder& decoder) +void NetworkProcess::didReceiveMessage(IPC::Connection& connection, IPC::Decoder& decoder) { if (messageReceiverMap().dispatchMessage(connection, decoder)) return; + if (decoder.messageReceiverName() == Messages::ChildProcess::messageReceiverName()) { + ChildProcess::didReceiveMessage(connection, decoder); + return; + } + didReceiveNetworkProcessMessage(connection, decoder); } -void NetworkProcess::didReceiveSyncMessage(IPC::Connection* connection, IPC::MessageDecoder& decoder, std::unique_ptr<IPC::MessageEncoder>& replyEncoder) +void NetworkProcess::didReceiveSyncMessage(IPC::Connection& connection, IPC::Decoder& decoder, std::unique_ptr<IPC::Encoder>& replyEncoder) { - messageReceiverMap().dispatchSyncMessage(connection, decoder, replyEncoder); -} + if (messageReceiverMap().dispatchSyncMessage(connection, decoder, replyEncoder)) + return; -void NetworkProcess::didClose(IPC::Connection*) -{ - // The UIProcess just exited. - RunLoop::current()->stop(); + didReceiveSyncNetworkProcessMessage(connection, decoder, replyEncoder); } -void NetworkProcess::didReceiveInvalidMessage(IPC::Connection*, IPC::StringReference, IPC::StringReference) +void NetworkProcess::didClose(IPC::Connection&) { - RunLoop::current()->stop(); + // The UIProcess just exited. + stopRunLoop(); } void NetworkProcess::didCreateDownload() @@ -151,123 +191,463 @@ AuthenticationManager& NetworkProcess::downloadsAuthenticationManager() return authenticationManager(); } -void NetworkProcess::initializeNetworkProcess(const NetworkProcessCreationParameters& parameters) +void NetworkProcess::lowMemoryHandler(Critical critical) +{ + if (m_suppressMemoryPressureHandler) + return; + + WTF::releaseFastMallocFreeMemory(); +} + +void NetworkProcess::initializeNetworkProcess(NetworkProcessCreationParameters&& parameters) { platformInitializeNetworkProcess(parameters); - setCacheModel(static_cast<uint32_t>(parameters.cacheModel)); + WTF::setCurrentThreadIsUserInitiated(); -#if PLATFORM(MAC) || USE(CFNETWORK) - SessionTracker::setIdentifierBase(parameters.uiProcessBundleIdentifier); + m_suppressMemoryPressureHandler = parameters.shouldSuppressMemoryPressureHandler; + m_loadThrottleLatency = parameters.loadThrottleLatency; + if (!m_suppressMemoryPressureHandler) { + auto& memoryPressureHandler = MemoryPressureHandler::singleton(); +#if OS(LINUX) + if (parameters.memoryPressureMonitorHandle.fileDescriptor() != -1) + memoryPressureHandler.setMemoryPressureMonitorHandle(parameters.memoryPressureMonitorHandle.releaseFileDescriptor()); #endif + memoryPressureHandler.setLowMemoryHandler([this] (Critical critical, Synchronous) { + lowMemoryHandler(critical); + }); + memoryPressureHandler.install(); + } + +#if ENABLE(NETWORK_CAPTURE) + NetworkCapture::Manager::singleton().initialize( + parameters.recordReplayMode, + parameters.recordReplayCacheLocation); +#endif + + m_diskCacheIsDisabledForTesting = parameters.shouldUseTestingNetworkSession; + + m_diskCacheSizeOverride = parameters.diskCacheSizeOverride; + setCacheModel(static_cast<uint32_t>(parameters.cacheModel)); + + setCanHandleHTTPSServerTrustEvaluation(parameters.canHandleHTTPSServerTrustEvaluation); // FIXME: instead of handling this here, a message should be sent later (scales to multiple sessions) if (parameters.privateBrowsingEnabled) - RemoteNetworkingContext::ensurePrivateBrowsingSession(SessionTracker::legacyPrivateSessionID); + RemoteNetworkingContext::ensurePrivateBrowsingSession(SessionID::legacyPrivateSessionID()); if (parameters.shouldUseTestingNetworkSession) NetworkStorageSession::switchToNewTestingSession(); - NetworkProcessSupplementMap::const_iterator it = m_supplements.begin(); - NetworkProcessSupplementMap::const_iterator end = m_supplements.end(); - for (; it != end; ++it) - it->value->initialize(parameters); + for (auto& supplement : m_supplements.values()) + supplement->initialize(parameters); } void NetworkProcess::initializeConnection(IPC::Connection* connection) { ChildProcess::initializeConnection(connection); -#if ENABLE(SEC_ITEM_SHIM) - SecItemShim::shared().initializeConnection(connection); -#endif - - NetworkProcessSupplementMap::const_iterator it = m_supplements.begin(); - NetworkProcessSupplementMap::const_iterator end = m_supplements.end(); - for (; it != end; ++it) - it->value->initializeConnection(connection); + for (auto& supplement : m_supplements.values()) + supplement->initializeConnection(connection); } void NetworkProcess::createNetworkConnectionToWebProcess() { -#if PLATFORM(MAC) +#if USE(UNIX_DOMAIN_SOCKETS) + IPC::Connection::SocketPair socketPair = IPC::Connection::createPlatformConnection(); + + auto connection = NetworkConnectionToWebProcess::create(socketPair.server); + m_webProcessConnections.append(WTFMove(connection)); + + IPC::Attachment clientSocket(socketPair.client); + parentProcessConnection()->send(Messages::NetworkProcessProxy::DidCreateNetworkConnectionToWebProcess(clientSocket), 0); +#elif OS(DARWIN) // Create the listening port. mach_port_t listeningPort; mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, &listeningPort); // Create a listening connection. - RefPtr<NetworkConnectionToWebProcess> connection = NetworkConnectionToWebProcess::create(IPC::Connection::Identifier(listeningPort)); - m_webProcessConnections.append(connection.release()); + auto connection = NetworkConnectionToWebProcess::create(IPC::Connection::Identifier(listeningPort)); + m_webProcessConnections.append(WTFMove(connection)); IPC::Attachment clientPort(listeningPort, MACH_MSG_TYPE_MAKE_SEND); parentProcessConnection()->send(Messages::NetworkProcessProxy::DidCreateNetworkConnectionToWebProcess(clientPort), 0); -#elif USE(UNIX_DOMAIN_SOCKETS) - IPC::Connection::SocketPair socketPair = IPC::Connection::createPlatformConnection(); - - RefPtr<NetworkConnectionToWebProcess> connection = NetworkConnectionToWebProcess::create(socketPair.server); - m_webProcessConnections.append(connection.release()); - - IPC::Attachment clientSocket(socketPair.client); - parentProcessConnection()->send(Messages::NetworkProcessProxy::DidCreateNetworkConnectionToWebProcess(clientSocket), 0); #else notImplemented(); #endif } -void NetworkProcess::ensurePrivateBrowsingSession(uint64_t sessionID) +void NetworkProcess::clearCachedCredentials() +{ + NetworkStorageSession::defaultStorageSession().credentialStorage().clearCredentials(); +#if USE(NETWORK_SESSION) + NetworkSession::defaultSession().clearCredentials(); +#endif +} + +void NetworkProcess::ensurePrivateBrowsingSession(SessionID sessionID) { RemoteNetworkingContext::ensurePrivateBrowsingSession(sessionID); } -void NetworkProcess::destroyPrivateBrowsingSession(uint64_t sessionID) +void NetworkProcess::destroyPrivateBrowsingSession(SessionID sessionID) { SessionTracker::destroySession(sessionID); } -void NetworkProcess::downloadRequest(uint64_t downloadID, const ResourceRequest& request) +void NetworkProcess::grantSandboxExtensionsToDatabaseProcessForBlobs(const Vector<String>& filenames, Function<void ()>&& completionHandler) +{ + static uint64_t lastRequestID; + + uint64_t requestID = ++lastRequestID; + m_sandboxExtensionForBlobsCompletionHandlers.set(requestID, WTFMove(completionHandler)); + parentProcessConnection()->send(Messages::NetworkProcessProxy::GrantSandboxExtensionsToDatabaseProcessForBlobs(requestID, filenames), 0); +} + +void NetworkProcess::didGrantSandboxExtensionsToDatabaseProcessForBlobs(uint64_t requestID) +{ + if (auto handler = m_sandboxExtensionForBlobsCompletionHandlers.take(requestID)) + handler(); +} + +static void fetchDiskCacheEntries(SessionID sessionID, OptionSet<WebsiteDataFetchOption> fetchOptions, Function<void (Vector<WebsiteData::Entry>)>&& completionHandler) +{ +#if ENABLE(NETWORK_CACHE) + if (NetworkCache::singleton().isEnabled()) { + HashMap<SecurityOriginData, uint64_t> originsAndSizes; + NetworkCache::singleton().traverse([fetchOptions, completionHandler = WTFMove(completionHandler), originsAndSizes = WTFMove(originsAndSizes)](auto* traversalEntry) mutable { + if (!traversalEntry) { + Vector<WebsiteData::Entry> entries; + + for (auto& originAndSize : originsAndSizes) + entries.append(WebsiteData::Entry { originAndSize.key, WebsiteDataType::DiskCache, originAndSize.value }); + + RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), entries = WTFMove(entries)] { + completionHandler(entries); + }); + + return; + } + + auto url = traversalEntry->entry.response().url(); + auto result = originsAndSizes.add({url.protocol().toString(), url.host(), url.port()}, 0); + + if (fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes)) + result.iterator->value += traversalEntry->entry.sourceStorageRecord().header.size() + traversalEntry->recordInfo.bodySize; + }); + + return; + } +#endif + + RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler)] { + completionHandler({ }); + }); +} + +void NetworkProcess::fetchWebsiteData(SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, uint64_t callbackID) +{ + struct CallbackAggregator final : public RefCounted<CallbackAggregator> { + explicit CallbackAggregator(Function<void (WebsiteData)>&& completionHandler) + : m_completionHandler(WTFMove(completionHandler)) + { + } + + ~CallbackAggregator() + { + ASSERT(RunLoop::isMain()); + + RunLoop::main().dispatch([completionHandler = WTFMove(m_completionHandler), websiteData = WTFMove(m_websiteData)] { + completionHandler(websiteData); + }); + } + + Function<void (WebsiteData)> m_completionHandler; + WebsiteData m_websiteData; + }; + + auto callbackAggregator = adoptRef(*new CallbackAggregator([this, callbackID] (WebsiteData websiteData) { + parentProcessConnection()->send(Messages::NetworkProcessProxy::DidFetchWebsiteData(callbackID, websiteData), 0); + })); + + if (websiteDataTypes.contains(WebsiteDataType::Cookies)) { + if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID)) + getHostnamesWithCookies(*networkStorageSession, callbackAggregator->m_websiteData.hostNamesWithCookies); + } + + if (websiteDataTypes.contains(WebsiteDataType::DiskCache)) { + fetchDiskCacheEntries(sessionID, fetchOptions, [callbackAggregator = WTFMove(callbackAggregator)](auto entries) mutable { + callbackAggregator->m_websiteData.entries.appendVector(entries); + }); + } +} + +void NetworkProcess::deleteWebsiteData(SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, std::chrono::system_clock::time_point modifiedSince, uint64_t callbackID) +{ +#if PLATFORM(COCOA) + if (websiteDataTypes.contains(WebsiteDataType::HSTSCache)) { + if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID)) + clearHSTSCache(*networkStorageSession, modifiedSince); + } +#endif + + if (websiteDataTypes.contains(WebsiteDataType::Cookies)) { + if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID)) + deleteAllCookiesModifiedSince(*networkStorageSession, modifiedSince); + } + + auto completionHandler = [this, callbackID] { + parentProcessConnection()->send(Messages::NetworkProcessProxy::DidDeleteWebsiteData(callbackID), 0); + }; + + if (websiteDataTypes.contains(WebsiteDataType::DiskCache) && !sessionID.isEphemeral()) { + clearDiskCache(modifiedSince, WTFMove(completionHandler)); + return; + } + + completionHandler(); +} + +static void clearDiskCacheEntries(const Vector<SecurityOriginData>& origins, Function<void ()>&& completionHandler) +{ +#if ENABLE(NETWORK_CACHE) + if (NetworkCache::singleton().isEnabled()) { + HashSet<RefPtr<SecurityOrigin>> originsToDelete; + for (auto& origin : origins) + originsToDelete.add(origin.securityOrigin()); + + Vector<NetworkCache::Key> cacheKeysToDelete; + NetworkCache::singleton().traverse([completionHandler = WTFMove(completionHandler), originsToDelete = WTFMove(originsToDelete), cacheKeysToDelete = WTFMove(cacheKeysToDelete)](auto* traversalEntry) mutable { + if (traversalEntry) { + if (originsToDelete.contains(SecurityOrigin::create(traversalEntry->entry.response().url()))) + cacheKeysToDelete.append(traversalEntry->entry.key()); + return; + } + + for (auto& key : cacheKeysToDelete) + NetworkCache::singleton().remove(key); + + RunLoop::main().dispatch(WTFMove(completionHandler)); + return; + }); + + return; + } +#endif + + RunLoop::main().dispatch(WTFMove(completionHandler)); +} + +void NetworkProcess::deleteWebsiteDataForOrigins(SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, const Vector<SecurityOriginData>& origins, const Vector<String>& cookieHostNames, uint64_t callbackID) +{ + if (websiteDataTypes.contains(WebsiteDataType::Cookies)) { + if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID)) + deleteCookiesForHostnames(*networkStorageSession, cookieHostNames); + } + + auto completionHandler = [this, callbackID] { + parentProcessConnection()->send(Messages::NetworkProcessProxy::DidDeleteWebsiteDataForOrigins(callbackID), 0); + }; + + if (websiteDataTypes.contains(WebsiteDataType::DiskCache) && !sessionID.isEphemeral()) { + clearDiskCacheEntries(origins, WTFMove(completionHandler)); + return; + } + + completionHandler(); +} + +void NetworkProcess::downloadRequest(SessionID sessionID, DownloadID downloadID, const ResourceRequest& request, const String& suggestedFilename) +{ + downloadManager().startDownload(nullptr, sessionID, downloadID, request, suggestedFilename); +} + +void NetworkProcess::resumeDownload(SessionID sessionID, DownloadID downloadID, const IPC::DataReference& resumeData, const String& path, const WebKit::SandboxExtension::Handle& sandboxExtensionHandle) { - downloadManager().startDownload(downloadID, request); + downloadManager().resumeDownload(sessionID, downloadID, resumeData, path, sandboxExtensionHandle); } -void NetworkProcess::cancelDownload(uint64_t downloadID) +void NetworkProcess::cancelDownload(DownloadID downloadID) { downloadManager().cancelDownload(downloadID); } + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void NetworkProcess::canAuthenticateAgainstProtectionSpace(NetworkResourceLoader& loader, const WebCore::ProtectionSpace& protectionSpace) +{ + static uint64_t lastLoaderID = 0; + uint64_t loaderID = ++lastLoaderID; + m_waitingNetworkResourceLoaders.set(lastLoaderID, loader); + parentProcessConnection()->send(Messages::NetworkProcessProxy::CanAuthenticateAgainstProtectionSpace(loaderID, loader.pageID(), loader.frameID(), protectionSpace), 0); +} + +void NetworkProcess::continueCanAuthenticateAgainstProtectionSpace(uint64_t loaderID, bool canAuthenticate) +{ + m_waitingNetworkResourceLoaders.take(loaderID).value()->continueCanAuthenticateAgainstProtectionSpace(canAuthenticate); +} +#endif + +#if USE(NETWORK_SESSION) +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void NetworkProcess::continueCanAuthenticateAgainstProtectionSpaceDownload(DownloadID downloadID, bool canAuthenticate) +{ + downloadManager().continueCanAuthenticateAgainstProtectionSpace(downloadID, canAuthenticate); +} +#endif + +void NetworkProcess::continueWillSendRequest(DownloadID downloadID, WebCore::ResourceRequest&& request) +{ + downloadManager().continueWillSendRequest(downloadID, WTFMove(request)); +} + +void NetworkProcess::pendingDownloadCanceled(DownloadID downloadID) +{ + downloadProxyConnection()->send(Messages::DownloadProxy::DidCancel({ }), downloadID.downloadID()); +} + +void NetworkProcess::findPendingDownloadLocation(NetworkDataTask& networkDataTask, ResponseCompletionHandler&& completionHandler, const ResourceResponse& response) +{ + uint64_t destinationID = networkDataTask.pendingDownloadID().downloadID(); + downloadProxyConnection()->send(Messages::DownloadProxy::DidReceiveResponse(response), destinationID); + + downloadManager().willDecidePendingDownloadDestination(networkDataTask, WTFMove(completionHandler)); + + // As per https://html.spec.whatwg.org/#as-a-download (step 2), the filename from the Content-Disposition header + // should override the suggested filename from the download attribute. + String suggestedFilename = response.isAttachmentWithFilename() ? response.suggestedFilename() : networkDataTask.suggestedFilename(); + suggestedFilename = MIMETypeRegistry::appendFileExtensionIfNecessary(suggestedFilename, response.mimeType()); + + downloadProxyConnection()->send(Messages::DownloadProxy::DecideDestinationWithSuggestedFilenameAsync(networkDataTask.pendingDownloadID(), suggestedFilename), destinationID); +} +#endif + +void NetworkProcess::continueDecidePendingDownloadDestination(DownloadID downloadID, String destination, const SandboxExtension::Handle& sandboxExtensionHandle, bool allowOverwrite) +{ + if (destination.isEmpty()) + downloadManager().cancelDownload(downloadID); + else + downloadManager().continueDecidePendingDownloadDestination(downloadID, destination, sandboxExtensionHandle, allowOverwrite); +} void NetworkProcess::setCacheModel(uint32_t cm) { CacheModel cacheModel = static_cast<CacheModel>(cm); - if (!m_hasSetCacheModel || cacheModel != m_cacheModel) { - m_hasSetCacheModel = true; - m_cacheModel = cacheModel; - platformSetCacheModel(cacheModel); + if (m_hasSetCacheModel && (cacheModel == m_cacheModel)) + return; + + m_hasSetCacheModel = true; + m_cacheModel = cacheModel; + + unsigned urlCacheMemoryCapacity = 0; + uint64_t urlCacheDiskCapacity = 0; + uint64_t diskFreeSize = 0; + if (WebCore::getVolumeFreeSpace(m_diskCacheDirectory, diskFreeSize)) { + // As a fudge factor, use 1000 instead of 1024, in case the reported byte + // count doesn't align exactly to a megabyte boundary. + diskFreeSize /= KB * 1000; + calculateURLCacheSizes(cacheModel, diskFreeSize, urlCacheMemoryCapacity, urlCacheDiskCapacity); } + + if (m_diskCacheSizeOverride >= 0) + urlCacheDiskCapacity = m_diskCacheSizeOverride; + +#if ENABLE(NETWORK_CACHE) + auto& networkCache = NetworkCache::singleton(); + if (networkCache.isEnabled()) { + networkCache.setCapacity(urlCacheDiskCapacity); + return; + } +#endif + + platformSetURLCacheSize(urlCacheMemoryCapacity, urlCacheDiskCapacity); } -void NetworkProcess::getNetworkProcessStatistics(uint64_t callbackID) +void NetworkProcess::setCanHandleHTTPSServerTrustEvaluation(bool value) { - NetworkResourceLoadScheduler& scheduler = NetworkProcess::shared().networkResourceLoadScheduler(); + m_canHandleHTTPSServerTrustEvaluation = value; +} +void NetworkProcess::getNetworkProcessStatistics(uint64_t callbackID) +{ StatisticsData data; - data.statisticsNumbers.set("HostsPendingCount", scheduler.hostsPendingCount()); - data.statisticsNumbers.set("HostsActiveCount", scheduler.hostsActiveCount()); - data.statisticsNumbers.set("LoadsPendingCount", scheduler.loadsPendingCount()); - data.statisticsNumbers.set("LoadsActiveCount", scheduler.loadsActiveCount()); - data.statisticsNumbers.set("DownloadsActiveCount", shared().downloadManager().activeDownloadCount()); - data.statisticsNumbers.set("OutstandingAuthenticationChallengesCount", shared().authenticationManager().outstandingAuthenticationChallengeCount()); + auto& networkProcess = NetworkProcess::singleton(); + data.statisticsNumbers.set("DownloadsActiveCount", networkProcess.downloadManager().activeDownloadCount()); + data.statisticsNumbers.set("OutstandingAuthenticationChallengesCount", networkProcess.authenticationManager().outstandingAuthenticationChallengeCount()); + + parentProcessConnection()->send(Messages::WebProcessPool::DidGetStatistics(data, callbackID), 0); +} + +void NetworkProcess::logDiagnosticMessage(uint64_t webPageID, const String& message, const String& description, ShouldSample shouldSample) +{ + if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample)) + return; + + parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessage(webPageID, message, description, ShouldSample::No), 0); +} + +void NetworkProcess::logDiagnosticMessageWithResult(uint64_t webPageID, const String& message, const String& description, DiagnosticLoggingResultType result, ShouldSample shouldSample) +{ + if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample)) + return; + + parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessageWithResult(webPageID, message, description, result, ShouldSample::No), 0); +} + +void NetworkProcess::logDiagnosticMessageWithValue(uint64_t webPageID, const String& message, const String& description, double value, unsigned significantFigures, ShouldSample shouldSample) +{ + if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample)) + return; - parentProcessConnection()->send(Messages::WebContext::DidGetStatistics(data, callbackID), 0); + parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessageWithValue(webPageID, message, description, value, significantFigures, ShouldSample::No), 0); } void NetworkProcess::terminate() { +#if ENABLE(NETWORK_CAPTURE) + NetworkCapture::Manager::singleton().terminate(); +#endif + platformTerminate(); ChildProcess::terminate(); } -#if !PLATFORM(MAC) +void NetworkProcess::processWillSuspendImminently(bool& handled) +{ + lowMemoryHandler(Critical::Yes); + handled = true; +} + +void NetworkProcess::prepareToSuspend() +{ + RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::prepareToSuspend()", this); + lowMemoryHandler(Critical::Yes); + + RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::prepareToSuspend() Sending ProcessReadyToSuspend IPC message", this); + parentProcessConnection()->send(Messages::NetworkProcessProxy::ProcessReadyToSuspend(), 0); +} + +void NetworkProcess::cancelPrepareToSuspend() +{ + // Although it is tempting to send a NetworkProcessProxy::DidCancelProcessSuspension message from here + // we do not because prepareToSuspend() already replied with a NetworkProcessProxy::ProcessReadyToSuspend + // message. And NetworkProcessProxy expects to receive either a NetworkProcessProxy::ProcessReadyToSuspend- + // or NetworkProcessProxy::DidCancelProcessSuspension- message, but not both. + RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::cancelPrepareToSuspend()", this); +} + +void NetworkProcess::processDidResume() +{ + RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::processDidResume()", this); +} + +void NetworkProcess::prefetchDNS(const String& hostname) +{ + WebCore::prefetchDNS(hostname); +} + +#if !PLATFORM(COCOA) void NetworkProcess::initializeProcess(const ChildProcessInitializationParameters&) { } @@ -282,5 +662,3 @@ void NetworkProcess::initializeSandbox(const ChildProcessInitializationParameter #endif } // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/NetworkProcess.h b/Source/WebKit2/NetworkProcess/NetworkProcess.h index ad5a8c11b..5575eda7d 100644 --- a/Source/WebKit2/NetworkProcess/NetworkProcess.h +++ b/Source/WebKit2/NetworkProcess/NetworkProcess.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, 2013 Apple Inc. All rights reserved. + * Copyright (C) 2012-2017 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -23,27 +23,43 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef NetworkProcess_h -#define NetworkProcess_h - -#if ENABLE(NETWORK_PROCESS) +#pragma once #include "CacheModel.h" #include "ChildProcess.h" #include "DownloadManager.h" #include "MessageReceiverMap.h" -#include "NetworkResourceLoadScheduler.h" +#include <WebCore/DiagnosticLoggingClient.h> +#include <WebCore/MemoryPressureHandler.h> +#include <WebCore/SessionID.h> +#include <memory> #include <wtf/Forward.h> +#include <wtf/Function.h> #include <wtf/NeverDestroyed.h> +#include <wtf/RetainPtr.h> + +#if PLATFORM(IOS) +#include "WebSQLiteDatabaseTracker.h" +#endif namespace WebCore { +class DownloadID; class CertificateInfo; +class NetworkStorageSession; +class ProtectionSpace; +class SecurityOrigin; +class SessionID; +struct SecurityOriginData; +struct SoupNetworkProxySettings; } namespace WebKit { class AuthenticationManager; class NetworkConnectionToWebProcess; class NetworkProcessSupplement; +class NetworkResourceLoader; +enum class WebsiteDataFetchOption; +enum class WebsiteDataType; struct NetworkProcessCreationParameters; class NetworkProcess : public ChildProcess, private DownloadManager::Client { @@ -51,7 +67,7 @@ class NetworkProcess : public ChildProcess, private DownloadManager::Client { friend class NeverDestroyed<NetworkProcess>; friend class NeverDestroyed<DownloadManager>; public: - static NetworkProcess& shared(); + static NetworkProcess& singleton(); template <typename T> T* supplement() @@ -62,15 +78,45 @@ public: template <typename T> void addSupplement() { - m_supplements.add(T::supplementName(), adoptPtr<NetworkProcessSupplement>(new T(this))); + m_supplements.add(T::supplementName(), std::make_unique<T>(this)); } void removeNetworkConnectionToWebProcess(NetworkConnectionToWebProcess*); - NetworkResourceLoadScheduler& networkResourceLoadScheduler() { return m_networkResourceLoadScheduler; } - AuthenticationManager& authenticationManager(); DownloadManager& downloadManager(); + bool canHandleHTTPSServerTrustEvaluation() const { return m_canHandleHTTPSServerTrustEvaluation; } + + void processWillSuspendImminently(bool& handled); + void prepareToSuspend(); + void cancelPrepareToSuspend(); + void processDidResume(); + + // Diagnostic messages logging. + void logDiagnosticMessage(uint64_t webPageID, const String& message, const String& description, WebCore::ShouldSample); + void logDiagnosticMessageWithResult(uint64_t webPageID, const String& message, const String& description, WebCore::DiagnosticLoggingResultType, WebCore::ShouldSample); + void logDiagnosticMessageWithValue(uint64_t webPageID, const String& message, const String& description, double value, unsigned significantFigures, WebCore::ShouldSample); + +#if PLATFORM(COCOA) + RetainPtr<CFDataRef> sourceApplicationAuditData() const; + void clearHSTSCache(WebCore::NetworkStorageSession&, std::chrono::system_clock::time_point modifiedSince); +#endif + +#if USE(NETWORK_SESSION) + void findPendingDownloadLocation(NetworkDataTask&, ResponseCompletionHandler&&, const WebCore::ResourceResponse&); +#endif + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void canAuthenticateAgainstProtectionSpace(NetworkResourceLoader&, const WebCore::ProtectionSpace&); +#endif + + void prefetchDNS(const String&); + + void ensurePrivateBrowsingSession(WebCore::SessionID); + + void grantSandboxExtensionsToDatabaseProcessForBlobs(const Vector<String>& filenames, Function<void ()>&& completionHandler); + + std::chrono::milliseconds loadThrottleLatency() const { return m_loadThrottleLatency; } private: NetworkProcess(); @@ -78,71 +124,114 @@ private: void platformInitializeNetworkProcess(const NetworkProcessCreationParameters&); - virtual void terminate() override; + void terminate() override; void platformTerminate(); + void lowMemoryHandler(WebCore::Critical); + // ChildProcess - virtual void initializeProcess(const ChildProcessInitializationParameters&) override; - virtual void initializeProcessName(const ChildProcessInitializationParameters&) override; - virtual void initializeSandbox(const ChildProcessInitializationParameters&, SandboxInitializationParameters&) override; - virtual void initializeConnection(IPC::Connection*) override; - virtual bool shouldTerminate() override; + void initializeProcess(const ChildProcessInitializationParameters&) override; + void initializeProcessName(const ChildProcessInitializationParameters&) override; + void initializeSandbox(const ChildProcessInitializationParameters&, SandboxInitializationParameters&) override; + void initializeConnection(IPC::Connection*) override; + bool shouldTerminate() override; // IPC::Connection::Client - virtual void didReceiveMessage(IPC::Connection*, IPC::MessageDecoder&) override; - virtual void didReceiveSyncMessage(IPC::Connection*, IPC::MessageDecoder&, std::unique_ptr<IPC::MessageEncoder>&); - virtual void didClose(IPC::Connection*) override; - virtual void didReceiveInvalidMessage(IPC::Connection*, IPC::StringReference messageReceiverName, IPC::StringReference messageName) override; + void didReceiveMessage(IPC::Connection&, IPC::Decoder&) override; + void didReceiveSyncMessage(IPC::Connection&, IPC::Decoder&, std::unique_ptr<IPC::Encoder>&) override; + void didClose(IPC::Connection&) override; // DownloadManager::Client - virtual void didCreateDownload() override; - virtual void didDestroyDownload() override; - virtual IPC::Connection* downloadProxyConnection() override; - virtual AuthenticationManager& downloadsAuthenticationManager() override; + void didCreateDownload() override; + void didDestroyDownload() override; + IPC::Connection* downloadProxyConnection() override; + AuthenticationManager& downloadsAuthenticationManager() override; +#if USE(NETWORK_SESSION) + void pendingDownloadCanceled(DownloadID) override; +#endif // Message Handlers - void didReceiveNetworkProcessMessage(IPC::Connection*, IPC::MessageDecoder&); - void initializeNetworkProcess(const NetworkProcessCreationParameters&); + void didReceiveNetworkProcessMessage(IPC::Connection&, IPC::Decoder&); + void didReceiveSyncNetworkProcessMessage(IPC::Connection&, IPC::Decoder&, std::unique_ptr<IPC::Encoder>&); + void initializeNetworkProcess(NetworkProcessCreationParameters&&); void createNetworkConnectionToWebProcess(); - void ensurePrivateBrowsingSession(uint64_t sessionID); - void destroyPrivateBrowsingSession(uint64_t sessionID); - void downloadRequest(uint64_t downloadID, const WebCore::ResourceRequest&); - void cancelDownload(uint64_t downloadID); + void destroyPrivateBrowsingSession(WebCore::SessionID); + + void fetchWebsiteData(WebCore::SessionID, OptionSet<WebsiteDataType>, OptionSet<WebsiteDataFetchOption>, uint64_t callbackID); + void deleteWebsiteData(WebCore::SessionID, OptionSet<WebsiteDataType>, std::chrono::system_clock::time_point modifiedSince, uint64_t callbackID); + void deleteWebsiteDataForOrigins(WebCore::SessionID, OptionSet<WebsiteDataType>, const Vector<WebCore::SecurityOriginData>& origins, const Vector<String>& cookieHostNames, uint64_t callbackID); + + void clearCachedCredentials(); + + // FIXME: This should take a session ID so we can identify which disk cache to delete. + void clearDiskCache(std::chrono::system_clock::time_point modifiedSince, std::function<void ()> completionHandler); + + void downloadRequest(WebCore::SessionID, DownloadID, const WebCore::ResourceRequest&, const String& suggestedFilename); + void resumeDownload(WebCore::SessionID, DownloadID, const IPC::DataReference& resumeData, const String& path, const SandboxExtension::Handle&); + void cancelDownload(DownloadID); +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void continueCanAuthenticateAgainstProtectionSpace(uint64_t resourceLoadIdentifier, bool canAuthenticate); +#endif +#if USE(NETWORK_SESSION) +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void continueCanAuthenticateAgainstProtectionSpaceDownload(DownloadID, bool canAuthenticate); +#endif + void continueWillSendRequest(DownloadID, WebCore::ResourceRequest&&); +#endif + void continueDecidePendingDownloadDestination(DownloadID, String destination, const SandboxExtension::Handle& sandboxExtensionHandle, bool allowOverwrite); + void setCacheModel(uint32_t); void allowSpecificHTTPSCertificateForHost(const WebCore::CertificateInfo&, const String& host); + void setCanHandleHTTPSServerTrustEvaluation(bool); void getNetworkProcessStatistics(uint64_t callbackID); void clearCacheForAllOrigins(uint32_t cachesToClear); + void didGrantSandboxExtensionsToDatabaseProcessForBlobs(uint64_t requestID); + #if USE(SOUP) void setIgnoreTLSErrors(bool); void userPreferredLanguagesChanged(const Vector<String>&); + void setNetworkProxySettings(const WebCore::SoupNetworkProxySettings&); #endif // Platform Helpers - void platformSetCacheModel(CacheModel); + void platformSetURLCacheSize(unsigned urlCacheMemoryCapacity, uint64_t urlCacheDiskCapacity); // Connections to WebProcesses. Vector<RefPtr<NetworkConnectionToWebProcess>> m_webProcessConnections; - NetworkResourceLoadScheduler m_networkResourceLoadScheduler; - String m_diskCacheDirectory; bool m_hasSetCacheModel; CacheModel m_cacheModel; + int64_t m_diskCacheSizeOverride { -1 }; + bool m_suppressMemoryPressureHandler { false }; + bool m_diskCacheIsDisabledForTesting; + bool m_canHandleHTTPSServerTrustEvaluation; + std::chrono::milliseconds m_loadThrottleLatency; - typedef HashMap<const char*, OwnPtr<NetworkProcessSupplement>, PtrHash<const char*>> NetworkProcessSupplementMap; + typedef HashMap<const char*, std::unique_ptr<NetworkProcessSupplement>, PtrHash<const char*>> NetworkProcessSupplementMap; NetworkProcessSupplementMap m_supplements; -#if PLATFORM(MAC) + HashMap<uint64_t, Function<void ()>> m_sandboxExtensionForBlobsCompletionHandlers; + HashMap<uint64_t, Ref<NetworkResourceLoader>> m_waitingNetworkResourceLoaders; + +#if ENABLE(WEB_RTC) + bool m_webRTCEnabled { false }; +#endif + +#if PLATFORM(COCOA) + void platformInitializeNetworkProcessCocoa(const NetworkProcessCreationParameters&); + void setCookieStoragePartitioningEnabled(bool); + // FIXME: We'd like to be able to do this without the #ifdef, but WorkQueue + BinarySemaphore isn't good enough since // multiple requests to clear the cache can come in before previous requests complete, and we need to wait for all of them. // In the future using WorkQueue and a counting semaphore would work, as would WorkQueue supporting the libdispatch concept of "work groups". dispatch_group_t m_clearCacheDispatchGroup; #endif + +#if PLATFORM(IOS) + WebSQLiteDatabaseTracker m_webSQLiteDatabaseTracker; +#endif }; } // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) - -#endif // NetworkProcess_h diff --git a/Source/WebKit2/NetworkProcess/NetworkProcess.messages.in b/Source/WebKit2/NetworkProcess/NetworkProcess.messages.in index 566deb401..0a9916903 100644 --- a/Source/WebKit2/NetworkProcess/NetworkProcess.messages.in +++ b/Source/WebKit2/NetworkProcess/NetworkProcess.messages.in @@ -20,11 +20,9 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#if ENABLE(NETWORK_PROCESS) - messages -> NetworkProcess LegacyReceiver { # Initializes the network process. - InitializeNetworkProcess(WebKit::NetworkProcessCreationParameters processCreationParameters) + InitializeNetworkProcess(struct WebKit::NetworkProcessCreationParameters processCreationParameters) # Creates a connection for communication with a WebProcess CreateNetworkConnectionToWebProcess() @@ -32,24 +30,51 @@ messages -> NetworkProcess LegacyReceiver { #if USE(SOUP) SetIgnoreTLSErrors(bool ignoreTLSErrors) UserPreferredLanguagesChanged(Vector<String> languages) + SetNetworkProxySettings(struct WebCore::SoupNetworkProxySettings settings) #endif - EnsurePrivateBrowsingSession(uint64_t sessionID) - DestroyPrivateBrowsingSession(uint64_t sessionID) + ClearCachedCredentials() + + EnsurePrivateBrowsingSession(WebCore::SessionID sessionID) + DestroyPrivateBrowsingSession(WebCore::SessionID sessionID) + + FetchWebsiteData(WebCore::SessionID sessionID, OptionSet<WebKit::WebsiteDataType> websiteDataTypes, OptionSet<WebKit::WebsiteDataFetchOption> fetchOptions, uint64_t callbackID) + DeleteWebsiteData(WebCore::SessionID sessionID, OptionSet<WebKit::WebsiteDataType> websiteDataTypes, std::chrono::system_clock::time_point modifiedSince, uint64_t callbackID) + DeleteWebsiteDataForOrigins(WebCore::SessionID sessionID, OptionSet<WebKit::WebsiteDataType> websiteDataTypes, Vector<WebCore::SecurityOriginData> origins, Vector<String> cookieHostNames, uint64_t callbackID) - DownloadRequest(uint64_t downloadID, WebCore::ResourceRequest request) - CancelDownload(uint64_t downloadID) + DownloadRequest(WebCore::SessionID sessionID, WebKit::DownloadID downloadID, WebCore::ResourceRequest request, String suggestedFilename) + ResumeDownload(WebCore::SessionID sessionID, WebKit::DownloadID downloadID, IPC::DataReference resumeData, String path, WebKit::SandboxExtension::Handle sandboxExtensionHandle) + CancelDownload(WebKit::DownloadID downloadID) + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + ContinueCanAuthenticateAgainstProtectionSpace(uint64_t loaderID, bool canAuthenticate) +#endif +#if USE(NETWORK_SESSION) +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + ContinueCanAuthenticateAgainstProtectionSpaceDownload(WebKit::DownloadID downloadID, bool canAuthenticate) +#endif + ContinueWillSendRequest(WebKit::DownloadID downloadID, WebCore::ResourceRequest request) +#endif + ContinueDecidePendingDownloadDestination(WebKit::DownloadID downloadID, String destination, WebKit::SandboxExtension::Handle sandboxExtensionHandle, bool allowOverwrite) -#if PLATFORM(MAC) SetProcessSuppressionEnabled(bool flag) +#if PLATFORM(COCOA) SetQOS(int latencyQOS, int throughputQOS) + SetCookieStoragePartitioningEnabled(bool enabled) #endif AllowSpecificHTTPSCertificateForHost(WebCore::CertificateInfo certificate, String host) + SetCanHandleHTTPSServerTrustEvaluation(bool value) GetNetworkProcessStatistics(uint64_t callbackID) ClearCacheForAllOrigins(uint32_t cachesToClear) -} + SetCacheModel(uint32_t cacheModel); -#endif // ENABLE(NETWORK_PROCESS) + ProcessWillSuspendImminently() -> (bool handled) + PrepareToSuspend() + CancelPrepareToSuspend() + ProcessDidResume() + + DidGrantSandboxExtensionsToDatabaseProcessForBlobs(uint64_t requestID) +} diff --git a/Source/WebKit2/NetworkProcess/NetworkProcessCreationParameters.cpp b/Source/WebKit2/NetworkProcess/NetworkProcessCreationParameters.cpp new file mode 100644 index 000000000..5dfabd689 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkProcessCreationParameters.cpp @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2012-2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkProcessCreationParameters.h" + +#include "ArgumentCoders.h" + +#if PLATFORM(COCOA) +#include "ArgumentCodersCF.h" +#endif + +#if USE(SOUP) +#include "WebCoreArgumentCoders.h" +#endif + +namespace WebKit { + +NetworkProcessCreationParameters::NetworkProcessCreationParameters() +{ +} + +void NetworkProcessCreationParameters::encode(IPC::Encoder& encoder) const +{ + encoder << privateBrowsingEnabled; + encoder.encodeEnum(cacheModel); + encoder << diskCacheSizeOverride; + encoder << canHandleHTTPSServerTrustEvaluation; + encoder << diskCacheDirectory; + encoder << diskCacheDirectoryExtensionHandle; +#if ENABLE(NETWORK_CACHE) + encoder << shouldEnableNetworkCache; + encoder << shouldEnableNetworkCacheEfficacyLogging; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + encoder << shouldEnableNetworkCacheSpeculativeRevalidation; +#endif +#endif +#if PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100 + encoder << uiProcessCookieStorageIdentifier; +#endif +#if PLATFORM(IOS) + encoder << cookieStorageDirectoryExtensionHandle; + encoder << containerCachesDirectoryExtensionHandle; + encoder << parentBundleDirectoryExtensionHandle; +#endif + encoder << shouldSuppressMemoryPressureHandler; + encoder << shouldUseTestingNetworkSession; + encoder << loadThrottleLatency; + encoder << urlSchemesRegisteredForCustomProtocols; +#if PLATFORM(COCOA) + encoder << parentProcessName; + encoder << uiProcessBundleIdentifier; + encoder << nsURLCacheMemoryCapacity; + encoder << nsURLCacheDiskCapacity; + encoder << sourceApplicationBundleIdentifier; + encoder << sourceApplicationSecondaryIdentifier; +#if PLATFORM(IOS) + encoder << ctDataConnectionServiceType; +#endif + encoder << httpProxy; + encoder << httpsProxy; +#if TARGET_OS_IPHONE || (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100) + IPC::encode(encoder, networkATSContext.get()); +#endif + encoder << cookieStoragePartitioningEnabled; +#endif +#if USE(SOUP) + encoder << cookiePersistentStoragePath; + encoder << cookiePersistentStorageType; + encoder.encodeEnum(cookieAcceptPolicy); + encoder << ignoreTLSErrors; + encoder << languages; + encoder << proxySettings; +#endif +#if OS(LINUX) + encoder << memoryPressureMonitorHandle; +#endif +#if ENABLE(NETWORK_CAPTURE) + encoder << recordReplayMode; + encoder << recordReplayCacheLocation; +#endif +#if ENABLE(WEB_RTC) + encoder << webRTCEnabled; +#endif +} + +bool NetworkProcessCreationParameters::decode(IPC::Decoder& decoder, NetworkProcessCreationParameters& result) +{ + if (!decoder.decode(result.privateBrowsingEnabled)) + return false; + if (!decoder.decodeEnum(result.cacheModel)) + return false; + if (!decoder.decode(result.diskCacheSizeOverride)) + return false; + if (!decoder.decode(result.canHandleHTTPSServerTrustEvaluation)) + return false; + if (!decoder.decode(result.diskCacheDirectory)) + return false; + if (!decoder.decode(result.diskCacheDirectoryExtensionHandle)) + return false; +#if ENABLE(NETWORK_CACHE) + if (!decoder.decode(result.shouldEnableNetworkCache)) + return false; + if (!decoder.decode(result.shouldEnableNetworkCacheEfficacyLogging)) + return false; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (!decoder.decode(result.shouldEnableNetworkCacheSpeculativeRevalidation)) + return false; +#endif +#endif +#if PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100 + if (!decoder.decode(result.uiProcessCookieStorageIdentifier)) + return false; +#endif +#if PLATFORM(IOS) + if (!decoder.decode(result.cookieStorageDirectoryExtensionHandle)) + return false; + if (!decoder.decode(result.containerCachesDirectoryExtensionHandle)) + return false; + if (!decoder.decode(result.parentBundleDirectoryExtensionHandle)) + return false; +#endif + if (!decoder.decode(result.shouldSuppressMemoryPressureHandler)) + return false; + if (!decoder.decode(result.shouldUseTestingNetworkSession)) + return false; + if (!decoder.decode(result.loadThrottleLatency)) + return false; + if (!decoder.decode(result.urlSchemesRegisteredForCustomProtocols)) + return false; +#if PLATFORM(COCOA) + if (!decoder.decode(result.parentProcessName)) + return false; + if (!decoder.decode(result.uiProcessBundleIdentifier)) + return false; + if (!decoder.decode(result.nsURLCacheMemoryCapacity)) + return false; + if (!decoder.decode(result.nsURLCacheDiskCapacity)) + return false; + if (!decoder.decode(result.sourceApplicationBundleIdentifier)) + return false; + if (!decoder.decode(result.sourceApplicationSecondaryIdentifier)) + return false; +#if PLATFORM(IOS) + if (!decoder.decode(result.ctDataConnectionServiceType)) + return false; +#endif + if (!decoder.decode(result.httpProxy)) + return false; + if (!decoder.decode(result.httpsProxy)) + return false; +#if TARGET_OS_IPHONE || (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100) + if (!IPC::decode(decoder, result.networkATSContext)) + return false; +#endif + if (!decoder.decode(result.cookieStoragePartitioningEnabled)) + return false; +#endif + +#if USE(SOUP) + if (!decoder.decode(result.cookiePersistentStoragePath)) + return false; + if (!decoder.decode(result.cookiePersistentStorageType)) + return false; + if (!decoder.decodeEnum(result.cookieAcceptPolicy)) + return false; + if (!decoder.decode(result.ignoreTLSErrors)) + return false; + if (!decoder.decode(result.languages)) + return false; + if (!decoder.decode(result.proxySettings)) + return false; +#endif + +#if OS(LINUX) + if (!decoder.decode(result.memoryPressureMonitorHandle)) + return false; +#endif + +#if ENABLE(NETWORK_CAPTURE) + if (!decoder.decode(result.recordReplayMode)) + return false; + if (!decoder.decode(result.recordReplayCacheLocation)) + return false; +#endif +#if ENABLE(WEB_RTC) + if (!decoder.decode(result.webRTCEnabled)) + return false; +#endif + + return true; +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/NetworkProcessCreationParameters.h b/Source/WebKit2/NetworkProcess/NetworkProcessCreationParameters.h new file mode 100644 index 000000000..2b88e88f4 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkProcessCreationParameters.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2012-2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "Attachment.h" +#include "CacheModel.h" +#include "SandboxExtension.h" +#include <wtf/Vector.h> +#include <wtf/text/WTFString.h> + +#if USE(SOUP) +#include "HTTPCookieAcceptPolicy.h" +#include <WebCore/SoupNetworkProxySettings.h> +#endif + +namespace IPC { +class Decoder; +class Encoder; +} + +namespace WebKit { + +struct NetworkProcessCreationParameters { + NetworkProcessCreationParameters(); + + void encode(IPC::Encoder&) const; + static bool decode(IPC::Decoder&, NetworkProcessCreationParameters&); + + bool privateBrowsingEnabled; + CacheModel cacheModel; + int64_t diskCacheSizeOverride { -1 }; + bool canHandleHTTPSServerTrustEvaluation; + + String diskCacheDirectory; + SandboxExtension::Handle diskCacheDirectoryExtensionHandle; +#if ENABLE(NETWORK_CACHE) + bool shouldEnableNetworkCache; + bool shouldEnableNetworkCacheEfficacyLogging; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + bool shouldEnableNetworkCacheSpeculativeRevalidation; +#endif +#endif +#if PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100 + Vector<uint8_t> uiProcessCookieStorageIdentifier; +#endif +#if PLATFORM(IOS) + SandboxExtension::Handle cookieStorageDirectoryExtensionHandle; + SandboxExtension::Handle containerCachesDirectoryExtensionHandle; + SandboxExtension::Handle parentBundleDirectoryExtensionHandle; +#endif + bool shouldSuppressMemoryPressureHandler { false }; + bool shouldUseTestingNetworkSession; + std::chrono::milliseconds loadThrottleLatency { 0ms }; + + Vector<String> urlSchemesRegisteredForCustomProtocols; + +#if PLATFORM(COCOA) + String parentProcessName; + String uiProcessBundleIdentifier; + uint64_t nsURLCacheMemoryCapacity; + uint64_t nsURLCacheDiskCapacity; + String sourceApplicationBundleIdentifier; + String sourceApplicationSecondaryIdentifier; +#if PLATFORM(IOS) + String ctDataConnectionServiceType; +#endif + String httpProxy; + String httpsProxy; +#if TARGET_OS_IPHONE || (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100) + RetainPtr<CFDataRef> networkATSContext; +#endif + bool cookieStoragePartitioningEnabled; +#endif + +#if USE(SOUP) + String cookiePersistentStoragePath; + uint32_t cookiePersistentStorageType; + HTTPCookieAcceptPolicy cookieAcceptPolicy; + bool ignoreTLSErrors; + Vector<String> languages; + WebCore::SoupNetworkProxySettings proxySettings; +#endif + +#if OS(LINUX) + IPC::Attachment memoryPressureMonitorHandle; +#endif + +#if ENABLE(NETWORK_CAPTURE) + String recordReplayMode; + String recordReplayCacheLocation; +#endif + +#if ENABLE(WEB_RTC) + bool webRTCEnabled { false }; +#endif +}; + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.cpp b/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.cpp index 5ca2a227a..e6d679f8f 100644 --- a/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.cpp +++ b/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.cpp @@ -41,62 +41,22 @@ void NetworkProcessPlatformStrategies::initialize() CookiesStrategy* NetworkProcessPlatformStrategies::createCookiesStrategy() { - return 0; -} - -DatabaseStrategy* NetworkProcessPlatformStrategies::createDatabaseStrategy() -{ - return 0; + return nullptr; } LoaderStrategy* NetworkProcessPlatformStrategies::createLoaderStrategy() { - return this; + return nullptr; } PasteboardStrategy* NetworkProcessPlatformStrategies::createPasteboardStrategy() { - return 0; -} - -PluginStrategy* NetworkProcessPlatformStrategies::createPluginStrategy() -{ - return 0; -} - -SharedWorkerStrategy* NetworkProcessPlatformStrategies::createSharedWorkerStrategy() -{ - return 0; -} - -StorageStrategy* NetworkProcessPlatformStrategies::createStorageStrategy() -{ - return 0; + return nullptr; } -VisitedLinkStrategy* NetworkProcessPlatformStrategies::createVisitedLinkStrategy() -{ - return 0; -} - -ResourceLoadScheduler* NetworkProcessPlatformStrategies::resourceLoadScheduler() -{ - ASSERT_NOT_REACHED(); - return 0; -} - -void NetworkProcessPlatformStrategies::loadResourceSynchronously(NetworkingContext*, unsigned long, const ResourceRequest&, StoredCredentials, ClientCredentialPolicy, ResourceError&, ResourceResponse&, Vector<char>&) -{ - ASSERT_NOT_REACHED(); -} - -#if ENABLE(BLOB) BlobRegistry* NetworkProcessPlatformStrategies::createBlobRegistry() { return new BlobRegistryImpl; } -#endif - - } diff --git a/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.h b/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.h index 9dfc8f3af..3a04c9c59 100644 --- a/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.h +++ b/Source/WebKit2/NetworkProcess/NetworkProcessPlatformStrategies.h @@ -31,27 +31,16 @@ namespace WebKit { -class NetworkProcessPlatformStrategies : public WebCore::PlatformStrategies, private WebCore::LoaderStrategy { +class NetworkProcessPlatformStrategies : public WebCore::PlatformStrategies { public: static void initialize(); private: // WebCore::PlatformStrategies - virtual WebCore::CookiesStrategy* createCookiesStrategy() override; - virtual WebCore::DatabaseStrategy* createDatabaseStrategy() override; - virtual WebCore::LoaderStrategy* createLoaderStrategy() override; - virtual WebCore::PasteboardStrategy* createPasteboardStrategy() override; - virtual WebCore::PluginStrategy* createPluginStrategy() override; - virtual WebCore::SharedWorkerStrategy* createSharedWorkerStrategy() override; - virtual WebCore::StorageStrategy* createStorageStrategy() override; - virtual WebCore::VisitedLinkStrategy* createVisitedLinkStrategy() override; - - // WebCore::LoaderStrategy - virtual WebCore::ResourceLoadScheduler* resourceLoadScheduler() override; - virtual void loadResourceSynchronously(WebCore::NetworkingContext*, unsigned long resourceLoadIdentifier, const WebCore::ResourceRequest&, WebCore::StoredCredentials, WebCore::ClientCredentialPolicy, WebCore::ResourceError&, WebCore::ResourceResponse&, Vector<char>& data) override; -#if ENABLE(BLOB) - virtual WebCore::BlobRegistry* createBlobRegistry() override; -#endif + WebCore::CookiesStrategy* createCookiesStrategy() override; + WebCore::LoaderStrategy* createLoaderStrategy() override; + WebCore::PasteboardStrategy* createPasteboardStrategy() override; + WebCore::BlobRegistry* createBlobRegistry() override; }; } // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/NetworkProcessSupplement.h b/Source/WebKit2/NetworkProcess/NetworkProcessSupplement.h new file mode 100644 index 000000000..e2b46b391 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkProcessSupplement.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2012 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkProcessSupplement_h +#define NetworkProcessSupplement_h + +#include "ChildProcessSupplement.h" + +namespace WebKit { + +struct NetworkProcessCreationParameters; + +class NetworkProcessSupplement : public ChildProcessSupplement { +public: + virtual void initialize(const NetworkProcessCreationParameters&) + { + } +}; + +} // namespace WebKit + +#endif // NetworkProcessSupplement_h diff --git a/Source/WebKit2/NetworkProcess/NetworkResourceLoadParameters.cpp b/Source/WebKit2/NetworkProcess/NetworkResourceLoadParameters.cpp new file mode 100644 index 000000000..4e197c8eb --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkResourceLoadParameters.cpp @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2012 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkResourceLoadParameters.h" + +#include "ArgumentCoders.h" +#include "DataReference.h" +#include "WebCoreArgumentCoders.h" + +using namespace WebCore; + +namespace WebKit { + +void NetworkResourceLoadParameters::encode(IPC::Encoder& encoder) const +{ + encoder << identifier; + encoder << webPageID; + encoder << webFrameID; + encoder << sessionID; + encoder << request; + + encoder << static_cast<bool>(request.httpBody()); + if (request.httpBody()) { + request.httpBody()->encode(encoder); + + const Vector<FormDataElement>& elements = request.httpBody()->elements(); + size_t fileCount = 0; + for (size_t i = 0, count = elements.size(); i < count; ++i) { + if (elements[i].m_type == FormDataElement::Type::EncodedFile) + ++fileCount; + } + + SandboxExtension::HandleArray requestBodySandboxExtensions; + requestBodySandboxExtensions.allocate(fileCount); + size_t extensionIndex = 0; + for (size_t i = 0, count = elements.size(); i < count; ++i) { + const FormDataElement& element = elements[i]; + if (element.m_type == FormDataElement::Type::EncodedFile) { + const String& path = element.m_shouldGenerateFile ? element.m_generatedFilename : element.m_filename; + SandboxExtension::createHandle(path, SandboxExtension::ReadOnly, requestBodySandboxExtensions[extensionIndex++]); + } + } + encoder << requestBodySandboxExtensions; + } + + if (request.url().isLocalFile()) { + SandboxExtension::Handle requestSandboxExtension; + SandboxExtension::createHandle(request.url().fileSystemPath(), SandboxExtension::ReadOnly, requestSandboxExtension); + encoder << requestSandboxExtension; + } + + encoder.encodeEnum(contentSniffingPolicy); + encoder.encodeEnum(allowStoredCredentials); + encoder.encodeEnum(clientCredentialPolicy); + encoder << shouldFollowRedirects; + encoder << shouldClearReferrerOnHTTPSToHTTPRedirect; + encoder << defersLoading; + encoder << needsCertificateInfo; + encoder << maximumBufferingTime; + encoder << derivedCachedDataTypesToRetrieve; +} + +bool NetworkResourceLoadParameters::decode(IPC::Decoder& decoder, NetworkResourceLoadParameters& result) +{ + if (!decoder.decode(result.identifier)) + return false; + + if (!decoder.decode(result.webPageID)) + return false; + + if (!decoder.decode(result.webFrameID)) + return false; + + if (!decoder.decode(result.sessionID)) + return false; + + if (!decoder.decode(result.request)) + return false; + + bool hasHTTPBody; + if (!decoder.decode(hasHTTPBody)) + return false; + + if (hasHTTPBody) { + RefPtr<FormData> formData = FormData::decode(decoder); + if (!formData) + return false; + result.request.setHTTPBody(WTFMove(formData)); + + SandboxExtension::HandleArray requestBodySandboxExtensionHandles; + if (!decoder.decode(requestBodySandboxExtensionHandles)) + return false; + for (size_t i = 0; i < requestBodySandboxExtensionHandles.size(); ++i) { + if (auto extension = SandboxExtension::create(requestBodySandboxExtensionHandles[i])) + result.requestBodySandboxExtensions.append(WTFMove(extension)); + } + } + + if (result.request.url().isLocalFile()) { + SandboxExtension::Handle resourceSandboxExtensionHandle; + if (!decoder.decode(resourceSandboxExtensionHandle)) + return false; + result.resourceSandboxExtension = SandboxExtension::create(resourceSandboxExtensionHandle); + } + + if (!decoder.decodeEnum(result.contentSniffingPolicy)) + return false; + if (!decoder.decodeEnum(result.allowStoredCredentials)) + return false; + if (!decoder.decodeEnum(result.clientCredentialPolicy)) + return false; + if (!decoder.decode(result.shouldFollowRedirects)) + return false; + if (!decoder.decode(result.shouldClearReferrerOnHTTPSToHTTPRedirect)) + return false; + if (!decoder.decode(result.defersLoading)) + return false; + if (!decoder.decode(result.needsCertificateInfo)) + return false; + if (!decoder.decode(result.maximumBufferingTime)) + return false; + if (!decoder.decode(result.derivedCachedDataTypesToRetrieve)) + return false; + + return true; +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/NetworkResourceLoadParameters.h b/Source/WebKit2/NetworkProcess/NetworkResourceLoadParameters.h new file mode 100644 index 000000000..ee6f9cbb7 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkResourceLoadParameters.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2012, 2013 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkResourceLoadParameters_h +#define NetworkResourceLoadParameters_h + +#include "NetworkLoadParameters.h" +#include "SandboxExtension.h" +#include <WebCore/ResourceHandle.h> +#include <WebCore/ResourceLoaderOptions.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/SessionID.h> + +namespace IPC { +class Decoder; +class Encoder; +} + +namespace WebKit { + +typedef uint64_t ResourceLoadIdentifier; + +class NetworkResourceLoadParameters : public NetworkLoadParameters { +public: + void encode(IPC::Encoder&) const; + static bool decode(IPC::Decoder&, NetworkResourceLoadParameters&); + + ResourceLoadIdentifier identifier { 0 }; + Vector<RefPtr<SandboxExtension>> requestBodySandboxExtensions; // Created automatically for the sender. + RefPtr<SandboxExtension> resourceSandboxExtension; // Created automatically for the sender. + std::chrono::milliseconds maximumBufferingTime { 0ms }; + Vector<String> derivedCachedDataTypesToRetrieve; +}; + +} // namespace WebKit + +#endif // NetworkResourceLoadParameters_h diff --git a/Source/WebKit2/NetworkProcess/NetworkResourceLoadScheduler.cpp b/Source/WebKit2/NetworkProcess/NetworkResourceLoadScheduler.cpp deleted file mode 100644 index ff42743ed..000000000 --- a/Source/WebKit2/NetworkProcess/NetworkResourceLoadScheduler.cpp +++ /dev/null @@ -1,227 +0,0 @@ -#include "config.h" -#include "NetworkResourceLoadScheduler.h" - -#include "HostRecord.h" -#include "Logging.h" -#include "NetworkProcess.h" -#include "NetworkResourceLoadParameters.h" -#include "NetworkResourceLoader.h" -#include <wtf/MainThread.h> -#include <wtf/text/CString.h> - -#if ENABLE(NETWORK_PROCESS) - -using namespace WebCore; - -namespace WebKit { - -static const unsigned maxRequestsInFlightForNonHTTPProtocols = 20; - -NetworkResourceLoadScheduler::NetworkResourceLoadScheduler() - : m_nonHTTPProtocolHost(HostRecord::create(String(), maxRequestsInFlightForNonHTTPProtocols)) - , m_requestTimer(this, &NetworkResourceLoadScheduler::requestTimerFired) - -{ - platformInitializeMaximumHTTPConnectionCountPerHost(); -} - -void NetworkResourceLoadScheduler::scheduleServePendingRequests() -{ - if (!m_requestTimer.isActive()) - m_requestTimer.startOneShot(0); -} - -void NetworkResourceLoadScheduler::requestTimerFired(WebCore::Timer<NetworkResourceLoadScheduler>*) -{ - servePendingRequests(); -} - -void NetworkResourceLoadScheduler::scheduleLoader(PassRefPtr<NetworkResourceLoader> loader) -{ - ResourceLoadPriority priority = loader->priority(); - const ResourceRequest& resourceRequest = loader->request(); - - LOG(NetworkScheduling, "(NetworkProcess) NetworkResourceLoadScheduler::scheduleLoader resource '%s'", resourceRequest.url().string().utf8().data()); - - HostRecord* host = hostForURL(resourceRequest.url(), CreateIfNotFound); - bool hadRequests = host->hasRequests(); - host->scheduleResourceLoader(loader); - - if (priority > ResourceLoadPriorityLow || !resourceRequest.url().protocolIsInHTTPFamily() || (priority == ResourceLoadPriorityLow && !hadRequests)) { - // Try to request important resources immediately. - host->servePendingRequests(priority); - return; - } - - // Handle asynchronously so early low priority requests don't get scheduled before later high priority ones. - scheduleServePendingRequests(); -} - -HostRecord* NetworkResourceLoadScheduler::hostForURL(const WebCore::URL& url, CreateHostPolicy createHostPolicy) -{ - if (!url.protocolIsInHTTPFamily()) - return m_nonHTTPProtocolHost.get(); - - m_hosts.checkConsistency(); - String hostName = url.host(); - HostRecord* host = m_hosts.get(hostName); - if (!host && createHostPolicy == CreateIfNotFound) { - RefPtr<HostRecord> newHost = HostRecord::create(hostName, m_maxRequestsInFlightPerHost); - host = newHost.get(); - m_hosts.add(hostName, newHost.release()); - } - - return host; -} - -void NetworkResourceLoadScheduler::removeLoader(NetworkResourceLoader* loader) -{ - ASSERT(isMainThread()); - ASSERT(loader); - - LOG(NetworkScheduling, "(NetworkProcess) NetworkResourceLoadScheduler::removeLoadIdentifier removing loader %s", loader->request().url().string().utf8().data()); - - HostRecord* host = loader->hostRecord(); - - // Due to a race condition the WebProcess might have messaged the NetworkProcess to remove this identifier - // after the NetworkProcess has already removed it internally. - // In this situation we might not have a HostRecord to clean up. - if (host) - host->removeLoader(loader); - - scheduleServePendingRequests(); -} - -void NetworkResourceLoadScheduler::receivedRedirect(NetworkResourceLoader* loader, const WebCore::URL& redirectURL) -{ - ASSERT(isMainThread()); - LOG(NetworkScheduling, "(NetworkProcess) NetworkResourceLoadScheduler::receivedRedirect loader originally for '%s' redirected to '%s'", loader->request().url().string().utf8().data(), redirectURL.string().utf8().data()); - - HostRecord* oldHost = loader->hostRecord(); - - // The load may have been cancelled while the message was in flight from network thread to main thread. - if (!oldHost) - return; - - HostRecord* newHost = hostForURL(redirectURL, CreateIfNotFound); - - if (oldHost->name() == newHost->name()) - return; - - oldHost->removeLoader(loader); - newHost->addLoaderInProgress(loader); -} - -void NetworkResourceLoadScheduler::servePendingRequests(ResourceLoadPriority minimumPriority) -{ - LOG(NetworkScheduling, "(NetworkProcess) NetworkResourceLoadScheduler::servePendingRequests Serving requests for up to %i hosts with minimum priority %i", m_hosts.size(), minimumPriority); - - m_requestTimer.stop(); - - m_nonHTTPProtocolHost->servePendingRequests(minimumPriority); - - m_hosts.checkConsistency(); - Vector<RefPtr<HostRecord>> hostsToServe; - copyValuesToVector(m_hosts, hostsToServe); - - size_t size = hostsToServe.size(); - for (size_t i = 0; i < size; ++i) { - HostRecord* host = hostsToServe[i].get(); - if (host->hasRequests()) - host->servePendingRequests(minimumPriority); - else - m_hosts.remove(host->name()); - } -} - -static bool removeScheduledLoadersCalled = false; - -void NetworkResourceLoadScheduler::removeScheduledLoaders(void* context) -{ - ASSERT(isMainThread()); - ASSERT(removeScheduledLoadersCalled); - - NetworkResourceLoadScheduler* scheduler = static_cast<NetworkResourceLoadScheduler*>(context); - scheduler->removeScheduledLoaders(); -} - -void NetworkResourceLoadScheduler::removeScheduledLoaders() -{ - Vector<RefPtr<NetworkResourceLoader>> loadersToRemove; - { - MutexLocker locker(m_loadersToRemoveMutex); - loadersToRemove = m_loadersToRemove; - m_loadersToRemove.clear(); - removeScheduledLoadersCalled = false; - } - - for (size_t i = 0; i < loadersToRemove.size(); ++i) - removeLoader(loadersToRemove[i].get()); -} - -void NetworkResourceLoadScheduler::scheduleRemoveLoader(NetworkResourceLoader* loader) -{ - MutexLocker locker(m_loadersToRemoveMutex); - - m_loadersToRemove.append(loader); - - if (!removeScheduledLoadersCalled) { - removeScheduledLoadersCalled = true; - callOnMainThread(NetworkResourceLoadScheduler::removeScheduledLoaders, this); - } -} - -uint64_t NetworkResourceLoadScheduler::hostsPendingCount() const -{ - uint64_t count = m_nonHTTPProtocolHost->pendingRequestCount() ? 1 : 0; - - HostMap::const_iterator end = m_hosts.end(); - for (HostMap::const_iterator i = m_hosts.begin(); i != end; ++i) { - if (i->value->pendingRequestCount()) - ++count; - } - - return count; -} - -uint64_t NetworkResourceLoadScheduler::loadsPendingCount() const -{ - uint64_t count = m_nonHTTPProtocolHost->pendingRequestCount(); - - HostMap::const_iterator end = m_hosts.end(); - for (HostMap::const_iterator i = m_hosts.begin(); i != end; ++i) - count += i->value->pendingRequestCount(); - - return count; -} - -uint64_t NetworkResourceLoadScheduler::hostsActiveCount() const -{ - uint64_t count = 0; - - if (m_nonHTTPProtocolHost->activeLoadCount()) - count = 1; - - HostMap::const_iterator end = m_hosts.end(); - for (HostMap::const_iterator i = m_hosts.begin(); i != end; ++i) { - if (i->value->activeLoadCount()) - ++count; - } - - return count; -} - -uint64_t NetworkResourceLoadScheduler::loadsActiveCount() const -{ - uint64_t count = m_nonHTTPProtocolHost->activeLoadCount(); - - HostMap::const_iterator end = m_hosts.end(); - for (HostMap::const_iterator i = m_hosts.begin(); i != end; ++i) - count += i->value->activeLoadCount(); - - return count; -} - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/NetworkResourceLoadScheduler.h b/Source/WebKit2/NetworkProcess/NetworkResourceLoadScheduler.h deleted file mode 100644 index 840a27829..000000000 --- a/Source/WebKit2/NetworkProcess/NetworkResourceLoadScheduler.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) 2012 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef NetworkResourceLoadScheduler_h -#define NetworkResourceLoadScheduler_h - -#include <WebCore/ResourceLoadPriority.h> -#include <WebCore/Timer.h> -#include <wtf/HashMap.h> -#include <wtf/HashSet.h> -#include <wtf/text/StringHash.h> - -#if ENABLE(NETWORK_PROCESS) - -namespace WebCore { -class URL; -} - -namespace WebKit { - -class HostRecord; -class NetworkResourceLoader; - -class NetworkResourceLoadScheduler { - WTF_MAKE_NONCOPYABLE(NetworkResourceLoadScheduler); WTF_MAKE_FAST_ALLOCATED; - -public: - NetworkResourceLoadScheduler(); - - // Adds the request to the queue for its host. - void scheduleLoader(PassRefPtr<NetworkResourceLoader>); - - // Called by the WebProcess when a ResourceLoader is being cleaned up. - void removeLoader(NetworkResourceLoader*); - - // Called within the NetworkProcess on a background thread when a resource load has finished. - void scheduleRemoveLoader(NetworkResourceLoader*); - - void receivedRedirect(NetworkResourceLoader*, const WebCore::URL& redirectURL); - void servePendingRequests(WebCore::ResourceLoadPriority = WebCore::ResourceLoadPriorityVeryLow); - - // For NetworkProcess statistics reporting. - uint64_t hostsPendingCount() const; - uint64_t loadsPendingCount() const; - uint64_t hostsActiveCount() const; - uint64_t loadsActiveCount() const; - -private: - enum CreateHostPolicy { - CreateIfNotFound, - FindOnly - }; - - HostRecord* hostForURL(const WebCore::URL&, CreateHostPolicy = FindOnly); - - void scheduleServePendingRequests(); - void requestTimerFired(WebCore::Timer<NetworkResourceLoadScheduler>*); - - void platformInitializeMaximumHTTPConnectionCountPerHost(); - - static void removeScheduledLoaders(void* context); - void removeScheduledLoaders(); - - typedef HashMap<String, RefPtr<HostRecord>, StringHash> HostMap; - HostMap m_hosts; - - typedef HashSet<RefPtr<NetworkResourceLoader>> NetworkResourceLoaderSet; - NetworkResourceLoaderSet m_loaders; - - RefPtr<HostRecord> m_nonHTTPProtocolHost; - - bool m_isSerialLoadingEnabled; - - WebCore::Timer<NetworkResourceLoadScheduler> m_requestTimer; - - Mutex m_loadersToRemoveMutex; - Vector<RefPtr<NetworkResourceLoader>> m_loadersToRemove; - - unsigned m_maxRequestsInFlightPerHost; -}; - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) - -#endif // NetworkResourceLoadScheduler_h diff --git a/Source/WebKit2/NetworkProcess/NetworkResourceLoader.cpp b/Source/WebKit2/NetworkProcess/NetworkResourceLoader.cpp index 60d13a338..bab56b3b4 100644 --- a/Source/WebKit2/NetworkProcess/NetworkResourceLoader.cpp +++ b/Source/WebKit2/NetworkProcess/NetworkResourceLoader.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 Apple Inc. All rights reserved. + * Copyright (C) 2012-2015 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,393 +26,682 @@ #include "config.h" #include "NetworkResourceLoader.h" -#if ENABLE(NETWORK_PROCESS) - -#include "AsynchronousNetworkLoaderClient.h" -#include "AuthenticationManager.h" #include "DataReference.h" #include "Logging.h" #include "NetworkBlobRegistry.h" +#include "NetworkCache.h" #include "NetworkConnectionToWebProcess.h" +#include "NetworkLoad.h" #include "NetworkProcess.h" #include "NetworkProcessConnectionMessages.h" -#include "NetworkResourceLoadParameters.h" -#include "RemoteNetworkingContext.h" -#include "ShareableResource.h" -#include "SharedMemory.h" -#include "SynchronousNetworkLoaderClient.h" +#include "SessionTracker.h" #include "WebCoreArgumentCoders.h" #include "WebErrors.h" #include "WebResourceLoaderMessages.h" -#include <WebCore/NotImplemented.h> -#include <WebCore/ResourceBuffer.h> -#include <WebCore/ResourceHandle.h> -#include <wtf/MainThread.h> +#include <WebCore/BlobDataFileReference.h> +#include <WebCore/CertificateInfo.h> +#include <WebCore/DiagnosticLoggingKeys.h> +#include <WebCore/HTTPHeaderNames.h> +#include <WebCore/ProtectionSpace.h> +#include <WebCore/SharedBuffer.h> +#include <WebCore/SynchronousLoaderClient.h> +#include <wtf/CurrentTime.h> +#include <wtf/RunLoop.h> using namespace WebCore; +#define RELEASE_LOG_IF_ALLOWED(fmt, ...) RELEASE_LOG_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__) +#define RELEASE_LOG_ERROR_IF_ALLOWED(fmt, ...) RELEASE_LOG_ERROR_IF(isAlwaysOnLoggingAllowed(), Network, "%p - NetworkResourceLoader::" fmt, this, ##__VA_ARGS__) + namespace WebKit { -NetworkResourceLoader::NetworkResourceLoader(const NetworkResourceLoadParameters& parameters, NetworkConnectionToWebProcess* connection, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> reply) - : m_bytesReceived(0) - , m_handleConvertedToDownload(false) - , m_identifier(parameters.identifier) - , m_webPageID(parameters.webPageID) - , m_webFrameID(parameters.webFrameID) - , m_sessionID(parameters.sessionID) - , m_request(parameters.request) - , m_priority(parameters.priority) - , m_contentSniffingPolicy(parameters.contentSniffingPolicy) - , m_allowStoredCredentials(parameters.allowStoredCredentials) - , m_clientCredentialPolicy(parameters.clientCredentialPolicy) - , m_shouldClearReferrerOnHTTPSToHTTPRedirect(parameters.shouldClearReferrerOnHTTPSToHTTPRedirect) - , m_isLoadingMainResource(parameters.isMainResource) - , m_sandboxExtensionsAreConsumed(false) +struct NetworkResourceLoader::SynchronousLoadData { + SynchronousLoadData(RefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>&& reply) + : delayedReply(WTFMove(reply)) + { + ASSERT(delayedReply); + } + ResourceRequest currentRequest; + RefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> delayedReply; + ResourceResponse response; + ResourceError error; +}; + +static void sendReplyToSynchronousRequest(NetworkResourceLoader::SynchronousLoadData& data, const SharedBuffer* buffer) +{ + ASSERT(data.delayedReply); + ASSERT(!data.response.isNull() || !data.error.isNull()); + + Vector<char> responseBuffer; + if (buffer && buffer->size()) + responseBuffer.append(buffer->data(), buffer->size()); + + data.delayedReply->send(data.error, data.response, responseBuffer); + data.delayedReply = nullptr; +} + +NetworkResourceLoader::NetworkResourceLoader(const NetworkResourceLoadParameters& parameters, NetworkConnectionToWebProcess& connection, RefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>&& synchronousReply) + : m_parameters(parameters) , m_connection(connection) + , m_defersLoading(parameters.defersLoading) + , m_bufferingTimer(*this, &NetworkResourceLoader::bufferingTimerFired) { - // Either this loader has both a webPageID and webFrameID, or it is not allowed to ask the client for authentication credentials. + ASSERT(RunLoop::isMain()); // FIXME: This is necessary because of the existence of EmptyFrameLoaderClient in WebCore. // Once bug 116233 is resolved, this ASSERT can just be "m_webPageID && m_webFrameID" - ASSERT((m_webPageID && m_webFrameID) || m_clientCredentialPolicy == DoNotAskClientForAnyCredentials); + ASSERT((m_parameters.webPageID && m_parameters.webFrameID) || m_parameters.clientCredentialPolicy == ClientCredentialPolicy::CannotAskClientForCredentials); - for (size_t i = 0, count = parameters.requestBodySandboxExtensions.size(); i < count; ++i) { - if (RefPtr<SandboxExtension> extension = SandboxExtension::create(parameters.requestBodySandboxExtensions[i])) - m_requestBodySandboxExtensions.append(extension); - } - -#if ENABLE(BLOB) - if (m_request.httpBody()) { - const Vector<FormDataElement>& elements = m_request.httpBody()->elements(); - for (size_t i = 0, count = elements.size(); i < count; ++i) { - if (elements[i].m_type == FormDataElement::encodedBlob) { - Vector<RefPtr<SandboxExtension>> blobElementExtensions = NetworkBlobRegistry::shared().sandboxExtensions(elements[i].m_url); - m_requestBodySandboxExtensions.appendVector(blobElementExtensions); - } + if (originalRequest().httpBody()) { + for (const auto& element : originalRequest().httpBody()->elements()) { + if (element.m_type == FormDataElement::Type::EncodedBlob) + m_fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(connection, element.m_url)); } } - if (m_request.url().protocolIs("blob")) { - ASSERT(!SandboxExtension::create(parameters.resourceSandboxExtension)); - m_resourceSandboxExtensions = NetworkBlobRegistry::shared().sandboxExtensions(m_request.url()); - } else +#if !USE(NETWORK_SESSION) + if (originalRequest().url().protocolIsBlob()) { + ASSERT(!m_parameters.resourceSandboxExtension); + m_fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(connection, originalRequest().url())); + } #endif - if (RefPtr<SandboxExtension> resourceSandboxExtension = SandboxExtension::create(parameters.resourceSandboxExtension)) - m_resourceSandboxExtensions.append(resourceSandboxExtension); - ASSERT(isMainThread()); - - if (reply) - m_networkLoaderClient = std::make_unique<SynchronousNetworkLoaderClient>(m_request, reply); - else - m_networkLoaderClient = std::make_unique<AsynchronousNetworkLoaderClient>(); + + if (synchronousReply) + m_synchronousLoadData = std::make_unique<SynchronousLoadData>(WTFMove(synchronousReply)); } NetworkResourceLoader::~NetworkResourceLoader() { - ASSERT(isMainThread()); - ASSERT(!m_handle); - ASSERT(!m_hostRecord); + ASSERT(RunLoop::isMain()); + ASSERT(!m_networkLoad); + ASSERT(!isSynchronous() || !m_synchronousLoadData->delayedReply); } +#if ENABLE(NETWORK_CACHE) +bool NetworkResourceLoader::canUseCache(const ResourceRequest& request) const +{ + if (!NetworkCache::singleton().isEnabled()) + return false; + if (sessionID().isEphemeral()) + return false; + if (!request.url().protocolIsInHTTPFamily()) + return false; + if (originalRequest().cachePolicy() == WebCore::DoNotUseAnyCache) + return false; + + return true; +} + +bool NetworkResourceLoader::canUseCachedRedirect(const ResourceRequest& request) const +{ + if (!canUseCache(request)) + return false; + // Limit cached redirects to avoid cycles and other trouble. + // Networking layer follows over 30 redirects but caching that many seems unnecessary. + static const unsigned maximumCachedRedirectCount { 5 }; + if (m_redirectCount > maximumCachedRedirectCount) + return false; + + return true; +} +#endif + bool NetworkResourceLoader::isSynchronous() const { - return m_networkLoaderClient->isSynchronous(); + return !!m_synchronousLoadData; } void NetworkResourceLoader::start() { - ASSERT(isMainThread()); + ASSERT(RunLoop::isMain()); + + if (m_defersLoading) { + RELEASE_LOG_IF_ALLOWED("start: Loading is deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous()); + return; + } + +#if ENABLE(NETWORK_CACHE) + if (canUseCache(originalRequest())) { + RELEASE_LOG_IF_ALLOWED("start: Retrieving resource from cache (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous()); + retrieveCacheEntry(originalRequest()); + return; + } +#endif - // Explicit ref() balanced by a deref() in NetworkResourceLoader::resourceHandleStopped() - ref(); + startNetworkLoad(originalRequest()); +} - // FIXME (NetworkProcess): Set platform specific settings. - m_networkingContext = RemoteNetworkingContext::create(m_sessionID, m_shouldClearReferrerOnHTTPSToHTTPRedirect); +#if ENABLE(NETWORK_CACHE) +void NetworkResourceLoader::retrieveCacheEntry(const ResourceRequest& request) +{ + ASSERT(canUseCache(request)); + + RefPtr<NetworkResourceLoader> loader(this); + NetworkCache::singleton().retrieve(request, { m_parameters.webPageID, m_parameters.webFrameID }, [loader = WTFMove(loader), request](auto entry) { + if (loader->hasOneRef()) { + // The loader has been aborted and is only held alive by this lambda. + return; + } + if (!entry) { + loader->startNetworkLoad(request); + return; + } + if (entry->redirectRequest()) { + loader->dispatchWillSendRequestForCacheEntry(WTFMove(entry)); + return; + } + if (loader->m_parameters.needsCertificateInfo && !entry->response().certificateInfo()) { + loader->startNetworkLoad(request); + return; + } + if (entry->needsValidation() || request.cachePolicy() == WebCore::RefreshAnyCacheData) { + loader->validateCacheEntry(WTFMove(entry)); + return; + } + loader->didRetrieveCacheEntry(WTFMove(entry)); + }); +} +#endif + +void NetworkResourceLoader::startNetworkLoad(const ResourceRequest& request) +{ + RELEASE_LOG_IF_ALLOWED("startNetworkLoad: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isMainResource = %d, isSynchronous = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, isMainResource(), isSynchronous()); consumeSandboxExtensions(); - // FIXME (NetworkProcess): Pass an actual value for defersLoading - m_handle = ResourceHandle::create(m_networkingContext.get(), m_request, this, false /* defersLoading */, m_contentSniffingPolicy == SniffContent); + if (isSynchronous() || m_parameters.maximumBufferingTime > 0ms) + m_bufferedData = SharedBuffer::create(); + +#if ENABLE(NETWORK_CACHE) + if (canUseCache(request)) + m_bufferedDataForCache = SharedBuffer::create(); +#endif + + NetworkLoadParameters parameters = m_parameters; + parameters.defersLoading = m_defersLoading; + parameters.request = request; + +#if USE(NETWORK_SESSION) + if (request.url().protocolIsBlob()) + parameters.blobFileReferences = NetworkBlobRegistry::singleton().filesInBlob(m_connection, originalRequest().url()); + + auto* networkSession = SessionTracker::networkSession(parameters.sessionID); + if (!networkSession) { + WTFLogAlways("Attempted to create a NetworkLoad with a session (id=%" PRIu64 ") that does not exist.", parameters.sessionID.sessionID()); + RELEASE_LOG_ERROR_IF_ALLOWED("startNetworkLoad: Attempted to create a NetworkLoad with a session that does not exist (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", sessionID=%" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, parameters.sessionID.sessionID()); + NetworkProcess::singleton().logDiagnosticMessage(m_parameters.webPageID, WebCore::DiagnosticLoggingKeys::internalErrorKey(), WebCore::DiagnosticLoggingKeys::invalidSessionIDKey(), WebCore::ShouldSample::No); + didFailLoading(internalError(request.url())); + return; + } + m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters), *networkSession); +#else + m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters)); +#endif + + if (m_defersLoading) { + RELEASE_LOG_IF_ALLOWED("startNetworkLoad: Created, but deferred (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", + m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + } +} + +void NetworkResourceLoader::setDefersLoading(bool defers) +{ + if (m_defersLoading == defers) + return; + m_defersLoading = defers; + + if (defers) + RELEASE_LOG_IF_ALLOWED("setDefersLoading: Deferring resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + else + RELEASE_LOG_IF_ALLOWED("setDefersLoading: Resuming deferred resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + + if (m_networkLoad) { + m_networkLoad->setDefersLoading(defers); + return; + } + + if (!m_defersLoading) + start(); + else + RELEASE_LOG_IF_ALLOWED("setDefersLoading: defers = TRUE, but nothing to stop (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); } void NetworkResourceLoader::cleanup() { - ASSERT(isMainThread()); + ASSERT(RunLoop::isMain()); - invalidateSandboxExtensions(); + m_bufferingTimer.stop(); - if (FormData* formData = request().httpBody()) - formData->removeGeneratedFilesIfNeeded(); + invalidateSandboxExtensions(); - // Tell the scheduler about this finished loader soon so it can start more network requests. - NetworkProcess::shared().networkResourceLoadScheduler().scheduleRemoveLoader(this); + m_networkLoad = nullptr; - if (m_handle) { - // Explicit deref() balanced by a ref() in NetworkResourceLoader::start() - // This might cause the NetworkResourceLoader to be destroyed and therefore we do it last. - m_handle = 0; - deref(); - } + // This will cause NetworkResourceLoader to be destroyed and therefore we do it last. + m_connection->didCleanupResourceLoader(*this); } -void NetworkResourceLoader::didConvertHandleToDownload() +void NetworkResourceLoader::convertToDownload(DownloadID downloadID, const ResourceRequest& request, const ResourceResponse& response) { - ASSERT(m_handle); - m_handleConvertedToDownload = true; + ASSERT(m_networkLoad); + NetworkProcess::singleton().downloadManager().convertNetworkLoadToDownload(downloadID, std::exchange(m_networkLoad, nullptr), WTFMove(m_fileReferences), request, response); } void NetworkResourceLoader::abort() { - ASSERT(isMainThread()); + ASSERT(RunLoop::isMain()); - if (m_handle && !m_handleConvertedToDownload) - m_handle->cancel(); + RELEASE_LOG_IF_ALLOWED("abort: Canceling resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", + m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + + if (m_networkLoad) { +#if ENABLE(NETWORK_CACHE) + if (canUseCache(m_networkLoad->currentRequest())) { + // We might already have used data from this incomplete load. Ensure older versions don't remain in the cache after cancel. + if (!m_response.isNull()) + NetworkCache::singleton().remove(m_networkLoad->currentRequest()); + } +#endif + m_networkLoad->cancel(); + } cleanup(); } -void NetworkResourceLoader::didReceiveResponseAsync(ResourceHandle* handle, const ResourceResponse& response) +auto NetworkResourceLoader::didReceiveResponse(ResourceResponse&& receivedResponse) -> ShouldContinueDidReceiveResponse { - ASSERT_UNUSED(handle, handle == m_handle); - - // FIXME (NetworkProcess): Cache the response. - if (FormData* formData = request().httpBody()) - formData->removeGeneratedFilesIfNeeded(); + RELEASE_LOG_IF_ALLOWED("didReceiveResponse: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", httpStatusCode = %d, length = %" PRId64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, receivedResponse.httpStatusCode(), receivedResponse.expectedContentLength()); + + m_response = WTFMove(receivedResponse); + + // For multipart/x-mixed-replace didReceiveResponseAsync gets called multiple times and buffering would require special handling. + if (!isSynchronous() && m_response.isMultipart()) + m_bufferedData = nullptr; + + bool shouldSendDidReceiveResponse = true; +#if ENABLE(NETWORK_CACHE) + if (m_response.isMultipart()) + m_bufferedDataForCache = nullptr; + + if (m_cacheEntryForValidation) { + bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified + if (validationSucceeded) { + m_cacheEntryForValidation = NetworkCache::singleton().update(originalRequest(), { m_parameters.webPageID, m_parameters.webFrameID }, *m_cacheEntryForValidation, m_response); + // If the request was conditional then this revalidation was not triggered by the network cache and we pass the 304 response to WebCore. + if (originalRequest().isConditional()) + m_cacheEntryForValidation = nullptr; + } else + m_cacheEntryForValidation = nullptr; + } + shouldSendDidReceiveResponse = !m_cacheEntryForValidation; +#endif - m_networkLoaderClient->didReceiveResponse(this, response); + bool shouldWaitContinueDidReceiveResponse = isMainResource(); + if (shouldSendDidReceiveResponse) { + if (isSynchronous()) + m_synchronousLoadData->response = m_response; + else + send(Messages::WebResourceLoader::DidReceiveResponse(m_response, shouldWaitContinueDidReceiveResponse)); + } - // m_handle will be 0 if the request got aborted above. - if (!m_handle) - return; + // For main resources, the web process is responsible for sending back a NetworkResourceLoader::ContinueDidReceiveResponse message. + bool shouldContinueDidReceiveResponse = !shouldWaitContinueDidReceiveResponse; +#if ENABLE(NETWORK_CACHE) + shouldContinueDidReceiveResponse = shouldContinueDidReceiveResponse || m_cacheEntryForValidation; +#endif - if (!m_isLoadingMainResource) { - // For main resources, the web process is responsible for sending back a NetworkResourceLoader::ContinueDidReceiveResponse message. - m_handle->continueDidReceiveResponse(); + if (shouldContinueDidReceiveResponse) { + RELEASE_LOG_IF_ALLOWED("didReceiveResponse: Should not wait for message from WebContent process before continuing resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + return ShouldContinueDidReceiveResponse::Yes; } -} -void NetworkResourceLoader::didReceiveData(ResourceHandle*, const char* data, unsigned length, int encodedDataLength) -{ - // The NetworkProcess should never get a didReceiveData callback. - // We should always be using didReceiveBuffer. - ASSERT_NOT_REACHED(); + RELEASE_LOG_IF_ALLOWED("didReceiveResponse: Should wait for message from WebContent process before continuing resource load (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + return ShouldContinueDidReceiveResponse::No; } -void NetworkResourceLoader::didReceiveBuffer(ResourceHandle* handle, PassRefPtr<SharedBuffer> buffer, int encodedDataLength) +void NetworkResourceLoader::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength) { - ASSERT_UNUSED(handle, handle == m_handle); + if (!m_hasReceivedData) { + RELEASE_LOG_IF_ALLOWED("didReceiveBuffer: Started receiving data (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + m_hasReceivedData = true; + } + +#if ENABLE(NETWORK_CACHE) + ASSERT(!m_cacheEntryForValidation); + + if (m_bufferedDataForCache) { + // Prevent memory growth in case of streaming data. + const size_t maximumCacheBufferSize = 10 * 1024 * 1024; + if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize) + m_bufferedDataForCache->append(buffer.get()); + else + m_bufferedDataForCache = nullptr; + } +#endif + // FIXME: At least on OS X Yosemite we always get -1 from the resource handle. + unsigned encodedDataLength = reportedEncodedDataLength >= 0 ? reportedEncodedDataLength : buffer->size(); - // FIXME (NetworkProcess): For the memory cache we'll also need to cache the response data here. - // Such buffering will need to be thread safe, as this callback is happening on a background thread. - m_bytesReceived += buffer->size(); - m_networkLoaderClient->didReceiveBuffer(this, buffer.get(), encodedDataLength); + if (m_bufferedData) { + m_bufferedData->append(buffer.get()); + m_bufferedDataEncodedDataLength += encodedDataLength; + startBufferingTimerIfNeeded(); + return; + } + sendBuffer(buffer, encodedDataLength); } -void NetworkResourceLoader::didFinishLoading(ResourceHandle* handle, double finishTime) +void NetworkResourceLoader::didFinishLoading(double finishTime) { - ASSERT_UNUSED(handle, handle == m_handle); + RELEASE_LOG_IF_ALLOWED("didFinishLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); + +#if ENABLE(NETWORK_CACHE) + if (m_cacheEntryForValidation) { + // 304 Not Modified + ASSERT(m_response.httpStatusCode() == 304); + LOG(NetworkCache, "(NetworkProcess) revalidated"); + didRetrieveCacheEntry(WTFMove(m_cacheEntryForValidation)); + return; + } +#endif - m_networkLoaderClient->didFinishLoading(this, finishTime); + if (isSynchronous()) + sendReplyToSynchronousRequest(*m_synchronousLoadData, m_bufferedData.get()); + else { + if (m_bufferedData && !m_bufferedData->isEmpty()) { + // FIXME: Pass a real value or remove the encoded data size feature. + sendBuffer(*m_bufferedData, -1); + } + send(Messages::WebResourceLoader::DidFinishResourceLoad(finishTime)); + } + +#if ENABLE(NETWORK_CACHE) + tryStoreAsCacheEntry(); +#endif cleanup(); } -void NetworkResourceLoader::didFail(ResourceHandle* handle, const ResourceError& error) +void NetworkResourceLoader::didFailLoading(const ResourceError& error) { - ASSERT_UNUSED(handle, handle == m_handle); + RELEASE_LOG_IF_ALLOWED("didFailLoading: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ", isTimeout = %d, isCancellation = %d, errCode = %d)", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier, error.isTimeout(), error.isCancellation(), error.errorCode()); + + ASSERT(!error.isNull()); - m_networkLoaderClient->didFail(this, error); +#if ENABLE(NETWORK_CACHE) + m_cacheEntryForValidation = nullptr; +#endif + + if (isSynchronous()) { + m_synchronousLoadData->error = error; + sendReplyToSynchronousRequest(*m_synchronousLoadData, nullptr); + } else if (auto* connection = messageSenderConnection()) + connection->send(Messages::WebResourceLoader::DidFailResourceLoad(error), messageSenderDestinationID()); cleanup(); } -void NetworkResourceLoader::willSendRequestAsync(ResourceHandle* handle, const ResourceRequest& request, const ResourceResponse& redirectResponse) +void NetworkResourceLoader::willSendRedirectedRequest(ResourceRequest&& request, WebCore::ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse) { - ASSERT_UNUSED(handle, handle == m_handle); - - // We only expect to get the willSendRequest callback from ResourceHandle as the result of a redirect. - ASSERT(!redirectResponse.isNull()); - ASSERT(isMainThread()); - - ResourceRequest proposedRequest = request; - m_suggestedRequestForWillSendRequest = request; + ++m_redirectCount; + + if (isSynchronous()) { + ResourceRequest overridenRequest = redirectRequest; + // FIXME: This needs to be fixed to follow the redirect correctly even for cross-domain requests. + // This includes at least updating host records, and comparing the current request instead of the original request here. + if (!protocolHostAndPortAreEqual(originalRequest().url(), redirectRequest.url())) { + ASSERT(m_synchronousLoadData->error.isNull()); + m_synchronousLoadData->error = SynchronousLoaderClient::platformBadResponseError(); + m_networkLoad->clearCurrentRequest(); + overridenRequest = ResourceRequest(); + } + continueWillSendRequest(WTFMove(overridenRequest)); + return; + } + send(Messages::WebResourceLoader::WillSendRequest(redirectRequest, redirectResponse)); - m_networkLoaderClient->willSendRequest(this, proposedRequest, redirectResponse); +#if ENABLE(NETWORK_CACHE) + if (canUseCachedRedirect(request)) + NetworkCache::singleton().storeRedirect(request, redirectResponse, redirectRequest); +#else + UNUSED_PARAM(request); +#endif } -void NetworkResourceLoader::continueWillSendRequest(const ResourceRequest& newRequest) +void NetworkResourceLoader::continueWillSendRequest(ResourceRequest&& newRequest) { -#if PLATFORM(MAC) - m_suggestedRequestForWillSendRequest.updateFromDelegatePreservingOldHTTPBody(newRequest.nsURLRequest(DoNotUpdateHTTPBody)); -#elif USE(SOUP) - // FIXME: Implement ResourceRequest::updateFromDelegatePreservingOldHTTPBody. See https://bugs.webkit.org/show_bug.cgi?id=126127. - m_suggestedRequestForWillSendRequest.updateFromDelegatePreservingOldHTTPBody(newRequest); -#endif + RELEASE_LOG_IF_ALLOWED("continueWillSendRequest: (pageID = %" PRIu64 ", frameID = %" PRIu64 ", resourceID = %" PRIu64 ")", m_parameters.webPageID, m_parameters.webFrameID, m_parameters.identifier); - RunLoop::main()->dispatch(bind(&NetworkResourceLoadScheduler::receivedRedirect, &NetworkProcess::shared().networkResourceLoadScheduler(), this, m_suggestedRequestForWillSendRequest.url())); + // If there is a match in the network cache, we need to reuse the original cache policy. + newRequest.setCachePolicy(originalRequest().cachePolicy()); - m_request = m_suggestedRequestForWillSendRequest; - m_suggestedRequestForWillSendRequest = ResourceRequest(); +#if ENABLE(NETWORK_CACHE) + if (m_isWaitingContinueWillSendRequestForCachedRedirect) { + m_isWaitingContinueWillSendRequestForCachedRedirect = false; + + LOG(NetworkCache, "(NetworkProcess) Retrieving cached redirect"); + + if (canUseCachedRedirect(newRequest)) + retrieveCacheEntry(newRequest); + else + startNetworkLoad(newRequest); - if (m_request.isNull()) { - m_handle->cancel(); - didFail(m_handle.get(), cancelledError(m_request)); return; } +#endif - m_handle->continueWillSendRequest(m_request); + if (m_networkLoad) + m_networkLoad->continueWillSendRequest(WTFMove(newRequest)); } void NetworkResourceLoader::continueDidReceiveResponse() { // FIXME: Remove this check once BlobResourceHandle implements didReceiveResponseAsync correctly. // Currently, it does not wait for response, so the load is likely to finish before continueDidReceiveResponse. - if (!m_handle) - return; - - m_handle->continueDidReceiveResponse(); + if (m_networkLoad) + m_networkLoad->continueDidReceiveResponse(); } -void NetworkResourceLoader::didSendData(ResourceHandle* handle, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) +void NetworkResourceLoader::didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) { - ASSERT_UNUSED(handle, handle == m_handle); - - m_networkLoaderClient->didSendData(this, bytesSent, totalBytesToBeSent); + if (!isSynchronous()) + send(Messages::WebResourceLoader::DidSendData(bytesSent, totalBytesToBeSent)); } -void NetworkResourceLoader::wasBlocked(ResourceHandle* handle) +void NetworkResourceLoader::startBufferingTimerIfNeeded() { - ASSERT_UNUSED(handle, handle == m_handle); - - didFail(handle, WebKit::blockedError(request())); + if (isSynchronous()) + return; + if (m_bufferingTimer.isActive()) + return; + m_bufferingTimer.startOneShot(m_parameters.maximumBufferingTime); } -void NetworkResourceLoader::cannotShowURL(ResourceHandle* handle) +void NetworkResourceLoader::bufferingTimerFired() { - ASSERT_UNUSED(handle, handle == m_handle); + ASSERT(m_bufferedData); + ASSERT(m_networkLoad); - didFail(handle, WebKit::cannotShowURLError(request())); + if (m_bufferedData->isEmpty()) + return; + + IPC::SharedBufferDataReference dataReference(m_bufferedData.get()); + size_t encodedLength = m_bufferedDataEncodedDataLength; + + m_bufferedData = SharedBuffer::create(); + m_bufferedDataEncodedDataLength = 0; + + send(Messages::WebResourceLoader::DidReceiveData(dataReference, encodedLength)); } -bool NetworkResourceLoader::shouldUseCredentialStorage(ResourceHandle* handle) +void NetworkResourceLoader::sendBuffer(SharedBuffer& buffer, size_t encodedDataLength) { - ASSERT_UNUSED(handle, handle == m_handle || !m_handle); // m_handle will be 0 if called from ResourceHandle::start(). + ASSERT(!isSynchronous()); - // When the WebProcess is handling loading a client is consulted each time this shouldUseCredentialStorage question is asked. - // In NetworkProcess mode we ask the WebProcess client up front once and then reuse the cached answer. + IPC::SharedBufferDataReference dataReference(&buffer); + send(Messages::WebResourceLoader::DidReceiveData(dataReference, encodedDataLength)); +} - // We still need this sync version, because ResourceHandle itself uses it internally, even when the delegate uses an async one. +#if ENABLE(NETWORK_CACHE) +void NetworkResourceLoader::tryStoreAsCacheEntry() +{ + if (!canUseCache(m_networkLoad->currentRequest())) + return; + if (!m_bufferedDataForCache) + return; - return m_allowStoredCredentials == AllowStoredCredentials; + NetworkCache::singleton().store(m_networkLoad->currentRequest(), m_response, WTFMove(m_bufferedDataForCache), [loader = makeRef(*this)](auto& mappedBody) mutable { +#if ENABLE(SHAREABLE_RESOURCE) + if (mappedBody.shareableResourceHandle.isNull()) + return; + LOG(NetworkCache, "(NetworkProcess) sending DidCacheResource"); + loader->send(Messages::NetworkProcessConnection::DidCacheResource(loader->originalRequest(), mappedBody.shareableResourceHandle, loader->sessionID())); +#endif + }); } -void NetworkResourceLoader::shouldUseCredentialStorageAsync(ResourceHandle* handle) +void NetworkResourceLoader::didRetrieveCacheEntry(std::unique_ptr<NetworkCache::Entry> entry) { - ASSERT_UNUSED(handle, handle == m_handle); + if (isSynchronous()) { + m_synchronousLoadData->response = entry->response(); + sendReplyToSynchronousRequest(*m_synchronousLoadData, entry->buffer()); + cleanup(); + return; + } - handle->continueShouldUseCredentialStorage(shouldUseCredentialStorage(handle)); + bool needsContinueDidReceiveResponseMessage = isMainResource(); + send(Messages::WebResourceLoader::DidReceiveResponse(entry->response(), needsContinueDidReceiveResponseMessage)); + + if (entry->sourceStorageRecord().bodyHash && !m_parameters.derivedCachedDataTypesToRetrieve.isEmpty()) { + auto bodyHash = *entry->sourceStorageRecord().bodyHash; + auto* entryPtr = entry.release(); + auto retrieveCount = m_parameters.derivedCachedDataTypesToRetrieve.size(); + + for (auto& type : m_parameters.derivedCachedDataTypesToRetrieve) { + NetworkCache::DataKey key { originalRequest().cachePartition(), type, bodyHash }; + NetworkCache::singleton().retrieveData(key, [loader = makeRef(*this), entryPtr, type, retrieveCount] (const uint8_t* data, size_t size) mutable { + loader->m_retrievedDerivedDataCount++; + bool retrievedAll = loader->m_retrievedDerivedDataCount == retrieveCount; + std::unique_ptr<NetworkCache::Entry> entry(retrievedAll ? entryPtr : nullptr); + if (loader->hasOneRef()) + return; + if (data) { + IPC::DataReference dataReference(data, size); + loader->send(Messages::WebResourceLoader::DidRetrieveDerivedData(type, dataReference)); + } + if (retrievedAll) { + loader->sendResultForCacheEntry(WTFMove(entry)); + loader->cleanup(); + } + }); + } + return; + } + + sendResultForCacheEntry(WTFMove(entry)); + + cleanup(); } -void NetworkResourceLoader::didReceiveAuthenticationChallenge(ResourceHandle* handle, const AuthenticationChallenge& challenge) +void NetworkResourceLoader::sendResultForCacheEntry(std::unique_ptr<NetworkCache::Entry> entry) { - ASSERT_UNUSED(handle, handle == m_handle); - - // FIXME (http://webkit.org/b/115291): Since we go straight to the UI process for authentication we don't get WebCore's - // cross-origin check before asking the client for credentials. - // Therefore we are too permissive in the case where the ClientCredentialPolicy is DoNotAskClientForCrossOriginCredentials. - if (m_clientCredentialPolicy == DoNotAskClientForAnyCredentials) { - challenge.authenticationClient()->receivedRequestToContinueWithoutCredential(challenge); +#if ENABLE(SHAREABLE_RESOURCE) + if (!entry->shareableResourceHandle().isNull()) { + send(Messages::WebResourceLoader::DidReceiveResource(entry->shareableResourceHandle(), currentTime())); return; } +#endif - NetworkProcess::shared().authenticationManager().didReceiveAuthenticationChallenge(m_webPageID, m_webFrameID, challenge); + sendBuffer(*entry->buffer(), entry->buffer()->size()); + send(Messages::WebResourceLoader::DidFinishResourceLoad(currentTime())); } -void NetworkResourceLoader::didCancelAuthenticationChallenge(ResourceHandle* handle, const AuthenticationChallenge& challenge) +void NetworkResourceLoader::validateCacheEntry(std::unique_ptr<NetworkCache::Entry> entry) { - ASSERT_UNUSED(handle, handle == m_handle); + ASSERT(!m_networkLoad); + + // If the request is already conditional then the revalidation was not triggered by the disk cache + // and we should not overwrite the existing conditional headers. + ResourceRequest revalidationRequest = originalRequest(); + if (!revalidationRequest.isConditional()) { + String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag); + String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified); + if (!eTag.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag); + if (!lastModified.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified); + } + + m_cacheEntryForValidation = WTFMove(entry); - // This function is probably not needed (see <rdar://problem/8960124>). - notImplemented(); + startNetworkLoad(revalidationRequest); } -void NetworkResourceLoader::receivedCancellation(ResourceHandle* handle, const AuthenticationChallenge& challenge) +void NetworkResourceLoader::dispatchWillSendRequestForCacheEntry(std::unique_ptr<NetworkCache::Entry> entry) { - ASSERT_UNUSED(handle, handle == m_handle); + ASSERT(entry->redirectRequest()); + ASSERT(!m_isWaitingContinueWillSendRequestForCachedRedirect); - m_handle->cancel(); - didFail(m_handle.get(), cancelledError(m_request)); + LOG(NetworkCache, "(NetworkProcess) Executing cached redirect"); + + ++m_redirectCount; + send(Messages::WebResourceLoader::WillSendRequest(*entry->redirectRequest(), entry->response())); + m_isWaitingContinueWillSendRequestForCachedRedirect = true; } +#endif IPC::Connection* NetworkResourceLoader::messageSenderConnection() { - return connectionToWebProcess()->connection(); + return &connectionToWebProcess().connection(); } void NetworkResourceLoader::consumeSandboxExtensions() { - for (size_t i = 0, count = m_requestBodySandboxExtensions.size(); i < count; ++i) - m_requestBodySandboxExtensions[i]->consume(); + ASSERT(!m_didConsumeSandboxExtensions); - for (size_t i = 0, count = m_resourceSandboxExtensions.size(); i < count; ++i) - m_resourceSandboxExtensions[i]->consume(); + for (auto& extension : m_parameters.requestBodySandboxExtensions) + extension->consume(); - m_sandboxExtensionsAreConsumed = true; + if (auto& extension = m_parameters.resourceSandboxExtension) + extension->consume(); + + for (auto& fileReference : m_fileReferences) + fileReference->prepareForFileAccess(); + + m_didConsumeSandboxExtensions = true; } void NetworkResourceLoader::invalidateSandboxExtensions() { - if (m_sandboxExtensionsAreConsumed) { - for (size_t i = 0, count = m_requestBodySandboxExtensions.size(); i < count; ++i) - m_requestBodySandboxExtensions[i]->revoke(); - for (size_t i = 0, count = m_resourceSandboxExtensions.size(); i < count; ++i) - m_resourceSandboxExtensions[i]->revoke(); + if (m_didConsumeSandboxExtensions) { + for (auto& extension : m_parameters.requestBodySandboxExtensions) + extension->revoke(); + if (auto& extension = m_parameters.resourceSandboxExtension) + extension->revoke(); + for (auto& fileReference : m_fileReferences) + fileReference->revokeFileAccess(); + + m_didConsumeSandboxExtensions = false; } - m_requestBodySandboxExtensions.clear(); - m_resourceSandboxExtensions.clear(); - - m_sandboxExtensionsAreConsumed = false; + m_fileReferences.clear(); } #if USE(PROTECTION_SPACE_AUTH_CALLBACK) -void NetworkResourceLoader::canAuthenticateAgainstProtectionSpaceAsync(ResourceHandle* handle, const ProtectionSpace& protectionSpace) +void NetworkResourceLoader::canAuthenticateAgainstProtectionSpaceAsync(const ProtectionSpace& protectionSpace) { - ASSERT(isMainThread()); - ASSERT_UNUSED(handle, handle == m_handle); - - m_networkLoaderClient->canAuthenticateAgainstProtectionSpace(this, protectionSpace); + NetworkProcess::singleton().canAuthenticateAgainstProtectionSpace(*this, protectionSpace); } void NetworkResourceLoader::continueCanAuthenticateAgainstProtectionSpace(bool result) { - m_handle->continueCanAuthenticateAgainstProtectionSpace(result); + if (m_networkLoad) + m_networkLoad->continueCanAuthenticateAgainstProtectionSpace(result); } - #endif -#if USE(NETWORK_CFDATA_ARRAY_CALLBACK) -bool NetworkResourceLoader::supportsDataArray() +bool NetworkResourceLoader::isAlwaysOnLoggingAllowed() const { - notImplemented(); - return false; + return sessionID().isAlwaysOnLoggingAllowed(); } -void NetworkResourceLoader::didReceiveDataArray(ResourceHandle*, CFArrayRef) -{ - ASSERT_NOT_REACHED(); - notImplemented(); -} -#endif - -#if PLATFORM(MAC) && !PLATFORM(IOS) -void NetworkResourceLoader::willStopBufferingData(ResourceHandle*, const char*, unsigned) -{ - notImplemented(); -} -#endif // PLATFORM(MAC) - } // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/NetworkResourceLoader.h b/Source/WebKit2/NetworkProcess/NetworkResourceLoader.h index 7b69a5911..fe93ee054 100644 --- a/Source/WebKit2/NetworkProcess/NetworkResourceLoader.h +++ b/Source/WebKit2/NetworkProcess/NetworkResourceLoader.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 Apple Inc. All rights reserved. + * Copyright (C) 2012-2015 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,174 +26,145 @@ #ifndef NetworkResourceLoader_h #define NetworkResourceLoader_h -#if ENABLE(NETWORK_PROCESS) - -#include "HostRecord.h" +#include "DownloadID.h" #include "MessageSender.h" #include "NetworkConnectionToWebProcessMessages.h" +#include "NetworkLoadClient.h" +#include "NetworkResourceLoadParameters.h" #include "ShareableResource.h" -#include <WebCore/ResourceHandleClient.h> -#include <WebCore/ResourceLoaderOptions.h> -#include <WebCore/ResourceRequest.h> -#include <wtf/MainThread.h> -#include <wtf/RunLoop.h> - -typedef const struct _CFCachedURLResponse* CFCachedURLResponseRef; +#include <WebCore/Timer.h> namespace WebCore { -class ResourceBuffer; -class ResourceHandle; +class BlobDataFileReference; class ResourceRequest; } namespace WebKit { class NetworkConnectionToWebProcess; -class NetworkLoaderClient; -class NetworkResourceLoadParameters; -class RemoteNetworkingContext; +class NetworkLoad; class SandboxExtension; -class NetworkResourceLoader : public RefCounted<NetworkResourceLoader>, public WebCore::ResourceHandleClient, public IPC::MessageSender { +namespace NetworkCache { +class Entry; +} + +class NetworkResourceLoader final : public RefCounted<NetworkResourceLoader>, public NetworkLoadClient, public IPC::MessageSender { public: - static RefPtr<NetworkResourceLoader> create(const NetworkResourceLoadParameters& parameters, NetworkConnectionToWebProcess* connection) + static Ref<NetworkResourceLoader> create(const NetworkResourceLoadParameters& parameters, NetworkConnectionToWebProcess& connection, RefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>&& reply = nullptr) { - return adoptRef(new NetworkResourceLoader(parameters, connection, nullptr)); + return adoptRef(*new NetworkResourceLoader(parameters, connection, WTFMove(reply))); } - - static RefPtr<NetworkResourceLoader> create(const NetworkResourceLoadParameters& parameters, NetworkConnectionToWebProcess* connection, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> reply) - { - return adoptRef(new NetworkResourceLoader(parameters, connection, reply)); - } - ~NetworkResourceLoader(); + virtual ~NetworkResourceLoader(); - NetworkConnectionToWebProcess* connectionToWebProcess() const { return m_connection.get(); } + const WebCore::ResourceRequest& originalRequest() const { return m_parameters.request; } - WebCore::ResourceLoadPriority priority() { return m_priority; } - WebCore::ResourceRequest& request() { return m_request; } - - WebCore::ResourceHandle* handle() const { return m_handle.get(); } - void didConvertHandleToDownload(); + NetworkLoad* networkLoad() const { return m_networkLoad.get(); } void start(); void abort(); - // ResourceHandleClient methods - virtual void willSendRequestAsync(WebCore::ResourceHandle*, const WebCore::ResourceRequest&, const WebCore::ResourceResponse& redirectResponse) override; - virtual void didSendData(WebCore::ResourceHandle*, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override; - virtual void didReceiveResponseAsync(WebCore::ResourceHandle*, const WebCore::ResourceResponse&) override; - virtual void didReceiveData(WebCore::ResourceHandle*, const char*, unsigned, int encodedDataLength) override; - virtual void didReceiveBuffer(WebCore::ResourceHandle*, PassRefPtr<WebCore::SharedBuffer>, int encodedDataLength) override; - virtual void didFinishLoading(WebCore::ResourceHandle*, double finishTime) override; - virtual void didFail(WebCore::ResourceHandle*, const WebCore::ResourceError&) override; - virtual void wasBlocked(WebCore::ResourceHandle*) override; - virtual void cannotShowURL(WebCore::ResourceHandle*) override; - virtual bool shouldUseCredentialStorage(WebCore::ResourceHandle*) override; - virtual void shouldUseCredentialStorageAsync(WebCore::ResourceHandle*) override; - virtual void didReceiveAuthenticationChallenge(WebCore::ResourceHandle*, const WebCore::AuthenticationChallenge&) override; - virtual void didCancelAuthenticationChallenge(WebCore::ResourceHandle*, const WebCore::AuthenticationChallenge&) override; - virtual void receivedCancellation(WebCore::ResourceHandle*, const WebCore::AuthenticationChallenge&) override; - virtual bool usesAsyncCallbacks() override { return true; } - -#if USE(PROTECTION_SPACE_AUTH_CALLBACK) - virtual void canAuthenticateAgainstProtectionSpaceAsync(WebCore::ResourceHandle*, const WebCore::ProtectionSpace&) override; -#endif - -#if USE(NETWORK_CFDATA_ARRAY_CALLBACK) - virtual bool supportsDataArray() override; - virtual void didReceiveDataArray(WebCore::ResourceHandle*, CFArrayRef) override; -#endif - -#if PLATFORM(MAC) - static size_t fileBackedResourceMinimumSize(); -#if !PLATFORM(IOS) - virtual void willCacheResponseAsync(WebCore::ResourceHandle*, NSCachedURLResponse *) override; - virtual void willStopBufferingData(WebCore::ResourceHandle*, const char*, unsigned) override; -#endif -#endif // PLATFORM(MAC) + void setDefersLoading(bool); // Message handlers. - void didReceiveNetworkResourceLoaderMessage(IPC::Connection*, IPC::MessageDecoder&); + void didReceiveNetworkResourceLoaderMessage(IPC::Connection&, IPC::Decoder&); -#if PLATFORM(IOS) || (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) - static void tryGetShareableHandleFromCFURLCachedResponse(ShareableResource::Handle&, CFCachedURLResponseRef); +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void continueCanAuthenticateAgainstProtectionSpace(bool); #endif + void continueWillSendRequest(WebCore::ResourceRequest&& newRequest); - bool isSynchronous() const; - bool isLoadingMainResource() const { return m_isLoadingMainResource; } - - void setHostRecord(HostRecord* hostRecord) { ASSERT(isMainThread()); m_hostRecord = hostRecord; } - HostRecord* hostRecord() const { ASSERT(isMainThread()); return m_hostRecord.get(); } + WebCore::SharedBuffer* bufferedData() { return m_bufferedData.get(); } + const WebCore::ResourceResponse& response() const { return m_response; } - template<typename T> - bool sendAbortingOnFailure(T&& message, unsigned messageSendFlags = 0) - { - bool result = messageSenderConnection()->send(std::forward<T>(message), messageSenderDestinationID(), messageSendFlags); - if (!result) - abort(); - return result; - } + NetworkConnectionToWebProcess& connectionToWebProcess() { return m_connection; } + WebCore::SessionID sessionID() const { return m_parameters.sessionID; } + ResourceLoadIdentifier identifier() const { return m_parameters.identifier; } + uint64_t frameID() const { return m_parameters.webFrameID; } + uint64_t pageID() const { return m_parameters.webPageID; } + struct SynchronousLoadData; + // NetworkLoadClient. + void didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override; #if USE(PROTECTION_SPACE_AUTH_CALLBACK) - void continueCanAuthenticateAgainstProtectionSpace(bool); + void canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) override; #endif - void continueWillSendRequest(const WebCore::ResourceRequest& newRequest); + bool isSynchronous() const override; + void willSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceRequest&& redirectRequest, WebCore::ResourceResponse&&) override; + ShouldContinueDidReceiveResponse didReceiveResponse(WebCore::ResourceResponse&&) override; + void didReceiveBuffer(Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) override; + void didFinishLoading(double finishTime) override; + void didFailLoading(const WebCore::ResourceError&) override; -#if PLATFORM(IOS) || (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) - static void tryGetShareableHandleFromSharedBuffer(ShareableResource::Handle&, WebCore::SharedBuffer*); -#endif + void convertToDownload(DownloadID, const WebCore::ResourceRequest&, const WebCore::ResourceResponse&); + + bool isMainResource() const { return m_parameters.request.requester() == WebCore::ResourceRequest::Requester::Main; } + bool isAlwaysOnLoggingAllowed() const; private: - NetworkResourceLoader(const NetworkResourceLoadParameters&, NetworkConnectionToWebProcess*, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>); + NetworkResourceLoader(const NetworkResourceLoadParameters&, NetworkConnectionToWebProcess&, RefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>&&); // IPC::MessageSender - virtual IPC::Connection* messageSenderConnection() override; - virtual uint64_t messageSenderDestinationID() override { return m_identifier; } + IPC::Connection* messageSenderConnection() override; + uint64_t messageSenderDestinationID() override { return m_parameters.identifier; } + +#if ENABLE(NETWORK_CACHE) + bool canUseCache(const WebCore::ResourceRequest&) const; + bool canUseCachedRedirect(const WebCore::ResourceRequest&) const; + + void tryStoreAsCacheEntry(); + void retrieveCacheEntry(const WebCore::ResourceRequest&); + void didRetrieveCacheEntry(std::unique_ptr<NetworkCache::Entry>); + void sendResultForCacheEntry(std::unique_ptr<NetworkCache::Entry>); + void validateCacheEntry(std::unique_ptr<NetworkCache::Entry>); + void dispatchWillSendRequestForCacheEntry(std::unique_ptr<NetworkCache::Entry>); +#endif + void startNetworkLoad(const WebCore::ResourceRequest&); void continueDidReceiveResponse(); void cleanup(); void platformDidReceiveResponse(const WebCore::ResourceResponse&); + void startBufferingTimerIfNeeded(); + void bufferingTimerFired(); + void sendBuffer(WebCore::SharedBuffer&, size_t encodedDataLength); + void consumeSandboxExtensions(); void invalidateSandboxExtensions(); - RefPtr<RemoteNetworkingContext> m_networkingContext; - RefPtr<WebCore::ResourceHandle> m_handle; + const NetworkResourceLoadParameters m_parameters; - // Keep the suggested request around while asynchronously asking to update it, because some parts of the request don't survive IPC. - WebCore::ResourceRequest m_suggestedRequestForWillSendRequest; + Ref<NetworkConnectionToWebProcess> m_connection; - uint64_t m_bytesReceived; + std::unique_ptr<NetworkLoad> m_networkLoad; - bool m_handleConvertedToDownload; - std::unique_ptr<NetworkLoaderClient> m_networkLoaderClient; + WebCore::ResourceResponse m_response; - ResourceLoadIdentifier m_identifier; - uint64_t m_webPageID; - uint64_t m_webFrameID; - uint64_t m_sessionID; - WebCore::ResourceRequest m_request; - WebCore::ResourceLoadPriority m_priority; - WebCore::ContentSniffingPolicy m_contentSniffingPolicy; - WebCore::StoredCredentials m_allowStoredCredentials; - WebCore::ClientCredentialPolicy m_clientCredentialPolicy; - bool m_shouldClearReferrerOnHTTPSToHTTPRedirect; - bool m_isLoadingMainResource; + size_t m_bytesReceived { 0 }; + size_t m_bufferedDataEncodedDataLength { 0 }; + RefPtr<WebCore::SharedBuffer> m_bufferedData; + unsigned m_redirectCount { 0 }; - Vector<RefPtr<SandboxExtension>> m_requestBodySandboxExtensions; - Vector<RefPtr<SandboxExtension>> m_resourceSandboxExtensions; - bool m_sandboxExtensionsAreConsumed; + std::unique_ptr<SynchronousLoadData> m_synchronousLoadData; + Vector<RefPtr<WebCore::BlobDataFileReference>> m_fileReferences; - RefPtr<NetworkConnectionToWebProcess> m_connection; - - RefPtr<HostRecord> m_hostRecord; + bool m_didConsumeSandboxExtensions { false }; + bool m_defersLoading { false }; + bool m_hasReceivedData { false }; + + unsigned m_retrievedDerivedDataCount { 0 }; + + WebCore::Timer m_bufferingTimer; +#if ENABLE(NETWORK_CACHE) + RefPtr<WebCore::SharedBuffer> m_bufferedDataForCache; + std::unique_ptr<NetworkCache::Entry> m_cacheEntryForValidation; + bool m_isWaitingContinueWillSendRequestForCachedRedirect { false }; +#endif }; } // namespace WebKit -#endif // ENABLE(NETWORK_PROCESS) - #endif // NetworkResourceLoader_h diff --git a/Source/WebKit2/NetworkProcess/NetworkResourceLoader.messages.in b/Source/WebKit2/NetworkProcess/NetworkResourceLoader.messages.in index 79a35af56..2cb76e4c5 100644 --- a/Source/WebKit2/NetworkProcess/NetworkResourceLoader.messages.in +++ b/Source/WebKit2/NetworkProcess/NetworkResourceLoader.messages.in @@ -20,15 +20,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#if ENABLE(NETWORK_PROCESS) - messages -> NetworkResourceLoader LegacyReceiver { ContinueWillSendRequest(WebCore::ResourceRequest request) ContinueDidReceiveResponse() -#if USE(PROTECTION_SPACE_AUTH_CALLBACK) - ContinueCanAuthenticateAgainstProtectionSpace(bool canAuthenticate) -#endif } - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/NetworkSession.cpp b/Source/WebKit2/NetworkProcess/NetworkSession.cpp new file mode 100644 index 000000000..1e5e011ac --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkSession.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkSession.h" + +#if USE(NETWORK_SESSION) + +#include "NetworkDataTask.h" +#include <WebCore/NetworkStorageSession.h> +#include <wtf/MainThread.h> + +#if PLATFORM(COCOA) +#include "NetworkSessionCocoa.h" +#endif +#if USE(SOUP) +#include "NetworkSessionSoup.h" +#endif + + +using namespace WebCore; + +namespace WebKit { + +Ref<NetworkSession> NetworkSession::create(SessionID sessionID, CustomProtocolManager* customProtocolManager) +{ +#if PLATFORM(COCOA) + return NetworkSessionCocoa::create(sessionID, customProtocolManager); +#endif +#if USE(SOUP) + UNUSED_PARAM(customProtocolManager); + return NetworkSessionSoup::create(sessionID); +#endif +} + +NetworkSession& NetworkSession::defaultSession() +{ +#if PLATFORM(COCOA) + return NetworkSessionCocoa::defaultSession(); +#else + ASSERT(isMainThread()); + static NetworkSession* session = &NetworkSession::create(SessionID::defaultSessionID()).leakRef(); + return *session; +#endif +} + +NetworkStorageSession& NetworkSession::networkStorageSession() const +{ + auto* storageSession = NetworkStorageSession::storageSession(m_sessionID); + RELEASE_ASSERT(storageSession); + return *storageSession; +} + +NetworkSession::NetworkSession(SessionID sessionID) + : m_sessionID(sessionID) +{ +} + +NetworkSession::~NetworkSession() +{ +} + +void NetworkSession::invalidateAndCancel() +{ + for (auto* task : m_dataTaskSet) + task->invalidateAndCancel(); +} + +} // namespace WebKit + +#endif // USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/NetworkSession.h b/Source/WebKit2/NetworkProcess/NetworkSession.h new file mode 100644 index 000000000..d5b016f28 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/NetworkSession.h @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(NETWORK_SESSION) + +#include <WebCore/SessionID.h> +#include <wtf/HashSet.h> +#include <wtf/Ref.h> +#include <wtf/RefCounted.h> + +namespace WebCore { +class NetworkStorageSession; +} + +namespace WebKit { + +class CustomProtocolManager; +class NetworkDataTask; + +class NetworkSession : public RefCounted<NetworkSession> { +public: + static Ref<NetworkSession> create(WebCore::SessionID, CustomProtocolManager* = nullptr); + static NetworkSession& defaultSession(); + virtual ~NetworkSession(); + + virtual void invalidateAndCancel(); + virtual void clearCredentials() { }; + + WebCore::SessionID sessionID() const { return m_sessionID; } + WebCore::NetworkStorageSession& networkStorageSession() const; + + void registerNetworkDataTask(NetworkDataTask& task) { m_dataTaskSet.add(&task); } + void unregisterNetworkDataTask(NetworkDataTask& task) { m_dataTaskSet.remove(&task); } + +protected: + NetworkSession(WebCore::SessionID); + + WebCore::SessionID m_sessionID; + + HashSet<NetworkDataTask*> m_dataTaskSet; +}; + +} // namespace WebKit + +#endif // USE(NETWORK_SESSION) diff --git a/Source/WebKit2/NetworkProcess/PingLoad.h b/Source/WebKit2/NetworkProcess/PingLoad.h new file mode 100644 index 000000000..300c30812 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/PingLoad.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PingLoad_h +#define PingLoad_h + +#include "AuthenticationManager.h" +#include "NetworkDataTask.h" +#include "SessionTracker.h" + +namespace WebKit { + +class PingLoad final : private NetworkDataTaskClient { +public: + PingLoad(const NetworkResourceLoadParameters& parameters) + : m_timeoutTimer(*this, &PingLoad::timeoutTimerFired) + , m_shouldFollowRedirects(parameters.shouldFollowRedirects) + { + if (auto* networkSession = SessionTracker::networkSession(parameters.sessionID)) { + m_task = NetworkDataTask::create(*networkSession, *this, parameters); + m_task->resume(); + } else + ASSERT_NOT_REACHED(); + + // If the server never responds, this object will hang around forever. + // Set a very generous timeout, just in case. + m_timeoutTimer.startOneShot(60000); + } + +private: + void willPerformHTTPRedirection(WebCore::ResourceResponse&&, WebCore::ResourceRequest&& request, RedirectCompletionHandler&& completionHandler) final + { + completionHandler(m_shouldFollowRedirects ? request : WebCore::ResourceRequest()); + } + void didReceiveChallenge(const WebCore::AuthenticationChallenge&, ChallengeCompletionHandler&& completionHandler) final + { + completionHandler(AuthenticationChallengeDisposition::Cancel, { }); + delete this; + } + void didReceiveResponseNetworkSession(WebCore::ResourceResponse&&, ResponseCompletionHandler&& completionHandler) final + { + completionHandler(WebCore::PolicyAction::PolicyIgnore); + delete this; + } + void didReceiveData(Ref<WebCore::SharedBuffer>&&) final { ASSERT_NOT_REACHED(); } + void didCompleteWithError(const WebCore::ResourceError&) final { delete this; } + void didSendData(uint64_t totalBytesSent, uint64_t totalBytesExpectedToSend) final { } + void wasBlocked() final { delete this; } + void cannotShowURL() final { delete this; } + + void timeoutTimerFired() { delete this; } + + virtual ~PingLoad() + { + if (m_task) { + ASSERT(m_task->client() == this); + m_task->clearClient(); + m_task->cancel(); + } + } + + RefPtr<NetworkDataTask> m_task; + WebCore::Timer m_timeoutTimer; + bool m_shouldFollowRedirects; +}; + +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/RemoteNetworkingContext.h b/Source/WebKit2/NetworkProcess/RemoteNetworkingContext.h index e3ed12b90..d22fc9d67 100644 --- a/Source/WebKit2/NetworkProcess/RemoteNetworkingContext.h +++ b/Source/WebKit2/NetworkProcess/RemoteNetworkingContext.h @@ -28,50 +28,46 @@ #define RemoteNetworkingContext_h #include <WebCore/NetworkingContext.h> +#include <WebCore/SessionID.h> namespace WebKit { class RemoteNetworkingContext final : public WebCore::NetworkingContext { public: - static PassRefPtr<RemoteNetworkingContext> create(uint64_t sessionID, bool shouldClearReferrerOnHTTPSToHTTPRedirect) + static Ref<RemoteNetworkingContext> create(WebCore::SessionID sessionID, bool shouldClearReferrerOnHTTPSToHTTPRedirect) { - return adoptRef(new RemoteNetworkingContext(sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect)); + return adoptRef(*new RemoteNetworkingContext(sessionID, shouldClearReferrerOnHTTPSToHTTPRedirect)); } virtual ~RemoteNetworkingContext(); // FIXME: Remove platform-specific code and use SessionTracker. - static void ensurePrivateBrowsingSession(uint64_t sessionID); + static void ensurePrivateBrowsingSession(WebCore::SessionID); - virtual bool shouldClearReferrerOnHTTPSToHTTPRedirect() const override { return m_shouldClearReferrerOnHTTPSToHTTPRedirect; } + bool shouldClearReferrerOnHTTPSToHTTPRedirect() const override { return m_shouldClearReferrerOnHTTPSToHTTPRedirect; } private: - RemoteNetworkingContext(uint64_t sessionID, bool shouldClearReferrerOnHTTPSToHTTPRedirect) + RemoteNetworkingContext(WebCore::SessionID sessionID, bool shouldClearReferrerOnHTTPSToHTTPRedirect) : m_sessionID(sessionID) , m_shouldClearReferrerOnHTTPSToHTTPRedirect(shouldClearReferrerOnHTTPSToHTTPRedirect) -#if PLATFORM(MAC) - , m_needsSiteSpecificQuirks(false) - , m_localFileContentSniffingEnabled(false) -#endif - { } + { + } - virtual bool isValid() const override; - virtual WebCore::NetworkStorageSession& storageSession() const override; + bool isValid() const override; + WebCore::NetworkStorageSession& storageSession() const override; -#if PLATFORM(MAC) - void setNeedsSiteSpecificQuirks(bool value) { m_needsSiteSpecificQuirks = value; } - virtual bool needsSiteSpecificQuirks() const override; +#if PLATFORM(COCOA) void setLocalFileContentSniffingEnabled(bool value) { m_localFileContentSniffingEnabled = value; } - virtual bool localFileContentSniffingEnabled() const override; - virtual RetainPtr<CFDataRef> sourceApplicationAuditData() const override; - virtual WebCore::ResourceError blockedError(const WebCore::ResourceRequest&) const override; + bool localFileContentSniffingEnabled() const override; + RetainPtr<CFDataRef> sourceApplicationAuditData() const override; + String sourceApplicationIdentifier() const override; + WebCore::ResourceError blockedError(const WebCore::ResourceRequest&) const override; #endif - uint64_t m_sessionID; + WebCore::SessionID m_sessionID; bool m_shouldClearReferrerOnHTTPSToHTTPRedirect; -#if PLATFORM(MAC) - bool m_needsSiteSpecificQuirks; - bool m_localFileContentSniffingEnabled; +#if PLATFORM(COCOA) + bool m_localFileContentSniffingEnabled = false; #endif }; diff --git a/Source/WebKit2/NetworkProcess/SynchronousNetworkLoaderClient.cpp b/Source/WebKit2/NetworkProcess/SynchronousNetworkLoaderClient.cpp deleted file mode 100644 index 07b148cfc..000000000 --- a/Source/WebKit2/NetworkProcess/SynchronousNetworkLoaderClient.cpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2013 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "SynchronousNetworkLoaderClient.h" - -#if ENABLE(NETWORK_PROCESS) - -#include "DataReference.h" -#include "NetworkResourceLoader.h" -#include "WebErrors.h" -#include <WebCore/ResourceRequest.h> -#include <WebCore/SharedBuffer.h> -#include <WebCore/SynchronousLoaderClient.h> - -using namespace WebCore; - -namespace WebKit { - -SynchronousNetworkLoaderClient::SynchronousNetworkLoaderClient(const ResourceRequest& request, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> reply) - : m_originalRequest(request) - , m_delayedReply(reply) -{ - ASSERT(m_delayedReply); -} - -SynchronousNetworkLoaderClient::~SynchronousNetworkLoaderClient() -{ - // By the time a SynchronousNetworkLoaderClient is being destroyed, it must always have sent its reply to the WebProcess. - ASSERT(!m_delayedReply); -} - -void SynchronousNetworkLoaderClient::willSendRequest(NetworkResourceLoader* loader, ResourceRequest& proposedRequest, const ResourceResponse& redirectResponse) -{ - // FIXME: This needs to be fixed to follow the redirect correctly even for cross-domain requests. - // This includes at least updating host records, and comparing the current request instead of the original request here. - if (!protocolHostAndPortAreEqual(m_originalRequest.url(), proposedRequest.url())) { - ASSERT(m_error.isNull()); - m_error = SynchronousLoaderClient::platformBadResponseError(); - proposedRequest = ResourceRequest(); - } - - m_currentRequest = proposedRequest; - loader->continueWillSendRequest(m_currentRequest); -} - -#if USE(PROTECTION_SPACE_AUTH_CALLBACK) -void SynchronousNetworkLoaderClient::canAuthenticateAgainstProtectionSpace(NetworkResourceLoader* loader, const ProtectionSpace&) -{ - // FIXME: We should ask the WebProcess like the asynchronous case below does. - // This is currently impossible as the WebProcess is blocked waiting on this synchronous load. - // It's possible that we can jump straight to the UI process to resolve this. - loader->continueCanAuthenticateAgainstProtectionSpace(true); -} -#endif - -void SynchronousNetworkLoaderClient::didReceiveResponse(NetworkResourceLoader*, const ResourceResponse& response) -{ - m_response = response; -} - -void SynchronousNetworkLoaderClient::didReceiveBuffer(NetworkResourceLoader*, SharedBuffer* buffer, int encodedDataLength) -{ - // FIXME: There's a potential performance improvement here by preallocating a SharedMemory region - // of the expected content length to avoid a copy when we send it to the WebProcess on completion. - // It's unclear if the potential complexities of that approach are worth it. - - if (!m_responseData) - m_responseData = adoptPtr(new Vector<char>); - - m_responseData->append(buffer->data(), buffer->size()); -} - -void SynchronousNetworkLoaderClient::didFinishLoading(NetworkResourceLoader*, double /* finishTime */) -{ - sendDelayedReply(); -} - -void SynchronousNetworkLoaderClient::didFail(NetworkResourceLoader*, const ResourceError& error) -{ - m_error = error; - sendDelayedReply(); -} - -void SynchronousNetworkLoaderClient::sendDelayedReply() -{ - ASSERT(m_delayedReply); - - if (m_response.isNull()) { - ASSERT(!m_error.isNull()); - //platformSynthesizeErrorResponse(); - } - - m_delayedReply->send(m_error, m_response, m_responseData ? *m_responseData : Vector<char>()); - m_delayedReply = nullptr; -} - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/SynchronousNetworkLoaderClient.h b/Source/WebKit2/NetworkProcess/SynchronousNetworkLoaderClient.h deleted file mode 100644 index 9470678e9..000000000 --- a/Source/WebKit2/NetworkProcess/SynchronousNetworkLoaderClient.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2013 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef SynchronousNetworkLoaderClient_h -#define SynchronousNetworkLoaderClient_h - -#include "NetworkConnectionToWebProcessMessages.h" -#include "NetworkLoaderClient.h" -#include <WebCore/ResourceError.h> -#include <WebCore/ResourceRequest.h> -#include <WebCore/ResourceResponse.h> -#include <wtf/RefCounted.h> - -#if ENABLE(NETWORK_PROCESS) - -namespace WebCore { -class SharedBuffer; -} - -namespace WebKit { - -class SynchronousNetworkLoaderClient : public NetworkLoaderClient { -public: - SynchronousNetworkLoaderClient(const WebCore::ResourceRequest& originalRequest, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply>); - virtual ~SynchronousNetworkLoaderClient() override; - - virtual bool isSynchronous() override { return true; } - -private: - virtual void willSendRequest(NetworkResourceLoader*, WebCore::ResourceRequest& proposedRequest, const WebCore::ResourceResponse& redirectResponse) override; -#if USE(PROTECTION_SPACE_AUTH_CALLBACK) - virtual void canAuthenticateAgainstProtectionSpace(NetworkResourceLoader*, const WebCore::ProtectionSpace&) override; -#endif - virtual void didReceiveResponse(NetworkResourceLoader*, const WebCore::ResourceResponse&) override; - virtual void didReceiveBuffer(NetworkResourceLoader*, WebCore::SharedBuffer*, int encodedDataLength) override; - virtual void didSendData(NetworkResourceLoader*, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override { } - virtual void didFinishLoading(NetworkResourceLoader*, double finishTime) override; - virtual void didFail(NetworkResourceLoader*, const WebCore::ResourceError&) override; - - void sendDelayedReply(); - - WebCore::ResourceRequest m_originalRequest; - WebCore::ResourceRequest m_currentRequest; - RefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> m_delayedReply; - WebCore::ResourceResponse m_response; - WebCore::ResourceError m_error; - OwnPtr<Vector<char>> m_responseData; -}; - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) - -#endif // SynchronousNetworkLoaderClient_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp new file mode 100644 index 000000000..bbedce3f1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp @@ -0,0 +1,612 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCache.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheSpeculativeLoadManager.h" +#include "NetworkCacheStatistics.h" +#include "NetworkCacheStorage.h" +#include <WebCore/CacheValidation.h> +#include <WebCore/FileSystem.h> +#include <WebCore/HTTPHeaderNames.h> +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/PlatformCookieJar.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> +#include <wtf/MainThread.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/RunLoop.h> +#include <wtf/text/StringBuilder.h> + +#if PLATFORM(COCOA) +#include <notify.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +static const AtomicString& resourceType() +{ + ASSERT(WTF::isMainThread()); + static NeverDestroyed<const AtomicString> resource("Resource", AtomicString::ConstructFromLiteral); + return resource; +} + +Cache& singleton() +{ + static NeverDestroyed<Cache> instance; + return instance; +} + +#if PLATFORM(GTK) +static void dumpFileChanged(Cache* cache) +{ + cache->dumpContentsToFile(); +} +#endif + +bool Cache::initialize(const String& cachePath, const Parameters& parameters) +{ + m_storage = Storage::open(cachePath); + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (parameters.enableNetworkCacheSpeculativeRevalidation) + m_speculativeLoadManager = std::make_unique<SpeculativeLoadManager>(*m_storage); +#endif + + if (parameters.enableEfficacyLogging) + m_statistics = Statistics::open(cachePath); + +#if PLATFORM(COCOA) + // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump". + if (m_storage) { + int token; + notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) { + dumpContentsToFile(); + }); + } +#endif +#if PLATFORM(GTK) + // Triggers with "touch $cachePath/dump". + if (m_storage) { + CString dumpFilePath = WebCore::fileSystemRepresentation(WebCore::pathByAppendingComponent(m_storage->basePath(), "dump")); + GRefPtr<GFile> dumpFile = adoptGRef(g_file_new_for_path(dumpFilePath.data())); + GFileMonitor* monitor = g_file_monitor_file(dumpFile.get(), G_FILE_MONITOR_NONE, nullptr, nullptr); + g_signal_connect_swapped(monitor, "changed", G_CALLBACK(dumpFileChanged), this); + } +#endif + + LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage); + return !!m_storage; +} + +void Cache::setCapacity(size_t maximumSize) +{ + if (!m_storage) + return; + m_storage->setCapacity(maximumSize); +} + +Key Cache::makeCacheKey(const WebCore::ResourceRequest& request) +{ + // FIXME: This implements minimal Range header disk cache support. We don't parse + // ranges so only the same exact range request will be served from the cache. + String range = request.httpHeaderField(WebCore::HTTPHeaderName::Range); + return { request.cachePartition(), resourceType(), range, request.url().string(), m_storage->salt() }; +} + +static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy) +{ + switch (policy) { + case WebCore::ReturnCacheDataElseLoad: + case WebCore::ReturnCacheDataDontLoad: + return true; + case WebCore::UseProtocolCachePolicy: + case WebCore::ReloadIgnoringCacheData: + case WebCore::RefreshAnyCacheData: + return false; + case WebCore::DoNotUseAnyCache: + ASSERT_NOT_REACHED(); + return false; + } + return false; +} + +static bool responseHasExpired(const WebCore::ResourceResponse& response, std::chrono::system_clock::time_point timestamp, std::optional<std::chrono::microseconds> maxStale) +{ + if (response.cacheControlContainsNoCache()) + return true; + + auto age = WebCore::computeCurrentAge(response, timestamp); + auto lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(response, timestamp); + + auto maximumStaleness = maxStale ? maxStale.value() : 0ms; + bool hasExpired = age - lifetime > maximumStaleness; + +#ifndef LOG_DISABLED + if (hasExpired) + LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasExpired age=%f lifetime=%f max-stale=%g", age, lifetime, maxStale); +#endif + + return hasExpired; +} + +static bool responseNeedsRevalidation(const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& request, std::chrono::system_clock::time_point timestamp) +{ + auto requestDirectives = WebCore::parseCacheControlDirectives(request.httpHeaderFields()); + if (requestDirectives.noCache) + return true; + // For requests we ignore max-age values other than zero. + if (requestDirectives.maxAge && requestDirectives.maxAge.value() == 0ms) + return true; + + return responseHasExpired(response, timestamp, requestDirectives.maxStale); +} + +static UseDecision makeUseDecision(const Entry& entry, const WebCore::ResourceRequest& request) +{ + // The request is conditional so we force revalidation from the network. We merely check the disk cache + // so we can update the cache entry. + if (request.isConditional() && !entry.redirectRequest()) + return UseDecision::Validate; + + if (!WebCore::verifyVaryingRequestHeaders(entry.varyingRequestHeaders(), request)) + return UseDecision::NoDueToVaryingHeaderMismatch; + + // We never revalidate in the case of a history navigation. + if (cachePolicyAllowsExpired(request.cachePolicy())) + return UseDecision::Use; + + if (!responseNeedsRevalidation(entry.response(), request, entry.timeStamp())) + return UseDecision::Use; + + if (!entry.response().hasCacheValidatorFields()) + return UseDecision::NoDueToMissingValidatorFields; + + return entry.redirectRequest() ? UseDecision::NoDueToExpiredRedirect : UseDecision::Validate; +} + +static RetrieveDecision makeRetrieveDecision(const WebCore::ResourceRequest& request) +{ + ASSERT(request.cachePolicy() != WebCore::DoNotUseAnyCache); + + // FIXME: Support HEAD requests. + if (request.httpMethod() != "GET") + return RetrieveDecision::NoDueToHTTPMethod; + if (request.requester() == WebCore::ResourceRequest::Requester::Media) + return RetrieveDecision::NoDueToStreamingMedia; + if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData && !request.isConditional()) + return RetrieveDecision::NoDueToReloadIgnoringCache; + + return RetrieveDecision::Yes; +} + +static bool isMediaMIMEType(const String& mimeType) +{ + if (mimeType.startsWith("video/", /*caseSensitive*/ false)) + return true; + if (mimeType.startsWith("audio/", /*caseSensitive*/ false)) + return true; + return false; +} + +static StoreDecision makeStoreDecision(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response) +{ + if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP()) + return StoreDecision::NoDueToProtocol; + + if (originalRequest.httpMethod() != "GET") + return StoreDecision::NoDueToHTTPMethod; + + auto requestDirectives = WebCore::parseCacheControlDirectives(originalRequest.httpHeaderFields()); + if (requestDirectives.noStore) + return StoreDecision::NoDueToNoStoreRequest; + + if (response.cacheControlContainsNoStore()) + return StoreDecision::NoDueToNoStoreResponse; + + if (!WebCore::isStatusCodeCacheableByDefault(response.httpStatusCode())) { + // http://tools.ietf.org/html/rfc7234#section-4.3.2 + bool hasExpirationHeaders = response.expires() || response.cacheControlMaxAge(); + bool expirationHeadersAllowCaching = WebCore::isStatusCodePotentiallyCacheable(response.httpStatusCode()) && hasExpirationHeaders; + if (!expirationHeadersAllowCaching) + return StoreDecision::NoDueToHTTPStatusCode; + } + + bool isMainResource = originalRequest.requester() == WebCore::ResourceRequest::Requester::Main; + bool storeUnconditionallyForHistoryNavigation = isMainResource || originalRequest.priority() == WebCore::ResourceLoadPriority::VeryHigh; + if (!storeUnconditionallyForHistoryNavigation) { + auto now = std::chrono::system_clock::now(); + bool hasNonZeroLifetime = !response.cacheControlContainsNoCache() && WebCore::computeFreshnessLifetimeForHTTPFamily(response, now) > 0ms; + + bool possiblyReusable = response.hasCacheValidatorFields() || hasNonZeroLifetime; + if (!possiblyReusable) + return StoreDecision::NoDueToUnlikelyToReuse; + } + + // Media loaded via XHR is likely being used for MSE streaming (YouTube and Netflix for example). + // Streaming media fills the cache quickly and is unlikely to be reused. + // FIXME: We should introduce a separate media cache partition that doesn't affect other resources. + // FIXME: We should also make sure make the MSE paths are copy-free so we can use mapped buffers from disk effectively. + auto requester = originalRequest.requester(); + bool isDefinitelyStreamingMedia = requester == WebCore::ResourceRequest::Requester::Media; + bool isLikelyStreamingMedia = requester == WebCore::ResourceRequest::Requester::XHR && isMediaMIMEType(response.mimeType()); + if (isLikelyStreamingMedia || isDefinitelyStreamingMedia) + return StoreDecision::NoDueToStreamingMedia; + + return StoreDecision::Yes; +} + +void Cache::retrieve(const WebCore::ResourceRequest& request, const GlobalFrameID& frameID, Function<void (std::unique_ptr<Entry>)>&& completionHandler) +{ + ASSERT(isEnabled()); + ASSERT(request.url().protocolIsInHTTPFamily()); + + LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %d", request.url().string().ascii().data(), static_cast<int>(request.priority())); + + if (m_statistics) + m_statistics->recordRetrievalRequest(frameID.first); + + Key storageKey = makeCacheKey(request); + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + bool canUseSpeculativeRevalidation = m_speculativeLoadManager && !request.isConditional(); + if (canUseSpeculativeRevalidation) + m_speculativeLoadManager->registerLoad(frameID, request, storageKey); +#endif + + auto retrieveDecision = makeRetrieveDecision(request); + if (retrieveDecision != RetrieveDecision::Yes) { + if (m_statistics) + m_statistics->recordNotUsingCacheForRequest(frameID.first, storageKey, request, retrieveDecision); + + completionHandler(nullptr); + return; + } + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (canUseSpeculativeRevalidation && m_speculativeLoadManager->canRetrieve(storageKey, request, frameID)) { + m_speculativeLoadManager->retrieve(storageKey, [request, completionHandler = WTFMove(completionHandler)](std::unique_ptr<Entry> entry) { + if (entry && WebCore::verifyVaryingRequestHeaders(entry->varyingRequestHeaders(), request)) + completionHandler(WTFMove(entry)); + else + completionHandler(nullptr); + }); + return; + } +#endif + + auto startTime = std::chrono::system_clock::now(); + auto priority = static_cast<unsigned>(request.priority()); + + m_storage->retrieve(storageKey, priority, [this, request, completionHandler = WTFMove(completionHandler), startTime, storageKey, frameID](auto record) { + if (!record) { + LOG(NetworkCache, "(NetworkProcess) not found in storage"); + + if (m_statistics) + m_statistics->recordRetrievalFailure(frameID.first, storageKey, request); + + completionHandler(nullptr); + return false; + } + + ASSERT(record->key == storageKey); + + auto entry = Entry::decodeStorageRecord(*record); + + auto useDecision = entry ? makeUseDecision(*entry, request) : UseDecision::NoDueToDecodeFailure; + switch (useDecision) { + case UseDecision::Use: + break; + case UseDecision::Validate: + entry->setNeedsValidation(true); + break; + default: + entry = nullptr; + }; + +#if !LOG_DISABLED + auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); + LOG(NetworkCache, "(NetworkProcess) retrieve complete useDecision=%d priority=%d time=%" PRIi64 "ms", static_cast<int>(useDecision), static_cast<int>(request.priority()), elapsedMS); +#endif + completionHandler(WTFMove(entry)); + + if (m_statistics) + m_statistics->recordRetrievedCachedEntry(frameID.first, storageKey, request, useDecision); + return useDecision != UseDecision::NoDueToDecodeFailure; + }); +} + + +std::unique_ptr<Entry> Cache::makeEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData) +{ + return std::make_unique<Entry>(makeCacheKey(request), response, WTFMove(responseData), WebCore::collectVaryingRequestHeaders(request, response)); +} + +std::unique_ptr<Entry> Cache::makeRedirectEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest) +{ + return std::make_unique<Entry>(makeCacheKey(request), response, redirectRequest, WebCore::collectVaryingRequestHeaders(request, response)); +} + +std::unique_ptr<Entry> Cache::store(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, Function<void (MappedBody&)>&& completionHandler) +{ + ASSERT(isEnabled()); + ASSERT(responseData); + + LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", request.url().string().latin1().data(), makeCacheKey(request).partition().latin1().data()); + + StoreDecision storeDecision = makeStoreDecision(request, response); + if (storeDecision != StoreDecision::Yes) { + LOG(NetworkCache, "(NetworkProcess) didn't store, storeDecision=%d", static_cast<int>(storeDecision)); + auto key = makeCacheKey(request); + + auto isSuccessfulRevalidation = response.httpStatusCode() == 304; + if (!isSuccessfulRevalidation) { + // Make sure we don't keep a stale entry in the cache. + remove(key); + } + + if (m_statistics) + m_statistics->recordNotCachingResponse(key, storeDecision); + + return nullptr; + } + + auto cacheEntry = makeEntry(request, response, WTFMove(responseData)); + auto record = cacheEntry->encodeAsStorageRecord(); + + m_storage->store(record, [this, completionHandler = WTFMove(completionHandler)](const Data& bodyData) { + MappedBody mappedBody; +#if ENABLE(SHAREABLE_RESOURCE) + if (canUseSharedMemoryForBodyData()) { + if (auto sharedMemory = bodyData.tryCreateSharedMemory()) { + mappedBody.shareableResource = ShareableResource::create(sharedMemory.releaseNonNull(), 0, bodyData.size()); + ASSERT(mappedBody.shareableResource); + mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle); + } + } +#endif + completionHandler(mappedBody); + LOG(NetworkCache, "(NetworkProcess) stored"); + }); + + return cacheEntry; +} + +std::unique_ptr<Entry> Cache::storeRedirect(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest) +{ + ASSERT(isEnabled()); + + LOG(NetworkCache, "(NetworkProcess) storing redirect %s -> %s", request.url().string().latin1().data(), redirectRequest.url().string().latin1().data()); + + StoreDecision storeDecision = makeStoreDecision(request, response); + if (storeDecision != StoreDecision::Yes) { + LOG(NetworkCache, "(NetworkProcess) didn't store redirect, storeDecision=%d", static_cast<int>(storeDecision)); + auto key = makeCacheKey(request); + if (m_statistics) + m_statistics->recordNotCachingResponse(key, storeDecision); + + return nullptr; + } + + auto cacheEntry = makeRedirectEntry(request, response, redirectRequest); + auto record = cacheEntry->encodeAsStorageRecord(); + + m_storage->store(record, nullptr); + + return cacheEntry; +} + +std::unique_ptr<Entry> Cache::update(const WebCore::ResourceRequest& originalRequest, const GlobalFrameID& frameID, const Entry& existingEntry, const WebCore::ResourceResponse& validatingResponse) +{ + LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data()); + + WebCore::ResourceResponse response = existingEntry.response(); + WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse); + + auto updateEntry = std::make_unique<Entry>(existingEntry.key(), response, existingEntry.buffer(), WebCore::collectVaryingRequestHeaders(originalRequest, response)); + auto updateRecord = updateEntry->encodeAsStorageRecord(); + + m_storage->store(updateRecord, { }); + + if (m_statistics) + m_statistics->recordRevalidationSuccess(frameID.first, existingEntry.key(), originalRequest); + + return updateEntry; +} + +void Cache::remove(const Key& key) +{ + ASSERT(isEnabled()); + + m_storage->remove(key); +} + +void Cache::remove(const WebCore::ResourceRequest& request) +{ + remove(makeCacheKey(request)); +} + +void Cache::traverse(Function<void (const TraversalEntry*)>&& traverseHandler) +{ + ASSERT(isEnabled()); + + // Protect against clients making excessive traversal requests. + const unsigned maximumTraverseCount = 3; + if (m_traverseCount >= maximumTraverseCount) { + WTFLogAlways("Maximum parallel cache traverse count exceeded. Ignoring traversal request."); + + RunLoop::main().dispatch([traverseHandler = WTFMove(traverseHandler)] { + traverseHandler(nullptr); + }); + return; + } + + ++m_traverseCount; + + m_storage->traverse(resourceType(), 0, [this, traverseHandler = WTFMove(traverseHandler)](const Storage::Record* record, const Storage::RecordInfo& recordInfo) { + if (!record) { + --m_traverseCount; + traverseHandler(nullptr); + return; + } + + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) + return; + + TraversalEntry traversalEntry { *entry, recordInfo }; + traverseHandler(&traversalEntry); + }); +} + +String Cache::dumpFilePath() const +{ + return WebCore::pathByAppendingComponent(m_storage->versionPath(), "dump.json"); +} + +void Cache::dumpContentsToFile() +{ + if (!m_storage) + return; + auto fd = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite); + if (!WebCore::isHandleValid(fd)) + return; + auto prologue = String("{\n\"entries\": [\n").utf8(); + WebCore::writeToFile(fd, prologue.data(), prologue.length()); + + struct Totals { + unsigned count { 0 }; + double worth { 0 }; + size_t bodySize { 0 }; + }; + Totals totals; + auto flags = Storage::TraverseFlag::ComputeWorth | Storage::TraverseFlag::ShareCount; + size_t capacity = m_storage->capacity(); + m_storage->traverse(resourceType(), flags, [fd, totals, capacity](const Storage::Record* record, const Storage::RecordInfo& info) mutable { + if (!record) { + StringBuilder epilogue; + epilogue.appendLiteral("{}\n],\n"); + epilogue.appendLiteral("\"totals\": {\n"); + epilogue.appendLiteral("\"capacity\": "); + epilogue.appendNumber(capacity); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"count\": "); + epilogue.appendNumber(totals.count); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"bodySize\": "); + epilogue.appendNumber(totals.bodySize); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"averageWorth\": "); + epilogue.appendNumber(totals.count ? totals.worth / totals.count : 0); + epilogue.appendLiteral("\n"); + epilogue.appendLiteral("}\n}\n"); + auto writeData = epilogue.toString().utf8(); + WebCore::writeToFile(fd, writeData.data(), writeData.length()); + WebCore::closeFile(fd); + return; + } + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) + return; + ++totals.count; + totals.worth += info.worth; + totals.bodySize += info.bodySize; + + StringBuilder json; + entry->asJSON(json, info); + json.appendLiteral(",\n"); + auto writeData = json.toString().utf8(); + WebCore::writeToFile(fd, writeData.data(), writeData.length()); + }); +} + +void Cache::deleteDumpFile() +{ + WorkQueue::create("com.apple.WebKit.Cache.delete")->dispatch([path = dumpFilePath().isolatedCopy()] { + WebCore::deleteFile(path); + }); +} + +void Cache::clear(std::chrono::system_clock::time_point modifiedSince, Function<void ()>&& completionHandler) +{ + LOG(NetworkCache, "(NetworkProcess) clearing cache"); + + if (m_statistics) + m_statistics->clear(); + + if (!m_storage) { + RunLoop::main().dispatch(WTFMove(completionHandler)); + return; + } + String anyType; + m_storage->clear(anyType, modifiedSince, WTFMove(completionHandler)); + + deleteDumpFile(); +} + +void Cache::clear() +{ + clear(std::chrono::system_clock::time_point::min(), nullptr); +} + +String Cache::recordsPath() const +{ + return m_storage ? m_storage->recordsPath() : String(); +} + +void Cache::retrieveData(const DataKey& dataKey, Function<void (const uint8_t* data, size_t size)> completionHandler) +{ + ASSERT(isEnabled()); + + Key key { dataKey, m_storage->salt() }; + m_storage->retrieve(key, 4, [completionHandler = WTFMove(completionHandler)] (auto record) { + if (!record || !record->body.size()) { + completionHandler(nullptr, 0); + return true; + } + completionHandler(record->body.data(), record->body.size()); + return true; + }); +} + +void Cache::storeData(const DataKey& dataKey, const uint8_t* data, size_t size) +{ + if (!m_storage) + return; + Key key { dataKey, m_storage->salt() }; + Storage::Record record { key, std::chrono::system_clock::now(), { }, Data { data, size }, { } }; + m_storage->store(record, { }); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.h b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h new file mode 100644 index 000000000..ca2bbc2dc --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCache_h +#define NetworkCache_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheEntry.h" +#include "NetworkCacheStorage.h" +#include "ShareableResource.h" +#include <WebCore/ResourceResponse.h> +#include <wtf/Function.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class ResourceRequest; +class SharedBuffer; +class URL; +} + +namespace WebKit { +namespace NetworkCache { + +class Cache; +class SpeculativeLoadManager; +class Statistics; + +Cache& singleton(); + +struct MappedBody { +#if ENABLE(SHAREABLE_RESOURCE) + RefPtr<ShareableResource> shareableResource; + ShareableResource::Handle shareableResourceHandle; +#endif +}; + +enum class RetrieveDecision { + Yes, + NoDueToHTTPMethod, + NoDueToConditionalRequest, + NoDueToReloadIgnoringCache, + NoDueToStreamingMedia, +}; + +// FIXME: This enum is used in the Statistics code in a way that prevents removing or reordering anything. +enum class StoreDecision { + Yes, + NoDueToProtocol, + NoDueToHTTPMethod, + NoDueToAttachmentResponse, // Unused. + NoDueToNoStoreResponse, + NoDueToHTTPStatusCode, + NoDueToNoStoreRequest, + NoDueToUnlikelyToReuse, + NoDueToStreamingMedia, +}; + +enum class UseDecision { + Use, + Validate, + NoDueToVaryingHeaderMismatch, + NoDueToMissingValidatorFields, + NoDueToDecodeFailure, + NoDueToExpiredRedirect, +}; + +using GlobalFrameID = std::pair<uint64_t /*webPageID*/, uint64_t /*webFrameID*/>; + +class Cache { + WTF_MAKE_NONCOPYABLE(Cache); + friend class WTF::NeverDestroyed<Cache>; +public: + struct Parameters { + bool enableEfficacyLogging; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + bool enableNetworkCacheSpeculativeRevalidation; +#endif + }; + bool initialize(const String& cachePath, const Parameters&); + void setCapacity(size_t); + + bool isEnabled() const { return !!m_storage; } + + // Completion handler may get called back synchronously on failure. + void retrieve(const WebCore::ResourceRequest&, const GlobalFrameID&, Function<void (std::unique_ptr<Entry>)>&&); + std::unique_ptr<Entry> store(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, Function<void (MappedBody&)>&&); + std::unique_ptr<Entry> storeRedirect(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest); + std::unique_ptr<Entry> update(const WebCore::ResourceRequest&, const GlobalFrameID&, const Entry&, const WebCore::ResourceResponse& validatingResponse); + + struct TraversalEntry { + const Entry& entry; + const Storage::RecordInfo& recordInfo; + }; + void traverse(Function<void (const TraversalEntry*)>&&); + void remove(const Key&); + void remove(const WebCore::ResourceRequest&); + + void clear(); + void clear(std::chrono::system_clock::time_point modifiedSince, Function<void ()>&& completionHandler); + + void retrieveData(const DataKey&, Function<void (const uint8_t* data, size_t size)>); + void storeData(const DataKey&, const uint8_t* data, size_t); + + std::unique_ptr<Entry> makeEntry(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&); + std::unique_ptr<Entry> makeRedirectEntry(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest); + + void dumpContentsToFile(); + + String recordsPath() const; + bool canUseSharedMemoryForBodyData() const { return m_storage && m_storage->canUseSharedMemoryForBodyData(); } + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + SpeculativeLoadManager* speculativeLoadManager() { return m_speculativeLoadManager.get(); } +#endif + +private: + Cache() = default; + ~Cache() = delete; + + Key makeCacheKey(const WebCore::ResourceRequest&); + + String dumpFilePath() const; + void deleteDumpFile(); + + std::unique_ptr<Storage> m_storage; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + std::unique_ptr<SpeculativeLoadManager> m_speculativeLoadManager; +#endif + std::unique_ptr<Statistics> m_statistics; + + unsigned m_traverseCount { 0 }; +}; + +} +} +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp new file mode 100644 index 000000000..267caf6a0 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheBlobStorage.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheFileSystem.h" +#include <WebCore/FileSystem.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <wtf/RunLoop.h> +#include <wtf/SHA1.h> + +namespace WebKit { +namespace NetworkCache { + +BlobStorage::BlobStorage(const String& blobDirectoryPath, Salt salt) + : m_blobDirectoryPath(blobDirectoryPath) + , m_salt(salt) +{ +} + +String BlobStorage::blobDirectoryPath() const +{ + return m_blobDirectoryPath.isolatedCopy(); +} + +void BlobStorage::synchronize() +{ + ASSERT(!RunLoop::isMain()); + + WebCore::makeAllDirectories(blobDirectoryPath()); + + m_approximateSize = 0; + auto blobDirectory = blobDirectoryPath(); + traverseDirectory(blobDirectory, [this, &blobDirectory](const String& name, DirectoryEntryType type) { + if (type != DirectoryEntryType::File) + return; + auto path = WebCore::pathByAppendingComponent(blobDirectory, name); + auto filePath = WebCore::fileSystemRepresentation(path); + struct stat stat; + ::stat(filePath.data(), &stat); + // No clients left for this blob. + if (stat.st_nlink == 1) + unlink(filePath.data()); + else + m_approximateSize += stat.st_size; + }); + + LOG(NetworkCacheStorage, "(NetworkProcess) blob synchronization completed approximateSize=%zu", approximateSize()); +} + +String BlobStorage::blobPathForHash(const SHA1::Digest& hash) const +{ + auto hashAsString = SHA1::hexDigest(hash); + return WebCore::pathByAppendingComponent(blobDirectoryPath(), String::fromUTF8(hashAsString)); +} + +BlobStorage::Blob BlobStorage::add(const String& path, const Data& data) +{ + ASSERT(!RunLoop::isMain()); + + auto hash = computeSHA1(data, m_salt); + if (data.isEmpty()) + return { data, hash }; + + auto blobPath = WebCore::fileSystemRepresentation(blobPathForHash(hash)); + auto linkPath = WebCore::fileSystemRepresentation(path); + unlink(linkPath.data()); + + bool blobExists = access(blobPath.data(), F_OK) != -1; + if (blobExists) { + auto existingData = mapFile(blobPath.data()); + if (bytesEqual(existingData, data)) { + if (link(blobPath.data(), linkPath.data()) == -1) + WTFLogAlways("Failed to create hard link from %s to %s", blobPath.data(), linkPath.data()); + return { existingData, hash }; + } + unlink(blobPath.data()); + } + + auto mappedData = data.mapToFile(blobPath.data()); + if (mappedData.isNull()) + return { }; + + if (link(blobPath.data(), linkPath.data()) == -1) + WTFLogAlways("Failed to create hard link from %s to %s", blobPath.data(), linkPath.data()); + + m_approximateSize += mappedData.size(); + + return { mappedData, hash }; +} + +BlobStorage::Blob BlobStorage::get(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + auto data = mapFile(linkPath.data()); + + return { data, computeSHA1(data, m_salt) }; +} + +void BlobStorage::remove(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + unlink(linkPath.data()); +} + +unsigned BlobStorage::shareCount(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + struct stat stat; + if (::stat(linkPath.data(), &stat) < 0) + return 0; + // Link count is 2 in the single client case (the blob file and a link). + return stat.st_nlink - 1; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h new file mode 100644 index 000000000..adf0b085e --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheBlobStorage_h +#define NetworkCacheBlobStorage_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include "NetworkCacheKey.h" +#include <wtf/SHA1.h> + +namespace WebKit { +namespace NetworkCache { + +// BlobStorage deduplicates the data using SHA1 hash computed over the blob bytes. +class BlobStorage { + WTF_MAKE_NONCOPYABLE(BlobStorage); +public: + BlobStorage(const String& blobDirectoryPath, Salt); + + struct Blob { + Data data; + SHA1::Digest hash; + }; + // These are all synchronous and should not be used from the main thread. + Blob add(const String& path, const Data&); + Blob get(const String& path); + + // Blob won't be removed until synchronization. + void remove(const String& path); + + unsigned shareCount(const String& path); + + size_t approximateSize() const { return m_approximateSize; } + + void synchronize(); + +private: + String blobDirectoryPath() const; + String blobPathForHash(const SHA1::Digest&) const; + + const String m_blobDirectoryPath; + const Salt m_salt; + + std::atomic<size_t> m_approximateSize { 0 }; +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp new file mode 100644 index 000000000..8423b31e6 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheCoders.h" + +#if ENABLE(NETWORK_CACHE) + +namespace WTF { +namespace Persistence { + +// Store common HTTP headers as strings instead of using their value in the HTTPHeaderName enumeration +// so that the headers stored in the cache stays valid even after HTTPHeaderName.in gets updated. +void Coder<WebCore::HTTPHeaderMap>::encode(Encoder& encoder, const WebCore::HTTPHeaderMap& headers) +{ + encoder << static_cast<uint64_t>(headers.size()); + for (auto& keyValue : headers) { + encoder << keyValue.key; + encoder << keyValue.value; + } +} + +bool Coder<WebCore::HTTPHeaderMap>::decode(Decoder& decoder, WebCore::HTTPHeaderMap& headers) +{ + uint64_t headersSize; + if (!decoder.decode(headersSize)) + return false; + for (uint64_t i = 0; i < headersSize; ++i) { + String name; + if (!decoder.decode(name)) + return false; + String value; + if (!decoder.decode(value)) + return false; + headers.add(name, value); + } + return true; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h new file mode 100644 index 000000000..8cac8c448 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2010, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/CertificateInfo.h> +#include <WebCore/HTTPHeaderMap.h> +#include <utility> +#include <wtf/Forward.h> +#include <wtf/HashMap.h> +#include <wtf/HashSet.h> +#include <wtf/SHA1.h> +#include <wtf/Vector.h> +#include <wtf/persistence/Coders.h> +#include <wtf/persistence/Decoder.h> +#include <wtf/persistence/Encoder.h> + +namespace WTF { +namespace Persistence { + +template<> struct Coder<WebCore::CertificateInfo> { + static void encode(Encoder&, const WebCore::CertificateInfo&); + static bool decode(Decoder&, WebCore::CertificateInfo&); +}; + +template<> struct Coder<WebCore::HTTPHeaderMap> { + static void encode(Encoder&, const WebCore::HTTPHeaderMap&); + static bool decode(Decoder&, WebCore::HTTPHeaderMap&); +}; + +} +} +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp new file mode 100644 index 000000000..5de5095d3 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersCocoa.cpp @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheCoders.h" + +#if ENABLE(NETWORK_CACHE) + +#if PLATFORM(COCOA) +#include <Security/SecCertificate.h> +#include <Security/SecTrust.h> +#include <wtf/spi/cocoa/SecuritySPI.h> +#endif + +namespace WTF { +namespace Persistence { + +static void encodeCFData(Encoder& encoder, CFDataRef data) +{ + uint64_t length = CFDataGetLength(data); + const uint8_t* bytePtr = CFDataGetBytePtr(data); + + encoder << length; + encoder.encodeFixedLengthData(bytePtr, length); +} + +static bool decodeCFData(Decoder& decoder, RetainPtr<CFDataRef>& data) +{ + uint64_t size = 0; + if (!decoder.decode(size)) + return false; + + Vector<uint8_t> vector(size); + if (!decoder.decodeFixedLengthData(vector.data(), vector.size())) + return false; + + data = adoptCF(CFDataCreate(nullptr, vector.data(), vector.size())); + return true; +} + + +#if HAVE(SEC_TRUST_SERIALIZATION) +static void encodeSecTrustRef(Encoder& encoder, SecTrustRef trust) +{ + auto data = adoptCF(SecTrustSerialize(trust, nullptr)); + if (!data) { + encoder << false; + return; + } + + encoder << true; + encodeCFData(encoder, data.get()); +} + +static bool decodeSecTrustRef(Decoder& decoder, RetainPtr<SecTrustRef>& result) +{ + bool hasTrust; + if (!decoder.decode(hasTrust)) + return false; + + if (!hasTrust) + return true; + + RetainPtr<CFDataRef> trustData; + if (!decodeCFData(decoder, trustData)) + return false; + + auto trust = adoptCF(SecTrustDeserialize(trustData.get(), nullptr)); + if (!trust) + return false; + + result = WTFMove(trust); + return true; +} +#endif + +static void encodeCertificateChain(Encoder& encoder, CFArrayRef certificateChain) +{ + CFIndex size = CFArrayGetCount(certificateChain); + Vector<CFTypeRef, 32> values(size); + + CFArrayGetValues(certificateChain, CFRangeMake(0, size), values.data()); + + encoder << static_cast<uint64_t>(size); + + for (CFIndex i = 0; i < size; ++i) { + ASSERT(values[i]); + ASSERT(CFGetTypeID(values[i]) == SecCertificateGetTypeID()); + + auto data = adoptCF(SecCertificateCopyData((SecCertificateRef)values[i])); + encodeCFData(encoder, data.get()); + } +} + +static bool decodeCertificateChain(Decoder& decoder, RetainPtr<CFArrayRef>& certificateChain) +{ + uint64_t size; + if (!decoder.decode(size)) + return false; + + auto array = adoptCF(CFArrayCreateMutable(0, 0, &kCFTypeArrayCallBacks)); + + for (size_t i = 0; i < size; ++i) { + RetainPtr<CFDataRef> data; + if (!decodeCFData(decoder, data)) + return false; + + auto certificate = adoptCF(SecCertificateCreateWithData(0, data.get())); + CFArrayAppendValue(array.get(), certificate.get()); + } + + certificateChain = WTFMove(array); + return true; +} + +void Coder<WebCore::CertificateInfo>::encode(Encoder& encoder, const WebCore::CertificateInfo& certificateInfo) +{ + encoder.encodeEnum(certificateInfo.type()); + + switch (certificateInfo.type()) { +#if HAVE(SEC_TRUST_SERIALIZATION) + case WebCore::CertificateInfo::Type::Trust: + encodeSecTrustRef(encoder, certificateInfo.trust()); + break; +#endif + case WebCore::CertificateInfo::Type::CertificateChain: { + encodeCertificateChain(encoder, certificateInfo.certificateChain()); + break; + } + case WebCore::CertificateInfo::Type::None: + // Do nothing. + break; + } +} + +bool Coder<WebCore::CertificateInfo>::decode(Decoder& decoder, WebCore::CertificateInfo& certificateInfo) +{ + WebCore::CertificateInfo::Type certificateInfoType; + if (!decoder.decodeEnum(certificateInfoType)) + return false; + + switch (certificateInfoType) { +#if HAVE(SEC_TRUST_SERIALIZATION) + case WebCore::CertificateInfo::Type::Trust: { + RetainPtr<SecTrustRef> trust; + if (!decodeSecTrustRef(decoder, trust)) + return false; + + certificateInfo = WebCore::CertificateInfo(WTFMove(trust)); + return true; + } +#endif + case WebCore::CertificateInfo::Type::CertificateChain: { + RetainPtr<CFArrayRef> certificateChain; + if (!decodeCertificateChain(decoder, certificateChain)) + return false; + + certificateInfo = WebCore::CertificateInfo(WTFMove(certificateChain)); + return true; + } + case WebCore::CertificateInfo::Type::None: + // Do nothing. + break; + } + + return true; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp new file mode 100644 index 000000000..0cef6ae03 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCodersSoup.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheCoders.h" + +#if ENABLE(NETWORK_CACHE) + +namespace WTF { +namespace Persistence { + +void Coder<WebCore::CertificateInfo>::encode(Encoder& encoder, const WebCore::CertificateInfo& certificateInfo) +{ + if (!certificateInfo.certificate()) { + encoder << false; + return; + } + + GByteArray* certificateData = 0; + g_object_get(G_OBJECT(certificateInfo.certificate()), "certificate", &certificateData, NULL); + if (!certificateData) { + encoder << false; + return; + } + + encoder << true; + + GRefPtr<GByteArray> certificate = adoptGRef(certificateData); + encoder << static_cast<uint64_t>(certificate->len); + encoder.encodeFixedLengthData(certificate->data, certificate->len); + + encoder << static_cast<uint32_t>(certificateInfo.tlsErrors()); +} + +bool Coder<WebCore::CertificateInfo>::decode(Decoder& decoder, WebCore::CertificateInfo& certificateInfo) +{ + bool hasCertificate; + if (!decoder.decode(hasCertificate)) + return false; + + if (!hasCertificate) + return true; + + uint64_t size = 0; + if (!decoder.decode(size)) + return false; + + Vector<uint8_t> vector(size); + if (!decoder.decodeFixedLengthData(vector.data(), vector.size())) + return false; + + GByteArray* certificateData = g_byte_array_sized_new(vector.size()); + certificateData = g_byte_array_append(certificateData, vector.data(), vector.size()); + GRefPtr<GByteArray> certificateBytes = adoptGRef(certificateData); + + GTlsBackend* backend = g_tls_backend_get_default(); + GRefPtr<GTlsCertificate> certificate = adoptGRef(G_TLS_CERTIFICATE(g_initable_new( + g_tls_backend_get_certificate_type(backend), 0, 0, "certificate", certificateBytes.get(), nullptr))); + certificateInfo.setCertificate(certificate.get()); + + uint32_t tlsErrors; + if (!decoder.decode(tlsErrors)) + return false; + certificateInfo.setTLSErrors(static_cast<GTlsCertificateFlags>(tlsErrors)); + + return true; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp new file mode 100644 index 000000000..31ae48fe5 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheData.h" + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/FileSystem.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <wtf/CryptographicallyRandomNumber.h> + +namespace WebKit { +namespace NetworkCache { + +Data Data::mapToFile(const char* path) const +{ + int fd = open(path, O_CREAT | O_EXCL | O_RDWR , S_IRUSR | S_IWUSR); + if (fd < 0) + return { }; + + if (ftruncate(fd, m_size) < 0) { + close(fd); + return { }; + } + + void* map = mmap(nullptr, m_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + close(fd); + return { }; + } + + uint8_t* mapData = static_cast<uint8_t*>(map); + apply([&mapData](const uint8_t* bytes, size_t bytesSize) { + memcpy(mapData, bytes, bytesSize); + mapData += bytesSize; + return true; + }); + + // Drop the write permission. + mprotect(map, m_size, PROT_READ); + + // Flush (asynchronously) to file, turning this into clean memory. + msync(map, m_size, MS_ASYNC); + + return Data::adoptMap(map, m_size, fd); +} + +Data mapFile(const char* path) +{ + int fd = open(path, O_RDONLY, 0); + if (fd < 0) + return { }; + struct stat stat; + if (fstat(fd, &stat) < 0) { + close(fd); + return { }; + } + size_t size = stat.st_size; + if (!size) { + close(fd); + return Data::empty(); + } + + return adoptAndMapFile(fd, 0, size); +} + +Data adoptAndMapFile(int fd, size_t offset, size_t size) +{ + if (!size) { + close(fd); + return Data::empty(); + } + + void* map = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, offset); + if (map == MAP_FAILED) { + close(fd); + return { }; + } + + return Data::adoptMap(map, size, fd); +} + +SHA1::Digest computeSHA1(const Data& data, const Salt& salt) +{ + SHA1 sha1; + sha1.addBytes(salt.data(), salt.size()); + data.apply([&sha1](const uint8_t* data, size_t size) { + sha1.addBytes(data, size); + return true; + }); + + SHA1::Digest digest; + sha1.computeHash(digest); + return digest; +} + +bool bytesEqual(const Data& a, const Data& b) +{ + if (a.isNull() || b.isNull()) + return false; + if (a.size() != b.size()) + return false; + return !memcmp(a.data(), b.data(), a.size()); +} + +static Salt makeSalt() +{ + Salt salt; + static_assert(salt.size() == 8, "Salt size"); + *reinterpret_cast<uint32_t*>(&salt[0]) = cryptographicallyRandomNumber(); + *reinterpret_cast<uint32_t*>(&salt[4]) = cryptographicallyRandomNumber(); + return salt; +} + +std::optional<Salt> readOrMakeSalt(const String& path) +{ + auto cpath = WebCore::fileSystemRepresentation(path); + auto fd = open(cpath.data(), O_RDONLY, 0); + Salt salt; + auto bytesRead = read(fd, salt.data(), salt.size()); + close(fd); + if (bytesRead != salt.size()) { + salt = makeSalt(); + + unlink(cpath.data()); + fd = open(cpath.data(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + bool success = write(fd, salt.data(), salt.size()) == salt.size(); + close(fd); + if (!success) + return { }; + } + return salt; +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif // #if ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h new file mode 100644 index 000000000..29c264b4b --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheData_h +#define NetworkCacheData_h + +#if ENABLE(NETWORK_CACHE) + +#include <functional> +#include <wtf/FunctionDispatcher.h> +#include <wtf/SHA1.h> +#include <wtf/ThreadSafeRefCounted.h> +#include <wtf/text/WTFString.h> + +#if USE(SOUP) +#include <WebCore/GRefPtrSoup.h> +#endif + +namespace WebKit { + +class SharedMemory; + +namespace NetworkCache { + +#if PLATFORM(COCOA) +template <typename T> class DispatchPtr; +template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject); + +// FIXME: Use OSObjectPtr instead when it works with dispatch_data_t on all platforms. +template<typename T> class DispatchPtr { +public: + DispatchPtr() + : m_ptr(nullptr) + { + } + explicit DispatchPtr(T ptr) + : m_ptr(ptr) + { + if (m_ptr) + dispatch_retain(m_ptr); + } + DispatchPtr(const DispatchPtr& other) + : m_ptr(other.m_ptr) + { + if (m_ptr) + dispatch_retain(m_ptr); + } + ~DispatchPtr() + { + if (m_ptr) + dispatch_release(m_ptr); + } + + DispatchPtr& operator=(const DispatchPtr& other) + { + auto copy = other; + std::swap(m_ptr, copy.m_ptr); + return *this; + } + + T get() const { return m_ptr; } + explicit operator bool() const { return m_ptr; } + + friend DispatchPtr adoptDispatch<T>(T); + +private: + struct Adopt { }; + DispatchPtr(Adopt, T data) + : m_ptr(data) + { + } + + T m_ptr; +}; + +template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject) +{ + return DispatchPtr<T>(typename DispatchPtr<T>::Adopt { }, dispatchObject); +} +#endif + +class Data { +public: + Data() { } + Data(const uint8_t*, size_t); + + ~Data() { } + + static Data empty(); + static Data adoptMap(void* map, size_t, int fd); + +#if PLATFORM(COCOA) + enum class Backing { Buffer, Map }; + Data(DispatchPtr<dispatch_data_t>, Backing = Backing::Buffer); +#endif +#if USE(SOUP) + Data(GRefPtr<SoupBuffer>&&, int fd = -1); +#endif + bool isNull() const; + bool isEmpty() const { return !m_size; } + + const uint8_t* data() const; + size_t size() const { return m_size; } + bool isMap() const { return m_isMap; } + RefPtr<SharedMemory> tryCreateSharedMemory() const; + + Data subrange(size_t offset, size_t) const; + + bool apply(const Function<bool (const uint8_t*, size_t)>&) const; + + Data mapToFile(const char* path) const; + +#if PLATFORM(COCOA) + dispatch_data_t dispatchData() const { return m_dispatchData.get(); } +#endif + +#if USE(SOUP) + SoupBuffer* soupBuffer() const { return m_buffer.get(); } +#endif +private: +#if PLATFORM(COCOA) + mutable DispatchPtr<dispatch_data_t> m_dispatchData; +#endif +#if USE(SOUP) + mutable GRefPtr<SoupBuffer> m_buffer; + int m_fileDescriptor { -1 }; +#endif + mutable const uint8_t* m_data { nullptr }; + size_t m_size { 0 }; + bool m_isMap { false }; +}; + +Data concatenate(const Data&, const Data&); +bool bytesEqual(const Data&, const Data&); +Data adoptAndMapFile(int fd, size_t offset, size_t); +Data mapFile(const char* path); + +using Salt = std::array<uint8_t, 8>; + +std::optional<Salt> readOrMakeSalt(const String& path); +SHA1::Digest computeSHA1(const Data&, const Salt&); + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp new file mode 100644 index 000000000..21b58fa7e --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2015 Igalia S.L + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheData.h" + +#if ENABLE(NETWORK_CACHE) + +#include "SharedMemory.h" +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +namespace WebKit { +namespace NetworkCache { + +Data::Data(const uint8_t* data, size_t size) + : m_size(size) +{ + uint8_t* copiedData = static_cast<uint8_t*>(fastMalloc(size)); + memcpy(copiedData, data, size); + m_buffer = adoptGRef(soup_buffer_new_with_owner(copiedData, size, copiedData, fastFree)); +} + +Data::Data(GRefPtr<SoupBuffer>&& buffer, int fd) + : m_buffer(buffer) + , m_fileDescriptor(fd) + , m_size(buffer ? buffer->length : 0) + , m_isMap(m_size && fd != -1) +{ +} + +Data Data::empty() +{ + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new(SOUP_MEMORY_TAKE, nullptr, 0)); + return { WTFMove(buffer) }; +} + +const uint8_t* Data::data() const +{ + return m_buffer ? reinterpret_cast<const uint8_t*>(m_buffer->data) : nullptr; +} + +bool Data::isNull() const +{ + return !m_buffer; +} + +bool Data::apply(const Function<bool (const uint8_t*, size_t)>& applier) const +{ + if (!m_size) + return false; + + return applier(reinterpret_cast<const uint8_t*>(m_buffer->data), m_buffer->length); +} + +Data Data::subrange(size_t offset, size_t size) const +{ + if (!m_buffer) + return { }; + + GRefPtr<SoupBuffer> subBuffer = adoptGRef(soup_buffer_new_subbuffer(m_buffer.get(), offset, size)); + return { WTFMove(subBuffer) }; +} + +Data concatenate(const Data& a, const Data& b) +{ + if (a.isNull()) + return b; + if (b.isNull()) + return a; + + size_t size = a.size() + b.size(); + uint8_t* data = static_cast<uint8_t*>(fastMalloc(size)); + memcpy(data, a.soupBuffer()->data, a.soupBuffer()->length); + memcpy(data + a.soupBuffer()->length, b.soupBuffer()->data, b.soupBuffer()->length); + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(data, size, data, fastFree)); + return { WTFMove(buffer) }; +} + +struct MapWrapper { + ~MapWrapper() + { + munmap(map, size); + close(fileDescriptor); + } + + void* map; + size_t size; + int fileDescriptor; +}; + +static void deleteMapWrapper(MapWrapper* wrapper) +{ + delete wrapper; +} + +Data Data::adoptMap(void* map, size_t size, int fd) +{ + ASSERT(map); + ASSERT(map != MAP_FAILED); + MapWrapper* wrapper = new MapWrapper { map, size, fd }; + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(map, size, wrapper, reinterpret_cast<GDestroyNotify>(deleteMapWrapper))); + return { WTFMove(buffer), fd }; +} + +RefPtr<SharedMemory> Data::tryCreateSharedMemory() const +{ + if (isNull() || !isMap()) + return nullptr; + + return SharedMemory::wrapMap(const_cast<char*>(m_buffer->data), m_buffer->length, m_fileDescriptor); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp new file mode 100644 index 000000000..b96bbcd5a --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCache.h" + +#include "Logging.h" +#include "NetworkCacheCoders.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/SharedBuffer.h> +#include <wtf/text/StringBuilder.h> + +#if ENABLE(NETWORK_CACHE) + +namespace WebKit { +namespace NetworkCache { + +Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& buffer, const Vector<std::pair<String, String>>& varyingRequestHeaders) + : m_key(key) + , m_timeStamp(std::chrono::system_clock::now()) + , m_response(response) + , m_varyingRequestHeaders(varyingRequestHeaders) + , m_buffer(WTFMove(buffer)) +{ + ASSERT(m_key.type() == "Resource"); +} + +Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders) + : m_key(key) + , m_timeStamp(std::chrono::system_clock::now()) + , m_response(response) + , m_varyingRequestHeaders(varyingRequestHeaders) +{ + ASSERT(m_key.type() == "Resource"); + + m_redirectRequest.emplace(); + m_redirectRequest->setAsIsolatedCopy(redirectRequest); + // Redirect body is not needed even if exists. + m_redirectRequest->setHTTPBody(nullptr); +} + +Entry::Entry(const Entry& other) + : m_key(other.m_key) + , m_timeStamp(other.m_timeStamp) + , m_response(other.m_response) + , m_varyingRequestHeaders(other.m_varyingRequestHeaders) + , m_redirectRequest(other.m_redirectRequest) + , m_buffer(other.m_buffer) + , m_sourceStorageRecord(other.m_sourceStorageRecord) +{ +} + +Entry::Entry(const Storage::Record& storageEntry) + : m_key(storageEntry.key) + , m_timeStamp(storageEntry.timeStamp) + , m_sourceStorageRecord(storageEntry) +{ + ASSERT(m_key.type() == "Resource"); +} + +Storage::Record Entry::encodeAsStorageRecord() const +{ + WTF::Persistence::Encoder encoder; + encoder << m_response; + + bool hasVaryingRequestHeaders = !m_varyingRequestHeaders.isEmpty(); + encoder << hasVaryingRequestHeaders; + if (hasVaryingRequestHeaders) + encoder << m_varyingRequestHeaders; + + bool isRedirect = !!m_redirectRequest; + encoder << isRedirect; + if (isRedirect) + m_redirectRequest->encodeWithoutPlatformData(encoder); + + encoder.encodeChecksum(); + + Data header(encoder.buffer(), encoder.bufferSize()); + Data body; + if (m_buffer) + body = { reinterpret_cast<const uint8_t*>(m_buffer->data()), m_buffer->size() }; + + return { m_key, m_timeStamp, header, body, { } }; +} + +std::unique_ptr<Entry> Entry::decodeStorageRecord(const Storage::Record& storageEntry) +{ + auto entry = std::make_unique<Entry>(storageEntry); + + WTF::Persistence::Decoder decoder(storageEntry.header.data(), storageEntry.header.size()); + if (!decoder.decode(entry->m_response)) + return nullptr; + entry->m_response.setSource(WebCore::ResourceResponse::Source::DiskCache); + if (storageEntry.bodyHash) + entry->m_response.setCacheBodyKey(*storageEntry.bodyHash); + + bool hasVaryingRequestHeaders; + if (!decoder.decode(hasVaryingRequestHeaders)) + return nullptr; + + if (hasVaryingRequestHeaders) { + if (!decoder.decode(entry->m_varyingRequestHeaders)) + return nullptr; + } + + bool isRedirect; + if (!decoder.decode(isRedirect)) + return nullptr; + + if (isRedirect) { + entry->m_redirectRequest.emplace(); + if (!entry->m_redirectRequest->decodeWithoutPlatformData(decoder)) + return nullptr; + } + + if (!decoder.verifyChecksum()) { + LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n"); + return nullptr; + } + + return entry; +} + +#if ENABLE(SHAREABLE_RESOURCE) +void Entry::initializeShareableResourceHandleFromStorageRecord() const +{ + if (!NetworkCache::singleton().canUseSharedMemoryForBodyData()) + return; + + auto sharedMemory = m_sourceStorageRecord.body.tryCreateSharedMemory(); + if (!sharedMemory) + return; + + auto shareableResource = ShareableResource::create(sharedMemory.releaseNonNull(), 0, m_sourceStorageRecord.body.size()); + shareableResource->createHandle(m_shareableResourceHandle); +} +#endif + +void Entry::initializeBufferFromStorageRecord() const +{ +#if ENABLE(SHAREABLE_RESOURCE) + if (!shareableResourceHandle().isNull()) { + m_buffer = m_shareableResourceHandle.tryWrapInSharedBuffer(); + if (m_buffer) + return; + } +#endif + m_buffer = WebCore::SharedBuffer::create(m_sourceStorageRecord.body.data(), m_sourceStorageRecord.body.size()); +} + +WebCore::SharedBuffer* Entry::buffer() const +{ + if (!m_buffer) + initializeBufferFromStorageRecord(); + + return m_buffer.get(); +} + +#if ENABLE(SHAREABLE_RESOURCE) +ShareableResource::Handle& Entry::shareableResourceHandle() const +{ + if (m_shareableResourceHandle.isNull()) + initializeShareableResourceHandleFromStorageRecord(); + + return m_shareableResourceHandle; +} +#endif + +bool Entry::needsValidation() const +{ + return m_response.source() == WebCore::ResourceResponse::Source::DiskCacheAfterValidation; +} + +void Entry::setNeedsValidation(bool value) +{ + m_response.setSource(value ? WebCore::ResourceResponse::Source::DiskCacheAfterValidation : WebCore::ResourceResponse::Source::DiskCache); +} + +void Entry::asJSON(StringBuilder& json, const Storage::RecordInfo& info) const +{ + json.appendLiteral("{\n"); + json.appendLiteral("\"hash\": "); + json.appendQuotedJSONString(m_key.hashAsString()); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodySize\": "); + json.appendNumber(info.bodySize); + json.appendLiteral(",\n"); + json.appendLiteral("\"worth\": "); + json.appendNumber(info.worth); + json.appendLiteral(",\n"); + json.appendLiteral("\"partition\": "); + json.appendQuotedJSONString(m_key.partition()); + json.appendLiteral(",\n"); + json.appendLiteral("\"timestamp\": "); + json.appendNumber(std::chrono::duration_cast<std::chrono::milliseconds>(m_timeStamp.time_since_epoch()).count()); + json.appendLiteral(",\n"); + json.appendLiteral("\"URL\": "); + json.appendQuotedJSONString(m_response.url().string()); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodyHash\": "); + json.appendQuotedJSONString(info.bodyHash); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodyShareCount\": "); + json.appendNumber(info.bodyShareCount); + json.appendLiteral(",\n"); + json.appendLiteral("\"headers\": {\n"); + bool firstHeader = true; + for (auto& header : m_response.httpHeaderFields()) { + if (!firstHeader) + json.appendLiteral(",\n"); + firstHeader = false; + json.appendLiteral(" "); + json.appendQuotedJSONString(header.key); + json.appendLiteral(": "); + json.appendQuotedJSONString(header.value); + } + json.appendLiteral("\n}\n"); + json.appendLiteral("}"); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h new file mode 100644 index 000000000..c03910894 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheEntry_h +#define NetworkCacheEntry_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheStorage.h" +#include "ShareableResource.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <wtf/Noncopyable.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class SharedBuffer; +} + +namespace WebKit { +namespace NetworkCache { + +class Entry { + WTF_MAKE_FAST_ALLOCATED; +public: + Entry(const Key&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, const Vector<std::pair<String, String>>& varyingRequestHeaders); + Entry(const Key&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders); + explicit Entry(const Storage::Record&); + Entry(const Entry&); + + Storage::Record encodeAsStorageRecord() const; + static std::unique_ptr<Entry> decodeStorageRecord(const Storage::Record&); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; } + const WebCore::ResourceResponse& response() const { return m_response; } + const Vector<std::pair<String, String>>& varyingRequestHeaders() const { return m_varyingRequestHeaders; } + + WebCore::SharedBuffer* buffer() const; + const std::optional<WebCore::ResourceRequest>& redirectRequest() const { return m_redirectRequest; } + +#if ENABLE(SHAREABLE_RESOURCE) + ShareableResource::Handle& shareableResourceHandle() const; +#endif + + bool needsValidation() const; + void setNeedsValidation(bool); + + const Storage::Record& sourceStorageRecord() const { return m_sourceStorageRecord; } + + void asJSON(StringBuilder&, const Storage::RecordInfo&) const; + +private: + void initializeBufferFromStorageRecord() const; +#if ENABLE(SHAREABLE_RESOURCE) + void initializeShareableResourceHandleFromStorageRecord() const; +#endif + + Key m_key; + std::chrono::system_clock::time_point m_timeStamp; + WebCore::ResourceResponse m_response; + Vector<std::pair<String, String>> m_varyingRequestHeaders; + + std::optional<WebCore::ResourceRequest> m_redirectRequest; + mutable RefPtr<WebCore::SharedBuffer> m_buffer; +#if ENABLE(SHAREABLE_RESOURCE) + mutable ShareableResource::Handle m_shareableResourceHandle; +#endif + + Storage::Record m_sourceStorageRecord { }; +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp new file mode 100644 index 000000000..52ac56696 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheFileSystem.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include <WebCore/FileSystem.h> +#include <dirent.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <wtf/Assertions.h> +#include <wtf/Function.h> +#include <wtf/text/CString.h> + +#if PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR) +#include <sys/attr.h> +#include <unistd.h> +#endif + +#if USE(SOUP) +#include <gio/gio.h> +#include <wtf/glib/GRefPtr.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +static DirectoryEntryType directoryEntryType(uint8_t dtype) +{ + switch (dtype) { + case DT_DIR: + return DirectoryEntryType::Directory; + case DT_REG: + return DirectoryEntryType::File; + default: + ASSERT_NOT_REACHED(); + return DirectoryEntryType::File; + } +} + +void traverseDirectory(const String& path, const Function<void (const String&, DirectoryEntryType)>& function) +{ + DIR* dir = opendir(WebCore::fileSystemRepresentation(path).data()); + if (!dir) + return; + dirent* dp; + while ((dp = readdir(dir))) { + if (dp->d_type != DT_DIR && dp->d_type != DT_REG) + continue; + const char* name = dp->d_name; + if (!strcmp(name, ".") || !strcmp(name, "..")) + continue; + auto nameString = String::fromUTF8(name); + if (nameString.isNull()) + continue; + function(nameString, directoryEntryType(dp->d_type)); + } + closedir(dir); +} + +void deleteDirectoryRecursively(const String& path) +{ + traverseDirectory(path, [&path](const String& name, DirectoryEntryType type) { + String entryPath = WebCore::pathByAppendingComponent(path, name); + switch (type) { + case DirectoryEntryType::File: + WebCore::deleteFile(entryPath); + break; + case DirectoryEntryType::Directory: + deleteDirectoryRecursively(entryPath); + break; + // This doesn't follow symlinks. + } + }); + WebCore::deleteEmptyDirectory(path); +} + +FileTimes fileTimes(const String& path) +{ +#if HAVE(STAT_BIRTHTIME) + struct stat fileInfo; + if (stat(WebCore::fileSystemRepresentation(path).data(), &fileInfo)) + return { }; + return { std::chrono::system_clock::from_time_t(fileInfo.st_birthtime), std::chrono::system_clock::from_time_t(fileInfo.st_mtime) }; +#elif USE(SOUP) + // There's no st_birthtime in some operating systems like Linux, so we use xattrs to set/get the creation time. + GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(WebCore::fileSystemRepresentation(path).data())); + GRefPtr<GFileInfo> fileInfo = adoptGRef(g_file_query_info(file.get(), "xattr::birthtime,time::modified", G_FILE_QUERY_INFO_NONE, nullptr, nullptr)); + if (!fileInfo) + return { }; + const char* birthtimeString = g_file_info_get_attribute_string(fileInfo.get(), "xattr::birthtime"); + if (!birthtimeString) + return { }; + return { std::chrono::system_clock::from_time_t(g_ascii_strtoull(birthtimeString, nullptr, 10)), + std::chrono::system_clock::from_time_t(g_file_info_get_attribute_uint64(fileInfo.get(), "time::modified")) }; +#endif +} + +void updateFileModificationTimeIfNeeded(const String& path) +{ + auto times = fileTimes(path); + if (times.creation != times.modification) { + // Don't update more than once per hour. + if (std::chrono::system_clock::now() - times.modification < std::chrono::hours(1)) + return; + } + // This really updates both the access time and the modification time. + utimes(WebCore::fileSystemRepresentation(path).data(), nullptr); +} + +bool canUseSharedMemoryForPath(const String& path) +{ +#if PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR) + struct { + uint32_t length; + uint32_t protectionClass; + } attrBuffer; + + attrlist attrList = { }; + attrList.bitmapcount = ATTR_BIT_MAP_COUNT; + attrList.commonattr = ATTR_CMN_DATA_PROTECT_FLAGS; + int32_t error = getattrlist(WebCore::fileSystemRepresentation(path).data(), &attrList, &attrBuffer, sizeof(attrBuffer), FSOPT_NOFOLLOW); + if (error) { + RELEASE_LOG_ERROR(Network, "Unable to get cache directory protection class, disabling use of shared mapped memory"); + return false; + } + + // For stricter protection classes shared maps could disappear when device is locked. + const uint32_t fileProtectionCompleteUntilFirstUserAuthentication = 3; + bool isSafe = attrBuffer.protectionClass >= fileProtectionCompleteUntilFirstUserAuthentication; + + if (!isSafe) + RELEASE_LOG(Network, "Disallowing use of shared mapped memory due to container protection class %u", attrBuffer.protectionClass); + + return isSafe; +#else + UNUSED_PARAM(path); + return true; +#endif +} + +} +} + +#endif // ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h new file mode 100644 index 000000000..22e716f52 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheFileSystem_h +#define NetworkCacheFileSystem_h + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/FileSystem.h> +#include <functional> + +namespace WebKit { +namespace NetworkCache { + +enum class DirectoryEntryType { Directory, File }; +void traverseDirectory(const String& path, const Function<void (const String& fileName, DirectoryEntryType)>&); + +void deleteDirectoryRecursively(const String& path); + +struct FileTimes { + std::chrono::system_clock::time_point creation; + std::chrono::system_clock::time_point modification; +}; +FileTimes fileTimes(const String& path); +void updateFileModificationTimeIfNeeded(const String& path); + +bool canUseSharedMemoryForPath(const String& path); + +} +} + +#endif + +#endif + diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h new file mode 100644 index 000000000..f19db6fa1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheIOChannel_h +#define NetworkCacheIOChannel_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include <functional> +#include <wtf/ThreadSafeRefCounted.h> +#include <wtf/WorkQueue.h> +#include <wtf/text/WTFString.h> + +#if USE(SOUP) +#include <wtf/glib/GRefPtr.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +class IOChannel : public ThreadSafeRefCounted<IOChannel> { +public: + enum class Type { Read, Write, Create }; + static Ref<IOChannel> open(const String& file, Type); + + // Using nullptr as queue submits the result to the main queue. + // FIXME: We should add WorkQueue::main() instead. + void read(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>); + void write(size_t offset, const Data&, WorkQueue*, std::function<void (int error)>); + + const String& path() const { return m_path; } + Type type() const { return m_type; } + + int fileDescriptor() const { return m_fileDescriptor; } + + ~IOChannel(); + +private: + IOChannel(const String& filePath, IOChannel::Type); + +#if USE(SOUP) + void readSyncInThread(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>); +#endif + + String m_path; + Type m_type; + + int m_fileDescriptor { 0 }; + std::atomic<bool> m_wasDeleted { false }; // Try to narrow down a crash, https://bugs.webkit.org/show_bug.cgi?id=165659 +#if PLATFORM(COCOA) + DispatchPtr<dispatch_io_t> m_dispatchIO; +#endif +#if USE(SOUP) + GRefPtr<GInputStream> m_inputStream; + GRefPtr<GOutputStream> m_outputStream; + GRefPtr<GFileIOStream> m_ioStream; +#endif +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp new file mode 100644 index 000000000..ef7ba8998 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp @@ -0,0 +1,285 @@ +/* + * Copyright (C) 2015 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheIOChannel.h" + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheFileSystem.h" +#include <wtf/MainThread.h> +#include <wtf/RunLoop.h> +#include <wtf/glib/GUniquePtr.h> + +namespace WebKit { +namespace NetworkCache { + +static const size_t gDefaultReadBufferSize = 4096; + +IOChannel::IOChannel(const String& filePath, Type type) + : m_path(filePath) + , m_type(type) +{ + auto path = WebCore::fileSystemRepresentation(filePath); + GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(path.data())); + switch (m_type) { + case Type::Create: { + g_file_delete(file.get(), nullptr, nullptr); + m_outputStream = adoptGRef(G_OUTPUT_STREAM(g_file_create(file.get(), static_cast<GFileCreateFlags>(G_FILE_CREATE_PRIVATE), nullptr, nullptr))); +#if !HAVE(STAT_BIRTHTIME) + GUniquePtr<char> birthtimeString(g_strdup_printf("%" G_GUINT64_FORMAT, std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()))); + g_file_set_attribute_string(file.get(), "xattr::birthtime", birthtimeString.get(), G_FILE_QUERY_INFO_NONE, nullptr, nullptr); +#endif + break; + } + case Type::Write: { + m_ioStream = adoptGRef(g_file_open_readwrite(file.get(), nullptr, nullptr)); + break; + } + case Type::Read: + m_inputStream = adoptGRef(G_INPUT_STREAM(g_file_read(file.get(), nullptr, nullptr))); + break; + } +} + +IOChannel::~IOChannel() +{ + RELEASE_ASSERT(!m_wasDeleted.exchange(true)); +} + +Ref<IOChannel> IOChannel::open(const String& filePath, IOChannel::Type type) +{ + return adoptRef(*new IOChannel(filePath, type)); +} + +static inline void runTaskInQueue(Function<void ()>&& task, WorkQueue* queue) +{ + if (queue) { + queue->dispatch(WTFMove(task)); + return; + } + + // Using nullptr as queue submits the result to the main context. + RunLoop::main().dispatch(WTFMove(task)); +} + +static void fillDataFromReadBuffer(SoupBuffer* readBuffer, size_t size, Data& data) +{ + GRefPtr<SoupBuffer> buffer; + if (size != readBuffer->length) { + // The subbuffer does not copy the data. + buffer = adoptGRef(soup_buffer_new_subbuffer(readBuffer, 0, size)); + } else + buffer = readBuffer; + + if (data.isNull()) { + // First chunk, we need to force the data to be copied. + data = { reinterpret_cast<const uint8_t*>(buffer->data), size }; + } else { + Data dataRead(WTFMove(buffer)); + // Concatenate will copy the data. + data = concatenate(data, dataRead); + } +} + +struct ReadAsyncData { + RefPtr<IOChannel> channel; + GRefPtr<SoupBuffer> buffer; + RefPtr<WorkQueue> queue; + size_t bytesToRead; + std::function<void (Data&, int error)> completionHandler; + Data data; +}; + +static void inputStreamReadReadyCallback(GInputStream* stream, GAsyncResult* result, gpointer userData) +{ + std::unique_ptr<ReadAsyncData> asyncData(static_cast<ReadAsyncData*>(userData)); + gssize bytesRead = g_input_stream_read_finish(stream, result, nullptr); + if (bytesRead == -1) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(asyncData->data, -1); + }, queue); + return; + } + + if (!bytesRead) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(asyncData->data, 0); + }, queue); + return; + } + + ASSERT(bytesRead > 0); + fillDataFromReadBuffer(asyncData->buffer.get(), static_cast<size_t>(bytesRead), asyncData->data); + + size_t pendingBytesToRead = asyncData->bytesToRead - asyncData->data.size(); + if (!pendingBytesToRead) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(asyncData->data, 0); + }, queue); + return; + } + + size_t bytesToRead = std::min(pendingBytesToRead, asyncData->buffer->length); + // Use a local variable for the data buffer to pass it to g_input_stream_read_async(), because ReadAsyncData is released. + auto data = const_cast<char*>(asyncData->buffer->data); + g_input_stream_read_async(stream, data, bytesToRead, G_PRIORITY_DEFAULT, nullptr, + reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData.release()); +} + +void IOChannel::read(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler) +{ + RefPtr<IOChannel> channel(this); + if (!m_inputStream) { + runTaskInQueue([channel, completionHandler] { + Data data; + completionHandler(data, -1); + }, queue); + return; + } + + if (!isMainThread()) { + readSyncInThread(offset, size, queue, completionHandler); + return; + } + + size_t bufferSize = std::min(size, gDefaultReadBufferSize); + uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize)); + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree)); + ReadAsyncData* asyncData = new ReadAsyncData { this, buffer.get(), queue, size, completionHandler, { } }; + + // FIXME: implement offset. + g_input_stream_read_async(m_inputStream.get(), const_cast<char*>(buffer->data), bufferSize, G_PRIORITY_DEFAULT, nullptr, + reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData); +} + +void IOChannel::readSyncInThread(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler) +{ + ASSERT(!isMainThread()); + + RefPtr<IOChannel> channel(this); + detachThread(createThread("IOChannel::readSync", [channel, size, queue, completionHandler] { + size_t bufferSize = std::min(size, gDefaultReadBufferSize); + uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize)); + GRefPtr<SoupBuffer> readBuffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree)); + Data data; + size_t pendingBytesToRead = size; + size_t bytesToRead = bufferSize; + do { + // FIXME: implement offset. + gssize bytesRead = g_input_stream_read(channel->m_inputStream.get(), const_cast<char*>(readBuffer->data), bytesToRead, nullptr, nullptr); + if (bytesRead == -1) { + runTaskInQueue([channel, completionHandler] { + Data data; + completionHandler(data, -1); + }, queue); + return; + } + + if (!bytesRead) + break; + + ASSERT(bytesRead > 0); + fillDataFromReadBuffer(readBuffer.get(), static_cast<size_t>(bytesRead), data); + + pendingBytesToRead = size - data.size(); + bytesToRead = std::min(pendingBytesToRead, readBuffer->length); + } while (pendingBytesToRead); + + GRefPtr<SoupBuffer> bufferCapture = data.soupBuffer(); + runTaskInQueue([channel, bufferCapture, completionHandler] { + GRefPtr<SoupBuffer> buffer = bufferCapture; + Data data = { WTFMove(buffer) }; + completionHandler(data, 0); + }, queue); + })); +} + +struct WriteAsyncData { + RefPtr<IOChannel> channel; + GRefPtr<SoupBuffer> buffer; + RefPtr<WorkQueue> queue; + std::function<void (int error)> completionHandler; +}; + +static void outputStreamWriteReadyCallback(GOutputStream* stream, GAsyncResult* result, gpointer userData) +{ + std::unique_ptr<WriteAsyncData> asyncData(static_cast<WriteAsyncData*>(userData)); + gssize bytesWritten = g_output_stream_write_finish(stream, result, nullptr); + if (bytesWritten == -1) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(-1); + }, queue); + return; + } + + gssize pendingBytesToWrite = asyncData->buffer->length - bytesWritten; + if (!pendingBytesToWrite) { + WorkQueue* queue = asyncData->queue.get(); + runTaskInQueue([asyncData = WTFMove(asyncData)] { + asyncData->completionHandler(0); + }, queue); + return; + } + + asyncData->buffer = adoptGRef(soup_buffer_new_subbuffer(asyncData->buffer.get(), bytesWritten, pendingBytesToWrite)); + // Use a local variable for the data buffer to pass it to g_output_stream_write_async(), because WriteAsyncData is released. + auto data = asyncData->buffer->data; + g_output_stream_write_async(stream, data, pendingBytesToWrite, G_PRIORITY_DEFAULT_IDLE, nullptr, + reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData.release()); +} + +void IOChannel::write(size_t offset, const Data& data, WorkQueue* queue, std::function<void (int error)> completionHandler) +{ + RefPtr<IOChannel> channel(this); + if (!m_outputStream && !m_ioStream) { + runTaskInQueue([channel, completionHandler] { + completionHandler(-1); + }, queue); + return; + } + + GOutputStream* stream = m_outputStream ? m_outputStream.get() : g_io_stream_get_output_stream(G_IO_STREAM(m_ioStream.get())); + if (!stream) { + runTaskInQueue([channel, completionHandler] { + completionHandler(-1); + }, queue); + return; + } + + WriteAsyncData* asyncData = new WriteAsyncData { this, data.soupBuffer(), queue, completionHandler }; + // FIXME: implement offset. + g_output_stream_write_async(stream, asyncData->buffer->data, data.size(), G_PRIORITY_DEFAULT_IDLE, nullptr, + reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp new file mode 100644 index 000000000..cce50a7d0 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheKey.h" + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheCoders.h" +#include <wtf/ASCIICType.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/persistence/Decoder.h> +#include <wtf/persistence/Encoder.h> +#include <wtf/text/CString.h> +#include <wtf/text/StringBuilder.h> + +namespace WebKit { +namespace NetworkCache { + +Key::Key(const Key& o) + : m_partition(o.m_partition.isolatedCopy()) + , m_type(o.m_type.isolatedCopy()) + , m_identifier(o.m_identifier.isolatedCopy()) + , m_range(o.m_range.isolatedCopy()) + , m_hash(o.m_hash) + , m_partitionHash(o.m_partitionHash) +{ +} + +Key::Key(const String& partition, const String& type, const String& range, const String& identifier, const Salt& salt) + : m_partition(partition) + , m_type(type) + , m_identifier(identifier) + , m_range(range) + , m_hash(computeHash(salt)) + , m_partitionHash(computePartitionHash(salt)) +{ +} + +Key::Key(WTF::HashTableDeletedValueType) + : m_identifier(WTF::HashTableDeletedValue) +{ +} + +Key::Key(const DataKey& dataKey, const Salt& salt) + : m_partition(dataKey.partition) + , m_type(dataKey.type) + , m_identifier(hashAsString(dataKey.identifier)) + , m_hash(computeHash(salt)) + , m_partitionHash(computePartitionHash(salt)) +{ +} + +Key& Key::operator=(const Key& other) +{ + m_partition = other.m_partition.isolatedCopy(); + m_type = other.m_type.isolatedCopy(); + m_identifier = other.m_identifier.isolatedCopy(); + m_range = other.m_range.isolatedCopy(); + m_hash = other.m_hash; + m_partitionHash = other.m_partitionHash; + return *this; +} + +static void hashString(SHA1& sha1, const String& string) +{ + if (string.isNull()) + return; + + if (string.is8Bit() && string.containsOnlyASCII()) { + const uint8_t nullByte = 0; + sha1.addBytes(string.characters8(), string.length()); + sha1.addBytes(&nullByte, 1); + return; + } + auto cString = string.utf8(); + // Include terminating null byte. + sha1.addBytes(reinterpret_cast<const uint8_t*>(cString.data()), cString.length() + 1); +} + +Key::HashType Key::computeHash(const Salt& salt) const +{ + // We don't really need a cryptographic hash. The key is always verified against the entry header. + // SHA1 just happens to be suitably sized, fast and available. + SHA1 sha1; + sha1.addBytes(salt.data(), salt.size()); + + hashString(sha1, m_partition); + hashString(sha1, m_type); + hashString(sha1, m_identifier); + hashString(sha1, m_range); + + SHA1::Digest hash; + sha1.computeHash(hash); + return hash; +} + +Key::HashType Key::computePartitionHash(const Salt& salt) const +{ + SHA1 sha1; + sha1.addBytes(salt.data(), salt.size()); + + hashString(sha1, m_partition); + + SHA1::Digest hash; + sha1.computeHash(hash); + return hash; +} + +String Key::hashAsString(const HashType& hash) +{ + StringBuilder builder; + builder.reserveCapacity(hashStringLength()); + for (auto byte : hash) { + builder.append(upperNibbleToASCIIHexDigit(byte)); + builder.append(lowerNibbleToASCIIHexDigit(byte)); + } + return builder.toString(); +} + +template <typename CharType> bool hexDigitsToHash(CharType* characters, Key::HashType& hash) +{ + for (unsigned i = 0; i < sizeof(hash); ++i) { + auto high = characters[2 * i]; + auto low = characters[2 * i + 1]; + if (!isASCIIHexDigit(high) || !isASCIIHexDigit(low)) + return false; + hash[i] = toASCIIHexValue(high, low); + } + return true; +} + +bool Key::stringToHash(const String& string, HashType& hash) +{ + if (string.length() != hashStringLength()) + return false; + if (string.is8Bit()) + return hexDigitsToHash(string.characters8(), hash); + return hexDigitsToHash(string.characters16(), hash); +} + +bool Key::operator==(const Key& other) const +{ + return m_hash == other.m_hash && m_partition == other.m_partition && m_type == other.m_type && m_identifier == other.m_identifier && m_range == other.m_range; +} + +void Key::encode(WTF::Persistence::Encoder& encoder) const +{ + encoder << m_partition; + encoder << m_type; + encoder << m_identifier; + encoder << m_range; + encoder << m_hash; + encoder << m_partitionHash; +} + +bool Key::decode(WTF::Persistence::Decoder& decoder, Key& key) +{ + return decoder.decode(key.m_partition) && decoder.decode(key.m_type) && decoder.decode(key.m_identifier) && decoder.decode(key.m_range) && decoder.decode(key.m_hash) && decoder.decode(key.m_partitionHash); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h new file mode 100644 index 000000000..439ebf0f7 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheKey_h +#define NetworkCacheKey_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include <wtf/SHA1.h> +#include <wtf/persistence/Coder.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { +namespace NetworkCache { + +struct DataKey { + String partition; + String type; + SHA1::Digest identifier; + + template <class Encoder> void encode(Encoder& encoder) const + { + encoder << partition << type << identifier; + } + + template <class Decoder> static bool decode(Decoder& decoder, DataKey& dataKey) + { + return decoder.decode(dataKey.partition) && decoder.decode(dataKey.type) && decoder.decode(dataKey.identifier); + } +}; + +class Key { +public: + typedef SHA1::Digest HashType; + + Key() { } + Key(const Key&); + Key(Key&&) = default; + Key(const String& partition, const String& type, const String& range, const String& identifier, const Salt&); + Key(const DataKey&, const Salt&); + + Key& operator=(const Key&); + Key& operator=(Key&&) = default; + + Key(WTF::HashTableDeletedValueType); + bool isHashTableDeletedValue() const { return m_identifier.isHashTableDeletedValue(); } + + bool isNull() const { return m_identifier.isNull(); } + + const String& partition() const { return m_partition; } + const String& identifier() const { return m_identifier; } + const String& type() const { return m_type; } + const String& range() const { return m_range; } + + const HashType& hash() const { return m_hash; } + const HashType& partitionHash() const { return m_partitionHash; } + + static bool stringToHash(const String&, HashType&); + + static size_t hashStringLength() { return 2 * sizeof(m_hash); } + String hashAsString() const { return hashAsString(m_hash); } + String partitionHashAsString() const { return hashAsString(m_partitionHash); } + + void encode(WTF::Persistence::Encoder&) const; + static bool decode(WTF::Persistence::Decoder&, Key&); + + bool operator==(const Key&) const; + bool operator!=(const Key& other) const { return !(*this == other); } + +private: + static String hashAsString(const HashType&); + HashType computeHash(const Salt&) const; + HashType computePartitionHash(const Salt&) const; + + String m_partition; + String m_type; + String m_identifier; + String m_range; + HashType m_hash; + HashType m_partitionHash; +}; + +} +} + +namespace WTF { + +struct NetworkCacheKeyHash { + static unsigned hash(const WebKit::NetworkCache::Key& key) + { + static_assert(SHA1::hashSize >= sizeof(unsigned), "Hash size must be greater than sizeof(unsigned)"); + return *reinterpret_cast<const unsigned*>(key.hash().data()); + } + + static bool equal(const WebKit::NetworkCache::Key& a, const WebKit::NetworkCache::Key& b) + { + return a == b; + } + + static const bool safeToCompareToEmptyOrDeleted = false; +}; + +template<typename T> struct DefaultHash; +template<> struct DefaultHash<WebKit::NetworkCache::Key> { + typedef NetworkCacheKeyHash Hash; +}; + +template<> struct HashTraits<WebKit::NetworkCache::Key> : SimpleClassHashTraits<WebKit::NetworkCache::Key> { + static const bool emptyValueIsZero = false; + + static const bool hasIsEmptyValueFunction = true; + static bool isEmptyValue(const WebKit::NetworkCache::Key& key) { return key.isNull(); } +}; + +} // namespace WTF + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp new file mode 100644 index 000000000..6db85d2de --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSpeculativeLoad.h" + +#include "Logging.h" +#include "NetworkCache.h" +#include "NetworkLoad.h" +#include "NetworkSession.h" +#include <WebCore/SessionID.h> +#include <wtf/CurrentTime.h> +#include <wtf/RunLoop.h> + +namespace WebKit { +namespace NetworkCache { + +using namespace WebCore; + +SpeculativeLoad::SpeculativeLoad(const GlobalFrameID& frameID, const ResourceRequest& request, std::unique_ptr<NetworkCache::Entry> cacheEntryForValidation, RevalidationCompletionHandler&& completionHandler) + : m_frameID(frameID) + , m_completionHandler(WTFMove(completionHandler)) + , m_originalRequest(request) + , m_bufferedDataForCache(SharedBuffer::create()) + , m_cacheEntry(WTFMove(cacheEntryForValidation)) +{ + ASSERT(!m_cacheEntry || m_cacheEntry->needsValidation()); + + NetworkLoadParameters parameters; + parameters.sessionID = SessionID::defaultSessionID(); + parameters.allowStoredCredentials = AllowStoredCredentials; + parameters.contentSniffingPolicy = DoNotSniffContent; + parameters.request = m_originalRequest; +#if USE(NETWORK_SESSION) + m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters), NetworkSession::defaultSession()); +#else + m_networkLoad = std::make_unique<NetworkLoad>(*this, WTFMove(parameters)); +#endif +} + +SpeculativeLoad::~SpeculativeLoad() +{ + ASSERT(!m_networkLoad); +} + +void SpeculativeLoad::willSendRedirectedRequest(ResourceRequest&& request, ResourceRequest&& redirectRequest, ResourceResponse&& redirectResponse) +{ + LOG(NetworkCacheSpeculativePreloading, "Speculative redirect %s -> %s", request.url().string().utf8().data(), redirectRequest.url().string().utf8().data()); + + m_cacheEntry = NetworkCache::singleton().storeRedirect(request, redirectResponse, redirectRequest); + // Create a synthetic cache entry if we can't store. + if (!m_cacheEntry) + m_cacheEntry = NetworkCache::singleton().makeRedirectEntry(request, redirectResponse, redirectRequest); + + // Don't follow the redirect. The redirect target will be registered for speculative load when it is loaded. + didComplete(); +} + +auto SpeculativeLoad::didReceiveResponse(ResourceResponse&& receivedResponse) -> ShouldContinueDidReceiveResponse +{ + m_response = receivedResponse; + + if (m_response.isMultipart()) + m_bufferedDataForCache = nullptr; + + bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified + if (validationSucceeded && m_cacheEntry) + m_cacheEntry = NetworkCache::singleton().update(m_originalRequest, m_frameID, *m_cacheEntry, m_response); + else + m_cacheEntry = nullptr; + + return ShouldContinueDidReceiveResponse::Yes; +} + +void SpeculativeLoad::didReceiveBuffer(Ref<SharedBuffer>&& buffer, int reportedEncodedDataLength) +{ + ASSERT(!m_cacheEntry); + + if (m_bufferedDataForCache) { + // Prevent memory growth in case of streaming data. + const size_t maximumCacheBufferSize = 10 * 1024 * 1024; + if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize) + m_bufferedDataForCache->append(buffer.get()); + else + m_bufferedDataForCache = nullptr; + } +} + +void SpeculativeLoad::didFinishLoading(double finishTime) +{ + if (m_didComplete) + return; + if (!m_cacheEntry && m_bufferedDataForCache) { + m_cacheEntry = NetworkCache::singleton().store(m_originalRequest, m_response, m_bufferedDataForCache.copyRef(), [](auto& mappedBody) { }); + // Create a synthetic cache entry if we can't store. + if (!m_cacheEntry && isStatusCodeCacheableByDefault(m_response.httpStatusCode())) + m_cacheEntry = NetworkCache::singleton().makeEntry(m_originalRequest, m_response, WTFMove(m_bufferedDataForCache)); + } + + didComplete(); +} + +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) +void SpeculativeLoad::canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) +{ + m_networkLoad->continueCanAuthenticateAgainstProtectionSpace(false); +} +#endif + +void SpeculativeLoad::didFailLoading(const ResourceError&) +{ + if (m_didComplete) + return; + m_cacheEntry = nullptr; + + didComplete(); +} + +void SpeculativeLoad::didComplete() +{ + RELEASE_ASSERT(RunLoop::isMain()); + + if (m_didComplete) + return; + m_didComplete = true; + m_networkLoad = nullptr; + + // Make sure speculatively revalidated resources do not get validated by the NetworkResourceLoader again. + if (m_cacheEntry) + m_cacheEntry->setNeedsValidation(false); + + m_completionHandler(WTFMove(m_cacheEntry)); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h new file mode 100644 index 000000000..3bc2d2e65 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSpeculativeLoad_h +#define NetworkCacheSpeculativeLoad_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCache.h" +#include "NetworkCacheEntry.h" +#include "NetworkLoadClient.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> + +namespace WebKit { + +class NetworkLoad; + +namespace NetworkCache { + +class SpeculativeLoad final : public NetworkLoadClient { + WTF_MAKE_FAST_ALLOCATED; +public: + typedef std::function<void (std::unique_ptr<NetworkCache::Entry>)> RevalidationCompletionHandler; + SpeculativeLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, std::unique_ptr<NetworkCache::Entry>, RevalidationCompletionHandler&&); + + virtual ~SpeculativeLoad(); + + const WebCore::ResourceRequest& originalRequest() const { return m_originalRequest; } + +private: + // NetworkLoadClient. + void didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override { } +#if USE(PROTECTION_SPACE_AUTH_CALLBACK) + void canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) override; +#endif + bool isSynchronous() const override { return false; } + void willSendRedirectedRequest(WebCore::ResourceRequest&&, WebCore::ResourceRequest&& redirectRequest, WebCore::ResourceResponse&& redirectResponse) override; + ShouldContinueDidReceiveResponse didReceiveResponse(WebCore::ResourceResponse&&) override; + void didReceiveBuffer(Ref<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) override; + void didFinishLoading(double finishTime) override; + void didFailLoading(const WebCore::ResourceError&) override; + + void didComplete(); + + GlobalFrameID m_frameID; + RevalidationCompletionHandler m_completionHandler; + WebCore::ResourceRequest m_originalRequest; + + std::unique_ptr<NetworkLoad> m_networkLoad; + + WebCore::ResourceResponse m_response; + + RefPtr<WebCore::SharedBuffer> m_bufferedDataForCache; + std::unique_ptr<NetworkCache::Entry> m_cacheEntry; + bool m_didComplete { false }; +}; + +} // namespace NetworkCache +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#endif // NetworkCacheSpeculativeLoad_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp new file mode 100644 index 000000000..db36bb8eb --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp @@ -0,0 +1,589 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSpeculativeLoadManager.h" + +#include "Logging.h" +#include "NetworkCacheEntry.h" +#include "NetworkCacheSpeculativeLoad.h" +#include "NetworkCacheSubresourcesEntry.h" +#include "NetworkProcess.h" +#include <WebCore/DiagnosticLoggingKeys.h> +#include <WebCore/HysteresisActivity.h> +#include <wtf/HashCountedSet.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/RefCounted.h> +#include <wtf/RunLoop.h> + +namespace WebKit { + +namespace NetworkCache { + +using namespace WebCore; + +static const auto preloadedEntryLifetime = 10s; + +#if !LOG_DISABLED +static HashCountedSet<String>& allSpeculativeLoadingDiagnosticMessages() +{ + static NeverDestroyed<HashCountedSet<String>> messages; + return messages; +} + +static void printSpeculativeLoadingDiagnosticMessageCounts() +{ + LOG(NetworkCacheSpeculativePreloading, "-- Speculative loading statistics --"); + for (auto& pair : allSpeculativeLoadingDiagnosticMessages()) + LOG(NetworkCacheSpeculativePreloading, "%s: %u", pair.key.utf8().data(), pair.value); +} +#endif + +static void logSpeculativeLoadingDiagnosticMessage(const GlobalFrameID& frameID, const String& message) +{ +#if !LOG_DISABLED + if (WebKit2LogNetworkCacheSpeculativePreloading.state == WTFLogChannelOn) + allSpeculativeLoadingDiagnosticMessages().add(message); +#endif + NetworkProcess::singleton().logDiagnosticMessage(frameID.first, WebCore::DiagnosticLoggingKeys::networkCacheKey(), message, WebCore::ShouldSample::Yes); +} + +static const AtomicString& subresourcesType() +{ + ASSERT(RunLoop::isMain()); + static NeverDestroyed<const AtomicString> resource("SubResources", AtomicString::ConstructFromLiteral); + return resource; +} + +static inline Key makeSubresourcesKey(const Key& resourceKey, const Salt& salt) +{ + return Key(resourceKey.partition(), subresourcesType(), resourceKey.range(), resourceKey.identifier(), salt); +} + +static inline ResourceRequest constructRevalidationRequest(const Key& key, const SubresourceInfo& subResourceInfo, const Entry* entry) +{ + ResourceRequest revalidationRequest(key.identifier()); + revalidationRequest.setHTTPHeaderFields(subResourceInfo.requestHeaders()); + revalidationRequest.setFirstPartyForCookies(subResourceInfo.firstPartyForCookies()); + if (!key.partition().isEmpty()) + revalidationRequest.setCachePartition(key.partition()); + ASSERT_WITH_MESSAGE(key.range().isEmpty(), "range is not supported"); + + revalidationRequest.makeUnconditional(); + if (entry) { + String eTag = entry->response().httpHeaderField(HTTPHeaderName::ETag); + if (!eTag.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag); + + String lastModified = entry->response().httpHeaderField(HTTPHeaderName::LastModified); + if (!lastModified.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified); + } + + revalidationRequest.setPriority(subResourceInfo.priority()); + + return revalidationRequest; +} + +static bool responseNeedsRevalidation(const ResourceResponse& response, std::chrono::system_clock::time_point timestamp) +{ + if (response.cacheControlContainsNoCache()) + return true; + + auto age = computeCurrentAge(response, timestamp); + auto lifetime = computeFreshnessLifetimeForHTTPFamily(response, timestamp); + return age - lifetime > 0ms; +} + +class SpeculativeLoadManager::ExpiringEntry { + WTF_MAKE_FAST_ALLOCATED; +public: + explicit ExpiringEntry(std::function<void()>&& expirationHandler) + : m_lifetimeTimer(WTFMove(expirationHandler)) + { + m_lifetimeTimer.startOneShot(preloadedEntryLifetime); + } + +private: + Timer m_lifetimeTimer; +}; + +class SpeculativeLoadManager::PreloadedEntry : private ExpiringEntry { + WTF_MAKE_FAST_ALLOCATED; +public: + PreloadedEntry(std::unique_ptr<Entry> entry, std::optional<ResourceRequest>&& speculativeValidationRequest, std::function<void()>&& lifetimeReachedHandler) + : ExpiringEntry(WTFMove(lifetimeReachedHandler)) + , m_entry(WTFMove(entry)) + , m_speculativeValidationRequest(WTFMove(speculativeValidationRequest)) + { } + + std::unique_ptr<Entry> takeCacheEntry() + { + ASSERT(m_entry); + return WTFMove(m_entry); + } + + const std::optional<ResourceRequest>& revalidationRequest() const { return m_speculativeValidationRequest; } + bool wasRevalidated() const { return !!m_speculativeValidationRequest; } + +private: + std::unique_ptr<Entry> m_entry; + std::optional<ResourceRequest> m_speculativeValidationRequest; +}; + +class SpeculativeLoadManager::PendingFrameLoad : public RefCounted<PendingFrameLoad> { +public: + static Ref<PendingFrameLoad> create(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler) + { + return adoptRef(*new PendingFrameLoad(storage, mainResourceKey, WTFMove(loadCompletionHandler))); + } + + ~PendingFrameLoad() + { + ASSERT(m_didFinishLoad); + ASSERT(m_didRetrieveExistingEntry); + } + + void registerSubresourceLoad(const ResourceRequest& request, const Key& subresourceKey) + { + ASSERT(RunLoop::isMain()); + m_subresourceLoads.append(std::make_unique<SubresourceLoad>(request, subresourceKey)); + m_loadHysteresisActivity.impulse(); + } + + void markLoadAsCompleted() + { + ASSERT(RunLoop::isMain()); + if (m_didFinishLoad) + return; + +#if !LOG_DISABLED + printSpeculativeLoadingDiagnosticMessageCounts(); +#endif + + m_didFinishLoad = true; + saveToDiskIfReady(); + m_loadCompletionHandler(); + } + + void setExistingSubresourcesEntry(std::unique_ptr<SubresourcesEntry> entry) + { + ASSERT(!m_existingEntry); + ASSERT(!m_didRetrieveExistingEntry); + + m_existingEntry = WTFMove(entry); + m_didRetrieveExistingEntry = true; + saveToDiskIfReady(); + } + +private: + PendingFrameLoad(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler) + : m_storage(storage) + , m_mainResourceKey(mainResourceKey) + , m_loadCompletionHandler(WTFMove(loadCompletionHandler)) + , m_loadHysteresisActivity([this](HysteresisState state) { if (state == HysteresisState::Stopped) markLoadAsCompleted(); }) + { + m_loadHysteresisActivity.impulse(); + } + + void saveToDiskIfReady() + { + if (!m_didFinishLoad || !m_didRetrieveExistingEntry) + return; + + if (m_subresourceLoads.isEmpty()) + return; + +#if !LOG_DISABLED + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Saving to disk list of subresources for '%s':", m_mainResourceKey.identifier().utf8().data()); + for (auto& subresourceLoad : m_subresourceLoads) + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) * Subresource: '%s'.", subresourceLoad->key.identifier().utf8().data()); +#endif + + if (m_existingEntry) { + m_existingEntry->updateSubresourceLoads(m_subresourceLoads); + m_storage.store(m_existingEntry->encodeAsStorageRecord(), [](const Data&) { }); + } else { + SubresourcesEntry entry(makeSubresourcesKey(m_mainResourceKey, m_storage.salt()), m_subresourceLoads); + m_storage.store(entry.encodeAsStorageRecord(), [](const Data&) { }); + } + } + + Storage& m_storage; + Key m_mainResourceKey; + Vector<std::unique_ptr<SubresourceLoad>> m_subresourceLoads; + std::function<void()> m_loadCompletionHandler; + HysteresisActivity m_loadHysteresisActivity; + std::unique_ptr<SubresourcesEntry> m_existingEntry; + bool m_didFinishLoad { false }; + bool m_didRetrieveExistingEntry { false }; +}; + +SpeculativeLoadManager::SpeculativeLoadManager(Storage& storage) + : m_storage(storage) +{ +} + +SpeculativeLoadManager::~SpeculativeLoadManager() +{ +} + +#if !LOG_DISABLED + +static void dumpHTTPHeadersDiff(const HTTPHeaderMap& headersA, const HTTPHeaderMap& headersB) +{ + auto aEnd = headersA.end(); + for (auto it = headersA.begin(); it != aEnd; ++it) { + String valueB = headersB.get(it->key); + if (valueB.isNull()) + LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header is only in first request (value: %s)", it->key.utf8().data(), it->value.utf8().data()); + else if (it->value != valueB) + LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header differs in both requests: %s != %s", it->key.utf8().data(), it->value.utf8().data(), valueB.utf8().data()); + } + auto bEnd = headersB.end(); + for (auto it = headersB.begin(); it != bEnd; ++it) { + if (!headersA.contains(it->key)) + LOG(NetworkCacheSpeculativePreloading, "* '%s' HTTP header is only in second request (value: %s)", it->key.utf8().data(), it->value.utf8().data()); + } +} + +#endif + +static bool requestsHeadersMatch(const ResourceRequest& speculativeValidationRequest, const ResourceRequest& actualRequest) +{ + ASSERT(!actualRequest.isConditional()); + ResourceRequest speculativeRequest = speculativeValidationRequest; + speculativeRequest.makeUnconditional(); + + if (speculativeRequest.httpHeaderFields() != actualRequest.httpHeaderFields()) { + LOG(NetworkCacheSpeculativePreloading, "Cannot reuse speculatively validated entry because HTTP headers used for validation do not match"); +#if !LOG_DISABLED + dumpHTTPHeadersDiff(speculativeRequest.httpHeaderFields(), actualRequest.httpHeaderFields()); +#endif + return false; + } + return true; +} + +bool SpeculativeLoadManager::canUsePreloadedEntry(const PreloadedEntry& entry, const ResourceRequest& actualRequest) +{ + if (!entry.wasRevalidated()) + return true; + + ASSERT(entry.revalidationRequest()); + return requestsHeadersMatch(*entry.revalidationRequest(), actualRequest); +} + +bool SpeculativeLoadManager::canUsePendingPreload(const SpeculativeLoad& load, const ResourceRequest& actualRequest) +{ + return requestsHeadersMatch(load.originalRequest(), actualRequest); +} + +bool SpeculativeLoadManager::canRetrieve(const Key& storageKey, const WebCore::ResourceRequest& request, const GlobalFrameID& frameID) const +{ + // Check already preloaded entries. + if (auto preloadedEntry = m_preloadedEntries.get(storageKey)) { + if (!canUsePreloadedEntry(*preloadedEntry, request)) { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: Could not use preloaded entry to satisfy request for '%s' due to HTTP headers mismatch:", storageKey.identifier().utf8().data()); + logSpeculativeLoadingDiagnosticMessage(frameID, preloadedEntry->wasRevalidated() ? DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey() : DiagnosticLoggingKeys::wastedSpeculativeWarmupWithoutRevalidationKey()); + return false; + } + + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: Using preloaded entry to satisfy request for '%s':", storageKey.identifier().utf8().data()); + logSpeculativeLoadingDiagnosticMessage(frameID, preloadedEntry->wasRevalidated() ? DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey() : DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey()); + return true; + } + + // Check pending speculative revalidations. + auto* pendingPreload = m_pendingPreloads.get(storageKey); + if (!pendingPreload) { + if (m_notPreloadedEntries.get(storageKey)) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryWronglyNotWarmedUpKey()); + else + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::unknownEntryRequestKey()); + + return false; + } + + if (!canUsePendingPreload(*pendingPreload, request)) { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: revalidation already in progress for '%s' but unusable due to HTTP headers mismatch:", storageKey.identifier().utf8().data()); + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey()); + return false; + } + + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: revalidation already in progress for '%s':", storageKey.identifier().utf8().data()); + + return true; +} + +void SpeculativeLoadManager::retrieve(const Key& storageKey, RetrieveCompletionHandler&& completionHandler) +{ + if (auto preloadedEntry = m_preloadedEntries.take(storageKey)) { + RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), cacheEntry = preloadedEntry->takeCacheEntry()] () mutable { + completionHandler(WTFMove(cacheEntry)); + }); + return; + } + ASSERT(m_pendingPreloads.contains(storageKey)); + // FIXME: This breaks incremental loading when the revalidation is not successful. + auto addResult = m_pendingRetrieveRequests.ensure(storageKey, [] { + return std::make_unique<Vector<RetrieveCompletionHandler>>(); + }); + addResult.iterator->value->append(WTFMove(completionHandler)); +} + +void SpeculativeLoadManager::registerLoad(const GlobalFrameID& frameID, const ResourceRequest& request, const Key& resourceKey) +{ + ASSERT(RunLoop::isMain()); + ASSERT(request.url().protocolIsInHTTPFamily()); + + if (request.httpMethod() != "GET") + return; + + auto isMainResource = request.requester() == ResourceRequest::Requester::Main; + if (isMainResource) { + // Mark previous load in this frame as completed if necessary. + if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID)) + pendingFrameLoad->markLoadAsCompleted(); + + ASSERT(!m_pendingFrameLoads.contains(frameID)); + + // Start tracking loads in this frame. + RefPtr<PendingFrameLoad> pendingFrameLoad = PendingFrameLoad::create(m_storage, resourceKey, [this, frameID] { + bool wasRemoved = m_pendingFrameLoads.remove(frameID); + ASSERT_UNUSED(wasRemoved, wasRemoved); + }); + m_pendingFrameLoads.add(frameID, pendingFrameLoad); + + // Retrieve the subresources entry if it exists to start speculative revalidation and to update it. + retrieveSubresourcesEntry(resourceKey, [this, frameID, pendingFrameLoad](std::unique_ptr<SubresourcesEntry> entry) { + if (entry) + startSpeculativeRevalidation(frameID, *entry); + + pendingFrameLoad->setExistingSubresourcesEntry(WTFMove(entry)); + }); + return; + } + + if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID)) + pendingFrameLoad->registerSubresourceLoad(request, resourceKey); +} + +void SpeculativeLoadManager::addPreloadedEntry(std::unique_ptr<Entry> entry, const GlobalFrameID& frameID, std::optional<ResourceRequest>&& revalidationRequest) +{ + ASSERT(entry); + ASSERT(!entry->needsValidation()); + auto key = entry->key(); + m_preloadedEntries.add(key, std::make_unique<PreloadedEntry>(WTFMove(entry), WTFMove(revalidationRequest), [this, key, frameID] { + auto preloadedEntry = m_preloadedEntries.take(key); + ASSERT(preloadedEntry); + if (preloadedEntry->wasRevalidated()) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey()); + else + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithoutRevalidationKey()); + })); +} + +void SpeculativeLoadManager::retrieveEntryFromStorage(const SubresourceInfo& info, RetrieveCompletionHandler&& completionHandler) +{ + m_storage.retrieve(info.key(), static_cast<unsigned>(info.priority()), [completionHandler = WTFMove(completionHandler)](auto record) { + if (!record) { + completionHandler(nullptr); + return false; + } + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) { + completionHandler(nullptr); + return false; + } + + auto& response = entry->response(); + if (responseNeedsRevalidation(response, entry->timeStamp())) { + // Do not use cached redirects that have expired. + if (entry->redirectRequest()) { + completionHandler(nullptr); + return true; + } + entry->setNeedsValidation(true); + } + + completionHandler(WTFMove(entry)); + return true; + }); +} + +bool SpeculativeLoadManager::satisfyPendingRequests(const Key& key, Entry* entry) +{ + auto completionHandlers = m_pendingRetrieveRequests.take(key); + if (!completionHandlers) + return false; + + for (auto& completionHandler : *completionHandlers) + completionHandler(entry ? std::make_unique<Entry>(*entry) : nullptr); + + return true; +} + +void SpeculativeLoadManager::revalidateSubresource(const SubresourceInfo& subresourceInfo, std::unique_ptr<Entry> entry, const GlobalFrameID& frameID) +{ + ASSERT(!entry || entry->needsValidation()); + + auto& key = subresourceInfo.key(); + + // Range is not supported. + if (!key.range().isEmpty()) + return; + + ResourceRequest revalidationRequest = constructRevalidationRequest(key, subresourceInfo, entry.get()); + + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculatively revalidating '%s':", key.identifier().utf8().data()); + + auto revalidator = std::make_unique<SpeculativeLoad>(frameID, revalidationRequest, WTFMove(entry), [this, key, revalidationRequest, frameID](std::unique_ptr<Entry> revalidatedEntry) { + ASSERT(!revalidatedEntry || !revalidatedEntry->needsValidation()); + ASSERT(!revalidatedEntry || revalidatedEntry->key() == key); + + auto protectRevalidator = m_pendingPreloads.take(key); + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculative revalidation completed for '%s':", key.identifier().utf8().data()); + + if (satisfyPendingRequests(key, revalidatedEntry.get())) { + if (revalidatedEntry) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey()); + return; + } + + if (revalidatedEntry) + addPreloadedEntry(WTFMove(revalidatedEntry), frameID, revalidationRequest); + }); + m_pendingPreloads.add(key, WTFMove(revalidator)); +} + +static bool canRevalidate(const SubresourceInfo& subresourceInfo, const Entry* entry) +{ + ASSERT(!subresourceInfo.isTransient()); + ASSERT(!entry || entry->needsValidation()); + + if (entry && entry->response().hasCacheValidatorFields()) + return true; + + auto seenAge = subresourceInfo.lastSeen() - subresourceInfo.firstSeen(); + if (seenAge == 0ms) { + LOG(NetworkCacheSpeculativePreloading, "Speculative load: Seen only once"); + return false; + } + + auto now = std::chrono::system_clock::now(); + auto firstSeenAge = now - subresourceInfo.firstSeen(); + auto lastSeenAge = now - subresourceInfo.lastSeen(); + // Sanity check. + if (seenAge <= 0ms || firstSeenAge <= 0ms || lastSeenAge <= 0ms) + return false; + + // Load full resources speculatively if they seem to stay the same. + const auto minimumAgeRatioToLoad = 2. / 3; + const auto recentMinimumAgeRatioToLoad = 1. / 3; + const auto recentThreshold = 5min; + + auto ageRatio = std::chrono::duration_cast<std::chrono::duration<double>>(seenAge) / firstSeenAge; + auto minimumAgeRatio = lastSeenAge > recentThreshold ? minimumAgeRatioToLoad : recentMinimumAgeRatioToLoad; + + LOG(NetworkCacheSpeculativePreloading, "Speculative load: ok=%d ageRatio=%f entry=%d", ageRatio > minimumAgeRatio, ageRatio, !!entry); + + if (ageRatio > minimumAgeRatio) + return true; + + return false; +} + +void SpeculativeLoadManager::preloadEntry(const Key& key, const SubresourceInfo& subresourceInfo, const GlobalFrameID& frameID) +{ + if (m_pendingPreloads.contains(key)) + return; + m_pendingPreloads.add(key, nullptr); + + retrieveEntryFromStorage(subresourceInfo, [this, key, subresourceInfo, frameID](std::unique_ptr<Entry> entry) { + ASSERT(!m_pendingPreloads.get(key)); + bool removed = m_pendingPreloads.remove(key); + ASSERT_UNUSED(removed, removed); + + if (satisfyPendingRequests(key, entry.get())) { + if (entry) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey()); + return; + } + + if (!entry || entry->needsValidation()) { + if (canRevalidate(subresourceInfo, entry.get())) + revalidateSubresource(subresourceInfo, WTFMove(entry), frameID); + return; + } + + addPreloadedEntry(WTFMove(entry), frameID); + }); +} + +void SpeculativeLoadManager::startSpeculativeRevalidation(const GlobalFrameID& frameID, SubresourcesEntry& entry) +{ + for (auto& subresourceInfo : entry.subresources()) { + auto& key = subresourceInfo.key(); + if (!subresourceInfo.isTransient()) + preloadEntry(key, subresourceInfo, frameID); + else { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Not preloading '%s' because it is marked as transient", key.identifier().utf8().data()); + m_notPreloadedEntries.add(key, std::make_unique<ExpiringEntry>([this, key, frameID] { + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryRightlyNotWarmedUpKey()); + m_notPreloadedEntries.remove(key); + })); + } + } +} + +void SpeculativeLoadManager::retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)>&& completionHandler) +{ + ASSERT(storageKey.type() == "Resource"); + auto subresourcesStorageKey = makeSubresourcesKey(storageKey, m_storage.salt()); + m_storage.retrieve(subresourcesStorageKey, static_cast<unsigned>(ResourceLoadPriority::Medium), [completionHandler = WTFMove(completionHandler)](auto record) { + if (!record) { + completionHandler(nullptr); + return false; + } + + auto subresourcesEntry = SubresourcesEntry::decodeStorageRecord(*record); + if (!subresourcesEntry) { + completionHandler(nullptr); + return false; + } + + completionHandler(WTFMove(subresourcesEntry)); + return true; + }); +} + +} // namespace NetworkCache + +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h new file mode 100644 index 000000000..e9c621a91 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSpeculativeLoadManager_h +#define NetworkCacheSpeculativeLoadManager_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCache.h" +#include "NetworkCacheStorage.h" +#include <WebCore/ResourceRequest.h> +#include <wtf/HashMap.h> +#include <wtf/Vector.h> + +namespace WebKit { + +namespace NetworkCache { + +class Entry; +class SpeculativeLoad; +class SubresourceInfo; +class SubresourcesEntry; + +class SpeculativeLoadManager { + WTF_MAKE_FAST_ALLOCATED; +public: + explicit SpeculativeLoadManager(Storage&); + ~SpeculativeLoadManager(); + + void registerLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, const Key& resourceKey); + + typedef Function<void (std::unique_ptr<Entry>)> RetrieveCompletionHandler; + + bool canRetrieve(const Key& storageKey, const WebCore::ResourceRequest&, const GlobalFrameID&) const; + void retrieve(const Key& storageKey, RetrieveCompletionHandler&&); + +private: + class PreloadedEntry; + + void addPreloadedEntry(std::unique_ptr<Entry>, const GlobalFrameID&, std::optional<WebCore::ResourceRequest>&& revalidationRequest = std::nullopt); + void preloadEntry(const Key&, const SubresourceInfo&, const GlobalFrameID&); + void retrieveEntryFromStorage(const SubresourceInfo&, RetrieveCompletionHandler&&); + void revalidateSubresource(const SubresourceInfo&, std::unique_ptr<Entry>, const GlobalFrameID&); + bool satisfyPendingRequests(const Key&, Entry*); + void retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)>&&); + void startSpeculativeRevalidation(const GlobalFrameID&, SubresourcesEntry&); + + static bool canUsePreloadedEntry(const PreloadedEntry&, const WebCore::ResourceRequest& actualRequest); + static bool canUsePendingPreload(const SpeculativeLoad&, const WebCore::ResourceRequest& actualRequest); + + Storage& m_storage; + + class PendingFrameLoad; + HashMap<GlobalFrameID, RefPtr<PendingFrameLoad>> m_pendingFrameLoads; + + HashMap<Key, std::unique_ptr<SpeculativeLoad>> m_pendingPreloads; + HashMap<Key, std::unique_ptr<Vector<RetrieveCompletionHandler>>> m_pendingRetrieveRequests; + + HashMap<Key, std::unique_ptr<PreloadedEntry>> m_preloadedEntries; + + class ExpiringEntry; + HashMap<Key, std::unique_ptr<ExpiringEntry>> m_notPreloadedEntries; // For logging. +}; + +} // namespace NetworkCache + +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#endif // NetworkCacheSpeculativeLoadManager_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp new file mode 100644 index 000000000..0fc8d1d3d --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp @@ -0,0 +1,450 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE) +#include "NetworkCacheStatistics.h" + +#include "Logging.h" +#include "NetworkCache.h" +#include "NetworkCacheFileSystem.h" +#include "NetworkProcess.h" +#include <WebCore/DiagnosticLoggingKeys.h> +#include <WebCore/DiagnosticLoggingResultType.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/SQLiteDatabaseTracker.h> +#include <WebCore/SQLiteStatement.h> +#include <WebCore/SQLiteTransaction.h> +#include <wtf/RunLoop.h> + +namespace WebKit { +namespace NetworkCache { + +static const char* StatisticsDatabaseName = "WebKitCacheStatistics.db"; +static const std::chrono::milliseconds mininumWriteInterval = std::chrono::milliseconds(10000); + +static bool executeSQLCommand(WebCore::SQLiteDatabase& database, const String& sql) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(database.isOpen()); + + bool result = database.executeCommand(sql); + if (!result) + LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", sql.utf8().data(), database.lastErrorMsg()); + + return result; +} + +static bool executeSQLStatement(WebCore::SQLiteStatement& statement) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(statement.database().isOpen()); + + if (statement.step() != SQLITE_DONE) { + LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", statement.query().utf8().data(), statement.database().lastErrorMsg()); + return false; + } + + return true; +} + +std::unique_ptr<Statistics> Statistics::open(const String& cachePath) +{ + ASSERT(RunLoop::isMain()); + + String databasePath = WebCore::pathByAppendingComponent(cachePath, StatisticsDatabaseName); + return std::make_unique<Statistics>(databasePath); +} + +Statistics::Statistics(const String& databasePath) + : m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Statistics.Background", WorkQueue::Type::Serial, WorkQueue::QOS::Background)) + , m_writeTimer(*this, &Statistics::writeTimerFired) +{ + initialize(databasePath); +} + +void Statistics::initialize(const String& databasePath) +{ + ASSERT(RunLoop::isMain()); + + auto startTime = std::chrono::system_clock::now(); + + serialBackgroundIOQueue().dispatch([this, databasePath = databasePath.isolatedCopy(), networkCachePath = singleton().recordsPath().isolatedCopy(), startTime] { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + + if (!WebCore::makeAllDirectories(WebCore::directoryName(databasePath))) + return; + + LOG(NetworkCache, "(NetworkProcess) Opening network cache statistics database at %s...", databasePath.utf8().data()); + m_database.open(databasePath); + m_database.disableThreadingChecks(); + if (!m_database.isOpen()) { + LOG_ERROR("Network cache statistics: Failed to open / create the network cache statistics database"); + return; + } + + executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS AlreadyRequested (hash TEXT PRIMARY KEY)")); + executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS UncachedReason (hash TEXT PRIMARY KEY, reason INTEGER)")); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT count(*) FROM AlreadyRequested")); + if (statement.prepareAndStep() != SQLITE_ROW) { + LOG_ERROR("Network cache statistics: Failed to count the number of rows in AlreadyRequested table"); + return; + } + + m_approximateEntryCount = statement.getColumnInt(0); + +#if !LOG_DISABLED + auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); +#endif + LOG(NetworkCache, "(NetworkProcess) Network cache statistics database load complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS); + + if (!m_approximateEntryCount) { + bootstrapFromNetworkCache(networkCachePath); +#if !LOG_DISABLED + elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); +#endif + LOG(NetworkCache, "(NetworkProcess) Network cache statistics database bootstrapping complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS); + } + }); +} + +void Statistics::bootstrapFromNetworkCache(const String& networkCachePath) +{ + ASSERT(!RunLoop::isMain()); + + LOG(NetworkCache, "(NetworkProcess) Bootstrapping the network cache statistics database from the network cache..."); + + HashSet<String> hashes; + traverseRecordsFiles(networkCachePath, ASCIILiteral("Resource"), [&hashes](const String& fileName, const String& hashString, const String& type, bool isBodyBlob, const String& recordDirectoryPath) { + if (isBodyBlob) + return; + + Key::HashType hash; + if (!Key::stringToHash(hashString, hash)) + return; + + hashes.add(hashString); + }); + + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction writeTransaction(m_database); + writeTransaction.begin(); + + addHashesToDatabase(hashes); + + writeTransaction.commit(); +} + +void Statistics::shrinkIfNeeded() +{ + ASSERT(RunLoop::isMain()); + const size_t maxEntries = 100000; + + if (m_approximateEntryCount < maxEntries) + return; + + LOG(NetworkCache, "(NetworkProcess) shrinking statistics cache m_approximateEntryCount=%lu, maxEntries=%lu", static_cast<size_t>(m_approximateEntryCount), maxEntries); + + clear(); + + serialBackgroundIOQueue().dispatch([this, networkCachePath = singleton().recordsPath().isolatedCopy()] { + bootstrapFromNetworkCache(networkCachePath); + LOG(NetworkCache, "(NetworkProcess) statistics cache shrink completed m_approximateEntryCount=%lu", static_cast<size_t>(m_approximateEntryCount)); + }); +} + +void Statistics::recordRetrievalRequest(uint64_t webPageID) +{ + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalRequestKey(), WebCore::ShouldSample::Yes); +} + +void Statistics::recordNotCachingResponse(const Key& key, StoreDecision storeDecision) +{ + ASSERT(storeDecision != StoreDecision::Yes); + + m_storeDecisionsToAdd.set(key.hashAsString(), storeDecision); + if (!m_writeTimer.isActive()) + m_writeTimer.startOneShot(mininumWriteInterval); +} + +static String retrieveDecisionToDiagnosticKey(RetrieveDecision retrieveDecision) +{ + switch (retrieveDecision) { + case RetrieveDecision::NoDueToHTTPMethod: + return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey(); + case RetrieveDecision::NoDueToConditionalRequest: + return WebCore::DiagnosticLoggingKeys::isConditionalRequestKey(); + case RetrieveDecision::NoDueToReloadIgnoringCache: + return WebCore::DiagnosticLoggingKeys::isReloadIgnoringCacheDataKey(); + case RetrieveDecision::NoDueToStreamingMedia: + return WebCore::DiagnosticLoggingKeys::streamingMedia(); + case RetrieveDecision::Yes: + ASSERT_NOT_REACHED(); + break; + } + return emptyString(); +} + +void Statistics::recordNotUsingCacheForRequest(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, RetrieveDecision retrieveDecision) +{ + ASSERT(retrieveDecision != RetrieveDecision::Yes); + + auto hash = key.hashAsString(); + queryWasEverRequested(hash, NeedUncachedReason::No, [this, hash, requestURL = request.url(), webPageID, retrieveDecision](bool wasEverRequested, const std::optional<StoreDecision>&) { + if (wasEverRequested) { + String diagnosticKey = retrieveDecisionToDiagnosticKey(retrieveDecision); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously requested but we are not using the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheUnusedReasonKey(), diagnosticKey, WebCore::ShouldSample::Yes); + } else { + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheUnusedReasonKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes); + markAsRequested(hash); + } + }); +} + +static String storeDecisionToDiagnosticKey(StoreDecision storeDecision) +{ + switch (storeDecision) { + case StoreDecision::NoDueToProtocol: + return WebCore::DiagnosticLoggingKeys::notHTTPFamilyKey(); + case StoreDecision::NoDueToHTTPMethod: + return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey(); + case StoreDecision::NoDueToAttachmentResponse: + return WebCore::DiagnosticLoggingKeys::isAttachmentKey(); + case StoreDecision::NoDueToNoStoreResponse: + case StoreDecision::NoDueToNoStoreRequest: + return WebCore::DiagnosticLoggingKeys::cacheControlNoStoreKey(); + case StoreDecision::NoDueToHTTPStatusCode: + return WebCore::DiagnosticLoggingKeys::uncacheableStatusCodeKey(); + case StoreDecision::NoDueToUnlikelyToReuse: + return WebCore::DiagnosticLoggingKeys::unlikelyToReuseKey(); + case StoreDecision::NoDueToStreamingMedia: + return WebCore::DiagnosticLoggingKeys::streamingMedia(); + case StoreDecision::Yes: + // It was stored but could not be retrieved so it must have been pruned from the cache. + return WebCore::DiagnosticLoggingKeys::noLongerInCacheKey(); + } + return String(); +} + +void Statistics::recordRetrievalFailure(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request) +{ + auto hash = key.hashAsString(); + queryWasEverRequested(hash, NeedUncachedReason::Yes, [this, hash, requestURL = request.url(), webPageID](bool wasPreviouslyRequested, const std::optional<StoreDecision>& storeDecision) { + if (wasPreviouslyRequested) { + String diagnosticKey = storeDecisionToDiagnosticKey(storeDecision.value()); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously request but is not in the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheFailureReasonKey(), diagnosticKey, WebCore::ShouldSample::Yes); + } else { + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheFailureReasonKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes); + markAsRequested(hash); + } + }); +} + +static String cachedEntryReuseFailureToDiagnosticKey(UseDecision decision) +{ + switch (decision) { + case UseDecision::NoDueToVaryingHeaderMismatch: + return WebCore::DiagnosticLoggingKeys::varyingHeaderMismatchKey(); + case UseDecision::NoDueToMissingValidatorFields: + return WebCore::DiagnosticLoggingKeys::missingValidatorFieldsKey(); + case UseDecision::NoDueToDecodeFailure: + case UseDecision::NoDueToExpiredRedirect: + return WebCore::DiagnosticLoggingKeys::otherKey(); + case UseDecision::Use: + case UseDecision::Validate: + ASSERT_NOT_REACHED(); + break; + } + return emptyString(); +} + +void Statistics::recordRetrievedCachedEntry(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, UseDecision decision) +{ + WebCore::URL requestURL = request.url(); + if (decision == UseDecision::Use) { + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache and is used", webPageID, requestURL.string().ascii().data()); + NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes); + return; + } + + if (decision == UseDecision::Validate) { + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but needs revalidation", webPageID, requestURL.string().ascii().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::needsRevalidationKey(), WebCore::ShouldSample::Yes); + return; + } + + String diagnosticKey = cachedEntryReuseFailureToDiagnosticKey(decision); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but wasn't used, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheReuseFailureKey(), diagnosticKey, WebCore::ShouldSample::Yes); +} + +void Statistics::recordRevalidationSuccess(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request) +{ + WebCore::URL requestURL = request.url(); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was successfully revalidated", webPageID, requestURL.string().ascii().data()); + + NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::revalidatingKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes); +} + +void Statistics::markAsRequested(const String& hash) +{ + ASSERT(RunLoop::isMain()); + + m_hashesToAdd.add(hash); + if (!m_writeTimer.isActive()) + m_writeTimer.startOneShot(mininumWriteInterval); +} + +void Statistics::writeTimerFired() +{ + ASSERT(RunLoop::isMain()); + + serialBackgroundIOQueue().dispatch([this, hashesToAdd = WTFMove(m_hashesToAdd), storeDecisionsToAdd = WTFMove(m_storeDecisionsToAdd)] { + if (!m_database.isOpen()) + return; + + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction writeTransaction(m_database); + writeTransaction.begin(); + + addHashesToDatabase(hashesToAdd); + addStoreDecisionsToDatabase(storeDecisionsToAdd); + + writeTransaction.commit(); + }); + + shrinkIfNeeded(); +} + +void Statistics::queryWasEverRequested(const String& hash, NeedUncachedReason needUncachedReason, RequestedCompletionHandler&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + + // Query pending writes first. + bool wasAlreadyRequested = m_hashesToAdd.contains(hash); + if (wasAlreadyRequested && needUncachedReason == NeedUncachedReason::No) { + completionHandler(true, std::nullopt); + return; + } + if (needUncachedReason == NeedUncachedReason::Yes && m_storeDecisionsToAdd.contains(hash)) { + completionHandler(true, m_storeDecisionsToAdd.get(hash)); + return; + } + + // Query the database. + auto everRequestedQuery = std::make_unique<EverRequestedQuery>(EverRequestedQuery { hash, needUncachedReason == NeedUncachedReason::Yes, WTFMove(completionHandler) }); + auto& query = *everRequestedQuery; + m_activeQueries.add(WTFMove(everRequestedQuery)); + serialBackgroundIOQueue().dispatch([this, wasAlreadyRequested, &query] () mutable { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + std::optional<StoreDecision> storeDecision; + if (m_database.isOpen()) { + if (!wasAlreadyRequested) { + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT hash FROM AlreadyRequested WHERE hash=?")); + if (statement.prepare() == SQLITE_OK) { + statement.bindText(1, query.hash); + wasAlreadyRequested = (statement.step() == SQLITE_ROW); + } + } + if (wasAlreadyRequested && query.needUncachedReason) { + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT reason FROM UncachedReason WHERE hash=?")); + storeDecision = StoreDecision::Yes; + if (statement.prepare() == SQLITE_OK) { + statement.bindText(1, query.hash); + if (statement.step() == SQLITE_ROW) + storeDecision = static_cast<StoreDecision>(statement.getColumnInt(0)); + } + } + } + RunLoop::main().dispatch([this, &query, wasAlreadyRequested, storeDecision] { + query.completionHandler(wasAlreadyRequested, storeDecision); + m_activeQueries.remove(&query); + }); + }); +} + +void Statistics::clear() +{ + ASSERT(RunLoop::isMain()); + + serialBackgroundIOQueue().dispatch([this] { + if (m_database.isOpen()) { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction deleteTransaction(m_database); + deleteTransaction.begin(); + executeSQLCommand(m_database, ASCIILiteral("DELETE FROM AlreadyRequested")); + executeSQLCommand(m_database, ASCIILiteral("DELETE FROM UncachedReason")); + deleteTransaction.commit(); + m_approximateEntryCount = 0; + } + }); +} + +void Statistics::addHashesToDatabase(const HashSet<String>& hashes) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(m_database.isOpen()); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR IGNORE INTO AlreadyRequested (hash) VALUES (?)")); + if (statement.prepare() != SQLITE_OK) + return; + + for (auto& hash : hashes) { + statement.bindText(1, hash); + if (executeSQLStatement(statement)) + ++m_approximateEntryCount; + statement.reset(); + } +} + +void Statistics::addStoreDecisionsToDatabase(const HashMap<String, NetworkCache::StoreDecision>& storeDecisions) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(m_database.isOpen()); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR REPLACE INTO UncachedReason (hash, reason) VALUES (?, ?)")); + if (statement.prepare() != SQLITE_OK) + return; + + for (auto& pair : storeDecisions) { + statement.bindText(1, pair.key); + statement.bindInt(2, static_cast<int>(pair.value)); + executeSQLStatement(statement); + statement.reset(); + } +} + +} +} + +#endif // ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h new file mode 100644 index 000000000..b67c260e1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheStatistics_h +#define NetworkCacheStatistics_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCache.h" +#include "NetworkCacheKey.h" +#include <WebCore/SQLiteDatabase.h> +#include <WebCore/Timer.h> +#include <wtf/WorkQueue.h> + +namespace WebCore { +class ResourceRequest; +} + +namespace WebKit { +namespace NetworkCache { + +class Statistics { +public: + static std::unique_ptr<Statistics> open(const String& cachePath); + explicit Statistics(const String& databasePath); + + void clear(); + + void recordRetrievalRequest(uint64_t webPageID); + void recordNotCachingResponse(const Key&, StoreDecision); + void recordNotUsingCacheForRequest(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, RetrieveDecision); + void recordRetrievalFailure(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&); + void recordRetrievedCachedEntry(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, UseDecision); + void recordRevalidationSuccess(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&); + +private: + WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); } + + void initialize(const String& databasePath); + void bootstrapFromNetworkCache(const String& networkCachePath); + void shrinkIfNeeded(); + + void addHashesToDatabase(const HashSet<String>& hashes); + void addStoreDecisionsToDatabase(const HashMap<String, NetworkCache::StoreDecision>&); + void writeTimerFired(); + + typedef std::function<void (bool wasEverRequested, const std::optional<StoreDecision>&)> RequestedCompletionHandler; + enum class NeedUncachedReason { No, Yes }; + void queryWasEverRequested(const String&, NeedUncachedReason, RequestedCompletionHandler&&); + void markAsRequested(const String& hash); + + struct EverRequestedQuery { + String hash; + bool needUncachedReason; + RequestedCompletionHandler completionHandler; + }; + + std::atomic<size_t> m_approximateEntryCount { 0 }; + + mutable Ref<WorkQueue> m_serialBackgroundIOQueue; + mutable HashSet<std::unique_ptr<const EverRequestedQuery>> m_activeQueries; + WebCore::SQLiteDatabase m_database; + HashSet<String> m_hashesToAdd; + HashMap<String, NetworkCache::StoreDecision> m_storeDecisionsToAdd; + WebCore::Timer m_writeTimer; +}; + +} +} + +#endif // ENABLE(NETWORK_CACHE) + +#endif // NetworkCacheStatistics_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp new file mode 100644 index 000000000..4f272449f --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp @@ -0,0 +1,1048 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheStorage.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheCoders.h" +#include "NetworkCacheFileSystem.h" +#include "NetworkCacheIOChannel.h" +#include <mutex> +#include <wtf/Condition.h> +#include <wtf/Lock.h> +#include <wtf/RandomNumber.h> +#include <wtf/RunLoop.h> +#include <wtf/text/CString.h> + +namespace WebKit { +namespace NetworkCache { + +static const char saltFileName[] = "salt"; +static const char versionDirectoryPrefix[] = "Version "; +static const char recordsDirectoryName[] = "Records"; +static const char blobsDirectoryName[] = "Blobs"; +static const char blobSuffix[] = "-blob"; + +static double computeRecordWorth(FileTimes); + +struct Storage::ReadOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + ReadOperation(const Key& key, RetrieveCompletionHandler&& completionHandler) + : key(key) + , completionHandler(WTFMove(completionHandler)) + { } + + void cancel(); + bool finish(); + + const Key key; + const RetrieveCompletionHandler completionHandler; + + std::unique_ptr<Record> resultRecord; + SHA1::Digest expectedBodyHash; + BlobStorage::Blob resultBodyBlob; + std::atomic<unsigned> activeCount { 0 }; + bool isCanceled { false }; +}; + +void Storage::ReadOperation::cancel() +{ + ASSERT(RunLoop::isMain()); + + if (isCanceled) + return; + isCanceled = true; + completionHandler(nullptr); +} + +bool Storage::ReadOperation::finish() +{ + ASSERT(RunLoop::isMain()); + + if (isCanceled) + return false; + if (resultRecord && resultRecord->body.isNull()) { + if (resultBodyBlob.hash == expectedBodyHash) + resultRecord->body = resultBodyBlob.data; + else + resultRecord = nullptr; + } + return completionHandler(WTFMove(resultRecord)); +} + +struct Storage::WriteOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + WriteOperation(const Record& record, MappedBodyHandler&& mappedBodyHandler) + : record(record) + , mappedBodyHandler(WTFMove(mappedBodyHandler)) + { } + + const Record record; + const MappedBodyHandler mappedBodyHandler; + + std::atomic<unsigned> activeCount { 0 }; +}; + +struct Storage::TraverseOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + TraverseOperation(const String& type, TraverseFlags flags, TraverseHandler&& handler) + : type(type) + , flags(flags) + , handler(WTFMove(handler)) + { } + + const String type; + const TraverseFlags flags; + const TraverseHandler handler; + + Lock activeMutex; + Condition activeCondition; + unsigned activeCount { 0 }; +}; + +static String makeVersionedDirectoryPath(const String& baseDirectoryPath) +{ + String versionSubdirectory = versionDirectoryPrefix + String::number(Storage::version); + return WebCore::pathByAppendingComponent(baseDirectoryPath, versionSubdirectory); +} + +static String makeRecordsDirectoryPath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), recordsDirectoryName); +} + +static String makeBlobDirectoryPath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), blobsDirectoryName); +} + +static String makeSaltFilePath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), saltFileName); +} + +std::unique_ptr<Storage> Storage::open(const String& cachePath) +{ + ASSERT(RunLoop::isMain()); + + if (!WebCore::makeAllDirectories(makeVersionedDirectoryPath(cachePath))) + return nullptr; + auto salt = readOrMakeSalt(makeSaltFilePath(cachePath)); + if (!salt) + return nullptr; + return std::unique_ptr<Storage>(new Storage(cachePath, *salt)); +} + +void traverseRecordsFiles(const String& recordsPath, const String& expectedType, const RecordFileTraverseFunction& function) +{ + traverseDirectory(recordsPath, [&](const String& partitionName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName); + traverseDirectory(partitionPath, [&](const String& actualType, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + if (!expectedType.isEmpty() && expectedType != actualType) + return; + String recordDirectoryPath = WebCore::pathByAppendingComponent(partitionPath, actualType); + traverseDirectory(recordDirectoryPath, [&function, &recordDirectoryPath, &actualType](const String& fileName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::File || fileName.length() < Key::hashStringLength()) + return; + + String hashString = fileName.substring(0, Key::hashStringLength()); + auto isBlob = fileName.length() > Key::hashStringLength() && fileName.endsWith(blobSuffix); + function(fileName, hashString, actualType, isBlob, recordDirectoryPath); + }); + }); + }); +} + +static void deleteEmptyRecordsDirectories(const String& recordsPath) +{ + traverseDirectory(recordsPath, [&recordsPath](const String& partitionName, DirectoryEntryType type) { + if (type != DirectoryEntryType::Directory) + return; + + // Delete [type] sub-folders. + String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName); + traverseDirectory(partitionPath, [&partitionPath](const String& subdirName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + + // Let system figure out if it is really empty. + WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(partitionPath, subdirName)); + }); + + // Delete [Partition] folders. + // Let system figure out if it is really empty. + WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(recordsPath, partitionName)); + }); +} + +Storage::Storage(const String& baseDirectoryPath, Salt salt) + : m_basePath(baseDirectoryPath) + , m_recordsPath(makeRecordsDirectoryPath(baseDirectoryPath)) + , m_salt(salt) + , m_canUseSharedMemoryForBodyData(canUseSharedMemoryForPath(baseDirectoryPath)) + , m_readOperationTimeoutTimer(*this, &Storage::cancelAllReadOperations) + , m_writeOperationDispatchTimer(*this, &Storage::dispatchPendingWriteOperations) + , m_ioQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage", WorkQueue::Type::Concurrent)) + , m_backgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.background", WorkQueue::Type::Concurrent, WorkQueue::QOS::Background)) + , m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.serialBackground", WorkQueue::Type::Serial, WorkQueue::QOS::Background)) + , m_blobStorage(makeBlobDirectoryPath(baseDirectoryPath), m_salt) +{ + deleteOldVersions(); + synchronize(); +} + +Storage::~Storage() +{ +} + +String Storage::basePath() const +{ + return m_basePath.isolatedCopy(); +} + +String Storage::versionPath() const +{ + return makeVersionedDirectoryPath(basePath()); +} + +String Storage::recordsPath() const +{ + return m_recordsPath.isolatedCopy(); +} + +size_t Storage::approximateSize() const +{ + return m_approximateRecordsSize + m_blobStorage.approximateSize(); +} + +void Storage::synchronize() +{ + ASSERT(RunLoop::isMain()); + + if (m_synchronizationInProgress || m_shrinkInProgress) + return; + m_synchronizationInProgress = true; + + LOG(NetworkCacheStorage, "(NetworkProcess) synchronizing cache"); + + backgroundIOQueue().dispatch([this] { + auto recordFilter = std::make_unique<ContentsFilter>(); + auto blobFilter = std::make_unique<ContentsFilter>(); + size_t recordsSize = 0; + unsigned count = 0; + String anyType; + traverseRecordsFiles(recordsPath(), anyType, [&recordFilter, &blobFilter, &recordsSize, &count](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + + Key::HashType hash; + if (!Key::stringToHash(hashString, hash)) { + WebCore::deleteFile(filePath); + return; + } + long long fileSize = 0; + WebCore::getFileSize(filePath, fileSize); + if (!fileSize) { + WebCore::deleteFile(filePath); + return; + } + + if (isBlob) { + blobFilter->add(hash); + return; + } + + recordFilter->add(hash); + recordsSize += fileSize; + ++count; + }); + + RunLoop::main().dispatch([this, recordFilter = WTFMove(recordFilter), blobFilter = WTFMove(blobFilter), recordsSize]() mutable { + for (auto& recordFilterKey : m_recordFilterHashesAddedDuringSynchronization) + recordFilter->add(recordFilterKey); + m_recordFilterHashesAddedDuringSynchronization.clear(); + + for (auto& hash : m_blobFilterHashesAddedDuringSynchronization) + blobFilter->add(hash); + m_blobFilterHashesAddedDuringSynchronization.clear(); + + m_recordFilter = WTFMove(recordFilter); + m_blobFilter = WTFMove(blobFilter); + m_approximateRecordsSize = recordsSize; + m_synchronizationInProgress = false; + }); + + m_blobStorage.synchronize(); + + deleteEmptyRecordsDirectories(recordsPath()); + + LOG(NetworkCacheStorage, "(NetworkProcess) cache synchronization completed size=%zu count=%u", recordsSize, count); + }); +} + +void Storage::addToRecordFilter(const Key& key) +{ + ASSERT(RunLoop::isMain()); + + if (m_recordFilter) + m_recordFilter->add(key.hash()); + + // If we get new entries during filter synchronization take care to add them to the new filter as well. + if (m_synchronizationInProgress) + m_recordFilterHashesAddedDuringSynchronization.append(key.hash()); +} + +bool Storage::mayContain(const Key& key) const +{ + ASSERT(RunLoop::isMain()); + return !m_recordFilter || m_recordFilter->mayContain(key.hash()); +} + +bool Storage::mayContainBlob(const Key& key) const +{ + ASSERT(RunLoop::isMain()); + return !m_blobFilter || m_blobFilter->mayContain(key.hash()); +} + +String Storage::recordDirectoryPathForKey(const Key& key) const +{ + ASSERT(!key.type().isEmpty()); + return WebCore::pathByAppendingComponent(WebCore::pathByAppendingComponent(recordsPath(), key.partitionHashAsString()), key.type()); +} + +String Storage::recordPathForKey(const Key& key) const +{ + return WebCore::pathByAppendingComponent(recordDirectoryPathForKey(key), key.hashAsString()); +} + +static String blobPathForRecordPath(const String& recordPath) +{ + return recordPath + blobSuffix; +} + +String Storage::blobPathForKey(const Key& key) const +{ + return blobPathForRecordPath(recordPathForKey(key)); +} + +struct RecordMetaData { + RecordMetaData() { } + explicit RecordMetaData(const Key& key) + : cacheStorageVersion(Storage::version) + , key(key) + { } + + unsigned cacheStorageVersion; + Key key; + std::chrono::system_clock::time_point timeStamp; + SHA1::Digest headerHash; + uint64_t headerSize; + SHA1::Digest bodyHash; + uint64_t bodySize; + bool isBodyInline; + + // Not encoded as a field. Header starts immediately after meta data. + uint64_t headerOffset; +}; + +static bool decodeRecordMetaData(RecordMetaData& metaData, const Data& fileData) +{ + bool success = false; + fileData.apply([&metaData, &success](const uint8_t* data, size_t size) { + WTF::Persistence::Decoder decoder(data, size); + if (!decoder.decode(metaData.cacheStorageVersion)) + return false; + if (!decoder.decode(metaData.key)) + return false; + if (!decoder.decode(metaData.timeStamp)) + return false; + if (!decoder.decode(metaData.headerHash)) + return false; + if (!decoder.decode(metaData.headerSize)) + return false; + if (!decoder.decode(metaData.bodyHash)) + return false; + if (!decoder.decode(metaData.bodySize)) + return false; + if (!decoder.decode(metaData.isBodyInline)) + return false; + if (!decoder.verifyChecksum()) + return false; + metaData.headerOffset = decoder.currentOffset(); + success = true; + return false; + }); + return success; +} + +static bool decodeRecordHeader(const Data& fileData, RecordMetaData& metaData, Data& headerData, const Salt& salt) +{ + if (!decodeRecordMetaData(metaData, fileData)) { + LOG(NetworkCacheStorage, "(NetworkProcess) meta data decode failure"); + return false; + } + + if (metaData.cacheStorageVersion != Storage::version) { + LOG(NetworkCacheStorage, "(NetworkProcess) version mismatch"); + return false; + } + + headerData = fileData.subrange(metaData.headerOffset, metaData.headerSize); + if (metaData.headerHash != computeSHA1(headerData, salt)) { + LOG(NetworkCacheStorage, "(NetworkProcess) header checksum mismatch"); + return false; + } + return true; +} + +void Storage::readRecord(ReadOperation& readOperation, const Data& recordData) +{ + ASSERT(!RunLoop::isMain()); + + RecordMetaData metaData; + Data headerData; + if (!decodeRecordHeader(recordData, metaData, headerData, m_salt)) + return; + + if (metaData.key != readOperation.key) + return; + + // Sanity check against time stamps in future. + if (metaData.timeStamp > std::chrono::system_clock::now()) + return; + + Data bodyData; + if (metaData.isBodyInline) { + size_t bodyOffset = metaData.headerOffset + headerData.size(); + if (bodyOffset + metaData.bodySize != recordData.size()) + return; + bodyData = recordData.subrange(bodyOffset, metaData.bodySize); + if (metaData.bodyHash != computeSHA1(bodyData, m_salt)) + return; + } + + readOperation.expectedBodyHash = metaData.bodyHash; + readOperation.resultRecord = std::make_unique<Storage::Record>(Storage::Record { + metaData.key, + metaData.timeStamp, + headerData, + bodyData, + metaData.bodyHash + }); +} + +static Data encodeRecordMetaData(const RecordMetaData& metaData) +{ + WTF::Persistence::Encoder encoder; + + encoder << metaData.cacheStorageVersion; + encoder << metaData.key; + encoder << metaData.timeStamp; + encoder << metaData.headerHash; + encoder << metaData.headerSize; + encoder << metaData.bodyHash; + encoder << metaData.bodySize; + encoder << metaData.isBodyInline; + + encoder.encodeChecksum(); + + return Data(encoder.buffer(), encoder.bufferSize()); +} + +std::optional<BlobStorage::Blob> Storage::storeBodyAsBlob(WriteOperation& writeOperation) +{ + auto blobPath = blobPathForKey(writeOperation.record.key); + + // Store the body. + auto blob = m_blobStorage.add(blobPath, writeOperation.record.body); + if (blob.data.isNull()) + return { }; + + ++writeOperation.activeCount; + + RunLoop::main().dispatch([this, blob, &writeOperation] { + if (m_blobFilter) + m_blobFilter->add(writeOperation.record.key.hash()); + if (m_synchronizationInProgress) + m_blobFilterHashesAddedDuringSynchronization.append(writeOperation.record.key.hash()); + + if (writeOperation.mappedBodyHandler) + writeOperation.mappedBodyHandler(blob.data); + + finishWriteOperation(writeOperation); + }); + return blob; +} + +Data Storage::encodeRecord(const Record& record, std::optional<BlobStorage::Blob> blob) +{ + ASSERT(!blob || bytesEqual(blob.value().data, record.body)); + + RecordMetaData metaData(record.key); + metaData.timeStamp = record.timeStamp; + metaData.headerHash = computeSHA1(record.header, m_salt); + metaData.headerSize = record.header.size(); + metaData.bodyHash = blob ? blob.value().hash : computeSHA1(record.body, m_salt); + metaData.bodySize = record.body.size(); + metaData.isBodyInline = !blob; + + auto encodedMetaData = encodeRecordMetaData(metaData); + auto headerData = concatenate(encodedMetaData, record.header); + + if (metaData.isBodyInline) + return concatenate(headerData, record.body); + + return { headerData }; +} + +void Storage::removeFromPendingWriteOperations(const Key& key) +{ + while (true) { + auto found = m_pendingWriteOperations.findIf([&key](auto& operation) { + return operation->record.key == key; + }); + + if (found == m_pendingWriteOperations.end()) + break; + + m_pendingWriteOperations.remove(found); + } +} + +void Storage::remove(const Key& key) +{ + ASSERT(RunLoop::isMain()); + + if (!mayContain(key)) + return; + + // We can't remove the key from the Bloom filter (but some false positives are expected anyway). + // For simplicity we also don't reduce m_approximateSize on removals. + // The next synchronization will update everything. + + removeFromPendingWriteOperations(key); + + serialBackgroundIOQueue().dispatch([this, key] { + WebCore::deleteFile(recordPathForKey(key)); + m_blobStorage.remove(blobPathForKey(key)); + }); +} + +void Storage::updateFileModificationTime(const String& path) +{ + serialBackgroundIOQueue().dispatch([path = path.isolatedCopy()] { + updateFileModificationTimeIfNeeded(path); + }); +} + +void Storage::dispatchReadOperation(std::unique_ptr<ReadOperation> readOperationPtr) +{ + ASSERT(RunLoop::isMain()); + + auto& readOperation = *readOperationPtr; + m_activeReadOperations.add(WTFMove(readOperationPtr)); + + // I/O pressure may make disk operations slow. If they start taking very long time we rather go to network. + const auto readTimeout = 1500ms; + m_readOperationTimeoutTimer.startOneShot(readTimeout); + + bool shouldGetBodyBlob = mayContainBlob(readOperation.key); + + ioQueue().dispatch([this, &readOperation, shouldGetBodyBlob] { + auto recordPath = recordPathForKey(readOperation.key); + + ++readOperation.activeCount; + if (shouldGetBodyBlob) + ++readOperation.activeCount; + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Read); + channel->read(0, std::numeric_limits<size_t>::max(), &ioQueue(), [this, &readOperation](const Data& fileData, int error) { + if (!error) + readRecord(readOperation, fileData); + finishReadOperation(readOperation); + }); + + if (shouldGetBodyBlob) { + // Read the blob in parallel with the record read. + auto blobPath = blobPathForKey(readOperation.key); + readOperation.resultBodyBlob = m_blobStorage.get(blobPath); + finishReadOperation(readOperation); + } + }); +} + +void Storage::finishReadOperation(ReadOperation& readOperation) +{ + ASSERT(readOperation.activeCount); + // Record and blob reads must finish. + if (--readOperation.activeCount) + return; + + RunLoop::main().dispatch([this, &readOperation] { + bool success = readOperation.finish(); + if (success) + updateFileModificationTime(recordPathForKey(readOperation.key)); + else if (!readOperation.isCanceled) + remove(readOperation.key); + + ASSERT(m_activeReadOperations.contains(&readOperation)); + m_activeReadOperations.remove(&readOperation); + + if (m_activeReadOperations.isEmpty()) + m_readOperationTimeoutTimer.stop(); + + dispatchPendingReadOperations(); + + LOG(NetworkCacheStorage, "(NetworkProcess) read complete success=%d", success); + }); +} + +void Storage::cancelAllReadOperations() +{ + ASSERT(RunLoop::isMain()); + + for (auto& readOperation : m_activeReadOperations) + readOperation->cancel(); + + size_t pendingCount = 0; + for (int priority = maximumRetrievePriority; priority >= 0; --priority) { + auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority]; + pendingCount += pendingRetrieveQueue.size(); + for (auto it = pendingRetrieveQueue.rbegin(), end = pendingRetrieveQueue.rend(); it != end; ++it) + (*it)->cancel(); + pendingRetrieveQueue.clear(); + } + + LOG(NetworkCacheStorage, "(NetworkProcess) retrieve timeout, canceled %u active and %zu pending", m_activeReadOperations.size(), pendingCount); +} + +void Storage::dispatchPendingReadOperations() +{ + ASSERT(RunLoop::isMain()); + + const int maximumActiveReadOperationCount = 5; + + for (int priority = maximumRetrievePriority; priority >= 0; --priority) { + if (m_activeReadOperations.size() > maximumActiveReadOperationCount) { + LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel retrieves"); + return; + } + auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority]; + if (pendingRetrieveQueue.isEmpty()) + continue; + dispatchReadOperation(pendingRetrieveQueue.takeLast()); + } +} + +template <class T> bool retrieveFromMemory(const T& operations, const Key& key, Storage::RetrieveCompletionHandler& completionHandler) +{ + for (auto& operation : operations) { + if (operation->record.key == key) { + LOG(NetworkCacheStorage, "(NetworkProcess) found write operation in progress"); + RunLoop::main().dispatch([record = operation->record, completionHandler = WTFMove(completionHandler)] { + completionHandler(std::make_unique<Storage::Record>(record)); + }); + return true; + } + } + return false; +} + +void Storage::dispatchPendingWriteOperations() +{ + ASSERT(RunLoop::isMain()); + + const int maximumActiveWriteOperationCount { 1 }; + + while (!m_pendingWriteOperations.isEmpty()) { + if (m_activeWriteOperations.size() >= maximumActiveWriteOperationCount) { + LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel writes"); + return; + } + dispatchWriteOperation(m_pendingWriteOperations.takeLast()); + } +} + +static bool shouldStoreBodyAsBlob(const Data& bodyData) +{ + const size_t maximumInlineBodySize { 16 * 1024 }; + return bodyData.size() > maximumInlineBodySize; +} + +void Storage::dispatchWriteOperation(std::unique_ptr<WriteOperation> writeOperationPtr) +{ + ASSERT(RunLoop::isMain()); + + auto& writeOperation = *writeOperationPtr; + m_activeWriteOperations.add(WTFMove(writeOperationPtr)); + + // This was added already when starting the store but filter might have been wiped. + addToRecordFilter(writeOperation.record.key); + + backgroundIOQueue().dispatch([this, &writeOperation] { + auto recordDirectorPath = recordDirectoryPathForKey(writeOperation.record.key); + auto recordPath = recordPathForKey(writeOperation.record.key); + + WebCore::makeAllDirectories(recordDirectorPath); + + ++writeOperation.activeCount; + + bool shouldStoreAsBlob = shouldStoreBodyAsBlob(writeOperation.record.body); + auto blob = shouldStoreAsBlob ? storeBodyAsBlob(writeOperation) : std::nullopt; + + auto recordData = encodeRecord(writeOperation.record, blob); + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Create); + size_t recordSize = recordData.size(); + channel->write(0, recordData, nullptr, [this, &writeOperation, recordSize](int error) { + // On error the entry still stays in the contents filter until next synchronization. + m_approximateRecordsSize += recordSize; + finishWriteOperation(writeOperation); + + LOG(NetworkCacheStorage, "(NetworkProcess) write complete error=%d", error); + }); + }); +} + +void Storage::finishWriteOperation(WriteOperation& writeOperation) +{ + ASSERT(RunLoop::isMain()); + ASSERT(writeOperation.activeCount); + ASSERT(m_activeWriteOperations.contains(&writeOperation)); + + if (--writeOperation.activeCount) + return; + + m_activeWriteOperations.remove(&writeOperation); + dispatchPendingWriteOperations(); + + shrinkIfNeeded(); +} + +void Storage::retrieve(const Key& key, unsigned priority, RetrieveCompletionHandler&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(priority <= maximumRetrievePriority); + ASSERT(!key.isNull()); + + if (!m_capacity) { + completionHandler(nullptr); + return; + } + + if (!mayContain(key)) { + completionHandler(nullptr); + return; + } + + if (retrieveFromMemory(m_pendingWriteOperations, key, completionHandler)) + return; + if (retrieveFromMemory(m_activeWriteOperations, key, completionHandler)) + return; + + auto readOperation = std::make_unique<ReadOperation>(key, WTFMove(completionHandler)); + m_pendingReadOperationsByPriority[priority].prepend(WTFMove(readOperation)); + dispatchPendingReadOperations(); +} + +void Storage::store(const Record& record, MappedBodyHandler&& mappedBodyHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(!record.key.isNull()); + + if (!m_capacity) + return; + + auto writeOperation = std::make_unique<WriteOperation>(record, WTFMove(mappedBodyHandler)); + m_pendingWriteOperations.prepend(WTFMove(writeOperation)); + + // Add key to the filter already here as we do lookups from the pending operations too. + addToRecordFilter(record.key); + + bool isInitialWrite = m_pendingWriteOperations.size() == 1; + if (!isInitialWrite) + return; + + // Delay the start of writes a bit to avoid affecting early page load. + // Completing writes will dispatch more writes without delay. + static const auto initialWriteDelay = 1s; + m_writeOperationDispatchTimer.startOneShot(initialWriteDelay); +} + +void Storage::traverse(const String& type, TraverseFlags flags, TraverseHandler&& traverseHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(traverseHandler); + // Avoid non-thread safe Function copies. + + auto traverseOperationPtr = std::make_unique<TraverseOperation>(type, flags, WTFMove(traverseHandler)); + auto& traverseOperation = *traverseOperationPtr; + m_activeTraverseOperations.add(WTFMove(traverseOperationPtr)); + + ioQueue().dispatch([this, &traverseOperation] { + traverseRecordsFiles(recordsPath(), traverseOperation.type, [this, &traverseOperation](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + ASSERT(type == traverseOperation.type); + if (isBlob) + return; + + auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + + double worth = -1; + if (traverseOperation.flags & TraverseFlag::ComputeWorth) + worth = computeRecordWorth(fileTimes(recordPath)); + unsigned bodyShareCount = 0; + if (traverseOperation.flags & TraverseFlag::ShareCount) + bodyShareCount = m_blobStorage.shareCount(blobPathForRecordPath(recordPath)); + + std::unique_lock<Lock> lock(traverseOperation.activeMutex); + ++traverseOperation.activeCount; + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Read); + channel->read(0, std::numeric_limits<size_t>::max(), nullptr, [this, &traverseOperation, worth, bodyShareCount](Data& fileData, int) { + RecordMetaData metaData; + Data headerData; + if (decodeRecordHeader(fileData, metaData, headerData, m_salt)) { + Record record { + metaData.key, + metaData.timeStamp, + headerData, + { }, + metaData.bodyHash + }; + RecordInfo info { + static_cast<size_t>(metaData.bodySize), + worth, + bodyShareCount, + String::fromUTF8(SHA1::hexDigest(metaData.bodyHash)) + }; + traverseOperation.handler(&record, info); + } + + std::lock_guard<Lock> lock(traverseOperation.activeMutex); + --traverseOperation.activeCount; + traverseOperation.activeCondition.notifyOne(); + }); + + const unsigned maximumParallelReadCount = 5; + traverseOperation.activeCondition.wait(lock, [&traverseOperation] { + return traverseOperation.activeCount <= maximumParallelReadCount; + }); + }); + { + // Wait for all reads to finish. + std::unique_lock<Lock> lock(traverseOperation.activeMutex); + traverseOperation.activeCondition.wait(lock, [&traverseOperation] { + return !traverseOperation.activeCount; + }); + } + RunLoop::main().dispatch([this, &traverseOperation] { + traverseOperation.handler(nullptr, { }); + m_activeTraverseOperations.remove(&traverseOperation); + }); + }); +} + +void Storage::setCapacity(size_t capacity) +{ + ASSERT(RunLoop::isMain()); + +#if !ASSERT_DISABLED + const size_t assumedAverageRecordSize = 50 << 10; + size_t maximumRecordCount = capacity / assumedAverageRecordSize; + // ~10 bits per element are required for <1% false positive rate. + size_t effectiveBloomFilterCapacity = ContentsFilter::tableSize / 10; + // If this gets hit it might be time to increase the filter size. + ASSERT(maximumRecordCount < effectiveBloomFilterCapacity); +#endif + + m_capacity = capacity; + + shrinkIfNeeded(); +} + +void Storage::clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, Function<void ()>&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + LOG(NetworkCacheStorage, "(NetworkProcess) clearing cache"); + + if (m_recordFilter) + m_recordFilter->clear(); + if (m_blobFilter) + m_blobFilter->clear(); + m_approximateRecordsSize = 0; + + ioQueue().dispatch([this, modifiedSinceTime, completionHandler = WTFMove(completionHandler), type = type.isolatedCopy()] () mutable { + auto recordsPath = this->recordsPath(); + traverseRecordsFiles(recordsPath, type, [modifiedSinceTime](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + if (modifiedSinceTime > std::chrono::system_clock::time_point::min()) { + auto times = fileTimes(filePath); + if (times.modification < modifiedSinceTime) + return; + } + WebCore::deleteFile(filePath); + }); + + deleteEmptyRecordsDirectories(recordsPath); + + // This cleans unreferenced blobs. + m_blobStorage.synchronize(); + + if (completionHandler) { + RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler)] { + completionHandler(); + }); + } + }); +} + +static double computeRecordWorth(FileTimes times) +{ + using namespace std::chrono; + auto age = system_clock::now() - times.creation; + // File modification time is updated manually on cache read. We don't use access time since OS may update it automatically. + auto accessAge = times.modification - times.creation; + + // For sanity. + if (age <= 0s || accessAge < 0s || accessAge > age) + return 0; + + // We like old entries that have been accessed recently. + return duration<double>(accessAge) / age; +} + +static double deletionProbability(FileTimes times, unsigned bodyShareCount) +{ + static const double maximumProbability { 0.33 }; + static const unsigned maximumEffectiveShareCount { 5 }; + + auto worth = computeRecordWorth(times); + + // Adjust a bit so the most valuable entries don't get deleted at all. + auto effectiveWorth = std::min(1.1 * worth, 1.); + + auto probability = (1 - effectiveWorth) * maximumProbability; + + // It is less useful to remove an entry that shares its body data. + if (bodyShareCount) + probability /= std::min(bodyShareCount, maximumEffectiveShareCount); + + return probability; +} + +void Storage::shrinkIfNeeded() +{ + ASSERT(RunLoop::isMain()); + + if (approximateSize() > m_capacity) + shrink(); +} + +void Storage::shrink() +{ + ASSERT(RunLoop::isMain()); + + if (m_shrinkInProgress || m_synchronizationInProgress) + return; + m_shrinkInProgress = true; + + LOG(NetworkCacheStorage, "(NetworkProcess) shrinking cache approximateSize=%zu capacity=%zu", approximateSize(), m_capacity); + + backgroundIOQueue().dispatch([this] { + auto recordsPath = this->recordsPath(); + String anyType; + traverseRecordsFiles(recordsPath, anyType, [this](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + if (isBlob) + return; + + auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + auto blobPath = blobPathForRecordPath(recordPath); + + auto times = fileTimes(recordPath); + unsigned bodyShareCount = m_blobStorage.shareCount(blobPath); + auto probability = deletionProbability(times, bodyShareCount); + + bool shouldDelete = randomNumber() < probability; + + LOG(NetworkCacheStorage, "Deletion probability=%f bodyLinkCount=%d shouldDelete=%d", probability, bodyShareCount, shouldDelete); + + if (shouldDelete) { + WebCore::deleteFile(recordPath); + m_blobStorage.remove(blobPath); + } + }); + + RunLoop::main().dispatch([this] { + m_shrinkInProgress = false; + // We could synchronize during the shrink traversal. However this is fast and it is better to have just one code path. + synchronize(); + }); + + LOG(NetworkCacheStorage, "(NetworkProcess) cache shrink completed"); + }); +} + +void Storage::deleteOldVersions() +{ + backgroundIOQueue().dispatch([this] { + auto cachePath = basePath(); + traverseDirectory(cachePath, [&cachePath](const String& subdirName, DirectoryEntryType type) { + if (type != DirectoryEntryType::Directory) + return; + if (!subdirName.startsWith(versionDirectoryPrefix)) + return; + auto versionString = subdirName.substring(strlen(versionDirectoryPrefix)); + bool success; + unsigned directoryVersion = versionString.toUIntStrict(&success); + if (!success) + return; + if (directoryVersion >= version) + return; +#if PLATFORM(MAC) + if (directoryVersion == lastStableVersion) + return; +#endif + + auto oldVersionPath = WebCore::pathByAppendingComponent(cachePath, subdirName); + LOG(NetworkCacheStorage, "(NetworkProcess) deleting old cache version, path %s", oldVersionPath.utf8().data()); + + deleteDirectoryRecursively(oldVersionPath); + }); + }); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h new file mode 100644 index 000000000..30b5f1368 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheStorage_h +#define NetworkCacheStorage_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheBlobStorage.h" +#include "NetworkCacheData.h" +#include "NetworkCacheKey.h" +#include <WebCore/Timer.h> +#include <wtf/BloomFilter.h> +#include <wtf/Deque.h> +#include <wtf/Function.h> +#include <wtf/HashSet.h> +#include <wtf/Optional.h> +#include <wtf/WorkQueue.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { +namespace NetworkCache { + +class IOChannel; + +class Storage { + WTF_MAKE_NONCOPYABLE(Storage); +public: + static std::unique_ptr<Storage> open(const String& cachePath); + + struct Record { + WTF_MAKE_FAST_ALLOCATED; + public: + Key key; + std::chrono::system_clock::time_point timeStamp; + Data header; + Data body; + std::optional<SHA1::Digest> bodyHash; + }; + // This may call completion handler synchronously on failure. + typedef Function<bool (std::unique_ptr<Record>)> RetrieveCompletionHandler; + void retrieve(const Key&, unsigned priority, RetrieveCompletionHandler&&); + + typedef Function<void (const Data& mappedBody)> MappedBodyHandler; + void store(const Record&, MappedBodyHandler&&); + + void remove(const Key&); + void clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, Function<void ()>&& completionHandler); + + struct RecordInfo { + size_t bodySize; + double worth; // 0-1 where 1 is the most valuable. + unsigned bodyShareCount; + String bodyHash; + }; + enum TraverseFlag { + ComputeWorth = 1 << 0, + ShareCount = 1 << 1, + }; + typedef unsigned TraverseFlags; + typedef Function<void (const Record*, const RecordInfo&)> TraverseHandler; + // Null record signals end. + void traverse(const String& type, TraverseFlags, TraverseHandler&&); + + void setCapacity(size_t); + size_t capacity() const { return m_capacity; } + size_t approximateSize() const; + + static const unsigned version = 11; +#if PLATFORM(MAC) + /// Allow the last stable version of the cache to co-exist with the latest development one. + static const unsigned lastStableVersion = 9; +#endif + + String basePath() const; + String versionPath() const; + String recordsPath() const; + + const Salt& salt() const { return m_salt; } + + bool canUseSharedMemoryForBodyData() const { return m_canUseSharedMemoryForBodyData; } + + ~Storage(); + +private: + Storage(const String& directoryPath, Salt); + + String recordDirectoryPathForKey(const Key&) const; + String recordPathForKey(const Key&) const; + String blobPathForKey(const Key&) const; + + void synchronize(); + void deleteOldVersions(); + void shrinkIfNeeded(); + void shrink(); + + struct ReadOperation; + void dispatchReadOperation(std::unique_ptr<ReadOperation>); + void dispatchPendingReadOperations(); + void finishReadOperation(ReadOperation&); + void cancelAllReadOperations(); + + struct WriteOperation; + void dispatchWriteOperation(std::unique_ptr<WriteOperation>); + void dispatchPendingWriteOperations(); + void finishWriteOperation(WriteOperation&); + + std::optional<BlobStorage::Blob> storeBodyAsBlob(WriteOperation&); + Data encodeRecord(const Record&, std::optional<BlobStorage::Blob>); + void readRecord(ReadOperation&, const Data&); + + void updateFileModificationTime(const String& path); + void removeFromPendingWriteOperations(const Key&); + + WorkQueue& ioQueue() { return m_ioQueue.get(); } + WorkQueue& backgroundIOQueue() { return m_backgroundIOQueue.get(); } + WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); } + + bool mayContain(const Key&) const; + bool mayContainBlob(const Key&) const; + + void addToRecordFilter(const Key&); + + const String m_basePath; + const String m_recordsPath; + + const Salt m_salt; + + const bool m_canUseSharedMemoryForBodyData; + + size_t m_capacity { std::numeric_limits<size_t>::max() }; + size_t m_approximateRecordsSize { 0 }; + + // 2^18 bit filter can support up to 26000 entries with false positive rate < 1%. + using ContentsFilter = BloomFilter<18>; + std::unique_ptr<ContentsFilter> m_recordFilter; + std::unique_ptr<ContentsFilter> m_blobFilter; + + bool m_synchronizationInProgress { false }; + bool m_shrinkInProgress { false }; + + Vector<Key::HashType> m_recordFilterHashesAddedDuringSynchronization; + Vector<Key::HashType> m_blobFilterHashesAddedDuringSynchronization; + + static const int maximumRetrievePriority = 4; + Deque<std::unique_ptr<ReadOperation>> m_pendingReadOperationsByPriority[maximumRetrievePriority + 1]; + HashSet<std::unique_ptr<ReadOperation>> m_activeReadOperations; + WebCore::Timer m_readOperationTimeoutTimer; + + Deque<std::unique_ptr<WriteOperation>> m_pendingWriteOperations; + HashSet<std::unique_ptr<WriteOperation>> m_activeWriteOperations; + WebCore::Timer m_writeOperationDispatchTimer; + + struct TraverseOperation; + HashSet<std::unique_ptr<TraverseOperation>> m_activeTraverseOperations; + + Ref<WorkQueue> m_ioQueue; + Ref<WorkQueue> m_backgroundIOQueue; + Ref<WorkQueue> m_serialBackgroundIOQueue; + + BlobStorage m_blobStorage; +}; + +// FIXME: Remove, used by NetworkCacheStatistics only. +using RecordFileTraverseFunction = Function<void (const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath)>; +void traverseRecordsFiles(const String& recordsPath, const String& type, const RecordFileTraverseFunction&); + +} +} +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp new file mode 100644 index 000000000..561e4e6b8 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSubresourcesEntry.h" + +#include "Logging.h" +#include "NetworkCacheCoders.h" + +using namespace std::chrono; + +namespace WebKit { +namespace NetworkCache { + +void SubresourceInfo::encode(WTF::Persistence::Encoder& encoder) const +{ + encoder << m_key; + encoder << m_lastSeen; + encoder << m_firstSeen; + encoder << m_isTransient; + + // Do not bother serializing other data members of transient resources as they are empty. + if (m_isTransient) + return; + + encoder << m_firstPartyForCookies; + encoder << m_requestHeaders; + encoder.encodeEnum(m_priority); +} + +bool SubresourceInfo::decode(WTF::Persistence::Decoder& decoder, SubresourceInfo& info) +{ + if (!decoder.decode(info.m_key)) + return false; + if (!decoder.decode(info.m_lastSeen)) + return false; + if (!decoder.decode(info.m_firstSeen)) + return false; + + if (!decoder.decode(info.m_isTransient)) + return false; + + if (info.m_isTransient) + return true; + + if (!decoder.decode(info.m_firstPartyForCookies)) + return false; + + if (!decoder.decode(info.m_requestHeaders)) + return false; + + if (!decoder.decodeEnum(info.m_priority)) + return false; + + return true; +} + +Storage::Record SubresourcesEntry::encodeAsStorageRecord() const +{ + WTF::Persistence::Encoder encoder; + encoder << m_subresources; + + encoder.encodeChecksum(); + + return { m_key, m_timeStamp, { encoder.buffer(), encoder.bufferSize() }, { }, { }}; +} + +std::unique_ptr<SubresourcesEntry> SubresourcesEntry::decodeStorageRecord(const Storage::Record& storageEntry) +{ + auto entry = std::make_unique<SubresourcesEntry>(storageEntry); + + WTF::Persistence::Decoder decoder(storageEntry.header.data(), storageEntry.header.size()); + if (!decoder.decode(entry->m_subresources)) + return nullptr; + + if (!decoder.verifyChecksum()) { + LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n"); + return nullptr; + } + + return entry; +} + +SubresourcesEntry::SubresourcesEntry(const Storage::Record& storageEntry) + : m_key(storageEntry.key) + , m_timeStamp(storageEntry.timeStamp) +{ + ASSERT(m_key.type() == "SubResources"); +} + +SubresourceInfo::SubresourceInfo(const Key& key, const WebCore::ResourceRequest& request, const SubresourceInfo* previousInfo) + : m_key(key) + , m_lastSeen(std::chrono::system_clock::now()) + , m_firstSeen(previousInfo ? previousInfo->firstSeen() : m_lastSeen) + , m_isTransient(!previousInfo) + , m_firstPartyForCookies(request.firstPartyForCookies()) + , m_requestHeaders(request.httpHeaderFields()) + , m_priority(request.priority()) +{ +} + +static Vector<SubresourceInfo> makeSubresourceInfoVector(const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads, Vector<SubresourceInfo>* previousSubresources) +{ + Vector<SubresourceInfo> result; + result.reserveInitialCapacity(subresourceLoads.size()); + + HashMap<Key, unsigned> previousMap; + if (previousSubresources) { + for (unsigned i = 0; i < previousSubresources->size(); ++i) + previousMap.add(previousSubresources->at(i).key(), i); + } + + HashSet<Key> deduplicationSet; + for (auto& load : subresourceLoads) { + if (!deduplicationSet.add(load->key).isNewEntry) + continue; + + SubresourceInfo* previousInfo = nullptr; + if (previousSubresources) { + auto it = previousMap.find(load->key); + if (it != previousMap.end()) + previousInfo = &(*previousSubresources)[it->value]; + } + + result.uncheckedAppend({ load->key, load->request, previousInfo }); + + // FIXME: We should really consider all resources seen for the first time transient. + if (!previousSubresources) + result.last().setNonTransient(); + } + + return result; +} + +SubresourcesEntry::SubresourcesEntry(Key&& key, const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads) + : m_key(WTFMove(key)) + , m_timeStamp(std::chrono::system_clock::now()) + , m_subresources(makeSubresourceInfoVector(subresourceLoads, nullptr)) +{ + ASSERT(m_key.type() == "SubResources"); +} + +void SubresourcesEntry::updateSubresourceLoads(const Vector<std::unique_ptr<SubresourceLoad>>& subresourceLoads) +{ + m_subresources = makeSubresourceInfoVector(subresourceLoads, &m_subresources); +} + +} // namespace WebKit +} // namespace NetworkCache + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h new file mode 100644 index 000000000..c2d5da8b1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSubresourcesEntry_h +#define NetworkCacheSubresourcesEntry_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCacheStorage.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/URL.h> +#include <wtf/HashMap.h> + +namespace WebKit { +namespace NetworkCache { + +class SubresourceInfo { + WTF_MAKE_FAST_ALLOCATED; +public: + void encode(WTF::Persistence::Encoder&) const; + static bool decode(WTF::Persistence::Decoder&, SubresourceInfo&); + + SubresourceInfo() = default; + SubresourceInfo(const Key&, const WebCore::ResourceRequest&, const SubresourceInfo* previousInfo); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point lastSeen() const { return m_lastSeen; } + std::chrono::system_clock::time_point firstSeen() const { return m_firstSeen; } + + bool isTransient() const { return m_isTransient; } + const WebCore::URL& firstPartyForCookies() const { ASSERT(!m_isTransient); return m_firstPartyForCookies; } + const WebCore::HTTPHeaderMap& requestHeaders() const { ASSERT(!m_isTransient); return m_requestHeaders; } + WebCore::ResourceLoadPriority priority() const { ASSERT(!m_isTransient); return m_priority; } + + void setNonTransient() { m_isTransient = false; } + +private: + Key m_key; + std::chrono::system_clock::time_point m_lastSeen; + std::chrono::system_clock::time_point m_firstSeen; + bool m_isTransient { false }; + WebCore::URL m_firstPartyForCookies; + WebCore::HTTPHeaderMap m_requestHeaders; + WebCore::ResourceLoadPriority m_priority; +}; + +struct SubresourceLoad { + WTF_MAKE_NONCOPYABLE(SubresourceLoad); WTF_MAKE_FAST_ALLOCATED; +public: + SubresourceLoad(const WebCore::ResourceRequest& request, const Key& key) + : request(request) + , key(key) + { } + + WebCore::ResourceRequest request; + Key key; +}; + +class SubresourcesEntry { + WTF_MAKE_NONCOPYABLE(SubresourcesEntry); WTF_MAKE_FAST_ALLOCATED; +public: + SubresourcesEntry(Key&&, const Vector<std::unique_ptr<SubresourceLoad>>&); + explicit SubresourcesEntry(const Storage::Record&); + + Storage::Record encodeAsStorageRecord() const; + static std::unique_ptr<SubresourcesEntry> decodeStorageRecord(const Storage::Record&); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; } + const Vector<SubresourceInfo>& subresources() const { return m_subresources; } + + void updateSubresourceLoads(const Vector<std::unique_ptr<SubresourceLoad>>&); + +private: + Key m_key; + std::chrono::system_clock::time_point m_timeStamp; + Vector<SubresourceInfo> m_subresources; +}; + +} // namespace WebKit +} // namespace NetworkCache + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#endif // NetworkCacheSubresourcesEntry_h diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureEvent.cpp b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureEvent.cpp new file mode 100644 index 000000000..cb7bbdc49 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureEvent.cpp @@ -0,0 +1,504 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCaptureEvent.h" + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureLogging.h" +#include "json.hpp" +#include <WebCore/ResourceError.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/URLParser.h> +#include <wtf/Assertions.h> +#include <wtf/Brigand.h> +#include <wtf/text/Base64.h> + +namespace WebKit { +namespace NetworkCapture { + +const char RequestSentEvent::typeName[] = "RequestSentEvent"; +const char ResponseReceivedEvent::typeName[] = "ResponseReceivedEvent"; +const char RedirectReceivedEvent::typeName[] = "RedirectReceivedEvent"; +const char RedirectSentEvent::typeName[] = "RedirectSentEvent"; +const char DataReceivedEvent::typeName[] = "DataReceivedEvent"; +const char FinishedEvent::typeName[] = "FinishedEvent"; + +static Headers copyHeaders(const WebCore::HTTPHeaderMap& httpHeaders) +{ + Headers headers; + for (const auto& header : httpHeaders) + headers.append(std::make_pair(header.key, header.value)); + return headers; +} + +// ---------- + +Request::Request(String&& url, String&& referrer, int policy, String&& method, Headers&& headers) + : url(WTFMove(url)) + , referrer(WTFMove(referrer)) + , policy(WTFMove(policy)) + , method(WTFMove(method)) + , headers(WTFMove(headers)) +{ +} + +Request::Request(const WebCore::ResourceRequest& request) + : url(request.url().string()) + , referrer(request.httpReferrer()) + , policy(static_cast<int>(request.cachePolicy())) + , method(request.httpMethod()) + , headers(copyHeaders(request.httpHeaderFields())) +{ +} + +Request::operator WebCore::ResourceRequest() const +{ + WebCore::URLParser parser(url); + WebCore::ResourceRequest request(parser.result(), referrer, static_cast<WebCore::ResourceRequestCachePolicy>(policy)); + request.setHTTPMethod(method); + + for (const auto& header : headers) + request.setHTTPHeaderField(header.first, header.second); + + return request; +} + +// ---------- + +Response::Response(String&& url, String&& mimeType, long long expectedLength, String&& textEncodingName, String&& version, int status, String&& reason, Headers&& headers) + : url(WTFMove(url)) + , mimeType(WTFMove(mimeType)) + , expectedLength(WTFMove(expectedLength)) + , textEncodingName(WTFMove(textEncodingName)) + , status(WTFMove(status)) + , reason(WTFMove(reason)) + , headers(WTFMove(headers)) +{ +} + +Response::Response(const WebCore::ResourceResponse& response) + : url(response.url().string()) + , mimeType(response.mimeType()) + , expectedLength(response.expectedContentLength()) + , textEncodingName(response.textEncodingName()) + , version(response.httpVersion()) + , status(response.httpStatusCode()) + , reason(response.httpStatusText()) + , headers(copyHeaders(response.httpHeaderFields())) +{ +} + +Response::operator WebCore::ResourceResponse() const +{ + WebCore::URLParser parser(url); + WebCore::ResourceResponse response(parser.result(), mimeType, expectedLength, textEncodingName); + response.setHTTPVersion(version); + response.setHTTPStatusCode(status); + response.setHTTPStatusText(reason); + + for (const auto& header : headers) + response.setHTTPHeaderField(header.first, header.second); + + return response; +} + +// ---------- + +Error::Error(String&& domain, String&& failingURL, String&& localizedDescription, int errorCode, int type) + : domain(WTFMove(domain)) + , failingURL(WTFMove(failingURL)) + , localizedDescription(WTFMove(localizedDescription)) + , errorCode(WTFMove(errorCode)) + , type(WTFMove(type)) +{ +} + +Error::Error(const WebCore::ResourceError& error) + : domain(error.domain()) + , failingURL(error.failingURL().string()) + , localizedDescription(error.localizedDescription()) + , errorCode(error.errorCode()) + , type(static_cast<int>(error.type())) +{ +} + +Error::operator WebCore::ResourceError() const +{ + WebCore::URLParser parser(failingURL); + WebCore::ResourceError error(domain, errorCode, parser.result(), localizedDescription, static_cast<WebCore::ResourceError::Type>(type)); + + return error; +} + +// ---------- + +// SEE THE NOTE IN json.hpp REGARDING ITS USE IN THIS PROJECT. IN SHORT, DO NOT +// USE json.hpp ANYWHERE ELSE. IT WILL BE GOING AWAY. +using json = nlohmann::basic_json<>; + +template<typename Type> +struct JSONCoder { + static json encode(Type val) + { + return json(val); + } + + static Type decode(const json& jVal) + { + return jVal.get<Type>(); + } +}; + +template<> +struct JSONCoder<const char*> { + static json encode(const char* val) + { + return json(val); + } +}; + +template<> +struct JSONCoder<String> { + static json encode(const String& val) + { + return json(static_cast<const char*>(val.utf8().data())); + } + + static String decode(const json& jVal) + { + return String(jVal.get_ref<const std::string&>().c_str()); + } +}; + +template<> +struct JSONCoder<CaptureTimeType> { + static json encode(const CaptureTimeType& time) + { + return JSONCoder<double>::encode(time.secondsSinceEpoch().seconds()); + } + + static CaptureTimeType decode(const json& jTime) + { + return CaptureTimeType::fromRawSeconds(JSONCoder<double>::decode(jTime)); + } +}; + +template<> +struct JSONCoder<KeyValuePair> { + static json encode(const KeyValuePair& pair) + { + return json { + JSONCoder<String>::encode(pair.first), + JSONCoder<String>::encode(pair.second) + }; + } + + static KeyValuePair decode(const json& jPair) + { + return KeyValuePair { + JSONCoder<String>::decode(jPair[0]), + JSONCoder<String>::decode(jPair[1]) + }; + } +}; + +template<typename T> +struct JSONCoder<Vector<T>> { + static json encode(const Vector<T>& vector) + { + json jVector; + + for (const auto& element : vector) + jVector.push_back(JSONCoder<T>::encode(element)); + + return jVector; + } + + static Vector<T> decode(const json& jVector) + { + Vector<T> vector; + + for (const auto& element : jVector) + vector.append(JSONCoder<T>::decode(element)); + + return vector; + } +}; + +template<> +struct JSONCoder<Request> { + static json encode(const Request& request) + { + return json { + { "url", JSONCoder<String>::encode(request.url) }, + { "referrer", JSONCoder<String>::encode(request.referrer) }, + { "policy", JSONCoder<int>::encode(request.policy) }, + { "method", JSONCoder<String>::encode(request.method) }, + { "headers", JSONCoder<Headers>::encode(request.headers) } + }; + } + + static Request decode(const json& jRequest) + { + return Request { + JSONCoder<String>::decode(jRequest["url"]), + JSONCoder<String>::decode(jRequest["referrer"]), + JSONCoder<int>::decode(jRequest["policy"]), + JSONCoder<String>::decode(jRequest["method"]), + JSONCoder<Headers>::decode(jRequest["headers"]) + }; + } +}; + +template<> +struct JSONCoder<Response> { + static json encode(const Response& response) + { + return json { + { "url", JSONCoder<String>::encode(response.url) }, + { "mimeType", JSONCoder<String>::encode(response.mimeType) }, + { "expectedLength", JSONCoder<long long>::encode(response.expectedLength) }, + { "textEncodingName", JSONCoder<String>::encode(response.textEncodingName) }, + { "version", JSONCoder<String>::encode(response.version) }, + { "status", JSONCoder<int>::encode(response.status) }, + { "reason", JSONCoder<String>::encode(response.reason) }, + { "headers", JSONCoder<Headers>::encode(response.headers) } + }; + } + + static Response decode(const json& jResponse) + { + return Response { + JSONCoder<String>::decode(jResponse["url"]), + JSONCoder<String>::decode(jResponse["mimeType"]), + JSONCoder<long long>::decode(jResponse["expectedLength"]), + JSONCoder<String>::decode(jResponse["textEncodingName"]), + JSONCoder<String>::decode(jResponse["version"]), + JSONCoder<int>::decode(jResponse["status"]), + JSONCoder<String>::decode(jResponse["reason"]), + JSONCoder<Headers>::decode(jResponse["headers"]) + }; + } +}; + +template<> +struct JSONCoder<Error> { + static json encode(const Error& error) + { + return json { + { "domain", JSONCoder<String>::encode(error.domain) }, + { "failingURL", JSONCoder<String>::encode(error.failingURL) }, + { "localizedDescription", JSONCoder<String>::encode(error.localizedDescription) }, + { "errorCode", JSONCoder<int>::encode(error.errorCode) }, + { "type", JSONCoder<int>::encode(error.type) } + }; + } + + static Error decode(const json& jError) + { + return Error { + JSONCoder<String>::decode(jError["domain"]), + JSONCoder<String>::decode(jError["failingURL"]), + JSONCoder<String>::decode(jError["localizedDescription"]), + JSONCoder<int>::decode(jError["errorCode"]), + JSONCoder<int>::decode(jError["type"]) + }; + } +}; + +template<> +struct JSONCoder<WebCore::SharedBuffer> { + static json encode(const WebCore::SharedBuffer& data) + { + Vector<char> buffer; + base64Encode(data.data(), data.size(), buffer); + return json(&buffer[0], buffer.size()); + } + + static Ref<WebCore::SharedBuffer> decode(const json& jData) + { + Vector<char> data; + const auto& str = jData.get_ref<const std::string&>(); + auto result = base64Decode(str.c_str(), str.size(), data); + ASSERT_UNUSED(result, result); + return WebCore::SharedBuffer::adoptVector(data); + } +}; + +template<> +struct JSONCoder<RequestSentEvent> { + static json encode(const RequestSentEvent& event) + { + return json { + { "type", JSONCoder<const char*>::encode(event.typeName) }, + { "time", JSONCoder<CaptureTimeType>::encode(event.time) }, + { "request", JSONCoder<Request>::encode(event.request) } + }; + } + + static RequestSentEvent decode(const json& jEvent) + { + return RequestSentEvent { + JSONCoder<CaptureTimeType>::decode(jEvent["time"]), + JSONCoder<Request>::decode(jEvent["request"]) + }; + } +}; + +template<> +struct JSONCoder<ResponseReceivedEvent> { + static json encode(const ResponseReceivedEvent& event) + { + return json { + { "type", JSONCoder<const char*>::encode(event.typeName) }, + { "time", JSONCoder<CaptureTimeType>::encode(event.time) }, + { "response", JSONCoder<Response>::encode(event.response) } + }; + } + + static ResponseReceivedEvent decode(const json& jEvent) + { + return ResponseReceivedEvent { + JSONCoder<CaptureTimeType>::decode(jEvent["time"]), + JSONCoder<Response>::decode(jEvent["response"]) + }; + } +}; + +template<> +struct JSONCoder<RedirectReceivedEvent> { + static json encode(const RedirectReceivedEvent& event) + { + return json { + { "type", JSONCoder<const char*>::encode(event.typeName) }, + { "time", JSONCoder<CaptureTimeType>::encode(event.time) }, + { "request", JSONCoder<Request>::encode(event.request) }, + { "response", JSONCoder<Response>::encode(event.response) } + }; + } + + static RedirectReceivedEvent decode(const json& jEvent) + { + return RedirectReceivedEvent { + JSONCoder<CaptureTimeType>::decode(jEvent["time"]), + JSONCoder<Request>::decode(jEvent["request"]), + JSONCoder<Response>::decode(jEvent["response"]) + }; + } +}; + +template<> +struct JSONCoder<RedirectSentEvent> { + static json encode(const RedirectSentEvent& event) + { + return json { + { "type", JSONCoder<const char*>::encode(event.typeName) }, + { "time", JSONCoder<CaptureTimeType>::encode(event.time) }, + { "request", JSONCoder<Request>::encode(event.request) }, + }; + } + + static RedirectSentEvent decode(const json& jEvent) + { + return RedirectSentEvent { + JSONCoder<CaptureTimeType>::decode(jEvent["time"]), + JSONCoder<Request>::decode(jEvent["request"]) + }; + } +}; + +template<> +struct JSONCoder<DataReceivedEvent> { + static json encode(const DataReceivedEvent& event) + { + return json { + { "type", JSONCoder<const char*>::encode(event.typeName) }, + { "time", JSONCoder<CaptureTimeType>::encode(event.time) }, + { "data", JSONCoder<WebCore::SharedBuffer>::encode(event.data.get()) } + }; + } + + static DataReceivedEvent decode(const json& jEvent) + { + return DataReceivedEvent { + JSONCoder<CaptureTimeType>::decode(jEvent["time"]), + JSONCoder<WebCore::SharedBuffer>::decode(jEvent["data"]) + }; + } +}; + +template<> +struct JSONCoder<FinishedEvent> { + static json encode(const FinishedEvent& event) + { + return json { + { "type", JSONCoder<const char*>::encode(event.typeName) }, + { "time", JSONCoder<CaptureTimeType>::encode(event.time) }, + { "error", JSONCoder<Error>::encode(event.error) } + }; + } + + static FinishedEvent decode(const json& jEvent) + { + return FinishedEvent { + JSONCoder<CaptureTimeType>::decode(jEvent["time"]), + JSONCoder<Error>::decode(jEvent["error"]) + }; + } +}; + +std::string eventToString(const CaptureEvent& event) +{ + json result; + + WTF::visit([&result](const auto& event) { + using EventType = std::decay_t<decltype(event)>; + result = JSONCoder<EventType>::encode(event); + }, event); + + return result.dump(4); +} + +OptionalCaptureEvent stringToEvent(const char* jsonStr) +{ + auto jValue = json::parse(jsonStr); + const auto& type = jValue["type"].get_ref<const std::string&>(); + + OptionalCaptureEvent result { std::nullopt }; + brigand::for_each<CaptureEvent>([&](auto T) { + using Type = typename decltype(T)::type; + if (!result && type == Type::typeName) + result = OptionalCaptureEvent(JSONCoder<Type>::decode(jValue)); + }); + return result; +} + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureEvent.h b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureEvent.h new file mode 100644 index 000000000..73b240b0c --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureEvent.h @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CAPTURE) + +#include <WebCore/SharedBuffer.h> +#include <wtf/MonotonicTime.h> +#include <wtf/Optional.h> +#include <wtf/Variant.h> +#include <wtf/Vector.h> + +namespace WebCore { +class ResourceError; +class ResourceRequest; +class ResourceResponse; +} + +namespace WebKit { +namespace NetworkCapture { + +struct RequestSentEvent; +struct ResponseReceivedEvent; +struct RedirectReceivedEvent; +struct RedirectSentEvent; +struct DataReceivedEvent; +struct FinishedEvent; + +using CaptureEvent = WTF::Variant<RequestSentEvent, ResponseReceivedEvent, RedirectReceivedEvent, RedirectSentEvent, DataReceivedEvent, FinishedEvent>; +using OptionalCaptureEvent = std::optional<CaptureEvent>; +using CaptureEvents = Vector<CaptureEvent>; +using CaptureClockType = WTF::MonotonicTime; +using CaptureTimeType = WTF::MonotonicTime; +using KeyValuePair = std::pair<String, String>; +using Headers = Vector<KeyValuePair>; + +std::string eventToString(const CaptureEvent&); +OptionalCaptureEvent stringToEvent(const char*); + +struct Request { + // See comment for RequestSentEvent for why we need this default constructor. + Request() + : url() + , referrer() + , policy(0) + , method() + , headers() { } + Request(String&& url, String&& referrer, int policy, String&& method, Headers&&); + Request(const WebCore::ResourceRequest&); + operator WebCore::ResourceRequest() const; + + String url; + String referrer; + int policy; + String method; + Headers headers; +}; +static_assert(std::is_default_constructible<Request>::value, "Request is not default constructible"); +static_assert(std::is_move_constructible<Request>::value, "Request is not move constructible"); +static_assert(std::is_move_assignable<Request>::value, "Request is not move assignable"); + +struct Response { + Response(String&& url, String&& mimeType, long long expectedLength, String&& textEncodingName, String&& version, int status, String&& reason, Headers&&); + Response(const WebCore::ResourceResponse&); + operator WebCore::ResourceResponse() const; + + String url; + String mimeType; + long long expectedLength; + String textEncodingName; + String version; + int status; + String reason; + Headers headers; +}; +static_assert(std::is_move_constructible<Response>::value, "Response is not move constructible"); +static_assert(std::is_move_assignable<Response>::value, "Response is not move assignable"); + +struct Error { + Error(String&& domain, String&& failingURL, String&& localizedDescription, int errorCode, int type); + Error(const WebCore::ResourceError&); + operator WebCore::ResourceError() const; + + String domain; + String failingURL; + String localizedDescription; + int errorCode; + int type; +}; +static_assert(std::is_move_constructible<Error>::value, "Error is not move constructible"); +static_assert(std::is_move_assignable<Error>::value, "Error is not move assignable"); + +struct TimedEvent { + TimedEvent() + : time(CaptureClockType::now()) { } + TimedEvent(CaptureTimeType&& time) + : time(WTFMove(time)) { } + + CaptureTimeType time; +}; +static_assert(std::is_move_constructible<TimedEvent>::value, "TimedEvent is not move constructible"); +static_assert(std::is_move_assignable<TimedEvent>::value, "TimedEvent is not move assignable"); + +struct RequestSentEvent final : public TimedEvent { + // This default constructor is needed only because this struct is the first + // type passed to CaptureEvent, which is a WTF::Variant. This means that if + // CaptureEvent is default-constructed, it needs to default-construct an + // instance of RequestSentEvent, the first type on its list. If we don't + // have a default constructor for this type, the CaptureEvent will be + // created in an invalid state and will crash when destructed. + RequestSentEvent() + : TimedEvent() + , request() { } + RequestSentEvent(const WebCore::ResourceRequest& request) + : TimedEvent() + , request(request) { } + RequestSentEvent(CaptureTimeType&& time, Request&& request) + : TimedEvent(WTFMove(time)) + , request(WTFMove(request)) { } + + Request request; + static const char typeName[]; +}; +static_assert(std::is_default_constructible<RequestSentEvent>::value, "RequestSentEvent is not default constructible"); +static_assert(std::is_move_constructible<RequestSentEvent>::value, "RequestSentEvent is not move constructible"); +static_assert(std::is_move_assignable<RequestSentEvent>::value, "RequestSentEvent is not move assignable"); + +struct ResponseReceivedEvent final : public TimedEvent { + ResponseReceivedEvent(const WebCore::ResourceResponse& response) + : TimedEvent() + , response(response) { } + ResponseReceivedEvent(CaptureTimeType&& time, Response&& response) + : TimedEvent(WTFMove(time)) + , response(WTFMove(response)) { } + + Response response; + static const char typeName[]; +}; +static_assert(std::is_move_constructible<ResponseReceivedEvent>::value, "ResponseReceivedEvent is not move constructible"); +static_assert(std::is_move_assignable<ResponseReceivedEvent>::value, "ResponseReceivedEvent is not move assignable"); + +struct RedirectReceivedEvent final : public TimedEvent { + RedirectReceivedEvent(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response) + : TimedEvent() + , request(request) + , response(response) { } + RedirectReceivedEvent(CaptureTimeType&& time, Request&& request, Response&& response) + : TimedEvent(WTFMove(time)) + , request(WTFMove(request)) + , response(WTFMove(response)) { } + + Request request; + Response response; + static const char typeName[]; +}; +static_assert(std::is_move_constructible<RedirectReceivedEvent>::value, "RedirectReceivedEvent is not move constructible"); +static_assert(std::is_move_assignable<RedirectReceivedEvent>::value, "RedirectReceivedEvent is not move assignable"); + +struct RedirectSentEvent final : public TimedEvent { + RedirectSentEvent(const WebCore::ResourceRequest& request) + : TimedEvent() + , request(request) { } + RedirectSentEvent(CaptureTimeType&& time, Request&& request) + : TimedEvent(WTFMove(time)) + , request(WTFMove(request)) { } + + Request request; + static const char typeName[]; +}; +static_assert(std::is_move_constructible<RedirectSentEvent>::value, "RedirectSentEvent is not move constructible"); +static_assert(std::is_move_assignable<RedirectSentEvent>::value, "RedirectSentEvent is not move assignable"); + +struct DataReceivedEvent final : public TimedEvent { + DataReceivedEvent() + : TimedEvent() + , data(WebCore::SharedBuffer::create()) { } + DataReceivedEvent(WebCore::SharedBuffer& data) + : TimedEvent() + , data(Ref<WebCore::SharedBuffer>(data)) { } + DataReceivedEvent(CaptureTimeType&& time, WebCore::SharedBuffer& data) + : TimedEvent(WTFMove(time)) + , data(Ref<WebCore::SharedBuffer>(data)) { } + + Ref<WebCore::SharedBuffer> data; + static const char typeName[]; +}; +static_assert(std::is_move_constructible<DataReceivedEvent>::value, "DataReceivedEvent is not move constructible"); +static_assert(std::is_move_assignable<DataReceivedEvent>::value, "DataReceivedEvent is not move assignable"); + +struct FinishedEvent final : public TimedEvent { + FinishedEvent(const WebCore::ResourceError& error) + : TimedEvent() + , error(error) { } + FinishedEvent(CaptureTimeType&& time, Error&& error) + : TimedEvent(WTFMove(time)) + , error(WTFMove(error)) { } + + Error error; + static const char typeName[]; +}; +static_assert(std::is_move_constructible<FinishedEvent>::value, "FinishedEvent is not move constructible"); +static_assert(std::is_move_assignable<FinishedEvent>::value, "FinishedEvent is not move assignable"); + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureLogging.h b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureLogging.h new file mode 100644 index 000000000..9aa84cf7a --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureLogging.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CAPTURE) + +#include "Logging.h" + +#define ENABLE_WTF_CAPTURE_INTERNAL_DEBUGGING 0 +#define ENABLE_WTF_VERBOSE_CAPTURE_INTERNAL_DEBUGGING 0 + +#define DEBUG_STR(s) (s).ascii().data() + +#if RELEASE_LOG_DISABLED +#define STRING_SPECIFIER "%s" +#else +#define STRING_SPECIFIER "%{public}s" +#endif + +#if ENABLE(WTF_CAPTURE_INTERNAL_DEBUGGING) +#define DEBUG_LOG_QUOTE(str) #str +#define DEBUG_LOG_EXPAND_AND_QUOTE(str) DEBUG_LOG_QUOTE(str) +#define DEBUG_LOG(format, ...) RELEASE_LOG(Network, "#PLT: %p - " STRING_SPECIFIER "::" STRING_SPECIFIER ": " format, this, DEBUG_LOG_EXPAND_AND_QUOTE(DEBUG_CLASS), __FUNCTION__, ##__VA_ARGS__) +#define DEBUG_LOG_ERROR(format, ...) RELEASE_LOG_ERROR(Network, "#PLT: %p - " STRING_SPECIFIER "::" STRING_SPECIFIER ": " format, this, DEBUG_LOG_EXPAND_AND_QUOTE(DEBUG_CLASS), __FUNCTION__, ##__VA_ARGS__) +#if ENABLE(WTF_VERBOSE_CAPTURE_INTERNAL_DEBUGGING) +#define DEBUG_LOG_VERBOSE(format, ...) DEBUG_LOG(format, ##__VA_ARGS__) +#else +#define DEBUG_LOG_VERBOSE(...) ((void)0) +#endif +#else +#define DEBUG_LOG(...) ((void)0) +#define DEBUG_LOG_ERROR(...) RELEASE_LOG_ERROR(Network, __VA_ARGS__) +#define DEBUG_LOG_VERBOSE(...) ((void)0) +#endif + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureManager.cpp b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureManager.cpp new file mode 100644 index 000000000..bf8ff4d50 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureManager.cpp @@ -0,0 +1,584 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCaptureManager.h" + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureLogging.h" +#include "NetworkCaptureResource.h" +#include "WebCore/ResourceRequest.h" +#include "WebCore/URL.h" +#include <wtf/MD5.h> +#include <wtf/text/Base64.h> +#include <wtf/text/StringBuilder.h> + +#include <algorithm> +#include <iterator> +#include <limits> + +#define DEBUG_CLASS Manager + +namespace WebKit { +namespace NetworkCapture { + +static const char* kDirNameRecordReplay = "WebKitPerf/record_replay"; +static const char* kDirNameResources = "resources"; +static const char* kFileNameReportLoad = "report_load.txt"; +static const char* kFileNameReportRecord = "report_record.txt"; +static const char* kFileNameReportReplay = "report_replay.txt"; + +static int kMaxMatch = std::numeric_limits<int>::max(); +static int kMinMatch = std::numeric_limits<int>::min(); + +Manager& Manager::singleton() +{ + static NeverDestroyed<Manager> instance; + return instance; +} + +void Manager::initialize(const String& recordReplayMode, const String& recordReplayCacheLocation) +{ + if (equalIgnoringASCIICase(recordReplayMode, "record")) { + DEBUG_LOG("Initializing: recording mode"); + m_recordReplayMode = Record; + } else if (equalIgnoringASCIICase(recordReplayMode, "replay")) { + DEBUG_LOG("Initializing: replay mode"); + m_recordReplayMode = Replay; + } else { + DEBUG_LOG("Initializing: disabled"); + m_recordReplayMode = Disabled; + } + + m_recordReplayCacheLocation = WebCore::pathByAppendingComponent(recordReplayCacheLocation, kDirNameRecordReplay); + DEBUG_LOG("Cache location = " STRING_SPECIFIER, DEBUG_STR(m_recordReplayCacheLocation)); + + if (isRecording()) { + m_recordFileHandle = WebCore::FileHandle(reportRecordPath(), WebCore::OpenForWrite); + } else if (isReplaying()) { + m_recordFileHandle = WebCore::FileHandle(reportRecordPath(), WebCore::OpenForRead); + m_loadFileHandle = WebCore::FileHandle(reportLoadPath(), WebCore::OpenForWrite); + m_replayFileHandle = WebCore::FileHandle(reportReplayPath(), WebCore::OpenForWrite); + loadResources(); + } +} + +void Manager::terminate() +{ + m_loadFileHandle.close(); + m_recordFileHandle.close(); + m_replayFileHandle.close(); +} + +Resource* Manager::findMatch(const WebCore::ResourceRequest& request) +{ + DEBUG_LOG_VERBOSE("URL = " STRING_SPECIFIER, DEBUG_STR(request.url().string())); + + auto bestMatch = findExactMatch(request); + if (!bestMatch) + bestMatch = findBestFuzzyMatch(request); + +#if ENABLE(WTF_CAPTURE_INTERNAL_DEBUGGING) + if (!bestMatch) + DEBUG_LOG("Could not find match for: " STRING_SPECIFIER, DEBUG_STR(request.url().string())); + else if (request.url() == bestMatch->url()) + DEBUG_LOG("Found exact match for: " STRING_SPECIFIER, DEBUG_STR(request.url().string())); + else { + DEBUG_LOG("Found fuzzy match for: " STRING_SPECIFIER, DEBUG_STR(request.url().string())); + DEBUG_LOG(" replaced with : " STRING_SPECIFIER, DEBUG_STR(bestMatch->url().string())); + } +#endif + + return bestMatch; +} + +Resource* Manager::findExactMatch(const WebCore::ResourceRequest& request) +{ + const auto& url = request.url(); + auto lower = std::lower_bound(std::begin(m_cachedResources), std::end(m_cachedResources), url, [](auto& resource, const auto& url) { + return WTF::codePointCompareLessThan(resource.url().string(), url.string()); + }); + + if (lower != std::end(m_cachedResources) && lower->url() == url) { + DEBUG_LOG_VERBOSE("Found exact match: " STRING_SPECIFIER, DEBUG_STR(lower->url().string())); + return &*lower; + } + + return nullptr; +} + +Resource* Manager::findBestFuzzyMatch(const WebCore::ResourceRequest& request) +{ + const auto& url = request.url(); + const auto& urlIdentifyingCommonDomain = Manager::urlIdentifyingCommonDomain(url); + + const auto& lower = std::lower_bound(std::begin(m_cachedResources), std::end(m_cachedResources), urlIdentifyingCommonDomain, [](auto& resource, const auto& urlIdentifyingCommonDomain) { + return WTF::codePointCompareLessThan(resource.urlIdentifyingCommonDomain(), urlIdentifyingCommonDomain); + }); + const auto& upper = std::upper_bound(lower, std::end(m_cachedResources), urlIdentifyingCommonDomain, [](const auto& urlIdentifyingCommonDomain, auto& resource) { + return WTF::codePointCompareLessThan(urlIdentifyingCommonDomain, resource.urlIdentifyingCommonDomain()); + }); + + Resource* bestMatch = nullptr; + int bestScore = kMinMatch; + const auto& requestParameters = WebCore::URLParser::parseURLEncodedForm(url.query()); + for (auto iResource = lower; iResource != upper; ++iResource) { + int thisScore = fuzzyMatchURLs(url, requestParameters, iResource->url(), iResource->queryParameters()); + // TODO: Consider ignoring any matches < 0 as being too different. + if (bestScore < thisScore) { + DEBUG_LOG("New best match (%d): " STRING_SPECIFIER, thisScore, DEBUG_STR(iResource->url().string())); + bestScore = thisScore; + bestMatch = &*iResource; + if (bestScore == kMaxMatch) + break; + } + } + + return bestMatch; +} + +// TODO: Convert to an interface based on ResourceRequest so that we can do +// deeper matching. + +int Manager::fuzzyMatchURLs(const WebCore::URL& requestURL, const WebCore::URLParser::URLEncodedForm& requestParameters, const WebCore::URL& resourceURL, const WebCore::URLParser::URLEncodedForm& resourceParameters) +{ + // TODO: consider requiring that any trailing suffixes (e.g., ".js", + // ".png", ".css", ".html", etc.) should be an exact match. + + // We do fuzzy matching on the path and query parameters. So let's first + // make sure that all the other parts are equal. + + // If scheme, host, and port don't all match, return this as the "worst" + // match. + + if (!protocolHostAndPortAreEqual(requestURL, resourceURL)) { + DEBUG_LOG("Scheme/host/port mismatch: " STRING_SPECIFIER " != " STRING_SPECIFIER, DEBUG_STR(requestURL.string()), DEBUG_STR(resourceURL.string())); + return kMinMatch; + } + + // If fragments don't match, return this as the "worst" match. + + if (requestURL.fragmentIdentifier() != resourceURL.fragmentIdentifier()) { + DEBUG_LOG("Fragments mismatch: " STRING_SPECIFIER " != " STRING_SPECIFIER, DEBUG_STR(requestURL.string()), DEBUG_STR(resourceURL.string())); + return kMinMatch; + } + + DEBUG_LOG("Fuzzy matching:"); + DEBUG_LOG(" : " STRING_SPECIFIER, DEBUG_STR(requestURL.string())); + DEBUG_LOG(" : " STRING_SPECIFIER, DEBUG_STR(resourceURL.string())); + + // Compare the path components and the query parameters. Score each partial + // match as +4, each mismatch as -1, and each missing component as -1. + // + // Note that at the current time these values are rather arbitrary and + // could fine-tuned. + + const int kPathMatchScore = 4; + const int kPathMismatchScore = -1; + const int kPathMissingScore = -1; + const int kParameterMatchScore = 4; + const int kParameterMismatchScore = -1; + const int kParameterMissingScore = -1; + + int score = 0; + + // Quantize the differences in URL paths. + // + // The approach here is to increase our score for each matching path + // component, and to subtract for each differing component as well as for + // components that exist in one path but not the other. + + const auto& requestPath = requestURL.path(); + const auto& resourcePath = resourceURL.path(); + + Vector<String> requestPathComponents, resourcePathComponents; + requestPath.split('/', requestPathComponents); + resourcePath.split('/', resourcePathComponents); + + auto updatedIterators = std::mismatch( + std::begin(requestPathComponents), std::end(requestPathComponents), + std::begin(resourcePathComponents), std::end(resourcePathComponents)); + + auto matchingDistance = std::distance(std::begin(requestPathComponents), updatedIterators.first); + auto requestPathMismatchDistance = std::distance(updatedIterators.first, std::end(requestPathComponents)); + auto resourcePathMismatchDistance = std::distance(updatedIterators.second, std::end(resourcePathComponents)); + decltype(matchingDistance) mismatchingDistance; + decltype(matchingDistance) missingDistance; + if (requestPathMismatchDistance < resourcePathMismatchDistance) { + mismatchingDistance = requestPathMismatchDistance; + missingDistance = resourcePathMismatchDistance - requestPathMismatchDistance; + } else { + mismatchingDistance = resourcePathMismatchDistance; + missingDistance = requestPathMismatchDistance - resourcePathMismatchDistance; + } + + DEBUG_LOG("Path matching results: matching = %d, mismatching = %d, missing = %d", + static_cast<int>(matchingDistance), + static_cast<int>(mismatchingDistance), + static_cast<int>(missingDistance)); + + score += matchingDistance * kPathMatchScore + + mismatchingDistance * kPathMismatchScore + + missingDistance * kPathMissingScore; + DEBUG_LOG("Score = %d", score); + + // Quantize the differences in query parameters. + // + // The approach here is to walk lock-step over the two sets of query + // parameters. For each pair of parameters for each URL, we compare their + // names and values. If the names and values match, we add a high score. If + // just the names match, we add a lower score. + // + // If the names don't match, we then assume that some intervening query + // parameters have been added to one or the other URL. We therefore try to + // sync up the iterators used to traverse the query parameter collections + // so that they're again pointing to parameters with the same names. We + // first start scanning forward down the query parameters for one URL, + // looking for one with the same name as the one we're on in the other URL. + // If that doesn't turn up a match, we reverse the roles of the query + // parameters perform the same process of scanning forward. If neither of + // these scans produces a match, we figure that each query parameter we're + // looking at from each of the query parameter collections is unique. We + // deduct points from the overall score and move on to the next query + // parameters in each set. + // + // If, on the other hand, the forward-scanning does turn up a match, we + // adjust out iterators so that they're now again pointing to query + // parameters with the same name. This synchronization involves skipping + // over any intervening query parameters in one collection or the other. + // The assumption here is that these intervening query parameters are + // insertions that exist in one URL but not the other. We treat them as + // such, subtracting from the overall score for each one. However, this + // assumption might easily be incorrect. It might be that the query + // parameters that we're skipping over in one URL might exist in the other + // URL. If so, then we are foregoing the possibility of using those matches + // to increase the overall match score between the two URLs. + // + // To address this problem, we might want to consider sorting the query + // parameters by their names. However, doing this may cause problems if the + // order of the parameters is significant. So if we decide to take the + // approach of sorting the parameters, keep in mind this possible drawback. + + auto requestParameter = std::begin(requestParameters); + auto resourceParameter = std::begin(resourceParameters); + + for (; requestParameter != std::end(requestParameters) && resourceParameter != std::end(resourceParameters); ++requestParameter, ++resourceParameter) { + if (requestParameter->key == resourceParameter->key) { +#if ENABLE(WTF_CAPTURE_INTERNAL_DEBUGGING) + if (requestParameter->value == resourceParameter->value) + DEBUG_LOG("Matching parameter names and values: \"" STRING_SPECIFIER "\" = \"" STRING_SPECIFIER "\"", DEBUG_STR(requestParameter->first), DEBUG_STR(requestParameter->second)); + else + DEBUG_LOG("Mismatching parameter values: \"" STRING_SPECIFIER "\" = \"" STRING_SPECIFIER "\" vs. \"" STRING_SPECIFIER "\"", DEBUG_STR(requestParameter->first), DEBUG_STR(requestParameter->second), DEBUG_STR(resourceParameter->second)); +#endif + score += (requestParameter->value == resourceParameter->value) ? kParameterMatchScore : kParameterMismatchScore; + DEBUG_LOG("Score = %d", score); + } else { + DEBUG_LOG("Mismatching parameter names: " STRING_SPECIFIER ", " STRING_SPECIFIER, DEBUG_STR(requestParameter->first), DEBUG_STR(resourceParameter->first)); + + const auto scanForwardForMatch = [this, &score, kParameterMatchScore, kParameterMismatchScore, kParameterMissingScore](const auto& fixedIter, auto& scanningIter, const auto& scannerEnd) { + auto scanner = scanningIter; + while (scanner != scannerEnd && scanner->key != fixedIter->key) + ++scanner; + if (scanner == scannerEnd) + return false; + DEBUG_LOG("Skipping past %d non-matching parameter names", static_cast<int>(std::distance(scanningIter, scanner))); + score += kParameterMissingScore * std::distance(scanningIter, scanner); + DEBUG_LOG("Score = %d", score); +#if ENABLE(WTF_CAPTURE_INTERNAL_DEBUGGING) + if (fixedIter->second == scanner->second) + DEBUG_LOG("Matching parameter names and values: \"" STRING_SPECIFIER "\" = \"" STRING_SPECIFIER "\"", DEBUG_STR(fixedIter->first), DEBUG_STR(fixedIter->second)); + else + DEBUG_LOG("Mismatching parameter values: \"" STRING_SPECIFIER "\" = \"" STRING_SPECIFIER "\" vs. \"" STRING_SPECIFIER "\"", DEBUG_STR(fixedIter->first), DEBUG_STR(fixedIter->second), DEBUG_STR(scanner->second)); +#endif + score += (fixedIter->value == scanner->value) ? kParameterMatchScore : kParameterMismatchScore; + DEBUG_LOG("Score = %d", score); + scanningIter = scanner; + return true; + }; + + if (!scanForwardForMatch(requestParameter, resourceParameter, std::end(resourceParameters))) { + if (!scanForwardForMatch(resourceParameter, requestParameter, std::end(requestParameters))) { + DEBUG_LOG("Unmatched parameter: " STRING_SPECIFIER "=" STRING_SPECIFIER, DEBUG_STR(requestParameter->first), DEBUG_STR(requestParameter->second)); + DEBUG_LOG("Unmatched parameter: " STRING_SPECIFIER "=" STRING_SPECIFIER, DEBUG_STR(resourceParameter->first), DEBUG_STR(resourceParameter->second)); + score += kParameterMissingScore + kParameterMissingScore; + DEBUG_LOG("Score = %d", score); + } + } + } + } + + DEBUG_LOG("Adjusting for trailing parameters"); + score += kParameterMissingScore + * (std::distance(requestParameter, std::end(requestParameters)) + + std::distance(resourceParameter, std::end(resourceParameters))); + DEBUG_LOG("Score = %d", score); + + return score; +} + +void Manager::loadResources() +{ + auto lines = readFile(reportRecordPath()); + if (!lines) + return; + + for (const auto& line : *lines) { + if (line.size() != 2) { + DEBUG_LOG_ERROR("line.size == %d", (int) line.size()); + continue; + } + + Resource newResource(hashToPath(line[0])); + m_cachedResources.append(WTFMove(newResource)); + } + + std::sort(std::begin(m_cachedResources), std::end(m_cachedResources), [](auto& left, auto& right) { + return WTF::codePointCompareLessThan(left.url().string(), right.url().string()); + }); + + for (auto& resource : m_cachedResources) + logLoadedResource(resource); +} + +String Manager::reportLoadPath() +{ + return WebCore::pathByAppendingComponent(m_recordReplayCacheLocation, kFileNameReportLoad); +} + +String Manager::reportRecordPath() +{ + return WebCore::pathByAppendingComponent(m_recordReplayCacheLocation, kFileNameReportRecord); +} + +String Manager::reportReplayPath() +{ + return WebCore::pathByAppendingComponent(m_recordReplayCacheLocation, kFileNameReportReplay); +} + +String Manager::requestToPath(const WebCore::ResourceRequest& request) +{ + // TODO: come up with a more comprehensive hash that includes HTTP method + // and possibly other values (such as headers). + + const auto& hash = stringToHash(request.url().string()); + const auto& path = hashToPath(hash); + return path; +} + +String Manager::stringToHash(const String& s) +{ + WTF::MD5 md5; + if (s.characters8()) + md5.addBytes(static_cast<const uint8_t*>(s.characters8()), s.length()); + else + md5.addBytes(reinterpret_cast<const uint8_t*>(s.characters16()), 2 * s.length()); + + WTF::MD5::Digest digest; + md5.checksum(digest); + + return WTF::base64URLEncode(&digest[0], WTF::MD5::hashSize); +} + +String Manager::hashToPath(const String& hash) +{ + auto hashHead = hash.substring(0, 2); + auto hashTail = hash.substring(2); + + StringBuilder fileName; + fileName.append(hashTail); + fileName.append(".data"); + + auto path = WebCore::pathByAppendingComponent(m_recordReplayCacheLocation, kDirNameResources); + path = WebCore::pathByAppendingComponent(path, hashHead); + path = WebCore::pathByAppendingComponent(path, fileName.toString()); + + return path; +} + +String Manager::urlIdentifyingCommonDomain(const WebCore::URL& url) +{ + return url.protocolHostAndPort(); +} + +void Manager::logRecordedResource(const WebCore::ResourceRequest& request) +{ + // Log network resources as they are cached to disk. + + const auto& url = request.url(); + m_recordFileHandle.printf("%s %s\n", DEBUG_STR(stringToHash(url.string())), DEBUG_STR(url.string())); +} + +void Manager::logLoadedResource(Resource& resource) +{ + // Log cached resources as they are loaded from disk. + + m_loadFileHandle.printf("%s\n", DEBUG_STR(resource.url().string())); +} + +void Manager::logPlayedBackResource(const WebCore::ResourceRequest& request, bool wasCacheMiss) +{ + // Log network resources that are requested during replay. + + const auto& url = request.url(); + + if (wasCacheMiss) + DEBUG_LOG("Cache miss: URL = " STRING_SPECIFIER, DEBUG_STR(url.string())); + else + DEBUG_LOG("Cache hit: URL = " STRING_SPECIFIER, DEBUG_STR(url.string())); + + m_replayFileHandle.printf("%s %s\n", wasCacheMiss ? "miss" : "hit ", DEBUG_STR(url.string())); +} + +WebCore::FileHandle Manager::openCacheFile(const String& filePath, WebCore::FileOpenMode mode) +{ + // If we can trivially open the file, then do that and return the new file + // handle. + + auto fileHandle = WebCore::FileHandle(filePath, mode); + if (fileHandle.open()) + return fileHandle; + + // If we're opening the file for writing (including appending), then try + // again after making sure all intermediate directories have been created. + + if (mode != WebCore::OpenForRead) { + const auto& parentDir = WebCore::directoryName(filePath); + if (!WebCore::makeAllDirectories(parentDir)) { + DEBUG_LOG_ERROR("Error %d trying to create intermediate directories: " STRING_SPECIFIER, errno, DEBUG_STR(parentDir)); + return fileHandle; + } + + fileHandle = WebCore::FileHandle(filePath, mode); + if (fileHandle.open()) + return fileHandle; + } + + // Could not open the file. Log the error and leave, returning the invalid + // file handle. + + if (mode == WebCore::OpenForRead) + DEBUG_LOG_ERROR("Error %d trying to open " STRING_SPECIFIER " for reading", errno, DEBUG_STR(filePath)); + else + DEBUG_LOG_ERROR("Error %d trying to open " STRING_SPECIFIER " for writing", errno, DEBUG_STR(filePath)); + + return fileHandle; +} + +std::optional<Vector<Vector<String>>> Manager::readFile(const String& filePath) +{ + bool success = false; + WebCore::MappedFileData file(filePath, success); + if (!success) + return std::nullopt; + + Vector<Vector<String>> lines; + auto begin = static_cast<const uint8_t*>(file.data()); + auto end = begin + file.size(); + + Vector<String> line; + while (getLine(begin, end, line)) + lines.append(WTFMove(line)); + + return WTFMove(lines); +} + +bool Manager::getLine(uint8_t const *& p, uint8_t const * const end, Vector<String>& line) +{ + // NB: Returns true if there may be more data to get, false if we've hit + // the end of the buffer. + + DEBUG_LOG_VERBOSE("Getting a line"); + + line.clear(); + + if (p == end) { + DEBUG_LOG_VERBOSE("Iterator at end; returning false"); + return false; + } + + String word; + while (getWord(p, end, word)) { + if (!word.isEmpty()) { + DEBUG_LOG_VERBOSE("Adding word: " STRING_SPECIFIER, DEBUG_STR(word)); + line.append(word); + } + } + + return true; +} + +bool Manager::getWord(uint8_t const *& p, uint8_t const * const end, String& word) +{ + // NB: Returns true if a (possibly empty) word was found and there may be + // more, false if we've hit the end of line or buffer. + + DEBUG_LOG_VERBOSE("Getting a word"); + + if (p == end) { + DEBUG_LOG_VERBOSE("Iterator at end; returning false"); + return false; + } + + if (*p == '\n') { + DEBUG_LOG_VERBOSE("Iterator hit EOL; returning false"); + ++p; + return false; + } + + bool escaping = false; + bool ignoring = false; + + word = String(); + + DEBUG_LOG_VERBOSE("Iterating"); + + for ( ; p != end; ++p) { + if (ignoring) { + if (*p == '\n') + break; + } else if (escaping) { + word.append(*p); + escaping = false; + } else if (*p == '#') { + ignoring = true; + } else if (*p == '\\') { + escaping = true; + } else if (*p == ' ') { + if (!word.isEmpty()) + break; + } else if (*p == '\n') + break; + else + word.append(*p); + } + + return true; +} + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureManager.h b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureManager.h new file mode 100644 index 000000000..66b639f48 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureManager.h @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CAPTURE) + +#include <WebCore/FileHandle.h> +#include <WebCore/FileSystem.h> +#include <WebCore/URLParser.h> +#include <wtf/Function.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/Vector.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class ResourceRequest; +} + +namespace WebKit { +namespace NetworkCapture { + +class Resource; + +/* + * NetworkCapture::Manager serves three purposes: + * + * * It keeps the state of whether we are recording, replaying, or neither. + * * It keeps the list of cached resources (if replaying), performs fuzzy + * matching on them. + * * It has utilities for logging and file management. + * + * TODO: Perhaps we should break this up into three classes? + */ +class Manager { + WTF_MAKE_NONCOPYABLE(Manager); + friend class WTF::NeverDestroyed<Manager>; + +public: + enum RecordReplayMode { + Disabled, + Record, + Replay + }; + + static Manager& singleton(); + + void initialize(const String& recordReplayMode, const String& recordReplayCacheLocation); + void terminate(); + + bool isRecording() const { return mode() == RecordReplayMode::Record; } + bool isReplaying() const { return mode() == RecordReplayMode::Replay; } + RecordReplayMode mode() const { return m_recordReplayMode; } + + Resource* findMatch(const WebCore::ResourceRequest&); + + void logRecordedResource(const WebCore::ResourceRequest&); + void logLoadedResource(Resource&); + void logPlayedBackResource(const WebCore::ResourceRequest&, bool wasCacheMiss); + + WebCore::FileHandle openCacheFile(const String&, WebCore::FileOpenMode); + + String requestToPath(const WebCore::ResourceRequest&); + static String urlIdentifyingCommonDomain(const WebCore::URL&); + +private: + Manager() = default; + ~Manager() = delete; + + Resource* findExactMatch(const WebCore::ResourceRequest&); + Resource* findBestFuzzyMatch(const WebCore::ResourceRequest&); + int fuzzyMatchURLs(const WebCore::URL& requestURL, const WebCore::URLParser::URLEncodedForm& requestParameters, const WebCore::URL& resourceURL, const WebCore::URLParser::URLEncodedForm& resourceParameters); + + void loadResources(); + + String reportLoadPath(); + String reportRecordPath(); + String reportReplayPath(); + + String stringToHash(const String&); + String hashToPath(const String& hash); + + std::optional<Vector<Vector<String>>> readFile(const String& filePath); + bool getLine(uint8_t const *& p, uint8_t const * const end, Vector<String>& line); + bool getWord(uint8_t const *& p, uint8_t const * const end, String& word); + + RecordReplayMode m_recordReplayMode { Disabled }; + String m_recordReplayCacheLocation; + + WebCore::FileHandle m_loadFileHandle; + WebCore::FileHandle m_recordFileHandle; + WebCore::FileHandle m_replayFileHandle; + + Vector<Resource> m_cachedResources; +}; + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureRecorder.cpp b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureRecorder.cpp new file mode 100644 index 000000000..e6603e636 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureRecorder.cpp @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCaptureRecorder.h" + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureLogging.h" +#include "NetworkCaptureManager.h" +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> + +#define DEBUG_CLASS Recorder + +namespace WebKit { +namespace NetworkCapture { + +void Recorder::recordRequestSent(const WebCore::ResourceRequest& request) +{ + // This class records NetworkLoad's process of loading a network resource. + // NetworkLoad does this by creating a NetworkDataTask and calling resume + // on it. This call to resume can be called immediately after the task has + // been created, or -- if the loading is marked as deferred -- it can be + // called later in NetworkLoad::setDeferred(true). In fact, the latter can + // be called multiple times if the loading is suspended and resumed + // multiple times. + // + // This method is called in both places where resume is called. Our task is + // to catch the call to NetworkDataTask::resume that starts the network + // loading process. We want to ignore the other calls to resume. Our + // approach to knowing which one is the first is to check our collection of + // recorded events. If it is empty, then this is the first call into the + // recorder, and we want to record the event. Otherwise, ignore it. + + if (m_events.size()) + return; + + DEBUG_LOG("Sent request for URL = " STRING_SPECIFIER, DEBUG_STR(request.url().string())); + + m_initialRequest = request; + recordEvent(RequestSentEvent(request)); +} + +void Recorder::recordResponseReceived(const WebCore::ResourceResponse& response) +{ + // Called when receiving a response other than a redirect or error. + + DEBUG_LOG("Received response from URL = " STRING_SPECIFIER, DEBUG_STR(response.url().string())); + ASSERT(m_events.size()); + + // TODO: Is there a better response to receiving a multi-part resource? + // Learn more about multi-part resources. Why don't we record these? (Note, + // this decision is based on some NetworkCache code.) + + if (!response.isMultipart()) + recordEvent(ResponseReceivedEvent(response)); + else + m_events.clear(); +} + +void Recorder::recordRedirectReceived(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response) +{ + DEBUG_LOG("Received redirect to URL = " STRING_SPECIFIER, DEBUG_STR(request.url().string())); + ASSERT(m_events.size()); + + recordEvent(RedirectReceivedEvent(request, response)); +} + +void Recorder::recordRedirectSent(const WebCore::ResourceRequest& request) +{ + DEBUG_LOG("Sent redirect for URL = " STRING_SPECIFIER, DEBUG_STR(request.url().string())); + ASSERT(m_events.size()); + + recordEvent(RedirectSentEvent(request)); +} + +void Recorder::recordDataReceived(WebCore::SharedBuffer& buffer) +{ + DEBUG_LOG("Received %u bytes of data", buffer.size()); + + if (!m_events.size()) + return; + + // Prevent memory growth in case of streaming data. TODO: Is there a better + // response to this? If we encounter this condition, all of our recording + // silently goes out the window. Replay will not work, and the user doesn't + // know that. + + constexpr size_t kMaximumCacheBufferSize = 10 * 1024 * 1024; + m_dataLength += buffer.size(); + if (m_dataLength <= kMaximumCacheBufferSize) + recordEvent(DataReceivedEvent(buffer)); + else + m_events.clear(); +} + +void Recorder::recordFinish(const WebCore::ResourceError& error) +{ + DEBUG_LOG("Finished"); + + if (!m_events.size()) + return; + + recordEvent(FinishedEvent(error)); + writeEvents(); +} + +void Recorder::writeEvents() +{ + auto path = Manager::singleton().requestToPath(m_initialRequest); + auto handle = Manager::singleton().openCacheFile(path, WebCore::OpenForWrite); + if (!handle) + return; + + for (auto const& event : m_events) { + auto asString = eventToString(event); + // Write out the JSON string with the terminating NUL. This allows us + // to better find the separate JSON objects that we write to a single + // file. It also works better with JSON parsers that expect to find a + // NUL at the end of their input. + if (handle.write(asString.c_str(), asString.size() + 1) == -1) { + DEBUG_LOG_ERROR("Error trying to write to file for URL = " STRING_SPECIFIER, DEBUG_STR(m_initialRequest.url().string())); + return; + } + } + + Manager::singleton().logRecordedResource(m_initialRequest); +} + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/AsynchronousNetworkLoaderClient.h b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureRecorder.h index ab8aacf2a..8fd531300 100644 --- a/Source/WebKit2/NetworkProcess/AsynchronousNetworkLoaderClient.h +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureRecorder.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Apple Inc. All rights reserved. + * Copyright (C) 2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -23,34 +23,46 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef AsynchronousNetworkLoaderClient_h -#define AsynchronousNetworkLoaderClient_h +#pragma once -#include "NetworkLoaderClient.h" -#include "ShareableResource.h" +#if ENABLE(NETWORK_CAPTURE) -#if ENABLE(NETWORK_PROCESS) +#include "NetworkCaptureEvent.h" +#include <WebCore/ResourceRequest.h> + +namespace WebCore { +class ResourceError; +class ResourceResponse; +class SharedBuffer; +} namespace WebKit { +namespace NetworkCapture { -class AsynchronousNetworkLoaderClient : public NetworkLoaderClient { +class Recorder { public: - AsynchronousNetworkLoaderClient(); + void recordRequestSent(const WebCore::ResourceRequest&); + void recordResponseReceived(const WebCore::ResourceResponse&); + void recordRedirectReceived(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&); + void recordRedirectSent(const WebCore::ResourceRequest&); + void recordDataReceived(WebCore::SharedBuffer&); + void recordFinish(const WebCore::ResourceError&); private: - virtual void willSendRequest(NetworkResourceLoader*, WebCore::ResourceRequest& newRequest, const WebCore::ResourceResponse& redirectResponse) override; -#if USE(PROTECTION_SPACE_AUTH_CALLBACK) - virtual void canAuthenticateAgainstProtectionSpace(NetworkResourceLoader*, const WebCore::ProtectionSpace&) override; -#endif - virtual void didReceiveResponse(NetworkResourceLoader*, const WebCore::ResourceResponse&) override; - virtual void didReceiveBuffer(NetworkResourceLoader*, WebCore::SharedBuffer*, int encodedDataLength) override; - virtual void didSendData(NetworkResourceLoader*, unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override; - virtual void didFinishLoading(NetworkResourceLoader*, double finishTime) override; - virtual void didFail(NetworkResourceLoader*, const WebCore::ResourceError&) override; + void writeEvents(); + + template <typename T> + void recordEvent(T&& event) + { + m_events.append(CaptureEvent(WTFMove(event))); + } + + CaptureEvents m_events; + WebCore::ResourceRequest m_initialRequest; + size_t m_dataLength { 0 }; }; +} // namespace NetworkCapture } // namespace WebKit -#endif // ENABLE(NETWORK_PROCESS) - -#endif // AsynchronousNetworkLoaderClient_h +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureReplayer.cpp b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureReplayer.cpp new file mode 100644 index 000000000..a04af69c7 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureReplayer.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCaptureReplayer.h" + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureLogging.h" +#include "NetworkCaptureManager.h" +#include "NetworkDataTaskReplay.h" +#include "NetworkLoadParameters.h" + +#define DEBUG_CLASS Replayer + +namespace WebKit { +namespace NetworkCapture { + +Ref<NetworkDataTask> Replayer::replayResource(NetworkSession& session, NetworkDataTaskClient& client, const NetworkLoadParameters& parameters) +{ + auto foundResource = Manager::singleton().findMatch(parameters.request); + Manager::singleton().logPlayedBackResource(parameters.request, !foundResource); + return NetworkDataTaskReplay::create(session, client, parameters, foundResource); +} + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureReplayer.h b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureReplayer.h new file mode 100644 index 000000000..087ec5060 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureReplayer.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CAPTURE) + +#include <wtf/Ref.h> + +namespace WebKit { +class NetworkDataTask; +class NetworkDataTaskClient; +class NetworkLoadParameters; +class NetworkSession; +} + +namespace WebKit { +namespace NetworkCapture { + +class Replayer { +public: + Ref<NetworkDataTask> replayResource(NetworkSession&, NetworkDataTaskClient&, const NetworkLoadParameters&); +}; + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureResource.cpp b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureResource.cpp new file mode 100644 index 000000000..704c0d6ad --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureResource.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCaptureResource.h" + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureEvent.h" +#include "NetworkCaptureLogging.h" +#include "NetworkCaptureManager.h" +#include "NetworkCaptureRecorder.h" + +namespace WebKit { +namespace NetworkCapture { + +Resource::Resource(const String& eventFilePath) + : m_eventFilePath(eventFilePath) +{ +} + +const WebCore::URL& Resource::url() +{ + if (!m_url.isValid()) { + auto events = eventStream(); + auto event = events.nextEvent(); + if (!event) + DEBUG_LOG_ERROR("Event stream does not contain events: file = " STRING_SPECIFIER, DEBUG_STR(m_eventFilePath)); + else if (!WTF::holds_alternative<RequestSentEvent>(*event)) + DEBUG_LOG_ERROR("Event stream does not have a requestSent event: file = " STRING_SPECIFIER, DEBUG_STR(m_eventFilePath)); + else { + auto requestSentEvent = WTF::get<RequestSentEvent>(*event); + WebCore::URLParser parser(requestSentEvent.request.url); + m_url = parser.result(); + } + } + + return m_url; +} + +const String& Resource::urlIdentifyingCommonDomain() +{ + if (m_urlIdentifyingCommonDomain.isNull()) + m_urlIdentifyingCommonDomain = Manager::urlIdentifyingCommonDomain(url()); + + return m_urlIdentifyingCommonDomain; +} + +WebCore::URLParser::URLEncodedForm Resource::queryParameters() +{ + if (!m_queryParameters) + m_queryParameters = WebCore::URLParser::parseURLEncodedForm(url().query()); + + return *m_queryParameters; +} + +Resource::EventStream Resource::eventStream() +{ + return EventStream(m_eventFilePath); +} + +Resource::EventStream::EventStream(const String& eventFilePath) + : m_eventFilePath(eventFilePath) + , m_mappedEventFile(m_eventFilePath, m_haveMappedEventFile) +{ +} + +OptionalCaptureEvent Resource::EventStream::nextEvent() +{ + if (m_offset == m_mappedEventFile.size()) { + DEBUG_LOG_ERROR("Unable to return event - at end of file: " STRING_SPECIFIER, DEBUG_STR(m_eventFilePath)); + return std::nullopt; + } + + const char* charBuffer = static_cast<const char*>(m_mappedEventFile.data()); + const char* current = charBuffer + m_offset; + + while (m_offset < m_mappedEventFile.size() && charBuffer[m_offset]) + ++m_offset; + + if (m_offset == m_mappedEventFile.size()) { + DEBUG_LOG_ERROR("Unable to return event - no terminating NUL: " STRING_SPECIFIER, DEBUG_STR(m_eventFilePath)); + return std::nullopt; + } + + ++m_offset; + + return stringToEvent(current); +} + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkCaptureResource.h b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureResource.h new file mode 100644 index 000000000..e5760ed1b --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkCaptureResource.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureEvent.h" +#include <WebCore/FileSystem.h> +#include <WebCore/URL.h> +#include <WebCore/URLParser.h> +#include <wtf/Optional.h> + +namespace WebKit { +namespace NetworkCapture { + +class Resource { +public: + class EventStream { + public: + EventStream() = default; + EventStream(const String& eventFilePath); + + OptionalCaptureEvent nextEvent(); + + private: + String m_eventFilePath; + bool m_haveMappedEventFile { false }; + WebCore::MappedFileData m_mappedEventFile; + size_t m_offset { 0 }; + }; + +public: + Resource(const String& eventFilePath); + + const WebCore::URL& url(); + const String& urlIdentifyingCommonDomain(); + WebCore::URLParser::URLEncodedForm queryParameters(); + EventStream eventStream(); + +private: + String m_eventFilePath; + WebCore::URL m_url; + String m_urlIdentifyingCommonDomain; + std::optional<WebCore::URLParser::URLEncodedForm> m_queryParameters; +}; + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkDataTaskReplay.cpp b/Source/WebKit2/NetworkProcess/capture/NetworkDataTaskReplay.cpp new file mode 100644 index 000000000..30d730252 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkDataTaskReplay.cpp @@ -0,0 +1,288 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkDataTaskReplay.h" + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureEvent.h" +#include "NetworkCaptureLogging.h" +#include "NetworkCaptureResource.h" +#include "NetworkLoadParameters.h" +#include "NetworkSession.h" +#include <WebCore/ResourceError.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <wtf/RunLoop.h> + +#define DEBUG_CLASS NetworkDataTaskReplay + +namespace WebKit { +namespace NetworkCapture { + +static const char* const webKitRelayDomain = "WebKitReplay"; + +NetworkDataTaskReplay::NetworkDataTaskReplay(NetworkSession& session, NetworkDataTaskClient& client, const NetworkLoadParameters& parameters, Resource* resource) + : NetworkDataTask(session, client, parameters.request, parameters.allowStoredCredentials, parameters.shouldClearReferrerOnHTTPSToHTTPRedirect) + , m_currentRequest(m_firstRequest) + , m_resource(resource) +{ + DEBUG_LOG("request URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + DEBUG_LOG("cached URL = " STRING_SPECIFIER, resource ? DEBUG_STR(resource->url().string()) : "<not found>"); + + m_session->registerNetworkDataTask(*this); + + if (resource) + m_eventStream = resource->eventStream(); +} + +NetworkDataTaskReplay::~NetworkDataTaskReplay() +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + m_session->unregisterNetworkDataTask(*this); +} + +void NetworkDataTaskReplay::resume() +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + if (m_state == State::Canceling || m_state == State::Completed) + return; + + m_state = State::Running; + + if (m_scheduledFailureType != NoFailure) { + ASSERT(m_failureTimer.isActive()); + return; + } + + enqueueEventHandler(); +} + +void NetworkDataTaskReplay::suspend() +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + if (m_state == State::Canceling || m_state == State::Completed) + return; + + m_state = State::Suspended; +} + +void NetworkDataTaskReplay::cancel() +{ + DEBUG_LOG(""); + + if (m_state == State::Canceling || m_state == State::Completed) + return; + + m_state = State::Canceling; +} + +void NetworkDataTaskReplay::complete() +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + if (m_state == State::Completed) + return; + + m_state = State::Completed; + m_resource = nullptr; +} + +void NetworkDataTaskReplay::invalidateAndCancel() +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + cancel(); + complete(); +} + +void NetworkDataTaskReplay::enqueueEventHandler() +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + RunLoop::main().dispatch([this, protectedThis = makeRef(*this)] { + DEBUG_LOG("enqueueEventHandler callback"); + + if (m_state == State::Suspended) + return; + + if (m_state == State::Canceling || m_state == State::Completed || !m_client) { + complete(); + return; + } + + if (!m_resource) { + DEBUG_LOG_ERROR("Error loading resource: could not find cached resource, URL = " STRING_SPECIFIER, DEBUG_STR(m_currentRequest.url().string())); + didFinish(Error::NotFoundError); // TODO: Turn this into a 404? + return; + } + + if (!equalLettersIgnoringASCIICase(m_currentRequest.httpMethod(), "get")) { + DEBUG_LOG_ERROR("Error loading resource: unsupported method (" STRING_SPECIFIER "), URL = " STRING_SPECIFIER, DEBUG_STR(m_currentRequest.httpMethod()), DEBUG_STR(m_currentRequest.url().string())); + didFinish(Error::MethodNotAllowed); + return; + } + + auto event = m_eventStream.nextEvent(); + if (!event) { + DEBUG_LOG_ERROR("Error loading resource: nextEvent return null, URL = " STRING_SPECIFIER, DEBUG_STR(m_currentRequest.url().string())); + didFinish(Error::NotFoundError); // TODO: Turn this into a 404? + return; + } + + const auto visitor = WTF::makeVisitor( + [this](const RequestSentEvent& event) { + replayRequestSent(event); + }, + [this](const ResponseReceivedEvent& event) { + replayResponseReceived(event); + }, + [this](const RedirectReceivedEvent& event) { + replayRedirectReceived(event); + }, + [this](const RedirectSentEvent& event) { + replayRedirectSent(event); + }, + [this](const DataReceivedEvent& event) { + replayDataReceived(event); + }, + [this](const FinishedEvent& event) { + replayFinished(event); + }); + + WTF::visit(visitor, *event); + }); +} + +void NetworkDataTaskReplay::replayRequestSent(const RequestSentEvent& event) +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + enqueueEventHandler(); +} + +void NetworkDataTaskReplay::replayResponseReceived(const ResponseReceivedEvent& event) +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + WebCore::ResourceResponse response(event.response); + didReceiveResponse(WTFMove(response)); +} + +void NetworkDataTaskReplay::replayRedirectReceived(const RedirectReceivedEvent& event) +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + WebCore::ResourceResponse receivedResponse = event.response; + WebCore::ResourceRequest receivedRequest = event.request; + + ASSERT(m_client); + m_client->willPerformHTTPRedirection(WTFMove(receivedResponse), WTFMove(receivedRequest), [this, protectedThis = makeRef(*this)] (const WebCore::ResourceRequest& updatedRequest) { + DEBUG_LOG("replayRedirectReceived callback: URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + m_currentRequest = updatedRequest; + enqueueEventHandler(); + }); +} + +void NetworkDataTaskReplay::replayRedirectSent(const RedirectSentEvent& event) +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + enqueueEventHandler(); +} + +void NetworkDataTaskReplay::replayDataReceived(const DataReceivedEvent& event) +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + ASSERT(m_client); + m_client->didReceiveData(event.data.copyRef()); + + enqueueEventHandler(); +} + +void NetworkDataTaskReplay::replayFinished(const FinishedEvent& event) +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + didFinish(event.error); +} + +void NetworkDataTaskReplay::didReceiveResponse(WebCore::ResourceResponse&& response) +{ + DEBUG_LOG("URL = " STRING_SPECIFIER, DEBUG_STR(m_firstRequest.url().string())); + + ASSERT(m_client); + m_client->didReceiveResponseNetworkSession(WTFMove(response), [this, protectedThis = makeRef(*this)](WebCore::PolicyAction policyAction) { + DEBUG_LOG("didReceiveResponse callback (%u)", static_cast<unsigned>(policyAction)); + + if (m_state == State::Canceling || m_state == State::Completed) { + complete(); + return; + } + + switch (policyAction) { + case WebCore::PolicyAction::PolicyUse: + enqueueEventHandler(); + break; + case WebCore::PolicyAction::PolicyIgnore: + complete(); + break; + case WebCore::PolicyAction::PolicyDownload: + DEBUG_LOG_ERROR("WebCore::PolicyAction::PolicyDownload"); + break; + } + }); +} + +void NetworkDataTaskReplay::didFinish() +{ + didFinish({ }); +} + +void NetworkDataTaskReplay::didFinish(Error errorCode) +{ + didFinish(WebCore::ResourceError(webKitRelayDomain, static_cast<int>(errorCode), m_firstRequest.url(), String())); +} + +void NetworkDataTaskReplay::didFinish(const WebCore::ResourceError& error) +{ + DEBUG_LOG("(%d)", error.errorCode()); + + complete(); + + ASSERT(m_client); + m_client->didCompleteWithError(error); +} + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/NetworkDataTaskReplay.h b/Source/WebKit2/NetworkProcess/capture/NetworkDataTaskReplay.h new file mode 100644 index 000000000..bd28a7d35 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/NetworkDataTaskReplay.h @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(NETWORK_CAPTURE) + +#include "NetworkCaptureResource.h" +#include "NetworkDataTask.h" +#include <WebCore/ResourceRequest.h> + +namespace WebCore { +class ResourceError; +class ResourceResponse; +} + +namespace WebKit { +class NetworkDataTaskClient; +class NetworkLoadParameters; +class NetworkSession; +} + +namespace WebKit { +namespace NetworkCapture { + +struct RequestSentEvent; +struct ResponseReceivedEvent; +struct RedirectReceivedEvent; +struct RedirectSentEvent; +struct DataReceivedEvent; +struct FinishedEvent; + +class NetworkDataTaskReplay : public NetworkDataTask { +public: + static Ref<NetworkDataTaskReplay> create(NetworkSession& session, NetworkDataTaskClient& client, const NetworkLoadParameters& parameters, Resource* resource) + { + return adoptRef(*new NetworkDataTaskReplay(session, client, parameters, resource)); + } + + void replayRequestSent(const RequestSentEvent&); + void replayResponseReceived(const ResponseReceivedEvent&); + void replayRedirectReceived(const RedirectReceivedEvent&); + void replayRedirectSent(const RedirectSentEvent&); + void replayDataReceived(const DataReceivedEvent&); + void replayFinished(const FinishedEvent&); + +private: + NetworkDataTaskReplay(NetworkSession&, NetworkDataTaskClient&, const NetworkLoadParameters&, Resource*); + ~NetworkDataTaskReplay() override; + + void resume() override; + void suspend() override; + void cancel() override; + void complete(); + void invalidateAndCancel() override; + + State state() const override { return m_state; } + + void enqueueEventHandler(); + + enum class Error { + NoError = 0, + NotFoundError = 1, + MethodNotAllowed = 5 + }; + + void didReceiveResponse(WebCore::ResourceResponse&&); + void didFinish(); + void didFinish(Error); + void didFinish(const WebCore::ResourceError&); + + State m_state { State::Suspended }; + WebCore::ResourceRequest m_currentRequest; + Resource* m_resource; + Resource::EventStream m_eventStream; +}; + +} // namespace NetworkCapture +} // namespace WebKit + +#endif // ENABLE(NETWORK_CAPTURE) diff --git a/Source/WebKit2/NetworkProcess/capture/json.hpp b/Source/WebKit2/NetworkProcess/capture/json.hpp new file mode 100644 index 000000000..10ac8e2fd --- /dev/null +++ b/Source/WebKit2/NetworkProcess/capture/json.hpp @@ -0,0 +1,10821 @@ +/* + * ************************************************************************** + * + * DO NOT USE THIS JSON LIBRARY! + * + * This JSON library exists as a temporary facility for reading and writing + * JSON files. Ultimately, we should be using the JSON facilities that are in + * JavaScriptCore. However, those are too heavy-weight for what we want in the + * NetworkProcess. The NetworkProcess currently doesn't use JavaScriptCore and + * doesn't allocate the data structures and memory-handling facilities that JSC + * requires. All we want is something that will do the simple streaming and + * parsing of JSON. To ultimately achieve this, we will be modifying the + * JSONParse in JSC to be lighter-weight, performing just the streaming or + * parsing and leaving the memory management and object creation to the caller. + * This will be similar to howe YarrParser works. The new JSONParse will be + * used here and in other places like in ContextExtensionParser. Until then, we + * use this library. + * + * In that context, this library should not be used by anyone else. It will be + * going away. Also, it has not been vetted for security issues that might + * arise if it were used in a context where customer data is being manipulated. + * + * DO NOT USE THIS JSON LIBRARY! + * + * ************************************************************************** + * + * This file was taken from <https://github.com/nlohmann/json> at revision + * e717492019687bf0edf2b884b51bf37db4831a6f and modified as follows for + * inclusion in WebKit: + * + * * Exceptions are disabled in WebKit, so all 'throw' statements were + * converted into a call to a Throw function. This function can be enabled + * to invoke 'throw' as before, but by default asserts and crashes. + * * try/catch statements are also disallowed, so these were converted into + * simple if/then checks where possible. Where not possible, the try/catch + * statements were simply removed with the expectation that no code paths + * would trigger the anticipated exceptions. + * * For efficiency, a basic_json constructor that takes a + * string_t::value_type* and a count was added. + * + * ************************************************************************** + */ + +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 2.0.7 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License <http://opensource.org/licenses/MIT>. +Copyright (c) 2013-2016 Niels Lohmann <http://nlohmann.me>. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#include <algorithm> // all_of, for_each, transform +#include <array> // array +#include <cassert> // assert +#include <cctype> // isdigit +#include <ciso646> // and, not, or +#include <cmath> // isfinite, signbit +#include <cstddef> // nullptr_t, ptrdiff_t, size_t +#include <cstdint> // int64_t, uint64_t +#include <cstdlib> // strtod, strtof, strtold, strtoul +#include <cstring> // strlen +#include <functional> // function, hash, less +#include <initializer_list> // initializer_list +#include <iomanip> // setw +#include <iostream> // istream, ostream +#include <iterator> // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator +#include <limits> // numeric_limits +#include <locale> // locale +#include <map> // map +#include <memory> // addressof, allocator, allocator_traits, unique_ptr +#include <numeric> // accumulate +#include <sstream> // stringstream +#include <stdexcept> // domain_error, invalid_argument, out_of_range +#include <string> // getline, stoi, string, to_string +#include <type_traits> // add_pointer, enable_if, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_floating_point, is_integral, is_nothrow_move_assignable, std::is_nothrow_move_constructible, std::is_pointer, std::is_reference, std::is_same, remove_const, remove_pointer, remove_reference +#include <utility> // declval, forward, make_pair, move, pair, swap +#include <vector> // vector + +#include <wtf/Assertions.h> // ASSERT +#include <wtf/Compiler.h> // NO_RETURN + +// exclude unsupported compilers +#if defined(__clang__) + #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) + #if CLANG_VERSION < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#elif defined(__GNUC__) + #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) + #if GCC_VERSION < 40900 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + + +/*! +@brief unnamed namespace with internal helper functions +@since version 1.0.0 +*/ +namespace +{ +/*! +@brief Helper to determine whether there's a key_type for T. + +Thus helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. + +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0, overworked in version 2.0.6 +*/ +template<typename T> +struct has_mapped_type +{ + private: + template <typename U, typename = typename U::mapped_type> + static int detect(U&&); + + static void detect(...); + public: + static constexpr bool value = + std::is_integral<decltype(detect(std::declval<T>()))>::value; +}; + +template <typename T, typename ... Types> +NO_RETURN void Throw(Types ... args) +{ +#if 0 + throw T(std::forward<Types>(args)...); +#else + ASSERT(false); + WTFCrash(); +#endif +} +} + +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null value. + - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + JSON values have + [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the class + has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](http://en.cppreference.com/w/cpp/concept/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. + +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 + +@nosubgrouping +*/ +template < + template<typename U, typename V, typename... Args> class ObjectType = std::map, + template<typename U, typename... Args> class ArrayType = std::vector, + class StringType = std::string, + class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template<typename U> class AllocatorType = std::allocator + > +class basic_json +{ + private: + /// workaround type for MSVC + using basic_json_t = basic_json<ObjectType, ArrayType, StringType, + BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType, + AllocatorType>; + + public: + // forward declarations + template<typename Base> class json_reverse_iterator; + class json_pointer; + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType<basic_json>; + + /// the type of an element pointer + using pointer = typename std::allocator_traits<allocator_type>::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer; + + /// an iterator for a basic_json container + class iterator; + /// a const iterator for a basic_json container + class const_iterator; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator<typename basic_json::iterator>; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator<typename basic_json::const_iterator>; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() + { + return allocator_type(); + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /*! + @brief a type for an object + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. + + To store objects in C++, a type is defined by the template parameters + described below. + + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less<StringType>` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) + + #### Default type + + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: + + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less<std::string>, // key_compare + std::allocator<std::pair<const std::string, basic_json>> // allocator_type + > + @endcode + + #### Behavior + + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: + + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, later stored name/value + pairs overwrite previously stored name/value pairs, leaving the used + names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will + be treated as equal and both stored as `{"key": 1}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType<StringType, + basic_json, + std::less<StringType>, + AllocatorType<std::pair<const StringType, + basic_json>>>; + + /*! + @brief a type for an array + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. + + To store objects in C++, a type is defined by the template parameters + explained below. + + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + + #### Default type + + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: + + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator<basic_json> // allocator_type + > + @endcode + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the array's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. + + #### Storage + + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value + + @since version 1.0.0 + */ + using array_t = ArrayType<basic_json, AllocatorType<basic_json>>; + + /*! + @brief a type for a string + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. + + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. + + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. + + #### Default type + + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: + + @code {.cpp} + std::string + @endcode + + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage + + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. + + @since version 1.0.0 + */ + using string_t = StringType; + + /*! + @brief a type for a boolean + + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. + + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. + + #### Default type + + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. + + @since version 1.0.0 + */ + using boolean_t = BooleanType; + + /*! + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. + + #### Default type + + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: + + @code {.cpp} + int64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_integer_t = NumberIntegerType; + + /*! + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. + + #### Default type + + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: + + @code {.cpp} + uint64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) + + @since version 2.0.0 + */ + using number_unsigned_t = NumberUnsignedType; + + /*! + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. + + #### Default type + + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: + + @code {.cpp} + double + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. + + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. + + #### Storage + + Floating-point number values are stored directly inside a @ref basic_json + type. + + @sa @ref number_integer_t -- type for number values (integer) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_float_t = NumberFloatType; + + /// @} + + + /////////////////////////// + // JSON type enumeration // + /////////////////////////// + + /*! + @brief the JSON type enumeration + + This enumeration collects the different JSON types. It is internally used + to distinguish the stored values, and the functions @ref is_null(), @ref + is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref + is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and + @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and + @ref is_structured() rely on it. + + @note There are three enumeration entries (number_integer, + number_unsigned, and number_float), because the library distinguishes + these three types for numbers: @ref number_unsigned_t is used for unsigned + integers, @ref number_integer_t is used for signed integers, and @ref + number_float_t is used for floating-point numbers or to approximate + integers which do not fit in the limits of their respective type. + + @sa @ref basic_json(const value_t value_type) -- create a JSON value with + the default value for a given type + + @since version 1.0.0 + */ + enum class value_t : uint8_t + { + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function + }; + + + private: + + /// helper for exception-safe object creation + template<typename T, typename... Args> + static T* create(Args&& ... args) + { + AllocatorType<T> alloc; + auto deleter = [&](T * object) + { + alloc.deallocate(object, 1); + }; + std::unique_ptr<T, decltype(deleter)> object(alloc.allocate(1), deleter); + alloc.construct(object.get(), std::forward<Args>(args)...); + assert(object.get() != nullptr); + return object.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create<object_t>(); + break; + } + + case value_t::array: + { + array = create<array_t>(); + break; + } + + case value_t::string: + { + string = create<string_t>(""); + break; + } + + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } + + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } + + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } + + default: + { + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) + { + string = create<string_t>(value); + } + + /// constructor for objects + json_value(const object_t& value) + { + object = create<object_t>(value); + } + + /// constructor for arrays + json_value(const array_t& value) + { + array = create<array_t>(value); + } + }; + + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + */ + void assert_invariant() const + { + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /*! + @brief JSON callback events + + This enumeration lists the parser events that can trigger calling a + callback function of type @ref parser_callback_t during parsing. + + @image html callback_events.png "Example when certain parse events are triggered" + + @since version 1.0.0 + */ + enum class parse_event_t : uint8_t + { + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; + + /*! + @brief per-element parser callback type + + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse(std::istream&, const + parser_callback_t) or @ref parse(const char*, const parser_callback_t), + it is called on certain events (passed as @ref parse_event_t via parameter + @a event) with a set recursion depth @a depth and context JSON value + @a parsed. The return value of the callback function is a boolean + indicating whether the element that emitted the callback shall be kept or + not. + + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. + + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: + + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. + + @param[in] depth the depth of the recursion during parsing + + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called + + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events + + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. + + @sa @ref parse(std::istream&, parser_callback_t) or + @ref parse(const char*, parser_callback_t) for examples + + @since version 1.0.0 + */ + using parser_callback_t = std::function<bool(int depth, + parse_event_t event, + basic_json& parsed)>; + + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /*! + @brief create an empty value with a given type + + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @param[in] value_type the type of the value to create + + @complexity Constant. + + @throw std::bad_alloc if allocation for object, array, or string value + fails + + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} + + @sa @ref basic_json(std::nullptr_t) -- create a `null` value + @sa @ref basic_json(boolean_t value) -- create a boolean value + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const object_t&) -- create a object value + @sa @ref basic_json(const array_t&) -- create a array value + @sa @ref basic_json(const number_float_t) -- create a number + (floating-point) value + @sa @ref basic_json(const number_integer_t) -- create a number (integer) + value + @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) + value + + @since version 1.0.0 + */ + basic_json(const value_t value_type) + : m_type(value_type), m_value(value_type) + { + assert_invariant(); + } + + /*! + @brief create a null object + + Create a `null` JSON value. It either takes a null pointer as parameter + (explicitly creating `null`) or no parameter (implicitly creating `null`). + The passed null pointer itself is not read -- it is only used to choose + the right constructor. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @liveexample{The following code shows the constructor with and without a + null pointer parameter.,basic_json__nullptr_t} + + @since version 1.0.0 + */ + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /*! + @brief create an object (explicit) + + Create an object JSON value with a given content. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with an @ref + object_t parameter.,basic_json__object_t} + + @sa @ref basic_json(const CompatibleObjectType&) -- create an object value + from a compatible STL container + + @since version 1.0.0 + */ + basic_json(const object_t& val) + : m_type(value_t::object), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an object (implicit) + + Create an object JSON value with a given content. This constructor allows + any type @a CompatibleObjectType that can be used to construct values of + type @ref object_t. + + @tparam CompatibleObjectType An object type whose `key_type` and + `value_type` is compatible to @ref object_t. Examples include `std::map`, + `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with + a `key_type` of `std::string`, and a `value_type` from which a @ref + basic_json value can be constructed. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with several + compatible object type parameters.,basic_json__CompatibleObjectType} + + @sa @ref basic_json(const object_t&) -- create an object value + + @since version 1.0.0 + */ + template<class CompatibleObjectType, typename std::enable_if< + std::is_constructible<typename object_t::key_type, typename CompatibleObjectType::key_type>::value and + std::is_constructible<basic_json, typename CompatibleObjectType::mapped_type>::value, int>::type = 0> + basic_json(const CompatibleObjectType& val) + : m_type(value_t::object) + { + using std::begin; + using std::end; + m_value.object = create<object_t>(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create an array (explicit) + + Create an array JSON value with a given content. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with an @ref array_t + parameter.,basic_json__array_t} + + @sa @ref basic_json(const CompatibleArrayType&) -- create an array value + from a compatible STL containers + + @since version 1.0.0 + */ + basic_json(const array_t& val) + : m_type(value_t::array), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an array (implicit) + + Create an array JSON value with a given content. This constructor allows + any type @a CompatibleArrayType that can be used to construct values of + type @ref array_t. + + @tparam CompatibleArrayType An object type whose `value_type` is + compatible to @ref array_t. Examples include `std::vector`, `std::deque`, + `std::list`, `std::forward_list`, `std::array`, `std::set`, + `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a + `value_type` from which a @ref basic_json value can be constructed. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with several + compatible array type parameters.,basic_json__CompatibleArrayType} + + @sa @ref basic_json(const array_t&) -- create an array value + + @since version 1.0.0 + */ + template<class CompatibleArrayType, typename std::enable_if< + not std::is_same<CompatibleArrayType, typename basic_json_t::iterator>::value and + not std::is_same<CompatibleArrayType, typename basic_json_t::const_iterator>::value and + not std::is_same<CompatibleArrayType, typename basic_json_t::reverse_iterator>::value and + not std::is_same<CompatibleArrayType, typename basic_json_t::const_reverse_iterator>::value and + not std::is_same<CompatibleArrayType, typename array_t::iterator>::value and + not std::is_same<CompatibleArrayType, typename array_t::const_iterator>::value and + std::is_constructible<basic_json, typename CompatibleArrayType::value_type>::value, int>::type = 0> + basic_json(const CompatibleArrayType& val) + : m_type(value_t::array) + { + using std::begin; + using std::end; + m_value.array = create<array_t>(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create an string JSON value with a given content. + + @param[in] val a value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with an @ref + string_t parameter.,basic_json__string_t} + + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const string_t& val) + : m_type(value_t::string), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create a string JSON value with a given content. + + @param[in] val a literal value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with string literal + parameter.,basic_json__string_t_value_type} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const typename string_t::value_type* val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + basic_json(const typename string_t::value_type* val, typename string_t::size_type count) + : basic_json(string_t(val, count)) + { + assert_invariant(); + } + + /*! + @brief create a string (implicit) + + Create a string JSON value with a given content. + + @param[in] val a value for the string + + @tparam CompatibleStringType an string type which is compatible to @ref + string_t, for instance `std::string`. + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the construction of a string value + from a compatible type.,basic_json__CompatibleStringType} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + + @since version 1.0.0 + */ + template<class CompatibleStringType, typename std::enable_if< + std::is_constructible<string_t, CompatibleStringType>::value, int>::type = 0> + basic_json(const CompatibleStringType& val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a boolean (explicit) + + Creates a JSON boolean type from a given value. + + @param[in] val a boolean value to store + + @complexity Constant. + + @liveexample{The example below demonstrates boolean + values.,basic_json__boolean_t} + + @since version 1.0.0 + */ + basic_json(boolean_t val) noexcept + : m_type(value_t::boolean), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number (explicit) + + Create an integer number JSON value with a given content. + + @tparam T A helper type to remove this function via SFINAE in case @ref + number_integer_t is the same as `int`. In this case, this constructor + would have the same signature as @ref basic_json(const int value). Note + the helper type @a T is not visible in this constructor's interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value.,basic_json__number_integer_t} + + @sa @ref basic_json(const int) -- create a number value (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + template<typename T, typename std::enable_if< + not (std::is_same<T, int>::value) and + std::is_same<T, number_integer_t>::value, int>::type = 0> + basic_json(const number_integer_t val) noexcept + : m_type(value_t::number_integer), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number from an enum type (explicit) + + Create an integer number JSON value with a given content. + + @param[in] val an integer to create a JSON number from + + @note This constructor allows to pass enums directly to a constructor. As + C++ has no way of specifying the type of an anonymous enum explicitly, we + can only rely on the fact that such values implicitly convert to int. As + int may already be the same type of number_integer_t, we may need to + switch off the constructor @ref basic_json(const number_integer_t). + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value from an anonymous enum.,basic_json__const_int} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const int val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast<number_integer_t>(val)) + { + assert_invariant(); + } + + /*! + @brief create an integer number (implicit) + + Create an integer number JSON value with a given content. This constructor + allows any type @a CompatibleNumberIntegerType that can be used to + construct values of type @ref number_integer_t. + + @tparam CompatibleNumberIntegerType An integer type which is compatible to + @ref number_integer_t. Examples include the types `int`, `int32_t`, + `long`, and `short`. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of several integer + number values from compatible + types.,basic_json__CompatibleIntegerNumberType} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const int) -- create a number value (integer) + + @since version 1.0.0 + */ + template<typename CompatibleNumberIntegerType, typename std::enable_if< + std::is_constructible<number_integer_t, CompatibleNumberIntegerType>::value and + std::numeric_limits<CompatibleNumberIntegerType>::is_integer and + std::numeric_limits<CompatibleNumberIntegerType>::is_signed, + CompatibleNumberIntegerType>::type = 0> + basic_json(const CompatibleNumberIntegerType val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast<number_integer_t>(val)) + { + assert_invariant(); + } + + /*! + @brief create an unsigned integer number (explicit) + + Create an unsigned integer number JSON value with a given content. + + @tparam T helper type to compare number_unsigned_t and unsigned int (not + visible in) the interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number + value (unsigned integer) from a compatible number type + + @since version 2.0.0 + */ + template<typename T, typename std::enable_if< + not (std::is_same<T, int>::value) and + std::is_same<T, number_unsigned_t>::value, int>::type = 0> + basic_json(const number_unsigned_t val) noexcept + : m_type(value_t::number_unsigned), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an unsigned number (implicit) + + Create an unsigned number JSON value with a given content. This + constructor allows any type @a CompatibleNumberUnsignedType that can be + used to construct values of type @ref number_unsigned_t. + + @tparam CompatibleNumberUnsignedType An integer type which is compatible + to @ref number_unsigned_t. Examples may include the types `unsigned int`, + `uint32_t`, or `unsigned short`. + + @param[in] val an unsigned integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const number_unsigned_t) -- create a number value + (unsigned) + + @since version 2.0.0 + */ + template<typename CompatibleNumberUnsignedType, typename std::enable_if < + std::is_constructible<number_unsigned_t, CompatibleNumberUnsignedType>::value and + std::numeric_limits<CompatibleNumberUnsignedType>::is_integer and + not std::numeric_limits<CompatibleNumberUnsignedType>::is_signed, + CompatibleNumberUnsignedType>::type = 0> + basic_json(const CompatibleNumberUnsignedType val) noexcept + : m_type(value_t::number_unsigned), + m_value(static_cast<number_unsigned_t>(val)) + { + assert_invariant(); + } + + /*! + @brief create a floating-point number (explicit) + + Create a floating-point number JSON value with a given content. + + @param[in] val a floating-point value to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is created + instead. + + @complexity Constant. + + @liveexample{The following example creates several floating-point + values.,basic_json__number_float_t} + + @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number + value (floating-point) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const number_float_t val) noexcept + : m_type(value_t::number_float), m_value(val) + { + // replace infinity and NAN by null + if (not std::isfinite(val)) + { + m_type = value_t::null; + m_value = json_value(); + } + + assert_invariant(); + } + + /*! + @brief create an floating-point number (implicit) + + Create an floating-point number JSON value with a given content. This + constructor allows any type @a CompatibleNumberFloatType that can be used + to construct values of type @ref number_float_t. + + @tparam CompatibleNumberFloatType A floating-point type which is + compatible to @ref number_float_t. Examples may include the types `float` + or `double`. + + @param[in] val a floating-point to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is + created instead. + + @complexity Constant. + + @liveexample{The example below shows the construction of several + floating-point number values from compatible + types.,basic_json__CompatibleNumberFloatType} + + @sa @ref basic_json(const number_float_t) -- create a number value + (floating-point) + + @since version 1.0.0 + */ + template<typename CompatibleNumberFloatType, typename = typename std::enable_if< + std::is_constructible<number_float_t, CompatibleNumberFloatType>::value and + std::is_floating_point<CompatibleNumberFloatType>::value>::type> + basic_json(const CompatibleNumberFloatType val) noexcept + : basic_json(number_float_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a container (array or object) from an initializer list + + Creates a JSON value of type array or object from the passed initializer + list @a init. In case @a type_deduction is `true` (default), the type of + the JSON value to be created is deducted from the initializer list @a init + according to the following rules: + + 1. If the list is empty, an empty JSON object value `{}` is created. + 2. If the list consists of pairs whose first element is a string, a JSON + object value is created where the first elements of the pairs are + treated as keys and the second elements are as values. + 3. In all other cases, an array is created. + + The rules aim to create the best fit between a C++ initializer list and + JSON values. The rationale is as follows: + + 1. The empty initializer list is written as `{}` which is exactly an empty + JSON object. + 2. C++ has now way of describing mapped types other than to list a list of + pairs. As JSON requires that keys must be of type string, rule 2 is the + weakest constraint one can pose on initializer lists to interpret them + as an object. + 3. In all other cases, the initializer list could not be interpreted as + JSON object type, so interpreting it as JSON array type is safe. + + With the rules described above, the following JSON values cannot be + expressed by an initializer list: + + - the empty array (`[]`): use @ref array(std::initializer_list<basic_json>) + with an empty initializer list in this case + - arrays whose elements satisfy rule 2: use @ref + array(std::initializer_list<basic_json>) with the same initializer list + in this case + + @note When used without parentheses around an empty initializer list, @ref + basic_json() is called instead of this function, yielding the JSON null + value. + + @param[in] init initializer list with JSON values + + @param[in] type_deduction internal parameter; when set to `true`, the type + of the JSON value is deducted from the initializer list @a init; when set + to `false`, the type provided via @a manual_type is forced. This mode is + used by the functions @ref array(std::initializer_list<basic_json>) and + @ref object(std::initializer_list<basic_json>). + + @param[in] manual_type internal parameter; when @a type_deduction is set + to `false`, the created JSON value will use the provided type (only @ref + value_t::array and @ref value_t::object are valid); when @a type_deduction + is set to `true`, this parameter has no effect + + @throw std::domain_error if @a type_deduction is `false`, @a manual_type + is `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string; example: `"cannot create object from + initializer list"` + + @complexity Linear in the size of the initializer list @a init. + + @liveexample{The example below shows how JSON values are created from + initializer lists.,basic_json__list_init_t} + + @sa @ref array(std::initializer_list<basic_json>) -- create a JSON array + value from an initializer list + @sa @ref object(std::initializer_list<basic_json>) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + basic_json(std::initializer_list<basic_json> init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const basic_json & element) + { + return element.is_array() and element.size() == 2 and element[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (not type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (manual_type == value_t::object and not is_an_object) + { + Throw<std::domain_error>("cannot create object from initializer list"); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + std::for_each(init.begin(), init.end(), [this](const basic_json & element) + { + m_value.object->emplace(*(element[0].m_value.string), element[1]); + }); + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create<array_t>(init); + } + + assert_invariant(); + } + + /*! + @brief explicitly create an array from an initializer list + + Creates a JSON array value from a given initializer list. That is, given a + list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the + initializer list is empty, the empty array `[]` is created. + + @note This function is only needed to express two edge cases that cannot + be realized with the initializer list constructor (@ref + basic_json(std::initializer_list<basic_json>, bool, value_t)). These cases + are: + 1. creating an array whose elements are all pairs whose first element is a + string -- in this case, the initializer list constructor would create an + object, taking the first elements as keys + 2. creating an empty array -- passing the empty initializer list to the + initializer list constructor yields an empty object + + @param[in] init initializer list with JSON values to create an array from + (optional) + + @return JSON array value + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `array` + function.,array} + + @sa @ref basic_json(std::initializer_list<basic_json>, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref object(std::initializer_list<basic_json>) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + static basic_json array(std::initializer_list<basic_json> init = + std::initializer_list<basic_json>()) + { + return basic_json(init, false, value_t::array); + } + + /*! + @brief explicitly create an object from an initializer list + + Creates a JSON object value from a given initializer list. The initializer + lists elements must be pairs, and their first elements must be strings. If + the initializer list is empty, the empty object `{}` is created. + + @note This function is only added for symmetry reasons. In contrast to the + related function @ref array(std::initializer_list<basic_json>), there are + no cases which can only be expressed by this function. That is, any + initializer list @a init can also be passed to the initializer list + constructor @ref basic_json(std::initializer_list<basic_json>, bool, + value_t). + + @param[in] init initializer list to create an object from (optional) + + @return JSON object value + + @throw std::domain_error if @a init is not a pair whose first elements are + strings; thrown by + @ref basic_json(std::initializer_list<basic_json>, bool, value_t) + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `object` + function.,object} + + @sa @ref basic_json(std::initializer_list<basic_json>, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref array(std::initializer_list<basic_json>) -- create a JSON array + value from an initializer list + + @since version 1.0.0 + */ + static basic_json object(std::initializer_list<basic_json> init = + std::initializer_list<basic_json>()) + { + return basic_json(init, false, value_t::object); + } + + /*! + @brief construct an array with count copies of given value + + Constructs a JSON array value by creating @a cnt copies of a passed value. + In case @a cnt is `0`, an empty array is created. As postcondition, + `std::distance(begin(),end()) == cnt` holds. + + @param[in] cnt the number of JSON copies of @a val to create + @param[in] val the JSON value to copy + + @complexity Linear in @a cnt. + + @liveexample{The following code shows examples for the @ref + basic_json(size_type\, const basic_json&) + constructor.,basic_json__size_type_basic_json} + + @since version 1.0.0 + */ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create<array_t>(cnt, val); + assert_invariant(); + } + + /*! + @brief construct a JSON container given an iterator range + + Constructs the JSON value with the contents of the range `[first, last)`. + The semantics depends on the different types a JSON value can have: + - In case of primitive types (number, boolean, or string), @a first must + be `begin()` and @a last must be `end()`. In this case, the value is + copied. Otherwise, std::out_of_range is thrown. + - In case of structured types (array, object), the constructor behaves as + similar versions for `std::vector`. + - In case of a null type, std::domain_error is thrown. + + @tparam InputIT an input iterator type (@ref iterator or @ref + const_iterator) + + @param[in] first begin of the range to copy from (included) + @param[in] last end of the range to copy from (excluded) + + @pre Iterators @a first and @a last must be initialized. **This + precondition is enforced with an assertion.** + + @throw std::domain_error if iterators are not compatible; that is, do not + belong to the same JSON value; example: `"iterators are not compatible"` + @throw std::out_of_range if iterators are for a primitive type (number, + boolean, or string) where an out of range error can be detected easily; + example: `"iterators out of range"` + @throw std::bad_alloc if allocation for object, array, or string fails + @throw std::domain_error if called with a null value; example: `"cannot + use construct with iterators from null"` + + @complexity Linear in distance between @a first and @a last. + + @liveexample{The example below shows several ways to create JSON values by + specifying a subrange with iterators.,basic_json__InputIt_InputIt} + + @since version 1.0.0 + */ + template<class InputIT, typename std::enable_if< + std::is_same<InputIT, typename basic_json_t::iterator>::value or + std::is_same<InputIT, typename basic_json_t::const_iterator>::value, int>::type = 0> + basic_json(InputIT first, InputIT last) + { + assert(first.m_object != nullptr); + assert(last.m_object != nullptr); + + // make sure iterator fits the current value + if (first.m_object != last.m_object) + { + Throw<std::domain_error>("iterators are not compatible"); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + Throw<std::out_of_range>("iterators out of range"); + } + break; + } + + default: + { + break; + } + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create<object_t>(first.m_it.object_iterator, last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create<array_t>(first.m_it.array_iterator, last.m_it.array_iterator); + break; + } + + default: + { + Throw<std::domain_error>("cannot use construct with iterators from " + first.m_object->type_name()); + } + } + + assert_invariant(); + } + + /*! + @brief construct a JSON value given an input stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @deprecated This constructor is deprecated and will be removed in version + 3.0.0 to unify the interface of the library. Deserialization will be + done by stream operators or by calling one of the `parse` functions, + e.g. @ref parse(std::istream&, const parser_callback_t). That is, calls + like `json j(i);` for an input stream @a i need to be replaced by + `json j = json::parse(i);`. See the example below. + + @liveexample{The example below demonstrates constructing a JSON value from + a `std::stringstream` with and without callback + function.,basic_json__istream} + + @since version 2.0.0, deprecated in version 2.0.3, to be removed in + version 3.0.0 + */ + JSON_DEPRECATED + explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) + { + *this = parser(i, cb).parse(); + assert_invariant(); + } + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + /*! + @brief copy constructor + + Creates a copy of a given JSON value. + + @param[in] other the JSON value to copy + + @complexity Linear in the size of @a other. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - As postcondition, it holds: `other == basic_json(other)`. + + @throw std::bad_alloc if allocation for object, array, or string fails. + + @liveexample{The following code shows an example for the copy + constructor.,basic_json__basic_json} + + @since version 1.0.0 + */ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + default: + { + break; + } + } + + assert_invariant(); + } + + /*! + @brief move constructor + + Move constructor. Constructs a JSON value with the contents of the given + value @a other using move semantics. It "steals" the resources from @a + other and leaves it as JSON null value. + + @param[in,out] other value to move to this object + + @post @a other is a JSON null value + + @complexity Constant. + + @liveexample{The code below shows the move constructor explicitly called + via std::move.,basic_json__moveconstructor} + + @since version 1.0.0 + */ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + assert_invariant(); + } + + /*! + @brief copy assignment + + Copy assignment operator. Copies a JSON value via the "copy and swap" + strategy: It is expressed in terms of the copy constructor, destructor, + and the swap() member function. + + @param[in] other value to copy from + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + + @liveexample{The code below shows and example for the copy assignment. It + creates a copy of value `a` which is then swapped with `b`. Finally\, the + copy of `a` (which is the null value after the swap) is + destroyed.,basic_json__copyassignment} + + @since version 1.0.0 + */ + reference& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible<value_t>::value and + std::is_nothrow_move_assignable<value_t>::value and + std::is_nothrow_move_constructible<json_value>::value and + std::is_nothrow_move_assignable<json_value>::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + assert_invariant(); + return *this; + } + + /*! + @brief destructor + + Destroys the JSON value and frees all allocated memory. + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - All stored elements are destroyed and all memory is freed. + + @since version 1.0.0 + */ + ~basic_json() + { + assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + AllocatorType<object_t> alloc; + alloc.destroy(m_value.object); + alloc.deallocate(m_value.object, 1); + break; + } + + case value_t::array: + { + AllocatorType<array_t> alloc; + alloc.destroy(m_value.array); + alloc.deallocate(m_value.array, 1); + break; + } + + case value_t::string: + { + AllocatorType<string_t> alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + break; + } + + default: + { + // all other types need no specific destructor + break; + } + } + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /*! + @brief serialization + + Serialization function for JSON values. The function tries to mimic + Python's `json.dumps()` function, and currently supports its @a indent + parameter. + + @param[in] indent If indent is nonnegative, then array elements and object + members will be pretty-printed with that indent level. An indent level of + `0` will only insert newlines. `-1` (the default) selects the most compact + representation. + + @return string containing the serialization of the JSON value + + @complexity Linear. + + @liveexample{The following example shows the effect of different @a indent + parameters to the result of the serialization.,dump} + + @see https://docs.python.org/2/library/json.html#json.dump + + @since version 1.0.0 + */ + string_t dump(const int indent = -1) const + { + std::stringstream ss; + // fix locale problems + ss.imbue(std::locale::classic()); + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits<number_float_t>::digits10 + ss.precision(std::numeric_limits<double>::digits10); + + if (indent >= 0) + { + dump(ss, true, static_cast<unsigned int>(indent)); + } + else + { + dump(ss, false, 0); + } + + return ss.str(); + } + + /*! + @brief return the type of the JSON value (explicit) + + Return the type of the JSON value as a value from the @ref value_t + enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `type()` for all JSON + types.,type} + + @since version 1.0.0 + */ + constexpr value_t type() const noexcept + { + return m_type; + } + + /*! + @brief return whether type is primitive + + This function returns true iff the JSON type is primitive (string, number, + boolean, or null). + + @return `true` if type is primitive (string, number, boolean, or null), + `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_primitive()` for all JSON + types.,is_primitive} + + @sa @ref is_structured() -- returns whether JSON value is structured + @sa @ref is_null() -- returns whether JSON value is `null` + @sa @ref is_string() -- returns whether JSON value is a string + @sa @ref is_boolean() -- returns whether JSON value is a boolean + @sa @ref is_number() -- returns whether JSON value is a number + + @since version 1.0.0 + */ + constexpr bool is_primitive() const noexcept + { + return is_null() or is_string() or is_boolean() or is_number(); + } + + /*! + @brief return whether type is structured + + This function returns true iff the JSON type is structured (array or + object). + + @return `true` if type is structured (array or object), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_structured()` for all JSON + types.,is_structured} + + @sa @ref is_primitive() -- returns whether value is primitive + @sa @ref is_array() -- returns whether value is an array + @sa @ref is_object() -- returns whether value is an object + + @since version 1.0.0 + */ + constexpr bool is_structured() const noexcept + { + return is_array() or is_object(); + } + + /*! + @brief return whether value is null + + This function returns true iff the JSON value is null. + + @return `true` if type is null, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_null()` for all JSON + types.,is_null} + + @since version 1.0.0 + */ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /*! + @brief return whether value is a boolean + + This function returns true iff the JSON value is a boolean. + + @return `true` if type is boolean, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_boolean()` for all JSON + types.,is_boolean} + + @since version 1.0.0 + */ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /*! + @brief return whether value is a number + + This function returns true iff the JSON value is a number. This includes + both integer and floating-point values. + + @return `true` if type is number (regardless whether integer, unsigned + integer or floating-type), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number()` for all JSON + types.,is_number} + + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number() const noexcept + { + return is_number_integer() or is_number_float(); + } + + /*! + @brief return whether value is an integer number + + This function returns true iff the JSON value is an integer or unsigned + integer number. This excludes floating-point values. + + @return `true` if type is an integer or unsigned integer number, `false` + otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_integer()` for all + JSON types.,is_number_integer} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer or m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is an unsigned integer number + + This function returns true iff the JSON value is an unsigned integer + number. This excludes floating-point and (signed) integer values. + + @return `true` if type is an unsigned integer number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_unsigned()` for all + JSON types.,is_number_unsigned} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 2.0.0 + */ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is a floating-point number + + This function returns true iff the JSON value is a floating-point number. + This excludes integer and unsigned integer values. + + @return `true` if type is a floating-point number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_float()` for all + JSON types.,is_number_float} + + @sa @ref is_number() -- check if value is number + @sa @ref is_number_integer() -- check if value is an integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + + @since version 1.0.0 + */ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /*! + @brief return whether value is an object + + This function returns true iff the JSON value is an object. + + @return `true` if type is object, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_object()` for all JSON + types.,is_object} + + @since version 1.0.0 + */ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /*! + @brief return whether value is an array + + This function returns true iff the JSON value is an array. + + @return `true` if type is array, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_array()` for all JSON + types.,is_array} + + @since version 1.0.0 + */ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /*! + @brief return whether value is a string + + This function returns true iff the JSON value is a string. + + @return `true` if type is string, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_string()` for all JSON + types.,is_string} + + @since version 1.0.0 + */ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /*! + @brief return whether value is discarded + + This function returns true iff the JSON value was discarded during parsing + with a callback function (see @ref parser_callback_t). + + @note This function will always be `false` for JSON values after parsing. + That is, discarded values can only occur during parsing, but will be + removed when inside a structured value or replaced by null in other cases. + + @return `true` if type is discarded, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_discarded()` for all JSON + types.,is_discarded} + + @since version 1.0.0 + */ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /*! + @brief return the type of the JSON value (implicit) + + Implicitly return the type of the JSON value as a value from the @ref + value_t enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies the @ref value_t operator for + all JSON types.,operator__value_t} + + @since version 1.0.0 + */ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get an object (explicit) + template<class T, typename std::enable_if< + std::is_convertible<typename object_t::key_type, typename T::key_type>::value and + std::is_convertible<basic_json_t, typename T::mapped_type>::value, int>::type = 0> + T get_impl(T*) const + { + if (is_object()) + { + return T(m_value.object->begin(), m_value.object->end()); + } + else + { + Throw<std::domain_error>("type must be object, but is " + type_name()); + } + } + + /// get an object (explicit) + object_t get_impl(object_t*) const + { + if (is_object()) + { + return *(m_value.object); + } + else + { + Throw<std::domain_error>("type must be object, but is " + type_name()); + } + } + + /// get an array (explicit) + template<class T, typename std::enable_if< + std::is_convertible<basic_json_t, typename T::value_type>::value and + not std::is_same<basic_json_t, typename T::value_type>::value and + not std::is_arithmetic<T>::value and + not std::is_convertible<std::string, T>::value and + not has_mapped_type<T>::value, int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + T to_vector; + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get<typename T::value_type>(); + }); + return to_vector; + } + else + { + Throw<std::domain_error>("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template<class T, typename std::enable_if< + std::is_convertible<basic_json_t, T>::value and + not std::is_same<basic_json_t, T>::value, int>::type = 0> + std::vector<T> get_impl(std::vector<T>*) const + { + if (is_array()) + { + std::vector<T> to_vector; + to_vector.reserve(m_value.array->size()); + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get<T>(); + }); + return to_vector; + } + else + { + Throw<std::domain_error>("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template<class T, typename std::enable_if< + std::is_same<basic_json, typename T::value_type>::value and + not has_mapped_type<T>::value, int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + return T(m_value.array->begin(), m_value.array->end()); + } + else + { + Throw<std::domain_error>("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + array_t get_impl(array_t*) const + { + if (is_array()) + { + return *(m_value.array); + } + else + { + Throw<std::domain_error>("type must be array, but is " + type_name()); + } + } + + /// get a string (explicit) + template<typename T, typename std::enable_if< + std::is_convertible<string_t, T>::value, int>::type = 0> + T get_impl(T*) const + { + if (is_string()) + { + return *m_value.string; + } + else + { + Throw<std::domain_error>("type must be string, but is " + type_name()); + } + } + + /// get a number (explicit) + template<typename T, typename std::enable_if< + std::is_arithmetic<T>::value, int>::type = 0> + T get_impl(T*) const + { + switch (m_type) + { + case value_t::number_integer: + { + return static_cast<T>(m_value.number_integer); + } + + case value_t::number_unsigned: + { + return static_cast<T>(m_value.number_unsigned); + } + + case value_t::number_float: + { + return static_cast<T>(m_value.number_float); + } + + default: + { + Throw<std::domain_error>("type must be number, but is " + type_name()); + } + } + } + + /// get a boolean (explicit) + constexpr boolean_t get_impl(boolean_t*) const + { + return is_boolean() + ? m_value.boolean + : Throw<std::domain_error>("type must be boolean, but is " + type_name()), false; + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t*) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t*) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t*) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t*) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t*) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t*) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t*) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t*) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t*) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This funcion helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw std::domain_error if ReferenceType does not match underlying value + type of the current JSON + */ + template<typename ReferenceType, typename ThisType> + static ReferenceType get_ref_impl(ThisType& obj) + { + // helper type + using PointerType = typename std::add_pointer<ReferenceType>::type; + + // delegate the call to get_ptr<>() + auto ptr = obj.template get_ptr<PointerType>(); + + if (ptr != nullptr) + { + return *ptr; + } + else + { + Throw<std::domain_error>("incompatible ReferenceType for get_ref, actual type is " + + obj.type_name()); + } + } + + public: + + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON; example: `"type must be object, but is null"` + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector<short>`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map<std::string\, + json>`.,get__ValueType_const} + + @internal + The idea of using a casted null pointer to choose the correct + implementation is from <http://stackoverflow.com/a/8315197/266378>. + @endinternal + + @sa @ref operator ValueType() const for implicit conversion + @sa @ref get() for pointer-member access + + @since version 1.0.0 + */ + template<typename ValueType, typename std::enable_if< + not std::is_pointer<ValueType>::value, int>::type = 0> + ValueType get() const + { + return get_impl(static_cast<ValueType*>(nullptr)); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value, int>::type = 0> + PointerType get() noexcept + { + // delegate the call to get_ptr + return get_ptr<PointerType>(); + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value, int>::type = 0> + constexpr const PointerType get() const noexcept + { + // delegate the call to get_ptr + return get_ptr<PointerType>(); + } + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are + made. + + @warning Writing data to the pointee of the result yields an undefined + state. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get_ptr} + + @since version 1.0.0 + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value, int>::type = 0> + PointerType get_ptr() noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const<typename + std::remove_pointer<typename + std::remove_const<PointerType>::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same<object_t, pointee_t>::value + or std::is_same<array_t, pointee_t>::value + or std::is_same<string_t, pointee_t>::value + or std::is_same<boolean_t, pointee_t>::value + or std::is_same<number_integer_t, pointee_t>::value + or std::is_same<number_unsigned_t, pointee_t>::value + or std::is_same<number_float_t, pointee_t>::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast<PointerType>(nullptr)); + } + + /*! + @brief get a pointer value (implicit) + @copydoc get_ptr() + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value and + std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0> + constexpr const PointerType get_ptr() const noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const<typename + std::remove_pointer<typename + std::remove_const<PointerType>::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same<object_t, pointee_t>::value + or std::is_same<array_t, pointee_t>::value + or std::is_same<string_t, pointee_t>::value + or std::is_same<boolean_t, pointee_t>::value + or std::is_same<number_integer_t, pointee_t>::value + or std::is_same<number_unsigned_t, pointee_t>::value + or std::is_same<number_float_t, pointee_t>::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast<const PointerType>(nullptr)); + } + + /*! + @brief get a reference value (implicit) + + Implict reference access to the internally stored JSON value. No copies + are made. + + @warning Writing data to the referee of the result yields an undefined + state. + + @tparam ReferenceType reference type; must be a reference to @ref array_t, + @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or + @ref number_float_t. Enforced by static assertion. + + @return reference to the internally stored JSON value if the requested + reference type @a ReferenceType fits to the JSON value; throws + std::domain_error otherwise + + @throw std::domain_error in case passed type @a ReferenceType is + incompatible with the stored JSON value + + @complexity Constant. + + @liveexample{The example shows several calls to `get_ref()`.,get_ref} + + @since version 1.1.0 + */ + template<typename ReferenceType, typename std::enable_if< + std::is_reference<ReferenceType>::value, int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl<ReferenceType>(*this); + } + + /*! + @brief get a reference value (implicit) + @copydoc get_ref() + */ + template<typename ReferenceType, typename std::enable_if< + std::is_reference<ReferenceType>::value and + std::is_const<typename std::remove_reference<ReferenceType>::type>::value, int>::type = 0> + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl<ReferenceType>(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON, thrown by @ref get() const + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector<short>`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map<std::string\, + json>`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename std::enable_if < + not std::is_pointer<ValueType>::value and + not std::is_same<ValueType, typename string_t::value_type>::value +#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 + and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value +#endif + , int >::type = 0 > + operator ValueType() const + { + // delegate the call to get<>() const + return get<ValueType>(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /*! + @brief access specified array element with bounds checking + + Returns a reference to the element at specified location @a idx, with + bounds checking. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read and + written using `at()`.,at__size_type} + + @since version 1.0.0 + */ + reference at(size_type idx) + { + // at only works for arrays + if (is_array()) + { + if (idx < m_value.array->size()) + return m_value.array->at(idx); + Throw<std::out_of_range>("array index " + std::to_string(idx) + " is out of range"); + } + else + { + Throw<std::domain_error>("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element with bounds checking + + Returns a const reference to the element at specified location @a idx, + with bounds checking. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + `at()`.,at__size_type_const} + + @since version 1.0.0 + */ + const_reference at(size_type idx) const + { + // at only works for arrays + if (is_array()) + { + if (idx < m_value.array->size()) + return m_value.array->at(idx); + Throw<std::out_of_range>("array index " + std::to_string(idx) + " is out of range"); + } + else + { + Throw<std::domain_error>("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a reference to the element at with specified key @a key, with + bounds checking. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using `at()`.,at__object_t_key_type} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (is_object()) + { + if (m_value.object->count(key) > 0) + return m_value.object->at(key); + Throw<std::out_of_range>("key '" + key + "' not found"); + } + else + { + Throw<std::domain_error>("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a const reference to the element at with specified key @a key, + with bounds checking. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + `at()`.,at__object_t_key_type_const} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (is_object()) + { + if (m_value.object->count(key) > 0) + return m_value.object->at(key); + Throw<std::out_of_range>("key '" + key + "' not found"); + } + else + { + Throw<std::domain_error>("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a reference to the element at specified location @a idx. + + @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), + then the array is silently filled up with `null` values to make `idx` a + valid reference to the last stored element. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array or null; example: + `"cannot use operator[] with string"` + + @complexity Constant if @a idx is in the range of the array. Otherwise + linear in `idx - size()`. + + @liveexample{The example below shows how array elements can be read and + written using `[]` operator. Note the addition of `null` + values.,operatorarray__size_type} + + @since version 1.0.0 + */ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create<array_t>(); + assert_invariant(); + } + + // operator[] only works for arrays + if (is_array()) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { + m_value.array->insert(m_value.array->end(), + idx - m_value.array->size() + 1, + basic_json()); + } + + return m_value.array->operator[](idx); + } + else + { + Throw<std::domain_error>("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a const reference to the element at specified location @a idx. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array; example: `"cannot use + operator[] with null"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + the `[]` operator.,operatorarray__size_type_const} + + @since version 1.0.0 + */ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (is_array()) + { + return m_value.array->operator[](idx); + } + else + { + Throw<std::domain_error>("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create<object_t>(); + assert_invariant(); + } + + // operator[] only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + Throw<std::domain_error>("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @pre The element with key @a key must exist. **This precondition is + enforced with an assertion.** + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + Throw<std::domain_error>("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template<typename T, std::size_t n> + reference operator[](T * (&key)[n]) + { + return operator[](static_cast<const T>(key)); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @note This function is required for compatibility reasons with Clang. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template<typename T, std::size_t n> + const_reference operator[](T * (&key)[n]) const + { + return operator[](static_cast<const T>(key)); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template<typename T> + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + Throw<std::domain_error>("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @pre The element with key @a key must exist. **This precondition is + enforced with an assertion.** + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template<typename T> + const_reference operator[](T* key) const + { + // at only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + Throw<std::domain_error>("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(key); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const typename object_t::key_type&), this function + does not throw if the given key @a key was not found. + + @note Unlike @ref operator[](const typename object_t::key_type& key), this + function does not implicitly add an element to the position defined by @a + key. This function is furthermore also applicable to const objects. + + @param[in] key key of the element to access + @param[in] default_value the value to return if @a key is not found + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + + @since version 1.0.0 + */ + template<class ValueType, typename std::enable_if< + std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0> + ValueType value(const typename object_t::key_type& key, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return *it; + } + else + { + return default_value; + } + } + else + { + Throw<std::domain_error>("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const + */ + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /*! + @brief access specified object element via JSON Pointer with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(ptr); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const json_pointer&), this function does not throw + if the given key @a key was not found. + + @param[in] ptr a JSON pointer to the element to access + @param[in] default_value the value to return if @a ptr found no value + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value_ptr} + + @sa @ref operator[](const json_pointer&) for unchecked access by reference + + @since version 2.0.2 + */ + template<class ValueType, typename std::enable_if< + std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0> + ValueType value(const json_pointer& ptr, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + return ptr.get_checked(this); + } + else + { + Throw<std::domain_error>("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const json_pointer&, ValueType) const + */ + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /*! + @brief access the first element + + Returns a reference to the first element in the container. For a JSON + container `c`, the expression `c.front()` is equivalent to `*c.begin()`. + + @return In case of a structured type (array or object), a reference to the + first element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, **guarded by + assertions**). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value + + @liveexample{The following code shows an example for `front()`.,front} + + @sa @ref back() -- access the last element + + @since version 1.0.0 + */ + reference front() + { + return *begin(); + } + + /*! + @copydoc basic_json::front() + */ + const_reference front() const + { + return *cbegin(); + } + + /*! + @brief access the last element + + Returns a reference to the last element in the container. For a JSON + container `c`, the expression `c.back()` is equivalent to + @code {.cpp} + auto tmp = c.end(); + --tmp; + return *tmp; + @endcode + + @return In case of a structured type (array or object), a reference to the + last element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, **guarded by + assertions**). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value. + + @liveexample{The following code shows an example for `back()`.,back} + + @sa @ref front() -- access the first element + + @since version 1.0.0 + */ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /*! + @copydoc basic_json::back() + */ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /*! + @brief remove element given an iterator + + Removes the element specified by iterator @a pos. The iterator @a pos must + be valid and dereferenceable. Thus the `end()` iterator (which is valid, + but is not dereferenceable) cannot be used as a value for @a pos. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] pos iterator to the element to remove + @return Iterator following the last removed element. If the iterator @a + pos refers to the last element, the `end()` iterator is returned. + + @tparam IteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on an iterator which does not belong to + the current JSON value; example: `"iterator does not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterator (i.e., any iterator which is not `begin()`); example: `"iterator + out of range"` + + @complexity The complexity depends on the type: + - objects: amortized constant + - arrays: linear in distance between pos and the end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType} + + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template<class IteratorType, typename std::enable_if< + std::is_same<IteratorType, typename basic_json_t::iterator>::value or + std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type + = 0> + IteratorType erase(IteratorType pos) + { + // make sure iterator fits the current value + if (this != pos.m_object) + { + Throw<std::domain_error>("iterator does not fit current value"); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not pos.m_it.primitive_iterator.is_begin()) + { + Throw<std::out_of_range>("iterator out of range"); + } + + if (is_string()) + { + AllocatorType<string_t> alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + default: + { + Throw<std::domain_error>("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove elements given an iterator range + + Removes the element specified by the range `[first; last)`. The iterator + @a first does not need to be dereferenceable if `first == last`: erasing + an empty range is a no-op. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] first iterator to the beginning of the range to remove + @param[in] last iterator past the end of the range to remove + @return Iterator following the last removed element. If the iterator @a + second refers to the last element, the `end()` iterator is returned. + + @tparam IteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on iterators which does not belong to + the current JSON value; example: `"iterators do not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterators (i.e., if `first != begin()` and `last != end()`); example: + `"iterators out of range"` + + @complexity The complexity depends on the type: + - objects: `log(size()) + std::distance(first, last)` + - arrays: linear in the distance between @a first and @a last, plus linear + in the distance between @a last and end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType_IteratorType} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template<class IteratorType, typename std::enable_if< + std::is_same<IteratorType, typename basic_json_t::iterator>::value or + std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type + = 0> + IteratorType erase(IteratorType first, IteratorType last) + { + // make sure iterator fits the current value + if (this != first.m_object or this != last.m_object) + { + Throw<std::domain_error>("iterators do not fit current value"); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + Throw<std::out_of_range>("iterators out of range"); + } + + if (is_string()) + { + AllocatorType<string_t> alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + { + Throw<std::domain_error>("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove element from a JSON object given a key + + Removes elements from a JSON object with the key value @a key. + + @param[in] key value of the elements to remove + + @return Number of elements removed. If @a ObjectType is the default + `std::map` type, the return value will always be `0` (@a key was not + found) or `1` (@a key was found). + + @post References and iterators to the erased elements are invalidated. + Other references and iterators are not affected. + + @throw std::domain_error when called on a type other than JSON object; + example: `"cannot use erase() with null"` + + @complexity `log(size()) + count(key)` + + @liveexample{The example shows the effect of `erase()`.,erase__key_type} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (is_object()) + { + return m_value.object->erase(key); + } + else + { + Throw<std::domain_error>("cannot use erase() with " + type_name()); + } + } + + /*! + @brief remove element from a JSON array given an index + + Removes element from a JSON array at the index @a idx. + + @param[in] idx index of the element to remove + + @throw std::domain_error when called on a type other than JSON array; + example: `"cannot use erase() with null"` + @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + is out of range"` + + @complexity Linear in distance between @a idx and the end of the container. + + @liveexample{The example shows the effect of `erase()`.,erase__size_type} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + + @since version 1.0.0 + */ + void erase(const size_type idx) + { + // this erase only works for arrays + if (is_array()) + { + if (idx >= size()) + { + Throw<std::out_of_range>("array index " + std::to_string(idx) + " is out of range"); + } + + m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx)); + } + else + { + Throw<std::domain_error>("cannot use erase() with " + type_name()); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /*! + @brief find an element in a JSON object + + Finds an element in a JSON object with key equivalent to @a key. If the + element is not found or the JSON value is not an object, end() is + returned. + + @param[in] key key value of the element to search for + + @return Iterator to an element with key equivalent to @a key. If no such + element is found, past-the-end (see end()) iterator is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `find()` is used.,find__key_type} + + @since version 1.0.0 + */ + iterator find(typename object_t::key_type key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief find an element in a JSON object + @copydoc find(typename object_t::key_type) + */ + const_iterator find(typename object_t::key_type key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief returns the number of occurrences of a key in a JSON object + + Returns the number of elements with key @a key. If ObjectType is the + default `std::map` type, the return value will always be `0` (@a key was + not found) or `1` (@a key was found). + + @param[in] key key value of the element to count + + @return Number of elements with key @a key. If the JSON value is not an + object, the return value will be `0`. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `count()` is used.,count} + + @since version 1.0.0 + */ + size_type count(typename object_t::key_type key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(key) : 0; + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /*! + @brief returns an iterator to the first element + + Returns an iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `begin()`.,begin} + + @sa @ref cbegin() -- returns a const iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /*! + @copydoc basic_json::cbegin() + */ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /*! + @brief returns a const iterator to the first element + + Returns a const iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).begin()`. + + @liveexample{The following code shows an example for `cbegin()`.,cbegin} + + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /*! + @brief returns an iterator to one past the last element + + Returns an iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `end()`.,end} + + @sa @ref cend() -- returns a const iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /*! + @copydoc basic_json::cend() + */ + const_iterator end() const noexcept + { + return cend(); + } + + /*! + @brief returns a const iterator to one past the last element + + Returns a const iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).end()`. + + @liveexample{The following code shows an example for `cend()`.,cend} + + @sa @ref end() -- returns an iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /*! + @brief returns an iterator to the reverse-beginning + + Returns an iterator to the reverse-beginning; that is, the last element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(end())`. + + @liveexample{The following code shows an example for `rbegin()`.,rbegin} + + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /*! + @copydoc basic_json::crbegin() + */ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /*! + @brief returns an iterator to the reverse-end + + Returns an iterator to the reverse-end; that is, one before the first + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(begin())`. + + @liveexample{The following code shows an example for `rend()`.,rend} + + @sa @ref crend() -- returns a const reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /*! + @copydoc basic_json::crend() + */ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /*! + @brief returns a const reverse iterator to the last element + + Returns a const iterator to the reverse-beginning; that is, the last + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`. + + @liveexample{The following code shows an example for `crbegin()`.,crbegin} + + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /*! + @brief returns a const reverse iterator to one before the first + + Returns a const reverse iterator to the reverse-end; that is, one before + the first element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).rend()`. + + @liveexample{The following code shows an example for `crend()`.,crend} + + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + private: + // forward declaration + template<typename IteratorType> class iteration_proxy; + + public: + /*! + @brief wrapper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + @note The name of this function is not yet final and may change in the + future. + */ + static iteration_proxy<iterator> iterator_wrapper(reference cont) + { + return iteration_proxy<iterator>(cont); + } + + /*! + @copydoc iterator_wrapper(reference) + */ + static iteration_proxy<const_iterator> iterator_wrapper(const_reference cont) + { + return iteration_proxy<const_iterator>(cont); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /*! + @brief checks whether the container is empty + + Checks if a JSON value has no elements. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `true` + boolean | `false` + string | `false` + number | `false` + object | result of function `object_t::empty()` + array | result of function `array_t::empty()` + + @note This function does not return whether a string stored as JSON value + is empty - it returns whether the JSON container itself is empty which is + false in the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `empty()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `begin() == end()`. + + @liveexample{The following code uses `empty()` to check if a JSON + object contains any elements.,empty} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + default: + { + // all other types are nonempty + return false; + } + } + } + + /*! + @brief returns the number of elements + + Returns the number of elements in a JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` + boolean | `1` + string | `1` + number | `1` + object | result of function object_t::size() + array | result of function array_t::size() + + @note This function does not return the length of a string stored as JSON + value - it returns the number of elements in the JSON value which is 1 in + the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their size() functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `std::distance(begin(), end())`. + + @liveexample{The following code calls `size()` on the different value + types.,size} + + @sa @ref empty() -- checks whether the container is empty + @sa @ref max_size() -- returns the maximal number of elements + + @since version 1.0.0 + */ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + default: + { + // all other types have size 1 + return 1; + } + } + } + + /*! + @brief returns the maximum possible number of elements + + Returns the maximum number of elements a JSON value is able to hold due to + system or library implementation limitations, i.e. `std::distance(begin(), + end())` for the JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` (same as `size()`) + boolean | `1` (same as `size()`) + string | `1` (same as `size()`) + number | `1` (same as `size()`) + object | result of function `object_t::max_size()` + array | result of function `array_t::max_size()` + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `max_size()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of returning `b.size()` where `b` is the largest + possible JSON value. + + @liveexample{The following code calls `max_size()` on the different value + types. Note the output is implementation specific.,max_size} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /*! + @brief clears the contents + + Clears the content of a JSON value and resets it to the default value as + if @ref basic_json(value_t) would have been called: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @note Floating-point numbers are set to `0.0` which will be serialized to + `0`. The vale type remains @ref number_float_t. + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows the effect of `clear()` to different + JSON types.,clear} + + @since version 1.0.0 + */ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + default: + { + break; + } + } + } + + /*! + @brief add an object to an array + + Appends the given element @a val to the end of the JSON value. If the + function is called on a JSON null value, an empty array is created before + appending @a val. + + @param[in] val the value to add to the JSON array + + @throw std::domain_error when called on a type other than JSON array or + null; example: `"cannot use push_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON array. Note how the `null` value was silently + converted to a JSON array.,push_back} + + @since version 1.0.0 + */ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + Throw<std::domain_error>("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + m_value.array->push_back(std::move(val)); + // invalidate object + val.m_type = value_t::null; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + Throw<std::domain_error>("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + m_value.array->push_back(val); + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + Inserts the given element @a val to the JSON object. If the function is + called on a JSON null value, an empty object is created before inserting + @a val. + + @param[in] val the value to add to the JSON object + + @throw std::domain_error when called on a type other than JSON object or + null; example: `"cannot use push_back() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON object. Note how the `null` value was silently + converted to a JSON object.,push_back__object_t__value} + + @since version 1.0.0 + */ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (not(is_null() or is_object())) + { + Throw<std::domain_error>("cannot use push_back() with " + type_name()); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array + m_value.object->insert(val); + } + + /*! + @brief add an object to an object + @copydoc push_back(const typename object_t::value_type&) + */ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + This function allows to use `push_back` with an initializer list. In case + + 1. the current value is an object, + 2. the initializer list @a init contains only two elements, and + 3. the first element of @a init is a string, + + @a init is converted into an object element and added using + @ref push_back(const typename object_t::value_type&). Otherwise, @a init + is converted to a JSON value and added using @ref push_back(basic_json&&). + + @param init an initializer list + + @complexity Linear in the size of the initializer list @a init. + + @note This function is required to resolve an ambiguous overload error, + because pairs like `{"key", "value"}` can be both interpreted as + `object_t::value_type` or `std::initializer_list<basic_json>`, see + https://github.com/nlohmann/json/issues/235 for more information. + + @liveexample{The example shows how initializer lists are treated as + objects when possible.,push_back__initializer_list} + */ + void push_back(std::initializer_list<basic_json> init) + { + if (is_object() and init.size() == 2 and init.begin()->is_string()) + { + const string_t key = *init.begin(); + push_back(typename object_t::value_type(key, *(init.begin() + 1))); + } + else + { + push_back(basic_json(init)); + } + } + + /*! + @brief add an object to an object + @copydoc push_back(std::initializer_list<basic_json>) + */ + reference operator+=(std::initializer_list<basic_json> init) + { + push_back(init); + return *this; + } + + /*! + @brief inserts element + + Inserts element @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] val element to insert + @return iterator pointing to the inserted @a val. + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Constant plus linear in the distance between pos and end of the + container. + + @liveexample{The example shows how `insert()` is used.,insert} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + Throw<std::domain_error>("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); + return result; + } + else + { + Throw<std::domain_error>("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts element + @copydoc insert(const_iterator, const basic_json&) + */ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /*! + @brief inserts elements + + Inserts @a cnt copies of @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] cnt number of copies of @a val to insert + @param[in] val element to insert + @return iterator pointing to the first element inserted, or @a pos if + `cnt==0` + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Linear in @a cnt plus linear in the distance between @a pos + and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__count} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + Throw<std::domain_error>("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + return result; + } + else + { + Throw<std::domain_error>("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts elements + + Inserts elements from range `[first, last)` before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + @throw std::domain_error if @a first and @a last do not belong to the same + JSON value; example: `"iterators do not fit"` + @throw std::domain_error if @a first or @a last are iterators into + container for which insert is called; example: `"passed iterators may not + belong to container"` + + @return iterator pointing to the first element inserted, or @a pos if + `first==last` + + @complexity Linear in `std::distance(first, last)` plus linear in the + distance between @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__range} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (not is_array()) + { + Throw<std::domain_error>("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + Throw<std::domain_error>("iterator does not fit current value"); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + Throw<std::domain_error>("iterators do not fit"); + } + + if (first.m_object == this or last.m_object == this) + { + Throw<std::domain_error>("passed iterators may not belong to container"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert( + pos.m_it.array_iterator, + first.m_it.array_iterator, + last.m_it.array_iterator); + return result; + } + + /*! + @brief inserts elements + + Inserts elements from initializer list @a ilist before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] ilist initializer list to insert the values from + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @return iterator pointing to the first element inserted, or @a pos if + `ilist` is empty + + @complexity Linear in `ilist.size()` plus linear in the distance between + @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__ilist} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, std::initializer_list<basic_json> ilist) + { + // insert only works for arrays + if (not is_array()) + { + Throw<std::domain_error>("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + Throw<std::domain_error>("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); + return result; + } + + /*! + @brief exchanges the values + + Exchanges the contents of the JSON value with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other JSON value to exchange the contents with + + @complexity Constant. + + @liveexample{The example below shows how JSON values can be swapped with + `swap()`.,swap__reference} + + @since version 1.0.0 + */ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible<value_t>::value and + std::is_nothrow_move_assignable<value_t>::value and + std::is_nothrow_move_constructible<json_value>::value and + std::is_nothrow_move_assignable<json_value>::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + assert_invariant(); + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON array with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other array to exchange the contents with + + @throw std::domain_error when JSON value is not an array; example: `"cannot + use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how arrays can be swapped with + `swap()`.,swap__array_t} + + @since version 1.0.0 + */ + void swap(array_t& other) + { + // swap only works for arrays + if (is_array()) + { + std::swap(*(m_value.array), other); + } + else + { + Throw<std::domain_error>("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON object with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other object to exchange the contents with + + @throw std::domain_error when JSON value is not an object; example: + `"cannot use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how objects can be swapped with + `swap()`.,swap__object_t} + + @since version 1.0.0 + */ + void swap(object_t& other) + { + // swap only works for objects + if (is_object()) + { + std::swap(*(m_value.object), other); + } + else + { + Throw<std::domain_error>("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON string with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other string to exchange the contents with + + @throw std::domain_error when JSON value is not a string; example: `"cannot + use swap() with boolean"` + + @complexity Constant. + + @liveexample{The example below shows how strings can be swapped with + `swap()`.,swap__string_t} + + @since version 1.0.0 + */ + void swap(string_t& other) + { + // swap only works for strings + if (is_string()) + { + std::swap(*(m_value.string), other); + } + else + { + Throw<std::domain_error>("cannot use swap() with " + type_name()); + } + } + + /// @} + + + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + private: + /*! + @brief comparison operator for JSON types + + Returns an ordering that is similar to Python: + - order: null < boolean < number < object < array < string + - furthermore, each type is not smaller than itself + + @since version 1.0.0 + */ + friend bool operator<(const value_t lhs, const value_t rhs) noexcept + { + static constexpr std::array<uint8_t, 8> order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) + { + return false; + } + + return order[static_cast<std::size_t>(lhs)] < order[static_cast<std::size_t>(rhs)]; + } + + public: + /*! + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same. + - Integer and floating-point numbers are automatically converted before + comparison. Floating-point numbers are compared indirectly: two + floating-point numbers `f1` and `f2` are considered equal if neither + `f1 > f2` nor `f2 > f1` holds. + - Two JSON null values are equal. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array == *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object == *rhs.m_value.object; + } + case value_t::null: + { + return true; + } + case value_t::string: + { + return *lhs.m_value.string == *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean == rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer == rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float == rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned); + } + + return false; + } + + /*! + @brief comparison: equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__equal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator==(const_reference v, std::nullptr_t) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, std::nullptr_t) + */ + friend bool operator==(std::nullptr_t, const_reference v) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs == rhs); + } + + /*! + @brief comparison: not equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `not v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is not null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__notequal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference v, std::nullptr_t) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, std::nullptr_t) + */ + friend bool operator!=(std::nullptr_t, const_reference v) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: less than + + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. + - Integer and floating-point numbers are automatically converted before + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__less} + + @since version 1.0.0 + */ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array < *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object < *rhs.m_value.object; + } + case value_t::null: + { + return false; + } + case value_t::string: + { + return *lhs.m_value.string < *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean < rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer < rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float < rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast<number_float_t>(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast<number_float_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast<number_integer_t>(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast<number_integer_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /*! + @brief comparison: less than or equal + + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} + + @since version 1.0.0 + */ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return not (rhs < lhs); + } + + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__lessequal} + + @since version 1.0.0 + */ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs <= rhs); + } + + /*! + @brief comparison: greater than or equal + + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} + + @since version 1.0.0 + */ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs < rhs); + } + + /// @} + + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ + + /*! + @brief serialize to stream + + Serialize the given JSON value @a j to the output stream @a o. The JSON + value will be serialized using the @ref dump member function. The + indentation of the output can be controlled with the member variable + `width` of the output stream @a o. For instance, using the manipulator + `std::setw(4)` on @a o sets the indentation level to `4` and the + serialization result is the same as calling `dump(4)`. + + @note During serializaion, the locale and the precision of the output + stream @a o are changed. The original values are restored when the + function returns. + + @param[in,out] o stream to serialize to + @param[in] j JSON value to serialize + + @return the stream @a o + + @complexity Linear. + + @liveexample{The example below shows the serialization with different + parameters to `width` to adjust the indentation level.,operator_serialize} + + @since version 1.0.0 + */ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = (o.width() > 0); + const auto indentation = (pretty_print ? o.width() : 0); + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // fix locale problems + const auto old_locale = o.imbue(std::locale::classic()); + // set precision + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits<number_float_t>::digits10 + const auto old_precision = o.precision(std::numeric_limits<double>::digits10); + + // do the actual serialization + j.dump(o, pretty_print, static_cast<unsigned int>(indentation)); + + // reset locale and precision + o.imbue(old_locale); + o.precision(old_precision); + return o; + } + + /*! + @brief serialize to stream + @copydoc operator<<(std::ostream&, const basic_json&) + */ + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } + + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /*! + @brief deserialize from an array + + This function reads from an array of 1-byte values. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @param[in] array array to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an array.,parse__array__parser_callback_t} + + @since version 2.0.3 + */ + template<class T, std::size_t N> + static basic_json parse(T (&array)[N], + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); + } + + /*! + @brief deserialize from string literal + + @tparam CharT character/literal type with size of 1 byte + @param[in] s string literal to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + @note String containers like `std::string` or @ref string_t can be parsed + with @ref parse(const ContiguousContainer&, const parser_callback_t) + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @sa @ref parse(std::istream&, const parser_callback_t) for a version that + reads from an input stream + + @since version 1.0.0 (originally for @ref string_t) + */ + template<typename CharPT, typename std::enable_if< + std::is_pointer<CharPT>::value and + std::is_integral<typename std::remove_pointer<CharPT>::type>::value and + sizeof(typename std::remove_pointer<CharPT>::type) == 1, int>::type = 0> + static basic_json parse(const CharPT s, + const parser_callback_t cb = nullptr) + { + return parser(reinterpret_cast<const char*>(s), cb).parse(); + } + + /*! + @brief deserialize from stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__istream__parser_callback_t} + + @sa @ref parse(const char*, const parser_callback_t) for a version + that reads from a string + + @since version 1.0.0 + */ + static basic_json parse(std::istream& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @copydoc parse(std::istream&, const parser_callback_t) + */ + static basic_json parse(std::istream&& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @brief deserialize from an iterator range with contiguous storage + + This function reads from an iterator range of a container with contiguous + storage of 1-byte values. Compatible container types include + `std::vector`, `std::string`, `std::array`, `std::valarray`, and + `std::initializer_list`. Furthermore, C-style arrays can be used with + `std::begin()`/`std::end()`. User-defined containers can be used as long + as they implement random-access iterators and a contiguous storage. + + @pre The iterator range is contiguous. Violating this precondition yields + undefined behavior. **This precondition is enforced with an assertion.** + @pre Each element in the range has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with noncompliant iterators and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @tparam IteratorType iterator of container with contiguous storage + @param[in] first begin of the range to parse (included) + @param[in] last end of the range to parse (excluded) + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an iterator range.,parse__iteratortype__parser_callback_t} + + @since version 2.0.3 + */ + template<class IteratorType, typename std::enable_if< + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0> + static basic_json parse(IteratorType first, IteratorType last, + const parser_callback_t cb = nullptr) + { + // assertion to check that the iterator range is indeed contiguous, + // see http://stackoverflow.com/a/35008842/266378 for more discussion + assert(std::accumulate(first, last, std::make_pair<bool, int>(true, 0), + [&first](std::pair<bool, int> res, decltype(*first) val) + { + res.first &= (val == *(std::next(std::addressof(*first), res.second++))); + return res; + }).first); + + // assertion to check that each element is 1 byte long + static_assert(sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1, + "each element in the iterator range must have the size of 1 byte"); + + // if iterator range is empty, create a parser with an empty string + // to generate "unexpected EOF" error message + if (std::distance(first, last) <= 0) + { + return parser("").parse(); + } + + return parser(first, last, cb).parse(); + } + + /*! + @brief deserialize from a container with contiguous storage + + This function reads from a container with contiguous storage of 1-byte + values. Compatible container types include `std::vector`, `std::string`, + `std::array`, and `std::initializer_list`. User-defined containers can be + used as long as they implement random-access iterators and a contiguous + storage. + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @tparam ContiguousContainer container type with contiguous storage + @param[in] c container to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from a contiguous container.,parse__contiguouscontainer__parser_callback_t} + + @since version 2.0.3 + */ + template<class ContiguousContainer, typename std::enable_if< + not std::is_pointer<ContiguousContainer>::value and + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value + , int>::type = 0> + static basic_json parse(const ContiguousContainer& c, + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(c), std::end(c), cb); + } + + /*! + @brief deserialize from stream + + Deserializes an input stream to a JSON value. + + @param[in,out] i input stream to read a serialized JSON value from + @param[in,out] j JSON value to write the deserialized input to + + @throw std::invalid_argument in case of parse errors + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below shows how a JSON value is constructed by + reading a serialization from a stream.,operator_deserialize} + + @sa parse(std::istream&, const parser_callback_t) for a variant with a + parser callback function to filter values while parsing + + @since version 1.0.0 + */ + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; + } + + /*! + @brief deserialize from stream + @copydoc operator<<(basic_json&, std::istream&) + */ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + j = parser(i).parse(); + return i; + } + + /// @} + + + private: + /////////////////////////// + // convenience functions // + /////////////////////////// + + /*! + @brief return the type as string + + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. + + @return basically a string representation of a the @a m_type member + + @complexity Constant. + + @since version 1.0.0 + */ + std::string type_name() const + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; + } + } + + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + else + { + return res; + } + } + } + }); + } + + /*! + @brief escape a string + + Escape a string by replacing certain special characters by a sequence of + an escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. + + @param[in] s the string to escape + @return the escaped string + + @complexity Linear in the length of string @a s. + */ + static string_t escape_string(const string_t& s) + { + const auto space = extra_space(s); + if (space == 0) + { + return s; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + } + else + { + // all other characters are added as-is + result[pos++] = c; + } + break; + } + } + } + + return result; + } + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. Note that + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[out] o stream to write to + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(std::ostream& o, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) const + { + // variable to hold indentation for recursive calls + unsigned int new_indent = current_indent; + + switch (m_type) + { + case value_t::object: + { + if (m_value.object->empty()) + { + o << "{}"; + return; + } + + o << "{"; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) + { + if (i != m_value.object->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' ') << "\"" + << escape_string(i->first) << "\":" + << (pretty_print ? " " : ""); + i->second.dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') + "}"; + return; + } + + case value_t::array: + { + if (m_value.array->empty()) + { + o << "[]"; + return; + } + + o << "["; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) + { + if (i != m_value.array->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' '); + i->dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') << "]"; + return; + } + + case value_t::string: + { + o << string_t("\"") << escape_string(*m_value.string) << "\""; + return; + } + + case value_t::boolean: + { + o << (m_value.boolean ? "true" : "false"); + return; + } + + case value_t::number_integer: + { + o << m_value.number_integer; + return; + } + + case value_t::number_unsigned: + { + o << m_value.number_unsigned; + return; + } + + case value_t::number_float: + { + if (m_value.number_float == 0) + { + // special case for zero to get "0.0"/"-0.0" + o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); + } + else + { + o << m_value.number_float; + } + return; + } + + case value_t::discarded: + { + o << "<discarded>"; + return; + } + + case value_t::null: + { + o << "null"; + return; + } + } + } + + private: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + + + private: + /////////////// + // iterators // + /////////////// + + /*! + @brief an iterator for primitive JSON types + + This class models an iterator for primitive JSON types (boolean, number, + string). It's only purpose is to allow the iterator/const_iterator classes + to "iterate" over primitive values. Internally, the iterator is modeled by + a `difference_type` variable. Value begin_value (`0`) models the begin, + end_value (`1`) models past the end. + */ + class primitive_iterator_t + { + public: + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return (m_it == begin_value); + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return (m_it == end_value); + } + + /// return reference to the value to change and compare + operator difference_type& () noexcept + { + return m_it; + } + + /// return value to compare + constexpr operator difference_type () const noexcept + { + return m_it; + } + + private: + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + /// iterator as signed integer type + difference_type m_it = std::numeric_limits<std::ptrdiff_t>::denorm_min(); + }; + + /*! + @brief an iterator value + + @note This structure could easily be a union, but MSVC currently does not + allow unions members with complex constructors, see + https://github.com/nlohmann/json/pull/105. + */ + struct internal_iterator + { + /// iterator for JSON objects + typename object_t::iterator object_iterator; + /// iterator for JSON arrays + typename array_t::iterator array_iterator; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator; + + /// create an uninitialized internal_iterator + internal_iterator() noexcept + : object_iterator(), array_iterator(), primitive_iterator() + {} + }; + + /// proxy class for the iterator_wrapper functions + template<typename IteratorType> + class iteration_proxy + { + private: + /// helper class for iteration + class iteration_proxy_internal + { + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + size_t array_index = 0; + + public: + explicit iteration_proxy_internal(IteratorType it) noexcept + : anchor(it) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_internal& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_internal& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// inequality operator (needed for range-based for) + bool operator!= (const iteration_proxy_internal& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + typename basic_json::string_t key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + return std::to_string(array_index); + } + + // use key from the object + case value_t::object: + { + return anchor.key(); + } + + // use an empty key for all primitive types + default: + { + return ""; + } + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } + }; + + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) + : container(cont) + {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_internal begin() noexcept + { + return iteration_proxy_internal(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_internal end() noexcept + { + return iteration_proxy_internal(container.end()); + } + }; + + public: + /*! + @brief a const random access iterator for the @ref basic_json class + + This class implements a const iterator for the @ref basic_json class. From + this class, the @ref iterator class is derived. + + @note An iterator is called *initialized* when a pointer to a JSON value + has been set (e.g., by a constructor or a copy assignment). If the + iterator is default-constructed, it is *uninitialized* and most + methods are undefined. **The library uses assertions to detect calls + on uninitialized iterators.** + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + + @since version 1.0.0 + */ + class const_iterator : public std::iterator<std::random_access_iterator_tag, const basic_json> + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /// the type of the values when the iterator is dereferenced + using value_type = typename basic_json::value_type; + /// a type to represent differences between iterators + using difference_type = typename basic_json::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename basic_json::const_pointer; + /// defines a reference to the type iterated over (value_type) + using reference = typename basic_json::const_reference; + /// the category of the iterator + using iterator_category = std::bidirectional_iterator_tag; + + /// default constructor + const_iterator() = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit const_iterator(pointer object) noexcept + : m_object(object) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @brief copy constructor given a non-const iterator + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + explicit const_iterator(const iterator& other) noexcept + : m_object(other.m_object) + { + if (m_object != nullptr) + { + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = other.m_it.object_iterator; + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = other.m_it.array_iterator; + break; + } + + default: + { + m_it.primitive_iterator = other.m_it.primitive_iterator; + break; + } + } + } + } + + /*! + @brief copy constructor + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator(const const_iterator& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief copy assignment + @param[in,out] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator& operator=(const_iterator other) noexcept( + std::is_nothrow_move_constructible<pointer>::value and + std::is_nothrow_move_assignable<pointer>::value and + std::is_nothrow_move_constructible<internal_iterator>::value and + std::is_nothrow_move_assignable<internal_iterator>::value + ) + { + std::swap(m_object, other.m_object); + std::swap(m_it, other.m_it); + return *this; + } + + private: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case basic_json::value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case basic_json::value_t::null: + { + Throw<std::out_of_range>("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return *m_object; + } + else + { + Throw<std::out_of_range>("cannot get value"); + } + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return m_object; + } + else + { + Throw<std::out_of_range>("cannot get value"); + } + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator++(int) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator++() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator--(int) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator--() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator==(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + Throw<std::domain_error>("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator == other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator!=(const const_iterator& other) const + { + return not operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + Throw<std::domain_error>("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + Throw<std::domain_error>("cannot compare order of object iterators"); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator < other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const const_iterator& other) const + { + return not other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const const_iterator& other) const + { + return not operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const const_iterator& other) const + { + return not operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator+=(difference_type i) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + Throw<std::domain_error>("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const const_iterator& other) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + Throw<std::domain_error>("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + return m_it.array_iterator - other.m_it.array_iterator; + } + + default: + { + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + Throw<std::domain_error>("cannot use operator[] for object iterators"); + } + + case basic_json::value_t::array: + { + return *std::next(m_it.array_iterator, n); + } + + case basic_json::value_t::null: + { + Throw<std::out_of_range>("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator == -n) + { + return *m_object; + } + else + { + Throw<std::out_of_range>("cannot get value"); + } + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + typename object_t::key_type key() const + { + assert(m_object != nullptr); + + if (m_object->is_object()) + { + return m_it.object_iterator->first; + } + else + { + Throw<std::domain_error>("cannot use key() for non-object iterators"); + } + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + private: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator m_it = internal_iterator(); + }; + + /*! + @brief a mutable random access iterator for the @ref basic_json class + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element. + + @since version 1.0.0 + */ + class iterator : public const_iterator + { + public: + using base_iterator = const_iterator; + using pointer = typename basic_json::pointer; + using reference = typename basic_json::reference; + + /// default constructor + iterator() = default; + + /// constructor for a given JSON instance + explicit iterator(pointer object) noexcept + : base_iterator(object) + {} + + /// copy constructor + iterator(const iterator& other) noexcept + : base_iterator(other) + {} + + /// copy assignment + iterator& operator=(iterator other) noexcept( + std::is_nothrow_move_constructible<pointer>::value and + std::is_nothrow_move_assignable<pointer>::value and + std::is_nothrow_move_constructible<internal_iterator>::value and + std::is_nothrow_move_assignable<internal_iterator>::value + ) + { + base_iterator::operator=(other); + return *this; + } + + /// return a reference to the value pointed to by the iterator + reference operator*() const + { + return const_cast<reference>(base_iterator::operator*()); + } + + /// dereference the iterator + pointer operator->() const + { + return const_cast<pointer>(base_iterator::operator->()); + } + + /// post-increment (it++) + iterator operator++(int) + { + iterator result = *this; + base_iterator::operator++(); + return result; + } + + /// pre-increment (++it) + iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + iterator operator--(int) + { + iterator result = *this; + base_iterator::operator--(); + return result; + } + + /// pre-decrement (--it) + iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// subtract from iterator + iterator& operator-=(difference_type i) + { + base_iterator::operator-=(i); + return *this; + } + + /// add to iterator + iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const iterator& other) const + { + return base_iterator::operator-(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return const_cast<reference>(base_iterator::operator[](n)); + } + + /// return the value of an iterator + reference value() const + { + return const_cast<reference>(base_iterator::value()); + } + }; + + /*! + @brief a template for a reverse iterator class + + @tparam Base the base iterator type to reverse. Valid types are @ref + iterator (to create @ref reverse_iterator) and @ref const_iterator (to + create @ref const_reverse_iterator). + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + + @since version 1.0.0 + */ + template<typename Base> + class json_reverse_iterator : public std::reverse_iterator<Base> + { + public: + /// shortcut to the reverse iterator adaptor + using base_iterator = std::reverse_iterator<Base>; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) + {} + + /// create reverse iterator from base class + json_reverse_iterator(const base_iterator& it) noexcept + : base_iterator(it) + {} + + /// post-increment (it++) + json_reverse_iterator operator++(int) + { + return base_iterator::operator++(1); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + json_reverse_iterator operator--(int) + { + return base_iterator::operator--(1); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return this->base() - other.base(); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + typename object_t::key_type key() const + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } + }; + + + private: + ////////////////////// + // lexer and parser // + ////////////////////// + + /*! + @brief lexical analysis + + This class organizes the lexical analysis during JSON deserialization. The + core of it is a scanner generated by [re2c](http://re2c.org) that + processes a buffer and recognizes tokens according to RFC 7159. + */ + class lexer + { + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_number, ///< a number -- use get_number() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input ///< indicating the end of the input buffer + }; + + /// the char type to use in the lexer + using lexer_char_t = unsigned char; + + /// a lexer from a buffer with given length + lexer(const lexer_char_t* buff, const size_t len) noexcept + : m_content(buff) + { + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + len; + } + + /// a lexer from an input stream + explicit lexer(std::istream& s) + : m_stream(&s), m_line_buffer() + { + // immediately abort if stream is erroneous + if (s.fail()) + { + Throw<std::invalid_argument>("stream error: " + std::string(strerror(errno))); + } + + // fill buffer + fill_line_buffer(); + + // skip UTF-8 byte-order mark + if (m_line_buffer.size() >= 3 and m_line_buffer.substr(0, 3) == "\xEF\xBB\xBF") + { + m_line_buffer[0] = ' '; + m_line_buffer[1] = ' '; + m_line_buffer[2] = ' '; + } + } + + // switch off unwanted functions (due to pointer members) + lexer() = delete; + lexer(const lexer&) = delete; + lexer operator=(const lexer&) = delete; + + /*! + @brief create a string from one or two Unicode code points + + There are two cases: (1) @a codepoint1 is in the Basic Multilingual + Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) + @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to + represent a code point above U+FFFF. + + @param[in] codepoint1 the code point (can be high surrogate) + @param[in] codepoint2 the code point (can be low surrogate or 0) + + @return string representation of the code point; the length of the + result string is between 1 and 4 characters. + + @throw std::out_of_range if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` + @throw std::invalid_argument if the low surrogate is invalid; example: + `""missing or wrong low surrogate""` + + @complexity Constant. + + @see <http://en.wikipedia.org/wiki/UTF-8#Sample_code> + */ + static string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) + { + // calculate the code point from the given code points + std::size_t codepoint = codepoint1; + + // check if codepoint1 is a high surrogate + if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) + { + // check if codepoint2 is a low surrogate + if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) + { + codepoint = + // high surrogate occupies the most significant 22 bits + (codepoint1 << 10) + // low surrogate occupies the least significant 15 bits + + codepoint2 + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00; + } + else + { + Throw<std::invalid_argument>("missing or wrong low surrogate"); + } + } + + string_t result; + + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + result.append(1, static_cast<typename string_t::value_type>(codepoint)); + } + else if (codepoint <= 0x7ff) + { + // 2-byte characters: 110xxxxx 10xxxxxx + result.append(1, static_cast<typename string_t::value_type>(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast<typename string_t::value_type>(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xffff) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast<typename string_t::value_type>(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast<typename string_t::value_type>(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast<typename string_t::value_type>(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0x10ffff) + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast<typename string_t::value_type>(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast<typename string_t::value_type>(0x80 | ((codepoint >> 12) & 0x3F))); + result.append(1, static_cast<typename string_t::value_type>(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast<typename string_t::value_type>(0x80 | (codepoint & 0x3F))); + } + else + { + Throw<std::out_of_range>("code points above 0x10FFFF are invalid"); + } + + return result; + } + + /// return name of values of type token_type (only used for errors) + static std::string token_type_name(const token_type t) + { + switch (t) + { + case token_type::uninitialized: + return "<uninitialized>"; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case token_type::value_number: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return "<parse error>"; + case token_type::end_of_input: + return "end of input"; + default: + { + // catch non-enum values + return "unknown token"; // LCOV_EXCL_LINE + } + } + } + + /*! + This function implements a scanner for JSON. It is specified using + regular expressions that try to follow RFC 7159 as close as possible. + These regular expressions are then translated into a minimized + deterministic finite automaton (DFA) by the tool + [re2c](http://re2c.org). As a result, the translated code for this + function consists of a large block of code with `goto` jumps. + + @return the class of the next token read from the buffer + + @complexity Linear in the length of the input.\n + + Proposition: The loop below will always terminate for finite input.\n + + Proof (by contradiction): Assume a finite input. To loop forever, the + loop must never hit code with a `break` statement. The only code + snippets without a `break` statement are the continue statements for + whitespace and byte-order-marks. To loop forever, the input must be an + infinite sequence of whitespace or byte-order-marks. This contradicts + the assumption of finite input, q.e.d. + */ + token_type scan() + { + while (true) + { + // pointer for backtracking information + m_marker = nullptr; + + // remember the begin of the token + m_start = m_cursor; + assert(m_start != nullptr); + + + { + lexer_char_t yych; + unsigned int yyaccept = 0; + static const unsigned char yybm[] = + { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 32, 0, 0, 32, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + if ((m_limit - m_cursor) < 5) + { + fill_line_buffer(5); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + if (yych <= '[') + { + if (yych <= '-') + { + if (yych <= '"') + { + if (yych <= 0x00) + { + goto basic_json_parser_2; + } + if (yych <= '!') + { + goto basic_json_parser_4; + } + goto basic_json_parser_9; + } + else + { + if (yych <= '+') + { + goto basic_json_parser_4; + } + if (yych <= ',') + { + goto basic_json_parser_10; + } + goto basic_json_parser_12; + } + } + else + { + if (yych <= '9') + { + if (yych <= '/') + { + goto basic_json_parser_4; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + goto basic_json_parser_15; + } + else + { + if (yych <= ':') + { + goto basic_json_parser_17; + } + if (yych <= 'Z') + { + goto basic_json_parser_4; + } + goto basic_json_parser_19; + } + } + } + else + { + if (yych <= 'n') + { + if (yych <= 'e') + { + if (yych == ']') + { + goto basic_json_parser_21; + } + goto basic_json_parser_4; + } + else + { + if (yych <= 'f') + { + goto basic_json_parser_23; + } + if (yych <= 'm') + { + goto basic_json_parser_4; + } + goto basic_json_parser_24; + } + } + else + { + if (yych <= 'z') + { + if (yych == 't') + { + goto basic_json_parser_25; + } + goto basic_json_parser_4; + } + else + { + if (yych <= '{') + { + goto basic_json_parser_26; + } + if (yych == '}') + { + goto basic_json_parser_28; + } + goto basic_json_parser_4; + } + } + } +basic_json_parser_2: + ++m_cursor; + { + last_token_type = token_type::end_of_input; + break; + } +basic_json_parser_4: + ++m_cursor; +basic_json_parser_5: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_6: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + { + continue; + } +basic_json_parser_9: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych <= 0x1F) + { + goto basic_json_parser_5; + } + if (yych <= 0x7F) + { + goto basic_json_parser_31; + } + if (yych <= 0xC1) + { + goto basic_json_parser_5; + } + if (yych <= 0xF4) + { + goto basic_json_parser_31; + } + goto basic_json_parser_5; +basic_json_parser_10: + ++m_cursor; + { + last_token_type = token_type::value_separator; + break; + } +basic_json_parser_12: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_5; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + if (yych <= '9') + { + goto basic_json_parser_15; + } + goto basic_json_parser_5; +basic_json_parser_13: + yyaccept = 1; + yych = *(m_marker = ++m_cursor); + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_43; + } + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_44; + } + if (yych == 'e') + { + goto basic_json_parser_44; + } + } +basic_json_parser_14: + { + last_token_type = token_type::value_number; + break; + } +basic_json_parser_15: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yybm[0 + yych] & 64) + { + goto basic_json_parser_15; + } + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_43; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_44; + } + if (yych == 'e') + { + goto basic_json_parser_44; + } + goto basic_json_parser_14; + } +basic_json_parser_17: + ++m_cursor; + { + last_token_type = token_type::name_separator; + break; + } +basic_json_parser_19: + ++m_cursor; + { + last_token_type = token_type::begin_array; + break; + } +basic_json_parser_21: + ++m_cursor; + { + last_token_type = token_type::end_array; + break; + } +basic_json_parser_23: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'a') + { + goto basic_json_parser_45; + } + goto basic_json_parser_5; +basic_json_parser_24: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'u') + { + goto basic_json_parser_46; + } + goto basic_json_parser_5; +basic_json_parser_25: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'r') + { + goto basic_json_parser_47; + } + goto basic_json_parser_5; +basic_json_parser_26: + ++m_cursor; + { + last_token_type = token_type::begin_object; + break; + } +basic_json_parser_28: + ++m_cursor; + { + last_token_type = token_type::end_object; + break; + } +basic_json_parser_30: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; +basic_json_parser_31: + if (yybm[0 + yych] & 128) + { + goto basic_json_parser_30; + } + if (yych <= 0xE0) + { + if (yych <= '\\') + { + if (yych <= 0x1F) + { + goto basic_json_parser_32; + } + if (yych <= '"') + { + goto basic_json_parser_33; + } + goto basic_json_parser_35; + } + else + { + if (yych <= 0xC1) + { + goto basic_json_parser_32; + } + if (yych <= 0xDF) + { + goto basic_json_parser_36; + } + goto basic_json_parser_37; + } + } + else + { + if (yych <= 0xEF) + { + if (yych == 0xED) + { + goto basic_json_parser_39; + } + goto basic_json_parser_38; + } + else + { + if (yych <= 0xF0) + { + goto basic_json_parser_40; + } + if (yych <= 0xF3) + { + goto basic_json_parser_41; + } + if (yych <= 0xF4) + { + goto basic_json_parser_42; + } + } + } +basic_json_parser_32: + m_cursor = m_marker; + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } +basic_json_parser_33: + ++m_cursor; + { + last_token_type = token_type::value_string; + break; + } +basic_json_parser_35: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 'e') + { + if (yych <= '/') + { + if (yych == '"') + { + goto basic_json_parser_30; + } + if (yych <= '.') + { + goto basic_json_parser_32; + } + goto basic_json_parser_30; + } + else + { + if (yych <= '\\') + { + if (yych <= '[') + { + goto basic_json_parser_32; + } + goto basic_json_parser_30; + } + else + { + if (yych == 'b') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + } + } + else + { + if (yych <= 'q') + { + if (yych <= 'f') + { + goto basic_json_parser_30; + } + if (yych == 'n') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 's') + { + if (yych <= 'r') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 't') + { + goto basic_json_parser_30; + } + if (yych <= 'u') + { + goto basic_json_parser_48; + } + goto basic_json_parser_32; + } + } + } +basic_json_parser_36: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; +basic_json_parser_37: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x9F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_36; + } + goto basic_json_parser_32; +basic_json_parser_38: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_36; + } + goto basic_json_parser_32; +basic_json_parser_39: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0x9F) + { + goto basic_json_parser_36; + } + goto basic_json_parser_32; +basic_json_parser_40: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x8F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_41: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_42: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0x8F) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_43: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_49; + } + goto basic_json_parser_32; +basic_json_parser_44: + yych = *++m_cursor; + if (yych <= ',') + { + if (yych == '+') + { + goto basic_json_parser_51; + } + goto basic_json_parser_32; + } + else + { + if (yych <= '-') + { + goto basic_json_parser_51; + } + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_52; + } + goto basic_json_parser_32; + } +basic_json_parser_45: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_54; + } + goto basic_json_parser_32; +basic_json_parser_46: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_55; + } + goto basic_json_parser_32; +basic_json_parser_47: + yych = *++m_cursor; + if (yych == 'u') + { + goto basic_json_parser_56; + } + goto basic_json_parser_32; +basic_json_parser_48: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_57; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_57; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_57; + } + goto basic_json_parser_32; + } +basic_json_parser_49: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 'D') + { + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_49; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_44; + } + if (yych == 'e') + { + goto basic_json_parser_44; + } + goto basic_json_parser_14; + } +basic_json_parser_51: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych >= ':') + { + goto basic_json_parser_32; + } +basic_json_parser_52: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_52; + } + goto basic_json_parser_14; +basic_json_parser_54: + yych = *++m_cursor; + if (yych == 's') + { + goto basic_json_parser_58; + } + goto basic_json_parser_32; +basic_json_parser_55: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_59; + } + goto basic_json_parser_32; +basic_json_parser_56: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_61; + } + goto basic_json_parser_32; +basic_json_parser_57: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_63; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_63; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_63; + } + goto basic_json_parser_32; + } +basic_json_parser_58: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_64; + } + goto basic_json_parser_32; +basic_json_parser_59: + ++m_cursor; + { + last_token_type = token_type::literal_null; + break; + } +basic_json_parser_61: + ++m_cursor; + { + last_token_type = token_type::literal_true; + break; + } +basic_json_parser_63: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_66; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_66; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_66; + } + goto basic_json_parser_32; + } +basic_json_parser_64: + ++m_cursor; + { + last_token_type = token_type::literal_false; + break; + } +basic_json_parser_66: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_30; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + } + + } + + return last_token_type; + } + + /*! + @brief append data from the stream to the line buffer + + This function is called by the scan() function when the end of the + buffer (`m_limit`) is reached and the `m_cursor` pointer cannot be + incremented without leaving the limits of the line buffer. Note re2c + decides when to call this function. + + If the lexer reads from contiguous storage, there is no trailing null + byte. Therefore, this function must make sure to add these padding + null bytes. + + If the lexer reads from an input stream, this function reads the next + line of the input. + + @pre + p p p p p p u u u u u x . . . . . . + ^ ^ ^ ^ + m_content m_start | m_limit + m_cursor + + @post + u u u u u x x x x x x x . . . . . . + ^ ^ ^ + | m_cursor m_limit + m_start + m_content + */ + void fill_line_buffer(size_t n = 0) + { + // if line buffer is used, m_content points to its data + assert(m_line_buffer.empty() + or m_content == reinterpret_cast<const lexer_char_t*>(m_line_buffer.data())); + + // if line buffer is used, m_limit is set past the end of its data + assert(m_line_buffer.empty() + or m_limit == m_content + m_line_buffer.size()); + + // pointer relationships + assert(m_content <= m_start); + assert(m_start <= m_cursor); + assert(m_cursor <= m_limit); + assert(m_marker == nullptr or m_marker <= m_limit); + + // number of processed characters (p) + const size_t num_processed_chars = static_cast<size_t>(m_start - m_content); + // offset for m_marker wrt. to m_start + const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start; + // number of unprocessed characters (u) + const auto offset_cursor = m_cursor - m_start; + + // no stream is used or end of file is reached + if (m_stream == nullptr or m_stream->eof()) + { + // m_start may or may not be pointing into m_line_buffer at + // this point. We trust the standand library to do the right + // thing. See http://stackoverflow.com/q/28142011/266378 + m_line_buffer.assign(m_start, m_limit); + + // append n characters to make sure that there is sufficient + // space between m_cursor and m_limit + m_line_buffer.append(1, '\x00'); + if (n > 0) + { + m_line_buffer.append(n - 1, '\x01'); + } + } + else + { + // delete processed characters from line buffer + m_line_buffer.erase(0, num_processed_chars); + // read next line from input stream + m_line_buffer_tmp.clear(); + std::getline(*m_stream, m_line_buffer_tmp, '\n'); + + // add line with newline symbol to the line buffer + m_line_buffer += m_line_buffer_tmp; + m_line_buffer.push_back('\n'); + } + + // set pointers + m_content = reinterpret_cast<const lexer_char_t*>(m_line_buffer.data()); + assert(m_content != nullptr); + m_start = m_content; + m_marker = m_start + offset_marker; + m_cursor = m_start + offset_cursor; + m_limit = m_start + m_line_buffer.size(); + } + + /// return string representation of last read token + string_t get_token_string() const + { + assert(m_start != nullptr); + return string_t(reinterpret_cast<typename string_t::const_pointer>(m_start), + static_cast<size_t>(m_cursor - m_start)); + } + + /*! + @brief return string value for string tokens + + The function iterates the characters between the opening and closing + quotes of the string value. The complete string is the range + [m_start,m_cursor). Consequently, we iterate from m_start+1 to + m_cursor-1. + + We differentiate two cases: + + 1. Escaped characters. In this case, a new character is constructed + according to the nature of the escape. Some escapes create new + characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied + as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape + `"\\uxxxx"` need special care. In this case, to_unicode takes care + of the construction of the values. + 2. Unescaped characters are copied as is. + + @pre `m_cursor - m_start >= 2`, meaning the length of the last token + is at least 2 bytes which is trivially true for any string (which + consists of at least two quotes). + + " c1 c2 c3 ... " + ^ ^ + m_start m_cursor + + @complexity Linear in the length of the string.\n + + Lemma: The loop body will always terminate.\n + + Proof (by contradiction): Assume the loop body does not terminate. As + the loop body does not contain another loop, one of the called + functions must never return. The called functions are `std::strtoul` + and to_unicode. Neither function can loop forever, so the loop body + will never loop forever which contradicts the assumption that the loop + body does not terminate, q.e.d.\n + + Lemma: The loop condition for the for loop is eventually false.\n + + Proof (by contradiction): Assume the loop does not terminate. Due to + the above lemma, this can only be due to a tautological loop + condition; that is, the loop condition i < m_cursor - 1 must always be + true. Let x be the change of i for any loop iteration. Then + m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This + can be rephrased to m_cursor - m_start - 2 > x. With the + precondition, we x <= 0, meaning that the loop condition holds + indefinitly if i is always decreased. However, observe that the value + of i is strictly increasing with each iteration, as it is incremented + by 1 in the iteration expression and never decremented inside the loop + body. Hence, the loop condition will eventually be false which + contradicts the assumption that the loop condition is a tautology, + q.e.d. + + @return string value of current token without opening and closing + quotes + @throw std::out_of_range if to_unicode fails + */ + string_t get_string() const + { + assert(m_cursor - m_start >= 2); + + string_t result; + result.reserve(static_cast<size_t>(m_cursor - m_start - 2)); + + // iterate the result between the quotes + for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) + { + // find next escape character + auto e = std::find(i, m_cursor - 1, '\\'); + if (e != i) + { + // see https://github.com/nlohmann/json/issues/365#issuecomment-262874705 + for (auto k = i; k < e; k++) + { + result.push_back(static_cast<typename string_t::value_type>(*k)); + } + i = e - 1; // -1 because of ++i + } + else + { + // processing escaped character + // read next character + ++i; + + switch (*i) + { + // the default escapes + case 't': + { + result += "\t"; + break; + } + case 'b': + { + result += "\b"; + break; + } + case 'f': + { + result += "\f"; + break; + } + case 'n': + { + result += "\n"; + break; + } + case 'r': + { + result += "\r"; + break; + } + case '\\': + { + result += "\\"; + break; + } + case '/': + { + result += "/"; + break; + } + case '"': + { + result += "\""; + break; + } + + // unicode + case 'u': + { + // get code xxxx from uxxxx + auto codepoint = std::strtoul(std::string(reinterpret_cast<typename string_t::const_pointer>(i + 1), + 4).c_str(), nullptr, 16); + + // check if codepoint is a high surrogate + if (codepoint >= 0xD800 and codepoint <= 0xDBFF) + { + // make sure there is a subsequent unicode + if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') + { + Throw<std::invalid_argument>("missing low surrogate"); + } + + // get code yyyy from uxxxx\uyyyy + auto codepoint2 = std::strtoul(std::string(reinterpret_cast<typename string_t::const_pointer> + (i + 7), 4).c_str(), nullptr, 16); + result += to_unicode(codepoint, codepoint2); + // skip the next 10 characters (xxxx\uyyyy) + i += 10; + } + else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF) + { + // we found a lone low surrogate + Throw<std::invalid_argument>("missing high surrogate"); + } + else + { + // add unicode character(s) + result += to_unicode(codepoint); + // skip the next four characters (xxxx) + i += 4; + } + break; + } + } + } + } + + return result; + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast<number_float_t*>(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + long double str_to_float_t(long double* /* type */, char** endptr) const + { + return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast<number_float_t*>(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + double str_to_float_t(double* /* type */, char** endptr) const + { + return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast<number_float_t*>(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + float str_to_float_t(float* /* type */, char** endptr) const + { + return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr); + } + + /*! + @brief return number value for number tokens + + This function translates the last token into the most appropriate + number type (either integer, unsigned integer or floating point), + which is passed back to the caller via the result parameter. + + This function parses the integer component up to the radix point or + exponent while collecting information about the 'floating point + representation', which it stores in the result parameter. If there is + no radix point or exponent, and the number can fit into a @ref + number_integer_t or @ref number_unsigned_t then it sets the result + parameter accordingly. + + If the number is a floating point number the number is then parsed + using @a std:strtod (or @a std:strtof or @a std::strtold). + + @param[out] result @ref basic_json object to receive the number, or + NAN if the conversion read past the current token. The latter case + needs to be treated by the caller function. + */ + void get_number(basic_json& result) const + { + assert(m_start != nullptr); + + const lexer::lexer_char_t* curptr = m_start; + + // accumulate the integer conversion result (unsigned for now) + number_unsigned_t value = 0; + + // maximum absolute value of the relevant integer type + number_unsigned_t max; + + // temporarily store the type to avoid unecessary bitfield access + value_t type; + + // look for sign + if (*curptr == '-') + { + type = value_t::number_integer; + max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1; + curptr++; + } + else + { + type = value_t::number_unsigned; + max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)()); + } + + // count the significant figures + for (; curptr < m_cursor; curptr++) + { + // quickly skip tests if a digit + if (*curptr < '0' || *curptr > '9') + { + if (*curptr == '.') + { + // don't count '.' but change to float + type = value_t::number_float; + continue; + } + // assume exponent (if not then will fail parse): change to + // float, stop counting and record exponent details + type = value_t::number_float; + break; + } + + // skip if definitely not an integer + if (type != value_t::number_float) + { + // multiply last value by ten and add the new digit + auto temp = value * 10 + *curptr - '0'; + + // test for overflow + if (temp < value || temp > max) + { + // overflow + type = value_t::number_float; + } + else + { + // no overflow - save it + value = temp; + } + } + } + + // save the value (if not a float) + if (type == value_t::number_unsigned) + { + result.m_value.number_unsigned = value; + } + else if (type == value_t::number_integer) + { + result.m_value.number_integer = -static_cast<number_integer_t>(value); + } + else + { + // parse with strtod + result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), NULL); + + // replace infinity and NAN by null + if (not std::isfinite(result.m_value.number_float)) + { + type = value_t::null; + result.m_value = basic_json::json_value(); + } + } + + // save the type + result.m_type = type; + } + + private: + /// optional input stream + std::istream* m_stream = nullptr; + /// line buffer buffer for m_stream + string_t m_line_buffer {}; + /// used for filling m_line_buffer + string_t m_line_buffer_tmp {}; + /// the buffer pointer + const lexer_char_t* m_content = nullptr; + /// pointer to the beginning of the current symbol + const lexer_char_t* m_start = nullptr; + /// pointer for backtracking information + const lexer_char_t* m_marker = nullptr; + /// pointer to the current symbol + const lexer_char_t* m_cursor = nullptr; + /// pointer to the end of the buffer + const lexer_char_t* m_limit = nullptr; + /// the last token type + token_type last_token_type = token_type::end_of_input; + }; + + /*! + @brief syntax analysis + + This class implements a recursive decent parser. + */ + class parser + { + public: + /// a parser reading from a string literal + parser(const char* buff, const parser_callback_t cb = nullptr) + : callback(cb), + m_lexer(reinterpret_cast<const typename lexer::lexer_char_t*>(buff), std::strlen(buff)) + {} + + /// a parser reading from an input stream + parser(std::istream& is, const parser_callback_t cb = nullptr) + : callback(cb), m_lexer(is) + {} + + /// a parser reading from an iterator range with contiguous storage + template<class IteratorType, typename std::enable_if< + std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> + parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) + : callback(cb), + m_lexer(reinterpret_cast<const typename lexer::lexer_char_t*>(&(*first)), + static_cast<size_t>(std::distance(first, last))) + {} + + /// public parser interface + basic_json parse() + { + // read first token + get_token(); + + basic_json result = parse_internal(true); + result.assert_invariant(); + + expect(lexer::token_type::end_of_input); + + // return parser result and replace it with null in case the + // top-level value was discarded by the callback function + return result.is_discarded() ? basic_json() : std::move(result); + } + + private: + /// the actual parser + basic_json parse_internal(bool keep) + { + auto result = basic_json(value_t::discarded); + + switch (last_token) + { + case lexer::token_type::begin_object: + { + if (keep and (not callback + or ((keep = callback(depth++, parse_event_t::object_start, result)) != 0))) + { + // explicitly set result to object to cope with {} + result.m_type = value_t::object; + result.m_value = value_t::object; + } + + // read next token + get_token(); + + // closing } -> we are done + if (last_token == lexer::token_type::end_object) + { + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse key-value pairs + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // store key + expect(lexer::token_type::value_string); + const auto key = m_lexer.get_string(); + + bool keep_tag = false; + if (keep) + { + if (callback) + { + basic_json k(key); + keep_tag = callback(depth, parse_event_t::key, k); + } + else + { + keep_tag = true; + } + } + + // parse separator (:) + get_token(); + expect(lexer::token_type::name_separator); + + // parse and add value + get_token(); + auto value = parse_internal(keep); + if (keep and keep_tag and not value.is_discarded()) + { + result[key] = std::move(value); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing } + expect(lexer::token_type::end_object); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::begin_array: + { + if (keep and (not callback + or ((keep = callback(depth++, parse_event_t::array_start, result)) != 0))) + { + // explicitly set result to object to cope with [] + result.m_type = value_t::array; + result.m_value = value_t::array; + } + + // read next token + get_token(); + + // closing ] -> we are done + if (last_token == lexer::token_type::end_array) + { + get_token(); + if (callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse values + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // parse value + auto value = parse_internal(keep); + if (keep and not value.is_discarded()) + { + result.push_back(std::move(value)); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing ] + expect(lexer::token_type::end_array); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::literal_null: + { + get_token(); + result.m_type = value_t::null; + break; + } + + case lexer::token_type::value_string: + { + const auto s = m_lexer.get_string(); + get_token(); + result = basic_json(s); + break; + } + + case lexer::token_type::literal_true: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = true; + break; + } + + case lexer::token_type::literal_false: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = false; + break; + } + + case lexer::token_type::value_number: + { + m_lexer.get_number(result); + get_token(); + break; + } + + default: + { + // the last token was unexpected + unexpect(last_token); + } + } + + if (keep and callback and not callback(depth, parse_event_t::value, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + /// get next token from lexer + typename lexer::token_type get_token() + { + last_token = m_lexer.scan(); + return last_token; + } + + void expect(typename lexer::token_type t) const + { + if (t != last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + error_msg += "; expected " + lexer::token_type_name(t); + Throw<std::invalid_argument>(error_msg); + } + } + + void unexpect(typename lexer::token_type t) const + { + if (t == last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + Throw<std::invalid_argument>(error_msg); + } + } + + private: + /// current level of recursion + int depth = 0; + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + typename lexer::token_type last_token = lexer::token_type::uninitialized; + /// the lexer + lexer m_lexer; + }; + + public: + /*! + @brief JSON Pointer + + A JSON pointer defines a string syntax for identifying a specific value + within a JSON document. It can be used with functions `at` and + `operator[]`. Furthermore, JSON pointers are the base for JSON patches. + + @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + class json_pointer + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /*! + @brief create JSON pointer + + Create a JSON pointer according to the syntax described in + [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). + + @param[in] s string representing the JSON pointer; if omitted, the + empty string is assumed which references the whole JSON + value + + @throw std::domain_error if reference token is nonempty and does not + begin with a slash (`/`); example: `"JSON pointer must be empty or + begin with /"` + @throw std::domain_error if a tilde (`~`) is not followed by `0` + (representing `~`) or `1` (representing `/`); example: `"escape error: + ~ must be followed with 0 or 1"` + + @liveexample{The example shows the construction several valid JSON + pointers as well as the exceptional behavior.,json_pointer} + + @since version 2.0.0 + */ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /*! + @brief return a string representation of the JSON pointer + + @invariant For each JSON pointer `ptr`, it holds: + @code {.cpp} + ptr == json_pointer(ptr.to_string()); + @endcode + + @return a string representation of the JSON pointer + + @liveexample{The example shows the result of `to_string`., + json_pointer__to_string} + + @since version 2.0.0 + */ + std::string to_string() const noexcept + { + return std::accumulate(reference_tokens.begin(), + reference_tokens.end(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + escape(b); + }); + } + + /// @copydoc to_string() + operator std::string() const + { + return to_string(); + } + + private: + /// remove and return last reference pointer + std::string pop_back() + { + if (is_root()) + { + Throw<std::domain_error>("JSON pointer has no parent"); + } + + auto last = reference_tokens.back(); + reference_tokens.pop_back(); + return last; + } + + /// return whether pointer points to the root document + bool is_root() const + { + return reference_tokens.empty(); + } + + json_pointer top() const + { + if (is_root()) + { + Throw<std::domain_error>("JSON pointer has no parent"); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + */ + reference get_and_create(reference j) const + { + pointer result = &j; + + // in case no reference tokens exist, return a reference to the + // JSON value j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case value_t::array: + { + // create an entry in the array + result = &result->operator[](static_cast<size_type>(std::stoi(reference_token))); + break; + } + + /* + The following code is only reached if there exists a + reference token _and_ the current value is primitive. In + this case, we have an error situation, because primitive + values may only occur as single value; that is, with an + empty list of reference tokens. + */ + default: + { + Throw<std::domain_error>("invalid value to unflatten"); + } + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @note This version does not throw if a value is not present, but tries + to create nested values instead. For instance, calling this function + with pointer `"/this/that"` on a null value is equivalent to calling + `operator[]("this").operator[]("that")` on that value, effectively + changing the null value to an object. + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + */ + reference get_unchecked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->m_type == value_t::null) + { + // check if reference token is a number + const bool nums = std::all_of(reference_token.begin(), + reference_token.end(), + [](const char x) + { + return std::isdigit(x); + }); + + // change value to array for numbers or "-" or to object + // otherwise + if (nums or reference_token == "-") + { + *ptr = value_t::array; + } + else + { + *ptr = value_t::object; + } + } + + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + Throw<std::domain_error>("array index must not begin with '0'"); + } + + if (reference_token == "-") + { + // explicityly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token))); + } + break; + } + + default: + { + Throw<std::out_of_range>("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + reference get_checked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + Throw<std::out_of_range>("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + Throw<std::domain_error>("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token))); + break; + } + + default: + { + Throw<std::out_of_range>("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + */ + const_reference get_unchecked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" cannot be used for const access + Throw<std::out_of_range>("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + Throw<std::domain_error>("array index must not begin with '0'"); + } + + // use unchecked array access + ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token))); + break; + } + + default: + { + Throw<std::out_of_range>("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + const_reference get_checked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + Throw<std::out_of_range>("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + Throw<std::domain_error>("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token))); + break; + } + + default: + { + Throw<std::out_of_range>("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /// split the string input to reference tokens + static std::vector<std::string> split(const std::string& reference_string) + { + std::vector<std::string> result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (reference_string[0] != '/') + { + Throw<std::domain_error>("JSON pointer must be empty or begin with '/'"); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + size_t slash = reference_string.find_first_of("/", 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == string::npos+1 = 0 + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = slash + 1, + // find next slash + slash = reference_string.find_first_of("/", start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (size_t pos = reference_token.find_first_of("~"); + pos != std::string::npos; + pos = reference_token.find_first_of("~", pos + 1)) + { + assert(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (pos == reference_token.size() - 1 or + (reference_token[pos + 1] != '0' and + reference_token[pos + 1] != '1')) + { + Throw<std::domain_error>("escape error: '~' must be followed with '0' or '1'"); + } + } + + // finally, store the reference token + unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @brief replace all occurrences of a substring by another string + + @param[in,out] s the string to manipulate + @param[in] f the substring to replace with @a t + @param[in] t the string to replace @a f + + @return The string @a s where all occurrences of @a f are replaced + with @a t. + + @pre The search string @a f must not be empty. + + @since version 2.0.0 + */ + static void replace_substring(std::string& s, + const std::string& f, + const std::string& t) + { + assert(not f.empty()); + + for ( + size_t pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t + pos = s.find(f, pos + t.size()) // find next occurrence of f + ); + } + + /// escape tilde and slash + static std::string escape(std::string s) + { + // escape "~"" to "~0" and "/" to "~1" + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; + } + + /// unescape tilde and slash + static void unescape(std::string& s) + { + // first transform any occurrence of the sequence '~1' to '/' + replace_substring(s, "~1", "/"); + // then transform any occurrence of the sequence '~0' to '~' + replace_substring(s, "~0", "~"); + } + + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const basic_json& value, + basic_json& result) + { + switch (value.m_type) + { + case value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), + element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + */ + static basic_json unflatten(const basic_json& value) + { + if (not value.is_object()) + { + Throw<std::domain_error>("only objects can be unflattened"); + } + + basic_json result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (not element.second.is_primitive()) + { + Throw<std::domain_error>("values in object must be primitive"); + } + + // assign value to reference pointed to by JSON pointer; Note + // that if the JSON pointer is "" (i.e., points to the whole + // value), function get_and_create returns a reference to + // result itself. An assignment will then create a primitive + // value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + private: + /// the reference tokens + std::vector<std::string> reference_tokens {}; + }; + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. Similar to @ref operator[](const typename + object_t::key_type&), `null` values are created in arrays and objects if + necessary. + + In particular: + - If the JSON pointer points to an object key that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. + - If the JSON pointer points to an array index that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. All indices between the current maximum and the given + index are also filled with `null`. + - The special value `-` is treated as a synonym for the index past the + end. + + @param[in] ptr a JSON pointer + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer} + + @since version 2.0.0 + */ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. The function does not change the JSON + value; no `null` values are created. In particular, the the special value + `-` yields an exception. + + @param[in] ptr JSON pointer to the desired element + + @return const reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} + + @since version 2.0.0 + */ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a reference to the element at with specified JSON pointer @a ptr, + with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer} + + @since version 2.0.0 + */ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a const reference to the element at with specified JSON pointer @a + ptr, with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} + + @since version 2.0.0 + */ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /*! + @brief return flattened JSON value + + The function creates a JSON object whose keys are JSON pointers (see [RFC + 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all + primitive. The original JSON value can be restored using the @ref + unflatten() function. + + @return an object that maps JSON pointers to primitve values + + @note Empty objects and arrays are flattened to `null` and will not be + reconstructed correctly by the @ref unflatten() function. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a JSON object is flattened to an + object whose keys consist of JSON pointers.,flatten} + + @sa @ref unflatten() for the reverse function + + @since version 2.0.0 + */ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /*! + @brief unflatten a previously flattened JSON value + + The function restores the arbitrary nesting of a JSON value that has been + flattened before using the @ref flatten() function. The JSON value must + meet certain constraints: + 1. The value must be an object. + 2. The keys must be JSON pointers (see + [RFC 6901](https://tools.ietf.org/html/rfc6901)) + 3. The mapped values must be primitive JSON types. + + @return the original JSON from a flattened version + + @note Empty objects and arrays are flattened by @ref flatten() to `null` + values and can not unflattened to their original type. Apart from + this example, for a JSON value `j`, the following is always true: + `j == j.flatten().unflatten()`. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a flattened JSON object is + unflattened into the original nested JSON object.,unflatten} + + @sa @ref flatten() for the reverse function + + @since version 2.0.0 + */ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /*! + @brief applies a JSON patch + + [JSON Patch](http://jsonpatch.com) defines a JSON document structure for + expressing a sequence of operations to apply to a JSON) document. With + this funcion, a JSON Patch is applied to the current JSON value by + executing all operations from the patch. + + @param[in] json_patch JSON patch document + @return patched document + + @note The application of a patch is atomic: Either all operations succeed + and the patched document is returned or an exception is thrown. In + any case, the original value is not changed: the patch is applied + to a copy of the value. + + @throw std::out_of_range if a JSON pointer inside the patch could not + be resolved successfully in the current JSON value; example: `"key baz + not found"` + @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + attributes are missing); example: `"operation add must have member path"` + + @complexity Linear in the size of the JSON value and the length of the + JSON patch. As usually only a fraction of the JSON value is affected by + the patch, the complexity can usually be neglected. + + @liveexample{The following code shows how a JSON patch is applied to a + value.,patch} + + @sa @ref diff -- create a JSON patch by comparing two JSON values + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.is_root()) + { + result = val; + } + else + { + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = std::stoi(last_path); + if (static_cast<size_type>(idx) > parent.size()) + { + // avoid undefined behavior + Throw<std::out_of_range>("array index " + std::to_string(idx) + " is out of range"); + } + else + { + // default case: insert add offset + parent.insert(parent.begin() + static_cast<difference_type>(idx), val); + } + } + break; + } + + default: + { + // if there exists a parent it cannot be primitive + assert(false); // LCOV_EXCL_LINE + } + } + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [&result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (it != parent.end()) + { + parent.erase(it); + } + else + { + Throw<std::out_of_range>("key '" + last_path + "' not found"); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(static_cast<size_type>(std::stoi(last_path))); + } + }; + + // type check + if (not json_patch.is_array()) + { + // a JSON patch must be an array of objects + Throw<std::invalid_argument>("JSON patch must be an array of objects"); + } + + // iterate and apply th eoperations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json& + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (it == val.m_value.object->end()) + { + Throw<std::invalid_argument>(error_msg + " must have member '" + member + "'"); + } + + // check if result is of type string + if (string_type and not it->second.is_string()) + { + Throw<std::invalid_argument>(error_msg + " must have string member '" + member + "'"); + } + + // no error: return value + return it->second; + }; + + // type check + if (not val.is_object()) + { + Throw<std::invalid_argument>("JSON patch must be an array of objects"); + } + + // collect mandatory members + const std::string op = get_value("op", "op", true); + const std::string path = get_value(op, "path", true); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const std::string from_path = get_value("move", "from", true); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const std::string from_path = get_value("copy", "from", true);; + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + result[ptr] = result.at(from_ptr); + break; + } + + case patch_operations::test: + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + bool success = (result.at(ptr) == get_value("test", "value", false)); + + // throw an exception if test fails + if (not success) + { + Throw<std::domain_error>("unsuccessful: " + val.dump()); + } + + break; + } + + case patch_operations::invalid: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + Throw<std::invalid_argument>("operation value '" + op + "' is invalid"); + } + } + } + + return result; + } + + /*! + @brief creates a diff as a JSON patch + + Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can + be changed into the value @a target by calling @ref patch function. + + @invariant For two JSON values @a source and @a target, the following code + yields always `true`: + @code {.cpp} + source.patch(diff(source, target)) == target; + @endcode + + @note Currently, only `remove`, `add`, and `replace` operations are + generated. + + @param[in] source JSON value to copare from + @param[in] target JSON value to copare against + @param[in] path helper value to create JSON pointers + + @return a JSON patch to convert the @a source to @a target + + @complexity Linear in the lengths of @a source and @a target. + + @liveexample{The following code shows how a JSON patch is created as a + diff for two JSON values.,diff} + + @sa @ref patch -- apply a JSON patch + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + + @since version 2.0.0 + */ + static basic_json diff(const basic_json& source, + const basic_json& target, + const std::string& path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + } + else + { + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + size_t i = 0; + while (i < source.size() and i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast<difference_type>(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.begin(); it != source.end(); ++it) + { + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, + {"path", path + "/" + key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.begin(); it != target.end(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + key}, + {"value", it.value()} + }); + } + } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + break; + } + } + } + + return result; + } + + /// @} +}; + + +///////////// +// presets // +///////////// + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} + + +/////////////////////// +// nonmember support // +/////////////////////// + +// specialization of std::swap, and std::hash +namespace std +{ +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template<> +inline void swap(nlohmann::json& j1, + nlohmann::json& j2) noexcept( + is_nothrow_move_constructible<nlohmann::json>::value and + is_nothrow_move_assignable<nlohmann::json>::value + ) +{ + j1.swap(j2); +} + +/// hash value for JSON objects +template<> +struct hash<nlohmann::json> +{ + /*! + @brief return a hash value for a JSON object + + @since version 1.0.0 + */ + std::size_t operator()(const nlohmann::json& j) const + { + // a naive hashing via the string representation + const auto& h = hash<nlohmann::json::string_t>(); + return h(j.dump()); + } +}; +} + +/*! +@brief user-defined string literal for JSON values + +This operator implements a user-defined string literal for JSON objects. It +can be used by adding `"_json"` to a string literal and returns a JSON object +if no parse error occurred. + +@param[in] s a string representation of a JSON object +@param[in] n the length of string @a s +@return a JSON object + +@since version 1.0.0 +*/ +inline nlohmann::json operator "" _json(const char* s, std::size_t n) +{ + return nlohmann::json::parse(s, s + n); +} + +/*! +@brief user-defined string literal for JSON pointer + +This operator implements a user-defined string literal for JSON Pointers. It +can be used by adding `"_json_pointer"` to a string literal and returns a JSON pointer +object if no parse error occurred. + +@param[in] s a string representation of a JSON Pointer +@param[in] n the length of string @a s +@return a JSON pointer object + +@since version 2.0.0 +*/ +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +{ + return nlohmann::json::json_pointer(std::string(s, n)); +} + +// restore GCC/clang diagnostic settings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic pop +#endif + +#endif diff --git a/Source/WebKit2/NetworkProcess/soup/NetworkDataTaskSoup.cpp b/Source/WebKit2/NetworkProcess/soup/NetworkDataTaskSoup.cpp new file mode 100644 index 000000000..8e77fc407 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/soup/NetworkDataTaskSoup.cpp @@ -0,0 +1,1102 @@ +/* + * Copyright (C) 2016 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkDataTaskSoup.h" + +#include "AuthenticationManager.h" +#include "DataReference.h" +#include "Download.h" +#include "DownloadSoupErrors.h" +#include "NetworkLoad.h" +#include "NetworkProcess.h" +#include "NetworkSessionSoup.h" +#include "WebErrors.h" +#include <WebCore/AuthenticationChallenge.h> +#include <WebCore/HTTPParsers.h> +#include <WebCore/MIMETypeRegistry.h> +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/SharedBuffer.h> +#include <WebCore/SoupNetworkSession.h> +#include <wtf/MainThread.h> + +using namespace WebCore; + +namespace WebKit { + +static const size_t gDefaultReadBufferSize = 8192; + +NetworkDataTaskSoup::NetworkDataTaskSoup(NetworkSession& session, NetworkDataTaskClient& client, const ResourceRequest& requestWithCredentials, StoredCredentials storedCredentials, ContentSniffingPolicy shouldContentSniff, bool shouldClearReferrerOnHTTPSToHTTPRedirect) + : NetworkDataTask(session, client, requestWithCredentials, storedCredentials, shouldClearReferrerOnHTTPSToHTTPRedirect) + , m_shouldContentSniff(shouldContentSniff) + , m_timeoutSource(RunLoop::main(), this, &NetworkDataTaskSoup::timeoutFired) +{ + m_session->registerNetworkDataTask(*this); + if (m_scheduledFailureType != NoFailure) + return; + + auto request = requestWithCredentials; + if (request.url().protocolIsInHTTPFamily()) { +#if ENABLE(WEB_TIMING) + m_startTime = monotonicallyIncreasingTimeMS(); +#endif + auto url = request.url(); + if (m_storedCredentials == AllowStoredCredentials) { + m_user = url.user(); + m_password = url.pass(); + request.removeCredentials(); + + if (m_user.isEmpty() && m_password.isEmpty()) + m_initialCredential = m_session->networkStorageSession().credentialStorage().get(m_partition, request.url()); + else + m_session->networkStorageSession().credentialStorage().set(m_partition, Credential(m_user, m_password, CredentialPersistenceNone), request.url()); + } + applyAuthenticationToRequest(request); + } + createRequest(WTFMove(request)); +} + +NetworkDataTaskSoup::~NetworkDataTaskSoup() +{ + clearRequest(); + m_session->unregisterNetworkDataTask(*this); +} + +String NetworkDataTaskSoup::suggestedFilename() const +{ + if (!m_suggestedFilename.isEmpty()) + return m_suggestedFilename; + + String suggestedFilename = m_response.suggestedFilename(); + if (!suggestedFilename.isEmpty()) + return suggestedFilename; + + return decodeURLEscapeSequences(m_response.url().lastPathComponent()); +} + +void NetworkDataTaskSoup::setPendingDownloadLocation(const String& filename, const SandboxExtension::Handle& sandboxExtensionHandle, bool allowOverwrite) +{ + NetworkDataTask::setPendingDownloadLocation(filename, sandboxExtensionHandle, allowOverwrite); + m_allowOverwriteDownload = allowOverwrite; +} + +void NetworkDataTaskSoup::createRequest(ResourceRequest&& request) +{ + m_currentRequest = WTFMove(request); + + GUniquePtr<SoupURI> soupURI = m_currentRequest.createSoupURI(); + if (!soupURI) { + scheduleFailure(InvalidURLFailure); + return; + } + + GRefPtr<SoupRequest> soupRequest = adoptGRef(soup_session_request_uri(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), soupURI.get(), nullptr)); + if (!soupRequest) { + scheduleFailure(InvalidURLFailure); + return; + } + + m_currentRequest.updateSoupRequest(soupRequest.get()); + + if (!m_currentRequest.url().protocolIsInHTTPFamily()) { + m_soupRequest = WTFMove(soupRequest); + return; + } + + // HTTP request. + GRefPtr<SoupMessage> soupMessage = adoptGRef(soup_request_http_get_message(SOUP_REQUEST_HTTP(soupRequest.get()))); + if (!soupMessage) { + scheduleFailure(InvalidURLFailure); + return; + } + + unsigned messageFlags = SOUP_MESSAGE_NO_REDIRECT; + + m_currentRequest.updateSoupMessage(soupMessage.get()); + if (m_shouldContentSniff == DoNotSniffContent) + soup_message_disable_feature(soupMessage.get(), SOUP_TYPE_CONTENT_SNIFFER); + if (m_user.isEmpty() && m_password.isEmpty() && m_storedCredentials == DoNotAllowStoredCredentials) { +#if SOUP_CHECK_VERSION(2, 57, 1) + messageFlags |= SOUP_MESSAGE_DO_NOT_USE_AUTH_CACHE; +#else + // In case credential is not available and credential storage should not to be used, + // disable authentication manager so that credentials stored in libsoup are not used. + soup_message_disable_feature(soupMessage.get(), SOUP_TYPE_AUTH_MANAGER); +#endif + } + + // Make sure we have an Accept header for subresources; some sites want this to serve some of their subresources. + if (!soup_message_headers_get_one(soupMessage->request_headers, "Accept")) + soup_message_headers_append(soupMessage->request_headers, "Accept", "*/*"); + + // In the case of XHR .send() and .send("") explicitly tell libsoup to send a zero content-lenght header + // for consistency with other UA implementations like Firefox. It's done in the backend here instead of + // in XHR code since in XHR CORS checking prevents us from this kind of late header manipulation. + if ((soupMessage->method == SOUP_METHOD_POST || soupMessage->method == SOUP_METHOD_PUT) && !soupMessage->request_body->length) + soup_message_headers_set_content_length(soupMessage->request_headers, 0); + + soup_message_set_flags(soupMessage.get(), static_cast<SoupMessageFlags>(soup_message_get_flags(soupMessage.get()) | messageFlags)); + +#if SOUP_CHECK_VERSION(2, 43, 1) + soup_message_set_priority(soupMessage.get(), toSoupMessagePriority(m_currentRequest.priority())); +#endif + + m_soupRequest = WTFMove(soupRequest); + m_soupMessage = WTFMove(soupMessage); + + g_signal_connect(m_soupMessage.get(), "notify::tls-errors", G_CALLBACK(tlsErrorsChangedCallback), this); + g_signal_connect(m_soupMessage.get(), "got-headers", G_CALLBACK(gotHeadersCallback), this); + g_signal_connect(m_soupMessage.get(), "wrote-body-data", G_CALLBACK(wroteBodyDataCallback), this); + g_signal_connect(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), "authenticate", G_CALLBACK(authenticateCallback), this); +#if ENABLE(WEB_TIMING) + g_signal_connect(m_soupMessage.get(), "network-event", G_CALLBACK(networkEventCallback), this); + g_signal_connect(m_soupMessage.get(), "restarted", G_CALLBACK(restartedCallback), this); +#if SOUP_CHECK_VERSION(2, 49, 91) + g_signal_connect(m_soupMessage.get(), "starting", G_CALLBACK(startingCallback), this); +#else + g_signal_connect(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), "request-started", G_CALLBACK(requestStartedCallback), this); +#endif +#endif +} + +void NetworkDataTaskSoup::clearRequest() +{ + if (m_state == State::Completed) + return; + + m_state = State::Completed; + + stopTimeout(); + m_pendingResult = nullptr; + m_soupRequest = nullptr; + m_inputStream = nullptr; + m_multipartInputStream = nullptr; + m_downloadOutputStream = nullptr; + g_cancellable_cancel(m_cancellable.get()); + m_cancellable = nullptr; + if (m_soupMessage) { + g_signal_handlers_disconnect_matched(m_soupMessage.get(), G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this); + soup_session_cancel_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get(), SOUP_STATUS_CANCELLED); + m_soupMessage = nullptr; + } + g_signal_handlers_disconnect_matched(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this); +} + +void NetworkDataTaskSoup::resume() +{ + ASSERT(m_state != State::Running); + if (m_state == State::Canceling || m_state == State::Completed) + return; + + m_state = State::Running; + + if (m_scheduledFailureType != NoFailure) { + ASSERT(m_failureTimer.isActive()); + return; + } + + startTimeout(); + + RefPtr<NetworkDataTaskSoup> protectedThis(this); + if (m_soupRequest && !m_cancellable) { + m_cancellable = adoptGRef(g_cancellable_new()); + soup_request_send_async(m_soupRequest.get(), m_cancellable.get(), reinterpret_cast<GAsyncReadyCallback>(sendRequestCallback), protectedThis.leakRef()); + return; + } + + if (m_pendingResult) { + GRefPtr<GAsyncResult> pendingResult = WTFMove(m_pendingResult); + if (m_inputStream) + readCallback(m_inputStream.get(), pendingResult.get(), protectedThis.leakRef()); + else if (m_multipartInputStream) + requestNextPartCallback(m_multipartInputStream.get(), pendingResult.get(), protectedThis.leakRef()); + else if (m_soupRequest) + sendRequestCallback(m_soupRequest.get(), pendingResult.get(), protectedThis.leakRef()); + else + ASSERT_NOT_REACHED(); + } +} + +void NetworkDataTaskSoup::suspend() +{ + ASSERT(m_state != State::Suspended); + if (m_state == State::Canceling || m_state == State::Completed) + return; + m_state = State::Suspended; + + stopTimeout(); +} + +void NetworkDataTaskSoup::cancel() +{ + if (m_state == State::Canceling || m_state == State::Completed) + return; + + m_state = State::Canceling; + + if (m_soupMessage) + soup_session_cancel_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get(), SOUP_STATUS_CANCELLED); + + g_cancellable_cancel(m_cancellable.get()); + + if (isDownload()) + cleanDownloadFiles(); +} + +void NetworkDataTaskSoup::invalidateAndCancel() +{ + cancel(); + clearRequest(); +} + +NetworkDataTask::State NetworkDataTaskSoup::state() const +{ + return m_state; +} + +void NetworkDataTaskSoup::timeoutFired() +{ + if (m_state == State::Canceling || m_state == State::Completed || !m_client) { + clearRequest(); + return; + } + + RefPtr<NetworkDataTaskSoup> protectedThis(this); + invalidateAndCancel(); + m_client->didCompleteWithError(ResourceError::timeoutError(m_firstRequest.url())); +} + +void NetworkDataTaskSoup::startTimeout() +{ + if (m_firstRequest.timeoutInterval() > 0) + m_timeoutSource.startOneShot(m_firstRequest.timeoutInterval()); +} + +void NetworkDataTaskSoup::stopTimeout() +{ + m_timeoutSource.stop(); +} + +void NetworkDataTaskSoup::sendRequestCallback(SoupRequest* soupRequest, GAsyncResult* result, NetworkDataTaskSoup* task) +{ + RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task); + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) { + task->clearRequest(); + return; + } + ASSERT(soupRequest == task->m_soupRequest.get()); + + if (task->state() == State::Suspended) { + ASSERT(!task->m_pendingResult); + task->m_pendingResult = result; + return; + } + + GUniqueOutPtr<GError> error; + GRefPtr<GInputStream> inputStream = adoptGRef(soup_request_send_finish(soupRequest, result, &error.outPtr())); + if (error) + task->didFail(ResourceError::httpError(task->m_soupMessage.get(), error.get(), soupRequest)); + else + task->didSendRequest(WTFMove(inputStream)); +} + +void NetworkDataTaskSoup::didSendRequest(GRefPtr<GInputStream>&& inputStream) +{ + if (m_soupMessage) { + if (m_shouldContentSniff == SniffContent && m_soupMessage->status_code != SOUP_STATUS_NOT_MODIFIED) + m_response.setSniffedContentType(soup_request_get_content_type(m_soupRequest.get())); + m_response.updateFromSoupMessage(m_soupMessage.get()); + if (m_response.mimeType().isEmpty() && m_soupMessage->status_code != SOUP_STATUS_NOT_MODIFIED) + m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(m_response.url().path())); + + if (shouldStartHTTPRedirection()) { + m_inputStream = WTFMove(inputStream); + skipInputStreamForRedirection(); + return; + } + + if (m_response.isMultipart()) + m_multipartInputStream = adoptGRef(soup_multipart_input_stream_new(m_soupMessage.get(), inputStream.get())); + else + m_inputStream = WTFMove(inputStream); + +#if ENABLE(WEB_TIMING) + m_response.networkLoadTiming().responseStart = monotonicallyIncreasingTimeMS() - m_startTime; +#endif + } else { + m_response.setURL(m_firstRequest.url()); + const gchar* contentType = soup_request_get_content_type(m_soupRequest.get()); + m_response.setMimeType(extractMIMETypeFromMediaType(contentType)); + m_response.setTextEncodingName(extractCharsetFromMediaType(contentType)); + m_response.setExpectedContentLength(soup_request_get_content_length(m_soupRequest.get())); + if (m_response.mimeType().isEmpty()) + m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(m_response.url().path())); + + m_inputStream = WTFMove(inputStream); + } + + dispatchDidReceiveResponse(); +} + +void NetworkDataTaskSoup::dispatchDidReceiveResponse() +{ + ASSERT(!m_response.isNull()); + + didReceiveResponse(ResourceResponse(m_response), [this, protectedThis = makeRef(*this)](PolicyAction policyAction) { + if (m_state == State::Canceling || m_state == State::Completed) { + clearRequest(); + return; + } + + switch (policyAction) { + case PolicyAction::PolicyUse: + if (m_inputStream) + read(); + else if (m_multipartInputStream) + requestNextPart(); + else + ASSERT_NOT_REACHED(); + + break; + case PolicyAction::PolicyIgnore: + clearRequest(); + break; + case PolicyAction::PolicyDownload: + download(); + break; + } + }); +} + +void NetworkDataTaskSoup::tlsErrorsChangedCallback(SoupMessage* soupMessage, GParamSpec*, NetworkDataTaskSoup* task) +{ + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) { + task->clearRequest(); + return; + } + + ASSERT(soupMessage == task->m_soupMessage.get()); + task->tlsErrorsChanged(); +} + +void NetworkDataTaskSoup::tlsErrorsChanged() +{ + ASSERT(m_soupRequest); + SoupNetworkSession::checkTLSErrors(m_soupRequest.get(), m_soupMessage.get(), [this] (const ResourceError& error) { + if (error.isNull()) + return; + + RefPtr<NetworkDataTaskSoup> protectedThis(this); + invalidateAndCancel(); + m_client->didCompleteWithError(error); + }); +} + +void NetworkDataTaskSoup::applyAuthenticationToRequest(ResourceRequest& request) +{ + if (m_user.isEmpty() && m_password.isEmpty()) + return; + + auto url = request.url(); + url.setUser(m_user); + url.setPass(m_password); + request.setURL(url); + + m_user = String(); + m_password = String(); +} + +void NetworkDataTaskSoup::authenticateCallback(SoupSession* session, SoupMessage* soupMessage, SoupAuth* soupAuth, gboolean retrying, NetworkDataTaskSoup* task) +{ + ASSERT(session == static_cast<NetworkSessionSoup&>(task->m_session.get()).soupSession()); + if (soupMessage != task->m_soupMessage.get()) + return; + + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) { + task->clearRequest(); + return; + } + + task->authenticate(AuthenticationChallenge(soupMessage, soupAuth, retrying)); +} + +static inline bool isAuthenticationFailureStatusCode(int httpStatusCode) +{ + return httpStatusCode == SOUP_STATUS_PROXY_AUTHENTICATION_REQUIRED || httpStatusCode == SOUP_STATUS_UNAUTHORIZED; +} + +void NetworkDataTaskSoup::authenticate(AuthenticationChallenge&& challenge) +{ + ASSERT(m_soupMessage); + if (m_storedCredentials == AllowStoredCredentials) { + if (!m_initialCredential.isEmpty() || challenge.previousFailureCount()) { + // The stored credential wasn't accepted, stop using it. There is a race condition + // here, since a different credential might have already been stored by another + // NetworkDataTask, but the observable effect should be very minor, if any. + m_session->networkStorageSession().credentialStorage().remove(m_partition, challenge.protectionSpace()); + } + + if (!challenge.previousFailureCount()) { + auto credential = m_session->networkStorageSession().credentialStorage().get(m_partition, challenge.protectionSpace()); + if (!credential.isEmpty() && credential != m_initialCredential) { + ASSERT(credential.persistence() == CredentialPersistenceNone); + + if (isAuthenticationFailureStatusCode(challenge.failureResponse().httpStatusCode())) { + // Store the credential back, possibly adding it as a default for this directory. + m_session->networkStorageSession().credentialStorage().set(m_partition, credential, challenge.protectionSpace(), challenge.failureResponse().url()); + } + soup_auth_authenticate(challenge.soupAuth(), credential.user().utf8().data(), credential.password().utf8().data()); + return; + } + } + } + + soup_session_pause_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get()); + + // We could also do this before we even start the request, but that would be at the expense + // of all request latency, versus a one-time latency for the small subset of requests that + // use HTTP authentication. In the end, this doesn't matter much, because persistent credentials + // will become session credentials after the first use. + if (m_storedCredentials == AllowStoredCredentials) { + auto protectionSpace = challenge.protectionSpace(); + m_session->networkStorageSession().getCredentialFromPersistentStorage(protectionSpace, + [this, protectedThis = makeRef(*this), authChallenge = WTFMove(challenge)] (Credential&& credential) mutable { + if (m_state == State::Canceling || m_state == State::Completed || !m_client) { + clearRequest(); + return; + } + + authChallenge.setProposedCredential(WTFMove(credential)); + continueAuthenticate(WTFMove(authChallenge)); + }); + } else + continueAuthenticate(WTFMove(challenge)); +} + +void NetworkDataTaskSoup::continueAuthenticate(AuthenticationChallenge&& challenge) +{ + m_client->didReceiveChallenge(challenge, [this, protectedThis = makeRef(*this), challenge](AuthenticationChallengeDisposition disposition, const Credential& credential) { + if (m_state == State::Canceling || m_state == State::Completed) { + clearRequest(); + return; + } + + if (disposition == AuthenticationChallengeDisposition::Cancel) { + cancel(); + didFail(cancelledError(m_soupRequest.get())); + return; + } + + if (disposition == AuthenticationChallengeDisposition::UseCredential && !credential.isEmpty()) { + if (m_storedCredentials == AllowStoredCredentials) { + // Eventually we will manage per-session credentials only internally or use some newly-exposed API from libsoup, + // because once we authenticate via libsoup, there is no way to ignore it for a particular request. Right now, + // we place the credentials in the store even though libsoup will never fire the authenticate signal again for + // this protection space. + if (credential.persistence() == CredentialPersistenceForSession || credential.persistence() == CredentialPersistencePermanent) + m_session->networkStorageSession().credentialStorage().set(m_partition, credential, challenge.protectionSpace(), challenge.failureResponse().url()); + + if (credential.persistence() == CredentialPersistencePermanent) { + m_protectionSpaceForPersistentStorage = challenge.protectionSpace(); + m_credentialForPersistentStorage = credential; + } + } + + soup_auth_authenticate(challenge.soupAuth(), credential.user().utf8().data(), credential.password().utf8().data()); + } + + soup_session_unpause_message(static_cast<NetworkSessionSoup&>(m_session.get()).soupSession(), m_soupMessage.get()); + }); +} + +void NetworkDataTaskSoup::skipInputStreamForRedirectionCallback(GInputStream* inputStream, GAsyncResult* result, NetworkDataTaskSoup* task) +{ + RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task); + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) { + task->clearRequest(); + return; + } + ASSERT(inputStream == task->m_inputStream.get()); + + GUniqueOutPtr<GError> error; + gssize bytesSkipped = g_input_stream_skip_finish(inputStream, result, &error.outPtr()); + if (error) + task->didFail(ResourceError::genericGError(error.get(), task->m_soupRequest.get())); + else if (bytesSkipped > 0) + task->skipInputStreamForRedirection(); + else + task->didFinishSkipInputStreamForRedirection(); +} + +void NetworkDataTaskSoup::skipInputStreamForRedirection() +{ + ASSERT(m_inputStream); + RefPtr<NetworkDataTaskSoup> protectedThis(this); + g_input_stream_skip_async(m_inputStream.get(), gDefaultReadBufferSize, G_PRIORITY_DEFAULT, m_cancellable.get(), + reinterpret_cast<GAsyncReadyCallback>(skipInputStreamForRedirectionCallback), protectedThis.leakRef()); +} + +void NetworkDataTaskSoup::didFinishSkipInputStreamForRedirection() +{ + g_input_stream_close(m_inputStream.get(), nullptr, nullptr); + continueHTTPRedirection(); +} + +static bool shouldRedirectAsGET(SoupMessage* message, bool crossOrigin) +{ + if (message->method == SOUP_METHOD_GET || message->method == SOUP_METHOD_HEAD) + return false; + + switch (message->status_code) { + case SOUP_STATUS_SEE_OTHER: + return true; + case SOUP_STATUS_FOUND: + case SOUP_STATUS_MOVED_PERMANENTLY: + if (message->method == SOUP_METHOD_POST) + return true; + break; + } + + if (crossOrigin && message->method == SOUP_METHOD_DELETE) + return true; + + return false; +} + +bool NetworkDataTaskSoup::shouldStartHTTPRedirection() +{ + ASSERT(m_soupMessage); + ASSERT(!m_response.isNull()); + + auto status = m_response.httpStatusCode(); + if (!SOUP_STATUS_IS_REDIRECTION(status)) + return false; + + // Some 3xx status codes aren't actually redirects. + if (status == 300 || status == 304 || status == 305 || status == 306) + return false; + + if (m_response.httpHeaderField(HTTPHeaderName::Location).isEmpty()) + return false; + + return true; +} + +void NetworkDataTaskSoup::continueHTTPRedirection() +{ + ASSERT(m_soupMessage); + ASSERT(!m_response.isNull()); + + static const unsigned maxRedirects = 20; + if (m_redirectCount++ > maxRedirects) { + didFail(ResourceError::transportError(m_soupRequest.get(), SOUP_STATUS_TOO_MANY_REDIRECTS, "Too many redirects")); + return; + } + + ResourceRequest request = m_currentRequest; + URL redirectedURL = URL(m_response.url(), m_response.httpHeaderField(HTTPHeaderName::Location)); + if (!redirectedURL.hasFragmentIdentifier() && request.url().hasFragmentIdentifier()) + redirectedURL.setFragmentIdentifier(request.url().fragmentIdentifier()); + request.setURL(redirectedURL); + + // Should not set Referer after a redirect from a secure resource to non-secure one. + if (m_shouldClearReferrerOnHTTPSToHTTPRedirect && !request.url().protocolIs("https") && protocolIs(request.httpReferrer(), "https")) + request.clearHTTPReferrer(); + + bool isCrossOrigin = !protocolHostAndPortAreEqual(m_currentRequest.url(), request.url()); + if (!equalLettersIgnoringASCIICase(request.httpMethod(), "get")) { + // Change newRequest method to GET if change was made during a previous redirection or if current redirection says so. + if (m_soupMessage->method == SOUP_METHOD_GET || !request.url().protocolIsInHTTPFamily() || shouldRedirectAsGET(m_soupMessage.get(), isCrossOrigin)) { + request.setHTTPMethod("GET"); + request.setHTTPBody(nullptr); + request.clearHTTPContentType(); + } + } + + const auto& url = request.url(); + m_user = url.user(); + m_password = url.pass(); + m_lastHTTPMethod = request.httpMethod(); + request.removeCredentials(); + + if (isCrossOrigin) { + // The network layer might carry over some headers from the original request that + // we want to strip here because the redirect is cross-origin. + request.clearHTTPAuthorization(); + request.clearHTTPOrigin(); + } else if (url.protocolIsInHTTPFamily() && m_storedCredentials == AllowStoredCredentials) { + if (m_user.isEmpty() && m_password.isEmpty()) { + auto credential = m_session->networkStorageSession().credentialStorage().get(m_partition, request.url()); + if (!credential.isEmpty()) + m_initialCredential = credential; + } + } + + clearRequest(); + + auto response = ResourceResponse(m_response); + m_client->willPerformHTTPRedirection(WTFMove(response), WTFMove(request), [this, protectedThis = makeRef(*this), isCrossOrigin](const ResourceRequest& newRequest) { + if (newRequest.isNull() || m_state == State::Canceling) + return; + + auto request = newRequest; + if (request.url().protocolIsInHTTPFamily()) { +#if ENABLE(WEB_TIMING) + if (isCrossOrigin) + m_startTime = monotonicallyIncreasingTimeMS(); +#endif + applyAuthenticationToRequest(request); + } + createRequest(WTFMove(request)); + if (m_soupRequest && m_state != State::Suspended) { + m_state = State::Suspended; + resume(); + } + }); +} + +void NetworkDataTaskSoup::readCallback(GInputStream* inputStream, GAsyncResult* result, NetworkDataTaskSoup* task) +{ + RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task); + if (task->state() == State::Canceling || task->state() == State::Completed || (!task->m_client && !task->isDownload())) { + task->clearRequest(); + return; + } + ASSERT(inputStream == task->m_inputStream.get()); + + if (task->state() == State::Suspended) { + ASSERT(!task->m_pendingResult); + task->m_pendingResult = result; + return; + } + + GUniqueOutPtr<GError> error; + gssize bytesRead = g_input_stream_read_finish(inputStream, result, &error.outPtr()); + if (error) + task->didFail(ResourceError::genericGError(error.get(), task->m_soupRequest.get())); + else if (bytesRead > 0) + task->didRead(bytesRead); + else + task->didFinishRead(); +} + +void NetworkDataTaskSoup::read() +{ + RefPtr<NetworkDataTaskSoup> protectedThis(this); + ASSERT(m_inputStream); + m_readBuffer.grow(gDefaultReadBufferSize); + g_input_stream_read_async(m_inputStream.get(), m_readBuffer.data(), m_readBuffer.size(), G_PRIORITY_DEFAULT, m_cancellable.get(), + reinterpret_cast<GAsyncReadyCallback>(readCallback), protectedThis.leakRef()); +} + +void NetworkDataTaskSoup::didRead(gssize bytesRead) +{ + m_readBuffer.shrink(bytesRead); + if (m_downloadOutputStream) { + ASSERT(isDownload()); + writeDownload(); + } else { + ASSERT(m_client); + m_client->didReceiveData(SharedBuffer::adoptVector(m_readBuffer)); + read(); + } +} + +void NetworkDataTaskSoup::didFinishRead() +{ + ASSERT(m_inputStream); + g_input_stream_close(m_inputStream.get(), nullptr, nullptr); + m_inputStream = nullptr; + if (m_multipartInputStream) { + requestNextPart(); + return; + } + + if (m_downloadOutputStream) { + didFinishDownload(); + return; + } + + clearRequest(); + ASSERT(m_client); + m_client->didCompleteWithError({ }); +} + +void NetworkDataTaskSoup::requestNextPartCallback(SoupMultipartInputStream* multipartInputStream, GAsyncResult* result, NetworkDataTaskSoup* task) +{ + RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task); + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) { + task->clearRequest(); + return; + } + ASSERT(multipartInputStream == task->m_multipartInputStream.get()); + + if (task->state() == State::Suspended) { + ASSERT(!task->m_pendingResult); + task->m_pendingResult = result; + return; + } + + GUniqueOutPtr<GError> error; + GRefPtr<GInputStream> inputStream = adoptGRef(soup_multipart_input_stream_next_part_finish(multipartInputStream, result, &error.outPtr())); + if (error) + task->didFail(ResourceError::httpError(task->m_soupMessage.get(), error.get(), task->m_soupRequest.get())); + else if (inputStream) + task->didRequestNextPart(WTFMove(inputStream)); + else + task->didFinishRequestNextPart(); +} + +void NetworkDataTaskSoup::requestNextPart() +{ + RefPtr<NetworkDataTaskSoup> protectedThis(this); + ASSERT(m_multipartInputStream); + ASSERT(!m_inputStream); + soup_multipart_input_stream_next_part_async(m_multipartInputStream.get(), G_PRIORITY_DEFAULT, m_cancellable.get(), + reinterpret_cast<GAsyncReadyCallback>(requestNextPartCallback), protectedThis.leakRef()); +} + +void NetworkDataTaskSoup::didRequestNextPart(GRefPtr<GInputStream>&& inputStream) +{ + ASSERT(!m_inputStream); + m_inputStream = WTFMove(inputStream); + m_response = ResourceResponse(); + m_response.setURL(m_firstRequest.url()); + m_response.updateFromSoupMessageHeaders(soup_multipart_input_stream_get_headers(m_multipartInputStream.get())); + dispatchDidReceiveResponse(); +} + +void NetworkDataTaskSoup::didFinishRequestNextPart() +{ + ASSERT(!m_inputStream); + ASSERT(m_multipartInputStream); + g_input_stream_close(G_INPUT_STREAM(m_multipartInputStream.get()), nullptr, nullptr); + clearRequest(); + m_client->didCompleteWithError({ }); +} + +void NetworkDataTaskSoup::gotHeadersCallback(SoupMessage* soupMessage, NetworkDataTaskSoup* task) +{ + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) { + task->clearRequest(); + return; + } + ASSERT(task->m_soupMessage.get() == soupMessage); + task->didGetHeaders(); +} + +void NetworkDataTaskSoup::didGetHeaders() +{ + // We are a bit more conservative with the persistent credential storage than the session store, + // since we are waiting until we know that this authentication succeeded before actually storing. + // This is because we want to avoid hitting the disk twice (once to add and once to remove) for + // incorrect credentials or polluting the keychain with invalid credentials. + if (!isAuthenticationFailureStatusCode(m_soupMessage->status_code) && m_soupMessage->status_code < 500) { + m_session->networkStorageSession().saveCredentialToPersistentStorage(m_protectionSpaceForPersistentStorage, m_credentialForPersistentStorage); + m_protectionSpaceForPersistentStorage = ProtectionSpace(); + m_credentialForPersistentStorage = Credential(); + } +} + +void NetworkDataTaskSoup::wroteBodyDataCallback(SoupMessage* soupMessage, SoupBuffer* buffer, NetworkDataTaskSoup* task) +{ + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) { + task->clearRequest(); + return; + } + ASSERT(task->m_soupMessage.get() == soupMessage); + task->didWriteBodyData(buffer->length); +} + +void NetworkDataTaskSoup::didWriteBodyData(uint64_t bytesSent) +{ + RefPtr<NetworkDataTaskSoup> protectedThis(this); + m_bodyDataTotalBytesSent += bytesSent; + m_client->didSendData(m_bodyDataTotalBytesSent, m_soupMessage->request_body->length); +} + +void NetworkDataTaskSoup::download() +{ + ASSERT(isDownload()); + ASSERT(m_pendingDownloadLocation); + ASSERT(!m_response.isNull()); + + if (m_response.httpStatusCode() >= 400) { + didFailDownload(platformDownloadNetworkError(m_response.httpStatusCode(), m_response.url(), m_response.httpStatusText())); + return; + } + + CString downloadDestinationPath = m_pendingDownloadLocation.utf8(); + m_downloadDestinationFile = adoptGRef(g_file_new_for_path(downloadDestinationPath.data())); + GRefPtr<GFileOutputStream> outputStream; + GUniqueOutPtr<GError> error; + if (m_allowOverwriteDownload) + outputStream = adoptGRef(g_file_replace(m_downloadDestinationFile.get(), nullptr, FALSE, G_FILE_CREATE_NONE, nullptr, &error.outPtr())); + else + outputStream = adoptGRef(g_file_create(m_downloadDestinationFile.get(), G_FILE_CREATE_NONE, nullptr, &error.outPtr())); + if (!outputStream) { + didFailDownload(platformDownloadDestinationError(m_response, error->message)); + return; + } + + GUniquePtr<char> intermediatePath(g_strdup_printf("%s.wkdownload", downloadDestinationPath.data())); + m_downloadIntermediateFile = adoptGRef(g_file_new_for_path(intermediatePath.get())); + outputStream = adoptGRef(g_file_replace(m_downloadIntermediateFile.get(), nullptr, TRUE, G_FILE_CREATE_NONE, nullptr, &error.outPtr())); + if (!outputStream) { + didFailDownload(platformDownloadDestinationError(m_response, error->message)); + return; + } + m_downloadOutputStream = adoptGRef(G_OUTPUT_STREAM(outputStream.leakRef())); + + auto& downloadManager = NetworkProcess::singleton().downloadManager(); + auto download = std::make_unique<Download>(downloadManager, m_pendingDownloadID, *this, m_session->sessionID(), suggestedFilename()); + auto* downloadPtr = download.get(); + downloadManager.dataTaskBecameDownloadTask(m_pendingDownloadID, WTFMove(download)); + downloadPtr->didCreateDestination(m_pendingDownloadLocation); + + ASSERT(!m_client); + read(); +} + +void NetworkDataTaskSoup::writeDownloadCallback(GOutputStream* outputStream, GAsyncResult* result, NetworkDataTaskSoup* task) +{ + RefPtr<NetworkDataTaskSoup> protectedThis = adoptRef(task); + if (task->state() == State::Canceling || task->state() == State::Completed || !task->isDownload()) { + task->clearRequest(); + return; + } + ASSERT(outputStream == task->m_downloadOutputStream.get()); + + GUniqueOutPtr<GError> error; + gsize bytesWritten; +#if GLIB_CHECK_VERSION(2, 44, 0) + g_output_stream_write_all_finish(outputStream, result, &bytesWritten, &error.outPtr()); +#else + gssize writeTaskResult = g_task_propagate_int(G_TASK(result), &error.outPtr()); + if (writeTaskResult != -1) + bytesWritten = writeTaskResult; +#endif + if (error) + task->didFailDownload(platformDownloadDestinationError(task->m_response, error->message)); + else + task->didWriteDownload(bytesWritten); +} + +void NetworkDataTaskSoup::writeDownload() +{ + RefPtr<NetworkDataTaskSoup> protectedThis(this); +#if GLIB_CHECK_VERSION(2, 44, 0) + g_output_stream_write_all_async(m_downloadOutputStream.get(), m_readBuffer.data(), m_readBuffer.size(), G_PRIORITY_DEFAULT, m_cancellable.get(), + reinterpret_cast<GAsyncReadyCallback>(writeDownloadCallback), protectedThis.leakRef()); +#else + GRefPtr<GTask> writeTask = adoptGRef(g_task_new(m_downloadOutputStream.get(), m_cancellable.get(), + reinterpret_cast<GAsyncReadyCallback>(writeDownloadCallback), protectedThis.leakRef())); + g_task_set_task_data(writeTask.get(), this, nullptr); + g_task_run_in_thread(writeTask.get(), [](GTask* writeTask, gpointer source, gpointer userData, GCancellable* cancellable) { + auto* task = static_cast<NetworkDataTaskSoup*>(userData); + GOutputStream* outputStream = G_OUTPUT_STREAM(source); + RELEASE_ASSERT(task->m_downloadOutputStream.get() == outputStream); + RELEASE_ASSERT(task->m_cancellable.get() == cancellable); + GError* error = nullptr; + if (g_cancellable_set_error_if_cancelled(cancellable, &error)) { + g_task_return_error(writeTask, error); + return; + } + + gsize bytesWritten; + if (g_output_stream_write_all(outputStream, task->m_readBuffer.data(), task->m_readBuffer.size(), &bytesWritten, cancellable, &error)) + g_task_return_int(writeTask, bytesWritten); + else + g_task_return_error(writeTask, error); + }); +#endif +} + +void NetworkDataTaskSoup::didWriteDownload(gsize bytesWritten) +{ + ASSERT(bytesWritten == m_readBuffer.size()); + auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID); + ASSERT(download); + download->didReceiveData(bytesWritten); + read(); +} + +void NetworkDataTaskSoup::didFinishDownload() +{ + ASSERT(!m_response.isNull()); + ASSERT(m_downloadOutputStream); + g_output_stream_close(m_downloadOutputStream.get(), nullptr, nullptr); + m_downloadOutputStream = nullptr; + + ASSERT(m_downloadDestinationFile); + ASSERT(m_downloadIntermediateFile); + GUniqueOutPtr<GError> error; + if (!g_file_move(m_downloadIntermediateFile.get(), m_downloadDestinationFile.get(), G_FILE_COPY_OVERWRITE, m_cancellable.get(), nullptr, nullptr, &error.outPtr())) { + didFailDownload(platformDownloadDestinationError(m_response, error->message)); + return; + } + + GRefPtr<GFileInfo> info = adoptGRef(g_file_info_new()); + CString uri = m_response.url().string().utf8(); + g_file_info_set_attribute_string(info.get(), "metadata::download-uri", uri.data()); + g_file_info_set_attribute_string(info.get(), "xattr::xdg.origin.url", uri.data()); + g_file_set_attributes_async(m_downloadDestinationFile.get(), info.get(), G_FILE_QUERY_INFO_NONE, G_PRIORITY_DEFAULT, nullptr, nullptr, nullptr); + + clearRequest(); + auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID); + ASSERT(download); + download->didFinish(); +} + +void NetworkDataTaskSoup::didFailDownload(const ResourceError& error) +{ + clearRequest(); + cleanDownloadFiles(); + if (m_client) + m_client->didCompleteWithError(error); + else { + auto* download = NetworkProcess::singleton().downloadManager().download(m_pendingDownloadID); + ASSERT(download); + download->didFail(error, IPC::DataReference()); + } +} + +void NetworkDataTaskSoup::cleanDownloadFiles() +{ + if (m_downloadDestinationFile) { + g_file_delete(m_downloadDestinationFile.get(), nullptr, nullptr); + m_downloadDestinationFile = nullptr; + } + if (m_downloadIntermediateFile) { + g_file_delete(m_downloadIntermediateFile.get(), nullptr, nullptr); + m_downloadIntermediateFile = nullptr; + } +} + +void NetworkDataTaskSoup::didFail(const ResourceError& error) +{ + if (isDownload()) { + didFailDownload(platformDownloadNetworkError(error.errorCode(), error.failingURL(), error.localizedDescription())); + return; + } + + clearRequest(); + ASSERT(m_client); + m_client->didCompleteWithError(error); +} + +#if ENABLE(WEB_TIMING) +void NetworkDataTaskSoup::networkEventCallback(SoupMessage* soupMessage, GSocketClientEvent event, GIOStream*, NetworkDataTaskSoup* task) +{ + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) + return; + + ASSERT(task->m_soupMessage.get() == soupMessage); + task->networkEvent(event); +} + +void NetworkDataTaskSoup::networkEvent(GSocketClientEvent event) +{ + double deltaTime = monotonicallyIncreasingTimeMS() - m_startTime; + auto& loadTiming = m_response.networkLoadTiming(); + switch (event) { + case G_SOCKET_CLIENT_RESOLVING: + loadTiming.domainLookupStart = deltaTime; + break; + case G_SOCKET_CLIENT_RESOLVED: + loadTiming.domainLookupEnd = deltaTime; + break; + case G_SOCKET_CLIENT_CONNECTING: + loadTiming.connectStart = deltaTime; + break; + case G_SOCKET_CLIENT_CONNECTED: + // Web Timing considers that connection time involves dns, proxy & TLS negotiation... + // so we better pick G_SOCKET_CLIENT_COMPLETE for connectEnd + break; + case G_SOCKET_CLIENT_PROXY_NEGOTIATING: + break; + case G_SOCKET_CLIENT_PROXY_NEGOTIATED: + break; + case G_SOCKET_CLIENT_TLS_HANDSHAKING: + loadTiming.secureConnectionStart = deltaTime; + break; + case G_SOCKET_CLIENT_TLS_HANDSHAKED: + break; + case G_SOCKET_CLIENT_COMPLETE: + loadTiming.connectEnd = deltaTime; + break; + default: + ASSERT_NOT_REACHED(); + break; + } +} + +#if SOUP_CHECK_VERSION(2, 49, 91) +void NetworkDataTaskSoup::startingCallback(SoupMessage* soupMessage, NetworkDataTaskSoup* task) +{ + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) + return; + + ASSERT(task->m_soupMessage.get() == soupMessage); + task->didStartRequest(); +} +#else +void NetworkDataTaskSoup::requestStartedCallback(SoupSession* session, SoupMessage* soupMessage, SoupSocket*, NetworkDataTaskSoup* task) +{ + ASSERT(session == static_cast<NetworkSessionSoup&>(task->m_session.get()).soupSession()); + if (soupMessage != task->m_soupMessage.get()) + return; + + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) + return; + + task->didStartRequest(); +} +#endif + +void NetworkDataTaskSoup::didStartRequest() +{ + m_response.networkLoadTiming().requestStart = monotonicallyIncreasingTimeMS() - m_startTime; +} + +void NetworkDataTaskSoup::restartedCallback(SoupMessage* soupMessage, NetworkDataTaskSoup* task) +{ + // Called each time the message is going to be sent again except the first time. + // This happens when libsoup handles HTTP authentication. + if (task->state() == State::Canceling || task->state() == State::Completed || !task->m_client) + return; + + ASSERT(task->m_soupMessage.get() == soupMessage); + task->didRestart(); +} + +void NetworkDataTaskSoup::didRestart() +{ + m_startTime = monotonicallyIncreasingTimeMS(); +} +#endif // ENABLE(WEB_TIMING) + +} // namespace WebKit + diff --git a/Source/WebKit2/NetworkProcess/soup/NetworkDataTaskSoup.h b/Source/WebKit2/NetworkProcess/soup/NetworkDataTaskSoup.h new file mode 100644 index 000000000..40ca87c85 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/soup/NetworkDataTaskSoup.h @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2016 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "NetworkDataTask.h" +#include <WebCore/ProtectionSpace.h> +#include <WebCore/ResourceResponse.h> +#include <wtf/RunLoop.h> +#include <wtf/glib/GRefPtr.h> + +namespace WebKit { + +class NetworkDataTaskSoup final : public NetworkDataTask { +public: + static Ref<NetworkDataTask> create(NetworkSession& session, NetworkDataTaskClient& client, const WebCore::ResourceRequest& request, WebCore::StoredCredentials storedCredentials, WebCore::ContentSniffingPolicy shouldContentSniff, bool shouldClearReferrerOnHTTPSToHTTPRedirect) + { + return adoptRef(*new NetworkDataTaskSoup(session, client, request, storedCredentials, shouldContentSniff, shouldClearReferrerOnHTTPSToHTTPRedirect)); + } + + ~NetworkDataTaskSoup(); + +private: + NetworkDataTaskSoup(NetworkSession&, NetworkDataTaskClient&, const WebCore::ResourceRequest&, WebCore::StoredCredentials, WebCore::ContentSniffingPolicy, bool shouldClearReferrerOnHTTPSToHTTPRedirect); + + void suspend() override; + void cancel() override; + void resume() override; + void invalidateAndCancel() override; + NetworkDataTask::State state() const override; + + void setPendingDownloadLocation(const String&, const SandboxExtension::Handle&, bool /*allowOverwrite*/) override; + String suggestedFilename() const override; + + void timeoutFired(); + void startTimeout(); + void stopTimeout(); + + void createRequest(WebCore::ResourceRequest&&); + void clearRequest(); + static void sendRequestCallback(SoupRequest*, GAsyncResult*, NetworkDataTaskSoup*); + void didSendRequest(GRefPtr<GInputStream>&&); + void dispatchDidReceiveResponse(); + + static void tlsErrorsChangedCallback(SoupMessage*, GParamSpec*, NetworkDataTaskSoup*); + void tlsErrorsChanged(); + + void applyAuthenticationToRequest(WebCore::ResourceRequest&); + static void authenticateCallback(SoupSession*, SoupMessage*, SoupAuth*, gboolean retrying, NetworkDataTaskSoup*); + void authenticate(WebCore::AuthenticationChallenge&&); + void continueAuthenticate(WebCore::AuthenticationChallenge&&); + + static void skipInputStreamForRedirectionCallback(GInputStream*, GAsyncResult*, NetworkDataTaskSoup*); + void skipInputStreamForRedirection(); + void didFinishSkipInputStreamForRedirection(); + bool shouldStartHTTPRedirection(); + void continueHTTPRedirection(); + + static void readCallback(GInputStream*, GAsyncResult*, NetworkDataTaskSoup*); + void read(); + void didRead(gssize bytesRead); + void didFinishRead(); + + static void requestNextPartCallback(SoupMultipartInputStream*, GAsyncResult*, NetworkDataTaskSoup*); + void requestNextPart(); + void didRequestNextPart(GRefPtr<GInputStream>&&); + void didFinishRequestNextPart(); + + static void gotHeadersCallback(SoupMessage*, NetworkDataTaskSoup*); + void didGetHeaders(); + + static void wroteBodyDataCallback(SoupMessage*, SoupBuffer*, NetworkDataTaskSoup*); + void didWriteBodyData(uint64_t bytesSent); + + void download(); + static void writeDownloadCallback(GOutputStream*, GAsyncResult*, NetworkDataTaskSoup*); + void writeDownload(); + void didWriteDownload(gsize bytesWritten); + void didFailDownload(const WebCore::ResourceError&); + void didFinishDownload(); + void cleanDownloadFiles(); + + void didFail(const WebCore::ResourceError&); + +#if ENABLE(WEB_TIMING) + static void networkEventCallback(SoupMessage*, GSocketClientEvent, GIOStream*, NetworkDataTaskSoup*); + void networkEvent(GSocketClientEvent); +#if SOUP_CHECK_VERSION(2, 49, 91) + static void startingCallback(SoupMessage*, NetworkDataTaskSoup*); +#else + static void requestStartedCallback(SoupSession*, SoupMessage*, SoupSocket*, NetworkDataTaskSoup*); +#endif + void didStartRequest(); + static void restartedCallback(SoupMessage*, NetworkDataTaskSoup*); + void didRestart(); +#endif // ENABLE(WEB_TIMING) + + State m_state { State::Suspended }; + WebCore::ContentSniffingPolicy m_shouldContentSniff; + GRefPtr<SoupRequest> m_soupRequest; + GRefPtr<SoupMessage> m_soupMessage; + GRefPtr<GInputStream> m_inputStream; + GRefPtr<SoupMultipartInputStream> m_multipartInputStream; + GRefPtr<GCancellable> m_cancellable; + GRefPtr<GAsyncResult> m_pendingResult; + WebCore::ProtectionSpace m_protectionSpaceForPersistentStorage; + WebCore::Credential m_credentialForPersistentStorage; + WebCore::ResourceRequest m_currentRequest; + WebCore::ResourceResponse m_response; + Vector<char> m_readBuffer; + unsigned m_redirectCount { 0 }; + uint64_t m_bodyDataTotalBytesSent { 0 }; + GRefPtr<GFile> m_downloadDestinationFile; + GRefPtr<GFile> m_downloadIntermediateFile; + GRefPtr<GOutputStream> m_downloadOutputStream; + bool m_allowOverwriteDownload { false }; +#if ENABLE(WEB_TIMING) + double m_startTime { 0 }; +#endif + RunLoop::Timer<NetworkDataTaskSoup> m_timeoutSource; +}; + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/soup/NetworkProcessMainSoup.cpp b/Source/WebKit2/NetworkProcess/soup/NetworkProcessMainSoup.cpp new file mode 100644 index 000000000..d5081acd3 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/soup/NetworkProcessMainSoup.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2014 Igalia S.L. + * Copyright (C) 2013 Company 100 Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkProcess.h" + +#include "ChildProcessMain.h" +#include "NetworkProcessMainUnix.h" + +namespace WebKit { + +int NetworkProcessMainUnix(int argc, char** argv) +{ + return ChildProcessMain<NetworkProcess, ChildProcessMainBase>(argc, argv); +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/soup/NetworkProcessSoup.cpp b/Source/WebKit2/NetworkProcess/soup/NetworkProcessSoup.cpp index 5730ca61c..d8dcde330 100644 --- a/Source/WebKit2/NetworkProcess/soup/NetworkProcessSoup.cpp +++ b/Source/WebKit2/NetworkProcess/soup/NetworkProcessSoup.cpp @@ -25,72 +25,100 @@ */ #include "config.h" -#if ENABLE(NETWORK_PROCESS) #include "NetworkProcess.h" +#include "NetworkCache.h" #include "NetworkProcessCreationParameters.h" #include "ResourceCachesToClear.h" #include "WebCookieManager.h" #include <WebCore/CertificateInfo.h> #include <WebCore/FileSystem.h> +#include <WebCore/NetworkStorageSession.h> #include <WebCore/NotImplemented.h> #include <WebCore/ResourceHandle.h> #include <WebCore/SoupNetworkSession.h> #include <libsoup/soup.h> -#include <wtf/gobject/GRefPtr.h> -#include <wtf/gobject/GUniquePtr.h> +#include <wtf/RAMSize.h> +#include <wtf/glib/GRefPtr.h> +#include <wtf/glib/GUniquePtr.h> +#include <wtf/text/CString.h> +#include <wtf/text/StringBuilder.h> using namespace WebCore; namespace WebKit { -static uint64_t getCacheDiskFreeSize(SoupCache* cache) +static CString buildAcceptLanguages(const Vector<String>& languages) { - ASSERT(cache); - - GUniqueOutPtr<char> cacheDir; - g_object_get(G_OBJECT(cache), "cache-dir", &cacheDir.outPtr(), NULL); - if (!cacheDir) - return 0; - - return WebCore::getVolumeFreeSizeForPath(cacheDir.get()); -} + size_t languagesCount = languages.size(); + + // Ignore "C" locale. + size_t cLocalePosition = languages.find("c"); + if (cLocalePosition != notFound) + languagesCount--; + + // Fallback to "en" if the list is empty. + if (!languagesCount) + return "en"; + + // Calculate deltas for the quality values. + int delta; + if (languagesCount < 10) + delta = 10; + else if (languagesCount < 20) + delta = 5; + else + delta = 1; + + // Set quality values for each language. + StringBuilder builder; + for (size_t i = 0; i < languages.size(); ++i) { + if (i == cLocalePosition) + continue; + + if (i) + builder.appendLiteral(", "); + + builder.append(languages[i]); + + int quality = 100 - i * delta; + if (quality > 0 && quality < 100) { + char buffer[8]; + g_ascii_formatd(buffer, 8, "%.2f", quality / 100.0); + builder.append(String::format(";q=%s", buffer)); + } + } -static uint64_t getMemorySize() -{ - static uint64_t kDefaultMemorySize = 512; -#if !OS(WINDOWS) - long pageSize = sysconf(_SC_PAGESIZE); - if (pageSize == -1) - return kDefaultMemorySize; - - long physPages = sysconf(_SC_PHYS_PAGES); - if (physPages == -1) - return kDefaultMemorySize; - - return ((pageSize / 1024) * physPages) / 1024; -#else - // Fallback to default for other platforms. - return kDefaultMemorySize; -#endif + return builder.toString().utf8(); } void NetworkProcess::userPreferredLanguagesChanged(const Vector<String>& languages) { - SoupNetworkSession::defaultSession().setAcceptLanguages(languages); + auto acceptLanguages = buildAcceptLanguages(languages); + SoupNetworkSession::setInitialAcceptLanguages(acceptLanguages); + NetworkStorageSession::forEach([&acceptLanguages](const WebCore::NetworkStorageSession& session) { + if (auto* soupSession = session.soupNetworkSession()) + soupSession->setAcceptLanguages(acceptLanguages); + }); } void NetworkProcess::platformInitializeNetworkProcess(const NetworkProcessCreationParameters& parameters) { + if (parameters.proxySettings.mode != SoupNetworkProxySettings::Mode::Default) + setNetworkProxySettings(parameters.proxySettings); + ASSERT(!parameters.diskCacheDirectory.isEmpty()); - GRefPtr<SoupCache> soupCache = adoptGRef(soup_cache_new(parameters.diskCacheDirectory.utf8().data(), SOUP_CACHE_SINGLE_USER)); - SoupNetworkSession::defaultSession().setCache(soupCache.get()); - // Set an initial huge max_size for the SoupCache so the call to soup_cache_load() won't evict any cached - // resource. The final size of the cache will be set by NetworkProcess::platformSetCacheModel(). - unsigned initialMaxSize = soup_cache_get_max_size(soupCache.get()); - soup_cache_set_max_size(soupCache.get(), G_MAXUINT); - soup_cache_load(soupCache.get()); - soup_cache_set_max_size(soupCache.get(), initialMaxSize); + m_diskCacheDirectory = parameters.diskCacheDirectory; + + SoupNetworkSession::clearOldSoupCache(WebCore::directoryName(m_diskCacheDirectory)); + + NetworkCache::Cache::Parameters cacheParameters { + parameters.shouldEnableNetworkCacheEfficacyLogging +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + , parameters.shouldEnableNetworkCacheSpeculativeRevalidation +#endif + }; + NetworkCache::singleton().initialize(m_diskCacheDirectory, cacheParameters); if (!parameters.cookiePersistentStoragePath.isEmpty()) { supplement<WebCookieManager>()->setCookiePersistentStorage(parameters.cookiePersistentStoragePath, @@ -104,37 +132,18 @@ void NetworkProcess::platformInitializeNetworkProcess(const NetworkProcessCreati setIgnoreTLSErrors(parameters.ignoreTLSErrors); } -void NetworkProcess::platformSetCacheModel(CacheModel cacheModel) +void NetworkProcess::platformSetURLCacheSize(unsigned, uint64_t) { - unsigned cacheTotalCapacity = 0; - unsigned cacheMinDeadCapacity = 0; - unsigned cacheMaxDeadCapacity = 0; - double deadDecodedDataDeletionInterval = 0; - unsigned pageCacheCapacity = 0; - - unsigned long urlCacheMemoryCapacity = 0; - unsigned long urlCacheDiskCapacity = 0; - - SoupCache* cache = SoupNetworkSession::defaultSession().cache(); - uint64_t diskFreeSize = getCacheDiskFreeSize(cache) / 1024 / 1024; - - uint64_t memSize = getMemorySize(); - calculateCacheSizes(cacheModel, memSize, diskFreeSize, - cacheTotalCapacity, cacheMinDeadCapacity, cacheMaxDeadCapacity, deadDecodedDataDeletionInterval, - pageCacheCapacity, urlCacheMemoryCapacity, urlCacheDiskCapacity); - - if (urlCacheDiskCapacity > soup_cache_get_max_size(cache)) - soup_cache_set_max_size(cache, urlCacheDiskCapacity); } void NetworkProcess::setIgnoreTLSErrors(bool ignoreTLSErrors) { - ResourceHandle::setIgnoreSSLErrors(ignoreTLSErrors); + SoupNetworkSession::setShouldIgnoreTLSErrors(ignoreTLSErrors); } void NetworkProcess::allowSpecificHTTPSCertificateForHost(const CertificateInfo& certificateInfo, const String& host) { - ResourceHandle::setClientCertificate(host, certificateInfo.certificate()); + SoupNetworkSession::allowSpecificHTTPSCertificateForHost(certificateInfo, host); } void NetworkProcess::clearCacheForAllOrigins(uint32_t cachesToClear) @@ -142,7 +151,12 @@ void NetworkProcess::clearCacheForAllOrigins(uint32_t cachesToClear) if (cachesToClear == InMemoryResourceCachesOnly) return; - soup_cache_clear(SoupNetworkSession::defaultSession().cache()); + clearDiskCache(std::chrono::system_clock::time_point::min(), [] { }); +} + +void NetworkProcess::clearDiskCache(std::chrono::system_clock::time_point modifiedSince, std::function<void ()> completionHandler) +{ + NetworkCache::singleton().clear(modifiedSince, WTFMove(completionHandler)); } void NetworkProcess::platformTerminate() @@ -150,6 +164,13 @@ void NetworkProcess::platformTerminate() notImplemented(); } -} // namespace WebKit +void NetworkProcess::setNetworkProxySettings(const SoupNetworkProxySettings& settings) +{ + SoupNetworkSession::setProxySettings(settings); + NetworkStorageSession::forEach([](const NetworkStorageSession& session) { + if (auto* soupSession = session.soupNetworkSession()) + soupSession->setupProxy(); + }); +} -#endif +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/soup/NetworkSessionSoup.cpp b/Source/WebKit2/NetworkProcess/soup/NetworkSessionSoup.cpp new file mode 100644 index 000000000..f6baedac1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/soup/NetworkSessionSoup.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2016 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkSessionSoup.h" + +#include "NetworkProcess.h" +#include "WebCookieManager.h" +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/SoupNetworkSession.h> +#include <libsoup/soup.h> + +using namespace WebCore; + +namespace WebKit { + +NetworkSessionSoup::NetworkSessionSoup(SessionID sessionID) + : NetworkSession(sessionID) +{ + networkStorageSession().setCookieObserverHandler([this] { + NetworkProcess::singleton().supplement<WebCookieManager>()->notifyCookiesDidChange(m_sessionID); + }); +} + +NetworkSessionSoup::~NetworkSessionSoup() +{ + if (auto* storageSession = NetworkStorageSession::storageSession(m_sessionID)) + storageSession->setCookieObserverHandler(nullptr); +} + +SoupSession* NetworkSessionSoup::soupSession() const +{ + return networkStorageSession().getOrCreateSoupNetworkSession().soupSession(); +} + +void NetworkSessionSoup::clearCredentials() +{ +#if SOUP_CHECK_VERSION(2, 57, 1) + soup_auth_manager_clear_cached_credentials(SOUP_AUTH_MANAGER(soup_session_get_feature(soupSession(), SOUP_TYPE_AUTH_MANAGER))); +#endif +} + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/soup/NetworkSessionSoup.h b/Source/WebKit2/NetworkProcess/soup/NetworkSessionSoup.h new file mode 100644 index 000000000..e0196029a --- /dev/null +++ b/Source/WebKit2/NetworkProcess/soup/NetworkSessionSoup.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2016 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "NetworkSession.h" + +typedef struct _SoupSession SoupSession; + +namespace WebKit { + +class NetworkSessionSoup final : public NetworkSession { +public: + static Ref<NetworkSession> create(WebCore::SessionID sessionID) + { + return adoptRef(*new NetworkSessionSoup(sessionID)); + } + ~NetworkSessionSoup(); + + SoupSession* soupSession() const; + +private: + NetworkSessionSoup(WebCore::SessionID); + + void clearCredentials() override; +}; + +} // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/soup/RemoteNetworkingContextSoup.cpp b/Source/WebKit2/NetworkProcess/soup/RemoteNetworkingContextSoup.cpp index d7cf86efb..2dba10521 100644 --- a/Source/WebKit2/NetworkProcess/soup/RemoteNetworkingContextSoup.cpp +++ b/Source/WebKit2/NetworkProcess/soup/RemoteNetworkingContextSoup.cpp @@ -26,9 +26,10 @@ */ #include "config.h" -#if ENABLE(NETWORK_PROCESS) #include "RemoteNetworkingContext.h" +#include "NetworkSession.h" +#include "SessionTracker.h" #include <WebCore/NetworkStorageSession.h> #include <WebCore/NotImplemented.h> #include <WebCore/ResourceHandle.h> @@ -46,16 +47,22 @@ bool RemoteNetworkingContext::isValid() const return true; } -void RemoteNetworkingContext::ensurePrivateBrowsingSession(uint64_t sessionID) +void RemoteNetworkingContext::ensurePrivateBrowsingSession(SessionID sessionID) { - notImplemented(); + ASSERT(sessionID.isEphemeral()); + + if (NetworkStorageSession::storageSession(sessionID)) + return; + + NetworkStorageSession::ensurePrivateBrowsingSession(sessionID, String::number(sessionID.sessionID())); + SessionTracker::setSession(sessionID, NetworkSession::create(sessionID)); } NetworkStorageSession& RemoteNetworkingContext::storageSession() const { + if (auto session = NetworkStorageSession::storageSession(m_sessionID)) + return *session; return NetworkStorageSession::defaultStorageSession(); } } - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/unix/NetworkProcessMainUnix.cpp b/Source/WebKit2/NetworkProcess/unix/NetworkProcessMainUnix.cpp deleted file mode 100644 index 84dc636db..000000000 --- a/Source/WebKit2/NetworkProcess/unix/NetworkProcessMainUnix.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2013 Apple Inc. All rights reserved. - * Copyright (C) 2013 Company 100 Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "NetworkProcessMainUnix.h" - -#if ENABLE(NETWORK_PROCESS) - -#include "WKBase.h" -#include "WebKit2Initialize.h" -#include <WebCore/SoupNetworkSession.h> -#include <WebKit2/NetworkProcess.h> -#include <runtime/InitializeThreading.h> -#include <stdlib.h> -#include <wtf/MainThread.h> -#include <wtf/RunLoop.h> -#include <wtf/gobject/GRefPtr.h> - -#if PLATFORM(EFL) -#include <Ecore.h> -#endif - -#if USE(SOUP) -#include <libsoup/soup.h> -#endif - -using namespace WebCore; - -namespace WebKit { - -WK_EXPORT int NetworkProcessMain(int argc, char* argv[]) -{ - if (argc != 2) - return 1; - -#if PLATFORM(EFL) - if (!ecore_init()) - return 1; - - if (!ecore_main_loop_glib_integrate()) - return 1; -#endif - - InitializeWebKit2(); - - SoupNetworkSession::defaultSession().setupHTTPProxyFromEnvironment(); - - int socket = atoi(argv[1]); - - WebKit::ChildProcessInitializationParameters parameters; - parameters.connectionIdentifier = int(socket); - - NetworkProcess::shared().initialize(parameters); - -#if USE(SOUP) - // Despite using system CAs to validate certificates we're - // accepting invalid certificates by default. New API will be - // added later to let client accept/discard invalid certificates. - SoupNetworkSession::defaultSession().setSSLPolicy(SoupNetworkSession::SSLUseSystemCAFile); -#endif - - RunLoop::run(); - -#if USE(SOUP) - if (SoupCache* soupCache = SoupNetworkSession::defaultSession().cache()) { - soup_cache_flush(soupCache); - soup_cache_dump(soupCache); - } -#endif - - return 0; -} - -} // namespace WebKit - -#endif // ENABLE(NETWORK_PROCESS) diff --git a/Source/WebKit2/NetworkProcess/unix/NetworkProcessMainUnix.h b/Source/WebKit2/NetworkProcess/unix/NetworkProcessMainUnix.h index cb62eda2d..fa064c358 100644 --- a/Source/WebKit2/NetworkProcess/unix/NetworkProcessMainUnix.h +++ b/Source/WebKit2/NetworkProcess/unix/NetworkProcessMainUnix.h @@ -27,15 +27,13 @@ #ifndef NetworkProcessMainUnix_h #define NetworkProcessMainUnix_h -#include <WebKit2/WKBase.h> +#include <WebKit/WKBase.h> namespace WebKit { -#ifdef __cplusplus extern "C" { -WK_EXPORT int NetworkProcessMain(int argc, char* argv[]); -} // extern "C" -#endif // __cplusplus +WK_EXPORT int NetworkProcessMainUnix(int argc, char** argv); +} } // namespace WebKit diff --git a/Source/WebKit2/NetworkProcess/webrtc/LibWebRTCSocketClient.cpp b/Source/WebKit2/NetworkProcess/webrtc/LibWebRTCSocketClient.cpp new file mode 100644 index 000000000..33c6be505 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/LibWebRTCSocketClient.cpp @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "LibWebRTCSocketClient.h" + +#if USE(LIBWEBRTC) + +#include "Connection.h" +#include "DataReference.h" +#include "NetworkRTCProvider.h" +#include "WebRTCSocketMessages.h" +#include <WebCore/SharedBuffer.h> +#include <wtf/Function.h> + +namespace WebKit { + +LibWebRTCSocketClient::LibWebRTCSocketClient(uint64_t identifier, NetworkRTCProvider& rtcProvider, std::unique_ptr<rtc::AsyncPacketSocket>&& socket, Type type) + : m_identifier(identifier) + , m_rtcProvider(rtcProvider) + , m_socket(WTFMove(socket)) +{ + if (!m_socket) { + rtcProvider.sendFromMainThread([identifier](IPC::Connection& connection) { + connection.send(Messages::WebRTCSocket::SignalClose(1), identifier); + }); + return; + } + + m_socket->SignalReadPacket.connect(this, &LibWebRTCSocketClient::signalReadPacket); + m_socket->SignalSentPacket.connect(this, &LibWebRTCSocketClient::signalSentPacket); + m_socket->SignalConnect.connect(this, &LibWebRTCSocketClient::signalConnect); + m_socket->SignalClose.connect(this, &LibWebRTCSocketClient::signalClose); + + if (type == Type::ClientTCP) { + m_socket->SignalAddressReady.connect(this, &LibWebRTCSocketClient::signalAddressReady); + return; + } + signalAddressReady(); +} + +void LibWebRTCSocketClient::sendTo(const WebCore::SharedBuffer& buffer, const rtc::SocketAddress& socketAddress, const rtc::PacketOptions& options) +{ + m_socket->SendTo(reinterpret_cast<const uint8_t*>(buffer.data()), buffer.size(), socketAddress, options); +} + +void LibWebRTCSocketClient::close() +{ + ASSERT(m_socket); + m_socket->Close(); + m_rtcProvider.takeSocket(m_identifier); +} + +void LibWebRTCSocketClient::setOption(int option, int value) +{ + ASSERT(m_socket); + m_socket->SetOption(static_cast<rtc::Socket::Option>(option), value); +} + +void LibWebRTCSocketClient::signalReadPacket(rtc::AsyncPacketSocket*, const char* value, size_t length, const rtc::SocketAddress& address, const rtc::PacketTime& packetTime) +{ + auto buffer = WebCore::SharedBuffer::create(value, length); + m_rtcProvider.sendFromMainThread([identifier = m_identifier, buffer = WTFMove(buffer), address = RTCNetwork::isolatedCopy(address), packetTime](IPC::Connection& connection) { + IPC::DataReference data(reinterpret_cast<const uint8_t*>(buffer->data()), buffer->size()); + connection.send(Messages::WebRTCSocket::SignalReadPacket(data, RTCNetwork::IPAddress(address.ipaddr()), address.port(), packetTime.timestamp), identifier); + }); +} + +void LibWebRTCSocketClient::signalSentPacket(rtc::AsyncPacketSocket*, const rtc::SentPacket& sentPacket) +{ + m_rtcProvider.sendFromMainThread([identifier = m_identifier, sentPacket](IPC::Connection& connection) { + connection.send(Messages::WebRTCSocket::SignalSentPacket(sentPacket.packet_id, sentPacket.send_time_ms), identifier); + }); +} + +void LibWebRTCSocketClient::signalAddressReady(rtc::AsyncPacketSocket*, const rtc::SocketAddress& address) +{ + m_rtcProvider.sendFromMainThread([identifier = m_identifier, address = RTCNetwork::isolatedCopy(address)](IPC::Connection& connection) { + connection.send(Messages::WebRTCSocket::SignalAddressReady(RTCNetwork::SocketAddress(address)), identifier); + }); +} + +void LibWebRTCSocketClient::signalAddressReady() +{ + signalAddressReady(m_socket.get(), m_socket->GetLocalAddress()); +} + +void LibWebRTCSocketClient::signalConnect(rtc::AsyncPacketSocket*) +{ + m_rtcProvider.sendFromMainThread([identifier = m_identifier](IPC::Connection& connection) { + connection.send(Messages::WebRTCSocket::SignalConnect(), identifier); + }); +} + +void LibWebRTCSocketClient::signalClose(rtc::AsyncPacketSocket*, int error) +{ + m_rtcProvider.sendFromMainThread([identifier = m_identifier, error](IPC::Connection& connection) { + connection.send(Messages::WebRTCSocket::SignalClose(error), identifier); + }); + m_rtcProvider.takeSocket(m_identifier); +} + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/LibWebRTCSocketClient.h b/Source/WebKit2/NetworkProcess/webrtc/LibWebRTCSocketClient.h new file mode 100644 index 000000000..375cfdf24 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/LibWebRTCSocketClient.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(LIBWEBRTC) + +#include <WebCore/LibWebRTCMacros.h> +#include <webrtc/base/asyncpacketsocket.h> +#include <webrtc/base/sigslot.h> + +namespace rtc { +class AsyncPacketSocket; +class SocketAddress; +struct PacketOptions; +struct PacketTime; +struct SentPacket; +} + +namespace WebCore { +class SharedBuffer; +} + +namespace WebKit { + +class NetworkRTCProvider; + +class LibWebRTCSocketClient final : public sigslot::has_slots<> { +public: + enum class Type { UDP, ServerTCP, ClientTCP }; + + LibWebRTCSocketClient(uint64_t identifier, NetworkRTCProvider&, std::unique_ptr<rtc::AsyncPacketSocket>&&, Type); + +private: + friend class NetworkRTCSocket; + + void close(); + void setOption(int option, int value); + void sendTo(const WebCore::SharedBuffer&, const rtc::SocketAddress&, const rtc::PacketOptions&); + + void signalReadPacket(rtc::AsyncPacketSocket*, const char*, size_t, const rtc::SocketAddress&, const rtc::PacketTime&); + void signalSentPacket(rtc::AsyncPacketSocket*, const rtc::SentPacket&); + void signalAddressReady(rtc::AsyncPacketSocket*, const rtc::SocketAddress&); + void signalConnect(rtc::AsyncPacketSocket*); + void signalClose(rtc::AsyncPacketSocket*, int); + + void signalAddressReady(); + + uint64_t m_identifier; + NetworkRTCProvider& m_rtcProvider; + std::unique_ptr<rtc::AsyncPacketSocket> m_socket; +}; + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.cpp b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.cpp new file mode 100644 index 000000000..e30b5eb07 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkRTCMonitor.h" + +#if USE(LIBWEBRTC) + +#include "Connection.h" +#include "NetworkRTCProvider.h" +#include "WebRTCMonitorMessages.h" +#include <wtf/Function.h> + +namespace WebKit { + +void NetworkRTCMonitor::startUpdating() +{ + m_isStarted = true; + m_rtcProvider.callOnRTCNetworkThread([this]() { + m_manager = std::make_unique<rtc::BasicNetworkManager>(); + m_manager->SignalNetworksChanged.connect(this, &NetworkRTCMonitor::onNetworksChanged); + m_manager->StartUpdating(); + }); +} + +void NetworkRTCMonitor::stopUpdating() +{ + m_isStarted = false; + m_rtcProvider.callOnRTCNetworkThread([this]() { + m_manager->StopUpdating(); + m_manager = nullptr; + }); +} + +void NetworkRTCMonitor::onNetworksChanged() +{ + rtc::BasicNetworkManager::NetworkList networks; + m_manager->GetNetworks(&networks); + + RTCNetwork::IPAddress ipv4; + m_manager->GetDefaultLocalAddress(AF_INET, &ipv4.value); + RTCNetwork::IPAddress ipv6; + m_manager->GetDefaultLocalAddress(AF_INET6, &ipv6.value); + + Vector<RTCNetwork> networkList; + networkList.reserveInitialCapacity(networks.size()); + for (auto* network : networks) { + ASSERT(network); + networkList.uncheckedAppend(RTCNetwork { *network }); + } + + m_rtcProvider.sendFromMainThread([this, networkList = WTFMove(networkList), ipv4 = WTFMove(ipv4), ipv6 = WTFMove(ipv6)](IPC::Connection& connection) { + if (!m_isStarted) + return; + connection.send(Messages::WebRTCMonitor::NetworksChanged(networkList, ipv4, ipv6), 0); + }); +} + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.h b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.h new file mode 100644 index 000000000..0e07d84b7 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(LIBWEBRTC) + +#include "WebCore/LibWebRTCMacros.h" +#include <webrtc/base/network.h> +#include <webrtc/base/sigslot.h> +#include <webrtc/base/thread.h> + +namespace IPC { +class Connection; +class Decoder; +} + +namespace WebKit { + +class NetworkRTCProvider; + +class NetworkRTCMonitor final : public sigslot::has_slots<> { +public: + explicit NetworkRTCMonitor(NetworkRTCProvider& rtcProvider) : m_rtcProvider(rtcProvider) { } + + void didReceiveMessage(IPC::Connection&, IPC::Decoder&); + void stopUpdating(); + +private: + void startUpdating(); + + void onNetworksChanged(); + + NetworkRTCProvider& m_rtcProvider; + std::unique_ptr<rtc::BasicNetworkManager> m_manager; + bool m_isStarted { false }; +}; + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.messages.in b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.messages.in new file mode 100644 index 000000000..51b15dd28 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCMonitor.messages.in @@ -0,0 +1,30 @@ +# Copyright (C) 2017 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#if USE(LIBWEBRTC) + +messages -> NetworkRTCMonitor { + void StartUpdating() + void StopUpdating() +} + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.cpp b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.cpp new file mode 100644 index 000000000..a64f0f2cd --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.cpp @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkRTCProvider.h" + +#if USE(LIBWEBRTC) + +#include "NetworkConnectionToWebProcess.h" +#include "NetworkProcess.h" +#include "NetworkRTCSocket.h" +#include "WebRTCResolverMessages.h" +#include <WebCore/LibWebRTCMacros.h> +#include <webrtc/base/asyncpacketsocket.h> +#include <wtf/MainThread.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { + +static inline std::unique_ptr<rtc::Thread> createThread() +{ + auto thread = rtc::Thread::CreateWithSocketServer(); + auto result = thread->Start(); + ASSERT_UNUSED(result, result); + // FIXME: Set thread name. + return thread; +} + +NetworkRTCProvider::NetworkRTCProvider(NetworkConnectionToWebProcess& connection) + : m_connection(&connection) + , m_rtcMonitor(*this) + , m_rtcNetworkThread(createThread()) + , m_packetSocketFactory(makeUniqueRef<rtc::BasicPacketSocketFactory>(m_rtcNetworkThread.get())) +{ +} + +void NetworkRTCProvider::close() +{ + m_connection = nullptr; + m_resolvers.clear(); + m_rtcMonitor.stopUpdating(); + + callOnRTCNetworkThread([this]() { + m_sockets.clear(); + }); +} + +void NetworkRTCProvider::createUDPSocket(uint64_t identifier, const RTCNetwork::SocketAddress& address, uint16_t minPort, uint16_t maxPort) +{ + callOnRTCNetworkThread([this, identifier, address = RTCNetwork::isolatedCopy(address.value), minPort, maxPort]() { + std::unique_ptr<rtc::AsyncPacketSocket> socket(m_packetSocketFactory->CreateUdpSocket(address, minPort, maxPort)); + addSocket(identifier, std::make_unique<LibWebRTCSocketClient>(identifier, *this, WTFMove(socket), LibWebRTCSocketClient::Type::UDP)); + }); +} + +void NetworkRTCProvider::createServerTCPSocket(uint64_t identifier, const RTCNetwork::SocketAddress& address, uint16_t minPort, uint16_t maxPort, int options) +{ + callOnRTCNetworkThread([this, identifier, address = RTCNetwork::isolatedCopy(address.value), minPort, maxPort, options]() { + std::unique_ptr<rtc::AsyncPacketSocket> socket(m_packetSocketFactory->CreateServerTcpSocket(address, minPort, maxPort, options)); + addSocket(identifier, std::make_unique<LibWebRTCSocketClient>(identifier, *this, WTFMove(socket), LibWebRTCSocketClient::Type::ServerTCP)); + }); +} + +void NetworkRTCProvider::createClientTCPSocket(uint64_t identifier, const RTCNetwork::SocketAddress& localAddress, const RTCNetwork::SocketAddress& remoteAddress, int options) +{ + callOnRTCNetworkThread([this, identifier, localAddress = RTCNetwork::isolatedCopy(localAddress.value), remoteAddress = RTCNetwork::isolatedCopy(remoteAddress.value), options]() { + std::unique_ptr<rtc::AsyncPacketSocket> socket(m_packetSocketFactory->CreateClientTcpSocket(localAddress, remoteAddress, { }, { }, options)); + addSocket(identifier, std::make_unique<LibWebRTCSocketClient>(identifier, *this, WTFMove(socket), LibWebRTCSocketClient::Type::ClientTCP)); + }); +} + +void NetworkRTCProvider::addSocket(uint64_t identifier, std::unique_ptr<LibWebRTCSocketClient>&& socket) +{ + m_sockets.add(identifier, WTFMove(socket)); +} + +std::unique_ptr<LibWebRTCSocketClient> NetworkRTCProvider::takeSocket(uint64_t identifier) +{ + return m_sockets.take(identifier); +} + +void NetworkRTCProvider::didReceiveNetworkRTCSocketMessage(IPC::Connection& connection, IPC::Decoder& decoder) +{ + NetworkRTCSocket(decoder.destinationID(), *this).didReceiveMessage(connection, decoder); +} + +void NetworkRTCProvider::createResolver(uint64_t identifier, const String& address) +{ + CFHostRef host = CFHostCreateWithName(kCFAllocatorDefault, address.createCFString().get()); + ASSERT(host); + + auto resolver = std::make_unique<Resolver>(identifier, *this, host); + + CFHostClientContext context = { 0, resolver.get(), nullptr, nullptr, nullptr }; + CFHostSetClient(host, NetworkRTCProvider::resolvedName, &context); + CFHostScheduleWithRunLoop(host, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode); + Boolean result = CFHostStartInfoResolution(host, kCFHostAddresses, nullptr); + ASSERT_UNUSED(result, result); + + m_resolvers.add(identifier, WTFMove(resolver)); +} + +void NetworkRTCProvider::stopResolver(uint64_t identifier) +{ + auto resolver = m_resolvers.take(identifier); + if (resolver) + CFHostCancelInfoResolution(resolver->host, CFHostInfoType::kCFHostAddresses); +} + +void NetworkRTCProvider::resolvedName(CFHostRef hostRef, CFHostInfoType typeInfo, const CFStreamError *error, void *info) +{ + ASSERT_UNUSED(typeInfo, !typeInfo); + + if (error->domain) { + // FIXME: Need to handle failure, but info is not provided in the callback. + return; + } + + ASSERT(info); + auto* resolverInfo = static_cast<Resolver*>(info); + auto resolver = resolverInfo->rtcProvider.m_resolvers.take(resolverInfo->identifier); + if (!resolver) + return; + + Boolean result; + CFArrayRef resolvedAddresses = (CFArrayRef)CFHostGetAddressing(hostRef, &result); + ASSERT_UNUSED(result, result); + + size_t count = CFArrayGetCount(resolvedAddresses); + Vector<RTCNetwork::IPAddress> addresses; + addresses.reserveInitialCapacity(count); + + for (size_t index = 0; index < count; ++index) { + CFDataRef data = (CFDataRef)CFArrayGetValueAtIndex(resolvedAddresses, index); + auto* address = reinterpret_cast<const struct sockaddr_in*>(CFDataGetBytePtr(data)); + addresses.uncheckedAppend(RTCNetwork::IPAddress(rtc::IPAddress(address->sin_addr))); + } + ASSERT(resolver->rtcProvider.m_connection); + resolver->rtcProvider.m_connection->connection().send(Messages::WebRTCResolver::SetResolvedAddress(addresses), resolver->identifier); +} + +struct NetworkMessageData : public rtc::MessageData { + NetworkMessageData(Ref<NetworkRTCProvider>&& rtcProvider, Function<void()>&& callback) + : rtcProvider(WTFMove(rtcProvider)) + , callback(WTFMove(callback)) + { } + Ref<NetworkRTCProvider> rtcProvider; + Function<void()> callback; +}; + +void NetworkRTCProvider::OnMessage(rtc::Message* message) +{ + ASSERT(message->message_id == 1); + static_cast<NetworkMessageData*>(message->pdata)->callback(); +} + +void NetworkRTCProvider::callOnRTCNetworkThread(Function<void()>&& callback) +{ + m_rtcNetworkThread->Post(RTC_FROM_HERE, this, 1, new NetworkMessageData(*this, WTFMove(callback))); +} + +void NetworkRTCProvider::callSocket(uint64_t identifier, Function<void(LibWebRTCSocketClient&)>&& callback) +{ + callOnRTCNetworkThread([this, identifier, callback = WTFMove(callback)]() { + if (auto* socket = m_sockets.get(identifier)) + callback(*socket); + }); +} + +void NetworkRTCProvider::sendFromMainThread(Function<void(IPC::Connection&)>&& callback) +{ + callOnMainThread([provider = makeRef(*this), callback = WTFMove(callback)]() { + if (provider->m_connection) + callback(provider->m_connection->connection()); + }); +} + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.h b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.h new file mode 100644 index 000000000..6a72182be --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.h @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(LIBWEBRTC) + +#include "LibWebRTCSocketClient.h" +#include "NetworkRTCMonitor.h" +#include "RTCNetwork.h" +#include <CFNetwork/CFHost.h> +#include <WebCore/LibWebRTCMacros.h> +#include <webrtc/base/sigslot.h> +#include <webrtc/p2p/base/basicpacketsocketfactory.h> +#include <wtf/HashMap.h> +#include <wtf/ThreadSafeRefCounted.h> +#include <wtf/UniqueRef.h> +#include <wtf/text/WTFString.h> + +namespace IPC { +class Connection; +class Decoder; +} + +namespace WebKit { + +class NetworkConnectionToWebProcess; +class NetworkRTCSocket; + +class NetworkRTCProvider : public ThreadSafeRefCounted<NetworkRTCProvider>, public rtc::MessageHandler { +public: + static Ref<NetworkRTCProvider> create(NetworkConnectionToWebProcess& connection) { return adoptRef(*new NetworkRTCProvider(connection)); } + + void didReceiveMessage(IPC::Connection&, IPC::Decoder&); + void didReceiveNetworkRTCMonitorMessage(IPC::Connection& connection, IPC::Decoder& decoder) { m_rtcMonitor.didReceiveMessage(connection, decoder); } + void didReceiveNetworkRTCSocketMessage(IPC::Connection&, IPC::Decoder&); + + std::unique_ptr<LibWebRTCSocketClient> takeSocket(uint64_t); + void resolverDone(uint64_t); + + void close(); + + void callSocket(uint64_t, Function<void(LibWebRTCSocketClient&)>&&); + void callOnRTCNetworkThread(Function<void()>&&); + void sendFromMainThread(Function<void(IPC::Connection&)>&&); + +private: + explicit NetworkRTCProvider(NetworkConnectionToWebProcess&); + + void createUDPSocket(uint64_t, const RTCNetwork::SocketAddress&, uint16_t, uint16_t); + void createClientTCPSocket(uint64_t, const RTCNetwork::SocketAddress&, const RTCNetwork::SocketAddress&, int); + void createServerTCPSocket(uint64_t, const RTCNetwork::SocketAddress&, uint16_t minPort, uint16_t maxPort, int); + void createResolver(uint64_t, const String&); + void stopResolver(uint64_t); + + void addSocket(uint64_t, std::unique_ptr<LibWebRTCSocketClient>&&); + + void OnMessage(rtc::Message*); + + static void resolvedName(CFHostRef, CFHostInfoType, const CFStreamError*, void*); + + struct Resolver { + Resolver(uint64_t identifier, NetworkRTCProvider& rtcProvider, RetainPtr<CFHostRef>&& host) + : identifier(identifier) + , rtcProvider(rtcProvider) + , host(WTFMove(host)) { } + ~Resolver() { CFRelease(host); } + + uint64_t identifier; + NetworkRTCProvider& rtcProvider; + CFHostRef host; + }; + + HashMap<uint64_t, std::unique_ptr<Resolver>> m_resolvers; + HashMap<uint64_t, std::unique_ptr<LibWebRTCSocketClient>> m_sockets; + NetworkConnectionToWebProcess* m_connection; + bool m_isStarted { true }; + + NetworkRTCMonitor m_rtcMonitor; + + std::unique_ptr<rtc::Thread> m_rtcNetworkThread; + UniqueRef<rtc::BasicPacketSocketFactory> m_packetSocketFactory; +}; + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.messages.in b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.messages.in new file mode 100644 index 000000000..940952144 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCProvider.messages.in @@ -0,0 +1,33 @@ +# Copyright (C) 2017 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#if USE(LIBWEBRTC) + +messages -> NetworkRTCProvider { + CreateUDPSocket(int identifier, WebKit::RTCNetwork::SocketAddress localAddress, uint16_t minPort, uint16_t maxPort) + CreateServerTCPSocket(int identifier, WebKit::RTCNetwork::SocketAddress localAddress, uint16_t minPort, uint16_t maxPort, int options) + CreateClientTCPSocket(int identifier, WebKit::RTCNetwork::SocketAddress localAddress, WebKit::RTCNetwork::SocketAddress remoteAddress, int options) + CreateResolver(uint64_t identifier, String address) + StopResolver(uint64_t identifier) +} + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.cpp b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.cpp new file mode 100644 index 000000000..fcd12498f --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkRTCSocket.h" + +#if USE(LIBWEBRTC) + +#include "DataReference.h" +#include "LibWebRTCSocketClient.h" +#include "NetworkRTCProvider.h" +#include <WebCore/SharedBuffer.h> +#include <wtf/Function.h> + +namespace WebKit { + +NetworkRTCSocket::NetworkRTCSocket(uint64_t identifier, NetworkRTCProvider& rtcProvider) + : m_identifier(identifier) + , m_rtcProvider(rtcProvider) +{ +} + +void NetworkRTCSocket::sendTo(const IPC::DataReference& data, const RTCNetwork::SocketAddress& socketAddress, int packetID, int rtpSendtimeExtensionID, String srtpAuth, int64_t srtpPacketIndex, int dscp) +{ + auto buffer = WebCore::SharedBuffer::create(data.data(), data.size()); + + rtc::PacketOptions options; + options.packet_id = packetID; + options.packet_time_params.rtp_sendtime_extension_id = rtpSendtimeExtensionID; + options.packet_time_params.srtp_packet_index = srtpPacketIndex; + options.dscp = static_cast<rtc::DiffServCodePoint>(dscp); + if (srtpAuth.utf8().length()) { + options.packet_time_params.srtp_auth_key = std::vector<char>(srtpAuth.utf8().data(), srtpAuth.utf8().data() + srtpAuth.utf8().length()); + options.packet_time_params.srtp_auth_tag_len = srtpAuth.utf8().length(); + } else + options.packet_time_params.srtp_auth_tag_len = -1; + + m_rtcProvider.callSocket(m_identifier, [buffer = WTFMove(buffer), socketAddress, options](LibWebRTCSocketClient& client) { + client.sendTo(buffer.get(), socketAddress.value, options); + }); +} + +void NetworkRTCSocket::close() +{ + m_rtcProvider.callSocket(m_identifier, [](LibWebRTCSocketClient& socket) { + socket.close(); + }); +} + +void NetworkRTCSocket::setOption(int option, int value) +{ + m_rtcProvider.callSocket(m_identifier, [option, value](LibWebRTCSocketClient& socket) { + socket.setOption(option, value); + }); +} + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.h b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.h new file mode 100644 index 000000000..ea99540e6 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2017 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(LIBWEBRTC) + +#include "RTCNetwork.h" + +#include <WebCore/LibWebRTCMacros.h> +#include <webrtc/base/asyncpacketsocket.h> +#include <webrtc/base/sigslot.h> + +namespace IPC { +class Connection; +class DataReference; +} + +namespace rtc { +class AsyncPacketSocket; +class SocketAddress; +struct PacketOptions; +struct PacketTime; +struct SentPacket; +} + +namespace WebCore { +class SharedBuffer; +} + +namespace WebKit { + +class NetworkConnectionToWebProcess; +class NetworkRTCProvider; + +class NetworkRTCSocket { +public: + NetworkRTCSocket(uint64_t, NetworkRTCProvider&); + void didReceiveMessage(IPC::Connection&, IPC::Decoder&); +private: + void sendTo(const IPC::DataReference&, const RTCNetwork::SocketAddress&, int packetID, int rtpSendtimeExtensionID, String srtpAuth, int64_t srtpPacketIndex, int dscp); + void close(); + void setOption(int option, int value); + + uint64_t m_identifier; + NetworkRTCProvider& m_rtcProvider; +}; + +} // namespace WebKit + +#endif // USE(LIBWEBRTC) diff --git a/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.messages.in b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.messages.in new file mode 100644 index 000000000..713d423e8 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/webrtc/NetworkRTCSocket.messages.in @@ -0,0 +1,31 @@ +# Copyright (C) 2017 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#if USE(LIBWEBRTC) + +messages -> NetworkRTCSocket { + void SendTo(IPC::DataReference data, WebKit::RTCNetwork::SocketAddress address, int packetID, int rtpSendtimeExtensionID, String srtpAuth, int64_t srtpPacketIndex, int dscp) + void Close() + void SetOption(int option, int value) +} + +#endif // USE(LIBWEBRTC)s |