diff options
Diffstat (limited to 'chromium/storage')
47 files changed, 3234 insertions, 1565 deletions
diff --git a/chromium/storage/OWNERS b/chromium/storage/OWNERS index 5c7f0175f40..ddd9fcfd45c 100644 --- a/chromium/storage/OWNERS +++ b/chromium/storage/OWNERS @@ -1,4 +1,5 @@ darin@chromium.org +jsbell@chromium.org kinuko@chromium.org michaeln@chromium.org piman@chromium.org diff --git a/chromium/storage/browser/BUILD.gn b/chromium/storage/browser/BUILD.gn index a4cc79a2e3f..8e91590e8a4 100644 --- a/chromium/storage/browser/BUILD.gn +++ b/chromium/storage/browser/BUILD.gn @@ -1,14 +1,11 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//testing/test.gni") component("browser") { output_name = "storage_browser" sources = [ - "blob/blob_async_builder_host.cc", - "blob/blob_async_builder_host.h", - "blob/blob_async_transport_request_builder.cc", - "blob/blob_async_transport_request_builder.h", "blob/blob_data_builder.cc", "blob/blob_data_builder.h", "blob/blob_data_handle.cc", @@ -17,18 +14,24 @@ component("browser") { "blob/blob_data_item.h", "blob/blob_data_snapshot.cc", "blob/blob_data_snapshot.h", + "blob/blob_entry.cc", + "blob/blob_entry.h", + "blob/blob_memory_controller.cc", + "blob/blob_memory_controller.h", "blob/blob_reader.cc", "blob/blob_reader.h", "blob/blob_storage_context.cc", "blob/blob_storage_context.h", "blob/blob_storage_registry.cc", "blob/blob_storage_registry.h", + "blob/blob_transport_host.cc", + "blob/blob_transport_host.h", + "blob/blob_transport_request_builder.cc", + "blob/blob_transport_request_builder.h", "blob/blob_url_request_job.cc", "blob/blob_url_request_job.h", "blob/blob_url_request_job_factory.cc", "blob/blob_url_request_job_factory.h", - "blob/internal_blob_data.cc", - "blob/internal_blob_data.h", "blob/scoped_file.cc", "blob/scoped_file.h", "blob/shareable_blob_data_item.cc", @@ -215,3 +218,21 @@ executable("dump_file_system") { "//storage/common", ] } + +test("storage_unittests") { + sources = [ + # Do NOT add storage/ tests here until this target is added to the build + # bots. http://crbug.com/653751 + + # If the sources list is empty, the win_clang builder fails. + # This file will be removed when the real tests are moved over from + # content_unittests to this target. + "crbug653751_unittest.cc", + ] + + deps = [ + "//base/test:run_all_unittests", + "//base/test:test_support", + "//testing/gtest", + ] +} diff --git a/chromium/storage/browser/blob/blob_async_builder_host.cc b/chromium/storage/browser/blob/blob_async_builder_host.cc deleted file mode 100644 index a73d9d68323..00000000000 --- a/chromium/storage/browser/blob/blob_async_builder_host.cc +++ /dev/null @@ -1,465 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "storage/browser/blob/blob_async_builder_host.h" - -#include <stddef.h> -#include <stdint.h> - -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/memory/ptr_util.h" -#include "base/memory/shared_memory.h" -#include "storage/browser/blob/blob_data_handle.h" -#include "storage/browser/blob/blob_storage_context.h" - -namespace storage { -namespace { - -bool CalculateBlobMemorySize(const std::vector<DataElement>& elements, - size_t* shortcut_bytes, - uint64_t* total_bytes) { - DCHECK(shortcut_bytes); - DCHECK(total_bytes); - base::CheckedNumeric<uint64_t> total_size_checked = 0; - base::CheckedNumeric<size_t> shortcut_size_checked = 0; - for (const auto& e : elements) { - if (e.type() == DataElement::TYPE_BYTES) { - total_size_checked += e.length(); - shortcut_size_checked += e.length(); - } else if (e.type() == DataElement::TYPE_BYTES_DESCRIPTION) { - total_size_checked += e.length(); - } else { - continue; - } - if (!total_size_checked.IsValid() || !shortcut_size_checked.IsValid()) { - return false; - } - } - *shortcut_bytes = shortcut_size_checked.ValueOrDie(); - *total_bytes = total_size_checked.ValueOrDie(); - return true; -} - -IPCBlobCreationCancelCode ConvertReferencedBlobErrorToConstructingError( - IPCBlobCreationCancelCode referenced_blob_error) { - switch (referenced_blob_error) { - // For most cases we propagate the error. - case IPCBlobCreationCancelCode::FILE_WRITE_FAILED: - case IPCBlobCreationCancelCode::SOURCE_DIED_IN_TRANSIT: - case IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN: - case IPCBlobCreationCancelCode::OUT_OF_MEMORY: - return referenced_blob_error; - // Others we report that the referenced blob is broken, as we don't know - // why (the BLOB_DEREFERENCED_WHILE_BUILDING should never happen, as we hold - // onto the reference of the blobs we're using). - case IPCBlobCreationCancelCode::BLOB_DEREFERENCED_WHILE_BUILDING: - DCHECK(false) << "Referenced blob should never be dereferenced while we " - << "are depending on it, as our system holds a handle."; - case IPCBlobCreationCancelCode::UNKNOWN: - return IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN; - } - NOTREACHED(); - return IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN; -} - -} // namespace - -using MemoryItemRequest = - BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest; - -BlobAsyncBuilderHost::BlobBuildingState::BlobBuildingState( - const std::string& uuid, - std::set<std::string> referenced_blob_uuids, - std::vector<std::unique_ptr<BlobDataHandle>>* referenced_blob_handles) - : data_builder(uuid), - referenced_blob_uuids(referenced_blob_uuids), - referenced_blob_handles(std::move(*referenced_blob_handles)) {} - -BlobAsyncBuilderHost::BlobBuildingState::~BlobBuildingState() {} - -BlobAsyncBuilderHost::BlobAsyncBuilderHost() : ptr_factory_(this) {} - -BlobAsyncBuilderHost::~BlobAsyncBuilderHost() {} - -BlobTransportResult BlobAsyncBuilderHost::RegisterBlobUUID( - const std::string& uuid, - const std::string& content_type, - const std::string& content_disposition, - const std::set<std::string>& referenced_blob_uuids, - BlobStorageContext* context) { - if (async_blob_map_.find(uuid) != async_blob_map_.end()) - return BlobTransportResult::BAD_IPC; - if (referenced_blob_uuids.find(uuid) != referenced_blob_uuids.end()) - return BlobTransportResult::BAD_IPC; - context->CreatePendingBlob(uuid, content_type, content_disposition); - std::vector<std::unique_ptr<BlobDataHandle>> handles; - for (const std::string& referenced_uuid : referenced_blob_uuids) { - std::unique_ptr<BlobDataHandle> handle = - context->GetBlobDataFromUUID(referenced_uuid); - if (!handle || handle->IsBroken()) { - // We cancel the blob right away, and don't bother storing our state. - context->CancelPendingBlob( - uuid, IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN); - return BlobTransportResult::CANCEL_REFERENCED_BLOB_BROKEN; - } - handles.emplace_back(std::move(handle)); - } - async_blob_map_[uuid] = base::MakeUnique<BlobBuildingState>( - uuid, referenced_blob_uuids, &handles); - return BlobTransportResult::DONE; -} - -BlobTransportResult BlobAsyncBuilderHost::StartBuildingBlob( - const std::string& uuid, - const std::vector<DataElement>& elements, - size_t memory_available, - BlobStorageContext* context, - const RequestMemoryCallback& request_memory) { - DCHECK(context); - DCHECK(async_blob_map_.find(uuid) != async_blob_map_.end()); - - // Step 1: Get the sizes. - size_t shortcut_memory_size_bytes = 0; - uint64_t total_memory_size_bytes = 0; - if (!CalculateBlobMemorySize(elements, &shortcut_memory_size_bytes, - &total_memory_size_bytes)) { - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context); - return BlobTransportResult::BAD_IPC; - } - - // Step 2: Check if we have enough memory to store the blob. - if (total_memory_size_bytes > memory_available) { - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::OUT_OF_MEMORY, context); - return BlobTransportResult::CANCEL_MEMORY_FULL; - } - - // From here on, we know we can fit the blob in memory. - BlobBuildingState* state_ptr = async_blob_map_[uuid].get(); - if (!state_ptr->request_builder.requests().empty()) { - // Check that we're not a duplicate call. - return BlobTransportResult::BAD_IPC; - } - state_ptr->request_memory_callback = request_memory; - - // Step 3: Check to make sure the referenced blob information we received - // earlier is correct: - std::set<std::string> extracted_blob_uuids; - for (const DataElement& e : elements) { - if (e.type() == DataElement::TYPE_BLOB) { - extracted_blob_uuids.insert(e.blob_uuid()); - // We can't depend on ourselves. - if (e.blob_uuid() == uuid) { - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context); - return BlobTransportResult::BAD_IPC; - } - } - } - if (extracted_blob_uuids != state_ptr->referenced_blob_uuids) { - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context); - return BlobTransportResult::BAD_IPC; - } - - // Step 4: Decide if we're using the shortcut method. This will also catch - // the case where we don't have any memory items. - if (shortcut_memory_size_bytes == total_memory_size_bytes && - shortcut_memory_size_bytes <= memory_available) { - for (const DataElement& e : elements) { - state_ptr->data_builder.AppendIPCDataElement(e); - } - FinishBuildingBlob(state_ptr, context); - return BlobTransportResult::DONE; - } - - // From here on, we know the blob's size is less than |memory_available|, - // so we know we're < max(size_t). - // Step 5: Decide if we're using shared memory. - if (total_memory_size_bytes > max_ipc_memory_size_) { - state_ptr->request_builder.InitializeForSharedMemoryRequests( - max_shared_memory_size_, total_memory_size_bytes, elements, - &(state_ptr->data_builder)); - } else { - // Step 6: We can fit in IPC. - state_ptr->request_builder.InitializeForIPCRequests( - max_ipc_memory_size_, total_memory_size_bytes, elements, - &(state_ptr->data_builder)); - } - // We initialize our requests received state now that they are populated. - state_ptr->request_received.resize( - state_ptr->request_builder.requests().size(), false); - return ContinueBlobMemoryRequests(uuid, context); -} - -BlobTransportResult BlobAsyncBuilderHost::OnMemoryResponses( - const std::string& uuid, - const std::vector<BlobItemBytesResponse>& responses, - BlobStorageContext* context) { - AsyncBlobMap::const_iterator state_it = async_blob_map_.find(uuid); - if (state_it == async_blob_map_.end()) { - DVLOG(1) << "Could not find blob " << uuid; - return BlobTransportResult::BAD_IPC; - } - if (responses.empty()) { - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context); - return BlobTransportResult::BAD_IPC; - } - BlobAsyncBuilderHost::BlobBuildingState* state = state_it->second.get(); - BlobAsyncTransportRequestBuilder& request_builder = state->request_builder; - const auto& requests = request_builder.requests(); - for (const BlobItemBytesResponse& response : responses) { - if (response.request_number >= requests.size()) { - // Bad IPC, so we delete our record and ignore. - DVLOG(1) << "Invalid request number " << response.request_number; - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context); - return BlobTransportResult::BAD_IPC; - } - DCHECK_LT(response.request_number, state->request_received.size()); - const MemoryItemRequest& request = requests[response.request_number]; - if (state->request_received[response.request_number]) { - // Bad IPC, so we delete our record. - DVLOG(1) << "Already received response for that request."; - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context); - return BlobTransportResult::BAD_IPC; - } - state->request_received[response.request_number] = true; - bool invalid_ipc = false; - bool memory_error = false; - switch (request.message.transport_strategy) { - case IPCBlobItemRequestStrategy::IPC: - if (response.inline_data.size() < request.message.size) { - DVLOG(1) << "Invalid data size " << response.inline_data.size() - << " vs requested size of " << request.message.size; - invalid_ipc = true; - break; - } - invalid_ipc = !state->data_builder.PopulateFutureData( - request.browser_item_index, &response.inline_data[0], - request.browser_item_offset, request.message.size); - break; - case IPCBlobItemRequestStrategy::SHARED_MEMORY: - if (state->num_shared_memory_requests == 0) { - DVLOG(1) << "Received too many responses for shared memory."; - invalid_ipc = true; - break; - } - state->num_shared_memory_requests--; - if (!state->shared_memory_block->memory()) { - // We just map the whole block, as we'll probably be accessing the - // whole thing in this group of responses. Another option is to use - // MapAt, remove the mapped boolean, and then exclude the - // handle_offset below. - size_t handle_size = request_builder.shared_memory_sizes() - [state->current_shared_memory_handle_index]; - if (!state->shared_memory_block->Map(handle_size)) { - DVLOG(1) << "Unable to map memory to size " << handle_size; - memory_error = true; - break; - } - } - - invalid_ipc = !state->data_builder.PopulateFutureData( - request.browser_item_index, - static_cast<const char*>(state->shared_memory_block->memory()) + - request.message.handle_offset, - request.browser_item_offset, request.message.size); - break; - case IPCBlobItemRequestStrategy::FILE: - case IPCBlobItemRequestStrategy::UNKNOWN: - DVLOG(1) << "Not implemented."; - invalid_ipc = true; - break; - } - if (invalid_ipc) { - // Bad IPC, so we delete our record and return false. - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context); - return BlobTransportResult::BAD_IPC; - } - if (memory_error) { - DVLOG(1) << "Shared memory error."; - CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::OUT_OF_MEMORY, - context); - return BlobTransportResult::CANCEL_MEMORY_FULL; - } - state->num_fulfilled_requests++; - } - return ContinueBlobMemoryRequests(uuid, context); -} - -void BlobAsyncBuilderHost::CancelBuildingBlob(const std::string& uuid, - IPCBlobCreationCancelCode code, - BlobStorageContext* context) { - DCHECK(context); - auto state_it = async_blob_map_.find(uuid); - if (state_it == async_blob_map_.end()) { - return; - } - // We can have the blob dereferenced by the renderer, but have it still being - // 'built'. In this case, it's destructed in the context, but we still have - // it in our map. Hence we make sure the context has the entry before - // calling cancel. - if (context->registry().HasEntry(uuid)) - context->CancelPendingBlob(uuid, code); - async_blob_map_.erase(state_it); -} - -void BlobAsyncBuilderHost::CancelAll(BlobStorageContext* context) { - DCHECK(context); - // If the blob still exists in the context (and is being built), then we know - // that someone else is expecting our blob, and we need to cancel it to let - // the dependency know it's gone. - std::vector<std::unique_ptr<BlobDataHandle>> referenced_pending_blobs; - for (const auto& uuid_state_pair : async_blob_map_) { - if (context->IsBeingBuilt(uuid_state_pair.first)) { - referenced_pending_blobs.emplace_back( - context->GetBlobDataFromUUID(uuid_state_pair.first)); - } - } - // We clear the map before canceling them to prevent any strange reentry into - // our class (see ReferencedBlobFinished) if any blobs were waiting for others - // to construct. - async_blob_map_.clear(); - for (const std::unique_ptr<BlobDataHandle>& handle : - referenced_pending_blobs) { - context->CancelPendingBlob( - handle->uuid(), IPCBlobCreationCancelCode::SOURCE_DIED_IN_TRANSIT); - } -} - -BlobTransportResult BlobAsyncBuilderHost::ContinueBlobMemoryRequests( - const std::string& uuid, - BlobStorageContext* context) { - AsyncBlobMap::const_iterator state_it = async_blob_map_.find(uuid); - DCHECK(state_it != async_blob_map_.end()); - BlobAsyncBuilderHost::BlobBuildingState* state = state_it->second.get(); - - BlobAsyncTransportRequestBuilder& request_builder = state->request_builder; - const std::vector<MemoryItemRequest>& requests = request_builder.requests(); - size_t num_requests = requests.size(); - if (state->num_fulfilled_requests == num_requests) { - FinishBuildingBlob(state, context); - return BlobTransportResult::DONE; - } - DCHECK_LT(state->num_fulfilled_requests, num_requests); - if (state->next_request == num_requests) { - // We are still waiting on other requests to come back. - return BlobTransportResult::PENDING_RESPONSES; - } - - std::unique_ptr<std::vector<BlobItemBytesRequest>> byte_requests( - new std::vector<BlobItemBytesRequest>()); - std::unique_ptr<std::vector<base::SharedMemoryHandle>> shared_memory( - new std::vector<base::SharedMemoryHandle>()); - - for (; state->next_request < num_requests; ++state->next_request) { - const MemoryItemRequest& request = requests[state->next_request]; - - bool stop_accumulating = false; - bool using_shared_memory_handle = state->num_shared_memory_requests > 0; - switch (request.message.transport_strategy) { - case IPCBlobItemRequestStrategy::IPC: - byte_requests->push_back(request.message); - break; - case IPCBlobItemRequestStrategy::SHARED_MEMORY: - if (using_shared_memory_handle && - state->current_shared_memory_handle_index != - request.message.handle_index) { - // We only want one shared memory per requesting blob. - stop_accumulating = true; - break; - } - using_shared_memory_handle = true; - state->current_shared_memory_handle_index = - request.message.handle_index; - state->num_shared_memory_requests++; - - if (!state->shared_memory_block) { - state->shared_memory_block.reset(new base::SharedMemory()); - size_t size = - request_builder - .shared_memory_sizes()[request.message.handle_index]; - if (!state->shared_memory_block->CreateAnonymous(size)) { - DVLOG(1) << "Unable to allocate shared memory for blob transfer."; - return BlobTransportResult::CANCEL_MEMORY_FULL; - } - } - shared_memory->push_back(state->shared_memory_block->handle()); - byte_requests->push_back(request.message); - // Since we are only using one handle at a time, transform our handle - // index correctly back to 0. - byte_requests->back().handle_index = 0; - break; - case IPCBlobItemRequestStrategy::FILE: - case IPCBlobItemRequestStrategy::UNKNOWN: - NOTREACHED() << "Not implemented yet."; - break; - } - if (stop_accumulating) { - break; - } - } - DCHECK(!requests.empty()); - - state->request_memory_callback.Run( - std::move(byte_requests), std::move(shared_memory), - base::MakeUnique<std::vector<base::File>>()); - return BlobTransportResult::PENDING_RESPONSES; -} - -void BlobAsyncBuilderHost::ReferencedBlobFinished( - const std::string& owning_blob_uuid, - base::WeakPtr<BlobStorageContext> context, - bool construction_success, - IPCBlobCreationCancelCode reason) { - if (!context) { - return; - } - auto state_it = async_blob_map_.find(owning_blob_uuid); - if (state_it == async_blob_map_.end()) { - return; - } - if (!construction_success) { - CancelBuildingBlob(owning_blob_uuid, - ConvertReferencedBlobErrorToConstructingError(reason), - context.get()); - return; - } - BlobBuildingState* state = state_it->second.get(); - DCHECK_GT(state->num_referenced_blobs_building, 0u); - if (--state->num_referenced_blobs_building == 0) { - context->CompletePendingBlob(state->data_builder); - async_blob_map_.erase(state->data_builder.uuid()); - } -} - -void BlobAsyncBuilderHost::FinishBuildingBlob(BlobBuildingState* state, - BlobStorageContext* context) { - if (!state->referenced_blob_uuids.empty()) { - DCHECK_EQ(0u, state->num_referenced_blobs_building); - state->num_referenced_blobs_building = 0; - // We assume re-entry is not possible, as RunOnConstructionComplete - // will schedule a task when the blob is being built. Thus we can't have the - // case where |num_referenced_blobs_building| reaches 0 in the - // ReferencedBlobFinished method before we're finished looping. - for (const std::string& referenced_uuid : state->referenced_blob_uuids) { - if (context->IsBeingBuilt(referenced_uuid)) { - state->num_referenced_blobs_building++; - context->RunOnConstructionComplete( - referenced_uuid, - base::Bind(&BlobAsyncBuilderHost::ReferencedBlobFinished, - ptr_factory_.GetWeakPtr(), state->data_builder.uuid(), - context->AsWeakPtr())); - } - } - if (state->num_referenced_blobs_building > 0) { - // We wait until referenced blobs are done. - return; - } - } - context->CompletePendingBlob(state->data_builder); - async_blob_map_.erase(state->data_builder.uuid()); -} - -} // namespace storage diff --git a/chromium/storage/browser/blob/blob_async_builder_host.h b/chromium/storage/browser/blob/blob_async_builder_host.h deleted file mode 100644 index b7e82fb3dd7..00000000000 --- a/chromium/storage/browser/blob/blob_async_builder_host.h +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef STORAGE_BROWSER_BLOB_BLOB_ASYNC_BUILDER_HOST_H_ -#define STORAGE_BROWSER_BLOB_BLOB_ASYNC_BUILDER_HOST_H_ - -#include <stddef.h> -#include <stdint.h> - -#include <map> -#include <memory> -#include <set> -#include <string> -#include <vector> - -#include "base/callback.h" -#include "base/files/file.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/shared_memory_handle.h" -#include "base/memory/weak_ptr.h" -#include "storage/browser/blob/blob_async_transport_request_builder.h" -#include "storage/browser/blob/blob_data_builder.h" -#include "storage/browser/blob/blob_transport_result.h" -#include "storage/browser/storage_browser_export.h" -#include "storage/common/blob_storage/blob_item_bytes_request.h" -#include "storage/common/blob_storage/blob_item_bytes_response.h" -#include "storage/common/blob_storage/blob_storage_constants.h" -#include "storage/common/data_element.h" - -namespace base { -class SharedMemory; -} - -namespace storage { -class BlobDataHandle; -class BlobStorageContext; - -// This class -// * holds all blobs that are currently being built asynchronously for a child -// process, -// * sends memory requests through the given callback in |StartBuildingBlob|, -// and uses the BlobTransportResult return value to signify other results, -// * includes all logic for deciding which async transport strategy to use, and -// * handles all blob construction communication with the BlobStorageContext. -// The method |CancelAll| must be called by the consumer, it is not called on -// destruction. -class STORAGE_EXPORT BlobAsyncBuilderHost { - public: - using RequestMemoryCallback = base::Callback<void( - std::unique_ptr<std::vector<storage::BlobItemBytesRequest>>, - std::unique_ptr<std::vector<base::SharedMemoryHandle>>, - std::unique_ptr<std::vector<base::File>>)>; - BlobAsyncBuilderHost(); - ~BlobAsyncBuilderHost(); - - // This registers the given blob internally and adds it to the storage. - // Calling this method also guarentees that the referenced blobs are kept - // alive for the duration of the construction of this blob. - // We return - // * BAD_IPC if we already have the blob registered or if we reference ourself - // in the referenced_blob_uuids. - // * CANCEL_REFERENCED_BLOB_BROKEN if one of the referenced blobs is broken or - // doesn't exist. We store the blob in the context as broken with code - // REFERENCED_BLOB_BROKEN. - // * DONE if we successfully registered the blob. - BlobTransportResult RegisterBlobUUID( - const std::string& uuid, - const std::string& content_type, - const std::string& content_disposition, - const std::set<std::string>& referenced_blob_uuids, - BlobStorageContext* context); - - // This method begins the construction of the blob given the descriptions. The - // blob uuid MUST be building in this object. - // When we return: - // * DONE: The blob is finished transfering right away, and is now - // successfully saved in the context. - // * PENDING_RESPONSES: The async builder host is waiting for responses from - // the renderer. It has called |request_memory| for these responses. - // * CANCEL_*: We have to cancel the blob construction. This function clears - // the blob's internal state and marks the blob as broken in the context - // before returning. - // * BAD_IPC: The arguments were invalid/bad. This marks the blob as broken in - // the context before returning. - BlobTransportResult StartBuildingBlob( - const std::string& uuid, - const std::vector<DataElement>& elements, - size_t memory_available, - BlobStorageContext* context, - const RequestMemoryCallback& request_memory); - - // This is called when we have responses from the Renderer to our calls to - // the request_memory callback above. See above for return value meaning. - BlobTransportResult OnMemoryResponses( - const std::string& uuid, - const std::vector<BlobItemBytesResponse>& responses, - BlobStorageContext* context); - - // This removes the BlobBuildingState from our map and flags the blob as - // broken in the context. This can be called both from our own logic to cancel - // the blob, or from the DispatcherHost (Renderer). The blob MUST be being - // built in this builder. - // Note: if the blob isn't in the context (renderer dereferenced it before we - // finished constructing), then we don't bother touching the context. - void CancelBuildingBlob(const std::string& uuid, - IPCBlobCreationCancelCode code, - BlobStorageContext* context); - - // This clears this object of pending construction. It also handles marking - // blobs that haven't been fully constructed as broken in the context if there - // are any references being held by anyone. We know that they're being used - // by someone else if they still exist in the context. - void CancelAll(BlobStorageContext* context); - - bool IsEmpty() const { return async_blob_map_.empty(); } - - size_t blob_building_count() const { return async_blob_map_.size(); } - - bool IsBeingBuilt(const std::string& key) const { - return async_blob_map_.find(key) != async_blob_map_.end(); - } - - // For testing use only. Must be called before StartBuildingBlob. - void SetMemoryConstantsForTesting(size_t max_ipc_memory_size, - size_t max_shared_memory_size, - uint64_t max_file_size) { - max_ipc_memory_size_ = max_ipc_memory_size; - max_shared_memory_size_ = max_shared_memory_size; - max_file_size_ = max_file_size; - } - - private: - struct BlobBuildingState { - // |refernced_blob_handles| should be all handles generated from the set - // of |refernced_blob_uuids|. - BlobBuildingState( - const std::string& uuid, - std::set<std::string> referenced_blob_uuids, - std::vector<std::unique_ptr<BlobDataHandle>>* referenced_blob_handles); - ~BlobBuildingState(); - - BlobAsyncTransportRequestBuilder request_builder; - BlobDataBuilder data_builder; - std::vector<bool> request_received; - size_t next_request = 0; - size_t num_fulfilled_requests = 0; - std::unique_ptr<base::SharedMemory> shared_memory_block; - // This is the number of requests that have been sent to populate the above - // shared data. We won't ask for more data in shared memory until all - // requests have been responded to. - size_t num_shared_memory_requests = 0; - // Only relevant if num_shared_memory_requests is > 0 - size_t current_shared_memory_handle_index = 0; - - // We save these to double check that the RegisterBlob and StartBuildingBlob - // messages are in sync. - std::set<std::string> referenced_blob_uuids; - // These are the blobs that are referenced in the newly constructed blob. - // We use these to make sure they stay alive while we create the new blob, - // and to wait until any blobs that are not done building are fully - // constructed. - std::vector<std::unique_ptr<BlobDataHandle>> referenced_blob_handles; - - // These are the number of blobs we're waiting for before we can start - // building. - size_t num_referenced_blobs_building = 0; - - BlobAsyncBuilderHost::RequestMemoryCallback request_memory_callback; - }; - - typedef std::map<std::string, std::unique_ptr<BlobBuildingState>> - AsyncBlobMap; - - // This is the 'main loop' of our memory requests to the renderer. - BlobTransportResult ContinueBlobMemoryRequests(const std::string& uuid, - BlobStorageContext* context); - - // This is our callback for when we want to finish the blob and we're waiting - // for blobs we reference to be built. When the last callback occurs, we - // complete the blob and erase our internal state. - void ReferencedBlobFinished(const std::string& uuid, - base::WeakPtr<BlobStorageContext> context, - bool construction_success, - IPCBlobCreationCancelCode reason); - - // This finishes creating the blob in the context, decrements blob references - // that we were holding during construction, and erases our state. - void FinishBuildingBlob(BlobBuildingState* state, - BlobStorageContext* context); - - AsyncBlobMap async_blob_map_; - - // Here for testing. - size_t max_ipc_memory_size_ = kBlobStorageIPCThresholdBytes; - size_t max_shared_memory_size_ = kBlobStorageMaxSharedMemoryBytes; - uint64_t max_file_size_ = kBlobStorageMaxFileSizeBytes; - - base::WeakPtrFactory<BlobAsyncBuilderHost> ptr_factory_; - - DISALLOW_COPY_AND_ASSIGN(BlobAsyncBuilderHost); -}; - -} // namespace storage -#endif // STORAGE_BROWSER_BLOB_BLOB_ASYNC_BUILDER_HOST_H_ diff --git a/chromium/storage/browser/blob/blob_data_builder.cc b/chromium/storage/browser/blob/blob_data_builder.cc index 1397cede297..204a4e81875 100644 --- a/chromium/storage/browser/blob/blob_data_builder.cc +++ b/chromium/storage/browser/blob/blob_data_builder.cc @@ -12,9 +12,12 @@ #include "base/numerics/safe_conversions.h" #include "base/numerics/safe_math.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_util.h" #include "base/time/time.h" #include "net/disk_cache/disk_cache.h" -#include "storage/browser/blob/shareable_file_reference.h" + +using base::FilePath; namespace storage { @@ -24,21 +27,48 @@ const static int kInvalidDiskCacheSideStreamIndex = -1; } // namespace -const char BlobDataBuilder::kAppendFutureFileTemporaryFileName[] = - "kFakeFilenameToBeChangedByPopulateFutureFile"; +const FilePath::CharType kFutureFileName[] = FILE_PATH_LITERAL("_future_name_"); + +/* static */ +base::FilePath BlobDataBuilder::GetFutureFileItemPath(uint64_t file_id) { + std::string file_id_str = base::Uint64ToString(file_id); + return base::FilePath(kFutureFileName) + .AddExtension( + base::FilePath::StringType(file_id_str.begin(), file_id_str.end())); +} -BlobDataBuilder::BlobDataBuilder(const std::string& uuid) : uuid_(uuid) { +/* static */ +bool BlobDataBuilder::IsFutureFileItem(const DataElement& element) { + const FilePath::StringType prefix(kFutureFileName); + // The prefix shouldn't occur unless the user used "AppendFutureFile". We + // DCHECK on AppendFile to make sure no one appends a future file. + return base::StartsWith(element.path().value(), prefix, + base::CompareCase::SENSITIVE); } -BlobDataBuilder::~BlobDataBuilder() { + +/* static */ +uint64_t BlobDataBuilder::GetFutureFileID(const DataElement& element) { + DCHECK(IsFutureFileItem(element)); + uint64_t id = 0; + bool success = + base::StringToUint64(element.path().Extension().substr(1), &id); + DCHECK(success) << element.path().Extension(); + return id; } +BlobDataBuilder::BlobDataBuilder(const std::string& uuid) : uuid_(uuid) {} + +BlobDataBuilder::BlobDataBuilder(BlobDataBuilder&&) = default; +BlobDataBuilder& BlobDataBuilder::operator=(BlobDataBuilder&&) = default; +BlobDataBuilder::~BlobDataBuilder() {} + void BlobDataBuilder::AppendIPCDataElement(const DataElement& ipc_data) { uint64_t length = ipc_data.length(); switch (ipc_data.type()) { case DataElement::TYPE_BYTES: DCHECK(!ipc_data.offset()); AppendData(ipc_data.bytes(), - base::checked_cast<size_t, uint64_t>(length)); + base::checked_cast<size_t>(length)); break; case DataElement::TYPE_FILE: AppendFile(ipc_data.path(), ipc_data.offset(), length, @@ -83,7 +113,7 @@ bool BlobDataBuilder::PopulateFutureData(size_t index, size_t length) { DCHECK_LT(index, items_.size()); DCHECK(data); - DataElement* element = items_.at(index)->data_element_ptr(); + DataElement* element = items_[index]->data_element_ptr(); // We lazily allocate our data buffer by waiting until the first // PopulateFutureData call. @@ -110,12 +140,13 @@ bool BlobDataBuilder::PopulateFutureData(size_t index, return true; } -size_t BlobDataBuilder::AppendFutureFile(uint64_t offset, uint64_t length) { +size_t BlobDataBuilder::AppendFutureFile(uint64_t offset, + uint64_t length, + uint64_t file_id) { CHECK_NE(length, 0ull); std::unique_ptr<DataElement> element(new DataElement()); - element->SetToFilePathRange(base::FilePath::FromUTF8Unsafe(std::string( - kAppendFutureFileTemporaryFileName)), - offset, length, base::Time()); + element->SetToFilePathRange(GetFutureFileItemPath(file_id), offset, length, + base::Time()); items_.push_back(new BlobDataItem(std::move(element))); return items_.size() - 1; } @@ -125,32 +156,31 @@ bool BlobDataBuilder::PopulateFutureFile( const scoped_refptr<ShareableFileReference>& file_reference, const base::Time& expected_modification_time) { DCHECK_LT(index, items_.size()); - DataElement* old_element = items_.at(index)->data_element_ptr(); + DataElement* element = items_[index]->data_element_ptr(); - if (old_element->type() != DataElement::TYPE_FILE) { + if (element->type() != DataElement::TYPE_FILE) { DVLOG(1) << "Invalid item type."; return false; - } else if (old_element->path().AsUTF8Unsafe() != - std::string(kAppendFutureFileTemporaryFileName)) { + } else if (!IsFutureFileItem(*element)) { DVLOG(1) << "Item not created by AppendFutureFile"; return false; } - uint64_t length = old_element->length(); - uint64_t offset = old_element->offset(); - std::unique_ptr<DataElement> element(new DataElement()); + uint64_t length = element->length(); + uint64_t offset = element->offset(); + items_[index]->data_handle_ = std::move(file_reference); element->SetToFilePathRange(file_reference->path(), offset, length, expected_modification_time); - items_[index] = new BlobDataItem(std::move(element), file_reference); return true; } -void BlobDataBuilder::AppendFile(const base::FilePath& file_path, +void BlobDataBuilder::AppendFile(const FilePath& file_path, uint64_t offset, uint64_t length, const base::Time& expected_modification_time) { std::unique_ptr<DataElement> element(new DataElement()); element->SetToFilePathRange(file_path, offset, length, expected_modification_time); + DCHECK(!IsFutureFileItem(*element)) << file_path.value(); items_.push_back(new BlobDataItem(std::move(element), ShareableFileReference::Get(file_path))); } diff --git a/chromium/storage/browser/blob/blob_data_builder.h b/chromium/storage/browser/blob/blob_data_builder.h index f304f446555..088ceb755d7 100644 --- a/chromium/storage/browser/blob/blob_data_builder.h +++ b/chromium/storage/browser/blob/blob_data_builder.h @@ -16,6 +16,7 @@ #include "base/memory/ref_counted.h" #include "storage/browser/blob/blob_data_item.h" #include "storage/browser/blob/blob_data_snapshot.h" +#include "storage/browser/blob/shareable_file_reference.h" #include "storage/browser/storage_browser_export.h" namespace disk_cache { @@ -26,15 +27,25 @@ namespace storage { class BlobStorageContext; class ShareableFileReference; +// This class is used to build blobs. It also facilitates the operation of +// 'pending' data, where the user knows the size and existence of a file or +// bytes item, but we don't have the memory or file yet. See AppendFuture* and +// PopulateFuture* methods for more description. Use +// BlobDataHandle::GetBlobStatus to check for an error after creating the blob. class STORAGE_EXPORT BlobDataBuilder { public: using DataHandle = BlobDataItem::DataHandle; + // Visible for testing. + static base::FilePath GetFutureFileItemPath(uint64_t file_id); - // This is the filename used for the temporary file items added by - // AppendFutureFile. - const static char kAppendFutureFileTemporaryFileName[]; + // Returns if the given item was created by AppendFutureFile. + static bool IsFutureFileItem(const DataElement& element); + // Returns |file_id| given to AppendFutureFile. + static uint64_t GetFutureFileID(const DataElement& element); explicit BlobDataBuilder(const std::string& uuid); + BlobDataBuilder(BlobDataBuilder&&); + BlobDataBuilder& operator=(BlobDataBuilder&&); ~BlobDataBuilder(); const std::string& uuid() const { return uuid_; } @@ -73,9 +84,12 @@ class STORAGE_EXPORT BlobDataBuilder { // Adds an item that is flagged for future data population. Use // 'PopulateFutureFile' to set the file path and expected modification time // of this file. Returns the index of the item (to be used in - // PopulateFutureFile). The temporary filename used by this method is - // kAppendFutureFileTemporaryFileName. |length| cannot be 0. - size_t AppendFutureFile(uint64_t offset, uint64_t length); + // PopulateFutureFile). |length| cannot be 0. + // Data for multiple items can be stored in the same 'future' file, just at + // different offsets and lengths. The |file_id| is used to differentiate + // between different 'future' files that will be used to store data for these + // items. + size_t AppendFutureFile(uint64_t offset, uint64_t length, uint64_t file_id); // Populates a part of an item previously allocated with AppendFutureFile. // Returns true if: @@ -106,6 +120,7 @@ class STORAGE_EXPORT BlobDataBuilder { void AppendDiskCacheEntry(const scoped_refptr<DataHandle>& data_handle, disk_cache::Entry* disk_cache_entry, int disk_cache_stream_index); + // The content of the side data is accessible with BlobReader::ReadSideData(). void AppendDiskCacheEntryWithSideData( const scoped_refptr<DataHandle>& data_handle, @@ -124,12 +139,14 @@ class STORAGE_EXPORT BlobDataBuilder { void Clear(); private: + friend class BlobMemoryControllerTest; friend class BlobStorageContext; - friend class BlobAsyncBuilderHostTest; friend bool operator==(const BlobDataBuilder& a, const BlobDataBuilder& b); friend bool operator==(const BlobDataSnapshot& a, const BlobDataBuilder& b); friend STORAGE_EXPORT void PrintTo(const BlobDataBuilder& x, ::std::ostream* os); + FRIEND_TEST_ALL_PREFIXES(BlobDataBuilderTest, TestFutureFiles); + FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, BuildBlobFuzzy); std::string uuid_; std::string content_type_; diff --git a/chromium/storage/browser/blob/blob_data_handle.cc b/chromium/storage/browser/blob/blob_data_handle.cc index 55e63a1b620..8d29cca76f3 100644 --- a/chromium/storage/browser/blob/blob_data_handle.cc +++ b/chromium/storage/browser/blob/blob_data_handle.cc @@ -26,8 +26,6 @@ #include "url/gurl.h" namespace storage { -using BlobState = BlobStorageRegistry::BlobState; - namespace { class FileStreamReaderProviderImpl @@ -67,10 +65,12 @@ BlobDataHandle::BlobDataHandleShared::BlobDataHandleShared( const std::string& uuid, const std::string& content_type, const std::string& content_disposition, + uint64_t size, BlobStorageContext* context) : uuid_(uuid), content_type_(content_type), content_disposition_(content_disposition), + size_(size), context_(context->AsWeakPtr()) { context_->IncrementBlobRefCount(uuid); } @@ -92,21 +92,20 @@ BlobDataHandle::BlobDataHandleShared::~BlobDataHandleShared() { BlobDataHandle::BlobDataHandle(const std::string& uuid, const std::string& content_type, const std::string& content_disposition, + uint64_t size, BlobStorageContext* context, base::SequencedTaskRunner* io_task_runner) : io_task_runner_(io_task_runner), shared_(new BlobDataHandleShared(uuid, content_type, content_disposition, + size, context)) { DCHECK(io_task_runner_.get()); DCHECK(io_task_runner_->RunsTasksOnCurrentThread()); } -BlobDataHandle::BlobDataHandle(const BlobDataHandle& other) { - io_task_runner_ = other.io_task_runner_; - shared_ = other.shared_; -} +BlobDataHandle::BlobDataHandle(const BlobDataHandle& other) = default; BlobDataHandle::~BlobDataHandle() { if (!io_task_runner_->RunsTasksOnCurrentThread()) { @@ -117,25 +116,31 @@ BlobDataHandle::~BlobDataHandle() { } } +BlobDataHandle& BlobDataHandle::operator=( + const BlobDataHandle& other) = default; + bool BlobDataHandle::IsBeingBuilt() const { DCHECK(io_task_runner_->RunsTasksOnCurrentThread()); if (!shared_->context_) return false; - return shared_->context_->IsBeingBuilt(shared_->uuid_); + return BlobStatusIsPending(GetBlobStatus()); } bool BlobDataHandle::IsBroken() const { DCHECK(io_task_runner_->RunsTasksOnCurrentThread()); if (!shared_->context_) return true; - return shared_->context_->IsBroken(shared_->uuid_); + return BlobStatusIsError(GetBlobStatus()); } -void BlobDataHandle::RunOnConstructionComplete( - const BlobConstructedCallback& done) { +BlobStatus BlobDataHandle::GetBlobStatus() const { + return shared_->context_->GetBlobStatus(shared_->uuid_); +} + +void BlobDataHandle::RunOnConstructionComplete(const BlobStatusCallback& done) { DCHECK(io_task_runner_->RunsTasksOnCurrentThread()); if (!shared_->context_.get()) { - done.Run(false, IPCBlobCreationCancelCode::UNKNOWN); + done.Run(BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS); return; } shared_->context_->RunOnConstructionComplete(shared_->uuid_, done); @@ -160,4 +165,8 @@ const std::string& BlobDataHandle::content_disposition() const { return shared_->content_disposition_; } +uint64_t BlobDataHandle::size() const { + return shared_->size_; +} + } // namespace storage diff --git a/chromium/storage/browser/blob/blob_data_handle.h b/chromium/storage/browser/blob/blob_data_handle.h index 3f4529346b1..4a46f934400 100644 --- a/chromium/storage/browser/blob/blob_data_handle.h +++ b/chromium/storage/browser/blob/blob_data_handle.h @@ -5,6 +5,7 @@ #ifndef STORAGE_BROWSER_BLOB_BLOB_DATA_HANDLE_H_ #define STORAGE_BROWSER_BLOB_BLOB_DATA_HANDLE_H_ +#include <limits> #include <memory> #include <string> @@ -39,14 +40,14 @@ class FileSystemContext; class STORAGE_EXPORT BlobDataHandle : public base::SupportsUserData::Data { public: - // True means the blob was constructed successfully, and false means that - // there was an error, which is reported in the second argument. - using BlobConstructedCallback = - base::Callback<void(bool, IPCBlobCreationCancelCode)>; + static constexpr uint64_t kUnknownSize = std::numeric_limits<uint64_t>::max(); BlobDataHandle(const BlobDataHandle& other); // May be copied on any thread. ~BlobDataHandle() override; // May be deleted on any thread. + // Assignment operator matching copy constructor. + BlobDataHandle& operator=(const BlobDataHandle& other); + // Returns if this blob is still constructing. If so, one can use the // RunOnConstructionComplete to wait. // Must be called on IO thread. @@ -56,13 +57,17 @@ class STORAGE_EXPORT BlobDataHandle // Must be called on IO thread. bool IsBroken() const; + // Returns the broken reason if this blob is broken. + // Must be called on IO thread. + BlobStatus GetBlobStatus() const; + // The callback will be run on the IO thread when construction of the blob // is complete. If construction is already complete, then the task is run // immediately on the current message loop (i.e. IO thread). // Must be called on IO thread. Returns if construction successful. // Calling this multiple times results in registering multiple // completion callbacks. - void RunOnConstructionComplete(const BlobConstructedCallback& done); + void RunOnConstructionComplete(const BlobStatusCallback& done); // A BlobReader is used to read the data from the blob. This object is // intended to be transient and should not be stored for any extended period @@ -77,6 +82,9 @@ class STORAGE_EXPORT BlobDataHandle const std::string& content_type() const; // May be accessed on any thread. const std::string& content_disposition() const; + // May be accessed on any thread. In rare cases where the blob is created + // as a file from javascript, this will be kUnknownSize. + uint64_t size() const; // This call and the destruction of the returned snapshot must be called // on the IO thread. If the blob is broken, then we return a nullptr here. @@ -94,6 +102,7 @@ class STORAGE_EXPORT BlobDataHandle BlobDataHandleShared(const std::string& uuid, const std::string& content_type, const std::string& content_disposition, + uint64_t size, BlobStorageContext* context); private: @@ -106,6 +115,7 @@ class STORAGE_EXPORT BlobDataHandle const std::string uuid_; const std::string content_type_; const std::string content_disposition_; + const uint64_t size_; base::WeakPtr<BlobStorageContext> context_; DISALLOW_COPY_AND_ASSIGN(BlobDataHandleShared); @@ -115,6 +125,7 @@ class STORAGE_EXPORT BlobDataHandle BlobDataHandle(const std::string& uuid, const std::string& content_type, const std::string& content_disposition, + uint64_t size, BlobStorageContext* context, base::SequencedTaskRunner* io_task_runner); diff --git a/chromium/storage/browser/blob/blob_data_item.h b/chromium/storage/browser/blob/blob_data_item.h index a711a4caf3b..bc03269212d 100644 --- a/chromium/storage/browser/blob/blob_data_item.h +++ b/chromium/storage/browser/blob/blob_data_item.h @@ -21,7 +21,9 @@ class Entry; namespace storage { class BlobDataBuilder; +class BlobMemoryController; class BlobStorageContext; +class DataElement; // Ref counted blob item. This class owns the backing data of the blob item. The // backing data is immutable, and cannot change after creation. The purpose of @@ -65,7 +67,11 @@ class STORAGE_EXPORT BlobDataItem : public base::RefCounted<BlobDataItem> { private: friend class BlobDataBuilder; + friend class BlobMemoryController; friend class BlobStorageContext; + friend struct BlobSlice; + friend class BlobSliceTest; + friend class BlobFlattenerTest; friend class base::RefCounted<BlobDataItem>; friend STORAGE_EXPORT void PrintTo(const BlobDataItem& x, ::std::ostream* os); @@ -77,6 +83,7 @@ class STORAGE_EXPORT BlobDataItem : public base::RefCounted<BlobDataItem> { disk_cache::Entry* entry, int disk_cache_stream_index, int disk_cache_side_stream_index); + virtual ~BlobDataItem(); std::unique_ptr<DataElement> item_; diff --git a/chromium/storage/browser/blob/blob_entry.cc b/chromium/storage/browser/blob/blob_entry.cc new file mode 100644 index 00000000000..094eb365641 --- /dev/null +++ b/chromium/storage/browser/blob/blob_entry.cc @@ -0,0 +1,69 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <utility> + +#include "base/callback.h" +#include "base/containers/hash_tables.h" +#include "base/metrics/histogram.h" +#include "storage/browser/blob/blob_data_handle.h" +#include "storage/browser/blob/blob_data_item.h" +#include "storage/browser/blob/blob_entry.h" +#include "storage/browser/blob/shareable_blob_data_item.h" +#include "storage/common/data_element.h" + +namespace storage { + +BlobEntry::ItemCopyEntry::ItemCopyEntry( + scoped_refptr<ShareableBlobDataItem> source_item, + size_t source_item_offset, + scoped_refptr<ShareableBlobDataItem> dest_item) + : source_item(std::move(source_item)), + source_item_offset(source_item_offset), + dest_item(std::move(dest_item)) {} + +BlobEntry::ItemCopyEntry::ItemCopyEntry(ItemCopyEntry&& other) = default; +BlobEntry::ItemCopyEntry& BlobEntry::ItemCopyEntry::operator=( + BlobEntry::ItemCopyEntry&& rhs) = default; +BlobEntry::ItemCopyEntry::~ItemCopyEntry() {} + +BlobEntry::BuildingState::BuildingState( + bool transport_items_present, + TransportAllowedCallback transport_allowed_callback, + size_t num_building_dependent_blobs) + : transport_items_present(transport_items_present), + transport_allowed_callback(transport_allowed_callback), + num_building_dependent_blobs(num_building_dependent_blobs) {} + +BlobEntry::BuildingState::~BuildingState() {} + +BlobEntry::BlobEntry(const std::string& content_type, + const std::string& content_disposition) + : content_type_(content_type), content_disposition_(content_disposition) {} +BlobEntry::~BlobEntry() {} + +void BlobEntry::AppendSharedBlobItem( + scoped_refptr<ShareableBlobDataItem> item) { + DCHECK(item); + if (!items_.empty()) { + offsets_.push_back(size_); + } + size_ += item->item()->length(); + items_.push_back(std::move(item)); +} + +const std::vector<scoped_refptr<ShareableBlobDataItem>>& BlobEntry::items() + const { + return items_; +} + +void BlobEntry::ClearItems() { + items_.clear(); +} + +void BlobEntry::ClearOffsets() { + offsets_.clear(); +} + +} // namespace storage diff --git a/chromium/storage/browser/blob/blob_entry.h b/chromium/storage/browser/blob/blob_entry.h new file mode 100644 index 00000000000..3c3869e89f4 --- /dev/null +++ b/chromium/storage/browser/blob/blob_entry.h @@ -0,0 +1,164 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef STORAGE_BROWSER_BLOB_BLOB_ENTRY_H_ +#define STORAGE_BROWSER_BLOB_BLOB_ENTRY_H_ + +#include <stddef.h> + +#include <memory> +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "storage/browser/blob/blob_memory_controller.h" +#include "storage/browser/storage_browser_export.h" + +namespace storage { +class BlobDataHandle; +class ShareableBlobDataItem; +class ViewBlobInternalsJob; + +// This class represents a blob in BlobStorageRegistry. We export this only for +// unit tests. +class STORAGE_EXPORT BlobEntry { + public: + using TransportAllowedCallback = + base::Callback<void(BlobStatus, + std::vector<BlobMemoryController::FileCreationInfo>)>; + + // This records a copy from a referenced blob. When we finish building our + // blob we perform all of these copies. + struct STORAGE_EXPORT ItemCopyEntry { + ItemCopyEntry(scoped_refptr<ShareableBlobDataItem> source_item, + size_t source_item_offset, + scoped_refptr<ShareableBlobDataItem> dest_item); + ~ItemCopyEntry(); + ItemCopyEntry(ItemCopyEntry&& other); + BlobEntry::ItemCopyEntry& operator=(BlobEntry::ItemCopyEntry&& rhs); + + scoped_refptr<ShareableBlobDataItem> source_item; + size_t source_item_offset = 0; + scoped_refptr<ShareableBlobDataItem> dest_item; + + private: + DISALLOW_COPY_AND_ASSIGN(ItemCopyEntry); + }; + + // This keeps track of our building state for our blob. While building, four + // things can be happening mostly simultaneously: + // 1. Waiting for quota to be reserved for memory needed (PENDING_QUOTA) + // 2. Waiting for user population of data after quota (PENDING_TRANSPORT) + // 3. Waiting for blobs we reference to complete (PENDING_INTERNALS) + struct STORAGE_EXPORT BuildingState { + // |transport_allowed_callback| is not null when data needs population. See + // BlobStorageContext::BuildBlob for when the callback is called. + BuildingState(bool transport_items_present, + TransportAllowedCallback transport_allowed_callback, + size_t num_building_dependent_blobs); + ~BuildingState(); + + const bool transport_items_present; + // We can have trasnport data that's either populated or unpopulated. If we + // need population, this is populated. + TransportAllowedCallback transport_allowed_callback; + std::vector<ShareableBlobDataItem*> transport_items; + + // Stores all blobs that we're depending on for building. This keeps the + // blobs alive while we build our blob. + std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs; + size_t num_building_dependent_blobs; + + base::WeakPtr<BlobMemoryController::QuotaAllocationTask> + memory_quota_request; + + // These are copies from a referenced blob item to our blob items. Some of + // these entries may have changed from bytes to files if they were paged. + std::vector<ItemCopyEntry> copies; + + // When our blob finishes building these callbacks are called. + std::vector<BlobStatusCallback> build_completion_callbacks; + + private: + DISALLOW_COPY_AND_ASSIGN(BuildingState); + }; + + BlobEntry(const std::string& content_type, + const std::string& content_disposition); + ~BlobEntry(); + + // Appends the given shared blob data item to this object. + void AppendSharedBlobItem(scoped_refptr<ShareableBlobDataItem> item); + + // Returns if we're a pending blob that can finish building. + bool CanFinishBuilding() const { + return status_ == BlobStatus::PENDING_INTERNALS && + building_state_->num_building_dependent_blobs == 0; + } + + BlobStatus status() const { return status_; } + + size_t refcount() const { return refcount_; } + + const std::string& content_type() const { return content_type_; } + + const std::string& content_disposition() const { + return content_disposition_; + } + + // Total size of this blob in bytes. + uint64_t total_size() const { return size_; }; + + // We record the cumulative offsets of each blob item (except for the first, + // which would always be 0) to enable binary searching for an item at a + // specific byte offset. + const std::vector<uint64_t>& offsets() const { return offsets_; } + + const std::vector<scoped_refptr<ShareableBlobDataItem>>& items() const; + + protected: + friend class BlobStorageContext; + + void IncrementRefCount() { ++refcount_; } + void DecrementRefCount() { --refcount_; } + + void set_status(BlobStatus status) { status_ = status; } + void set_size(uint64_t size) { size_ = size; } + + void ClearItems(); + void ClearOffsets(); + + void set_building_state(std::unique_ptr<BuildingState> building_state) { + building_state_ = std::move(building_state); + } + + private: + BlobStatus status_ = BlobStatus::PENDING_QUOTA; + size_t refcount_ = 0; + + // Metadata. + std::string content_type_; + std::string content_disposition_; + + std::vector<scoped_refptr<ShareableBlobDataItem>> items_; + + // Size in bytes. IFF we're a single file then this can be uint64_max. + uint64_t size_ = 0; + + // Only populated if len(items_) > 1. Used for binary search. + // Since the offset of the first item is always 0, we exclude this. + std::vector<uint64_t> offsets_; + + // Only populated if our status is PENDING_*. + std::unique_ptr<BuildingState> building_state_; + + DISALLOW_COPY_AND_ASSIGN(BlobEntry); +}; + +} // namespace storage +#endif // STORAGE_BROWSER_BLOB_BLOB_ENTRY_H_ diff --git a/chromium/storage/browser/blob/blob_memory_controller.cc b/chromium/storage/browser/blob/blob_memory_controller.cc new file mode 100644 index 00000000000..6cc6c3f291f --- /dev/null +++ b/chromium/storage/browser/blob/blob_memory_controller.cc @@ -0,0 +1,747 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "storage/browser/blob/blob_memory_controller.h" + +#include <algorithm> + +#include "base/callback.h" +#include "base/callback_helpers.h" +#include "base/containers/small_map.h" +#include "base/files/file_util.h" +#include "base/guid.h" +#include "base/location.h" +#include "base/memory/ptr_util.h" +#include "base/metrics/histogram_macros.h" +#include "base/numerics/safe_conversions.h" +#include "base/numerics/safe_math.h" +#include "base/single_thread_task_runner.h" +#include "base/stl_util.h" +#include "base/strings/string_number_conversions.h" +#include "base/task_runner.h" +#include "base/task_runner_util.h" +#include "base/threading/thread_restrictions.h" +#include "base/time/time.h" +#include "base/trace_event/trace_event.h" +#include "base/tuple.h" +#include "storage/browser/blob/blob_data_builder.h" +#include "storage/browser/blob/blob_data_item.h" +#include "storage/browser/blob/shareable_blob_data_item.h" +#include "storage/browser/blob/shareable_file_reference.h" +#include "storage/common/data_element.h" + +using base::File; +using base::FilePath; + +namespace storage { +namespace { +using FileCreationInfo = BlobMemoryController::FileCreationInfo; +using MemoryAllocation = BlobMemoryController::MemoryAllocation; +using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask; + +File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) { + File::Error error = File::FILE_OK; + base::CreateDirectoryAndGetError(blob_storage_dir, &error); + UMA_HISTOGRAM_ENUMERATION("Storage.Blob.CreateDirectoryResult", -error, + -File::FILE_ERROR_MAX); + return error; +} + +void DestructFile(File infos_without_references) {} + +// Used for new unpopulated file items. Caller must populate file reference in +// returned FileCreationInfos. +std::pair<std::vector<FileCreationInfo>, File::Error> CreateEmptyFiles( + const FilePath& blob_storage_dir, + scoped_refptr<base::TaskRunner> file_task_runner, + std::vector<base::FilePath> file_paths) { + base::ThreadRestrictions::AssertIOAllowed(); + + File::Error dir_create_status = CreateBlobDirectory(blob_storage_dir); + if (dir_create_status != File::FILE_OK) + return std::make_pair(std::vector<FileCreationInfo>(), dir_create_status); + + std::vector<FileCreationInfo> result; + for (const base::FilePath& file_path : file_paths) { + FileCreationInfo creation_info; + // Try to open our file. + File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE); + creation_info.path = std::move(file_path); + creation_info.file_deletion_runner = file_task_runner; + creation_info.error = file.error_details(); + if (creation_info.error != File::FILE_OK) { + return std::make_pair(std::vector<FileCreationInfo>(), + creation_info.error); + } + + // Grab the file info to get the "last modified" time and store the file. + File::Info file_info; + bool success = file.GetInfo(&file_info); + creation_info.error = success ? File::FILE_OK : File::FILE_ERROR_FAILED; + if (!success) { + return std::make_pair(std::vector<FileCreationInfo>(), + creation_info.error); + } + creation_info.file = std::move(file); + + result.push_back(std::move(creation_info)); + } + return std::make_pair(std::move(result), File::FILE_OK); +} + +// Used to evict multiple memory items out to a single file. Caller must +// populate file reference in returned FileCreationInfo. +FileCreationInfo CreateFileAndWriteItems( + const FilePath& blob_storage_dir, + const FilePath& file_path, + scoped_refptr<base::TaskRunner> file_task_runner, + std::vector<DataElement*> items, + size_t total_size_bytes) { + DCHECK_NE(0u, total_size_bytes); + UMA_HISTOGRAM_MEMORY_KB("Storage.Blob.PageFileSize", total_size_bytes / 1024); + base::ThreadRestrictions::AssertIOAllowed(); + + FileCreationInfo creation_info; + creation_info.file_deletion_runner = std::move(file_task_runner); + creation_info.error = CreateBlobDirectory(blob_storage_dir); + if (creation_info.error != File::FILE_OK) + return creation_info; + + // Create the page file. + File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE); + creation_info.path = file_path; + creation_info.error = file.error_details(); + if (creation_info.error != File::FILE_OK) + return creation_info; + + // Write data. + file.SetLength(total_size_bytes); + int bytes_written = 0; + for (DataElement* element : items) { + DCHECK_EQ(DataElement::TYPE_BYTES, element->type()); + size_t length = base::checked_cast<size_t>(element->length()); + size_t bytes_left = length; + while (bytes_left > 0) { + bytes_written = + file.WriteAtCurrentPos(element->bytes() + (length - bytes_left), + base::saturated_cast<int>(bytes_left)); + if (bytes_written < 0) + break; + DCHECK_LE(static_cast<size_t>(bytes_written), bytes_left); + bytes_left -= bytes_written; + } + if (bytes_written < 0) + break; + } + + File::Info info; + bool success = file.GetInfo(&info); + creation_info.error = + bytes_written < 0 || !success ? File::FILE_ERROR_FAILED : File::FILE_OK; + creation_info.last_modified = info.last_modified; + return creation_info; +} + +uint64_t GetTotalSizeAndFileSizes( + const std::vector<scoped_refptr<ShareableBlobDataItem>>& + unreserved_file_items, + std::vector<uint64_t>* file_sizes_output) { + uint64_t total_size_output = 0; + base::SmallMap<std::map<uint64_t, uint64_t>> file_id_to_sizes; + for (const auto& item : unreserved_file_items) { + const DataElement& element = item->item()->data_element(); + uint64_t file_id = BlobDataBuilder::GetFutureFileID(element); + auto it = file_id_to_sizes.find(file_id); + if (it != file_id_to_sizes.end()) + it->second = std::max(it->second, element.offset() + element.length()); + else + file_id_to_sizes[file_id] = element.offset() + element.length(); + total_size_output += element.length(); + } + for (const auto& size_pair : file_id_to_sizes) { + file_sizes_output->push_back(size_pair.second); + } + return total_size_output; +} + +} // namespace + +FileCreationInfo::FileCreationInfo() {} +FileCreationInfo::~FileCreationInfo() { + if (file.IsValid()) { + DCHECK(file_deletion_runner); + file_deletion_runner->PostTask( + FROM_HERE, base::Bind(&DestructFile, base::Passed(&file))); + } +} +FileCreationInfo::FileCreationInfo(FileCreationInfo&&) = default; +FileCreationInfo& FileCreationInfo::operator=(FileCreationInfo&&) = default; + +MemoryAllocation::MemoryAllocation( + base::WeakPtr<BlobMemoryController> controller, + uint64_t item_id, + size_t length) + : controller(controller), item_id(item_id), length(length) {} + +MemoryAllocation::~MemoryAllocation() { + if (controller) + controller->RevokeMemoryAllocation(item_id, length); +} + +BlobMemoryController::QuotaAllocationTask::~QuotaAllocationTask() {} + +class BlobMemoryController::MemoryQuotaAllocationTask + : public BlobMemoryController::QuotaAllocationTask { + public: + MemoryQuotaAllocationTask( + BlobMemoryController* controller, + size_t quota_request_size, + std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items, + MemoryQuotaRequestCallback done_callback) + : controller_(controller), + pending_items_(std::move(pending_items)), + done_callback_(std::move(done_callback)), + allocation_size_(quota_request_size), + weak_factory_(this) {} + + ~MemoryQuotaAllocationTask() override = default; + + void RunDoneCallback(bool success) { + // Make sure we clear the weak pointers we gave to the caller beforehand. + weak_factory_.InvalidateWeakPtrs(); + if (success) + controller_->GrantMemoryAllocations(&pending_items_, allocation_size_); + done_callback_.Run(success); + } + + base::WeakPtr<QuotaAllocationTask> GetWeakPtr() { + return weak_factory_.GetWeakPtr(); + } + + void Cancel() override { + DCHECK_GE(controller_->pending_memory_quota_total_size_, allocation_size_); + controller_->pending_memory_quota_total_size_ -= allocation_size_; + // This call destroys this object. + controller_->pending_memory_quota_tasks_.erase(my_list_position_); + } + + // The my_list_position_ iterator is stored so that we can remove ourself + // from the task list when we are cancelled. + void set_my_list_position( + PendingMemoryQuotaTaskList::iterator my_list_position) { + my_list_position_ = my_list_position; + } + + size_t allocation_size() const { return allocation_size_; } + + private: + BlobMemoryController* controller_; + std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items_; + MemoryQuotaRequestCallback done_callback_; + + size_t allocation_size_; + PendingMemoryQuotaTaskList::iterator my_list_position_; + + base::WeakPtrFactory<MemoryQuotaAllocationTask> weak_factory_; + DISALLOW_COPY_AND_ASSIGN(MemoryQuotaAllocationTask); +}; + +class BlobMemoryController::FileQuotaAllocationTask + : public BlobMemoryController::QuotaAllocationTask { + public: + // We post a task to create the file for the items right away. + FileQuotaAllocationTask( + BlobMemoryController* memory_controller, + std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items, + const FileQuotaRequestCallback& done_callback) + : controller_(memory_controller), + done_callback_(done_callback), + weak_factory_(this) { + // Get the file sizes and total size. + std::vector<uint64_t> file_sizes; + uint64_t total_size = + GetTotalSizeAndFileSizes(unreserved_file_items, &file_sizes); + DCHECK_LE(total_size, controller_->GetAvailableFileSpaceForBlobs()); + allocation_size_ = total_size; + + // Check & set our item states. + for (auto& shareable_item : unreserved_file_items) { + DCHECK_EQ(ShareableBlobDataItem::QUOTA_NEEDED, shareable_item->state()); + DCHECK_EQ(DataElement::TYPE_FILE, shareable_item->item()->type()); + shareable_item->set_state(ShareableBlobDataItem::QUOTA_REQUESTED); + } + pending_items_ = std::move(unreserved_file_items); + + // Increment disk usage and create our file references. + controller_->disk_used_ += allocation_size_; + std::vector<base::FilePath> file_paths; + std::vector<scoped_refptr<ShareableFileReference>> references; + for (size_t i = 0; i < file_sizes.size(); i++) { + file_paths.push_back(controller_->GenerateNextPageFileName()); + references.push_back(ShareableFileReference::GetOrCreate( + file_paths.back(), ShareableFileReference::DELETE_ON_FINAL_RELEASE, + controller_->file_runner_.get())); + references.back()->AddFinalReleaseCallback( + base::Bind(&BlobMemoryController::OnBlobFileDelete, + controller_->weak_factory_.GetWeakPtr(), file_sizes[i])); + } + + // Send file creation task to file thread. + base::PostTaskAndReplyWithResult( + controller_->file_runner_.get(), FROM_HERE, + base::Bind(&CreateEmptyFiles, controller_->blob_storage_dir_, + controller_->file_runner_, base::Passed(&file_paths)), + base::Bind(&FileQuotaAllocationTask::OnCreateEmptyFiles, + weak_factory_.GetWeakPtr(), base::Passed(&references))); + controller_->RecordTracingCounters(); + } + ~FileQuotaAllocationTask() override {} + + void RunDoneCallback(bool success, std::vector<FileCreationInfo> file_info) { + // Make sure we clear the weak pointers we gave to the caller beforehand. + weak_factory_.InvalidateWeakPtrs(); + + // We want to destroy this object on the exit of this method if we were + // successful. + std::unique_ptr<FileQuotaAllocationTask> this_object; + if (success) { + for (auto& item : pending_items_) { + item->set_state(ShareableBlobDataItem::QUOTA_GRANTED); + } + this_object = std::move(*my_list_position_); + controller_->pending_file_quota_tasks_.erase(my_list_position_); + } + + done_callback_.Run(success, std::move(file_info)); + } + + base::WeakPtr<QuotaAllocationTask> GetWeakPtr() { + return weak_factory_.GetWeakPtr(); + } + + void Cancel() override { + // This call destroys this object. We rely on ShareableFileReference's + // final release callback for disk_usage_ accounting. + controller_->pending_file_quota_tasks_.erase(my_list_position_); + } + + void OnCreateEmptyFiles( + std::vector<scoped_refptr<ShareableFileReference>> references, + std::pair<std::vector<FileCreationInfo>, File::Error> files_and_error) { + auto& files = files_and_error.first; + if (files.empty()) { + DCHECK_GE(controller_->disk_used_, allocation_size_); + controller_->disk_used_ -= allocation_size_; + // This will call our callback and delete the object correctly. + controller_->DisableFilePaging(files_and_error.second); + return; + } + DCHECK_EQ(files.size(), references.size()); + for (size_t i = 0; i < files.size(); i++) { + files[i].file_reference = std::move(references[i]); + } + RunDoneCallback(true, std::move(files)); + } + + // The my_list_position_ iterator is stored so that we can remove ourself + // from the task list when we are cancelled. + void set_my_list_position( + PendingFileQuotaTaskList::iterator my_list_position) { + my_list_position_ = my_list_position; + } + + private: + BlobMemoryController* controller_; + std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items_; + scoped_refptr<base::TaskRunner> file_runner_; + FileQuotaRequestCallback done_callback_; + + uint64_t allocation_size_; + PendingFileQuotaTaskList::iterator my_list_position_; + + base::WeakPtrFactory<FileQuotaAllocationTask> weak_factory_; + DISALLOW_COPY_AND_ASSIGN(FileQuotaAllocationTask); +}; + +BlobMemoryController::BlobMemoryController( + const base::FilePath& storage_directory, + scoped_refptr<base::TaskRunner> file_runner) + : file_paging_enabled_(file_runner.get() != nullptr), + blob_storage_dir_(storage_directory), + file_runner_(std::move(file_runner)), + populated_memory_items_( + base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT), + weak_factory_(this) {} + +BlobMemoryController::~BlobMemoryController() {} + +void BlobMemoryController::DisableFilePaging(base::File::Error reason) { + UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingDisabled", -reason, + -File::FILE_ERROR_MAX); + file_paging_enabled_ = false; + in_flight_memory_used_ = 0; + items_paging_to_file_.clear(); + pending_evictions_ = 0; + pending_memory_quota_total_size_ = 0; + populated_memory_items_.Clear(); + populated_memory_items_bytes_ = 0; + file_runner_ = nullptr; + + PendingMemoryQuotaTaskList old_memory_tasks; + PendingFileQuotaTaskList old_file_tasks; + std::swap(old_memory_tasks, pending_memory_quota_tasks_); + std::swap(old_file_tasks, pending_file_quota_tasks_); + + // Don't call the callbacks until we have a consistent state. + for (auto& memory_request : old_memory_tasks) { + memory_request->RunDoneCallback(false); + } + for (auto& file_request : old_file_tasks) { + file_request->RunDoneCallback(false, std::vector<FileCreationInfo>()); + } +} + +BlobMemoryController::Strategy BlobMemoryController::DetermineStrategy( + size_t preemptive_transported_bytes, + uint64_t total_transportation_bytes) const { + if (total_transportation_bytes == 0) + return Strategy::NONE_NEEDED; + if (!CanReserveQuota(total_transportation_bytes)) + return Strategy::TOO_LARGE; + + // Handle the case where we have all the bytes preemptively transported, and + // we can also fit them. + if (preemptive_transported_bytes == total_transportation_bytes && + pending_memory_quota_tasks_.empty() && + preemptive_transported_bytes < GetAvailableMemoryForBlobs()) { + return Strategy::NONE_NEEDED; + } + if (file_paging_enabled_ && + (total_transportation_bytes > limits_.memory_limit_before_paging())) { + return Strategy::FILE; + } + if (total_transportation_bytes > limits_.max_ipc_memory_size) + return Strategy::SHARED_MEMORY; + return Strategy::IPC; +} + +bool BlobMemoryController::CanReserveQuota(uint64_t size) const { + // We check each size independently as a blob can't be constructed in both + // disk and memory. + return size <= GetAvailableMemoryForBlobs() || + size <= GetAvailableFileSpaceForBlobs(); +} + +base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveMemoryQuota( + std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items, + const MemoryQuotaRequestCallback& done_callback) { + if (unreserved_memory_items.empty()) { + done_callback.Run(true); + return base::WeakPtr<QuotaAllocationTask>(); + } + + base::CheckedNumeric<uint64_t> unsafe_total_bytes_needed = 0; + for (auto& item : unreserved_memory_items) { + DCHECK_EQ(ShareableBlobDataItem::QUOTA_NEEDED, item->state()); + DCHECK(item->item()->type() == DataElement::TYPE_BYTES_DESCRIPTION || + item->item()->type() == DataElement::TYPE_BYTES); + DCHECK(item->item()->length() > 0); + unsafe_total_bytes_needed += item->item()->length(); + item->set_state(ShareableBlobDataItem::QUOTA_REQUESTED); + } + + uint64_t total_bytes_needed = unsafe_total_bytes_needed.ValueOrDie(); + DCHECK_GT(total_bytes_needed, 0ull); + + // If we're currently waiting for blobs to page already, then we add + // ourselves to the end of the queue. Once paging is complete, we'll schedule + // more paging for any more pending blobs. + if (!pending_memory_quota_tasks_.empty()) { + return AppendMemoryTask(total_bytes_needed, + std::move(unreserved_memory_items), done_callback); + } + + // Store right away if we can. + if (total_bytes_needed <= GetAvailableMemoryForBlobs()) { + GrantMemoryAllocations(&unreserved_memory_items, + static_cast<size_t>(total_bytes_needed)); + MaybeScheduleEvictionUntilSystemHealthy(); + done_callback.Run(true); + return base::WeakPtr<QuotaAllocationTask>(); + } + + // Size is larger than available memory. + DCHECK(pending_memory_quota_tasks_.empty()); + DCHECK_EQ(0u, pending_memory_quota_total_size_); + + auto weak_ptr = AppendMemoryTask( + total_bytes_needed, std::move(unreserved_memory_items), done_callback); + MaybeScheduleEvictionUntilSystemHealthy(); + return weak_ptr; +} + +base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveFileQuota( + std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items, + const FileQuotaRequestCallback& done_callback) { + pending_file_quota_tasks_.push_back(base::MakeUnique<FileQuotaAllocationTask>( + this, std::move(unreserved_file_items), done_callback)); + pending_file_quota_tasks_.back()->set_my_list_position( + --pending_file_quota_tasks_.end()); + return pending_file_quota_tasks_.back()->GetWeakPtr(); +} + +void BlobMemoryController::NotifyMemoryItemsUsed( + const std::vector<scoped_refptr<ShareableBlobDataItem>>& items) { + for (const auto& item : items) { + if (item->item()->type() != DataElement::TYPE_BYTES || + item->state() != ShareableBlobDataItem::POPULATED_WITH_QUOTA) { + continue; + } + // We don't want to re-add the item if we're currently paging it to disk. + if (items_paging_to_file_.find(item->item_id()) != + items_paging_to_file_.end()) { + return; + } + auto iterator = populated_memory_items_.Get(item->item_id()); + if (iterator == populated_memory_items_.end()) { + populated_memory_items_bytes_ += + static_cast<size_t>(item->item()->length()); + populated_memory_items_.Put(item->item_id(), item.get()); + } + } + MaybeScheduleEvictionUntilSystemHealthy(); +} + +base::WeakPtr<QuotaAllocationTask> BlobMemoryController::AppendMemoryTask( + uint64_t total_bytes_needed, + std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items, + const MemoryQuotaRequestCallback& done_callback) { + DCHECK(file_paging_enabled_) + << "Caller tried to reserve memory when CanReserveQuota(" + << total_bytes_needed << ") would have returned false."; + + pending_memory_quota_total_size_ += total_bytes_needed; + pending_memory_quota_tasks_.push_back( + base::MakeUnique<MemoryQuotaAllocationTask>( + this, total_bytes_needed, std::move(unreserved_memory_items), + std::move(done_callback))); + pending_memory_quota_tasks_.back()->set_my_list_position( + --pending_memory_quota_tasks_.end()); + + return pending_memory_quota_tasks_.back()->GetWeakPtr(); +} + +void BlobMemoryController::MaybeGrantPendingMemoryRequests() { + while (!pending_memory_quota_tasks_.empty() && + limits_.max_blob_in_memory_space - blob_memory_used_ >= + pending_memory_quota_tasks_.front()->allocation_size()) { + std::unique_ptr<MemoryQuotaAllocationTask> memory_task = + std::move(pending_memory_quota_tasks_.front()); + pending_memory_quota_tasks_.pop_front(); + pending_memory_quota_total_size_ -= memory_task->allocation_size(); + memory_task->RunDoneCallback(true); + } + RecordTracingCounters(); +} + +size_t BlobMemoryController::CollectItemsForEviction( + std::vector<scoped_refptr<ShareableBlobDataItem>>* output) { + base::CheckedNumeric<size_t> total_items_size = 0; + // Process the recent item list and remove items until we have at least a + // minimum file size or we're at the end of our items to page to disk. + while (total_items_size.ValueOrDie() < limits_.min_page_file_size && + !populated_memory_items_.empty()) { + auto iterator = --populated_memory_items_.end(); + ShareableBlobDataItem* item = iterator->second; + DCHECK_EQ(item->item()->type(), DataElement::TYPE_BYTES); + populated_memory_items_.Erase(iterator); + size_t size = base::checked_cast<size_t>(item->item()->length()); + populated_memory_items_bytes_ -= size; + total_items_size += size; + output->push_back(make_scoped_refptr(item)); + } + return total_items_size.ValueOrDie(); +} + +void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() { + // Don't do eviction when others are happening, as we don't change our + // pending_memory_quota_total_size_ value until after the paging files have + // been written. + if (pending_evictions_ != 0 || !file_paging_enabled_) + return; + + // We try to page items to disk until our current system size + requested + // memory is below our size limit. + while (pending_memory_quota_total_size_ + blob_memory_used_ > + limits_.memory_limit_before_paging()) { + // We only page when we have enough items to fill a whole page file. + if (populated_memory_items_bytes_ < limits_.min_page_file_size) + break; + DCHECK_LE(limits_.min_page_file_size, + static_cast<uint64_t>(blob_memory_used_)); + + std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap; + size_t total_items_size = CollectItemsForEviction(&items_to_swap); + if (total_items_size == 0) + break; + + std::vector<DataElement*> items_for_paging; + for (auto& shared_blob_item : items_to_swap) { + items_paging_to_file_.insert(shared_blob_item->item_id()); + items_for_paging.push_back(shared_blob_item->item()->data_element_ptr()); + } + + // Update our bookkeeping. + pending_evictions_++; + disk_used_ += total_items_size; + in_flight_memory_used_ += total_items_size; + + // Create our file reference. + FilePath page_file_path = GenerateNextPageFileName(); + scoped_refptr<ShareableFileReference> file_reference = + ShareableFileReference::GetOrCreate( + page_file_path, + ShareableFileReference::DELETE_ON_FINAL_RELEASE, + file_runner_.get()); + // Add the release callback so we decrement our disk usage on file deletion. + file_reference->AddFinalReleaseCallback( + base::Bind(&BlobMemoryController::OnBlobFileDelete, + weak_factory_.GetWeakPtr(), total_items_size)); + + // Post the file writing task. + base::PostTaskAndReplyWithResult( + file_runner_.get(), FROM_HERE, + base::Bind(&CreateFileAndWriteItems, blob_storage_dir_, + base::Passed(&page_file_path), file_runner_, + base::Passed(&items_for_paging), total_items_size), + base::Bind(&BlobMemoryController::OnEvictionComplete, + weak_factory_.GetWeakPtr(), base::Passed(&file_reference), + base::Passed(&items_to_swap), total_items_size)); + } + RecordTracingCounters(); +} + +void BlobMemoryController::OnEvictionComplete( + scoped_refptr<ShareableFileReference> file_reference, + std::vector<scoped_refptr<ShareableBlobDataItem>> items, + size_t total_items_size, + FileCreationInfo result) { + if (!file_paging_enabled_) + return; + + if (result.error != File::FILE_OK) { + DisableFilePaging(result.error); + return; + } + + DCHECK_LT(0, pending_evictions_); + pending_evictions_--; + + // Switch item from memory to the new file. + uint64_t offset = 0; + for (const scoped_refptr<ShareableBlobDataItem>& shareable_item : items) { + scoped_refptr<BlobDataItem> new_item( + new BlobDataItem(base::WrapUnique(new DataElement()), file_reference)); + new_item->data_element_ptr()->SetToFilePathRange( + file_reference->path(), offset, shareable_item->item()->length(), + result.last_modified); + DCHECK(shareable_item->memory_allocation_); + shareable_item->set_memory_allocation(nullptr); + shareable_item->set_item(new_item); + items_paging_to_file_.erase(shareable_item->item_id()); + offset += shareable_item->item()->length(); + } + in_flight_memory_used_ -= total_items_size; + + // We want callback on blobs up to the amount we've freed. + MaybeGrantPendingMemoryRequests(); + + // If we still have more blobs waiting and we're not waiting on more paging + // operations, schedule more. + MaybeScheduleEvictionUntilSystemHealthy(); +} + +FilePath BlobMemoryController::GenerateNextPageFileName() { + std::string file_name = base::Uint64ToString(current_file_num_++); + return blob_storage_dir_.Append(FilePath::FromUTF8Unsafe(file_name)); +} + +void BlobMemoryController::RecordTracingCounters() const { + TRACE_COUNTER2("Blob", "MemoryUsage", "TotalStorage", blob_memory_used_, + "InFlightToDisk", in_flight_memory_used_); + TRACE_COUNTER1("Blob", "DiskUsage", disk_used_); + TRACE_COUNTER1("Blob", "TranfersPendingOnDisk", + pending_memory_quota_tasks_.size()); + TRACE_COUNTER1("Blob", "TranfersBytesPendingOnDisk", + pending_memory_quota_total_size_); +} + +size_t BlobMemoryController::GetAvailableMemoryForBlobs() const { + if (limits_.max_blob_in_memory_space < memory_usage()) + return 0; + return limits_.max_blob_in_memory_space - memory_usage(); +} + +uint64_t BlobMemoryController::GetAvailableFileSpaceForBlobs() const { + if (!file_paging_enabled_) + return 0; + // Sometimes we're only paging part of what we need for the new blob, so add + // the rest of the size we need into our disk usage if this is the case. + uint64_t total_disk_used = disk_used_; + if (in_flight_memory_used_ < pending_memory_quota_total_size_) { + total_disk_used += + pending_memory_quota_total_size_ - in_flight_memory_used_; + } + if (limits_.max_blob_disk_space < total_disk_used) + return 0; + return limits_.max_blob_disk_space - total_disk_used; +} + +void BlobMemoryController::GrantMemoryAllocations( + std::vector<scoped_refptr<ShareableBlobDataItem>>* items, + size_t total_bytes) { + // These metrics let us calculate the global distribution of blob storage by + // subtracting the histograms. + UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend", + blob_memory_used_ / 1024); + blob_memory_used_ += total_bytes; + UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend", + blob_memory_used_ / 1024); + + for (auto& item : *items) { + item->set_state(ShareableBlobDataItem::QUOTA_GRANTED); + item->set_memory_allocation(base::MakeUnique<MemoryAllocation>( + weak_factory_.GetWeakPtr(), item->item_id(), + base::checked_cast<size_t>(item->item()->length()))); + } +} + +void BlobMemoryController::RevokeMemoryAllocation(uint64_t item_id, + size_t length) { + DCHECK_LE(length, blob_memory_used_); + + // These metrics let us calculate the global distribution of blob storage by + // subtracting the histograms. + UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend", + blob_memory_used_ / 1024); + blob_memory_used_ -= length; + UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend", + blob_memory_used_ / 1024); + + auto iterator = populated_memory_items_.Get(item_id); + if (iterator != populated_memory_items_.end()) { + DCHECK_GE(populated_memory_items_bytes_, length); + populated_memory_items_bytes_ -= length; + populated_memory_items_.Erase(iterator); + } + MaybeGrantPendingMemoryRequests(); +} + +void BlobMemoryController::OnBlobFileDelete(uint64_t size, + const FilePath& path) { + DCHECK_LE(size, disk_used_); + disk_used_ -= size; +} + +} // namespace storage diff --git a/chromium/storage/browser/blob/blob_memory_controller.h b/chromium/storage/browser/blob/blob_memory_controller.h new file mode 100644 index 00000000000..068e22b32ea --- /dev/null +++ b/chromium/storage/browser/blob/blob_memory_controller.h @@ -0,0 +1,252 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_ +#define STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_ + +#include <stdint.h> + +#include <list> +#include <map> +#include <memory> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +#include "base/callback.h" +#include "base/callback_helpers.h" +#include "base/containers/mru_cache.h" +#include "base/files/file.h" +#include "base/files/file_path.h" +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/time/time.h" +#include "storage/browser/storage_browser_export.h" +#include "storage/common/blob_storage/blob_storage_constants.h" + +namespace base { +class TaskRunner; +} + +namespace storage { +class DataElement; +class ShareableBlobDataItem; +class ShareableFileReference; + +// This class's main responsibility is deciding how blob data gets stored. +// This encompasses: +// * Keeping track of memory & file quota, +// * How to transport the blob data from the renderer (DetermineStrategy), +// * Allocating memory & file quota (ReserveMemoryQuota, ReserveFileQuota) +// * Paging memory quota to disk when we're nearing our memory limit, and +// * Maintaining an LRU of memory items to choose candidates to page to disk +// (NotifyMemoryItemsUsed). +// This class can only be interacted with on the IO thread. +class STORAGE_EXPORT BlobMemoryController { + public: + enum class Strategy { + // We don't have enough memory for this blob. + TOO_LARGE, + // There isn't any memory that needs transporting. + NONE_NEEDED, + // Transportation strategies. + IPC, + SHARED_MEMORY, + FILE + }; + + struct STORAGE_EXPORT FileCreationInfo { + FileCreationInfo(); + ~FileCreationInfo(); + FileCreationInfo(FileCreationInfo&& other); + FileCreationInfo& operator=(FileCreationInfo&&); + + base::File::Error error = base::File::FILE_ERROR_FAILED; + base::File file; + scoped_refptr<base::TaskRunner> file_deletion_runner; + base::FilePath path; + scoped_refptr<ShareableFileReference> file_reference; + base::Time last_modified; + }; + + struct MemoryAllocation { + MemoryAllocation(base::WeakPtr<BlobMemoryController> controller, + uint64_t item_id, + size_t length); + ~MemoryAllocation(); + + private: + base::WeakPtr<BlobMemoryController> controller; + uint64_t item_id; + size_t length; + + DISALLOW_COPY_AND_ASSIGN(MemoryAllocation); + }; + + class QuotaAllocationTask { + public: + // Operation is cancelled and the callback will NOT be called. This object + // will be destroyed by this call. + virtual void Cancel() = 0; + + protected: + virtual ~QuotaAllocationTask(); + }; + + // The bool argument is true if we successfully received memory quota. + using MemoryQuotaRequestCallback = base::Callback<void(bool)>; + // The bool argument is true if we successfully received file quota, and the + // vector argument provides the file info. + using FileQuotaRequestCallback = + base::Callback<void(bool, std::vector<FileCreationInfo>)>; + + // We enable file paging if |file_runner| isn't a nullptr. + BlobMemoryController(const base::FilePath& storage_directory, + scoped_refptr<base::TaskRunner> file_runner); + ~BlobMemoryController(); + + // Disables file paging. This cancels all pending file creations and paging + // operations. Reason is recorded in UMA. + void DisableFilePaging(base::File::Error reason); + + bool file_paging_enabled() const { return file_paging_enabled_; } + + // Returns the strategy the transportation layer should use to transport the + // given memory. |preemptive_transported_bytes| are the number of transport + // bytes that are already populated for us, so we don't haved to request them + // from the renderer. + Strategy DetermineStrategy(size_t preemptive_transported_bytes, + uint64_t total_transportation_bytes) const; + + // Checks to see if we can reserve quota (disk or memory) for the given size. + bool CanReserveQuota(uint64_t size) const; + + // Reserves quota for the given |unreserved_memory_items|. The items must be + // bytes items in QUOTA_NEEDED state which we change to QUOTA_REQUESTED. + // After we reserve memory quota we change their state to QUOTA_GRANTED, set + // the 'memory_allocation' on the item, and call |done_callback|. This can + // happen synchronously. + // Returns a task handle if the request is asynchronous for cancellation. + // NOTE: We don't inspect quota limits and assume the user checked + // CanReserveQuota before calling this. + base::WeakPtr<QuotaAllocationTask> ReserveMemoryQuota( + std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items, + const MemoryQuotaRequestCallback& done_callback); + + // Reserves quota for the given |unreserved_file_items|. The items must be + // temporary file items (BlobDataBuilder::IsTemporaryFileItem returns true) in + // QUOTA_NEEDED state, which we change to QUOTA_REQUESTED. After we reserve + // file quota we change their state to QUOTA_GRANTED and call + // |done_callback|. + // Returns a task handle for cancellation. + // NOTE: We don't inspect quota limits and assume the user checked + // CanReserveQuota before calling this. + base::WeakPtr<QuotaAllocationTask> ReserveFileQuota( + std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items, + const FileQuotaRequestCallback& done_callback); + + // Called when initially populated or upon later access. + void NotifyMemoryItemsUsed( + const std::vector<scoped_refptr<ShareableBlobDataItem>>& items); + + size_t memory_usage() const { return blob_memory_used_; } + uint64_t disk_usage() const { return disk_used_; } + + const BlobStorageLimits& limits() const { return limits_; } + void set_limits_for_testing(const BlobStorageLimits& limits) { + limits_ = limits; + } + + private: + class FileQuotaAllocationTask; + class MemoryQuotaAllocationTask; + + using PendingMemoryQuotaTaskList = + std::list<std::unique_ptr<MemoryQuotaAllocationTask>>; + using PendingFileQuotaTaskList = + std::list<std::unique_ptr<FileQuotaAllocationTask>>; + + base::WeakPtr<QuotaAllocationTask> AppendMemoryTask( + uint64_t total_bytes_needed, + std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items, + const MemoryQuotaRequestCallback& done_callback); + + void MaybeGrantPendingMemoryRequests(); + + size_t CollectItemsForEviction( + std::vector<scoped_refptr<ShareableBlobDataItem>>* output); + + // Schedule paging until our memory usage is below our memory limit. + void MaybeScheduleEvictionUntilSystemHealthy(); + + // Called when we've completed evicting a list of items to disk. This is where + // we swap the bytes items for file items, and update our bookkeeping. + void OnEvictionComplete( + scoped_refptr<ShareableFileReference> file_reference, + std::vector<scoped_refptr<ShareableBlobDataItem>> items, + size_t total_items_size, + FileCreationInfo result); + + size_t GetAvailableMemoryForBlobs() const; + uint64_t GetAvailableFileSpaceForBlobs() const; + + void GrantMemoryAllocations( + std::vector<scoped_refptr<ShareableBlobDataItem>>* items, + size_t total_bytes); + void RevokeMemoryAllocation(uint64_t item_id, size_t length); + + // This is registered as a callback for file deletions on the file reference + // of our paging files. We decrement the disk space used. + void OnBlobFileDelete(uint64_t size, const base::FilePath& path); + + base::FilePath GenerateNextPageFileName(); + + // This records diagnostic counters of our memory quotas. Called when usage + // changes. + void RecordTracingCounters() const; + + BlobStorageLimits limits_; + + // Memory bookkeeping. These numbers are all disjoint. + // This is the amount of memory we're using for blobs in RAM, including the + // in_flight_memory_used_. + size_t blob_memory_used_ = 0; + // This is memory we're temporarily using while we try to write blob items to + // disk. + size_t in_flight_memory_used_ = 0; + // This is the amount of memory we're using on disk. + uint64_t disk_used_ = 0; + + // State for GenerateNextPageFileName. + uint64_t current_file_num_ = 0; + + size_t pending_memory_quota_total_size_ = 0; + PendingMemoryQuotaTaskList pending_memory_quota_tasks_; + PendingFileQuotaTaskList pending_file_quota_tasks_; + + int pending_evictions_ = 0; + + bool file_paging_enabled_ = false; + base::FilePath blob_storage_dir_; + scoped_refptr<base::TaskRunner> file_runner_; + + // Lifetime of the ShareableBlobDataItem objects is handled externally in the + // BlobStorageContext class. + base::MRUCache<uint64_t, ShareableBlobDataItem*> populated_memory_items_; + size_t populated_memory_items_bytes_ = 0; + // We need to keep track of items currently being paged to disk so that if + // another blob successfully grabs a ref, we can prevent it from adding the + // item to the recent_item_cache_ above. + std::unordered_set<uint64_t> items_paging_to_file_; + + base::WeakPtrFactory<BlobMemoryController> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(BlobMemoryController); +}; +} // namespace storage +#endif // STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_ diff --git a/chromium/storage/browser/blob/blob_reader.cc b/chromium/storage/browser/blob/blob_reader.cc index 2afb964e88a..1ec3002a8d1 100644 --- a/chromium/storage/browser/blob/blob_reader.cc +++ b/chromium/storage/browser/blob/blob_reader.cc @@ -39,20 +39,25 @@ bool IsFileType(DataElement::Type type) { } } -int ConvertBlobErrorToNetError(IPCBlobCreationCancelCode reason) { +int ConvertBlobErrorToNetError(BlobStatus reason) { switch (reason) { - case IPCBlobCreationCancelCode::UNKNOWN: + case BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS: return net::ERR_FAILED; - case IPCBlobCreationCancelCode::OUT_OF_MEMORY: + case BlobStatus::ERR_OUT_OF_MEMORY: return net::ERR_OUT_OF_MEMORY; - case IPCBlobCreationCancelCode::FILE_WRITE_FAILED: + case BlobStatus::ERR_FILE_WRITE_FAILED: return net::ERR_FILE_NO_SPACE; - case IPCBlobCreationCancelCode::SOURCE_DIED_IN_TRANSIT: + case BlobStatus::ERR_SOURCE_DIED_IN_TRANSIT: return net::ERR_UNEXPECTED; - case IPCBlobCreationCancelCode::BLOB_DEREFERENCED_WHILE_BUILDING: + case BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING: return net::ERR_UNEXPECTED; - case IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN: + case BlobStatus::ERR_REFERENCED_BLOB_BROKEN: return net::ERR_INVALID_HANDLE; + case BlobStatus::DONE: + case BlobStatus::PENDING_QUOTA: + case BlobStatus::PENDING_TRANSPORT: + case BlobStatus::PENDING_INTERNALS: + NOTREACHED(); } NOTREACHED(); return net::ERR_FAILED; @@ -254,10 +259,9 @@ BlobReader::Status BlobReader::ReportError(int net_error) { } void BlobReader::AsyncCalculateSize(const net::CompletionCallback& done, - bool async_succeeded, - IPCBlobCreationCancelCode reason) { - if (!async_succeeded) { - InvalidateCallbacksAndDone(ConvertBlobErrorToNetError(reason), done); + BlobStatus status) { + if (BlobStatusIsError(status)) { + InvalidateCallbacksAndDone(ConvertBlobErrorToNetError(status), done); return; } DCHECK(!blob_handle_->IsBroken()) << "Callback should have returned false."; diff --git a/chromium/storage/browser/blob/blob_reader.h b/chromium/storage/browser/blob/blob_reader.h index 5a58f8aa255..a7b1d12be35 100644 --- a/chromium/storage/browser/blob/blob_reader.h +++ b/chromium/storage/browser/blob/blob_reader.h @@ -155,8 +155,7 @@ class STORAGE_EXPORT BlobReader { void InvalidateCallbacksAndDone(int net_error, net::CompletionCallback done); void AsyncCalculateSize(const net::CompletionCallback& done, - bool async_succeeded, - IPCBlobCreationCancelCode reason); + BlobStatus status); Status CalculateSizeImpl(const net::CompletionCallback& done); bool AddItemLength(size_t index, uint64_t length); bool ResolveFileItemLength(const BlobDataItem& item, diff --git a/chromium/storage/browser/blob/blob_storage_context.cc b/chromium/storage/browser/blob/blob_storage_context.cc index 6ce45e53f90..1fecda89f5b 100644 --- a/chromium/storage/browser/blob/blob_storage_context.cc +++ b/chromium/storage/browser/blob/blob_storage_context.cc @@ -4,72 +4,367 @@ #include "storage/browser/blob/blob_storage_context.h" -#include <stddef.h> -#include <stdint.h> - #include <algorithm> #include <limits> #include <memory> +#include <set> #include <utility> #include "base/bind.h" #include "base/callback.h" +#include "base/callback_helpers.h" #include "base/location.h" #include "base/logging.h" #include "base/memory/ptr_util.h" #include "base/message_loop/message_loop.h" -#include "base/metrics/histogram.h" +#include "base/metrics/histogram_macros.h" +#include "base/numerics/safe_conversions.h" +#include "base/numerics/safe_math.h" +#include "base/task_runner.h" #include "base/threading/thread_task_runner_handle.h" #include "base/trace_event/trace_event.h" #include "storage/browser/blob/blob_data_builder.h" -#include "storage/browser/blob/blob_data_handle.h" #include "storage/browser/blob/blob_data_item.h" #include "storage/browser/blob/blob_data_snapshot.h" #include "storage/browser/blob/shareable_blob_data_item.h" +#include "storage/common/data_element.h" #include "url/gurl.h" namespace storage { -using BlobRegistryEntry = BlobStorageRegistry::Entry; -using BlobState = BlobStorageRegistry::BlobState; +namespace { +using ItemCopyEntry = BlobEntry::ItemCopyEntry; +using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask; + +bool IsBytes(DataElement::Type type) { + return type == DataElement::TYPE_BYTES || + type == DataElement::TYPE_BYTES_DESCRIPTION; +} + +void RecordBlobItemSizeStats(const DataElement& input_element) { + uint64_t length = input_element.length(); + + switch (input_element.type()) { + case DataElement::TYPE_BYTES: + case DataElement::TYPE_BYTES_DESCRIPTION: + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024); + break; + case DataElement::TYPE_BLOB: + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob", + (length - input_element.offset()) / 1024); + break; + case DataElement::TYPE_FILE: { + bool full_file = (length == std::numeric_limits<uint64_t>::max()); + UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file); + if (!full_file) { + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File", + (length - input_element.offset()) / 1024); + } + break; + } + case DataElement::TYPE_FILE_FILESYSTEM: { + bool full_file = (length == std::numeric_limits<uint64_t>::max()); + UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown", + full_file); + if (!full_file) { + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem", + (length - input_element.offset()) / 1024); + } + break; + } + case DataElement::TYPE_DISK_CACHE_ENTRY: + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry", + (length - input_element.offset()) / 1024); + break; + case DataElement::TYPE_UNKNOWN: + NOTREACHED(); + break; + } +} +} // namespace + +BlobStorageContext::BlobFlattener::BlobFlattener( + const BlobDataBuilder& input_builder, + BlobEntry* output_blob, + BlobStorageRegistry* registry) { + const std::string& uuid = input_builder.uuid_; + std::set<std::string> dependent_blob_uuids; + + size_t num_files_with_unknown_size = 0; + size_t num_building_dependent_blobs = 0; + + base::CheckedNumeric<uint64_t> checked_total_size = 0; + base::CheckedNumeric<uint64_t> checked_total_memory_size = 0; + base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0; + + for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) { + const DataElement& input_element = input_item->data_element(); + DataElement::Type type = input_element.type(); + uint64_t length = input_element.length(); + + RecordBlobItemSizeStats(input_element); + + if (IsBytes(type)) { + DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length()); + checked_memory_quota_needed += length; + checked_total_size += length; + scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem( + std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED); + pending_memory_items.push_back(item); + transport_items.push_back(item.get()); + output_blob->AppendSharedBlobItem(std::move(item)); + continue; + } + if (type == DataElement::TYPE_BLOB) { + BlobEntry* ref_entry = registry->GetEntry(input_element.blob_uuid()); + + if (!ref_entry || input_element.blob_uuid() == uuid) { + status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + return; + } + + if (BlobStatusIsError(ref_entry->status())) { + status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN; + return; + } + + if (ref_entry->total_size() == DataElement::kUnknownSize) { + // We can't reference a blob with unknown size. + status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + return; + } + + if (dependent_blob_uuids.find(input_element.blob_uuid()) == + dependent_blob_uuids.end()) { + dependent_blobs.push_back( + std::make_pair(input_element.blob_uuid(), ref_entry)); + dependent_blob_uuids.insert(input_element.blob_uuid()); + if (BlobStatusIsPending(ref_entry->status())) { + num_building_dependent_blobs++; + } + } + + length = length == DataElement::kUnknownSize ? ref_entry->total_size() + : input_element.length(); + checked_total_size += length; + + // If we're referencing the whole blob, then we don't need to slice. + if (input_element.offset() == 0 && length == ref_entry->total_size()) { + for (const auto& shareable_item : ref_entry->items()) { + output_blob->AppendSharedBlobItem(shareable_item); + } + continue; + } + + // Validate our reference has good offset & length. + if (input_element.offset() + length > ref_entry->total_size()) { + status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + return; + } -BlobStorageContext::BlobStorageContext() : memory_usage_(0) {} + BlobSlice slice(*ref_entry, input_element.offset(), length); + + if (!slice.copying_memory_size.IsValid() || + !slice.total_memory_size.IsValid()) { + status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + return; + } + checked_total_memory_size += slice.total_memory_size; + + if (slice.first_source_item) { + copies.push_back(ItemCopyEntry(slice.first_source_item, + slice.first_item_slice_offset, + slice.dest_items.front())); + pending_memory_items.push_back(slice.dest_items.front()); + } + if (slice.last_source_item) { + copies.push_back( + ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back())); + pending_memory_items.push_back(slice.dest_items.back()); + } + checked_memory_quota_needed += slice.copying_memory_size; + + for (auto& shareable_item : slice.dest_items) { + output_blob->AppendSharedBlobItem(std::move(shareable_item)); + } + continue; + } + + DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element)) + << "File allocation not implemented."; + if (length == DataElement::kUnknownSize) + num_files_with_unknown_size++; + + scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem( + std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA); + + checked_total_size += length; + output_blob->AppendSharedBlobItem(std::move(item)); + } + + if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) { + status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + return; + } + if (!checked_total_size.IsValid() || !checked_total_memory_size.IsValid() || + !checked_memory_quota_needed.IsValid()) { + status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + return; + } + total_size = checked_total_size.ValueOrDie(); + total_memory_size = checked_total_memory_size.ValueOrDie(); + memory_quota_needed = checked_memory_quota_needed.ValueOrDie(); + if (memory_quota_needed) { + status = BlobStatus::PENDING_QUOTA; + } else { + status = BlobStatus::PENDING_INTERNALS; + } +} + +BlobStorageContext::BlobFlattener::~BlobFlattener() {} + +BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source, + uint64_t slice_offset, + uint64_t slice_size) { + const auto& source_items = source.items(); + const auto& offsets = source.offsets(); + DCHECK_LE(slice_offset + slice_size, source.total_size()); + size_t item_index = + std::upper_bound(offsets.begin(), offsets.end(), slice_offset) - + offsets.begin(); + uint64_t item_offset = + item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1]; + size_t num_items = source_items.size(); + + size_t first_item_index = item_index; + + // Read starting from 'first_item_index' and 'item_offset'. + for (uint64_t total_sliced = 0; + item_index < num_items && total_sliced < slice_size; item_index++) { + const scoped_refptr<BlobDataItem>& source_item = + source_items[item_index]->item(); + uint64_t source_length = source_item->length(); + DataElement::Type type = source_item->type(); + DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max()); + DCHECK_NE(source_length, 0ull); + + uint64_t read_size = + std::min(source_length - item_offset, slice_size - total_sliced); + total_sliced += read_size; + + bool reusing_blob_item = (read_size == source_length); + UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item); + if (reusing_blob_item) { + // We can share the entire item. + dest_items.push_back(source_items[item_index]); + if (IsBytes(type)) { + total_memory_size += source_length; + } + continue; + } + + scoped_refptr<BlobDataItem> data_item; + ShareableBlobDataItem::State state = + ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA; + switch (type) { + case DataElement::TYPE_BYTES_DESCRIPTION: + case DataElement::TYPE_BYTES: { + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes", + read_size / 1024); + if (item_index == first_item_index) { + first_item_slice_offset = item_offset; + first_source_item = source_items[item_index]; + } else { + last_source_item = source_items[item_index]; + } + copying_memory_size += read_size; + total_memory_size += read_size; + // Since we don't have quota yet for memory, we create temporary items + // for this data. When our blob is finished constructing, all dependent + // blobs are done, and we have enough memory quota, we'll copy the data + // over. + std::unique_ptr<DataElement> element(new DataElement()); + element->SetToBytesDescription(base::checked_cast<size_t>(read_size)); + data_item = new BlobDataItem(std::move(element)); + state = ShareableBlobDataItem::QUOTA_NEEDED; + break; + } + case DataElement::TYPE_FILE: { + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File", + read_size / 1024); + std::unique_ptr<DataElement> element(new DataElement()); + element->SetToFilePathRange( + source_item->path(), source_item->offset() + item_offset, read_size, + source_item->expected_modification_time()); + data_item = + new BlobDataItem(std::move(element), source_item->data_handle_); + + DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element())) + << "File allocation unimplemented."; + break; + } + case DataElement::TYPE_FILE_FILESYSTEM: { + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem", + read_size / 1024); + std::unique_ptr<DataElement> element(new DataElement()); + element->SetToFileSystemUrlRange( + source_item->filesystem_url(), source_item->offset() + item_offset, + read_size, source_item->expected_modification_time()); + data_item = new BlobDataItem(std::move(element)); + break; + } + case DataElement::TYPE_DISK_CACHE_ENTRY: { + UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.CacheEntry", + read_size / 1024); + std::unique_ptr<DataElement> element(new DataElement()); + element->SetToDiskCacheEntryRange(source_item->offset() + item_offset, + read_size); + data_item = + new BlobDataItem(std::move(element), source_item->data_handle_, + source_item->disk_cache_entry(), + source_item->disk_cache_stream_index(), + source_item->disk_cache_side_stream_index()); + break; + } + case DataElement::TYPE_BLOB: + case DataElement::TYPE_UNKNOWN: + CHECK(false) << "Illegal blob item type: " << type; + } + dest_items.push_back( + new ShareableBlobDataItem(std::move(data_item), state)); + item_offset = 0; + } +} + +BlobStorageContext::BlobSlice::~BlobSlice() {} + +BlobStorageContext::BlobStorageContext() + : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()), + ptr_factory_(this) {} BlobStorageContext::~BlobStorageContext() { } std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID( const std::string& uuid) { - BlobRegistryEntry* entry = registry_.GetEntry(uuid); - if (!entry) { + BlobEntry* entry = registry_.GetEntry(uuid); + if (!entry) return nullptr; - } - return base::WrapUnique( - new BlobDataHandle(uuid, entry->content_type, entry->content_disposition, - this, base::ThreadTaskRunnerHandle::Get().get())); + return CreateHandle(uuid, entry); } std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL( const GURL& url) { std::string uuid; - BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid); - if (!entry) { + BlobEntry* entry = registry_.GetEntryFromURL(url, &uuid); + if (!entry) return nullptr; - } - return base::WrapUnique( - new BlobDataHandle(uuid, entry->content_type, entry->content_disposition, - this, base::ThreadTaskRunnerHandle::Get().get())); + return CreateHandle(uuid, entry); } std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob( const BlobDataBuilder& external_builder) { TRACE_EVENT0("Blob", "Context::AddFinishedBlob"); - CreatePendingBlob(external_builder.uuid(), external_builder.content_type_, - external_builder.content_disposition_); - CompletePendingBlob(external_builder); - std::unique_ptr<BlobDataHandle> handle = - GetBlobDataFromUUID(external_builder.uuid_); - DecrementBlobRefCount(external_builder.uuid_); - return handle; + return BuildBlob(external_builder, TransportAllowedCallback()); } std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob( @@ -78,123 +373,149 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob( return AddFinishedBlob(*builder); } +std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob( + const std::string& uuid, + const std::string& content_type, + const std::string& content_disposition, + BlobStatus reason) { + DCHECK(!registry_.HasEntry(uuid)); + DCHECK(BlobStatusIsError(reason)); + BlobEntry* entry = + registry_.CreateEntry(uuid, content_type, content_disposition); + entry->set_status(reason); + FinishBuilding(entry); + return CreateHandle(uuid, entry); +} + bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url, const std::string& uuid) { - if (!registry_.CreateUrlMapping(blob_url, uuid)) { + if (!registry_.CreateUrlMapping(blob_url, uuid)) return false; - } IncrementBlobRefCount(uuid); return true; } void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) { std::string uuid; - if (!registry_.DeleteURLMapping(blob_url, &uuid)) { + if (!registry_.DeleteURLMapping(blob_url, &uuid)) return; - } DecrementBlobRefCount(uuid); } -void BlobStorageContext::CreatePendingBlob( - const std::string& uuid, - const std::string& content_type, - const std::string& content_disposition) { - DCHECK(!registry_.GetEntry(uuid) && !uuid.empty()); - registry_.CreateEntry(uuid, content_type, content_disposition); -} - -void BlobStorageContext::CompletePendingBlob( - const BlobDataBuilder& external_builder) { - BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid()); - DCHECK(entry); - DCHECK(!entry->data.get()) << "Blob already constructed: " - << external_builder.uuid(); - // We want to handle storing our broken blob as well. - switch (entry->state) { - case BlobState::PENDING: { - entry->data_builder.reset(new InternalBlobData::Builder()); - InternalBlobData::Builder* internal_data_builder = - entry->data_builder.get(); - - bool broken = false; - for (const auto& blob_item : external_builder.items_) { - IPCBlobCreationCancelCode error_code; - if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item, - internal_data_builder, &error_code)) { - broken = true; - memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage(); - entry->state = BlobState::BROKEN; - entry->broken_reason = error_code; - entry->data_builder.reset(new InternalBlobData::Builder()); - break; - } - } - entry->data = entry->data_builder->Build(); - entry->data_builder.reset(); - entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE; - break; - } - case BlobState::BROKEN: { - InternalBlobData::Builder builder; - entry->data = builder.Build(); - break; +std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob( + const BlobDataBuilder& content, + const TransportAllowedCallback& transport_allowed_callback) { + DCHECK(!registry_.HasEntry(content.uuid_)); + + BlobEntry* entry = registry_.CreateEntry( + content.uuid(), content.content_type_, content.content_disposition_); + + // This flattens all blob references in the transportion content out and + // stores the complete item representation in the internal data. + BlobFlattener flattener(content, entry, ®istry_); + + DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT || + !transport_allowed_callback) + << "There is no pending content for the user to populate, so the " + "callback should be null."; + DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT || + transport_allowed_callback) + << "If we have pending content then there needs to be a callback " + "present."; + + entry->set_size(flattener.total_size); + entry->set_status(flattener.status); + std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry); + + UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->items().size()); + UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", + flattener.total_memory_size / 1024); + UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize", + flattener.memory_quota_needed / 1024); + + size_t num_building_dependent_blobs = 0; + std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs; + // We hold a handle to all blobs we're using. This is important, as our memory + // accounting can be delayed until OnEnoughSizeForBlobData is called, and we + // only free memory on canceling when we've done this accounting. If a + // dependent blob is dereferenced, then we're the last blob holding onto that + // data item, and we need to account for that. So we prevent that case by + // holding onto all blobs. + for (const std::pair<std::string, BlobEntry*>& pending_blob : + flattener.dependent_blobs) { + dependent_blobs.push_back( + CreateHandle(pending_blob.first, pending_blob.second)); + if (BlobStatusIsPending(pending_blob.second->status())) { + pending_blob.second->building_state_->build_completion_callbacks + .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished, + ptr_factory_.GetWeakPtr(), content.uuid_)); + num_building_dependent_blobs++; } - case BlobState::COMPLETE: - DCHECK(false) << "Blob already constructed: " << external_builder.uuid(); - return; } - UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size()); - UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken", - entry->state == BlobState::BROKEN); - if (entry->state == BlobState::BROKEN) { - UMA_HISTOGRAM_ENUMERATION( - "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason), - (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1)); + entry->set_building_state(base::MakeUnique<BlobEntry::BuildingState>( + !flattener.transport_items.empty(), transport_allowed_callback, + num_building_dependent_blobs)); + BlobEntry::BuildingState* building_state = entry->building_state_.get(); + std::swap(building_state->copies, flattener.copies); + std::swap(building_state->dependent_blobs, dependent_blobs); + std::swap(building_state->transport_items, flattener.transport_items); + + // Break ourselves if we have an error. BuildingState must be set first so the + // callback is called correctly. + if (BlobStatusIsError(flattener.status)) { + CancelBuildingBlobInternal(entry, flattener.status); + return handle; } - size_t total_memory = 0, nonshared_memory = 0; - entry->data->GetMemoryUsage(&total_memory, &nonshared_memory); - UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024); - UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize", - nonshared_memory / 1024); - TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_); - auto runner = base::ThreadTaskRunnerHandle::Get(); - for (const auto& callback : entry->build_completion_callbacks) { - runner->PostTask(FROM_HERE, - base::Bind(callback, entry->state == BlobState::COMPLETE, - entry->broken_reason)); + if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) { + CancelBuildingBlobInternal(entry, BlobStatus::ERR_OUT_OF_MEMORY); + return handle; } - entry->build_completion_callbacks.clear(); + + if (flattener.memory_quota_needed > 0) { + // The blob can complete during the execution of this line. + base::WeakPtr<QuotaAllocationTask> pending_request = + memory_controller_.ReserveMemoryQuota( + std::move(flattener.pending_memory_items), + base::Bind(&BlobStorageContext::OnEnoughSizeForMemory, + ptr_factory_.GetWeakPtr(), content.uuid_)); + // Building state will be null if the blob is already finished. + if (entry->building_state_) + entry->building_state_->memory_quota_request = std::move(pending_request); + } + + if (entry->CanFinishBuilding()) + FinishBuilding(entry); + + return handle; +} + +void BlobStorageContext::CancelBuildingBlob(const std::string& uuid, + BlobStatus reason) { + CancelBuildingBlobInternal(registry_.GetEntry(uuid), reason); } -void BlobStorageContext::CancelPendingBlob(const std::string& uuid, - IPCBlobCreationCancelCode reason) { - BlobRegistryEntry* entry = registry_.GetEntry(uuid); - DCHECK(entry && entry->state == BlobState::PENDING); - entry->state = BlobState::BROKEN; - entry->broken_reason = reason; - CompletePendingBlob(BlobDataBuilder(uuid)); +void BlobStorageContext::NotifyTransportComplete(const std::string& uuid) { + BlobEntry* entry = registry_.GetEntry(uuid); + CHECK(entry) << "There is no blob entry with uuid " << uuid; + DCHECK(BlobStatusIsPending(entry->status())); + NotifyTransportCompleteInternal(entry); } void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) { - BlobRegistryEntry* entry = registry_.GetEntry(uuid); + BlobEntry* entry = registry_.GetEntry(uuid); DCHECK(entry); - ++(entry->refcount); + entry->IncrementRefCount(); } void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) { - BlobRegistryEntry* entry = registry_.GetEntry(uuid); + BlobEntry* entry = registry_.GetEntry(uuid); DCHECK(entry); - DCHECK_GT(entry->refcount, 0u); - if (--(entry->refcount) == 0) { - size_t memory_freeing = 0; - if (entry->state == BlobState::COMPLETE) { - memory_freeing = entry->data->GetUnsharedMemoryUsage(); - entry->data->RemoveBlobFromShareableItems(uuid); - } - DCHECK_LE(memory_freeing, memory_usage_); - memory_usage_ -= memory_freeing; + DCHECK_GT(entry->refcount(), 0u); + entry->DecrementRefCount(); + if (entry->refcount() == 0) { + ClearAndFreeMemory(entry); registry_.DeleteEntry(uuid); } } @@ -202,255 +523,208 @@ void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) { std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot( const std::string& uuid) { std::unique_ptr<BlobDataSnapshot> result; - BlobRegistryEntry* entry = registry_.GetEntry(uuid); - if (entry->state != BlobState::COMPLETE) { + BlobEntry* entry = registry_.GetEntry(uuid); + if (entry->status() != BlobStatus::DONE) return result; - } - const InternalBlobData& data = *entry->data; std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot( - uuid, entry->content_type, entry->content_disposition)); - snapshot->items_.reserve(data.items().size()); - for (const auto& shareable_item : data.items()) { + uuid, entry->content_type(), entry->content_disposition())); + snapshot->items_.reserve(entry->items().size()); + for (const auto& shareable_item : entry->items()) { snapshot->items_.push_back(shareable_item->item()); } + memory_controller_.NotifyMemoryItemsUsed(entry->items()); return snapshot; } -bool BlobStorageContext::IsBroken(const std::string& uuid) const { - const BlobRegistryEntry* entry = registry_.GetEntry(uuid); - if (!entry) { - return true; - } - return entry->state == BlobState::BROKEN; +BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const { + const BlobEntry* entry = registry_.GetEntry(uuid); + if (!entry) + return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + return entry->status(); } -bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const { - const BlobRegistryEntry* entry = registry_.GetEntry(uuid); - if (!entry) { - return false; +void BlobStorageContext::RunOnConstructionComplete( + const std::string& uuid, + const BlobStatusCallback& done) { + BlobEntry* entry = registry_.GetEntry(uuid); + DCHECK(entry); + if (BlobStatusIsPending(entry->status())) { + entry->building_state_->build_completion_callbacks.push_back(done); + return; } - return entry->state == BlobState::PENDING; + done.Run(entry->status()); } -void BlobStorageContext::RunOnConstructionComplete( +std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle( const std::string& uuid, - const BlobConstructedCallback& done) { - BlobRegistryEntry* entry = registry_.GetEntry(uuid); + BlobEntry* entry) { + return base::WrapUnique(new BlobDataHandle( + uuid, entry->content_type_, entry->content_disposition_, entry->size_, + this, base::ThreadTaskRunnerHandle::Get().get())); +} + +void BlobStorageContext::NotifyTransportCompleteInternal(BlobEntry* entry) { DCHECK(entry); - switch (entry->state) { - case BlobState::COMPLETE: - done.Run(true, IPCBlobCreationCancelCode::UNKNOWN); - return; - case BlobState::BROKEN: - done.Run(false, entry->broken_reason); - return; - case BlobState::PENDING: - entry->build_completion_callbacks.push_back(done); - return; + for (ShareableBlobDataItem* shareable_item : + entry->building_state_->transport_items) { + DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED); + shareable_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA); } - NOTREACHED(); + entry->set_status(BlobStatus::PENDING_INTERNALS); + if (entry->CanFinishBuilding()) + FinishBuilding(entry); } -bool BlobStorageContext::AppendAllocatedBlobItem( - const std::string& target_blob_uuid, - scoped_refptr<BlobDataItem> blob_item, - InternalBlobData::Builder* target_blob_builder, - IPCBlobCreationCancelCode* error_code) { - DCHECK(error_code); - *error_code = IPCBlobCreationCancelCode::UNKNOWN; - bool error = false; - - // The blob data is stored in the canonical way which only contains a - // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items - // are expanded into the primitive constituent types and reused if possible. - // 1) The Data item is denoted by the raw data and length. - // 2) The File item is denoted by the file path, the range and the expected - // modification time. - // 3) The FileSystem File item is denoted by the FileSystem URL, the range - // and the expected modification time. - // 4) The Blob item is denoted by the source blob and an offset and size. - // Internal items that are fully used by the new blob (not cut by the - // offset or size) are shared between the blobs. Otherwise, the relevant - // portion of the item is copied. - - DCHECK(blob_item->data_element_ptr()); - const DataElement& data_element = blob_item->data_element(); - uint64_t length = data_element.length(); - uint64_t offset = data_element.offset(); - UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend", - memory_usage_ / 1024); - switch (data_element.type()) { - case DataElement::TYPE_BYTES: - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024); - DCHECK(!offset); - if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) { - error = true; - *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY; - break; - } - memory_usage_ += length; - target_blob_builder->AppendSharedBlobItem( - new ShareableBlobDataItem(target_blob_uuid, blob_item)); - break; - case DataElement::TYPE_FILE: { - bool full_file = (length == std::numeric_limits<uint64_t>::max()); - UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file); - if (!full_file) { - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File", - (length - offset) / 1024); - } - target_blob_builder->AppendSharedBlobItem( - new ShareableBlobDataItem(target_blob_uuid, blob_item)); - break; - } - case DataElement::TYPE_FILE_FILESYSTEM: { - bool full_file = (length == std::numeric_limits<uint64_t>::max()); - UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown", - full_file); - if (!full_file) { - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem", - (length - offset) / 1024); - } - target_blob_builder->AppendSharedBlobItem( - new ShareableBlobDataItem(target_blob_uuid, blob_item)); - break; - } - case DataElement::TYPE_BLOB: { - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob", - (length - offset) / 1024); - // We grab the handle to ensure it stays around while we copy it. - std::unique_ptr<BlobDataHandle> src = - GetBlobDataFromUUID(data_element.blob_uuid()); - if (!src || src->IsBroken() || src->IsBeingBuilt()) { - error = true; - *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN; - break; - } - BlobRegistryEntry* other_entry = - registry_.GetEntry(data_element.blob_uuid()); - DCHECK(other_entry->data); - if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length, - target_blob_builder)) { - error = true; - *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY; +void BlobStorageContext::CancelBuildingBlobInternal(BlobEntry* entry, + BlobStatus reason) { + DCHECK(entry); + DCHECK(BlobStatusIsError(reason)); + TransportAllowedCallback transport_allowed_callback; + if (entry->building_state_ && + entry->building_state_->transport_allowed_callback) { + transport_allowed_callback = + entry->building_state_->transport_allowed_callback; + entry->building_state_->transport_allowed_callback.Reset(); + } + ClearAndFreeMemory(entry); + entry->set_status(reason); + if (transport_allowed_callback) { + transport_allowed_callback.Run( + reason, std::vector<BlobMemoryController::FileCreationInfo>()); + } + FinishBuilding(entry); +} + +void BlobStorageContext::FinishBuilding(BlobEntry* entry) { + DCHECK(entry); + + BlobStatus status = entry->status_; + DCHECK_NE(BlobStatus::DONE, status); + + bool error = BlobStatusIsError(status); + UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken", error); + if (error) { + UMA_HISTOGRAM_ENUMERATION("Storage.Blob.BrokenReason", + static_cast<int>(status), + (static_cast<int>(BlobStatus::LAST_ERROR) + 1)); + } + + if (BlobStatusIsPending(entry->status_)) { + for (const ItemCopyEntry& copy : entry->building_state_->copies) { + // Our source item can be a file if it was a slice of an unpopulated file, + // or a slice of data that was then paged to disk. + size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length()); + DataElement::Type dest_type = copy.dest_item->item()->type(); + switch (copy.source_item->item()->type()) { + case DataElement::TYPE_BYTES: { + DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION); + const char* src_data = + copy.source_item->item()->bytes() + copy.source_item_offset; + copy.dest_item->item()->item_->SetToBytes(src_data, dest_size); + } break; + case DataElement::TYPE_FILE: + case DataElement::TYPE_UNKNOWN: + case DataElement::TYPE_BLOB: + case DataElement::TYPE_BYTES_DESCRIPTION: + case DataElement::TYPE_FILE_FILESYSTEM: + case DataElement::TYPE_DISK_CACHE_ENTRY: + NOTREACHED(); + break; } - break; + copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA); } - case DataElement::TYPE_DISK_CACHE_ENTRY: { - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry", - (length - offset) / 1024); - target_blob_builder->AppendSharedBlobItem( - new ShareableBlobDataItem(target_blob_uuid, blob_item)); - break; - } - case DataElement::TYPE_BYTES_DESCRIPTION: - case DataElement::TYPE_UNKNOWN: - NOTREACHED(); - break; + + entry->set_status(BlobStatus::DONE); + } + + std::vector<BlobStatusCallback> callbacks; + if (entry->building_state_.get()) { + std::swap(callbacks, entry->building_state_->build_completion_callbacks); + entry->building_state_.reset(); + } + + memory_controller_.NotifyMemoryItemsUsed(entry->items()); + + auto runner = base::ThreadTaskRunnerHandle::Get(); + for (const auto& callback : callbacks) + runner->PostTask(FROM_HERE, base::Bind(callback, entry->status())); + + for (const auto& shareable_item : entry->items()) { + DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION, + shareable_item->item()->type()); + DCHECK(shareable_item->IsPopulated()) << shareable_item->state(); } - UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend", - memory_usage_ / 1024); - return !error; } -bool BlobStorageContext::AppendBlob( - const std::string& target_blob_uuid, - const InternalBlobData& blob, - uint64_t offset, - uint64_t length, - InternalBlobData::Builder* target_blob_builder) { - DCHECK_GT(length, 0ull); - - const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items(); - auto iter = items.begin(); - if (offset) { - for (; iter != items.end(); ++iter) { - const BlobDataItem& item = *(iter->get()->item()); - if (offset >= item.length()) - offset -= item.length(); - else - break; - } +void BlobStorageContext::RequestTransport( + BlobEntry* entry, + std::vector<BlobMemoryController::FileCreationInfo> files) { + BlobEntry::BuildingState* building_state = entry->building_state_.get(); + if (building_state->transport_allowed_callback) { + base::ResetAndReturn(&building_state->transport_allowed_callback) + .Run(BlobStatus::PENDING_TRANSPORT, std::move(files)); + return; } + DCHECK(files.empty()); + NotifyTransportCompleteInternal(entry); +} - for (; iter != items.end() && length > 0; ++iter) { - scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get(); - const BlobDataItem& item = *(shareable_item->item()); - uint64_t item_length = item.length(); - DCHECK_GT(item_length, offset); - uint64_t current_length = item_length - offset; - uint64_t new_length = current_length > length ? length : current_length; +void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid, + bool success) { + if (!success) { + CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY); + return; + } + BlobEntry* entry = registry_.GetEntry(uuid); + if (!entry || !entry->building_state_.get()) + return; + BlobEntry::BuildingState& building_state = *entry->building_state_; + DCHECK(!building_state.memory_quota_request); + + if (building_state.transport_items_present) { + entry->set_status(BlobStatus::PENDING_TRANSPORT); + RequestTransport(entry, + std::vector<BlobMemoryController::FileCreationInfo>()); + } else { + entry->set_status(BlobStatus::PENDING_INTERNALS); + } - bool reusing_blob_item = offset == 0 && new_length == item.length(); - UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item); - if (reusing_blob_item) { - shareable_item->referencing_blobs().insert(target_blob_uuid); - target_blob_builder->AppendSharedBlobItem(shareable_item); - length -= new_length; - continue; - } + if (entry->CanFinishBuilding()) + FinishBuilding(entry); +} - // We need to do copying of the items when we have a different offset or - // length - switch (item.type()) { - case DataElement::TYPE_BYTES: { - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes", - new_length / 1024); - if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) { - return false; - } - DCHECK(!item.offset()); - std::unique_ptr<DataElement> element(new DataElement()); - element->SetToBytes(item.bytes() + offset, - static_cast<int64_t>(new_length)); - memory_usage_ += new_length; - target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( - target_blob_uuid, new BlobDataItem(std::move(element)))); - } break; - case DataElement::TYPE_FILE: { - DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max()) - << "We cannot use a section of a file with an unknown length"; - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File", - new_length / 1024); - std::unique_ptr<DataElement> element(new DataElement()); - element->SetToFilePathRange(item.path(), item.offset() + offset, - new_length, - item.expected_modification_time()); - target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( - target_blob_uuid, - new BlobDataItem(std::move(element), item.data_handle_))); - } break; - case DataElement::TYPE_FILE_FILESYSTEM: { - UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem", - new_length / 1024); - std::unique_ptr<DataElement> element(new DataElement()); - element->SetToFileSystemUrlRange(item.filesystem_url(), - item.offset() + offset, new_length, - item.expected_modification_time()); - target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( - target_blob_uuid, new BlobDataItem(std::move(element)))); - } break; - case DataElement::TYPE_DISK_CACHE_ENTRY: { - std::unique_ptr<DataElement> element(new DataElement()); - element->SetToDiskCacheEntryRange(item.offset() + offset, - new_length); - target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( - target_blob_uuid, - new BlobDataItem(std::move(element), item.data_handle_, - item.disk_cache_entry(), - item.disk_cache_stream_index(), - item.disk_cache_side_stream_index()))); - } break; - case DataElement::TYPE_BYTES_DESCRIPTION: - case DataElement::TYPE_BLOB: - case DataElement::TYPE_UNKNOWN: - CHECK(false) << "Illegal blob item type: " << item.type(); +void BlobStorageContext::OnDependentBlobFinished( + const std::string& owning_blob_uuid, + BlobStatus status) { + BlobEntry* entry = registry_.GetEntry(owning_blob_uuid); + if (!entry || !entry->building_state_) + return; + + if (BlobStatusIsError(status)) { + DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status) + << "Referenced blob should never be dereferenced while we " + << "are depending on it, as our system holds a handle."; + CancelBuildingBlobInternal(entry, BlobStatus::ERR_REFERENCED_BLOB_BROKEN); + return; + } + DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u); + --entry->building_state_->num_building_dependent_blobs; + + if (entry->CanFinishBuilding()) + FinishBuilding(entry); +} + +void BlobStorageContext::ClearAndFreeMemory(BlobEntry* entry) { + if (entry->building_state_) { + BlobEntry::BuildingState* building_state = entry->building_state_.get(); + if (building_state->memory_quota_request) { + building_state->memory_quota_request->Cancel(); } - length -= new_length; - offset = 0; } - return true; + entry->ClearItems(); + entry->ClearOffsets(); + entry->set_size(0); } } // namespace storage diff --git a/chromium/storage/browser/blob/blob_storage_context.h b/chromium/storage/browser/blob/blob_storage_context.h index 8553d0266d2..c64581e91c7 100644 --- a/chromium/storage/browser/blob/blob_storage_context.h +++ b/chromium/storage/browser/blob/blob_storage_context.h @@ -19,17 +19,18 @@ #include "base/memory/ref_counted.h" #include "base/memory/weak_ptr.h" #include "storage/browser/blob/blob_data_handle.h" +#include "storage/browser/blob/blob_entry.h" +#include "storage/browser/blob/blob_memory_controller.h" #include "storage/browser/blob/blob_storage_registry.h" -#include "storage/browser/blob/internal_blob_data.h" #include "storage/browser/storage_browser_export.h" #include "storage/common/blob_storage/blob_storage_constants.h" -#include "storage/common/data_element.h" class GURL; namespace base { class FilePath; class Time; +class TaskRunner; } namespace content { @@ -38,130 +39,212 @@ class BlobDispatcherHostTest; } namespace storage { - class BlobDataBuilder; +class BlobDataHandle; class BlobDataItem; class BlobDataSnapshot; class ShareableBlobDataItem; -// This class handles the logistics of blob Storage within the browser process, -// and maintains a mapping from blob uuid to the data. The class is single -// threaded and should only be used on the IO thread. -// In chromium, there is one instance per profile. -class STORAGE_EXPORT BlobStorageContext - : public base::SupportsWeakPtr<BlobStorageContext> { +// This class handles the logistics of blob storage within the browser process. +// We are single threaded and should only be used on the IO thread. In Chromium +// there is one instance per profile. +class STORAGE_EXPORT BlobStorageContext { public: + using TransportAllowedCallback = BlobEntry::TransportAllowedCallback; + + // Initializes the context without disk support. BlobStorageContext(); ~BlobStorageContext(); std::unique_ptr<BlobDataHandle> GetBlobDataFromUUID(const std::string& uuid); std::unique_ptr<BlobDataHandle> GetBlobDataFromPublicURL(const GURL& url); - // Useful for coining blobs from within the browser process. If the - // blob cannot be added due to memory consumption, returns NULL. - // A builder should not be used twice to create blobs, as the internal - // resources are refcounted instead of copied for memory efficiency. - // To cleanly use a builder multiple times, please call Clone() on the - // builder, or even better for memory savings, clear the builder and append - // the previously constructed blob. + // Always returns a handle to a blob. Use BlobStatus::GetBlobStatus() and + // BlobStatus::RunOnConstructionComplete(callback) to determine construction + // completion and possible errors. std::unique_ptr<BlobDataHandle> AddFinishedBlob( const BlobDataBuilder& builder); - // Deprecated, use const ref version above. + // Deprecated, use const ref version above or BuildBlob below. std::unique_ptr<BlobDataHandle> AddFinishedBlob( const BlobDataBuilder* builder); + std::unique_ptr<BlobDataHandle> AddBrokenBlob( + const std::string& uuid, + const std::string& content_type, + const std::string& content_disposition, + BlobStatus reason); + // Useful for coining blob urls from within the browser process. bool RegisterPublicBlobURL(const GURL& url, const std::string& uuid); void RevokePublicBlobURL(const GURL& url); - size_t memory_usage() const { return memory_usage_; } size_t blob_count() const { return registry_.blob_count(); } - size_t memory_available() const { - return kBlobStorageMaxMemoryUsage - memory_usage_; - } const BlobStorageRegistry& registry() { return registry_; } - private: - using BlobRegistryEntry = BlobStorageRegistry::Entry; - using BlobConstructedCallback = BlobStorageRegistry::BlobConstructedCallback; + // This builds a blob with the given |input_builder| and returns a handle to + // the constructed Blob. Blob metadata and data should be accessed through + // this handle. + // If there is data present that needs further population then we will call + // |transport_allowed_callback| when we're ready for the user data to be + // populated with the PENDING_DATA_POPULATION status. This can happen + // synchronously or asynchronously. Otherwise |transport_allowed_callback| + // should be null. In the further population case, the caller must call either + // NotifyTransportComplete or CancelBuildingBlob after + // |transport_allowed_callback| is called to signify the data is finished + // populating or an error occurred (respectively). + // If the returned handle is broken, then the possible error cases are: + // * OUT_OF_MEMORY if we don't have enough memory to store the blob, + // * REFERENCED_BLOB_BROKEN if a referenced blob is broken or we're + // referencing ourself. + std::unique_ptr<BlobDataHandle> BuildBlob( + const BlobDataBuilder& input_builder, + const TransportAllowedCallback& transport_allowed_callback); + + // This breaks a blob that is currently being built by using the BuildBlob + // method above. Any callbacks waiting on this blob, including the + // |transport_allowed_callback| callback given to BuildBlob, will be called + // with this status code. + void CancelBuildingBlob(const std::string& uuid, BlobStatus code); + + // After calling BuildBlob above, the caller should call this method to + // notify the construction system that the unpopulated data in the given blob + // has been. populated. Caller must have all pending items populated in the + // original builder |input_builder| given in BuildBlob or we'll check-fail. + // If there is no pending data in the |input_builder| for the BuildBlob call, + // then this method doesn't need to be called. + void NotifyTransportComplete(const std::string& uuid); + + const BlobMemoryController& memory_controller() { return memory_controller_; } + + base::WeakPtr<BlobStorageContext> AsWeakPtr() { + return ptr_factory_.GetWeakPtr(); + } + + protected: friend class content::BlobDispatcherHost; - friend class BlobAsyncBuilderHost; - friend class BlobAsyncBuilderHostTest; + friend class content::BlobDispatcherHostTest; + friend class BlobTransportHost; + friend class BlobTransportHostTest; friend class BlobDataHandle; friend class BlobDataHandle::BlobDataHandleShared; - friend class BlobReaderTest; - FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, HandleBeforeAsyncCancel); - FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, ReadFromIncompleteBlob); + friend class BlobFlattenerTest; + friend class BlobSliceTest; friend class BlobStorageContextTest; - FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, IncrementDecrementRef); - FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, OnCancelBuildingBlob); - FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, PublicBlobUrls); - FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, - TestUnknownBrokenAndBuildingBlobReference); - friend class ViewBlobInternalsJob; - - // CompletePendingBlob or CancelPendingBlob should be called after this. - void CreatePendingBlob(const std::string& uuid, - const std::string& content_type, - const std::string& content_disposition); - - // This includes resolving blob references in the builder. This will run the - // callbacks given in RunOnConstructionComplete. - void CompletePendingBlob(const BlobDataBuilder& external_builder); - - // This will run the callbacks given in RunOnConstructionComplete. - void CancelPendingBlob(const std::string& uuid, - IPCBlobCreationCancelCode reason); + + // Transforms a BlobDataBuilder into a BlobEntry with no blob references. + // BlobSlice is used to flatten out these references. Records the total size + // and items for memory and file quota requests. + // Exposed in the header file for testing. + struct STORAGE_EXPORT BlobFlattener { + BlobFlattener(const BlobDataBuilder& input_builder, + BlobEntry* output_blob, + BlobStorageRegistry* registry); + ~BlobFlattener(); + + // This can be: + // * PENDING_QUOTA if we need memory quota, if we're populated and don't + // need quota. + // * PENDING_INTERNALS if we're waiting on dependent blobs or we're done. + // * INVALID_CONSTRUCTION_ARGUMENTS if we have invalid input. + // * REFERENCED_BLOB_BROKEN if one of the referenced blobs is broken or we + // reference ourself. + BlobStatus status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; + + // This is the total size of the blob, including all memory, files, etc. + uint64_t total_size = 0; + // Total memory size of the blob (not including files, etc). + uint64_t total_memory_size = 0; + + std::vector<std::pair<std::string, BlobEntry*>> dependent_blobs; + + uint64_t memory_quota_needed = 0; + std::vector<scoped_refptr<ShareableBlobDataItem>> pending_memory_items; + + std::vector<ShareableBlobDataItem*> transport_items; + + // These record all future copies we'll need to do from referenced blobs. + // This + // happens when we do a partial slice from a pending data or file item. + std::vector<BlobEntry::ItemCopyEntry> copies; + + private: + DISALLOW_COPY_AND_ASSIGN(BlobFlattener); + }; + + // Used when a blob reference has a size and offset. Records the source items + // and memory we need to copy if either side of slice intersects an item. + // Exposed in the header file for testing. + struct STORAGE_EXPORT BlobSlice { + BlobSlice(const BlobEntry& source, + uint64_t slice_offset, + uint64_t slice_size); + ~BlobSlice(); + + // Size of memory copying from the source blob. + base::CheckedNumeric<size_t> copying_memory_size = 0; + // Size of all memory for UMA stats. + base::CheckedNumeric<size_t> total_memory_size = 0; + + size_t first_item_slice_offset = 0; + // Populated if our first slice item is a temporary item that we'll copy to + // later from this |first_source_item|, at offset |first_item_slice_offset|. + scoped_refptr<ShareableBlobDataItem> first_source_item; + // Populated if our last slice item is a temporary item that we'll copy to + // later from this |last_source_item|. + scoped_refptr<ShareableBlobDataItem> last_source_item; + + std::vector<scoped_refptr<ShareableBlobDataItem>> dest_items; + + private: + DISALLOW_COPY_AND_ASSIGN(BlobSlice); + }; void IncrementBlobRefCount(const std::string& uuid); void DecrementBlobRefCount(const std::string& uuid); - // Methods called by BlobDataHandle: // This will return an empty snapshot until the blob is complete. // TODO(dmurph): After we make the snapshot method in BlobHandle private, then // make this DCHECK on the blob not being complete. std::unique_ptr<BlobDataSnapshot> CreateSnapshot(const std::string& uuid); - bool IsBroken(const std::string& uuid) const; - bool IsBeingBuilt(const std::string& uuid) const; - // Runs |done| when construction completes, with true if it was successful, - // and false if there was an error, which is reported in the second argument - // of the callback. + + BlobStatus GetBlobStatus(const std::string& uuid) const; + + // Runs |done| when construction completes with the final status of the blob. void RunOnConstructionComplete(const std::string& uuid, - const BlobConstructedCallback& done); - - // Appends the given blob item to the blob builder. The new blob - // retains ownership of data_item if applicable, and returns false if there - // was an error and pouplates the error_code. We can either have an error of: - // OUT_OF_MEMORY: We are out of memory to store this blob. - // REFERENCED_BLOB_BROKEN: One of the referenced blobs is broken. - bool AppendAllocatedBlobItem(const std::string& target_blob_uuid, - scoped_refptr<BlobDataItem> data_item, - InternalBlobData::Builder* target_blob_data, - IPCBlobCreationCancelCode* error_code); - - // Allocates a shareable blob data item, with blob references resolved. If - // there isn't enough memory, then a nullptr is returned. - scoped_refptr<ShareableBlobDataItem> AllocateShareableBlobDataItem( - const std::string& target_blob_uuid, - scoped_refptr<BlobDataItem> data_item); - - // Deconstructs the blob and appends it's contents to the target blob. Items - // are shared if possible, and copied if the given offset and length - // have to split an item. - bool AppendBlob(const std::string& target_blob_uuid, - const InternalBlobData& blob, - uint64_t offset, - uint64_t length, - InternalBlobData::Builder* target_blob_data); + const BlobStatusCallback& done_callback); - BlobStorageRegistry registry_; + BlobStorageRegistry* mutable_registry() { return ®istry_; } - // Used to keep track of how much memory is being utilized for blob data, - // we count only the items of TYPE_DATA which are held in memory and not - // items of TYPE_FILE. - size_t memory_usage_; + BlobMemoryController* mutable_memory_controller() { + return &memory_controller_; + } + + private: + std::unique_ptr<BlobDataHandle> CreateHandle(const std::string& uuid, + BlobEntry* entry); + + void NotifyTransportCompleteInternal(BlobEntry* entry); + + void CancelBuildingBlobInternal(BlobEntry* entry, BlobStatus reason); + + void FinishBuilding(BlobEntry* entry); + + void RequestTransport( + BlobEntry* entry, + std::vector<BlobMemoryController::FileCreationInfo> files); + + void OnEnoughSizeForMemory(const std::string& uuid, bool can_fit); + + void OnDependentBlobFinished(const std::string& owning_blob_uuid, + BlobStatus reason); + + void ClearAndFreeMemory(BlobEntry* entry); + + BlobStorageRegistry registry_; + BlobMemoryController memory_controller_; + base::WeakPtrFactory<BlobStorageContext> ptr_factory_; DISALLOW_COPY_AND_ASSIGN(BlobStorageContext); }; diff --git a/chromium/storage/browser/blob/blob_storage_registry.cc b/chromium/storage/browser/blob/blob_storage_registry.cc index 9ddcb99b555..47b4bd8d602 100644 --- a/chromium/storage/browser/blob/blob_storage_registry.cc +++ b/chromium/storage/browser/blob/blob_storage_registry.cc @@ -9,15 +9,13 @@ #include <memory> #include "base/bind.h" -#include "base/callback.h" #include "base/location.h" #include "base/logging.h" -#include "base/message_loop/message_loop.h" #include "base/stl_util.h" +#include "storage/browser/blob/blob_entry.h" #include "url/gurl.h" namespace storage { -using BlobState = BlobStorageRegistry::BlobState; namespace { // We can't use GURL directly for these hash fragment manipulations @@ -38,18 +36,6 @@ GURL ClearBlobUrlRef(const GURL& url) { } // namespace -BlobStorageRegistry::Entry::Entry(int refcount, BlobState state) - : refcount(refcount), state(state) {} - -BlobStorageRegistry::Entry::~Entry() {} - -bool BlobStorageRegistry::Entry::TestAndSetState(BlobState expected, - BlobState set) { - if (state != expected) - return false; - state = set; - return true; -} BlobStorageRegistry::BlobStorageRegistry() {} @@ -59,15 +45,14 @@ BlobStorageRegistry::~BlobStorageRegistry() { // So it shouldn't matter. } -BlobStorageRegistry::Entry* BlobStorageRegistry::CreateEntry( +BlobEntry* BlobStorageRegistry::CreateEntry( const std::string& uuid, const std::string& content_type, const std::string& content_disposition) { - DCHECK(!base::ContainsKey(blob_map_, uuid)); - std::unique_ptr<Entry> entry(new Entry(1, BlobState::PENDING)); - entry->content_type = content_type; - entry->content_disposition = content_disposition; - Entry* entry_ptr = entry.get(); + DCHECK(!ContainsKey(blob_map_, uuid)); + std::unique_ptr<BlobEntry> entry( + new BlobEntry(content_type, content_disposition)); + BlobEntry* entry_ptr = entry.get(); blob_map_.add(uuid, std::move(entry)); return entry_ptr; } @@ -80,16 +65,14 @@ bool BlobStorageRegistry::HasEntry(const std::string& uuid) const { return blob_map_.find(uuid) != blob_map_.end(); } -BlobStorageRegistry::Entry* BlobStorageRegistry::GetEntry( - const std::string& uuid) { +BlobEntry* BlobStorageRegistry::GetEntry(const std::string& uuid) { BlobMap::iterator found = blob_map_.find(uuid); if (found == blob_map_.end()) return nullptr; return found->second; } -const BlobStorageRegistry::Entry* BlobStorageRegistry::GetEntry( - const std::string& uuid) const { +const BlobEntry* BlobStorageRegistry::GetEntry(const std::string& uuid) const { return const_cast<BlobStorageRegistry*>(this)->GetEntry(uuid); } @@ -118,14 +101,13 @@ bool BlobStorageRegistry::IsURLMapped(const GURL& blob_url) const { return base::ContainsKey(url_to_uuid_, blob_url); } -BlobStorageRegistry::Entry* BlobStorageRegistry::GetEntryFromURL( - const GURL& url, - std::string* uuid) { +BlobEntry* BlobStorageRegistry::GetEntryFromURL(const GURL& url, + std::string* uuid) { URLMap::iterator found = url_to_uuid_.find(BlobUrlHasRef(url) ? ClearBlobUrlRef(url) : url); if (found == url_to_uuid_.end()) return nullptr; - Entry* entry = GetEntry(found->second); + BlobEntry* entry = GetEntry(found->second); if (entry && uuid) uuid->assign(found->second); return entry; diff --git a/chromium/storage/browser/blob/blob_storage_registry.h b/chromium/storage/browser/blob/blob_storage_registry.h index 5211094b61a..2b2dbdd13f2 100644 --- a/chromium/storage/browser/blob/blob_storage_registry.h +++ b/chromium/storage/browser/blob/blob_storage_registry.h @@ -15,73 +15,31 @@ #include "base/callback_forward.h" #include "base/containers/scoped_ptr_hash_map.h" #include "base/macros.h" -#include "storage/browser/blob/internal_blob_data.h" #include "storage/browser/storage_browser_export.h" #include "storage/common/blob_storage/blob_storage_constants.h" class GURL; namespace storage { +class BlobDataHandle; +class BlobEntry; +class ShareableBlobDataItem; // This class stores the blob data in the various states of construction, as // well as URL mappings to blob uuids. // Implementation notes: -// * There is no implicit refcounting in this class, except for setting the -// refcount to 1 on registration. // * When removing a uuid registration, we do not check for URL mappings to that // uuid. The user must keep track of these. class STORAGE_EXPORT BlobStorageRegistry { public: - // True means the blob was constructed successfully, and false means that - // there was an error, which is reported in the second argument. - using BlobConstructedCallback = - base::Callback<void(bool, IPCBlobCreationCancelCode)>; - - enum class BlobState { - // The blob is pending transportation from the renderer. This is the default - // state on entry construction. - PENDING, - // The blob is complete and can be read from. - COMPLETE, - // The blob is broken and no longer holds data. This happens when there was - // a problem constructing the blob, or we've run out of memory. - BROKEN - }; - - struct STORAGE_EXPORT Entry { - size_t refcount; - BlobState state; - std::vector<BlobConstructedCallback> build_completion_callbacks; - - // Only applicable if the state == BROKEN. - IPCBlobCreationCancelCode broken_reason = - IPCBlobCreationCancelCode::UNKNOWN; - - // data and data_builder are mutually exclusive. - std::unique_ptr<InternalBlobData> data; - std::unique_ptr<InternalBlobData::Builder> data_builder; - - std::string content_type; - std::string content_disposition; - - Entry() = delete; - Entry(int refcount, BlobState state); - ~Entry(); - - // Performs a test-and-set on the state of the given blob. If the state - // isn't as expected, we return false. Otherwise we set the new state and - // return true. - bool TestAndSetState(BlobState expected, BlobState set); - }; - BlobStorageRegistry(); ~BlobStorageRegistry(); // Creates the blob entry with a refcount of 1 and a state of PENDING. If // the blob is already in use, we return null. - Entry* CreateEntry(const std::string& uuid, - const std::string& content_type, - const std::string& content_disposition); + BlobEntry* CreateEntry(const std::string& uuid, + const std::string& content_type, + const std::string& content_disposition); // Removes the blob entry with the given uuid. This does not unmap any // URLs that are pointing to this uuid. Returns if the entry existed. @@ -91,8 +49,8 @@ class STORAGE_EXPORT BlobStorageRegistry { // Gets the blob entry for the given uuid. Returns nullptr if the entry // does not exist. - Entry* GetEntry(const std::string& uuid); - const Entry* GetEntry(const std::string& uuid) const; + BlobEntry* GetEntry(const std::string& uuid); + const BlobEntry* GetEntry(const std::string& uuid) const; // Creates a url mapping from blob uuid to the given url. Returns false if // the uuid isn't mapped to an entry or if there already is a map for the URL. @@ -107,14 +65,15 @@ class STORAGE_EXPORT BlobStorageRegistry { // Returns the entry from the given url, and optionally populates the uuid for // that entry. Returns a nullptr if the mapping or entry doesn't exist. - Entry* GetEntryFromURL(const GURL& url, std::string* uuid); + BlobEntry* GetEntryFromURL(const GURL& url, std::string* uuid); size_t blob_count() const { return blob_map_.size(); } size_t url_count() const { return url_to_uuid_.size(); } private: friend class ViewBlobInternalsJob; - using BlobMap = base::ScopedPtrHashMap<std::string, std::unique_ptr<Entry>>; + using BlobMap = + base::ScopedPtrHashMap<std::string, std::unique_ptr<BlobEntry>>; using URLMap = std::map<GURL, std::string>; BlobMap blob_map_; diff --git a/chromium/storage/browser/blob/blob_transport_host.cc b/chromium/storage/browser/blob/blob_transport_host.cc new file mode 100644 index 00000000000..032b9ba45ed --- /dev/null +++ b/chromium/storage/browser/blob/blob_transport_host.cc @@ -0,0 +1,506 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <stddef.h> +#include <stdint.h> + +#include <memory> +#include <utility> + +#include "base/bind.h" +#include "base/callback_helpers.h" +#include "base/memory/ptr_util.h" +#include "base/memory/shared_memory.h" +#include "base/stl_util.h" +#include "storage/browser/blob/blob_data_handle.h" +#include "storage/browser/blob/blob_memory_controller.h" +#include "storage/browser/blob/blob_storage_context.h" +#include "storage/browser/blob/blob_transport_host.h" + +namespace storage { +namespace { +using MemoryStrategy = BlobMemoryController::Strategy; +using MemoryItemRequest = + BlobTransportRequestBuilder::RendererMemoryItemRequest; + +bool CalculateBlobMemorySize(const std::vector<DataElement>& elements, + size_t* shortcut_bytes, + uint64_t* total_bytes) { + DCHECK(shortcut_bytes); + DCHECK(total_bytes); + + base::CheckedNumeric<uint64_t> total_size_checked = 0; + base::CheckedNumeric<size_t> shortcut_size_checked = 0; + for (const auto& e : elements) { + if (e.type() == DataElement::TYPE_BYTES) { + total_size_checked += e.length(); + shortcut_size_checked += e.length(); + } else if (e.type() == DataElement::TYPE_BYTES_DESCRIPTION) { + total_size_checked += e.length(); + } else { + continue; + } + if (!total_size_checked.IsValid() || !shortcut_size_checked.IsValid()) + return false; + } + *shortcut_bytes = shortcut_size_checked.ValueOrDie(); + *total_bytes = total_size_checked.ValueOrDie(); + return true; +} +} // namespace + +BlobTransportHost::TransportState::TransportState( + const std::string& uuid, + const std::string& content_type, + const std::string& content_disposition, + RequestMemoryCallback request_memory_callback, + BlobStatusCallback completion_callback) + : data_builder(uuid), + request_memory_callback(std::move(request_memory_callback)), + completion_callback(std::move(completion_callback)) { + data_builder.set_content_type(content_type); + data_builder.set_content_disposition(content_disposition); +} + +BlobTransportHost::TransportState::TransportState( + BlobTransportHost::TransportState&&) = default; +BlobTransportHost::TransportState& BlobTransportHost::TransportState::operator=( + BlobTransportHost::TransportState&&) = default; + +BlobTransportHost::TransportState::~TransportState() {} + +BlobTransportHost::BlobTransportHost() : ptr_factory_(this) {} + +BlobTransportHost::~BlobTransportHost() {} + +std::unique_ptr<BlobDataHandle> BlobTransportHost::StartBuildingBlob( + const std::string& uuid, + const std::string& content_type, + const std::string& content_disposition, + const std::vector<DataElement>& elements, + BlobStorageContext* context, + const RequestMemoryCallback& request_memory, + const BlobStatusCallback& completion_callback) { + DCHECK(context); + DCHECK(async_blob_map_.find(uuid) == async_blob_map_.end()); + std::unique_ptr<BlobDataHandle> handle; + // Validate that our referenced blobs aren't us. + for (const DataElement& e : elements) { + if (e.type() == DataElement::TYPE_BLOB && e.blob_uuid() == uuid) { + handle = context->AddBrokenBlob( + uuid, content_type, content_disposition, + BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS); + completion_callback.Run(BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS); + return handle; + } + } + uint64_t transport_memory_size = 0; + size_t shortcut_size = 0; + if (!CalculateBlobMemorySize(elements, &shortcut_size, + &transport_memory_size)) { + handle = + context->AddBrokenBlob(uuid, content_type, content_disposition, + BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS); + completion_callback.Run(BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS); + return handle; + } + + const BlobMemoryController& memory_controller = context->memory_controller(); + MemoryStrategy memory_strategy = + memory_controller.DetermineStrategy(shortcut_size, transport_memory_size); + TransportState state(uuid, content_type, content_disposition, request_memory, + completion_callback); + switch (memory_strategy) { + case MemoryStrategy::TOO_LARGE: { + handle = context->AddBrokenBlob(uuid, content_type, content_disposition, + BlobStatus::ERR_OUT_OF_MEMORY); + completion_callback.Run(BlobStatus::ERR_OUT_OF_MEMORY); + return handle; + } + case MemoryStrategy::NONE_NEEDED: { + // This means we don't need to transport anything, so just send the + // elements along and tell our callback we're done transporting. + for (const DataElement& e : elements) { + DCHECK_NE(e.type(), DataElement::TYPE_BYTES_DESCRIPTION); + state.data_builder.AppendIPCDataElement(e); + } + handle = context->BuildBlob( + state.data_builder, BlobStorageContext::TransportAllowedCallback()); + completion_callback.Run(BlobStatus::DONE); + return handle; + } + case MemoryStrategy::IPC: + state.strategy = IPCBlobItemRequestStrategy::IPC; + state.request_builder.InitializeForIPCRequests( + memory_controller.limits().max_ipc_memory_size, transport_memory_size, + elements, &(state.data_builder)); + break; + case MemoryStrategy::SHARED_MEMORY: + state.strategy = IPCBlobItemRequestStrategy::SHARED_MEMORY; + state.request_builder.InitializeForSharedMemoryRequests( + memory_controller.limits().max_shared_memory_size, + transport_memory_size, elements, &(state.data_builder)); + break; + case MemoryStrategy::FILE: + state.strategy = IPCBlobItemRequestStrategy::FILE; + state.request_builder.InitializeForFileRequests( + memory_controller.limits().max_file_size, transport_memory_size, + elements, &(state.data_builder)); + break; + } + // We initialize our requests received state now that they are populated. + state.request_received.resize(state.request_builder.requests().size(), false); + auto it_pair = async_blob_map_.insert(std::make_pair(uuid, std::move(state))); + + // OnReadyForTransport can be called synchronously. + handle = context->BuildBlob( + it_pair.first->second.data_builder, + base::Bind(&BlobTransportHost::OnReadyForTransport, + ptr_factory_.GetWeakPtr(), uuid, context->AsWeakPtr())); + return handle; +} + +void BlobTransportHost::OnMemoryResponses( + const std::string& uuid, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context) { + AsyncBlobMap::iterator state_it = async_blob_map_.find(uuid); + DCHECK(state_it != async_blob_map_.end()) << "Could not find blob " << uuid; + if (responses.empty()) { + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + + // Validate response sanity: it should refer to a legal request number, and + // we shouldn't have received an answer for that request yet. + TransportState& state = state_it->second; + const auto& requests = state.request_builder.requests(); + for (const BlobItemBytesResponse& response : responses) { + if (response.request_number >= requests.size()) { + // Bad IPC, so we delete our record and ignore. + DVLOG(1) << "Invalid request number " << response.request_number; + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + DCHECK_LT(response.request_number, state.request_received.size()); + if (state.request_received[response.request_number]) { + // Bad IPC, so we delete our record. + DVLOG(1) << "Already received response for that request."; + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + state.request_received[response.request_number] = true; + } + switch (state.strategy) { + case IPCBlobItemRequestStrategy::IPC: + OnIPCResponses(uuid, &state, responses, context); + break; + case IPCBlobItemRequestStrategy::SHARED_MEMORY: + OnSharedMemoryResponses(uuid, &state, responses, context); + break; + case IPCBlobItemRequestStrategy::FILE: + OnFileResponses(uuid, &state, responses, context); + break; + case IPCBlobItemRequestStrategy::UNKNOWN: + NOTREACHED(); + break; + } +} + +void BlobTransportHost::CancelBuildingBlob(const std::string& uuid, + BlobStatus code, + BlobStorageContext* context) { + DCHECK(context); + DCHECK(BlobStatusIsError(code)); + AsyncBlobMap::iterator state_it = async_blob_map_.find(uuid); + if (state_it == async_blob_map_.end()) + return; + // We can have the blob dereferenced by the renderer, but have it still being + // 'built'. In this case, it's destructed in the context, but we still have + // it in our map. Hence we make sure the context has the entry before + // calling cancel. + BlobStatusCallback completion_callback = state_it->second.completion_callback; + async_blob_map_.erase(state_it); + if (context->registry().HasEntry(uuid)) + context->CancelBuildingBlob(uuid, code); + + completion_callback.Run(code); +} + +void BlobTransportHost::CancelAll(BlobStorageContext* context) { + DCHECK(context); + // We cancel all of our blobs just in case another renderer referenced them. + std::vector<std::string> pending_blobs; + for (const auto& uuid_state_pair : async_blob_map_) { + pending_blobs.push_back(uuid_state_pair.second.data_builder.uuid()); + } + // We clear the map before canceling them to prevent any strange reentry into + // our class (see OnReadyForTransport) if any blobs were waiting for others + // to construct. + async_blob_map_.clear(); + for (auto& uuid : pending_blobs) { + if (context->registry().HasEntry(uuid)) + context->CancelBuildingBlob(uuid, BlobStatus::ERR_SOURCE_DIED_IN_TRANSIT); + } +} + +void BlobTransportHost::StartRequests( + const std::string& uuid, + TransportState* state, + BlobStorageContext* context, + std::vector<BlobMemoryController::FileCreationInfo> file_infos) { + switch (state->strategy) { + case IPCBlobItemRequestStrategy::IPC: + DCHECK(file_infos.empty()); + SendIPCRequests(state, context); + break; + case IPCBlobItemRequestStrategy::SHARED_MEMORY: + DCHECK(file_infos.empty()); + ContinueSharedMemoryRequests(uuid, state, context); + break; + case IPCBlobItemRequestStrategy::FILE: + DCHECK(!file_infos.empty()); + SendFileRequests(state, context, std::move(file_infos)); + break; + case IPCBlobItemRequestStrategy::UNKNOWN: + NOTREACHED(); + break; + } +} + +// Note: This can be called when we cancel a blob in the context. +void BlobTransportHost::OnReadyForTransport( + const std::string& uuid, + base::WeakPtr<BlobStorageContext> context, + BlobStatus status, + std::vector<BlobMemoryController::FileCreationInfo> file_infos) { + if (!context) { + async_blob_map_.erase(uuid); + return; + } + AsyncBlobMap::iterator state_it = async_blob_map_.find(uuid); + if (state_it == async_blob_map_.end()) + return; + + TransportState& state = state_it->second; + if (BlobStatusIsPending(status)) { + DCHECK(status == BlobStatus::PENDING_TRANSPORT); + StartRequests(uuid, &state, context.get(), std::move(file_infos)); + return; + } + // Done or error. + BlobStatusCallback completion_callback = state.completion_callback; + async_blob_map_.erase(state_it); + completion_callback.Run(status); +} + +void BlobTransportHost::SendIPCRequests(TransportState* state, + BlobStorageContext* context) { + const std::vector<MemoryItemRequest>& requests = + state->request_builder.requests(); + std::vector<BlobItemBytesRequest> byte_requests; + + DCHECK(!requests.empty()); + for (const MemoryItemRequest& request : requests) { + byte_requests.push_back(request.message); + } + + state->request_memory_callback.Run(std::move(byte_requests), + std::vector<base::SharedMemoryHandle>(), + std::vector<base::File>()); +} + +void BlobTransportHost::OnIPCResponses( + const std::string& uuid, + TransportState* state, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context) { + const auto& requests = state->request_builder.requests(); + size_t num_requests = requests.size(); + for (const BlobItemBytesResponse& response : responses) { + const MemoryItemRequest& request = requests[response.request_number]; + if (response.inline_data.size() < request.message.size) { + DVLOG(1) << "Invalid data size " << response.inline_data.size() + << " vs requested size of " << request.message.size; + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + bool success = state->data_builder.PopulateFutureData( + request.browser_item_index, response.inline_data.data(), + request.browser_item_offset, request.message.size); + if (!success) { + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + state->num_fulfilled_requests++; + } + if (state->num_fulfilled_requests == num_requests) + CompleteTransport(state, context); +} + +void BlobTransportHost::ContinueSharedMemoryRequests( + const std::string& uuid, + TransportState* state, + BlobStorageContext* context) { + BlobTransportRequestBuilder& request_builder = state->request_builder; + const std::vector<MemoryItemRequest>& requests = request_builder.requests(); + size_t num_requests = requests.size(); + DCHECK_LT(state->num_fulfilled_requests, num_requests); + if (state->next_request == num_requests) { + // We are still waiting on other requests to come back. + return; + } + + std::vector<BlobItemBytesRequest> byte_requests; + std::vector<base::SharedMemoryHandle> shared_memory; + + for (; state->next_request < num_requests; ++state->next_request) { + const MemoryItemRequest& request = requests[state->next_request]; + bool using_shared_memory_handle = state->num_shared_memory_requests > 0; + if (using_shared_memory_handle && + state->current_shared_memory_handle_index != + request.message.handle_index) { + // We only want one shared memory per requesting blob. + break; + } + state->current_shared_memory_handle_index = request.message.handle_index; + state->num_shared_memory_requests++; + + if (!state->shared_memory_block) { + state->shared_memory_block.reset(new base::SharedMemory()); + size_t size = + request_builder.shared_memory_sizes()[request.message.handle_index]; + if (!state->shared_memory_block->CreateAnonymous(size)) { + DVLOG(1) << "Unable to allocate shared memory for blob transfer."; + CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY, context); + return; + } + } + shared_memory.push_back(state->shared_memory_block->handle()); + byte_requests.push_back(request.message); + // Since we are only using one handle at a time, transform our handle + // index correctly back to 0. + byte_requests.back().handle_index = 0; + } + DCHECK(!requests.empty()); + + state->request_memory_callback.Run(std::move(byte_requests), + std::move(shared_memory), + std::vector<base::File>()); + return; +} + +void BlobTransportHost::OnSharedMemoryResponses( + const std::string& uuid, + TransportState* state, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context) { + BlobTransportRequestBuilder& request_builder = state->request_builder; + const auto& requests = request_builder.requests(); + for (const BlobItemBytesResponse& response : responses) { + const MemoryItemRequest& request = requests[response.request_number]; + if (state->num_shared_memory_requests == 0) { + DVLOG(1) << "Received too many responses for shared memory."; + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + state->num_shared_memory_requests--; + if (!state->shared_memory_block->memory()) { + // We just map the whole block, as we'll probably be accessing the + // whole thing in this group of responses. + size_t handle_size = + request_builder + .shared_memory_sizes()[state->current_shared_memory_handle_index]; + if (!state->shared_memory_block->Map(handle_size)) { + DVLOG(1) << "Unable to map memory to size " << handle_size; + CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY, context); + return; + } + } + + bool success = state->data_builder.PopulateFutureData( + request.browser_item_index, + static_cast<const char*>(state->shared_memory_block->memory()) + + request.message.handle_offset, + request.browser_item_offset, request.message.size); + + if (!success) { + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + state->num_fulfilled_requests++; + } + if (state->num_fulfilled_requests == requests.size()) { + CompleteTransport(state, context); + return; + } + ContinueSharedMemoryRequests(uuid, state, context); +} + +void BlobTransportHost::SendFileRequests( + TransportState* state, + BlobStorageContext* context, + std::vector<BlobMemoryController::FileCreationInfo> file_infos) { + std::vector<base::File> files; + + for (BlobMemoryController::FileCreationInfo& file_info : file_infos) { + state->files.push_back(std::move(file_info.file_reference)); + files.push_back(std::move(file_info.file)); + } + + const std::vector<MemoryItemRequest>& requests = + state->request_builder.requests(); + std::vector<BlobItemBytesRequest> byte_requests; + + DCHECK(!requests.empty()); + for (const MemoryItemRequest& request : requests) { + byte_requests.push_back(request.message); + } + + state->request_memory_callback.Run(std::move(byte_requests), + std::vector<base::SharedMemoryHandle>(), + std::move(files)); +} + +void BlobTransportHost::OnFileResponses( + const std::string& uuid, + TransportState* state, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context) { + BlobTransportRequestBuilder& request_builder = state->request_builder; + const auto& requests = request_builder.requests(); + for (const BlobItemBytesResponse& response : responses) { + const MemoryItemRequest& request = requests[response.request_number]; + const scoped_refptr<ShareableFileReference>& file_ref = + state->files[request.message.handle_index]; + bool success = state->data_builder.PopulateFutureFile( + request.browser_item_index, file_ref, response.time_file_modified); + if (!success) { + CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS, + context); + return; + } + state->num_fulfilled_requests++; + } + if (state->num_fulfilled_requests == requests.size()) + CompleteTransport(state, context); +} + +void BlobTransportHost::CompleteTransport(TransportState* state, + BlobStorageContext* context) { + std::string uuid = state->data_builder.uuid(); + BlobStatusCallback complete_callback = state->completion_callback; + async_blob_map_.erase(uuid); + context->NotifyTransportComplete(uuid); + complete_callback.Run(BlobStatus::DONE); +} + +} // namespace storage diff --git a/chromium/storage/browser/blob/blob_transport_host.h b/chromium/storage/browser/blob/blob_transport_host.h new file mode 100644 index 00000000000..f8e952a787a --- /dev/null +++ b/chromium/storage/browser/blob/blob_transport_host.h @@ -0,0 +1,186 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_HOST_H_ +#define STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_HOST_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "base/callback.h" +#include "base/files/file.h" +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/memory/shared_memory_handle.h" +#include "base/memory/weak_ptr.h" +#include "storage/browser/blob/blob_data_builder.h" +#include "storage/browser/blob/blob_memory_controller.h" +#include "storage/browser/blob/blob_transport_request_builder.h" +#include "storage/browser/storage_browser_export.h" +#include "storage/common/blob_storage/blob_item_bytes_request.h" +#include "storage/common/blob_storage/blob_item_bytes_response.h" +#include "storage/common/blob_storage/blob_storage_constants.h" +#include "storage/common/data_element.h" + +namespace base { +class SharedMemory; +} + +namespace storage { +class BlobDataHandle; +class BlobStorageContext; + +// This class facilitates moving memory from the renderer to the browser. +class STORAGE_EXPORT BlobTransportHost { + public: + // One is expected to use std::move when calling this callback. + using RequestMemoryCallback = + base::Callback<void(std::vector<storage::BlobItemBytesRequest>, + std::vector<base::SharedMemoryHandle>, + std::vector<base::File>)>; + + BlobTransportHost(); + ~BlobTransportHost(); + + // This registers the given blob internally and adds it to the storage with a + // refcount of 1. |completion_callback| is called synchronously or + // asynchronously with: + // * INVALID_CONSTRUCTION_ARGUMENTS if we have invalid input arguments/data. + // * REFERENCED_BLOB_BROKEN if one of the referenced blobs is broken or + // doesn't exist. + // * DONE if we don't need any more data transported and we can clean up. + // Returns a blob handle that is never null. + std::unique_ptr<BlobDataHandle> StartBuildingBlob( + const std::string& uuid, + const std::string& content_type, + const std::string& content_disposition, + const std::vector<DataElement>& elements, + BlobStorageContext* context, + const RequestMemoryCallback& request_memory, + const BlobStatusCallback& completion_callback); + + // This is called when we have responses from the Renderer to our calls to + // the request_memory callback above. The callbacks given in StartBuildingBlob + // will be used to request for more memory or signal completion. + // Note: The uuid must be being built in this host (IsBeingBuilt). + void OnMemoryResponses(const std::string& uuid, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context); + + // This removes the TransportState from our map and flags the blob as broken + // in the context. This can be called both from our own logic to cancel the + // blob, or from the DispatcherHost (Renderer). The blob MUST be being built + // in this builder. This also calls the |completion_callback|. + // Note: if the blob isn't in the context (renderer dereferenced it before we + // finished constructing), then we don't bother touching the context. + void CancelBuildingBlob(const std::string& uuid, + BlobStatus code, + BlobStorageContext* context); + + // This clears this object of pending construction and does NOT call transport + // complete callbacks. + void CancelAll(BlobStorageContext* context); + + bool IsEmpty() const { return async_blob_map_.empty(); } + + size_t blob_building_count() const { return async_blob_map_.size(); } + + bool IsBeingBuilt(const std::string& key) const { + return async_blob_map_.find(key) != async_blob_map_.end(); + } + + private: + struct TransportState { + TransportState(const std::string& uuid, + const std::string& content_type, + const std::string& content_disposition, + RequestMemoryCallback request_memory_callback, + BlobStatusCallback completion_callback); + TransportState(TransportState&&); + TransportState& operator=(TransportState&&); + DISALLOW_COPY_AND_ASSIGN(TransportState); + ~TransportState(); + + IPCBlobItemRequestStrategy strategy = IPCBlobItemRequestStrategy::UNKNOWN; + BlobTransportRequestBuilder request_builder; + BlobDataBuilder data_builder; + std::vector<bool> request_received; + size_t num_fulfilled_requests = 0; + + RequestMemoryCallback request_memory_callback; + BlobStatusCallback completion_callback; + + // Used by shared memory strategy. + size_t next_request = 0; + std::unique_ptr<base::SharedMemory> shared_memory_block; + // This is the number of requests that have been sent to populate the above + // shared data. We won't ask for more data in shared memory until all + // requests have been responded to. + size_t num_shared_memory_requests = 0; + // Only relevant if num_shared_memory_requests is > 0 + size_t current_shared_memory_handle_index = 0; + + // Used by file strategy. + std::vector<scoped_refptr<ShareableFileReference>> files; + }; + + typedef std::unordered_map<std::string, TransportState> AsyncBlobMap; + + void StartRequests( + const std::string& uuid, + TransportState* state, + BlobStorageContext* context, + std::vector<BlobMemoryController::FileCreationInfo> file_infos); + + void OnReadyForTransport( + const std::string& uuid, + base::WeakPtr<BlobStorageContext> context, + BlobStatus status, + std::vector<BlobMemoryController::FileCreationInfo> file_infos); + + void SendIPCRequests(TransportState* state, BlobStorageContext* context); + void OnIPCResponses(const std::string& uuid, + TransportState* state, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context); + + // This is the 'main loop' of our memory requests to the renderer. + void ContinueSharedMemoryRequests(const std::string& uuid, + TransportState* state, + BlobStorageContext* context); + + void OnSharedMemoryResponses( + const std::string& uuid, + TransportState* state, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context); + + void SendFileRequests( + TransportState* state, + BlobStorageContext* context, + std::vector<BlobMemoryController::FileCreationInfo> files); + + void OnFileResponses(const std::string& uuid, + TransportState* state, + const std::vector<BlobItemBytesResponse>& responses, + BlobStorageContext* context); + + // This finishes creating the blob in the context, decrements blob references + // that we were holding during construction, and erases our state. + void CompleteTransport(TransportState* state, BlobStorageContext* context); + + AsyncBlobMap async_blob_map_; + base::WeakPtrFactory<BlobTransportHost> ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(BlobTransportHost); +}; + +} // namespace storage +#endif // STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_HOST_H_ diff --git a/chromium/storage/browser/blob/blob_async_transport_request_builder.cc b/chromium/storage/browser/blob/blob_transport_request_builder.cc index a77d408ca8f..c9e3da0dfe7 100644 --- a/chromium/storage/browser/blob/blob_async_transport_request_builder.cc +++ b/chromium/storage/browser/blob/blob_transport_request_builder.cc @@ -8,7 +8,7 @@ #include <algorithm> #include "base/numerics/safe_math.h" -#include "storage/browser/blob/blob_async_transport_request_builder.h" +#include "storage/browser/blob/blob_transport_request_builder.h" #include "storage/common/blob_storage/blob_storage_constants.h" namespace storage { @@ -37,7 +37,7 @@ bool IsBytes(DataElement::Type type) { class FileStorageStrategy { public: FileStorageStrategy( - std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>* + std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>* requests, BlobDataBuilder* builder) : requests(requests), builder(builder), current_item_index(0) {} @@ -49,7 +49,7 @@ class FileStorageStrategy { size_t segment_index, uint64_t segment_offset, uint64_t size) { - BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest request; + BlobTransportRequestBuilder::RendererMemoryItemRequest request; request.browser_item_index = current_item_index; request.browser_item_offset = 0; request.message.request_number = requests->size(); @@ -61,7 +61,7 @@ class FileStorageStrategy { request.message.handle_offset = segment_offset; requests->push_back(request); - builder->AppendFutureFile(segment_offset, size); + builder->AppendFutureFile(segment_offset, size, segment_index); current_item_index++; } @@ -72,8 +72,7 @@ class FileStorageStrategy { void Done() {} - std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>* - requests; + std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>* requests; BlobDataBuilder* builder; size_t current_item_index; @@ -85,7 +84,7 @@ class SharedMemoryStorageStrategy { public: SharedMemoryStorageStrategy( size_t max_segment_size, - std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>* + std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>* requests, BlobDataBuilder* builder) : requests(requests), @@ -105,7 +104,7 @@ class SharedMemoryStorageStrategy { current_item_index++; current_item_size = 0; } - BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest request; + BlobTransportRequestBuilder::RendererMemoryItemRequest request; request.browser_item_index = current_item_index; request.browser_item_offset = current_item_size; request.message.request_number = requests->size(); @@ -137,8 +136,7 @@ class SharedMemoryStorageStrategy { } } - std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>* - requests; + std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>* requests; size_t max_segment_size; BlobDataBuilder* builder; @@ -193,17 +191,21 @@ void ForEachWithSegment(const std::vector<DataElement>& elements, } } // namespace -BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest:: +BlobTransportRequestBuilder::RendererMemoryItemRequest:: RendererMemoryItemRequest() : browser_item_index(0), browser_item_offset(0) {} -BlobAsyncTransportRequestBuilder::BlobAsyncTransportRequestBuilder() +BlobTransportRequestBuilder::BlobTransportRequestBuilder() : total_bytes_size_(0) {} -BlobAsyncTransportRequestBuilder::~BlobAsyncTransportRequestBuilder() {} +BlobTransportRequestBuilder::BlobTransportRequestBuilder( + BlobTransportRequestBuilder&&) = default; +BlobTransportRequestBuilder& BlobTransportRequestBuilder::operator=( + BlobTransportRequestBuilder&&) = default; +BlobTransportRequestBuilder::~BlobTransportRequestBuilder() {} // Initializes the transport strategy for file requests. -void BlobAsyncTransportRequestBuilder::InitializeForFileRequests( +void BlobTransportRequestBuilder::InitializeForFileRequests( size_t max_file_size, uint64_t blob_total_size, const std::vector<DataElement>& elements, @@ -215,7 +217,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForFileRequests( ForEachWithSegment(elements, static_cast<uint64_t>(max_file_size), &strategy); } -void BlobAsyncTransportRequestBuilder::InitializeForSharedMemoryRequests( +void BlobTransportRequestBuilder::InitializeForSharedMemoryRequests( size_t max_shared_memory_size, uint64_t blob_total_size, const std::vector<DataElement>& elements, @@ -231,7 +233,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForSharedMemoryRequests( &strategy); } -void BlobAsyncTransportRequestBuilder::InitializeForIPCRequests( +void BlobTransportRequestBuilder::InitializeForIPCRequests( size_t max_ipc_memory_size, uint64_t blob_total_size, const std::vector<DataElement>& elements, @@ -247,7 +249,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForIPCRequests( builder->AppendIPCDataElement(info); continue; } - BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest request; + BlobTransportRequestBuilder::RendererMemoryItemRequest request; request.browser_item_index = i; request.browser_item_offset = 0; request.message.request_number = requests_.size(); @@ -261,27 +263,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForIPCRequests( } /* static */ -bool BlobAsyncTransportRequestBuilder::ShouldBeShortcut( - const std::vector<DataElement>& elements, - size_t memory_available) { - base::CheckedNumeric<size_t> shortcut_bytes = 0; - for (const auto& element : elements) { - DataElement::Type type = element.type(); - if (type == DataElement::TYPE_BYTES_DESCRIPTION) { - return false; - } - if (type == DataElement::TYPE_BYTES) { - shortcut_bytes += element.length(); - if (!shortcut_bytes.IsValid()) { - return false; - } - } - } - return shortcut_bytes.ValueOrDie() <= memory_available; -} - -/* static */ -void BlobAsyncTransportRequestBuilder::ComputeHandleSizes( +void BlobTransportRequestBuilder::ComputeHandleSizes( uint64_t total_memory_size, size_t max_segment_size, std::vector<size_t>* segment_sizes) { diff --git a/chromium/storage/browser/blob/blob_async_transport_request_builder.h b/chromium/storage/browser/blob/blob_transport_request_builder.h index 10dde5ff6ff..4dce9a123a5 100644 --- a/chromium/storage/browser/blob/blob_async_transport_request_builder.h +++ b/chromium/storage/browser/blob/blob_transport_request_builder.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef STORAGE_BROWSER_BLOB_BLOB_ASYNC_TRANSPORT_REQUEST_BUILDER_H_ -#define STORAGE_BROWSER_BLOB_BLOB_ASYNC_TRANSPORT_REQUEST_BUILDER_H_ +#ifndef STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_REQUEST_BUILDER_H_ +#define STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_REQUEST_BUILDER_H_ #include <stddef.h> #include <stdint.h> @@ -28,7 +28,7 @@ namespace storage { // where the data is already present in the blob description, and will // always give the caller requests for requesting all data from the // renderer. -class STORAGE_EXPORT BlobAsyncTransportRequestBuilder { +class STORAGE_EXPORT BlobTransportRequestBuilder { public: struct RendererMemoryItemRequest { RendererMemoryItemRequest(); @@ -42,8 +42,10 @@ class STORAGE_EXPORT BlobAsyncTransportRequestBuilder { BlobItemBytesRequest message; }; - BlobAsyncTransportRequestBuilder(); - virtual ~BlobAsyncTransportRequestBuilder(); + BlobTransportRequestBuilder(); + BlobTransportRequestBuilder(BlobTransportRequestBuilder&&); + BlobTransportRequestBuilder& operator=(BlobTransportRequestBuilder&&); + virtual ~BlobTransportRequestBuilder(); // Initializes the request builder for file requests. One or more files are // created to hold the given data. Each file can hold data from multiple @@ -114,9 +116,6 @@ class STORAGE_EXPORT BlobAsyncTransportRequestBuilder { // The total bytes size of memory items in the blob. uint64_t total_bytes_size() const { return total_bytes_size_; } - static bool ShouldBeShortcut(const std::vector<DataElement>& items, - size_t memory_available); - private: static void ComputeHandleSizes(uint64_t total_memory_size, size_t max_segment_size, @@ -130,9 +129,9 @@ class STORAGE_EXPORT BlobAsyncTransportRequestBuilder { uint64_t total_bytes_size_; std::vector<RendererMemoryItemRequest> requests_; - DISALLOW_COPY_AND_ASSIGN(BlobAsyncTransportRequestBuilder); + DISALLOW_COPY_AND_ASSIGN(BlobTransportRequestBuilder); }; } // namespace storage -#endif // STORAGE_BROWSER_BLOB_BLOB_ASYNC_TRANSPORT_REQUEST_BUILDER_H_ +#endif // STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_REQUEST_BUILDER_H_ diff --git a/chromium/storage/browser/blob/blob_transport_result.h b/chromium/storage/browser/blob/blob_transport_result.h deleted file mode 100644 index 9aa2f72ad4a..00000000000 --- a/chromium/storage/browser/blob/blob_transport_result.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_RESULT_H_ -#define STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_RESULT_H_ - -namespace storage { - -// This 'result' enum is used in the blob transport logic. -enum class BlobTransportResult { - // This means we should flag the incoming IPC as a bad IPC. - BAD_IPC, - // Cancel reasons. - CANCEL_MEMORY_FULL, - CANCEL_FILE_ERROR, - CANCEL_REFERENCED_BLOB_BROKEN, - CANCEL_UNKNOWN, - // This means we're waiting on responses from the renderer. - PENDING_RESPONSES, - // We're done registering or transferring the blob. - DONE -}; - -} // namespace storage - -#endif // STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_RESULT_H_ diff --git a/chromium/storage/browser/blob/internal_blob_data.cc b/chromium/storage/browser/blob/internal_blob_data.cc deleted file mode 100644 index 115c6a65e1f..00000000000 --- a/chromium/storage/browser/blob/internal_blob_data.cc +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "storage/browser/blob/internal_blob_data.h" - -#include <stddef.h> - -#include <memory> -#include <utility> - -#include "base/containers/hash_tables.h" -#include "base/metrics/histogram.h" -#include "storage/browser/blob/blob_data_item.h" -#include "storage/common/data_element.h" - -namespace storage { - -InternalBlobData::Builder::Builder() : data_(new InternalBlobData()) { -} -InternalBlobData::Builder::~Builder() { -} - -void InternalBlobData::Builder::AppendSharedBlobItem( - scoped_refptr<ShareableBlobDataItem> item) { - DCHECK(item); - DCHECK(data_); - data_->items_.push_back(item); -} - -void InternalBlobData::Builder::RemoveBlobFromShareableItems( - const std::string& blob_uuid) { - DCHECK(data_); - data_->RemoveBlobFromShareableItems(blob_uuid); -} - -size_t InternalBlobData::Builder::GetNonsharedMemoryUsage() const { - DCHECK(data_); - return data_->GetUnsharedMemoryUsage(); -} - -std::unique_ptr<InternalBlobData> InternalBlobData::Builder::Build() { - DCHECK(data_); - return std::move(data_); -} - -InternalBlobData::InternalBlobData() { -} - -InternalBlobData::~InternalBlobData() { -} - -const std::vector<scoped_refptr<ShareableBlobDataItem>>& -InternalBlobData::items() const { - return items_; -} - -void InternalBlobData::RemoveBlobFromShareableItems( - const std::string& blob_uuid) { - for (auto& data_item : items_) { - data_item->referencing_blobs().erase(blob_uuid); - } -} - -size_t InternalBlobData::GetUnsharedMemoryUsage() const { - size_t memory = 0; - base::hash_set<void*> seen_items; - for (const auto& data_item : items_) { - if (data_item->item()->type() != DataElement::TYPE_BYTES || - data_item->referencing_blobs().size() > 1 || - seen_items.find(data_item.get()) != seen_items.end()) { - continue; - } - memory += data_item->item()->length(); - seen_items.insert(data_item.get()); - } - return memory; -} - -void InternalBlobData::GetMemoryUsage(size_t* total_memory, - size_t* unshared_memory) { - *total_memory = 0; - *unshared_memory = 0; - base::hash_set<void*> seen_items; - for (const auto& data_item : items_) { - if (data_item->item()->type() == DataElement::TYPE_BYTES) { - *total_memory += data_item->item()->length(); - if (data_item->referencing_blobs().size() == 1 && - seen_items.find(data_item.get()) == seen_items.end()) { - *unshared_memory += data_item->item()->length(); - seen_items.insert(data_item.get()); - } - } - } -} - -} // namespace storage diff --git a/chromium/storage/browser/blob/internal_blob_data.h b/chromium/storage/browser/blob/internal_blob_data.h deleted file mode 100644 index 65db2f3cdc3..00000000000 --- a/chromium/storage/browser/blob/internal_blob_data.h +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef STORAGE_BROWSER_BLOB_INTERNAL_BLOB_DATA_H_ -#define STORAGE_BROWSER_BLOB_INTERNAL_BLOB_DATA_H_ - -#include <stddef.h> - -#include <memory> -#include <string> -#include <vector> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "storage/browser/blob/shareable_blob_data_item.h" - -namespace storage { -class ViewBlobInternalsJob; - -// This class represents a blob in the BlobStorageContext. It is constructed -// using the internal Builder class. -class InternalBlobData { - public: - ~InternalBlobData(); - - protected: - friend class BlobStorageContext; - friend class BlobStorageRegistry; - friend class ViewBlobInternalsJob; - - // Removes the given blob uuid from the internal ShareableBlobDataItems. - // This is called when this blob is being destroyed. - void RemoveBlobFromShareableItems(const std::string& blob_uuid); - - const std::vector<scoped_refptr<ShareableBlobDataItem>>& items() const; - - // Gets the memory used by this blob that is not shared by other blobs. This - // also doesn't count duplicate items. - size_t GetUnsharedMemoryUsage() const; - - // Gets the memory used by this blob. Total memory includes memory of items - // possibly shared with other blobs, or items that appear multiple times in - // this blob. Unshared memory is memory used by this blob that is not shared - // by other blobs. - void GetMemoryUsage(size_t* total_memory, size_t* unshared_memory); - - private: - friend class Builder; - InternalBlobData(); - - std::string content_type_; - std::string content_disposition_; - std::vector<scoped_refptr<ShareableBlobDataItem>> items_; - - class Builder { - public: - Builder(); - ~Builder(); - - void AppendSharedBlobItem(scoped_refptr<ShareableBlobDataItem> item); - - // Gets the memory used by this builder that is not shared with other blobs. - size_t GetNonsharedMemoryUsage() const; - - // Removes the given blob uuid from the internal ShareableBlobDataItems. - // This is called on destruction of the blob if we're still building it. - void RemoveBlobFromShareableItems(const std::string& blob_uuid); - - // The builder is invalid after calling this method. - std::unique_ptr<::storage::InternalBlobData> Build(); - - private: - std::unique_ptr<::storage::InternalBlobData> data_; - - DISALLOW_COPY_AND_ASSIGN(Builder); - }; - - DISALLOW_COPY_AND_ASSIGN(InternalBlobData); -}; - -} // namespace storage -#endif // STORAGE_BROWSER_BLOB_INTERNAL_BLOB_DATA_H_ diff --git a/chromium/storage/browser/blob/shareable_blob_data_item.cc b/chromium/storage/browser/blob/shareable_blob_data_item.cc index 40aaf9c7e10..2b32349085a 100644 --- a/chromium/storage/browser/blob/shareable_blob_data_item.cc +++ b/chromium/storage/browser/blob/shareable_blob_data_item.cc @@ -7,20 +7,44 @@ #include "storage/browser/blob/blob_data_item.h" namespace storage { +namespace { + +uint64_t GetAndIncrementItemId() { + static uint64_t sNextItemId = 0; + return sNextItemId++; +} + +} // namespace ShareableBlobDataItem::ShareableBlobDataItem( - const std::string& blob_uuid, - const scoped_refptr<BlobDataItem>& item) - : item_(item) { + scoped_refptr<BlobDataItem> item, + ShareableBlobDataItem::State state) + : item_id_(GetAndIncrementItemId()), state_(state), item_(std::move(item)) { DCHECK_NE(item_->type(), DataElement::TYPE_BLOB); - referencing_blobs_.insert(blob_uuid); } ShareableBlobDataItem::~ShareableBlobDataItem() { } -const scoped_refptr<BlobDataItem>& ShareableBlobDataItem::item() { - return item_; +void ShareableBlobDataItem::set_item(scoped_refptr<BlobDataItem> item) { + item_ = std::move(item); +} + +void PrintTo(const ShareableBlobDataItem& x, ::std::ostream* os) { + *os << "<ShareableBlobDataItem>{ item_id: " << x.item_id_ + << ", state: " << x.state_ << ", item: "; + PrintTo(*x.item_, os); + *os << "]}"; +} + +bool operator==(const ShareableBlobDataItem& a, + const ShareableBlobDataItem& b) { + return a.item_id() == b.item_id() && *a.item() == *b.item(); +} + +bool operator!=(const ShareableBlobDataItem& a, + const ShareableBlobDataItem& b) { + return !(a == b); } } // namespace storage diff --git a/chromium/storage/browser/blob/shareable_blob_data_item.h b/chromium/storage/browser/blob/shareable_blob_data_item.h index 4c2f9c18332..7d6c15f8c2e 100644 --- a/chromium/storage/browser/blob/shareable_blob_data_item.h +++ b/chromium/storage/browser/blob/shareable_blob_data_item.h @@ -7,44 +7,94 @@ #include <string> +#include "base/callback_helpers.h" #include "base/containers/hash_tables.h" #include "base/hash.h" #include "base/macros.h" #include "base/memory/ref_counted.h" +#include "storage/browser/blob/blob_memory_controller.h" +#include "storage/browser/storage_browser_export.h" #include "storage/common/data_element.h" namespace storage { class BlobDataItem; -class InternalBlobData; -// This class allows blob items to be shared between blobs, and is only used by -// BlobStorageContext. This class contains both the blob data item and the uuids -// of all the blobs using this item. +// This class allows blob items to be shared between blobs. This class contains +// both the blob data item and the uuids of all the blobs using this item. // The data in this class (the item) is immutable, but the item itself can be // swapped out with an item with the same data but a different backing (think // RAM vs file backed). -class ShareableBlobDataItem : public base::RefCounted<ShareableBlobDataItem> { +// We also allow the storage of a memory quota allocation object which is used +// for memory quota reclamation. +class STORAGE_EXPORT ShareableBlobDataItem + : public base::RefCounted<ShareableBlobDataItem> { public: - ShareableBlobDataItem(const std::string& blob_uuid, - const scoped_refptr<BlobDataItem>& item); + enum State { + // We're an item that needs quota (either disk or memory). + QUOTA_NEEDED, + // We have requested quota from the BlobMemoryController. + QUOTA_REQUESTED, + // Space has been allocated for this item in the BlobMemoryController, but + // it may not yet be populated. + QUOTA_GRANTED, + // We're a populated item that needed quota. + POPULATED_WITH_QUOTA, + // We're a populated item that didn't need quota. + POPULATED_WITHOUT_QUOTA + }; - const scoped_refptr<BlobDataItem>& item(); + ShareableBlobDataItem(scoped_refptr<BlobDataItem> item, State state); - base::hash_set<std::string>& referencing_blobs() { - return referencing_blobs_; + const scoped_refptr<BlobDataItem>& item() const { return item_; } + + void set_item(scoped_refptr<BlobDataItem> item); + + // This is a unique auto-incrementing id assigned to this item on + // construction. It is used to keep track of this item in an LRU data + // structure for eviction to disk. + uint64_t item_id() const { return item_id_; } + + State state() const { return state_; } + void set_state(State state) { state_ = state; } + + bool IsPopulated() const { + return state_ == POPULATED_WITH_QUOTA || state_ == POPULATED_WITHOUT_QUOTA; + } + + bool HasGrantedQuota() const { + return state_ == POPULATED_WITH_QUOTA || state_ == QUOTA_GRANTED; } private: + friend class BlobMemoryController; + friend class BlobMemoryControllerTest; + friend class BlobStorageContext; friend class base::RefCounted<ShareableBlobDataItem>; - friend class InternalBlobData; + friend STORAGE_EXPORT void PrintTo(const ShareableBlobDataItem& x, + ::std::ostream* os); + ~ShareableBlobDataItem(); - scoped_refptr<BlobDataItem> item_; + void set_memory_allocation( + std::unique_ptr<BlobMemoryController::MemoryAllocation> allocation) { + memory_allocation_ = std::move(allocation); + } - base::hash_set<std::string> referencing_blobs_; + bool has_memory_allocation() { return static_cast<bool>(memory_allocation_); } + + // This is a unique identifier for this ShareableBlobDataItem. + const uint64_t item_id_; + State state_; + scoped_refptr<BlobDataItem> item_; + std::unique_ptr<BlobMemoryController::MemoryAllocation> memory_allocation_; DISALLOW_COPY_AND_ASSIGN(ShareableBlobDataItem); }; +STORAGE_EXPORT bool operator==(const ShareableBlobDataItem& a, + const ShareableBlobDataItem& b); +STORAGE_EXPORT bool operator!=(const ShareableBlobDataItem& a, + const ShareableBlobDataItem& b); + } // namespace storage #endif // STORAGE_BROWSER_BLOB_SHAREABLE_BLOB_DATA_ITEM_H_ diff --git a/chromium/storage/browser/blob/view_blob_internals_job.cc b/chromium/storage/browser/blob/view_blob_internals_job.cc index 269a56036fe..a076321f1ef 100644 --- a/chromium/storage/browser/blob/view_blob_internals_job.cc +++ b/chromium/storage/browser/blob/view_blob_internals_job.cc @@ -25,9 +25,10 @@ #include "net/disk_cache/disk_cache.h" #include "net/url_request/url_request.h" #include "storage/browser/blob/blob_data_item.h" +#include "storage/browser/blob/blob_entry.h" #include "storage/browser/blob/blob_storage_context.h" #include "storage/browser/blob/blob_storage_registry.h" -#include "storage/browser/blob/internal_blob_data.h" +#include "storage/browser/blob/shareable_blob_data_item.h" namespace { @@ -143,7 +144,7 @@ int ViewBlobInternalsJob::GetData( data->clear(); StartHTML(data); - if (blob_storage_context_->registry_.blob_map_.empty()) + if (blob_storage_context_->registry().blob_map_.empty()) data->append(kEmptyBlobStorageMessage); else GenerateHTML(data); @@ -153,18 +154,18 @@ int ViewBlobInternalsJob::GetData( void ViewBlobInternalsJob::GenerateHTML(std::string* out) const { for (BlobStorageRegistry::BlobMap::const_iterator iter = - blob_storage_context_->registry_.blob_map_.begin(); - iter != blob_storage_context_->registry_.blob_map_.end(); ++iter) { + blob_storage_context_->registry().blob_map_.begin(); + iter != blob_storage_context_->registry().blob_map_.end(); ++iter) { AddHTMLBoldText(iter->first, out); - GenerateHTMLForBlobData(*iter->second->data, iter->second->content_type, - iter->second->content_disposition, - iter->second->refcount, out); + GenerateHTMLForBlobData(*iter->second, iter->second->content_type(), + iter->second->content_disposition(), + iter->second->refcount(), out); } - if (!blob_storage_context_->registry_.url_to_uuid_.empty()) { + if (!blob_storage_context_->registry().url_to_uuid_.empty()) { AddHorizontalRule(out); for (BlobStorageRegistry::URLMap::const_iterator iter = - blob_storage_context_->registry_.url_to_uuid_.begin(); - iter != blob_storage_context_->registry_.url_to_uuid_.end(); ++iter) { + blob_storage_context_->registry().url_to_uuid_.begin(); + iter != blob_storage_context_->registry().url_to_uuid_.end(); ++iter) { AddHTMLBoldText(iter->first.spec(), out); StartHTMLList(out); AddHTMLListItem(kUUID, iter->second, out); @@ -174,7 +175,7 @@ void ViewBlobInternalsJob::GenerateHTML(std::string* out) const { } void ViewBlobInternalsJob::GenerateHTMLForBlobData( - const InternalBlobData& blob_data, + const BlobEntry& blob_data, const std::string& content_type, const std::string& content_disposition, int refcount, @@ -232,6 +233,7 @@ void ViewBlobInternalsJob::GenerateHTMLForBlobData( AddHTMLListItem(kURL, item.disk_cache_entry()->GetKey(), out); break; case DataElement::TYPE_BYTES_DESCRIPTION: + AddHTMLListItem(kType, "pending data", out); case DataElement::TYPE_UNKNOWN: NOTREACHED(); break; diff --git a/chromium/storage/browser/blob/view_blob_internals_job.h b/chromium/storage/browser/blob/view_blob_internals_job.h index ca6782be333..b1490fb86f6 100644 --- a/chromium/storage/browser/blob/view_blob_internals_job.h +++ b/chromium/storage/browser/blob/view_blob_internals_job.h @@ -18,7 +18,7 @@ class URLRequest; namespace storage { -class InternalBlobData; +class BlobEntry; class BlobStorageContext; // A job subclass that implements a protocol to inspect the internal @@ -42,7 +42,7 @@ class STORAGE_EXPORT ViewBlobInternalsJob ~ViewBlobInternalsJob() override; void GenerateHTML(std::string* out) const; - static void GenerateHTMLForBlobData(const InternalBlobData& blob_data, + static void GenerateHTMLForBlobData(const BlobEntry& blob_data, const std::string& content_type, const std::string& content_disposition, int refcount, diff --git a/chromium/storage/browser/crbug653751_unittest.cc b/chromium/storage/browser/crbug653751_unittest.cc new file mode 100644 index 00000000000..44b86fb6ee4 --- /dev/null +++ b/chromium/storage/browser/crbug653751_unittest.cc @@ -0,0 +1,13 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "testing/gtest/include/gtest/gtest.h" + +// This file will be removed when the real storage unit tests are migrated over +// from content_unittests to storage_unittests. Until then, it is needed to keep +// the win_clang builder happy. See http://crbug.com/653751 + +TEST(StorageUnittests, EmptyTest) { + +} diff --git a/chromium/storage/browser/database/database_tracker.cc b/chromium/storage/browser/database/database_tracker.cc index 42041c7869a..1539b0cd643 100644 --- a/chromium/storage/browser/database/database_tracker.cc +++ b/chromium/storage/browser/database/database_tracker.cc @@ -614,8 +614,8 @@ int64_t DatabaseTracker::UpdateOpenDatabaseInfoAndNotify( storage::GetOriginFromIdentifier(origin_id), storage::kStorageTypeTemporary, new_size - old_size); - FOR_EACH_OBSERVER(Observer, observers_, OnDatabaseSizeChanged( - origin_id, name, new_size)); + for (auto& observer : observers_) + observer.OnDatabaseSizeChanged(origin_id, name, new_size); } return new_size; } @@ -626,8 +626,8 @@ void DatabaseTracker::ScheduleDatabaseForDeletion( DCHECK(database_connections_.IsDatabaseOpened(origin_identifier, database_name)); dbs_to_be_deleted_[origin_identifier].insert(database_name); - FOR_EACH_OBSERVER(Observer, observers_, OnDatabaseScheduledForDeletion( - origin_identifier, database_name)); + for (auto& observer : observers_) + observer.OnDatabaseScheduledForDeletion(origin_identifier, database_name); } void DatabaseTracker::ScheduleDatabasesForDeletion( diff --git a/chromium/storage/browser/fileapi/file_system_context.cc b/chromium/storage/browser/fileapi/file_system_context.cc index 9c1de0ba4e9..f1fcf23eb9d 100644 --- a/chromium/storage/browser/fileapi/file_system_context.cc +++ b/chromium/storage/browser/fileapi/file_system_context.cc @@ -103,6 +103,7 @@ int FileSystemContext::GetPermissionPolicy(FileSystemType type) { return FILE_PERMISSION_USE_FILE_PERMISSION; case kFileSystemTypeRestrictedNativeLocal: + case kFileSystemTypeArcContent: return FILE_PERMISSION_READ_ONLY | FILE_PERMISSION_USE_FILE_PERMISSION; diff --git a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc index 06e9aefb3f1..efb51f74cb3 100644 --- a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc +++ b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc @@ -10,6 +10,8 @@ #include <memory> #include <utility> +#include "base/files/file_enumerator.h" +#include "base/files/file_path.h" #include "base/stl_util.h" #include "base/synchronization/lock.h" #include "base/task_runner_util.h" @@ -266,8 +268,75 @@ int64_t PluginPrivateFileSystemBackend::GetOriginUsageOnFileTaskRunner( FileSystemContext* context, const GURL& origin_url, FileSystemType type) { - // We don't track usage on this filesystem. - return 0; + DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); + + if (!CanHandleType(type)) + return 0; + + int64_t total_size; + base::Time last_modified_time; + GetOriginDetailsOnFileTaskRunner(context, origin_url, &total_size, + &last_modified_time); + return total_size; +} + +void PluginPrivateFileSystemBackend::GetOriginDetailsOnFileTaskRunner( + FileSystemContext* context, + const GURL& origin_url, + int64_t* total_size, + base::Time* last_modified_time) { + DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); + + *total_size = 0; + *last_modified_time = base::Time::UnixEpoch(); + std::string fsid = + storage::IsolatedContext::GetInstance()->RegisterFileSystemForVirtualPath( + storage::kFileSystemTypePluginPrivate, "pluginprivate", + base::FilePath()); + DCHECK(storage::ValidateIsolatedFileSystemId(fsid)); + + std::string root = storage::GetIsolatedFileSystemRootURIString( + origin_url, fsid, "pluginprivate"); + + std::unique_ptr<FileSystemOperationContext> operation_context( + new FileSystemOperationContext(context)); + + // Determine the available plugin private filesystem directories for this + // origin. Currently the plugin private filesystem is only used by Encrypted + // Media Content Decryption Modules. Each CDM gets a directory based on the + // mimetype (e.g. plugin application/x-ppapi-widevine-cdm uses directory + // application_x-ppapi-widevine-cdm). Enumerate through the set of + // directories so that data from any CDM used by this origin is counted. + base::File::Error error; + base::FilePath path = obfuscated_file_util()->GetDirectoryForOriginAndType( + origin_url, "", false, &error); + if (error != base::File::FILE_OK) { + DLOG(ERROR) << "Unable to read directory for " << origin_url; + return; + } + + base::FileEnumerator directory_enumerator(path, false, + base::FileEnumerator::DIRECTORIES); + base::FilePath plugin_path; + while (!(plugin_path = directory_enumerator.Next()).empty()) { + std::string plugin_name = plugin_path.BaseName().MaybeAsASCII(); + if (OpenFileSystemOnFileTaskRunner( + obfuscated_file_util(), plugin_map_, origin_url, fsid, plugin_name, + storage::OPEN_FILE_SYSTEM_FAIL_IF_NONEXISTENT) != + base::File::FILE_OK) { + continue; + } + + std::unique_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator( + obfuscated_file_util()->CreateFileEnumerator( + operation_context.get(), context->CrackURL(GURL(root)), true)); + + while (!enumerator->Next().empty()) { + *total_size += enumerator->Size(); + if (enumerator->LastModifiedTime() > *last_modified_time) + *last_modified_time = enumerator->LastModifiedTime(); + } + } } scoped_refptr<QuotaReservation> diff --git a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h index 95ffb647147..89da39bf290 100644 --- a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h +++ b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h @@ -118,6 +118,15 @@ class STORAGE_EXPORT PluginPrivateFileSystemBackend const GURL& origin_url, FileSystemType type) override; + // Get details on the files saved for the specified |origin_url|. Returns + // the total size and last modified time for the set of all files stored + // for the particular origin. |total_size| = 0 and |last_modified_time| = + // base::Time::UnixEpoch() if no files found. + void GetOriginDetailsOnFileTaskRunner(FileSystemContext* context, + const GURL& origin_url, + int64_t* total_size, + base::Time* last_modified_time); + private: friend class content::PluginPrivateFileSystemBackendTest; diff --git a/chromium/storage/browser/fileapi/sandbox_directory_database.cc b/chromium/storage/browser/fileapi/sandbox_directory_database.cc index a4e845aa61d..91c4914c7f2 100644 --- a/chromium/storage/browser/fileapi/sandbox_directory_database.cc +++ b/chromium/storage/browser/fileapi/sandbox_directory_database.cc @@ -17,7 +17,7 @@ #include "base/files/file_util.h" #include "base/location.h" #include "base/macros.h" -#include "base/metrics/histogram.h" +#include "base/metrics/histogram_macros.h" #include "base/pickle.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_util.h" diff --git a/chromium/storage/browser/fileapi/sandbox_origin_database.cc b/chromium/storage/browser/fileapi/sandbox_origin_database.cc index 992c5cfbe6b..2cfd3f3da9a 100644 --- a/chromium/storage/browser/fileapi/sandbox_origin_database.cc +++ b/chromium/storage/browser/fileapi/sandbox_origin_database.cc @@ -15,7 +15,7 @@ #include "base/format_macros.h" #include "base/location.h" #include "base/logging.h" -#include "base/metrics/histogram.h" +#include "base/metrics/histogram_macros.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" diff --git a/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc b/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc index 9174377c8fa..6cce670357c 100644 --- a/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc +++ b/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc @@ -9,7 +9,7 @@ #include <algorithm> #include "base/bind.h" -#include "base/metrics/histogram.h" +#include "base/metrics/histogram_macros.h" #include "storage/browser/quota/quota_manager.h" #include "url/gurl.h" diff --git a/chromium/storage/browser/quota/special_storage_policy.cc b/chromium/storage/browser/quota/special_storage_policy.cc index 6e774505df5..40125ac126e 100644 --- a/chromium/storage/browser/quota/special_storage_policy.cc +++ b/chromium/storage/browser/quota/special_storage_policy.cc @@ -22,17 +22,20 @@ void SpecialStoragePolicy::RemoveObserver(Observer* observer) { void SpecialStoragePolicy::NotifyGranted(const GURL& origin, int change_flags) { scoped_refptr<SpecialStoragePolicy> protect(this); - FOR_EACH_OBSERVER(Observer, observers_, OnGranted(origin, change_flags)); + for (auto& observer : observers_) + observer.OnGranted(origin, change_flags); } void SpecialStoragePolicy::NotifyRevoked(const GURL& origin, int change_flags) { scoped_refptr<SpecialStoragePolicy> protect(this); - FOR_EACH_OBSERVER(Observer, observers_, OnRevoked(origin, change_flags)); + for (auto& observer : observers_) + observer.OnRevoked(origin, change_flags); } void SpecialStoragePolicy::NotifyCleared() { scoped_refptr<SpecialStoragePolicy> protect(this); - FOR_EACH_OBSERVER(Observer, observers_, OnCleared()); + for (auto& observer : observers_) + observer.OnCleared(); } } // namespace storage diff --git a/chromium/storage/browser/quota/usage_tracker.cc b/chromium/storage/browser/quota/usage_tracker.cc index e324bfe7d28..041f94067ae 100644 --- a/chromium/storage/browser/quota/usage_tracker.cc +++ b/chromium/storage/browser/quota/usage_tracker.cc @@ -9,7 +9,7 @@ #include <algorithm> #include "base/bind.h" -#include "base/stl_util.h" +#include "base/memory/ptr_util.h" #include "storage/browser/quota/client_usage_tracker.h" #include "storage/browser/quota/storage_monitor.h" @@ -34,21 +34,18 @@ UsageTracker::UsageTracker(const QuotaClientList& clients, weak_factory_(this) { for (auto* client : clients) { if (client->DoesSupport(type)) { - client_tracker_map_[client->id()] = - new ClientUsageTracker(this, client, type, special_storage_policy, - storage_monitor_); + client_tracker_map_[client->id()] = base::MakeUnique<ClientUsageTracker>( + this, client, type, special_storage_policy, storage_monitor_); } } } -UsageTracker::~UsageTracker() { - base::STLDeleteValues(&client_tracker_map_); -} +UsageTracker::~UsageTracker() {} ClientUsageTracker* UsageTracker::GetClientTracker(QuotaClient::ID client_id) { - ClientTrackerMap::iterator found = client_tracker_map_.find(client_id); + auto found = client_tracker_map_.find(client_id); if (found != client_tracker_map_.end()) - return found->second; + return found->second.get(); return nullptr; } diff --git a/chromium/storage/browser/quota/usage_tracker.h b/chromium/storage/browser/quota/usage_tracker.h index 994fc584635..074b4f54e23 100644 --- a/chromium/storage/browser/quota/usage_tracker.h +++ b/chromium/storage/browser/quota/usage_tracker.h @@ -67,8 +67,6 @@ class STORAGE_EXPORT UsageTracker : public QuotaTaskObserver { int64_t unlimited_usage; }; - typedef std::map<QuotaClient::ID, ClientUsageTracker*> ClientTrackerMap; - typedef CallbackQueue<UsageCallback, int64_t> UsageCallbackQueue; typedef CallbackQueue<GlobalUsageCallback, int64_t, int64_t> GlobalUsageCallbackQueue; @@ -86,7 +84,8 @@ class STORAGE_EXPORT UsageTracker : public QuotaTaskObserver { int64_t usage); const StorageType type_; - ClientTrackerMap client_tracker_map_; + std::map<QuotaClient::ID, std::unique_ptr<ClientUsageTracker>> + client_tracker_map_; UsageCallbackQueue global_limited_usage_callbacks_; GlobalUsageCallbackQueue global_usage_callbacks_; diff --git a/chromium/storage/common/BUILD.gn b/chromium/storage/common/BUILD.gn index 03766c3a20b..1f15f489857 100644 --- a/chromium/storage/common/BUILD.gn +++ b/chromium/storage/common/BUILD.gn @@ -9,6 +9,7 @@ component("common") { "blob_storage/blob_item_bytes_request.h", "blob_storage/blob_item_bytes_response.cc", "blob_storage/blob_item_bytes_response.h", + "blob_storage/blob_storage_constants.cc", "blob_storage/blob_storage_constants.h", "data_element.cc", "data_element.h", diff --git a/chromium/storage/common/blob_storage/blob_storage_constants.cc b/chromium/storage/common/blob_storage/blob_storage_constants.cc new file mode 100644 index 00000000000..2d825f4534b --- /dev/null +++ b/chromium/storage/common/blob_storage/blob_storage_constants.cc @@ -0,0 +1,25 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "storage/common/blob_storage/blob_storage_constants.h" + +#include "base/logging.h" + +namespace storage { + +bool BlobStatusIsError(BlobStatus status) { + return static_cast<int>(status) <= static_cast<int>(BlobStatus::LAST_ERROR); +} + +bool BlobStatusIsPending(BlobStatus status) { + int status_int = static_cast<int>(status); + return status_int >= static_cast<int>(BlobStatus::PENDING_QUOTA) && + status_int <= static_cast<int>(BlobStatus::PENDING_INTERNALS); +} + +bool BlobStatusIsBadIPC(BlobStatus status) { + return status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; +} + +} // namespace storage diff --git a/chromium/storage/common/blob_storage/blob_storage_constants.h b/chromium/storage/common/blob_storage/blob_storage_constants.h index 0fc10c583ce..f54d20aaa70 100644 --- a/chromium/storage/common/blob_storage/blob_storage_constants.h +++ b/chromium/storage/common/blob_storage/blob_storage_constants.h @@ -8,17 +8,35 @@ #include <stddef.h> #include <stdint.h> +#include "base/callback_forward.h" +#include "storage/common/storage_common_export.h" + namespace storage { -// TODO(michaeln): use base::SysInfo::AmountOfPhysicalMemoryMB() in some -// way to come up with a better limit. -const int64_t kBlobStorageMaxMemoryUsage = 500 * 1024 * 1024; // Half a gig. -const size_t kBlobStorageIPCThresholdBytes = 250 * 1024; -const size_t kBlobStorageMaxSharedMemoryBytes = 10 * 1024 * 1024; -const uint64_t kBlobStorageMaxFileSizeBytes = 100 * 1024 * 1024; -const uint64_t kBlobStorageMinFileSizeBytes = 1 * 1024 * 1024; -const size_t kBlobStorageMaxBlobMemorySize = - kBlobStorageMaxMemoryUsage - kBlobStorageMinFileSizeBytes; +// All sizes are in bytes. +struct BlobStorageLimits { + size_t memory_limit_before_paging() const { + return max_blob_in_memory_space - min_page_file_size; + } + + // This is the maximum amount of memory we can send in an IPC. + size_t max_ipc_memory_size = 250 * 1024; + // This is the maximum size of a shared memory handle. + size_t max_shared_memory_size = 10 * 1024 * 1024; + + // This is the maximum amount of memory we can use to store blobs. + size_t max_blob_in_memory_space = 500 * 1024 * 1024; + + // This is the maximum amount of disk space we can use. + // TODO(dmurph): Consider storage size of the device. + uint64_t max_blob_disk_space = 5ull * 1024 * 1024 * 1024; + + // This is the minimum file size we can use when paging blob items to disk. + // We combine items until we reach at least this size. + uint64_t min_page_file_size = 5 * 1024 * 1024; + // This is the maximum file size we can create. + uint64_t max_file_size = 100 * 1024 * 1024; +}; enum class IPCBlobItemRequestStrategy { UNKNOWN = 0, @@ -28,27 +46,54 @@ enum class IPCBlobItemRequestStrategy { LAST = FILE }; -// These items cannot be reordered or renumbered because they're recorded to -// UMA. New items must be added immediately before LAST, and LAST must be set to -// the the last item. -enum class IPCBlobCreationCancelCode { - UNKNOWN = 0, - OUT_OF_MEMORY = 1, +// This is the enum to rule them all in the blob system. +// These values are used in UMA metrics, so they should not be changed. Please +// update LAST_ERROR if you add an error condition and LAST if you add new +// state. +enum class BlobStatus { + // Error case section: + // The construction arguments are invalid. This is considered a bad ipc. + ERR_INVALID_CONSTRUCTION_ARGUMENTS = 0, + // We don't have enough memory for the blob. + ERR_OUT_OF_MEMORY = 1, // We couldn't create or write to a file. File system error, like a full disk. - FILE_WRITE_FAILED = 2, + ERR_FILE_WRITE_FAILED = 2, // The renderer was destroyed while data was in transit. - SOURCE_DIED_IN_TRANSIT = 3, + ERR_SOURCE_DIED_IN_TRANSIT = 3, // The renderer destructed the blob before it was done transferring, and there // were no outstanding references (no one is waiting to read) to keep the // blob alive. - BLOB_DEREFERENCED_WHILE_BUILDING = 4, + ERR_BLOB_DEREFERENCED_WHILE_BUILDING = 4, // A blob that we referenced during construction is broken, or a browser-side // builder tries to build a blob with a blob reference that isn't finished // constructing. - REFERENCED_BLOB_BROKEN = 5, - LAST = REFERENCED_BLOB_BROKEN + ERR_REFERENCED_BLOB_BROKEN = 5, + LAST_ERROR = ERR_REFERENCED_BLOB_BROKEN, + + // Blob state section: + // The blob has finished. + DONE = 200, + // The system is pending on quota being granted, the transport layer + // populating pending data, and/or copying data from dependent blobs. See + // BlobEntry::BuildingState determine which of these are happening, as they + // all can happen concurrently. + PENDING_QUOTA = 201, + PENDING_TRANSPORT = 202, + PENDING_INTERNALS = 203, + LAST = PENDING_INTERNALS }; +using BlobStatusCallback = base::Callback<void(BlobStatus)>; + +// Returns if the status is an error code. +STORAGE_COMMON_EXPORT bool BlobStatusIsError(BlobStatus status); + +STORAGE_COMMON_EXPORT bool BlobStatusIsPending(BlobStatus status); + +// Returns if the status is a bad enough error to flag the IPC as bad. This is +// only INVALID_CONSTRUCTION_ARGUMENTS. +STORAGE_COMMON_EXPORT bool BlobStatusIsBadIPC(BlobStatus status); + } // namespace storage #endif // STORAGE_COMMON_BLOB_STORAGE_BLOB_STORAGE_CONSTANTS_H_ diff --git a/chromium/storage/common/data_element.h b/chromium/storage/common/data_element.h index 276181b6ec9..744cc99d11c 100644 --- a/chromium/storage/common/data_element.h +++ b/chromium/storage/common/data_element.h @@ -26,6 +26,8 @@ namespace storage { // bytes, file or blob data. class STORAGE_COMMON_EXPORT DataElement { public: + static const uint64_t kUnknownSize = std::numeric_limits<uint64_t>::max(); + enum Type { TYPE_UNKNOWN = -1, TYPE_BYTES, diff --git a/chromium/storage/common/fileapi/file_system_types.h b/chromium/storage/common/fileapi/file_system_types.h index 3f4f942cd9e..6ac262cc4ff 100644 --- a/chromium/storage/common/fileapi/file_system_types.h +++ b/chromium/storage/common/fileapi/file_system_types.h @@ -123,6 +123,9 @@ enum FileSystemType { // limited to media files. kFileSystemTypeDeviceMediaAsFileStorage, + // A filesystem to provide access to contents managed by ARC. + kFileSystemTypeArcContent, + // -------------------------------------------------------------------- // Marks the end of internal type enum. (This is not the actual fs type) // New internal filesystem types must be added above this line. diff --git a/chromium/storage/common/fileapi/file_system_util.cc b/chromium/storage/common/fileapi/file_system_util.cc index 0472c957d28..1116fcd5631 100644 --- a/chromium/storage/common/fileapi/file_system_util.cc +++ b/chromium/storage/common/fileapi/file_system_util.cc @@ -321,6 +321,8 @@ std::string GetFileSystemTypeString(FileSystemType type) { return "Provided"; case kFileSystemTypeDeviceMediaAsFileStorage: return "DeviceMediaStorage"; + case kFileSystemTypeArcContent: + return "ArcContent"; case kFileSystemInternalTypeEnumStart: case kFileSystemInternalTypeEnumEnd: NOTREACHED(); |