summaryrefslogtreecommitdiff
path: root/chromium/storage
diff options
context:
space:
mode:
authorAllan Sandfeld Jensen <allan.jensen@qt.io>2017-03-08 10:28:10 +0100
committerAllan Sandfeld Jensen <allan.jensen@qt.io>2017-03-20 13:40:30 +0000
commite733310db58160074f574c429d48f8308c0afe17 (patch)
treef8aef4b7e62a69928dbcf880620eece20f98c6df /chromium/storage
parent2f583e4aec1ae3a86fa047829c96b310dc12ecdf (diff)
downloadqtwebengine-chromium-e733310db58160074f574c429d48f8308c0afe17.tar.gz
BASELINE: Update Chromium to 56.0.2924.122
Change-Id: I4e04de8f47e47e501c46ed934c76a431c6337ced Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/storage')
-rw-r--r--chromium/storage/OWNERS1
-rw-r--r--chromium/storage/browser/BUILD.gn33
-rw-r--r--chromium/storage/browser/blob/blob_async_builder_host.cc465
-rw-r--r--chromium/storage/browser/blob/blob_async_builder_host.h206
-rw-r--r--chromium/storage/browser/blob/blob_data_builder.cc70
-rw-r--r--chromium/storage/browser/blob/blob_data_builder.h31
-rw-r--r--chromium/storage/browser/blob/blob_data_handle.cc31
-rw-r--r--chromium/storage/browser/blob/blob_data_handle.h21
-rw-r--r--chromium/storage/browser/blob/blob_data_item.h7
-rw-r--r--chromium/storage/browser/blob/blob_entry.cc69
-rw-r--r--chromium/storage/browser/blob/blob_entry.h164
-rw-r--r--chromium/storage/browser/blob/blob_memory_controller.cc747
-rw-r--r--chromium/storage/browser/blob/blob_memory_controller.h252
-rw-r--r--chromium/storage/browser/blob/blob_reader.cc26
-rw-r--r--chromium/storage/browser/blob/blob_reader.h3
-rw-r--r--chromium/storage/browser/blob/blob_storage_context.cc944
-rw-r--r--chromium/storage/browser/blob/blob_storage_context.h253
-rw-r--r--chromium/storage/browser/blob/blob_storage_registry.cc40
-rw-r--r--chromium/storage/browser/blob/blob_storage_registry.h63
-rw-r--r--chromium/storage/browser/blob/blob_transport_host.cc506
-rw-r--r--chromium/storage/browser/blob/blob_transport_host.h186
-rw-r--r--chromium/storage/browser/blob/blob_transport_request_builder.cc (renamed from chromium/storage/browser/blob/blob_async_transport_request_builder.cc)58
-rw-r--r--chromium/storage/browser/blob/blob_transport_request_builder.h (renamed from chromium/storage/browser/blob/blob_async_transport_request_builder.h)19
-rw-r--r--chromium/storage/browser/blob/blob_transport_result.h27
-rw-r--r--chromium/storage/browser/blob/internal_blob_data.cc97
-rw-r--r--chromium/storage/browser/blob/internal_blob_data.h83
-rw-r--r--chromium/storage/browser/blob/shareable_blob_data_item.cc36
-rw-r--r--chromium/storage/browser/blob/shareable_blob_data_item.h76
-rw-r--r--chromium/storage/browser/blob/view_blob_internals_job.cc24
-rw-r--r--chromium/storage/browser/blob/view_blob_internals_job.h4
-rw-r--r--chromium/storage/browser/crbug653751_unittest.cc13
-rw-r--r--chromium/storage/browser/database/database_tracker.cc8
-rw-r--r--chromium/storage/browser/fileapi/file_system_context.cc1
-rw-r--r--chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc73
-rw-r--r--chromium/storage/browser/fileapi/plugin_private_file_system_backend.h9
-rw-r--r--chromium/storage/browser/fileapi/sandbox_directory_database.cc2
-rw-r--r--chromium/storage/browser/fileapi/sandbox_origin_database.cc2
-rw-r--r--chromium/storage/browser/quota/quota_temporary_storage_evictor.cc2
-rw-r--r--chromium/storage/browser/quota/special_storage_policy.cc9
-rw-r--r--chromium/storage/browser/quota/usage_tracker.cc15
-rw-r--r--chromium/storage/browser/quota/usage_tracker.h5
-rw-r--r--chromium/storage/common/BUILD.gn1
-rw-r--r--chromium/storage/common/blob_storage/blob_storage_constants.cc25
-rw-r--r--chromium/storage/common/blob_storage/blob_storage_constants.h85
-rw-r--r--chromium/storage/common/data_element.h2
-rw-r--r--chromium/storage/common/fileapi/file_system_types.h3
-rw-r--r--chromium/storage/common/fileapi/file_system_util.cc2
47 files changed, 3234 insertions, 1565 deletions
diff --git a/chromium/storage/OWNERS b/chromium/storage/OWNERS
index 5c7f0175f40..ddd9fcfd45c 100644
--- a/chromium/storage/OWNERS
+++ b/chromium/storage/OWNERS
@@ -1,4 +1,5 @@
darin@chromium.org
+jsbell@chromium.org
kinuko@chromium.org
michaeln@chromium.org
piman@chromium.org
diff --git a/chromium/storage/browser/BUILD.gn b/chromium/storage/browser/BUILD.gn
index a4cc79a2e3f..8e91590e8a4 100644
--- a/chromium/storage/browser/BUILD.gn
+++ b/chromium/storage/browser/BUILD.gn
@@ -1,14 +1,11 @@
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+import("//testing/test.gni")
component("browser") {
output_name = "storage_browser"
sources = [
- "blob/blob_async_builder_host.cc",
- "blob/blob_async_builder_host.h",
- "blob/blob_async_transport_request_builder.cc",
- "blob/blob_async_transport_request_builder.h",
"blob/blob_data_builder.cc",
"blob/blob_data_builder.h",
"blob/blob_data_handle.cc",
@@ -17,18 +14,24 @@ component("browser") {
"blob/blob_data_item.h",
"blob/blob_data_snapshot.cc",
"blob/blob_data_snapshot.h",
+ "blob/blob_entry.cc",
+ "blob/blob_entry.h",
+ "blob/blob_memory_controller.cc",
+ "blob/blob_memory_controller.h",
"blob/blob_reader.cc",
"blob/blob_reader.h",
"blob/blob_storage_context.cc",
"blob/blob_storage_context.h",
"blob/blob_storage_registry.cc",
"blob/blob_storage_registry.h",
+ "blob/blob_transport_host.cc",
+ "blob/blob_transport_host.h",
+ "blob/blob_transport_request_builder.cc",
+ "blob/blob_transport_request_builder.h",
"blob/blob_url_request_job.cc",
"blob/blob_url_request_job.h",
"blob/blob_url_request_job_factory.cc",
"blob/blob_url_request_job_factory.h",
- "blob/internal_blob_data.cc",
- "blob/internal_blob_data.h",
"blob/scoped_file.cc",
"blob/scoped_file.h",
"blob/shareable_blob_data_item.cc",
@@ -215,3 +218,21 @@ executable("dump_file_system") {
"//storage/common",
]
}
+
+test("storage_unittests") {
+ sources = [
+ # Do NOT add storage/ tests here until this target is added to the build
+ # bots. http://crbug.com/653751
+
+ # If the sources list is empty, the win_clang builder fails.
+ # This file will be removed when the real tests are moved over from
+ # content_unittests to this target.
+ "crbug653751_unittest.cc",
+ ]
+
+ deps = [
+ "//base/test:run_all_unittests",
+ "//base/test:test_support",
+ "//testing/gtest",
+ ]
+}
diff --git a/chromium/storage/browser/blob/blob_async_builder_host.cc b/chromium/storage/browser/blob/blob_async_builder_host.cc
deleted file mode 100644
index a73d9d68323..00000000000
--- a/chromium/storage/browser/blob/blob_async_builder_host.cc
+++ /dev/null
@@ -1,465 +0,0 @@
-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "storage/browser/blob/blob_async_builder_host.h"
-
-#include <stddef.h>
-#include <stdint.h>
-
-#include <memory>
-#include <utility>
-
-#include "base/bind.h"
-#include "base/memory/ptr_util.h"
-#include "base/memory/shared_memory.h"
-#include "storage/browser/blob/blob_data_handle.h"
-#include "storage/browser/blob/blob_storage_context.h"
-
-namespace storage {
-namespace {
-
-bool CalculateBlobMemorySize(const std::vector<DataElement>& elements,
- size_t* shortcut_bytes,
- uint64_t* total_bytes) {
- DCHECK(shortcut_bytes);
- DCHECK(total_bytes);
- base::CheckedNumeric<uint64_t> total_size_checked = 0;
- base::CheckedNumeric<size_t> shortcut_size_checked = 0;
- for (const auto& e : elements) {
- if (e.type() == DataElement::TYPE_BYTES) {
- total_size_checked += e.length();
- shortcut_size_checked += e.length();
- } else if (e.type() == DataElement::TYPE_BYTES_DESCRIPTION) {
- total_size_checked += e.length();
- } else {
- continue;
- }
- if (!total_size_checked.IsValid() || !shortcut_size_checked.IsValid()) {
- return false;
- }
- }
- *shortcut_bytes = shortcut_size_checked.ValueOrDie();
- *total_bytes = total_size_checked.ValueOrDie();
- return true;
-}
-
-IPCBlobCreationCancelCode ConvertReferencedBlobErrorToConstructingError(
- IPCBlobCreationCancelCode referenced_blob_error) {
- switch (referenced_blob_error) {
- // For most cases we propagate the error.
- case IPCBlobCreationCancelCode::FILE_WRITE_FAILED:
- case IPCBlobCreationCancelCode::SOURCE_DIED_IN_TRANSIT:
- case IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN:
- case IPCBlobCreationCancelCode::OUT_OF_MEMORY:
- return referenced_blob_error;
- // Others we report that the referenced blob is broken, as we don't know
- // why (the BLOB_DEREFERENCED_WHILE_BUILDING should never happen, as we hold
- // onto the reference of the blobs we're using).
- case IPCBlobCreationCancelCode::BLOB_DEREFERENCED_WHILE_BUILDING:
- DCHECK(false) << "Referenced blob should never be dereferenced while we "
- << "are depending on it, as our system holds a handle.";
- case IPCBlobCreationCancelCode::UNKNOWN:
- return IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
- }
- NOTREACHED();
- return IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
-}
-
-} // namespace
-
-using MemoryItemRequest =
- BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest;
-
-BlobAsyncBuilderHost::BlobBuildingState::BlobBuildingState(
- const std::string& uuid,
- std::set<std::string> referenced_blob_uuids,
- std::vector<std::unique_ptr<BlobDataHandle>>* referenced_blob_handles)
- : data_builder(uuid),
- referenced_blob_uuids(referenced_blob_uuids),
- referenced_blob_handles(std::move(*referenced_blob_handles)) {}
-
-BlobAsyncBuilderHost::BlobBuildingState::~BlobBuildingState() {}
-
-BlobAsyncBuilderHost::BlobAsyncBuilderHost() : ptr_factory_(this) {}
-
-BlobAsyncBuilderHost::~BlobAsyncBuilderHost() {}
-
-BlobTransportResult BlobAsyncBuilderHost::RegisterBlobUUID(
- const std::string& uuid,
- const std::string& content_type,
- const std::string& content_disposition,
- const std::set<std::string>& referenced_blob_uuids,
- BlobStorageContext* context) {
- if (async_blob_map_.find(uuid) != async_blob_map_.end())
- return BlobTransportResult::BAD_IPC;
- if (referenced_blob_uuids.find(uuid) != referenced_blob_uuids.end())
- return BlobTransportResult::BAD_IPC;
- context->CreatePendingBlob(uuid, content_type, content_disposition);
- std::vector<std::unique_ptr<BlobDataHandle>> handles;
- for (const std::string& referenced_uuid : referenced_blob_uuids) {
- std::unique_ptr<BlobDataHandle> handle =
- context->GetBlobDataFromUUID(referenced_uuid);
- if (!handle || handle->IsBroken()) {
- // We cancel the blob right away, and don't bother storing our state.
- context->CancelPendingBlob(
- uuid, IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN);
- return BlobTransportResult::CANCEL_REFERENCED_BLOB_BROKEN;
- }
- handles.emplace_back(std::move(handle));
- }
- async_blob_map_[uuid] = base::MakeUnique<BlobBuildingState>(
- uuid, referenced_blob_uuids, &handles);
- return BlobTransportResult::DONE;
-}
-
-BlobTransportResult BlobAsyncBuilderHost::StartBuildingBlob(
- const std::string& uuid,
- const std::vector<DataElement>& elements,
- size_t memory_available,
- BlobStorageContext* context,
- const RequestMemoryCallback& request_memory) {
- DCHECK(context);
- DCHECK(async_blob_map_.find(uuid) != async_blob_map_.end());
-
- // Step 1: Get the sizes.
- size_t shortcut_memory_size_bytes = 0;
- uint64_t total_memory_size_bytes = 0;
- if (!CalculateBlobMemorySize(elements, &shortcut_memory_size_bytes,
- &total_memory_size_bytes)) {
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context);
- return BlobTransportResult::BAD_IPC;
- }
-
- // Step 2: Check if we have enough memory to store the blob.
- if (total_memory_size_bytes > memory_available) {
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::OUT_OF_MEMORY, context);
- return BlobTransportResult::CANCEL_MEMORY_FULL;
- }
-
- // From here on, we know we can fit the blob in memory.
- BlobBuildingState* state_ptr = async_blob_map_[uuid].get();
- if (!state_ptr->request_builder.requests().empty()) {
- // Check that we're not a duplicate call.
- return BlobTransportResult::BAD_IPC;
- }
- state_ptr->request_memory_callback = request_memory;
-
- // Step 3: Check to make sure the referenced blob information we received
- // earlier is correct:
- std::set<std::string> extracted_blob_uuids;
- for (const DataElement& e : elements) {
- if (e.type() == DataElement::TYPE_BLOB) {
- extracted_blob_uuids.insert(e.blob_uuid());
- // We can't depend on ourselves.
- if (e.blob_uuid() == uuid) {
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context);
- return BlobTransportResult::BAD_IPC;
- }
- }
- }
- if (extracted_blob_uuids != state_ptr->referenced_blob_uuids) {
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context);
- return BlobTransportResult::BAD_IPC;
- }
-
- // Step 4: Decide if we're using the shortcut method. This will also catch
- // the case where we don't have any memory items.
- if (shortcut_memory_size_bytes == total_memory_size_bytes &&
- shortcut_memory_size_bytes <= memory_available) {
- for (const DataElement& e : elements) {
- state_ptr->data_builder.AppendIPCDataElement(e);
- }
- FinishBuildingBlob(state_ptr, context);
- return BlobTransportResult::DONE;
- }
-
- // From here on, we know the blob's size is less than |memory_available|,
- // so we know we're < max(size_t).
- // Step 5: Decide if we're using shared memory.
- if (total_memory_size_bytes > max_ipc_memory_size_) {
- state_ptr->request_builder.InitializeForSharedMemoryRequests(
- max_shared_memory_size_, total_memory_size_bytes, elements,
- &(state_ptr->data_builder));
- } else {
- // Step 6: We can fit in IPC.
- state_ptr->request_builder.InitializeForIPCRequests(
- max_ipc_memory_size_, total_memory_size_bytes, elements,
- &(state_ptr->data_builder));
- }
- // We initialize our requests received state now that they are populated.
- state_ptr->request_received.resize(
- state_ptr->request_builder.requests().size(), false);
- return ContinueBlobMemoryRequests(uuid, context);
-}
-
-BlobTransportResult BlobAsyncBuilderHost::OnMemoryResponses(
- const std::string& uuid,
- const std::vector<BlobItemBytesResponse>& responses,
- BlobStorageContext* context) {
- AsyncBlobMap::const_iterator state_it = async_blob_map_.find(uuid);
- if (state_it == async_blob_map_.end()) {
- DVLOG(1) << "Could not find blob " << uuid;
- return BlobTransportResult::BAD_IPC;
- }
- if (responses.empty()) {
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context);
- return BlobTransportResult::BAD_IPC;
- }
- BlobAsyncBuilderHost::BlobBuildingState* state = state_it->second.get();
- BlobAsyncTransportRequestBuilder& request_builder = state->request_builder;
- const auto& requests = request_builder.requests();
- for (const BlobItemBytesResponse& response : responses) {
- if (response.request_number >= requests.size()) {
- // Bad IPC, so we delete our record and ignore.
- DVLOG(1) << "Invalid request number " << response.request_number;
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context);
- return BlobTransportResult::BAD_IPC;
- }
- DCHECK_LT(response.request_number, state->request_received.size());
- const MemoryItemRequest& request = requests[response.request_number];
- if (state->request_received[response.request_number]) {
- // Bad IPC, so we delete our record.
- DVLOG(1) << "Already received response for that request.";
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context);
- return BlobTransportResult::BAD_IPC;
- }
- state->request_received[response.request_number] = true;
- bool invalid_ipc = false;
- bool memory_error = false;
- switch (request.message.transport_strategy) {
- case IPCBlobItemRequestStrategy::IPC:
- if (response.inline_data.size() < request.message.size) {
- DVLOG(1) << "Invalid data size " << response.inline_data.size()
- << " vs requested size of " << request.message.size;
- invalid_ipc = true;
- break;
- }
- invalid_ipc = !state->data_builder.PopulateFutureData(
- request.browser_item_index, &response.inline_data[0],
- request.browser_item_offset, request.message.size);
- break;
- case IPCBlobItemRequestStrategy::SHARED_MEMORY:
- if (state->num_shared_memory_requests == 0) {
- DVLOG(1) << "Received too many responses for shared memory.";
- invalid_ipc = true;
- break;
- }
- state->num_shared_memory_requests--;
- if (!state->shared_memory_block->memory()) {
- // We just map the whole block, as we'll probably be accessing the
- // whole thing in this group of responses. Another option is to use
- // MapAt, remove the mapped boolean, and then exclude the
- // handle_offset below.
- size_t handle_size = request_builder.shared_memory_sizes()
- [state->current_shared_memory_handle_index];
- if (!state->shared_memory_block->Map(handle_size)) {
- DVLOG(1) << "Unable to map memory to size " << handle_size;
- memory_error = true;
- break;
- }
- }
-
- invalid_ipc = !state->data_builder.PopulateFutureData(
- request.browser_item_index,
- static_cast<const char*>(state->shared_memory_block->memory()) +
- request.message.handle_offset,
- request.browser_item_offset, request.message.size);
- break;
- case IPCBlobItemRequestStrategy::FILE:
- case IPCBlobItemRequestStrategy::UNKNOWN:
- DVLOG(1) << "Not implemented.";
- invalid_ipc = true;
- break;
- }
- if (invalid_ipc) {
- // Bad IPC, so we delete our record and return false.
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::UNKNOWN, context);
- return BlobTransportResult::BAD_IPC;
- }
- if (memory_error) {
- DVLOG(1) << "Shared memory error.";
- CancelBuildingBlob(uuid, IPCBlobCreationCancelCode::OUT_OF_MEMORY,
- context);
- return BlobTransportResult::CANCEL_MEMORY_FULL;
- }
- state->num_fulfilled_requests++;
- }
- return ContinueBlobMemoryRequests(uuid, context);
-}
-
-void BlobAsyncBuilderHost::CancelBuildingBlob(const std::string& uuid,
- IPCBlobCreationCancelCode code,
- BlobStorageContext* context) {
- DCHECK(context);
- auto state_it = async_blob_map_.find(uuid);
- if (state_it == async_blob_map_.end()) {
- return;
- }
- // We can have the blob dereferenced by the renderer, but have it still being
- // 'built'. In this case, it's destructed in the context, but we still have
- // it in our map. Hence we make sure the context has the entry before
- // calling cancel.
- if (context->registry().HasEntry(uuid))
- context->CancelPendingBlob(uuid, code);
- async_blob_map_.erase(state_it);
-}
-
-void BlobAsyncBuilderHost::CancelAll(BlobStorageContext* context) {
- DCHECK(context);
- // If the blob still exists in the context (and is being built), then we know
- // that someone else is expecting our blob, and we need to cancel it to let
- // the dependency know it's gone.
- std::vector<std::unique_ptr<BlobDataHandle>> referenced_pending_blobs;
- for (const auto& uuid_state_pair : async_blob_map_) {
- if (context->IsBeingBuilt(uuid_state_pair.first)) {
- referenced_pending_blobs.emplace_back(
- context->GetBlobDataFromUUID(uuid_state_pair.first));
- }
- }
- // We clear the map before canceling them to prevent any strange reentry into
- // our class (see ReferencedBlobFinished) if any blobs were waiting for others
- // to construct.
- async_blob_map_.clear();
- for (const std::unique_ptr<BlobDataHandle>& handle :
- referenced_pending_blobs) {
- context->CancelPendingBlob(
- handle->uuid(), IPCBlobCreationCancelCode::SOURCE_DIED_IN_TRANSIT);
- }
-}
-
-BlobTransportResult BlobAsyncBuilderHost::ContinueBlobMemoryRequests(
- const std::string& uuid,
- BlobStorageContext* context) {
- AsyncBlobMap::const_iterator state_it = async_blob_map_.find(uuid);
- DCHECK(state_it != async_blob_map_.end());
- BlobAsyncBuilderHost::BlobBuildingState* state = state_it->second.get();
-
- BlobAsyncTransportRequestBuilder& request_builder = state->request_builder;
- const std::vector<MemoryItemRequest>& requests = request_builder.requests();
- size_t num_requests = requests.size();
- if (state->num_fulfilled_requests == num_requests) {
- FinishBuildingBlob(state, context);
- return BlobTransportResult::DONE;
- }
- DCHECK_LT(state->num_fulfilled_requests, num_requests);
- if (state->next_request == num_requests) {
- // We are still waiting on other requests to come back.
- return BlobTransportResult::PENDING_RESPONSES;
- }
-
- std::unique_ptr<std::vector<BlobItemBytesRequest>> byte_requests(
- new std::vector<BlobItemBytesRequest>());
- std::unique_ptr<std::vector<base::SharedMemoryHandle>> shared_memory(
- new std::vector<base::SharedMemoryHandle>());
-
- for (; state->next_request < num_requests; ++state->next_request) {
- const MemoryItemRequest& request = requests[state->next_request];
-
- bool stop_accumulating = false;
- bool using_shared_memory_handle = state->num_shared_memory_requests > 0;
- switch (request.message.transport_strategy) {
- case IPCBlobItemRequestStrategy::IPC:
- byte_requests->push_back(request.message);
- break;
- case IPCBlobItemRequestStrategy::SHARED_MEMORY:
- if (using_shared_memory_handle &&
- state->current_shared_memory_handle_index !=
- request.message.handle_index) {
- // We only want one shared memory per requesting blob.
- stop_accumulating = true;
- break;
- }
- using_shared_memory_handle = true;
- state->current_shared_memory_handle_index =
- request.message.handle_index;
- state->num_shared_memory_requests++;
-
- if (!state->shared_memory_block) {
- state->shared_memory_block.reset(new base::SharedMemory());
- size_t size =
- request_builder
- .shared_memory_sizes()[request.message.handle_index];
- if (!state->shared_memory_block->CreateAnonymous(size)) {
- DVLOG(1) << "Unable to allocate shared memory for blob transfer.";
- return BlobTransportResult::CANCEL_MEMORY_FULL;
- }
- }
- shared_memory->push_back(state->shared_memory_block->handle());
- byte_requests->push_back(request.message);
- // Since we are only using one handle at a time, transform our handle
- // index correctly back to 0.
- byte_requests->back().handle_index = 0;
- break;
- case IPCBlobItemRequestStrategy::FILE:
- case IPCBlobItemRequestStrategy::UNKNOWN:
- NOTREACHED() << "Not implemented yet.";
- break;
- }
- if (stop_accumulating) {
- break;
- }
- }
- DCHECK(!requests.empty());
-
- state->request_memory_callback.Run(
- std::move(byte_requests), std::move(shared_memory),
- base::MakeUnique<std::vector<base::File>>());
- return BlobTransportResult::PENDING_RESPONSES;
-}
-
-void BlobAsyncBuilderHost::ReferencedBlobFinished(
- const std::string& owning_blob_uuid,
- base::WeakPtr<BlobStorageContext> context,
- bool construction_success,
- IPCBlobCreationCancelCode reason) {
- if (!context) {
- return;
- }
- auto state_it = async_blob_map_.find(owning_blob_uuid);
- if (state_it == async_blob_map_.end()) {
- return;
- }
- if (!construction_success) {
- CancelBuildingBlob(owning_blob_uuid,
- ConvertReferencedBlobErrorToConstructingError(reason),
- context.get());
- return;
- }
- BlobBuildingState* state = state_it->second.get();
- DCHECK_GT(state->num_referenced_blobs_building, 0u);
- if (--state->num_referenced_blobs_building == 0) {
- context->CompletePendingBlob(state->data_builder);
- async_blob_map_.erase(state->data_builder.uuid());
- }
-}
-
-void BlobAsyncBuilderHost::FinishBuildingBlob(BlobBuildingState* state,
- BlobStorageContext* context) {
- if (!state->referenced_blob_uuids.empty()) {
- DCHECK_EQ(0u, state->num_referenced_blobs_building);
- state->num_referenced_blobs_building = 0;
- // We assume re-entry is not possible, as RunOnConstructionComplete
- // will schedule a task when the blob is being built. Thus we can't have the
- // case where |num_referenced_blobs_building| reaches 0 in the
- // ReferencedBlobFinished method before we're finished looping.
- for (const std::string& referenced_uuid : state->referenced_blob_uuids) {
- if (context->IsBeingBuilt(referenced_uuid)) {
- state->num_referenced_blobs_building++;
- context->RunOnConstructionComplete(
- referenced_uuid,
- base::Bind(&BlobAsyncBuilderHost::ReferencedBlobFinished,
- ptr_factory_.GetWeakPtr(), state->data_builder.uuid(),
- context->AsWeakPtr()));
- }
- }
- if (state->num_referenced_blobs_building > 0) {
- // We wait until referenced blobs are done.
- return;
- }
- }
- context->CompletePendingBlob(state->data_builder);
- async_blob_map_.erase(state->data_builder.uuid());
-}
-
-} // namespace storage
diff --git a/chromium/storage/browser/blob/blob_async_builder_host.h b/chromium/storage/browser/blob/blob_async_builder_host.h
deleted file mode 100644
index b7e82fb3dd7..00000000000
--- a/chromium/storage/browser/blob/blob_async_builder_host.h
+++ /dev/null
@@ -1,206 +0,0 @@
-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef STORAGE_BROWSER_BLOB_BLOB_ASYNC_BUILDER_HOST_H_
-#define STORAGE_BROWSER_BLOB_BLOB_ASYNC_BUILDER_HOST_H_
-
-#include <stddef.h>
-#include <stdint.h>
-
-#include <map>
-#include <memory>
-#include <set>
-#include <string>
-#include <vector>
-
-#include "base/callback.h"
-#include "base/files/file.h"
-#include "base/macros.h"
-#include "base/memory/ref_counted.h"
-#include "base/memory/shared_memory_handle.h"
-#include "base/memory/weak_ptr.h"
-#include "storage/browser/blob/blob_async_transport_request_builder.h"
-#include "storage/browser/blob/blob_data_builder.h"
-#include "storage/browser/blob/blob_transport_result.h"
-#include "storage/browser/storage_browser_export.h"
-#include "storage/common/blob_storage/blob_item_bytes_request.h"
-#include "storage/common/blob_storage/blob_item_bytes_response.h"
-#include "storage/common/blob_storage/blob_storage_constants.h"
-#include "storage/common/data_element.h"
-
-namespace base {
-class SharedMemory;
-}
-
-namespace storage {
-class BlobDataHandle;
-class BlobStorageContext;
-
-// This class
-// * holds all blobs that are currently being built asynchronously for a child
-// process,
-// * sends memory requests through the given callback in |StartBuildingBlob|,
-// and uses the BlobTransportResult return value to signify other results,
-// * includes all logic for deciding which async transport strategy to use, and
-// * handles all blob construction communication with the BlobStorageContext.
-// The method |CancelAll| must be called by the consumer, it is not called on
-// destruction.
-class STORAGE_EXPORT BlobAsyncBuilderHost {
- public:
- using RequestMemoryCallback = base::Callback<void(
- std::unique_ptr<std::vector<storage::BlobItemBytesRequest>>,
- std::unique_ptr<std::vector<base::SharedMemoryHandle>>,
- std::unique_ptr<std::vector<base::File>>)>;
- BlobAsyncBuilderHost();
- ~BlobAsyncBuilderHost();
-
- // This registers the given blob internally and adds it to the storage.
- // Calling this method also guarentees that the referenced blobs are kept
- // alive for the duration of the construction of this blob.
- // We return
- // * BAD_IPC if we already have the blob registered or if we reference ourself
- // in the referenced_blob_uuids.
- // * CANCEL_REFERENCED_BLOB_BROKEN if one of the referenced blobs is broken or
- // doesn't exist. We store the blob in the context as broken with code
- // REFERENCED_BLOB_BROKEN.
- // * DONE if we successfully registered the blob.
- BlobTransportResult RegisterBlobUUID(
- const std::string& uuid,
- const std::string& content_type,
- const std::string& content_disposition,
- const std::set<std::string>& referenced_blob_uuids,
- BlobStorageContext* context);
-
- // This method begins the construction of the blob given the descriptions. The
- // blob uuid MUST be building in this object.
- // When we return:
- // * DONE: The blob is finished transfering right away, and is now
- // successfully saved in the context.
- // * PENDING_RESPONSES: The async builder host is waiting for responses from
- // the renderer. It has called |request_memory| for these responses.
- // * CANCEL_*: We have to cancel the blob construction. This function clears
- // the blob's internal state and marks the blob as broken in the context
- // before returning.
- // * BAD_IPC: The arguments were invalid/bad. This marks the blob as broken in
- // the context before returning.
- BlobTransportResult StartBuildingBlob(
- const std::string& uuid,
- const std::vector<DataElement>& elements,
- size_t memory_available,
- BlobStorageContext* context,
- const RequestMemoryCallback& request_memory);
-
- // This is called when we have responses from the Renderer to our calls to
- // the request_memory callback above. See above for return value meaning.
- BlobTransportResult OnMemoryResponses(
- const std::string& uuid,
- const std::vector<BlobItemBytesResponse>& responses,
- BlobStorageContext* context);
-
- // This removes the BlobBuildingState from our map and flags the blob as
- // broken in the context. This can be called both from our own logic to cancel
- // the blob, or from the DispatcherHost (Renderer). The blob MUST be being
- // built in this builder.
- // Note: if the blob isn't in the context (renderer dereferenced it before we
- // finished constructing), then we don't bother touching the context.
- void CancelBuildingBlob(const std::string& uuid,
- IPCBlobCreationCancelCode code,
- BlobStorageContext* context);
-
- // This clears this object of pending construction. It also handles marking
- // blobs that haven't been fully constructed as broken in the context if there
- // are any references being held by anyone. We know that they're being used
- // by someone else if they still exist in the context.
- void CancelAll(BlobStorageContext* context);
-
- bool IsEmpty() const { return async_blob_map_.empty(); }
-
- size_t blob_building_count() const { return async_blob_map_.size(); }
-
- bool IsBeingBuilt(const std::string& key) const {
- return async_blob_map_.find(key) != async_blob_map_.end();
- }
-
- // For testing use only. Must be called before StartBuildingBlob.
- void SetMemoryConstantsForTesting(size_t max_ipc_memory_size,
- size_t max_shared_memory_size,
- uint64_t max_file_size) {
- max_ipc_memory_size_ = max_ipc_memory_size;
- max_shared_memory_size_ = max_shared_memory_size;
- max_file_size_ = max_file_size;
- }
-
- private:
- struct BlobBuildingState {
- // |refernced_blob_handles| should be all handles generated from the set
- // of |refernced_blob_uuids|.
- BlobBuildingState(
- const std::string& uuid,
- std::set<std::string> referenced_blob_uuids,
- std::vector<std::unique_ptr<BlobDataHandle>>* referenced_blob_handles);
- ~BlobBuildingState();
-
- BlobAsyncTransportRequestBuilder request_builder;
- BlobDataBuilder data_builder;
- std::vector<bool> request_received;
- size_t next_request = 0;
- size_t num_fulfilled_requests = 0;
- std::unique_ptr<base::SharedMemory> shared_memory_block;
- // This is the number of requests that have been sent to populate the above
- // shared data. We won't ask for more data in shared memory until all
- // requests have been responded to.
- size_t num_shared_memory_requests = 0;
- // Only relevant if num_shared_memory_requests is > 0
- size_t current_shared_memory_handle_index = 0;
-
- // We save these to double check that the RegisterBlob and StartBuildingBlob
- // messages are in sync.
- std::set<std::string> referenced_blob_uuids;
- // These are the blobs that are referenced in the newly constructed blob.
- // We use these to make sure they stay alive while we create the new blob,
- // and to wait until any blobs that are not done building are fully
- // constructed.
- std::vector<std::unique_ptr<BlobDataHandle>> referenced_blob_handles;
-
- // These are the number of blobs we're waiting for before we can start
- // building.
- size_t num_referenced_blobs_building = 0;
-
- BlobAsyncBuilderHost::RequestMemoryCallback request_memory_callback;
- };
-
- typedef std::map<std::string, std::unique_ptr<BlobBuildingState>>
- AsyncBlobMap;
-
- // This is the 'main loop' of our memory requests to the renderer.
- BlobTransportResult ContinueBlobMemoryRequests(const std::string& uuid,
- BlobStorageContext* context);
-
- // This is our callback for when we want to finish the blob and we're waiting
- // for blobs we reference to be built. When the last callback occurs, we
- // complete the blob and erase our internal state.
- void ReferencedBlobFinished(const std::string& uuid,
- base::WeakPtr<BlobStorageContext> context,
- bool construction_success,
- IPCBlobCreationCancelCode reason);
-
- // This finishes creating the blob in the context, decrements blob references
- // that we were holding during construction, and erases our state.
- void FinishBuildingBlob(BlobBuildingState* state,
- BlobStorageContext* context);
-
- AsyncBlobMap async_blob_map_;
-
- // Here for testing.
- size_t max_ipc_memory_size_ = kBlobStorageIPCThresholdBytes;
- size_t max_shared_memory_size_ = kBlobStorageMaxSharedMemoryBytes;
- uint64_t max_file_size_ = kBlobStorageMaxFileSizeBytes;
-
- base::WeakPtrFactory<BlobAsyncBuilderHost> ptr_factory_;
-
- DISALLOW_COPY_AND_ASSIGN(BlobAsyncBuilderHost);
-};
-
-} // namespace storage
-#endif // STORAGE_BROWSER_BLOB_BLOB_ASYNC_BUILDER_HOST_H_
diff --git a/chromium/storage/browser/blob/blob_data_builder.cc b/chromium/storage/browser/blob/blob_data_builder.cc
index 1397cede297..204a4e81875 100644
--- a/chromium/storage/browser/blob/blob_data_builder.cc
+++ b/chromium/storage/browser/blob/blob_data_builder.cc
@@ -12,9 +12,12 @@
#include "base/numerics/safe_conversions.h"
#include "base/numerics/safe_math.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_util.h"
#include "base/time/time.h"
#include "net/disk_cache/disk_cache.h"
-#include "storage/browser/blob/shareable_file_reference.h"
+
+using base::FilePath;
namespace storage {
@@ -24,21 +27,48 @@ const static int kInvalidDiskCacheSideStreamIndex = -1;
} // namespace
-const char BlobDataBuilder::kAppendFutureFileTemporaryFileName[] =
- "kFakeFilenameToBeChangedByPopulateFutureFile";
+const FilePath::CharType kFutureFileName[] = FILE_PATH_LITERAL("_future_name_");
+
+/* static */
+base::FilePath BlobDataBuilder::GetFutureFileItemPath(uint64_t file_id) {
+ std::string file_id_str = base::Uint64ToString(file_id);
+ return base::FilePath(kFutureFileName)
+ .AddExtension(
+ base::FilePath::StringType(file_id_str.begin(), file_id_str.end()));
+}
-BlobDataBuilder::BlobDataBuilder(const std::string& uuid) : uuid_(uuid) {
+/* static */
+bool BlobDataBuilder::IsFutureFileItem(const DataElement& element) {
+ const FilePath::StringType prefix(kFutureFileName);
+ // The prefix shouldn't occur unless the user used "AppendFutureFile". We
+ // DCHECK on AppendFile to make sure no one appends a future file.
+ return base::StartsWith(element.path().value(), prefix,
+ base::CompareCase::SENSITIVE);
}
-BlobDataBuilder::~BlobDataBuilder() {
+
+/* static */
+uint64_t BlobDataBuilder::GetFutureFileID(const DataElement& element) {
+ DCHECK(IsFutureFileItem(element));
+ uint64_t id = 0;
+ bool success =
+ base::StringToUint64(element.path().Extension().substr(1), &id);
+ DCHECK(success) << element.path().Extension();
+ return id;
}
+BlobDataBuilder::BlobDataBuilder(const std::string& uuid) : uuid_(uuid) {}
+
+BlobDataBuilder::BlobDataBuilder(BlobDataBuilder&&) = default;
+BlobDataBuilder& BlobDataBuilder::operator=(BlobDataBuilder&&) = default;
+BlobDataBuilder::~BlobDataBuilder() {}
+
void BlobDataBuilder::AppendIPCDataElement(const DataElement& ipc_data) {
uint64_t length = ipc_data.length();
switch (ipc_data.type()) {
case DataElement::TYPE_BYTES:
DCHECK(!ipc_data.offset());
AppendData(ipc_data.bytes(),
- base::checked_cast<size_t, uint64_t>(length));
+ base::checked_cast<size_t>(length));
break;
case DataElement::TYPE_FILE:
AppendFile(ipc_data.path(), ipc_data.offset(), length,
@@ -83,7 +113,7 @@ bool BlobDataBuilder::PopulateFutureData(size_t index,
size_t length) {
DCHECK_LT(index, items_.size());
DCHECK(data);
- DataElement* element = items_.at(index)->data_element_ptr();
+ DataElement* element = items_[index]->data_element_ptr();
// We lazily allocate our data buffer by waiting until the first
// PopulateFutureData call.
@@ -110,12 +140,13 @@ bool BlobDataBuilder::PopulateFutureData(size_t index,
return true;
}
-size_t BlobDataBuilder::AppendFutureFile(uint64_t offset, uint64_t length) {
+size_t BlobDataBuilder::AppendFutureFile(uint64_t offset,
+ uint64_t length,
+ uint64_t file_id) {
CHECK_NE(length, 0ull);
std::unique_ptr<DataElement> element(new DataElement());
- element->SetToFilePathRange(base::FilePath::FromUTF8Unsafe(std::string(
- kAppendFutureFileTemporaryFileName)),
- offset, length, base::Time());
+ element->SetToFilePathRange(GetFutureFileItemPath(file_id), offset, length,
+ base::Time());
items_.push_back(new BlobDataItem(std::move(element)));
return items_.size() - 1;
}
@@ -125,32 +156,31 @@ bool BlobDataBuilder::PopulateFutureFile(
const scoped_refptr<ShareableFileReference>& file_reference,
const base::Time& expected_modification_time) {
DCHECK_LT(index, items_.size());
- DataElement* old_element = items_.at(index)->data_element_ptr();
+ DataElement* element = items_[index]->data_element_ptr();
- if (old_element->type() != DataElement::TYPE_FILE) {
+ if (element->type() != DataElement::TYPE_FILE) {
DVLOG(1) << "Invalid item type.";
return false;
- } else if (old_element->path().AsUTF8Unsafe() !=
- std::string(kAppendFutureFileTemporaryFileName)) {
+ } else if (!IsFutureFileItem(*element)) {
DVLOG(1) << "Item not created by AppendFutureFile";
return false;
}
- uint64_t length = old_element->length();
- uint64_t offset = old_element->offset();
- std::unique_ptr<DataElement> element(new DataElement());
+ uint64_t length = element->length();
+ uint64_t offset = element->offset();
+ items_[index]->data_handle_ = std::move(file_reference);
element->SetToFilePathRange(file_reference->path(), offset, length,
expected_modification_time);
- items_[index] = new BlobDataItem(std::move(element), file_reference);
return true;
}
-void BlobDataBuilder::AppendFile(const base::FilePath& file_path,
+void BlobDataBuilder::AppendFile(const FilePath& file_path,
uint64_t offset,
uint64_t length,
const base::Time& expected_modification_time) {
std::unique_ptr<DataElement> element(new DataElement());
element->SetToFilePathRange(file_path, offset, length,
expected_modification_time);
+ DCHECK(!IsFutureFileItem(*element)) << file_path.value();
items_.push_back(new BlobDataItem(std::move(element),
ShareableFileReference::Get(file_path)));
}
diff --git a/chromium/storage/browser/blob/blob_data_builder.h b/chromium/storage/browser/blob/blob_data_builder.h
index f304f446555..088ceb755d7 100644
--- a/chromium/storage/browser/blob/blob_data_builder.h
+++ b/chromium/storage/browser/blob/blob_data_builder.h
@@ -16,6 +16,7 @@
#include "base/memory/ref_counted.h"
#include "storage/browser/blob/blob_data_item.h"
#include "storage/browser/blob/blob_data_snapshot.h"
+#include "storage/browser/blob/shareable_file_reference.h"
#include "storage/browser/storage_browser_export.h"
namespace disk_cache {
@@ -26,15 +27,25 @@ namespace storage {
class BlobStorageContext;
class ShareableFileReference;
+// This class is used to build blobs. It also facilitates the operation of
+// 'pending' data, where the user knows the size and existence of a file or
+// bytes item, but we don't have the memory or file yet. See AppendFuture* and
+// PopulateFuture* methods for more description. Use
+// BlobDataHandle::GetBlobStatus to check for an error after creating the blob.
class STORAGE_EXPORT BlobDataBuilder {
public:
using DataHandle = BlobDataItem::DataHandle;
+ // Visible for testing.
+ static base::FilePath GetFutureFileItemPath(uint64_t file_id);
- // This is the filename used for the temporary file items added by
- // AppendFutureFile.
- const static char kAppendFutureFileTemporaryFileName[];
+ // Returns if the given item was created by AppendFutureFile.
+ static bool IsFutureFileItem(const DataElement& element);
+ // Returns |file_id| given to AppendFutureFile.
+ static uint64_t GetFutureFileID(const DataElement& element);
explicit BlobDataBuilder(const std::string& uuid);
+ BlobDataBuilder(BlobDataBuilder&&);
+ BlobDataBuilder& operator=(BlobDataBuilder&&);
~BlobDataBuilder();
const std::string& uuid() const { return uuid_; }
@@ -73,9 +84,12 @@ class STORAGE_EXPORT BlobDataBuilder {
// Adds an item that is flagged for future data population. Use
// 'PopulateFutureFile' to set the file path and expected modification time
// of this file. Returns the index of the item (to be used in
- // PopulateFutureFile). The temporary filename used by this method is
- // kAppendFutureFileTemporaryFileName. |length| cannot be 0.
- size_t AppendFutureFile(uint64_t offset, uint64_t length);
+ // PopulateFutureFile). |length| cannot be 0.
+ // Data for multiple items can be stored in the same 'future' file, just at
+ // different offsets and lengths. The |file_id| is used to differentiate
+ // between different 'future' files that will be used to store data for these
+ // items.
+ size_t AppendFutureFile(uint64_t offset, uint64_t length, uint64_t file_id);
// Populates a part of an item previously allocated with AppendFutureFile.
// Returns true if:
@@ -106,6 +120,7 @@ class STORAGE_EXPORT BlobDataBuilder {
void AppendDiskCacheEntry(const scoped_refptr<DataHandle>& data_handle,
disk_cache::Entry* disk_cache_entry,
int disk_cache_stream_index);
+
// The content of the side data is accessible with BlobReader::ReadSideData().
void AppendDiskCacheEntryWithSideData(
const scoped_refptr<DataHandle>& data_handle,
@@ -124,12 +139,14 @@ class STORAGE_EXPORT BlobDataBuilder {
void Clear();
private:
+ friend class BlobMemoryControllerTest;
friend class BlobStorageContext;
- friend class BlobAsyncBuilderHostTest;
friend bool operator==(const BlobDataBuilder& a, const BlobDataBuilder& b);
friend bool operator==(const BlobDataSnapshot& a, const BlobDataBuilder& b);
friend STORAGE_EXPORT void PrintTo(const BlobDataBuilder& x,
::std::ostream* os);
+ FRIEND_TEST_ALL_PREFIXES(BlobDataBuilderTest, TestFutureFiles);
+ FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, BuildBlobFuzzy);
std::string uuid_;
std::string content_type_;
diff --git a/chromium/storage/browser/blob/blob_data_handle.cc b/chromium/storage/browser/blob/blob_data_handle.cc
index 55e63a1b620..8d29cca76f3 100644
--- a/chromium/storage/browser/blob/blob_data_handle.cc
+++ b/chromium/storage/browser/blob/blob_data_handle.cc
@@ -26,8 +26,6 @@
#include "url/gurl.h"
namespace storage {
-using BlobState = BlobStorageRegistry::BlobState;
-
namespace {
class FileStreamReaderProviderImpl
@@ -67,10 +65,12 @@ BlobDataHandle::BlobDataHandleShared::BlobDataHandleShared(
const std::string& uuid,
const std::string& content_type,
const std::string& content_disposition,
+ uint64_t size,
BlobStorageContext* context)
: uuid_(uuid),
content_type_(content_type),
content_disposition_(content_disposition),
+ size_(size),
context_(context->AsWeakPtr()) {
context_->IncrementBlobRefCount(uuid);
}
@@ -92,21 +92,20 @@ BlobDataHandle::BlobDataHandleShared::~BlobDataHandleShared() {
BlobDataHandle::BlobDataHandle(const std::string& uuid,
const std::string& content_type,
const std::string& content_disposition,
+ uint64_t size,
BlobStorageContext* context,
base::SequencedTaskRunner* io_task_runner)
: io_task_runner_(io_task_runner),
shared_(new BlobDataHandleShared(uuid,
content_type,
content_disposition,
+ size,
context)) {
DCHECK(io_task_runner_.get());
DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
}
-BlobDataHandle::BlobDataHandle(const BlobDataHandle& other) {
- io_task_runner_ = other.io_task_runner_;
- shared_ = other.shared_;
-}
+BlobDataHandle::BlobDataHandle(const BlobDataHandle& other) = default;
BlobDataHandle::~BlobDataHandle() {
if (!io_task_runner_->RunsTasksOnCurrentThread()) {
@@ -117,25 +116,31 @@ BlobDataHandle::~BlobDataHandle() {
}
}
+BlobDataHandle& BlobDataHandle::operator=(
+ const BlobDataHandle& other) = default;
+
bool BlobDataHandle::IsBeingBuilt() const {
DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
if (!shared_->context_)
return false;
- return shared_->context_->IsBeingBuilt(shared_->uuid_);
+ return BlobStatusIsPending(GetBlobStatus());
}
bool BlobDataHandle::IsBroken() const {
DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
if (!shared_->context_)
return true;
- return shared_->context_->IsBroken(shared_->uuid_);
+ return BlobStatusIsError(GetBlobStatus());
}
-void BlobDataHandle::RunOnConstructionComplete(
- const BlobConstructedCallback& done) {
+BlobStatus BlobDataHandle::GetBlobStatus() const {
+ return shared_->context_->GetBlobStatus(shared_->uuid_);
+}
+
+void BlobDataHandle::RunOnConstructionComplete(const BlobStatusCallback& done) {
DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
if (!shared_->context_.get()) {
- done.Run(false, IPCBlobCreationCancelCode::UNKNOWN);
+ done.Run(BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
return;
}
shared_->context_->RunOnConstructionComplete(shared_->uuid_, done);
@@ -160,4 +165,8 @@ const std::string& BlobDataHandle::content_disposition() const {
return shared_->content_disposition_;
}
+uint64_t BlobDataHandle::size() const {
+ return shared_->size_;
+}
+
} // namespace storage
diff --git a/chromium/storage/browser/blob/blob_data_handle.h b/chromium/storage/browser/blob/blob_data_handle.h
index 3f4529346b1..4a46f934400 100644
--- a/chromium/storage/browser/blob/blob_data_handle.h
+++ b/chromium/storage/browser/blob/blob_data_handle.h
@@ -5,6 +5,7 @@
#ifndef STORAGE_BROWSER_BLOB_BLOB_DATA_HANDLE_H_
#define STORAGE_BROWSER_BLOB_BLOB_DATA_HANDLE_H_
+#include <limits>
#include <memory>
#include <string>
@@ -39,14 +40,14 @@ class FileSystemContext;
class STORAGE_EXPORT BlobDataHandle
: public base::SupportsUserData::Data {
public:
- // True means the blob was constructed successfully, and false means that
- // there was an error, which is reported in the second argument.
- using BlobConstructedCallback =
- base::Callback<void(bool, IPCBlobCreationCancelCode)>;
+ static constexpr uint64_t kUnknownSize = std::numeric_limits<uint64_t>::max();
BlobDataHandle(const BlobDataHandle& other); // May be copied on any thread.
~BlobDataHandle() override; // May be deleted on any thread.
+ // Assignment operator matching copy constructor.
+ BlobDataHandle& operator=(const BlobDataHandle& other);
+
// Returns if this blob is still constructing. If so, one can use the
// RunOnConstructionComplete to wait.
// Must be called on IO thread.
@@ -56,13 +57,17 @@ class STORAGE_EXPORT BlobDataHandle
// Must be called on IO thread.
bool IsBroken() const;
+ // Returns the broken reason if this blob is broken.
+ // Must be called on IO thread.
+ BlobStatus GetBlobStatus() const;
+
// The callback will be run on the IO thread when construction of the blob
// is complete. If construction is already complete, then the task is run
// immediately on the current message loop (i.e. IO thread).
// Must be called on IO thread. Returns if construction successful.
// Calling this multiple times results in registering multiple
// completion callbacks.
- void RunOnConstructionComplete(const BlobConstructedCallback& done);
+ void RunOnConstructionComplete(const BlobStatusCallback& done);
// A BlobReader is used to read the data from the blob. This object is
// intended to be transient and should not be stored for any extended period
@@ -77,6 +82,9 @@ class STORAGE_EXPORT BlobDataHandle
const std::string& content_type() const;
// May be accessed on any thread.
const std::string& content_disposition() const;
+ // May be accessed on any thread. In rare cases where the blob is created
+ // as a file from javascript, this will be kUnknownSize.
+ uint64_t size() const;
// This call and the destruction of the returned snapshot must be called
// on the IO thread. If the blob is broken, then we return a nullptr here.
@@ -94,6 +102,7 @@ class STORAGE_EXPORT BlobDataHandle
BlobDataHandleShared(const std::string& uuid,
const std::string& content_type,
const std::string& content_disposition,
+ uint64_t size,
BlobStorageContext* context);
private:
@@ -106,6 +115,7 @@ class STORAGE_EXPORT BlobDataHandle
const std::string uuid_;
const std::string content_type_;
const std::string content_disposition_;
+ const uint64_t size_;
base::WeakPtr<BlobStorageContext> context_;
DISALLOW_COPY_AND_ASSIGN(BlobDataHandleShared);
@@ -115,6 +125,7 @@ class STORAGE_EXPORT BlobDataHandle
BlobDataHandle(const std::string& uuid,
const std::string& content_type,
const std::string& content_disposition,
+ uint64_t size,
BlobStorageContext* context,
base::SequencedTaskRunner* io_task_runner);
diff --git a/chromium/storage/browser/blob/blob_data_item.h b/chromium/storage/browser/blob/blob_data_item.h
index a711a4caf3b..bc03269212d 100644
--- a/chromium/storage/browser/blob/blob_data_item.h
+++ b/chromium/storage/browser/blob/blob_data_item.h
@@ -21,7 +21,9 @@ class Entry;
namespace storage {
class BlobDataBuilder;
+class BlobMemoryController;
class BlobStorageContext;
+class DataElement;
// Ref counted blob item. This class owns the backing data of the blob item. The
// backing data is immutable, and cannot change after creation. The purpose of
@@ -65,7 +67,11 @@ class STORAGE_EXPORT BlobDataItem : public base::RefCounted<BlobDataItem> {
private:
friend class BlobDataBuilder;
+ friend class BlobMemoryController;
friend class BlobStorageContext;
+ friend struct BlobSlice;
+ friend class BlobSliceTest;
+ friend class BlobFlattenerTest;
friend class base::RefCounted<BlobDataItem>;
friend STORAGE_EXPORT void PrintTo(const BlobDataItem& x, ::std::ostream* os);
@@ -77,6 +83,7 @@ class STORAGE_EXPORT BlobDataItem : public base::RefCounted<BlobDataItem> {
disk_cache::Entry* entry,
int disk_cache_stream_index,
int disk_cache_side_stream_index);
+
virtual ~BlobDataItem();
std::unique_ptr<DataElement> item_;
diff --git a/chromium/storage/browser/blob/blob_entry.cc b/chromium/storage/browser/blob/blob_entry.cc
new file mode 100644
index 00000000000..094eb365641
--- /dev/null
+++ b/chromium/storage/browser/blob/blob_entry.cc
@@ -0,0 +1,69 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <utility>
+
+#include "base/callback.h"
+#include "base/containers/hash_tables.h"
+#include "base/metrics/histogram.h"
+#include "storage/browser/blob/blob_data_handle.h"
+#include "storage/browser/blob/blob_data_item.h"
+#include "storage/browser/blob/blob_entry.h"
+#include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
+
+namespace storage {
+
+BlobEntry::ItemCopyEntry::ItemCopyEntry(
+ scoped_refptr<ShareableBlobDataItem> source_item,
+ size_t source_item_offset,
+ scoped_refptr<ShareableBlobDataItem> dest_item)
+ : source_item(std::move(source_item)),
+ source_item_offset(source_item_offset),
+ dest_item(std::move(dest_item)) {}
+
+BlobEntry::ItemCopyEntry::ItemCopyEntry(ItemCopyEntry&& other) = default;
+BlobEntry::ItemCopyEntry& BlobEntry::ItemCopyEntry::operator=(
+ BlobEntry::ItemCopyEntry&& rhs) = default;
+BlobEntry::ItemCopyEntry::~ItemCopyEntry() {}
+
+BlobEntry::BuildingState::BuildingState(
+ bool transport_items_present,
+ TransportAllowedCallback transport_allowed_callback,
+ size_t num_building_dependent_blobs)
+ : transport_items_present(transport_items_present),
+ transport_allowed_callback(transport_allowed_callback),
+ num_building_dependent_blobs(num_building_dependent_blobs) {}
+
+BlobEntry::BuildingState::~BuildingState() {}
+
+BlobEntry::BlobEntry(const std::string& content_type,
+ const std::string& content_disposition)
+ : content_type_(content_type), content_disposition_(content_disposition) {}
+BlobEntry::~BlobEntry() {}
+
+void BlobEntry::AppendSharedBlobItem(
+ scoped_refptr<ShareableBlobDataItem> item) {
+ DCHECK(item);
+ if (!items_.empty()) {
+ offsets_.push_back(size_);
+ }
+ size_ += item->item()->length();
+ items_.push_back(std::move(item));
+}
+
+const std::vector<scoped_refptr<ShareableBlobDataItem>>& BlobEntry::items()
+ const {
+ return items_;
+}
+
+void BlobEntry::ClearItems() {
+ items_.clear();
+}
+
+void BlobEntry::ClearOffsets() {
+ offsets_.clear();
+}
+
+} // namespace storage
diff --git a/chromium/storage/browser/blob/blob_entry.h b/chromium/storage/browser/blob/blob_entry.h
new file mode 100644
index 00000000000..3c3869e89f4
--- /dev/null
+++ b/chromium/storage/browser/blob/blob_entry.h
@@ -0,0 +1,164 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef STORAGE_BROWSER_BLOB_BLOB_ENTRY_H_
+#define STORAGE_BROWSER_BLOB_BLOB_ENTRY_H_
+
+#include <stddef.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/callback_forward.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "storage/browser/blob/blob_memory_controller.h"
+#include "storage/browser/storage_browser_export.h"
+
+namespace storage {
+class BlobDataHandle;
+class ShareableBlobDataItem;
+class ViewBlobInternalsJob;
+
+// This class represents a blob in BlobStorageRegistry. We export this only for
+// unit tests.
+class STORAGE_EXPORT BlobEntry {
+ public:
+ using TransportAllowedCallback =
+ base::Callback<void(BlobStatus,
+ std::vector<BlobMemoryController::FileCreationInfo>)>;
+
+ // This records a copy from a referenced blob. When we finish building our
+ // blob we perform all of these copies.
+ struct STORAGE_EXPORT ItemCopyEntry {
+ ItemCopyEntry(scoped_refptr<ShareableBlobDataItem> source_item,
+ size_t source_item_offset,
+ scoped_refptr<ShareableBlobDataItem> dest_item);
+ ~ItemCopyEntry();
+ ItemCopyEntry(ItemCopyEntry&& other);
+ BlobEntry::ItemCopyEntry& operator=(BlobEntry::ItemCopyEntry&& rhs);
+
+ scoped_refptr<ShareableBlobDataItem> source_item;
+ size_t source_item_offset = 0;
+ scoped_refptr<ShareableBlobDataItem> dest_item;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ItemCopyEntry);
+ };
+
+ // This keeps track of our building state for our blob. While building, four
+ // things can be happening mostly simultaneously:
+ // 1. Waiting for quota to be reserved for memory needed (PENDING_QUOTA)
+ // 2. Waiting for user population of data after quota (PENDING_TRANSPORT)
+ // 3. Waiting for blobs we reference to complete (PENDING_INTERNALS)
+ struct STORAGE_EXPORT BuildingState {
+ // |transport_allowed_callback| is not null when data needs population. See
+ // BlobStorageContext::BuildBlob for when the callback is called.
+ BuildingState(bool transport_items_present,
+ TransportAllowedCallback transport_allowed_callback,
+ size_t num_building_dependent_blobs);
+ ~BuildingState();
+
+ const bool transport_items_present;
+ // We can have trasnport data that's either populated or unpopulated. If we
+ // need population, this is populated.
+ TransportAllowedCallback transport_allowed_callback;
+ std::vector<ShareableBlobDataItem*> transport_items;
+
+ // Stores all blobs that we're depending on for building. This keeps the
+ // blobs alive while we build our blob.
+ std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
+ size_t num_building_dependent_blobs;
+
+ base::WeakPtr<BlobMemoryController::QuotaAllocationTask>
+ memory_quota_request;
+
+ // These are copies from a referenced blob item to our blob items. Some of
+ // these entries may have changed from bytes to files if they were paged.
+ std::vector<ItemCopyEntry> copies;
+
+ // When our blob finishes building these callbacks are called.
+ std::vector<BlobStatusCallback> build_completion_callbacks;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(BuildingState);
+ };
+
+ BlobEntry(const std::string& content_type,
+ const std::string& content_disposition);
+ ~BlobEntry();
+
+ // Appends the given shared blob data item to this object.
+ void AppendSharedBlobItem(scoped_refptr<ShareableBlobDataItem> item);
+
+ // Returns if we're a pending blob that can finish building.
+ bool CanFinishBuilding() const {
+ return status_ == BlobStatus::PENDING_INTERNALS &&
+ building_state_->num_building_dependent_blobs == 0;
+ }
+
+ BlobStatus status() const { return status_; }
+
+ size_t refcount() const { return refcount_; }
+
+ const std::string& content_type() const { return content_type_; }
+
+ const std::string& content_disposition() const {
+ return content_disposition_;
+ }
+
+ // Total size of this blob in bytes.
+ uint64_t total_size() const { return size_; };
+
+ // We record the cumulative offsets of each blob item (except for the first,
+ // which would always be 0) to enable binary searching for an item at a
+ // specific byte offset.
+ const std::vector<uint64_t>& offsets() const { return offsets_; }
+
+ const std::vector<scoped_refptr<ShareableBlobDataItem>>& items() const;
+
+ protected:
+ friend class BlobStorageContext;
+
+ void IncrementRefCount() { ++refcount_; }
+ void DecrementRefCount() { --refcount_; }
+
+ void set_status(BlobStatus status) { status_ = status; }
+ void set_size(uint64_t size) { size_ = size; }
+
+ void ClearItems();
+ void ClearOffsets();
+
+ void set_building_state(std::unique_ptr<BuildingState> building_state) {
+ building_state_ = std::move(building_state);
+ }
+
+ private:
+ BlobStatus status_ = BlobStatus::PENDING_QUOTA;
+ size_t refcount_ = 0;
+
+ // Metadata.
+ std::string content_type_;
+ std::string content_disposition_;
+
+ std::vector<scoped_refptr<ShareableBlobDataItem>> items_;
+
+ // Size in bytes. IFF we're a single file then this can be uint64_max.
+ uint64_t size_ = 0;
+
+ // Only populated if len(items_) > 1. Used for binary search.
+ // Since the offset of the first item is always 0, we exclude this.
+ std::vector<uint64_t> offsets_;
+
+ // Only populated if our status is PENDING_*.
+ std::unique_ptr<BuildingState> building_state_;
+
+ DISALLOW_COPY_AND_ASSIGN(BlobEntry);
+};
+
+} // namespace storage
+#endif // STORAGE_BROWSER_BLOB_BLOB_ENTRY_H_
diff --git a/chromium/storage/browser/blob/blob_memory_controller.cc b/chromium/storage/browser/blob/blob_memory_controller.cc
new file mode 100644
index 00000000000..6cc6c3f291f
--- /dev/null
+++ b/chromium/storage/browser/blob/blob_memory_controller.cc
@@ -0,0 +1,747 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "storage/browser/blob/blob_memory_controller.h"
+
+#include <algorithm>
+
+#include "base/callback.h"
+#include "base/callback_helpers.h"
+#include "base/containers/small_map.h"
+#include "base/files/file_util.h"
+#include "base/guid.h"
+#include "base/location.h"
+#include "base/memory/ptr_util.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/single_thread_task_runner.h"
+#include "base/stl_util.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/task_runner.h"
+#include "base/task_runner_util.h"
+#include "base/threading/thread_restrictions.h"
+#include "base/time/time.h"
+#include "base/trace_event/trace_event.h"
+#include "base/tuple.h"
+#include "storage/browser/blob/blob_data_builder.h"
+#include "storage/browser/blob/blob_data_item.h"
+#include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/browser/blob/shareable_file_reference.h"
+#include "storage/common/data_element.h"
+
+using base::File;
+using base::FilePath;
+
+namespace storage {
+namespace {
+using FileCreationInfo = BlobMemoryController::FileCreationInfo;
+using MemoryAllocation = BlobMemoryController::MemoryAllocation;
+using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+
+File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) {
+ File::Error error = File::FILE_OK;
+ base::CreateDirectoryAndGetError(blob_storage_dir, &error);
+ UMA_HISTOGRAM_ENUMERATION("Storage.Blob.CreateDirectoryResult", -error,
+ -File::FILE_ERROR_MAX);
+ return error;
+}
+
+void DestructFile(File infos_without_references) {}
+
+// Used for new unpopulated file items. Caller must populate file reference in
+// returned FileCreationInfos.
+std::pair<std::vector<FileCreationInfo>, File::Error> CreateEmptyFiles(
+ const FilePath& blob_storage_dir,
+ scoped_refptr<base::TaskRunner> file_task_runner,
+ std::vector<base::FilePath> file_paths) {
+ base::ThreadRestrictions::AssertIOAllowed();
+
+ File::Error dir_create_status = CreateBlobDirectory(blob_storage_dir);
+ if (dir_create_status != File::FILE_OK)
+ return std::make_pair(std::vector<FileCreationInfo>(), dir_create_status);
+
+ std::vector<FileCreationInfo> result;
+ for (const base::FilePath& file_path : file_paths) {
+ FileCreationInfo creation_info;
+ // Try to open our file.
+ File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
+ creation_info.path = std::move(file_path);
+ creation_info.file_deletion_runner = file_task_runner;
+ creation_info.error = file.error_details();
+ if (creation_info.error != File::FILE_OK) {
+ return std::make_pair(std::vector<FileCreationInfo>(),
+ creation_info.error);
+ }
+
+ // Grab the file info to get the "last modified" time and store the file.
+ File::Info file_info;
+ bool success = file.GetInfo(&file_info);
+ creation_info.error = success ? File::FILE_OK : File::FILE_ERROR_FAILED;
+ if (!success) {
+ return std::make_pair(std::vector<FileCreationInfo>(),
+ creation_info.error);
+ }
+ creation_info.file = std::move(file);
+
+ result.push_back(std::move(creation_info));
+ }
+ return std::make_pair(std::move(result), File::FILE_OK);
+}
+
+// Used to evict multiple memory items out to a single file. Caller must
+// populate file reference in returned FileCreationInfo.
+FileCreationInfo CreateFileAndWriteItems(
+ const FilePath& blob_storage_dir,
+ const FilePath& file_path,
+ scoped_refptr<base::TaskRunner> file_task_runner,
+ std::vector<DataElement*> items,
+ size_t total_size_bytes) {
+ DCHECK_NE(0u, total_size_bytes);
+ UMA_HISTOGRAM_MEMORY_KB("Storage.Blob.PageFileSize", total_size_bytes / 1024);
+ base::ThreadRestrictions::AssertIOAllowed();
+
+ FileCreationInfo creation_info;
+ creation_info.file_deletion_runner = std::move(file_task_runner);
+ creation_info.error = CreateBlobDirectory(blob_storage_dir);
+ if (creation_info.error != File::FILE_OK)
+ return creation_info;
+
+ // Create the page file.
+ File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
+ creation_info.path = file_path;
+ creation_info.error = file.error_details();
+ if (creation_info.error != File::FILE_OK)
+ return creation_info;
+
+ // Write data.
+ file.SetLength(total_size_bytes);
+ int bytes_written = 0;
+ for (DataElement* element : items) {
+ DCHECK_EQ(DataElement::TYPE_BYTES, element->type());
+ size_t length = base::checked_cast<size_t>(element->length());
+ size_t bytes_left = length;
+ while (bytes_left > 0) {
+ bytes_written =
+ file.WriteAtCurrentPos(element->bytes() + (length - bytes_left),
+ base::saturated_cast<int>(bytes_left));
+ if (bytes_written < 0)
+ break;
+ DCHECK_LE(static_cast<size_t>(bytes_written), bytes_left);
+ bytes_left -= bytes_written;
+ }
+ if (bytes_written < 0)
+ break;
+ }
+
+ File::Info info;
+ bool success = file.GetInfo(&info);
+ creation_info.error =
+ bytes_written < 0 || !success ? File::FILE_ERROR_FAILED : File::FILE_OK;
+ creation_info.last_modified = info.last_modified;
+ return creation_info;
+}
+
+uint64_t GetTotalSizeAndFileSizes(
+ const std::vector<scoped_refptr<ShareableBlobDataItem>>&
+ unreserved_file_items,
+ std::vector<uint64_t>* file_sizes_output) {
+ uint64_t total_size_output = 0;
+ base::SmallMap<std::map<uint64_t, uint64_t>> file_id_to_sizes;
+ for (const auto& item : unreserved_file_items) {
+ const DataElement& element = item->item()->data_element();
+ uint64_t file_id = BlobDataBuilder::GetFutureFileID(element);
+ auto it = file_id_to_sizes.find(file_id);
+ if (it != file_id_to_sizes.end())
+ it->second = std::max(it->second, element.offset() + element.length());
+ else
+ file_id_to_sizes[file_id] = element.offset() + element.length();
+ total_size_output += element.length();
+ }
+ for (const auto& size_pair : file_id_to_sizes) {
+ file_sizes_output->push_back(size_pair.second);
+ }
+ return total_size_output;
+}
+
+} // namespace
+
+FileCreationInfo::FileCreationInfo() {}
+FileCreationInfo::~FileCreationInfo() {
+ if (file.IsValid()) {
+ DCHECK(file_deletion_runner);
+ file_deletion_runner->PostTask(
+ FROM_HERE, base::Bind(&DestructFile, base::Passed(&file)));
+ }
+}
+FileCreationInfo::FileCreationInfo(FileCreationInfo&&) = default;
+FileCreationInfo& FileCreationInfo::operator=(FileCreationInfo&&) = default;
+
+MemoryAllocation::MemoryAllocation(
+ base::WeakPtr<BlobMemoryController> controller,
+ uint64_t item_id,
+ size_t length)
+ : controller(controller), item_id(item_id), length(length) {}
+
+MemoryAllocation::~MemoryAllocation() {
+ if (controller)
+ controller->RevokeMemoryAllocation(item_id, length);
+}
+
+BlobMemoryController::QuotaAllocationTask::~QuotaAllocationTask() {}
+
+class BlobMemoryController::MemoryQuotaAllocationTask
+ : public BlobMemoryController::QuotaAllocationTask {
+ public:
+ MemoryQuotaAllocationTask(
+ BlobMemoryController* controller,
+ size_t quota_request_size,
+ std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items,
+ MemoryQuotaRequestCallback done_callback)
+ : controller_(controller),
+ pending_items_(std::move(pending_items)),
+ done_callback_(std::move(done_callback)),
+ allocation_size_(quota_request_size),
+ weak_factory_(this) {}
+
+ ~MemoryQuotaAllocationTask() override = default;
+
+ void RunDoneCallback(bool success) {
+ // Make sure we clear the weak pointers we gave to the caller beforehand.
+ weak_factory_.InvalidateWeakPtrs();
+ if (success)
+ controller_->GrantMemoryAllocations(&pending_items_, allocation_size_);
+ done_callback_.Run(success);
+ }
+
+ base::WeakPtr<QuotaAllocationTask> GetWeakPtr() {
+ return weak_factory_.GetWeakPtr();
+ }
+
+ void Cancel() override {
+ DCHECK_GE(controller_->pending_memory_quota_total_size_, allocation_size_);
+ controller_->pending_memory_quota_total_size_ -= allocation_size_;
+ // This call destroys this object.
+ controller_->pending_memory_quota_tasks_.erase(my_list_position_);
+ }
+
+ // The my_list_position_ iterator is stored so that we can remove ourself
+ // from the task list when we are cancelled.
+ void set_my_list_position(
+ PendingMemoryQuotaTaskList::iterator my_list_position) {
+ my_list_position_ = my_list_position;
+ }
+
+ size_t allocation_size() const { return allocation_size_; }
+
+ private:
+ BlobMemoryController* controller_;
+ std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items_;
+ MemoryQuotaRequestCallback done_callback_;
+
+ size_t allocation_size_;
+ PendingMemoryQuotaTaskList::iterator my_list_position_;
+
+ base::WeakPtrFactory<MemoryQuotaAllocationTask> weak_factory_;
+ DISALLOW_COPY_AND_ASSIGN(MemoryQuotaAllocationTask);
+};
+
+class BlobMemoryController::FileQuotaAllocationTask
+ : public BlobMemoryController::QuotaAllocationTask {
+ public:
+ // We post a task to create the file for the items right away.
+ FileQuotaAllocationTask(
+ BlobMemoryController* memory_controller,
+ std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
+ const FileQuotaRequestCallback& done_callback)
+ : controller_(memory_controller),
+ done_callback_(done_callback),
+ weak_factory_(this) {
+ // Get the file sizes and total size.
+ std::vector<uint64_t> file_sizes;
+ uint64_t total_size =
+ GetTotalSizeAndFileSizes(unreserved_file_items, &file_sizes);
+ DCHECK_LE(total_size, controller_->GetAvailableFileSpaceForBlobs());
+ allocation_size_ = total_size;
+
+ // Check & set our item states.
+ for (auto& shareable_item : unreserved_file_items) {
+ DCHECK_EQ(ShareableBlobDataItem::QUOTA_NEEDED, shareable_item->state());
+ DCHECK_EQ(DataElement::TYPE_FILE, shareable_item->item()->type());
+ shareable_item->set_state(ShareableBlobDataItem::QUOTA_REQUESTED);
+ }
+ pending_items_ = std::move(unreserved_file_items);
+
+ // Increment disk usage and create our file references.
+ controller_->disk_used_ += allocation_size_;
+ std::vector<base::FilePath> file_paths;
+ std::vector<scoped_refptr<ShareableFileReference>> references;
+ for (size_t i = 0; i < file_sizes.size(); i++) {
+ file_paths.push_back(controller_->GenerateNextPageFileName());
+ references.push_back(ShareableFileReference::GetOrCreate(
+ file_paths.back(), ShareableFileReference::DELETE_ON_FINAL_RELEASE,
+ controller_->file_runner_.get()));
+ references.back()->AddFinalReleaseCallback(
+ base::Bind(&BlobMemoryController::OnBlobFileDelete,
+ controller_->weak_factory_.GetWeakPtr(), file_sizes[i]));
+ }
+
+ // Send file creation task to file thread.
+ base::PostTaskAndReplyWithResult(
+ controller_->file_runner_.get(), FROM_HERE,
+ base::Bind(&CreateEmptyFiles, controller_->blob_storage_dir_,
+ controller_->file_runner_, base::Passed(&file_paths)),
+ base::Bind(&FileQuotaAllocationTask::OnCreateEmptyFiles,
+ weak_factory_.GetWeakPtr(), base::Passed(&references)));
+ controller_->RecordTracingCounters();
+ }
+ ~FileQuotaAllocationTask() override {}
+
+ void RunDoneCallback(bool success, std::vector<FileCreationInfo> file_info) {
+ // Make sure we clear the weak pointers we gave to the caller beforehand.
+ weak_factory_.InvalidateWeakPtrs();
+
+ // We want to destroy this object on the exit of this method if we were
+ // successful.
+ std::unique_ptr<FileQuotaAllocationTask> this_object;
+ if (success) {
+ for (auto& item : pending_items_) {
+ item->set_state(ShareableBlobDataItem::QUOTA_GRANTED);
+ }
+ this_object = std::move(*my_list_position_);
+ controller_->pending_file_quota_tasks_.erase(my_list_position_);
+ }
+
+ done_callback_.Run(success, std::move(file_info));
+ }
+
+ base::WeakPtr<QuotaAllocationTask> GetWeakPtr() {
+ return weak_factory_.GetWeakPtr();
+ }
+
+ void Cancel() override {
+ // This call destroys this object. We rely on ShareableFileReference's
+ // final release callback for disk_usage_ accounting.
+ controller_->pending_file_quota_tasks_.erase(my_list_position_);
+ }
+
+ void OnCreateEmptyFiles(
+ std::vector<scoped_refptr<ShareableFileReference>> references,
+ std::pair<std::vector<FileCreationInfo>, File::Error> files_and_error) {
+ auto& files = files_and_error.first;
+ if (files.empty()) {
+ DCHECK_GE(controller_->disk_used_, allocation_size_);
+ controller_->disk_used_ -= allocation_size_;
+ // This will call our callback and delete the object correctly.
+ controller_->DisableFilePaging(files_and_error.second);
+ return;
+ }
+ DCHECK_EQ(files.size(), references.size());
+ for (size_t i = 0; i < files.size(); i++) {
+ files[i].file_reference = std::move(references[i]);
+ }
+ RunDoneCallback(true, std::move(files));
+ }
+
+ // The my_list_position_ iterator is stored so that we can remove ourself
+ // from the task list when we are cancelled.
+ void set_my_list_position(
+ PendingFileQuotaTaskList::iterator my_list_position) {
+ my_list_position_ = my_list_position;
+ }
+
+ private:
+ BlobMemoryController* controller_;
+ std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items_;
+ scoped_refptr<base::TaskRunner> file_runner_;
+ FileQuotaRequestCallback done_callback_;
+
+ uint64_t allocation_size_;
+ PendingFileQuotaTaskList::iterator my_list_position_;
+
+ base::WeakPtrFactory<FileQuotaAllocationTask> weak_factory_;
+ DISALLOW_COPY_AND_ASSIGN(FileQuotaAllocationTask);
+};
+
+BlobMemoryController::BlobMemoryController(
+ const base::FilePath& storage_directory,
+ scoped_refptr<base::TaskRunner> file_runner)
+ : file_paging_enabled_(file_runner.get() != nullptr),
+ blob_storage_dir_(storage_directory),
+ file_runner_(std::move(file_runner)),
+ populated_memory_items_(
+ base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT),
+ weak_factory_(this) {}
+
+BlobMemoryController::~BlobMemoryController() {}
+
+void BlobMemoryController::DisableFilePaging(base::File::Error reason) {
+ UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingDisabled", -reason,
+ -File::FILE_ERROR_MAX);
+ file_paging_enabled_ = false;
+ in_flight_memory_used_ = 0;
+ items_paging_to_file_.clear();
+ pending_evictions_ = 0;
+ pending_memory_quota_total_size_ = 0;
+ populated_memory_items_.Clear();
+ populated_memory_items_bytes_ = 0;
+ file_runner_ = nullptr;
+
+ PendingMemoryQuotaTaskList old_memory_tasks;
+ PendingFileQuotaTaskList old_file_tasks;
+ std::swap(old_memory_tasks, pending_memory_quota_tasks_);
+ std::swap(old_file_tasks, pending_file_quota_tasks_);
+
+ // Don't call the callbacks until we have a consistent state.
+ for (auto& memory_request : old_memory_tasks) {
+ memory_request->RunDoneCallback(false);
+ }
+ for (auto& file_request : old_file_tasks) {
+ file_request->RunDoneCallback(false, std::vector<FileCreationInfo>());
+ }
+}
+
+BlobMemoryController::Strategy BlobMemoryController::DetermineStrategy(
+ size_t preemptive_transported_bytes,
+ uint64_t total_transportation_bytes) const {
+ if (total_transportation_bytes == 0)
+ return Strategy::NONE_NEEDED;
+ if (!CanReserveQuota(total_transportation_bytes))
+ return Strategy::TOO_LARGE;
+
+ // Handle the case where we have all the bytes preemptively transported, and
+ // we can also fit them.
+ if (preemptive_transported_bytes == total_transportation_bytes &&
+ pending_memory_quota_tasks_.empty() &&
+ preemptive_transported_bytes < GetAvailableMemoryForBlobs()) {
+ return Strategy::NONE_NEEDED;
+ }
+ if (file_paging_enabled_ &&
+ (total_transportation_bytes > limits_.memory_limit_before_paging())) {
+ return Strategy::FILE;
+ }
+ if (total_transportation_bytes > limits_.max_ipc_memory_size)
+ return Strategy::SHARED_MEMORY;
+ return Strategy::IPC;
+}
+
+bool BlobMemoryController::CanReserveQuota(uint64_t size) const {
+ // We check each size independently as a blob can't be constructed in both
+ // disk and memory.
+ return size <= GetAvailableMemoryForBlobs() ||
+ size <= GetAvailableFileSpaceForBlobs();
+}
+
+base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveMemoryQuota(
+ std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
+ const MemoryQuotaRequestCallback& done_callback) {
+ if (unreserved_memory_items.empty()) {
+ done_callback.Run(true);
+ return base::WeakPtr<QuotaAllocationTask>();
+ }
+
+ base::CheckedNumeric<uint64_t> unsafe_total_bytes_needed = 0;
+ for (auto& item : unreserved_memory_items) {
+ DCHECK_EQ(ShareableBlobDataItem::QUOTA_NEEDED, item->state());
+ DCHECK(item->item()->type() == DataElement::TYPE_BYTES_DESCRIPTION ||
+ item->item()->type() == DataElement::TYPE_BYTES);
+ DCHECK(item->item()->length() > 0);
+ unsafe_total_bytes_needed += item->item()->length();
+ item->set_state(ShareableBlobDataItem::QUOTA_REQUESTED);
+ }
+
+ uint64_t total_bytes_needed = unsafe_total_bytes_needed.ValueOrDie();
+ DCHECK_GT(total_bytes_needed, 0ull);
+
+ // If we're currently waiting for blobs to page already, then we add
+ // ourselves to the end of the queue. Once paging is complete, we'll schedule
+ // more paging for any more pending blobs.
+ if (!pending_memory_quota_tasks_.empty()) {
+ return AppendMemoryTask(total_bytes_needed,
+ std::move(unreserved_memory_items), done_callback);
+ }
+
+ // Store right away if we can.
+ if (total_bytes_needed <= GetAvailableMemoryForBlobs()) {
+ GrantMemoryAllocations(&unreserved_memory_items,
+ static_cast<size_t>(total_bytes_needed));
+ MaybeScheduleEvictionUntilSystemHealthy();
+ done_callback.Run(true);
+ return base::WeakPtr<QuotaAllocationTask>();
+ }
+
+ // Size is larger than available memory.
+ DCHECK(pending_memory_quota_tasks_.empty());
+ DCHECK_EQ(0u, pending_memory_quota_total_size_);
+
+ auto weak_ptr = AppendMemoryTask(
+ total_bytes_needed, std::move(unreserved_memory_items), done_callback);
+ MaybeScheduleEvictionUntilSystemHealthy();
+ return weak_ptr;
+}
+
+base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveFileQuota(
+ std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
+ const FileQuotaRequestCallback& done_callback) {
+ pending_file_quota_tasks_.push_back(base::MakeUnique<FileQuotaAllocationTask>(
+ this, std::move(unreserved_file_items), done_callback));
+ pending_file_quota_tasks_.back()->set_my_list_position(
+ --pending_file_quota_tasks_.end());
+ return pending_file_quota_tasks_.back()->GetWeakPtr();
+}
+
+void BlobMemoryController::NotifyMemoryItemsUsed(
+ const std::vector<scoped_refptr<ShareableBlobDataItem>>& items) {
+ for (const auto& item : items) {
+ if (item->item()->type() != DataElement::TYPE_BYTES ||
+ item->state() != ShareableBlobDataItem::POPULATED_WITH_QUOTA) {
+ continue;
+ }
+ // We don't want to re-add the item if we're currently paging it to disk.
+ if (items_paging_to_file_.find(item->item_id()) !=
+ items_paging_to_file_.end()) {
+ return;
+ }
+ auto iterator = populated_memory_items_.Get(item->item_id());
+ if (iterator == populated_memory_items_.end()) {
+ populated_memory_items_bytes_ +=
+ static_cast<size_t>(item->item()->length());
+ populated_memory_items_.Put(item->item_id(), item.get());
+ }
+ }
+ MaybeScheduleEvictionUntilSystemHealthy();
+}
+
+base::WeakPtr<QuotaAllocationTask> BlobMemoryController::AppendMemoryTask(
+ uint64_t total_bytes_needed,
+ std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
+ const MemoryQuotaRequestCallback& done_callback) {
+ DCHECK(file_paging_enabled_)
+ << "Caller tried to reserve memory when CanReserveQuota("
+ << total_bytes_needed << ") would have returned false.";
+
+ pending_memory_quota_total_size_ += total_bytes_needed;
+ pending_memory_quota_tasks_.push_back(
+ base::MakeUnique<MemoryQuotaAllocationTask>(
+ this, total_bytes_needed, std::move(unreserved_memory_items),
+ std::move(done_callback)));
+ pending_memory_quota_tasks_.back()->set_my_list_position(
+ --pending_memory_quota_tasks_.end());
+
+ return pending_memory_quota_tasks_.back()->GetWeakPtr();
+}
+
+void BlobMemoryController::MaybeGrantPendingMemoryRequests() {
+ while (!pending_memory_quota_tasks_.empty() &&
+ limits_.max_blob_in_memory_space - blob_memory_used_ >=
+ pending_memory_quota_tasks_.front()->allocation_size()) {
+ std::unique_ptr<MemoryQuotaAllocationTask> memory_task =
+ std::move(pending_memory_quota_tasks_.front());
+ pending_memory_quota_tasks_.pop_front();
+ pending_memory_quota_total_size_ -= memory_task->allocation_size();
+ memory_task->RunDoneCallback(true);
+ }
+ RecordTracingCounters();
+}
+
+size_t BlobMemoryController::CollectItemsForEviction(
+ std::vector<scoped_refptr<ShareableBlobDataItem>>* output) {
+ base::CheckedNumeric<size_t> total_items_size = 0;
+ // Process the recent item list and remove items until we have at least a
+ // minimum file size or we're at the end of our items to page to disk.
+ while (total_items_size.ValueOrDie() < limits_.min_page_file_size &&
+ !populated_memory_items_.empty()) {
+ auto iterator = --populated_memory_items_.end();
+ ShareableBlobDataItem* item = iterator->second;
+ DCHECK_EQ(item->item()->type(), DataElement::TYPE_BYTES);
+ populated_memory_items_.Erase(iterator);
+ size_t size = base::checked_cast<size_t>(item->item()->length());
+ populated_memory_items_bytes_ -= size;
+ total_items_size += size;
+ output->push_back(make_scoped_refptr(item));
+ }
+ return total_items_size.ValueOrDie();
+}
+
+void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
+ // Don't do eviction when others are happening, as we don't change our
+ // pending_memory_quota_total_size_ value until after the paging files have
+ // been written.
+ if (pending_evictions_ != 0 || !file_paging_enabled_)
+ return;
+
+ // We try to page items to disk until our current system size + requested
+ // memory is below our size limit.
+ while (pending_memory_quota_total_size_ + blob_memory_used_ >
+ limits_.memory_limit_before_paging()) {
+ // We only page when we have enough items to fill a whole page file.
+ if (populated_memory_items_bytes_ < limits_.min_page_file_size)
+ break;
+ DCHECK_LE(limits_.min_page_file_size,
+ static_cast<uint64_t>(blob_memory_used_));
+
+ std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap;
+ size_t total_items_size = CollectItemsForEviction(&items_to_swap);
+ if (total_items_size == 0)
+ break;
+
+ std::vector<DataElement*> items_for_paging;
+ for (auto& shared_blob_item : items_to_swap) {
+ items_paging_to_file_.insert(shared_blob_item->item_id());
+ items_for_paging.push_back(shared_blob_item->item()->data_element_ptr());
+ }
+
+ // Update our bookkeeping.
+ pending_evictions_++;
+ disk_used_ += total_items_size;
+ in_flight_memory_used_ += total_items_size;
+
+ // Create our file reference.
+ FilePath page_file_path = GenerateNextPageFileName();
+ scoped_refptr<ShareableFileReference> file_reference =
+ ShareableFileReference::GetOrCreate(
+ page_file_path,
+ ShareableFileReference::DELETE_ON_FINAL_RELEASE,
+ file_runner_.get());
+ // Add the release callback so we decrement our disk usage on file deletion.
+ file_reference->AddFinalReleaseCallback(
+ base::Bind(&BlobMemoryController::OnBlobFileDelete,
+ weak_factory_.GetWeakPtr(), total_items_size));
+
+ // Post the file writing task.
+ base::PostTaskAndReplyWithResult(
+ file_runner_.get(), FROM_HERE,
+ base::Bind(&CreateFileAndWriteItems, blob_storage_dir_,
+ base::Passed(&page_file_path), file_runner_,
+ base::Passed(&items_for_paging), total_items_size),
+ base::Bind(&BlobMemoryController::OnEvictionComplete,
+ weak_factory_.GetWeakPtr(), base::Passed(&file_reference),
+ base::Passed(&items_to_swap), total_items_size));
+ }
+ RecordTracingCounters();
+}
+
+void BlobMemoryController::OnEvictionComplete(
+ scoped_refptr<ShareableFileReference> file_reference,
+ std::vector<scoped_refptr<ShareableBlobDataItem>> items,
+ size_t total_items_size,
+ FileCreationInfo result) {
+ if (!file_paging_enabled_)
+ return;
+
+ if (result.error != File::FILE_OK) {
+ DisableFilePaging(result.error);
+ return;
+ }
+
+ DCHECK_LT(0, pending_evictions_);
+ pending_evictions_--;
+
+ // Switch item from memory to the new file.
+ uint64_t offset = 0;
+ for (const scoped_refptr<ShareableBlobDataItem>& shareable_item : items) {
+ scoped_refptr<BlobDataItem> new_item(
+ new BlobDataItem(base::WrapUnique(new DataElement()), file_reference));
+ new_item->data_element_ptr()->SetToFilePathRange(
+ file_reference->path(), offset, shareable_item->item()->length(),
+ result.last_modified);
+ DCHECK(shareable_item->memory_allocation_);
+ shareable_item->set_memory_allocation(nullptr);
+ shareable_item->set_item(new_item);
+ items_paging_to_file_.erase(shareable_item->item_id());
+ offset += shareable_item->item()->length();
+ }
+ in_flight_memory_used_ -= total_items_size;
+
+ // We want callback on blobs up to the amount we've freed.
+ MaybeGrantPendingMemoryRequests();
+
+ // If we still have more blobs waiting and we're not waiting on more paging
+ // operations, schedule more.
+ MaybeScheduleEvictionUntilSystemHealthy();
+}
+
+FilePath BlobMemoryController::GenerateNextPageFileName() {
+ std::string file_name = base::Uint64ToString(current_file_num_++);
+ return blob_storage_dir_.Append(FilePath::FromUTF8Unsafe(file_name));
+}
+
+void BlobMemoryController::RecordTracingCounters() const {
+ TRACE_COUNTER2("Blob", "MemoryUsage", "TotalStorage", blob_memory_used_,
+ "InFlightToDisk", in_flight_memory_used_);
+ TRACE_COUNTER1("Blob", "DiskUsage", disk_used_);
+ TRACE_COUNTER1("Blob", "TranfersPendingOnDisk",
+ pending_memory_quota_tasks_.size());
+ TRACE_COUNTER1("Blob", "TranfersBytesPendingOnDisk",
+ pending_memory_quota_total_size_);
+}
+
+size_t BlobMemoryController::GetAvailableMemoryForBlobs() const {
+ if (limits_.max_blob_in_memory_space < memory_usage())
+ return 0;
+ return limits_.max_blob_in_memory_space - memory_usage();
+}
+
+uint64_t BlobMemoryController::GetAvailableFileSpaceForBlobs() const {
+ if (!file_paging_enabled_)
+ return 0;
+ // Sometimes we're only paging part of what we need for the new blob, so add
+ // the rest of the size we need into our disk usage if this is the case.
+ uint64_t total_disk_used = disk_used_;
+ if (in_flight_memory_used_ < pending_memory_quota_total_size_) {
+ total_disk_used +=
+ pending_memory_quota_total_size_ - in_flight_memory_used_;
+ }
+ if (limits_.max_blob_disk_space < total_disk_used)
+ return 0;
+ return limits_.max_blob_disk_space - total_disk_used;
+}
+
+void BlobMemoryController::GrantMemoryAllocations(
+ std::vector<scoped_refptr<ShareableBlobDataItem>>* items,
+ size_t total_bytes) {
+ // These metrics let us calculate the global distribution of blob storage by
+ // subtracting the histograms.
+ UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
+ blob_memory_used_ / 1024);
+ blob_memory_used_ += total_bytes;
+ UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
+ blob_memory_used_ / 1024);
+
+ for (auto& item : *items) {
+ item->set_state(ShareableBlobDataItem::QUOTA_GRANTED);
+ item->set_memory_allocation(base::MakeUnique<MemoryAllocation>(
+ weak_factory_.GetWeakPtr(), item->item_id(),
+ base::checked_cast<size_t>(item->item()->length())));
+ }
+}
+
+void BlobMemoryController::RevokeMemoryAllocation(uint64_t item_id,
+ size_t length) {
+ DCHECK_LE(length, blob_memory_used_);
+
+ // These metrics let us calculate the global distribution of blob storage by
+ // subtracting the histograms.
+ UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
+ blob_memory_used_ / 1024);
+ blob_memory_used_ -= length;
+ UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
+ blob_memory_used_ / 1024);
+
+ auto iterator = populated_memory_items_.Get(item_id);
+ if (iterator != populated_memory_items_.end()) {
+ DCHECK_GE(populated_memory_items_bytes_, length);
+ populated_memory_items_bytes_ -= length;
+ populated_memory_items_.Erase(iterator);
+ }
+ MaybeGrantPendingMemoryRequests();
+}
+
+void BlobMemoryController::OnBlobFileDelete(uint64_t size,
+ const FilePath& path) {
+ DCHECK_LE(size, disk_used_);
+ disk_used_ -= size;
+}
+
+} // namespace storage
diff --git a/chromium/storage/browser/blob/blob_memory_controller.h b/chromium/storage/browser/blob/blob_memory_controller.h
new file mode 100644
index 00000000000..068e22b32ea
--- /dev/null
+++ b/chromium/storage/browser/blob/blob_memory_controller.h
@@ -0,0 +1,252 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+#define STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+
+#include <stdint.h>
+
+#include <list>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/callback_helpers.h"
+#include "base/containers/mru_cache.h"
+#include "base/files/file.h"
+#include "base/files/file_path.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/time/time.h"
+#include "storage/browser/storage_browser_export.h"
+#include "storage/common/blob_storage/blob_storage_constants.h"
+
+namespace base {
+class TaskRunner;
+}
+
+namespace storage {
+class DataElement;
+class ShareableBlobDataItem;
+class ShareableFileReference;
+
+// This class's main responsibility is deciding how blob data gets stored.
+// This encompasses:
+// * Keeping track of memory & file quota,
+// * How to transport the blob data from the renderer (DetermineStrategy),
+// * Allocating memory & file quota (ReserveMemoryQuota, ReserveFileQuota)
+// * Paging memory quota to disk when we're nearing our memory limit, and
+// * Maintaining an LRU of memory items to choose candidates to page to disk
+// (NotifyMemoryItemsUsed).
+// This class can only be interacted with on the IO thread.
+class STORAGE_EXPORT BlobMemoryController {
+ public:
+ enum class Strategy {
+ // We don't have enough memory for this blob.
+ TOO_LARGE,
+ // There isn't any memory that needs transporting.
+ NONE_NEEDED,
+ // Transportation strategies.
+ IPC,
+ SHARED_MEMORY,
+ FILE
+ };
+
+ struct STORAGE_EXPORT FileCreationInfo {
+ FileCreationInfo();
+ ~FileCreationInfo();
+ FileCreationInfo(FileCreationInfo&& other);
+ FileCreationInfo& operator=(FileCreationInfo&&);
+
+ base::File::Error error = base::File::FILE_ERROR_FAILED;
+ base::File file;
+ scoped_refptr<base::TaskRunner> file_deletion_runner;
+ base::FilePath path;
+ scoped_refptr<ShareableFileReference> file_reference;
+ base::Time last_modified;
+ };
+
+ struct MemoryAllocation {
+ MemoryAllocation(base::WeakPtr<BlobMemoryController> controller,
+ uint64_t item_id,
+ size_t length);
+ ~MemoryAllocation();
+
+ private:
+ base::WeakPtr<BlobMemoryController> controller;
+ uint64_t item_id;
+ size_t length;
+
+ DISALLOW_COPY_AND_ASSIGN(MemoryAllocation);
+ };
+
+ class QuotaAllocationTask {
+ public:
+ // Operation is cancelled and the callback will NOT be called. This object
+ // will be destroyed by this call.
+ virtual void Cancel() = 0;
+
+ protected:
+ virtual ~QuotaAllocationTask();
+ };
+
+ // The bool argument is true if we successfully received memory quota.
+ using MemoryQuotaRequestCallback = base::Callback<void(bool)>;
+ // The bool argument is true if we successfully received file quota, and the
+ // vector argument provides the file info.
+ using FileQuotaRequestCallback =
+ base::Callback<void(bool, std::vector<FileCreationInfo>)>;
+
+ // We enable file paging if |file_runner| isn't a nullptr.
+ BlobMemoryController(const base::FilePath& storage_directory,
+ scoped_refptr<base::TaskRunner> file_runner);
+ ~BlobMemoryController();
+
+ // Disables file paging. This cancels all pending file creations and paging
+ // operations. Reason is recorded in UMA.
+ void DisableFilePaging(base::File::Error reason);
+
+ bool file_paging_enabled() const { return file_paging_enabled_; }
+
+ // Returns the strategy the transportation layer should use to transport the
+ // given memory. |preemptive_transported_bytes| are the number of transport
+ // bytes that are already populated for us, so we don't haved to request them
+ // from the renderer.
+ Strategy DetermineStrategy(size_t preemptive_transported_bytes,
+ uint64_t total_transportation_bytes) const;
+
+ // Checks to see if we can reserve quota (disk or memory) for the given size.
+ bool CanReserveQuota(uint64_t size) const;
+
+ // Reserves quota for the given |unreserved_memory_items|. The items must be
+ // bytes items in QUOTA_NEEDED state which we change to QUOTA_REQUESTED.
+ // After we reserve memory quota we change their state to QUOTA_GRANTED, set
+ // the 'memory_allocation' on the item, and call |done_callback|. This can
+ // happen synchronously.
+ // Returns a task handle if the request is asynchronous for cancellation.
+ // NOTE: We don't inspect quota limits and assume the user checked
+ // CanReserveQuota before calling this.
+ base::WeakPtr<QuotaAllocationTask> ReserveMemoryQuota(
+ std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
+ const MemoryQuotaRequestCallback& done_callback);
+
+ // Reserves quota for the given |unreserved_file_items|. The items must be
+ // temporary file items (BlobDataBuilder::IsTemporaryFileItem returns true) in
+ // QUOTA_NEEDED state, which we change to QUOTA_REQUESTED. After we reserve
+ // file quota we change their state to QUOTA_GRANTED and call
+ // |done_callback|.
+ // Returns a task handle for cancellation.
+ // NOTE: We don't inspect quota limits and assume the user checked
+ // CanReserveQuota before calling this.
+ base::WeakPtr<QuotaAllocationTask> ReserveFileQuota(
+ std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
+ const FileQuotaRequestCallback& done_callback);
+
+ // Called when initially populated or upon later access.
+ void NotifyMemoryItemsUsed(
+ const std::vector<scoped_refptr<ShareableBlobDataItem>>& items);
+
+ size_t memory_usage() const { return blob_memory_used_; }
+ uint64_t disk_usage() const { return disk_used_; }
+
+ const BlobStorageLimits& limits() const { return limits_; }
+ void set_limits_for_testing(const BlobStorageLimits& limits) {
+ limits_ = limits;
+ }
+
+ private:
+ class FileQuotaAllocationTask;
+ class MemoryQuotaAllocationTask;
+
+ using PendingMemoryQuotaTaskList =
+ std::list<std::unique_ptr<MemoryQuotaAllocationTask>>;
+ using PendingFileQuotaTaskList =
+ std::list<std::unique_ptr<FileQuotaAllocationTask>>;
+
+ base::WeakPtr<QuotaAllocationTask> AppendMemoryTask(
+ uint64_t total_bytes_needed,
+ std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
+ const MemoryQuotaRequestCallback& done_callback);
+
+ void MaybeGrantPendingMemoryRequests();
+
+ size_t CollectItemsForEviction(
+ std::vector<scoped_refptr<ShareableBlobDataItem>>* output);
+
+ // Schedule paging until our memory usage is below our memory limit.
+ void MaybeScheduleEvictionUntilSystemHealthy();
+
+ // Called when we've completed evicting a list of items to disk. This is where
+ // we swap the bytes items for file items, and update our bookkeeping.
+ void OnEvictionComplete(
+ scoped_refptr<ShareableFileReference> file_reference,
+ std::vector<scoped_refptr<ShareableBlobDataItem>> items,
+ size_t total_items_size,
+ FileCreationInfo result);
+
+ size_t GetAvailableMemoryForBlobs() const;
+ uint64_t GetAvailableFileSpaceForBlobs() const;
+
+ void GrantMemoryAllocations(
+ std::vector<scoped_refptr<ShareableBlobDataItem>>* items,
+ size_t total_bytes);
+ void RevokeMemoryAllocation(uint64_t item_id, size_t length);
+
+ // This is registered as a callback for file deletions on the file reference
+ // of our paging files. We decrement the disk space used.
+ void OnBlobFileDelete(uint64_t size, const base::FilePath& path);
+
+ base::FilePath GenerateNextPageFileName();
+
+ // This records diagnostic counters of our memory quotas. Called when usage
+ // changes.
+ void RecordTracingCounters() const;
+
+ BlobStorageLimits limits_;
+
+ // Memory bookkeeping. These numbers are all disjoint.
+ // This is the amount of memory we're using for blobs in RAM, including the
+ // in_flight_memory_used_.
+ size_t blob_memory_used_ = 0;
+ // This is memory we're temporarily using while we try to write blob items to
+ // disk.
+ size_t in_flight_memory_used_ = 0;
+ // This is the amount of memory we're using on disk.
+ uint64_t disk_used_ = 0;
+
+ // State for GenerateNextPageFileName.
+ uint64_t current_file_num_ = 0;
+
+ size_t pending_memory_quota_total_size_ = 0;
+ PendingMemoryQuotaTaskList pending_memory_quota_tasks_;
+ PendingFileQuotaTaskList pending_file_quota_tasks_;
+
+ int pending_evictions_ = 0;
+
+ bool file_paging_enabled_ = false;
+ base::FilePath blob_storage_dir_;
+ scoped_refptr<base::TaskRunner> file_runner_;
+
+ // Lifetime of the ShareableBlobDataItem objects is handled externally in the
+ // BlobStorageContext class.
+ base::MRUCache<uint64_t, ShareableBlobDataItem*> populated_memory_items_;
+ size_t populated_memory_items_bytes_ = 0;
+ // We need to keep track of items currently being paged to disk so that if
+ // another blob successfully grabs a ref, we can prevent it from adding the
+ // item to the recent_item_cache_ above.
+ std::unordered_set<uint64_t> items_paging_to_file_;
+
+ base::WeakPtrFactory<BlobMemoryController> weak_factory_;
+
+ DISALLOW_COPY_AND_ASSIGN(BlobMemoryController);
+};
+} // namespace storage
+#endif // STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
diff --git a/chromium/storage/browser/blob/blob_reader.cc b/chromium/storage/browser/blob/blob_reader.cc
index 2afb964e88a..1ec3002a8d1 100644
--- a/chromium/storage/browser/blob/blob_reader.cc
+++ b/chromium/storage/browser/blob/blob_reader.cc
@@ -39,20 +39,25 @@ bool IsFileType(DataElement::Type type) {
}
}
-int ConvertBlobErrorToNetError(IPCBlobCreationCancelCode reason) {
+int ConvertBlobErrorToNetError(BlobStatus reason) {
switch (reason) {
- case IPCBlobCreationCancelCode::UNKNOWN:
+ case BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS:
return net::ERR_FAILED;
- case IPCBlobCreationCancelCode::OUT_OF_MEMORY:
+ case BlobStatus::ERR_OUT_OF_MEMORY:
return net::ERR_OUT_OF_MEMORY;
- case IPCBlobCreationCancelCode::FILE_WRITE_FAILED:
+ case BlobStatus::ERR_FILE_WRITE_FAILED:
return net::ERR_FILE_NO_SPACE;
- case IPCBlobCreationCancelCode::SOURCE_DIED_IN_TRANSIT:
+ case BlobStatus::ERR_SOURCE_DIED_IN_TRANSIT:
return net::ERR_UNEXPECTED;
- case IPCBlobCreationCancelCode::BLOB_DEREFERENCED_WHILE_BUILDING:
+ case BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING:
return net::ERR_UNEXPECTED;
- case IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN:
+ case BlobStatus::ERR_REFERENCED_BLOB_BROKEN:
return net::ERR_INVALID_HANDLE;
+ case BlobStatus::DONE:
+ case BlobStatus::PENDING_QUOTA:
+ case BlobStatus::PENDING_TRANSPORT:
+ case BlobStatus::PENDING_INTERNALS:
+ NOTREACHED();
}
NOTREACHED();
return net::ERR_FAILED;
@@ -254,10 +259,9 @@ BlobReader::Status BlobReader::ReportError(int net_error) {
}
void BlobReader::AsyncCalculateSize(const net::CompletionCallback& done,
- bool async_succeeded,
- IPCBlobCreationCancelCode reason) {
- if (!async_succeeded) {
- InvalidateCallbacksAndDone(ConvertBlobErrorToNetError(reason), done);
+ BlobStatus status) {
+ if (BlobStatusIsError(status)) {
+ InvalidateCallbacksAndDone(ConvertBlobErrorToNetError(status), done);
return;
}
DCHECK(!blob_handle_->IsBroken()) << "Callback should have returned false.";
diff --git a/chromium/storage/browser/blob/blob_reader.h b/chromium/storage/browser/blob/blob_reader.h
index 5a58f8aa255..a7b1d12be35 100644
--- a/chromium/storage/browser/blob/blob_reader.h
+++ b/chromium/storage/browser/blob/blob_reader.h
@@ -155,8 +155,7 @@ class STORAGE_EXPORT BlobReader {
void InvalidateCallbacksAndDone(int net_error, net::CompletionCallback done);
void AsyncCalculateSize(const net::CompletionCallback& done,
- bool async_succeeded,
- IPCBlobCreationCancelCode reason);
+ BlobStatus status);
Status CalculateSizeImpl(const net::CompletionCallback& done);
bool AddItemLength(size_t index, uint64_t length);
bool ResolveFileItemLength(const BlobDataItem& item,
diff --git a/chromium/storage/browser/blob/blob_storage_context.cc b/chromium/storage/browser/blob/blob_storage_context.cc
index 6ce45e53f90..1fecda89f5b 100644
--- a/chromium/storage/browser/blob/blob_storage_context.cc
+++ b/chromium/storage/browser/blob/blob_storage_context.cc
@@ -4,72 +4,367 @@
#include "storage/browser/blob/blob_storage_context.h"
-#include <stddef.h>
-#include <stdint.h>
-
#include <algorithm>
#include <limits>
#include <memory>
+#include <set>
#include <utility>
#include "base/bind.h"
#include "base/callback.h"
+#include "base/callback_helpers.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/message_loop/message_loop.h"
-#include "base/metrics/histogram.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/task_runner.h"
#include "base/threading/thread_task_runner_handle.h"
#include "base/trace_event/trace_event.h"
#include "storage/browser/blob/blob_data_builder.h"
-#include "storage/browser/blob/blob_data_handle.h"
#include "storage/browser/blob/blob_data_item.h"
#include "storage/browser/blob/blob_data_snapshot.h"
#include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
#include "url/gurl.h"
namespace storage {
-using BlobRegistryEntry = BlobStorageRegistry::Entry;
-using BlobState = BlobStorageRegistry::BlobState;
+namespace {
+using ItemCopyEntry = BlobEntry::ItemCopyEntry;
+using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+
+bool IsBytes(DataElement::Type type) {
+ return type == DataElement::TYPE_BYTES ||
+ type == DataElement::TYPE_BYTES_DESCRIPTION;
+}
+
+void RecordBlobItemSizeStats(const DataElement& input_element) {
+ uint64_t length = input_element.length();
+
+ switch (input_element.type()) {
+ case DataElement::TYPE_BYTES:
+ case DataElement::TYPE_BYTES_DESCRIPTION:
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024);
+ break;
+ case DataElement::TYPE_BLOB:
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob",
+ (length - input_element.offset()) / 1024);
+ break;
+ case DataElement::TYPE_FILE: {
+ bool full_file = (length == std::numeric_limits<uint64_t>::max());
+ UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file);
+ if (!full_file) {
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File",
+ (length - input_element.offset()) / 1024);
+ }
+ break;
+ }
+ case DataElement::TYPE_FILE_FILESYSTEM: {
+ bool full_file = (length == std::numeric_limits<uint64_t>::max());
+ UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown",
+ full_file);
+ if (!full_file) {
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem",
+ (length - input_element.offset()) / 1024);
+ }
+ break;
+ }
+ case DataElement::TYPE_DISK_CACHE_ENTRY:
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
+ (length - input_element.offset()) / 1024);
+ break;
+ case DataElement::TYPE_UNKNOWN:
+ NOTREACHED();
+ break;
+ }
+}
+} // namespace
+
+BlobStorageContext::BlobFlattener::BlobFlattener(
+ const BlobDataBuilder& input_builder,
+ BlobEntry* output_blob,
+ BlobStorageRegistry* registry) {
+ const std::string& uuid = input_builder.uuid_;
+ std::set<std::string> dependent_blob_uuids;
+
+ size_t num_files_with_unknown_size = 0;
+ size_t num_building_dependent_blobs = 0;
+
+ base::CheckedNumeric<uint64_t> checked_total_size = 0;
+ base::CheckedNumeric<uint64_t> checked_total_memory_size = 0;
+ base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0;
+
+ for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
+ const DataElement& input_element = input_item->data_element();
+ DataElement::Type type = input_element.type();
+ uint64_t length = input_element.length();
+
+ RecordBlobItemSizeStats(input_element);
+
+ if (IsBytes(type)) {
+ DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());
+ checked_memory_quota_needed += length;
+ checked_total_size += length;
+ scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+ std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
+ pending_memory_items.push_back(item);
+ transport_items.push_back(item.get());
+ output_blob->AppendSharedBlobItem(std::move(item));
+ continue;
+ }
+ if (type == DataElement::TYPE_BLOB) {
+ BlobEntry* ref_entry = registry->GetEntry(input_element.blob_uuid());
+
+ if (!ref_entry || input_element.blob_uuid() == uuid) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+
+ if (BlobStatusIsError(ref_entry->status())) {
+ status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN;
+ return;
+ }
+
+ if (ref_entry->total_size() == DataElement::kUnknownSize) {
+ // We can't reference a blob with unknown size.
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+
+ if (dependent_blob_uuids.find(input_element.blob_uuid()) ==
+ dependent_blob_uuids.end()) {
+ dependent_blobs.push_back(
+ std::make_pair(input_element.blob_uuid(), ref_entry));
+ dependent_blob_uuids.insert(input_element.blob_uuid());
+ if (BlobStatusIsPending(ref_entry->status())) {
+ num_building_dependent_blobs++;
+ }
+ }
+
+ length = length == DataElement::kUnknownSize ? ref_entry->total_size()
+ : input_element.length();
+ checked_total_size += length;
+
+ // If we're referencing the whole blob, then we don't need to slice.
+ if (input_element.offset() == 0 && length == ref_entry->total_size()) {
+ for (const auto& shareable_item : ref_entry->items()) {
+ output_blob->AppendSharedBlobItem(shareable_item);
+ }
+ continue;
+ }
+
+ // Validate our reference has good offset & length.
+ if (input_element.offset() + length > ref_entry->total_size()) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
-BlobStorageContext::BlobStorageContext() : memory_usage_(0) {}
+ BlobSlice slice(*ref_entry, input_element.offset(), length);
+
+ if (!slice.copying_memory_size.IsValid() ||
+ !slice.total_memory_size.IsValid()) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+ checked_total_memory_size += slice.total_memory_size;
+
+ if (slice.first_source_item) {
+ copies.push_back(ItemCopyEntry(slice.first_source_item,
+ slice.first_item_slice_offset,
+ slice.dest_items.front()));
+ pending_memory_items.push_back(slice.dest_items.front());
+ }
+ if (slice.last_source_item) {
+ copies.push_back(
+ ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
+ pending_memory_items.push_back(slice.dest_items.back());
+ }
+ checked_memory_quota_needed += slice.copying_memory_size;
+
+ for (auto& shareable_item : slice.dest_items) {
+ output_blob->AppendSharedBlobItem(std::move(shareable_item));
+ }
+ continue;
+ }
+
+ DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element))
+ << "File allocation not implemented.";
+ if (length == DataElement::kUnknownSize)
+ num_files_with_unknown_size++;
+
+ scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+ std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+
+ checked_total_size += length;
+ output_blob->AppendSharedBlobItem(std::move(item));
+ }
+
+ if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+ if (!checked_total_size.IsValid() || !checked_total_memory_size.IsValid() ||
+ !checked_memory_quota_needed.IsValid()) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+ total_size = checked_total_size.ValueOrDie();
+ total_memory_size = checked_total_memory_size.ValueOrDie();
+ memory_quota_needed = checked_memory_quota_needed.ValueOrDie();
+ if (memory_quota_needed) {
+ status = BlobStatus::PENDING_QUOTA;
+ } else {
+ status = BlobStatus::PENDING_INTERNALS;
+ }
+}
+
+BlobStorageContext::BlobFlattener::~BlobFlattener() {}
+
+BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source,
+ uint64_t slice_offset,
+ uint64_t slice_size) {
+ const auto& source_items = source.items();
+ const auto& offsets = source.offsets();
+ DCHECK_LE(slice_offset + slice_size, source.total_size());
+ size_t item_index =
+ std::upper_bound(offsets.begin(), offsets.end(), slice_offset) -
+ offsets.begin();
+ uint64_t item_offset =
+ item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1];
+ size_t num_items = source_items.size();
+
+ size_t first_item_index = item_index;
+
+ // Read starting from 'first_item_index' and 'item_offset'.
+ for (uint64_t total_sliced = 0;
+ item_index < num_items && total_sliced < slice_size; item_index++) {
+ const scoped_refptr<BlobDataItem>& source_item =
+ source_items[item_index]->item();
+ uint64_t source_length = source_item->length();
+ DataElement::Type type = source_item->type();
+ DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max());
+ DCHECK_NE(source_length, 0ull);
+
+ uint64_t read_size =
+ std::min(source_length - item_offset, slice_size - total_sliced);
+ total_sliced += read_size;
+
+ bool reusing_blob_item = (read_size == source_length);
+ UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item);
+ if (reusing_blob_item) {
+ // We can share the entire item.
+ dest_items.push_back(source_items[item_index]);
+ if (IsBytes(type)) {
+ total_memory_size += source_length;
+ }
+ continue;
+ }
+
+ scoped_refptr<BlobDataItem> data_item;
+ ShareableBlobDataItem::State state =
+ ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA;
+ switch (type) {
+ case DataElement::TYPE_BYTES_DESCRIPTION:
+ case DataElement::TYPE_BYTES: {
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes",
+ read_size / 1024);
+ if (item_index == first_item_index) {
+ first_item_slice_offset = item_offset;
+ first_source_item = source_items[item_index];
+ } else {
+ last_source_item = source_items[item_index];
+ }
+ copying_memory_size += read_size;
+ total_memory_size += read_size;
+ // Since we don't have quota yet for memory, we create temporary items
+ // for this data. When our blob is finished constructing, all dependent
+ // blobs are done, and we have enough memory quota, we'll copy the data
+ // over.
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToBytesDescription(base::checked_cast<size_t>(read_size));
+ data_item = new BlobDataItem(std::move(element));
+ state = ShareableBlobDataItem::QUOTA_NEEDED;
+ break;
+ }
+ case DataElement::TYPE_FILE: {
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
+ read_size / 1024);
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToFilePathRange(
+ source_item->path(), source_item->offset() + item_offset, read_size,
+ source_item->expected_modification_time());
+ data_item =
+ new BlobDataItem(std::move(element), source_item->data_handle_);
+
+ DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element()))
+ << "File allocation unimplemented.";
+ break;
+ }
+ case DataElement::TYPE_FILE_FILESYSTEM: {
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
+ read_size / 1024);
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToFileSystemUrlRange(
+ source_item->filesystem_url(), source_item->offset() + item_offset,
+ read_size, source_item->expected_modification_time());
+ data_item = new BlobDataItem(std::move(element));
+ break;
+ }
+ case DataElement::TYPE_DISK_CACHE_ENTRY: {
+ UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.CacheEntry",
+ read_size / 1024);
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToDiskCacheEntryRange(source_item->offset() + item_offset,
+ read_size);
+ data_item =
+ new BlobDataItem(std::move(element), source_item->data_handle_,
+ source_item->disk_cache_entry(),
+ source_item->disk_cache_stream_index(),
+ source_item->disk_cache_side_stream_index());
+ break;
+ }
+ case DataElement::TYPE_BLOB:
+ case DataElement::TYPE_UNKNOWN:
+ CHECK(false) << "Illegal blob item type: " << type;
+ }
+ dest_items.push_back(
+ new ShareableBlobDataItem(std::move(data_item), state));
+ item_offset = 0;
+ }
+}
+
+BlobStorageContext::BlobSlice::~BlobSlice() {}
+
+BlobStorageContext::BlobStorageContext()
+ : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()),
+ ptr_factory_(this) {}
BlobStorageContext::~BlobStorageContext() {
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
const std::string& uuid) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (!entry) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ if (!entry)
return nullptr;
- }
- return base::WrapUnique(
- new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
- this, base::ThreadTaskRunnerHandle::Get().get()));
+ return CreateHandle(uuid, entry);
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL(
const GURL& url) {
std::string uuid;
- BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid);
- if (!entry) {
+ BlobEntry* entry = registry_.GetEntryFromURL(url, &uuid);
+ if (!entry)
return nullptr;
- }
- return base::WrapUnique(
- new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
- this, base::ThreadTaskRunnerHandle::Get().get()));
+ return CreateHandle(uuid, entry);
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
const BlobDataBuilder& external_builder) {
TRACE_EVENT0("Blob", "Context::AddFinishedBlob");
- CreatePendingBlob(external_builder.uuid(), external_builder.content_type_,
- external_builder.content_disposition_);
- CompletePendingBlob(external_builder);
- std::unique_ptr<BlobDataHandle> handle =
- GetBlobDataFromUUID(external_builder.uuid_);
- DecrementBlobRefCount(external_builder.uuid_);
- return handle;
+ return BuildBlob(external_builder, TransportAllowedCallback());
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
@@ -78,123 +373,149 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
return AddFinishedBlob(*builder);
}
+std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob(
+ const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition,
+ BlobStatus reason) {
+ DCHECK(!registry_.HasEntry(uuid));
+ DCHECK(BlobStatusIsError(reason));
+ BlobEntry* entry =
+ registry_.CreateEntry(uuid, content_type, content_disposition);
+ entry->set_status(reason);
+ FinishBuilding(entry);
+ return CreateHandle(uuid, entry);
+}
+
bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url,
const std::string& uuid) {
- if (!registry_.CreateUrlMapping(blob_url, uuid)) {
+ if (!registry_.CreateUrlMapping(blob_url, uuid))
return false;
- }
IncrementBlobRefCount(uuid);
return true;
}
void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) {
std::string uuid;
- if (!registry_.DeleteURLMapping(blob_url, &uuid)) {
+ if (!registry_.DeleteURLMapping(blob_url, &uuid))
return;
- }
DecrementBlobRefCount(uuid);
}
-void BlobStorageContext::CreatePendingBlob(
- const std::string& uuid,
- const std::string& content_type,
- const std::string& content_disposition) {
- DCHECK(!registry_.GetEntry(uuid) && !uuid.empty());
- registry_.CreateEntry(uuid, content_type, content_disposition);
-}
-
-void BlobStorageContext::CompletePendingBlob(
- const BlobDataBuilder& external_builder) {
- BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid());
- DCHECK(entry);
- DCHECK(!entry->data.get()) << "Blob already constructed: "
- << external_builder.uuid();
- // We want to handle storing our broken blob as well.
- switch (entry->state) {
- case BlobState::PENDING: {
- entry->data_builder.reset(new InternalBlobData::Builder());
- InternalBlobData::Builder* internal_data_builder =
- entry->data_builder.get();
-
- bool broken = false;
- for (const auto& blob_item : external_builder.items_) {
- IPCBlobCreationCancelCode error_code;
- if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item,
- internal_data_builder, &error_code)) {
- broken = true;
- memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage();
- entry->state = BlobState::BROKEN;
- entry->broken_reason = error_code;
- entry->data_builder.reset(new InternalBlobData::Builder());
- break;
- }
- }
- entry->data = entry->data_builder->Build();
- entry->data_builder.reset();
- entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE;
- break;
- }
- case BlobState::BROKEN: {
- InternalBlobData::Builder builder;
- entry->data = builder.Build();
- break;
+std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
+ const BlobDataBuilder& content,
+ const TransportAllowedCallback& transport_allowed_callback) {
+ DCHECK(!registry_.HasEntry(content.uuid_));
+
+ BlobEntry* entry = registry_.CreateEntry(
+ content.uuid(), content.content_type_, content.content_disposition_);
+
+ // This flattens all blob references in the transportion content out and
+ // stores the complete item representation in the internal data.
+ BlobFlattener flattener(content, entry, &registry_);
+
+ DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+ !transport_allowed_callback)
+ << "There is no pending content for the user to populate, so the "
+ "callback should be null.";
+ DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+ transport_allowed_callback)
+ << "If we have pending content then there needs to be a callback "
+ "present.";
+
+ entry->set_size(flattener.total_size);
+ entry->set_status(flattener.status);
+ std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry);
+
+ UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->items().size());
+ UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize",
+ flattener.total_memory_size / 1024);
+ UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
+ flattener.memory_quota_needed / 1024);
+
+ size_t num_building_dependent_blobs = 0;
+ std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
+ // We hold a handle to all blobs we're using. This is important, as our memory
+ // accounting can be delayed until OnEnoughSizeForBlobData is called, and we
+ // only free memory on canceling when we've done this accounting. If a
+ // dependent blob is dereferenced, then we're the last blob holding onto that
+ // data item, and we need to account for that. So we prevent that case by
+ // holding onto all blobs.
+ for (const std::pair<std::string, BlobEntry*>& pending_blob :
+ flattener.dependent_blobs) {
+ dependent_blobs.push_back(
+ CreateHandle(pending_blob.first, pending_blob.second));
+ if (BlobStatusIsPending(pending_blob.second->status())) {
+ pending_blob.second->building_state_->build_completion_callbacks
+ .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished,
+ ptr_factory_.GetWeakPtr(), content.uuid_));
+ num_building_dependent_blobs++;
}
- case BlobState::COMPLETE:
- DCHECK(false) << "Blob already constructed: " << external_builder.uuid();
- return;
}
- UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size());
- UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken",
- entry->state == BlobState::BROKEN);
- if (entry->state == BlobState::BROKEN) {
- UMA_HISTOGRAM_ENUMERATION(
- "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason),
- (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1));
+ entry->set_building_state(base::MakeUnique<BlobEntry::BuildingState>(
+ !flattener.transport_items.empty(), transport_allowed_callback,
+ num_building_dependent_blobs));
+ BlobEntry::BuildingState* building_state = entry->building_state_.get();
+ std::swap(building_state->copies, flattener.copies);
+ std::swap(building_state->dependent_blobs, dependent_blobs);
+ std::swap(building_state->transport_items, flattener.transport_items);
+
+ // Break ourselves if we have an error. BuildingState must be set first so the
+ // callback is called correctly.
+ if (BlobStatusIsError(flattener.status)) {
+ CancelBuildingBlobInternal(entry, flattener.status);
+ return handle;
}
- size_t total_memory = 0, nonshared_memory = 0;
- entry->data->GetMemoryUsage(&total_memory, &nonshared_memory);
- UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024);
- UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
- nonshared_memory / 1024);
- TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_);
- auto runner = base::ThreadTaskRunnerHandle::Get();
- for (const auto& callback : entry->build_completion_callbacks) {
- runner->PostTask(FROM_HERE,
- base::Bind(callback, entry->state == BlobState::COMPLETE,
- entry->broken_reason));
+ if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) {
+ CancelBuildingBlobInternal(entry, BlobStatus::ERR_OUT_OF_MEMORY);
+ return handle;
}
- entry->build_completion_callbacks.clear();
+
+ if (flattener.memory_quota_needed > 0) {
+ // The blob can complete during the execution of this line.
+ base::WeakPtr<QuotaAllocationTask> pending_request =
+ memory_controller_.ReserveMemoryQuota(
+ std::move(flattener.pending_memory_items),
+ base::Bind(&BlobStorageContext::OnEnoughSizeForMemory,
+ ptr_factory_.GetWeakPtr(), content.uuid_));
+ // Building state will be null if the blob is already finished.
+ if (entry->building_state_)
+ entry->building_state_->memory_quota_request = std::move(pending_request);
+ }
+
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
+
+ return handle;
+}
+
+void BlobStorageContext::CancelBuildingBlob(const std::string& uuid,
+ BlobStatus reason) {
+ CancelBuildingBlobInternal(registry_.GetEntry(uuid), reason);
}
-void BlobStorageContext::CancelPendingBlob(const std::string& uuid,
- IPCBlobCreationCancelCode reason) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- DCHECK(entry && entry->state == BlobState::PENDING);
- entry->state = BlobState::BROKEN;
- entry->broken_reason = reason;
- CompletePendingBlob(BlobDataBuilder(uuid));
+void BlobStorageContext::NotifyTransportComplete(const std::string& uuid) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ CHECK(entry) << "There is no blob entry with uuid " << uuid;
+ DCHECK(BlobStatusIsPending(entry->status()));
+ NotifyTransportCompleteInternal(entry);
}
void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+ BlobEntry* entry = registry_.GetEntry(uuid);
DCHECK(entry);
- ++(entry->refcount);
+ entry->IncrementRefCount();
}
void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+ BlobEntry* entry = registry_.GetEntry(uuid);
DCHECK(entry);
- DCHECK_GT(entry->refcount, 0u);
- if (--(entry->refcount) == 0) {
- size_t memory_freeing = 0;
- if (entry->state == BlobState::COMPLETE) {
- memory_freeing = entry->data->GetUnsharedMemoryUsage();
- entry->data->RemoveBlobFromShareableItems(uuid);
- }
- DCHECK_LE(memory_freeing, memory_usage_);
- memory_usage_ -= memory_freeing;
+ DCHECK_GT(entry->refcount(), 0u);
+ entry->DecrementRefCount();
+ if (entry->refcount() == 0) {
+ ClearAndFreeMemory(entry);
registry_.DeleteEntry(uuid);
}
}
@@ -202,255 +523,208 @@ void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot(
const std::string& uuid) {
std::unique_ptr<BlobDataSnapshot> result;
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (entry->state != BlobState::COMPLETE) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ if (entry->status() != BlobStatus::DONE)
return result;
- }
- const InternalBlobData& data = *entry->data;
std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot(
- uuid, entry->content_type, entry->content_disposition));
- snapshot->items_.reserve(data.items().size());
- for (const auto& shareable_item : data.items()) {
+ uuid, entry->content_type(), entry->content_disposition()));
+ snapshot->items_.reserve(entry->items().size());
+ for (const auto& shareable_item : entry->items()) {
snapshot->items_.push_back(shareable_item->item());
}
+ memory_controller_.NotifyMemoryItemsUsed(entry->items());
return snapshot;
}
-bool BlobStorageContext::IsBroken(const std::string& uuid) const {
- const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (!entry) {
- return true;
- }
- return entry->state == BlobState::BROKEN;
+BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const {
+ const BlobEntry* entry = registry_.GetEntry(uuid);
+ if (!entry)
+ return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return entry->status();
}
-bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const {
- const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (!entry) {
- return false;
+void BlobStorageContext::RunOnConstructionComplete(
+ const std::string& uuid,
+ const BlobStatusCallback& done) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ DCHECK(entry);
+ if (BlobStatusIsPending(entry->status())) {
+ entry->building_state_->build_completion_callbacks.push_back(done);
+ return;
}
- return entry->state == BlobState::PENDING;
+ done.Run(entry->status());
}
-void BlobStorageContext::RunOnConstructionComplete(
+std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle(
const std::string& uuid,
- const BlobConstructedCallback& done) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+ BlobEntry* entry) {
+ return base::WrapUnique(new BlobDataHandle(
+ uuid, entry->content_type_, entry->content_disposition_, entry->size_,
+ this, base::ThreadTaskRunnerHandle::Get().get()));
+}
+
+void BlobStorageContext::NotifyTransportCompleteInternal(BlobEntry* entry) {
DCHECK(entry);
- switch (entry->state) {
- case BlobState::COMPLETE:
- done.Run(true, IPCBlobCreationCancelCode::UNKNOWN);
- return;
- case BlobState::BROKEN:
- done.Run(false, entry->broken_reason);
- return;
- case BlobState::PENDING:
- entry->build_completion_callbacks.push_back(done);
- return;
+ for (ShareableBlobDataItem* shareable_item :
+ entry->building_state_->transport_items) {
+ DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED);
+ shareable_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
}
- NOTREACHED();
+ entry->set_status(BlobStatus::PENDING_INTERNALS);
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
}
-bool BlobStorageContext::AppendAllocatedBlobItem(
- const std::string& target_blob_uuid,
- scoped_refptr<BlobDataItem> blob_item,
- InternalBlobData::Builder* target_blob_builder,
- IPCBlobCreationCancelCode* error_code) {
- DCHECK(error_code);
- *error_code = IPCBlobCreationCancelCode::UNKNOWN;
- bool error = false;
-
- // The blob data is stored in the canonical way which only contains a
- // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items
- // are expanded into the primitive constituent types and reused if possible.
- // 1) The Data item is denoted by the raw data and length.
- // 2) The File item is denoted by the file path, the range and the expected
- // modification time.
- // 3) The FileSystem File item is denoted by the FileSystem URL, the range
- // and the expected modification time.
- // 4) The Blob item is denoted by the source blob and an offset and size.
- // Internal items that are fully used by the new blob (not cut by the
- // offset or size) are shared between the blobs. Otherwise, the relevant
- // portion of the item is copied.
-
- DCHECK(blob_item->data_element_ptr());
- const DataElement& data_element = blob_item->data_element();
- uint64_t length = data_element.length();
- uint64_t offset = data_element.offset();
- UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
- memory_usage_ / 1024);
- switch (data_element.type()) {
- case DataElement::TYPE_BYTES:
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024);
- DCHECK(!offset);
- if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) {
- error = true;
- *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
- break;
- }
- memory_usage_ += length;
- target_blob_builder->AppendSharedBlobItem(
- new ShareableBlobDataItem(target_blob_uuid, blob_item));
- break;
- case DataElement::TYPE_FILE: {
- bool full_file = (length == std::numeric_limits<uint64_t>::max());
- UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file);
- if (!full_file) {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File",
- (length - offset) / 1024);
- }
- target_blob_builder->AppendSharedBlobItem(
- new ShareableBlobDataItem(target_blob_uuid, blob_item));
- break;
- }
- case DataElement::TYPE_FILE_FILESYSTEM: {
- bool full_file = (length == std::numeric_limits<uint64_t>::max());
- UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown",
- full_file);
- if (!full_file) {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem",
- (length - offset) / 1024);
- }
- target_blob_builder->AppendSharedBlobItem(
- new ShareableBlobDataItem(target_blob_uuid, blob_item));
- break;
- }
- case DataElement::TYPE_BLOB: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob",
- (length - offset) / 1024);
- // We grab the handle to ensure it stays around while we copy it.
- std::unique_ptr<BlobDataHandle> src =
- GetBlobDataFromUUID(data_element.blob_uuid());
- if (!src || src->IsBroken() || src->IsBeingBuilt()) {
- error = true;
- *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
- break;
- }
- BlobRegistryEntry* other_entry =
- registry_.GetEntry(data_element.blob_uuid());
- DCHECK(other_entry->data);
- if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length,
- target_blob_builder)) {
- error = true;
- *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
+void BlobStorageContext::CancelBuildingBlobInternal(BlobEntry* entry,
+ BlobStatus reason) {
+ DCHECK(entry);
+ DCHECK(BlobStatusIsError(reason));
+ TransportAllowedCallback transport_allowed_callback;
+ if (entry->building_state_ &&
+ entry->building_state_->transport_allowed_callback) {
+ transport_allowed_callback =
+ entry->building_state_->transport_allowed_callback;
+ entry->building_state_->transport_allowed_callback.Reset();
+ }
+ ClearAndFreeMemory(entry);
+ entry->set_status(reason);
+ if (transport_allowed_callback) {
+ transport_allowed_callback.Run(
+ reason, std::vector<BlobMemoryController::FileCreationInfo>());
+ }
+ FinishBuilding(entry);
+}
+
+void BlobStorageContext::FinishBuilding(BlobEntry* entry) {
+ DCHECK(entry);
+
+ BlobStatus status = entry->status_;
+ DCHECK_NE(BlobStatus::DONE, status);
+
+ bool error = BlobStatusIsError(status);
+ UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken", error);
+ if (error) {
+ UMA_HISTOGRAM_ENUMERATION("Storage.Blob.BrokenReason",
+ static_cast<int>(status),
+ (static_cast<int>(BlobStatus::LAST_ERROR) + 1));
+ }
+
+ if (BlobStatusIsPending(entry->status_)) {
+ for (const ItemCopyEntry& copy : entry->building_state_->copies) {
+ // Our source item can be a file if it was a slice of an unpopulated file,
+ // or a slice of data that was then paged to disk.
+ size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length());
+ DataElement::Type dest_type = copy.dest_item->item()->type();
+ switch (copy.source_item->item()->type()) {
+ case DataElement::TYPE_BYTES: {
+ DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION);
+ const char* src_data =
+ copy.source_item->item()->bytes() + copy.source_item_offset;
+ copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
+ } break;
+ case DataElement::TYPE_FILE:
+ case DataElement::TYPE_UNKNOWN:
+ case DataElement::TYPE_BLOB:
+ case DataElement::TYPE_BYTES_DESCRIPTION:
+ case DataElement::TYPE_FILE_FILESYSTEM:
+ case DataElement::TYPE_DISK_CACHE_ENTRY:
+ NOTREACHED();
+ break;
}
- break;
+ copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
}
- case DataElement::TYPE_DISK_CACHE_ENTRY: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
- (length - offset) / 1024);
- target_blob_builder->AppendSharedBlobItem(
- new ShareableBlobDataItem(target_blob_uuid, blob_item));
- break;
- }
- case DataElement::TYPE_BYTES_DESCRIPTION:
- case DataElement::TYPE_UNKNOWN:
- NOTREACHED();
- break;
+
+ entry->set_status(BlobStatus::DONE);
+ }
+
+ std::vector<BlobStatusCallback> callbacks;
+ if (entry->building_state_.get()) {
+ std::swap(callbacks, entry->building_state_->build_completion_callbacks);
+ entry->building_state_.reset();
+ }
+
+ memory_controller_.NotifyMemoryItemsUsed(entry->items());
+
+ auto runner = base::ThreadTaskRunnerHandle::Get();
+ for (const auto& callback : callbacks)
+ runner->PostTask(FROM_HERE, base::Bind(callback, entry->status()));
+
+ for (const auto& shareable_item : entry->items()) {
+ DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION,
+ shareable_item->item()->type());
+ DCHECK(shareable_item->IsPopulated()) << shareable_item->state();
}
- UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
- memory_usage_ / 1024);
- return !error;
}
-bool BlobStorageContext::AppendBlob(
- const std::string& target_blob_uuid,
- const InternalBlobData& blob,
- uint64_t offset,
- uint64_t length,
- InternalBlobData::Builder* target_blob_builder) {
- DCHECK_GT(length, 0ull);
-
- const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items();
- auto iter = items.begin();
- if (offset) {
- for (; iter != items.end(); ++iter) {
- const BlobDataItem& item = *(iter->get()->item());
- if (offset >= item.length())
- offset -= item.length();
- else
- break;
- }
+void BlobStorageContext::RequestTransport(
+ BlobEntry* entry,
+ std::vector<BlobMemoryController::FileCreationInfo> files) {
+ BlobEntry::BuildingState* building_state = entry->building_state_.get();
+ if (building_state->transport_allowed_callback) {
+ base::ResetAndReturn(&building_state->transport_allowed_callback)
+ .Run(BlobStatus::PENDING_TRANSPORT, std::move(files));
+ return;
}
+ DCHECK(files.empty());
+ NotifyTransportCompleteInternal(entry);
+}
- for (; iter != items.end() && length > 0; ++iter) {
- scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get();
- const BlobDataItem& item = *(shareable_item->item());
- uint64_t item_length = item.length();
- DCHECK_GT(item_length, offset);
- uint64_t current_length = item_length - offset;
- uint64_t new_length = current_length > length ? length : current_length;
+void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
+ bool success) {
+ if (!success) {
+ CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+ return;
+ }
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ if (!entry || !entry->building_state_.get())
+ return;
+ BlobEntry::BuildingState& building_state = *entry->building_state_;
+ DCHECK(!building_state.memory_quota_request);
+
+ if (building_state.transport_items_present) {
+ entry->set_status(BlobStatus::PENDING_TRANSPORT);
+ RequestTransport(entry,
+ std::vector<BlobMemoryController::FileCreationInfo>());
+ } else {
+ entry->set_status(BlobStatus::PENDING_INTERNALS);
+ }
- bool reusing_blob_item = offset == 0 && new_length == item.length();
- UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item);
- if (reusing_blob_item) {
- shareable_item->referencing_blobs().insert(target_blob_uuid);
- target_blob_builder->AppendSharedBlobItem(shareable_item);
- length -= new_length;
- continue;
- }
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
+}
- // We need to do copying of the items when we have a different offset or
- // length
- switch (item.type()) {
- case DataElement::TYPE_BYTES: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes",
- new_length / 1024);
- if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) {
- return false;
- }
- DCHECK(!item.offset());
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToBytes(item.bytes() + offset,
- static_cast<int64_t>(new_length));
- memory_usage_ += new_length;
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, new BlobDataItem(std::move(element))));
- } break;
- case DataElement::TYPE_FILE: {
- DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max())
- << "We cannot use a section of a file with an unknown length";
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
- new_length / 1024);
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToFilePathRange(item.path(), item.offset() + offset,
- new_length,
- item.expected_modification_time());
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid,
- new BlobDataItem(std::move(element), item.data_handle_)));
- } break;
- case DataElement::TYPE_FILE_FILESYSTEM: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
- new_length / 1024);
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToFileSystemUrlRange(item.filesystem_url(),
- item.offset() + offset, new_length,
- item.expected_modification_time());
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, new BlobDataItem(std::move(element))));
- } break;
- case DataElement::TYPE_DISK_CACHE_ENTRY: {
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToDiskCacheEntryRange(item.offset() + offset,
- new_length);
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid,
- new BlobDataItem(std::move(element), item.data_handle_,
- item.disk_cache_entry(),
- item.disk_cache_stream_index(),
- item.disk_cache_side_stream_index())));
- } break;
- case DataElement::TYPE_BYTES_DESCRIPTION:
- case DataElement::TYPE_BLOB:
- case DataElement::TYPE_UNKNOWN:
- CHECK(false) << "Illegal blob item type: " << item.type();
+void BlobStorageContext::OnDependentBlobFinished(
+ const std::string& owning_blob_uuid,
+ BlobStatus status) {
+ BlobEntry* entry = registry_.GetEntry(owning_blob_uuid);
+ if (!entry || !entry->building_state_)
+ return;
+
+ if (BlobStatusIsError(status)) {
+ DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status)
+ << "Referenced blob should never be dereferenced while we "
+ << "are depending on it, as our system holds a handle.";
+ CancelBuildingBlobInternal(entry, BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+ return;
+ }
+ DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u);
+ --entry->building_state_->num_building_dependent_blobs;
+
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
+}
+
+void BlobStorageContext::ClearAndFreeMemory(BlobEntry* entry) {
+ if (entry->building_state_) {
+ BlobEntry::BuildingState* building_state = entry->building_state_.get();
+ if (building_state->memory_quota_request) {
+ building_state->memory_quota_request->Cancel();
}
- length -= new_length;
- offset = 0;
}
- return true;
+ entry->ClearItems();
+ entry->ClearOffsets();
+ entry->set_size(0);
}
} // namespace storage
diff --git a/chromium/storage/browser/blob/blob_storage_context.h b/chromium/storage/browser/blob/blob_storage_context.h
index 8553d0266d2..c64581e91c7 100644
--- a/chromium/storage/browser/blob/blob_storage_context.h
+++ b/chromium/storage/browser/blob/blob_storage_context.h
@@ -19,17 +19,18 @@
#include "base/memory/ref_counted.h"
#include "base/memory/weak_ptr.h"
#include "storage/browser/blob/blob_data_handle.h"
+#include "storage/browser/blob/blob_entry.h"
+#include "storage/browser/blob/blob_memory_controller.h"
#include "storage/browser/blob/blob_storage_registry.h"
-#include "storage/browser/blob/internal_blob_data.h"
#include "storage/browser/storage_browser_export.h"
#include "storage/common/blob_storage/blob_storage_constants.h"
-#include "storage/common/data_element.h"
class GURL;
namespace base {
class FilePath;
class Time;
+class TaskRunner;
}
namespace content {
@@ -38,130 +39,212 @@ class BlobDispatcherHostTest;
}
namespace storage {
-
class BlobDataBuilder;
+class BlobDataHandle;
class BlobDataItem;
class BlobDataSnapshot;
class ShareableBlobDataItem;
-// This class handles the logistics of blob Storage within the browser process,
-// and maintains a mapping from blob uuid to the data. The class is single
-// threaded and should only be used on the IO thread.
-// In chromium, there is one instance per profile.
-class STORAGE_EXPORT BlobStorageContext
- : public base::SupportsWeakPtr<BlobStorageContext> {
+// This class handles the logistics of blob storage within the browser process.
+// We are single threaded and should only be used on the IO thread. In Chromium
+// there is one instance per profile.
+class STORAGE_EXPORT BlobStorageContext {
public:
+ using TransportAllowedCallback = BlobEntry::TransportAllowedCallback;
+
+ // Initializes the context without disk support.
BlobStorageContext();
~BlobStorageContext();
std::unique_ptr<BlobDataHandle> GetBlobDataFromUUID(const std::string& uuid);
std::unique_ptr<BlobDataHandle> GetBlobDataFromPublicURL(const GURL& url);
- // Useful for coining blobs from within the browser process. If the
- // blob cannot be added due to memory consumption, returns NULL.
- // A builder should not be used twice to create blobs, as the internal
- // resources are refcounted instead of copied for memory efficiency.
- // To cleanly use a builder multiple times, please call Clone() on the
- // builder, or even better for memory savings, clear the builder and append
- // the previously constructed blob.
+ // Always returns a handle to a blob. Use BlobStatus::GetBlobStatus() and
+ // BlobStatus::RunOnConstructionComplete(callback) to determine construction
+ // completion and possible errors.
std::unique_ptr<BlobDataHandle> AddFinishedBlob(
const BlobDataBuilder& builder);
- // Deprecated, use const ref version above.
+ // Deprecated, use const ref version above or BuildBlob below.
std::unique_ptr<BlobDataHandle> AddFinishedBlob(
const BlobDataBuilder* builder);
+ std::unique_ptr<BlobDataHandle> AddBrokenBlob(
+ const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition,
+ BlobStatus reason);
+
// Useful for coining blob urls from within the browser process.
bool RegisterPublicBlobURL(const GURL& url, const std::string& uuid);
void RevokePublicBlobURL(const GURL& url);
- size_t memory_usage() const { return memory_usage_; }
size_t blob_count() const { return registry_.blob_count(); }
- size_t memory_available() const {
- return kBlobStorageMaxMemoryUsage - memory_usage_;
- }
const BlobStorageRegistry& registry() { return registry_; }
- private:
- using BlobRegistryEntry = BlobStorageRegistry::Entry;
- using BlobConstructedCallback = BlobStorageRegistry::BlobConstructedCallback;
+ // This builds a blob with the given |input_builder| and returns a handle to
+ // the constructed Blob. Blob metadata and data should be accessed through
+ // this handle.
+ // If there is data present that needs further population then we will call
+ // |transport_allowed_callback| when we're ready for the user data to be
+ // populated with the PENDING_DATA_POPULATION status. This can happen
+ // synchronously or asynchronously. Otherwise |transport_allowed_callback|
+ // should be null. In the further population case, the caller must call either
+ // NotifyTransportComplete or CancelBuildingBlob after
+ // |transport_allowed_callback| is called to signify the data is finished
+ // populating or an error occurred (respectively).
+ // If the returned handle is broken, then the possible error cases are:
+ // * OUT_OF_MEMORY if we don't have enough memory to store the blob,
+ // * REFERENCED_BLOB_BROKEN if a referenced blob is broken or we're
+ // referencing ourself.
+ std::unique_ptr<BlobDataHandle> BuildBlob(
+ const BlobDataBuilder& input_builder,
+ const TransportAllowedCallback& transport_allowed_callback);
+
+ // This breaks a blob that is currently being built by using the BuildBlob
+ // method above. Any callbacks waiting on this blob, including the
+ // |transport_allowed_callback| callback given to BuildBlob, will be called
+ // with this status code.
+ void CancelBuildingBlob(const std::string& uuid, BlobStatus code);
+
+ // After calling BuildBlob above, the caller should call this method to
+ // notify the construction system that the unpopulated data in the given blob
+ // has been. populated. Caller must have all pending items populated in the
+ // original builder |input_builder| given in BuildBlob or we'll check-fail.
+ // If there is no pending data in the |input_builder| for the BuildBlob call,
+ // then this method doesn't need to be called.
+ void NotifyTransportComplete(const std::string& uuid);
+
+ const BlobMemoryController& memory_controller() { return memory_controller_; }
+
+ base::WeakPtr<BlobStorageContext> AsWeakPtr() {
+ return ptr_factory_.GetWeakPtr();
+ }
+
+ protected:
friend class content::BlobDispatcherHost;
- friend class BlobAsyncBuilderHost;
- friend class BlobAsyncBuilderHostTest;
+ friend class content::BlobDispatcherHostTest;
+ friend class BlobTransportHost;
+ friend class BlobTransportHostTest;
friend class BlobDataHandle;
friend class BlobDataHandle::BlobDataHandleShared;
- friend class BlobReaderTest;
- FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, HandleBeforeAsyncCancel);
- FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, ReadFromIncompleteBlob);
+ friend class BlobFlattenerTest;
+ friend class BlobSliceTest;
friend class BlobStorageContextTest;
- FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, IncrementDecrementRef);
- FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, OnCancelBuildingBlob);
- FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, PublicBlobUrls);
- FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest,
- TestUnknownBrokenAndBuildingBlobReference);
- friend class ViewBlobInternalsJob;
-
- // CompletePendingBlob or CancelPendingBlob should be called after this.
- void CreatePendingBlob(const std::string& uuid,
- const std::string& content_type,
- const std::string& content_disposition);
-
- // This includes resolving blob references in the builder. This will run the
- // callbacks given in RunOnConstructionComplete.
- void CompletePendingBlob(const BlobDataBuilder& external_builder);
-
- // This will run the callbacks given in RunOnConstructionComplete.
- void CancelPendingBlob(const std::string& uuid,
- IPCBlobCreationCancelCode reason);
+
+ // Transforms a BlobDataBuilder into a BlobEntry with no blob references.
+ // BlobSlice is used to flatten out these references. Records the total size
+ // and items for memory and file quota requests.
+ // Exposed in the header file for testing.
+ struct STORAGE_EXPORT BlobFlattener {
+ BlobFlattener(const BlobDataBuilder& input_builder,
+ BlobEntry* output_blob,
+ BlobStorageRegistry* registry);
+ ~BlobFlattener();
+
+ // This can be:
+ // * PENDING_QUOTA if we need memory quota, if we're populated and don't
+ // need quota.
+ // * PENDING_INTERNALS if we're waiting on dependent blobs or we're done.
+ // * INVALID_CONSTRUCTION_ARGUMENTS if we have invalid input.
+ // * REFERENCED_BLOB_BROKEN if one of the referenced blobs is broken or we
+ // reference ourself.
+ BlobStatus status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+
+ // This is the total size of the blob, including all memory, files, etc.
+ uint64_t total_size = 0;
+ // Total memory size of the blob (not including files, etc).
+ uint64_t total_memory_size = 0;
+
+ std::vector<std::pair<std::string, BlobEntry*>> dependent_blobs;
+
+ uint64_t memory_quota_needed = 0;
+ std::vector<scoped_refptr<ShareableBlobDataItem>> pending_memory_items;
+
+ std::vector<ShareableBlobDataItem*> transport_items;
+
+ // These record all future copies we'll need to do from referenced blobs.
+ // This
+ // happens when we do a partial slice from a pending data or file item.
+ std::vector<BlobEntry::ItemCopyEntry> copies;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(BlobFlattener);
+ };
+
+ // Used when a blob reference has a size and offset. Records the source items
+ // and memory we need to copy if either side of slice intersects an item.
+ // Exposed in the header file for testing.
+ struct STORAGE_EXPORT BlobSlice {
+ BlobSlice(const BlobEntry& source,
+ uint64_t slice_offset,
+ uint64_t slice_size);
+ ~BlobSlice();
+
+ // Size of memory copying from the source blob.
+ base::CheckedNumeric<size_t> copying_memory_size = 0;
+ // Size of all memory for UMA stats.
+ base::CheckedNumeric<size_t> total_memory_size = 0;
+
+ size_t first_item_slice_offset = 0;
+ // Populated if our first slice item is a temporary item that we'll copy to
+ // later from this |first_source_item|, at offset |first_item_slice_offset|.
+ scoped_refptr<ShareableBlobDataItem> first_source_item;
+ // Populated if our last slice item is a temporary item that we'll copy to
+ // later from this |last_source_item|.
+ scoped_refptr<ShareableBlobDataItem> last_source_item;
+
+ std::vector<scoped_refptr<ShareableBlobDataItem>> dest_items;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(BlobSlice);
+ };
void IncrementBlobRefCount(const std::string& uuid);
void DecrementBlobRefCount(const std::string& uuid);
- // Methods called by BlobDataHandle:
// This will return an empty snapshot until the blob is complete.
// TODO(dmurph): After we make the snapshot method in BlobHandle private, then
// make this DCHECK on the blob not being complete.
std::unique_ptr<BlobDataSnapshot> CreateSnapshot(const std::string& uuid);
- bool IsBroken(const std::string& uuid) const;
- bool IsBeingBuilt(const std::string& uuid) const;
- // Runs |done| when construction completes, with true if it was successful,
- // and false if there was an error, which is reported in the second argument
- // of the callback.
+
+ BlobStatus GetBlobStatus(const std::string& uuid) const;
+
+ // Runs |done| when construction completes with the final status of the blob.
void RunOnConstructionComplete(const std::string& uuid,
- const BlobConstructedCallback& done);
-
- // Appends the given blob item to the blob builder. The new blob
- // retains ownership of data_item if applicable, and returns false if there
- // was an error and pouplates the error_code. We can either have an error of:
- // OUT_OF_MEMORY: We are out of memory to store this blob.
- // REFERENCED_BLOB_BROKEN: One of the referenced blobs is broken.
- bool AppendAllocatedBlobItem(const std::string& target_blob_uuid,
- scoped_refptr<BlobDataItem> data_item,
- InternalBlobData::Builder* target_blob_data,
- IPCBlobCreationCancelCode* error_code);
-
- // Allocates a shareable blob data item, with blob references resolved. If
- // there isn't enough memory, then a nullptr is returned.
- scoped_refptr<ShareableBlobDataItem> AllocateShareableBlobDataItem(
- const std::string& target_blob_uuid,
- scoped_refptr<BlobDataItem> data_item);
-
- // Deconstructs the blob and appends it's contents to the target blob. Items
- // are shared if possible, and copied if the given offset and length
- // have to split an item.
- bool AppendBlob(const std::string& target_blob_uuid,
- const InternalBlobData& blob,
- uint64_t offset,
- uint64_t length,
- InternalBlobData::Builder* target_blob_data);
+ const BlobStatusCallback& done_callback);
- BlobStorageRegistry registry_;
+ BlobStorageRegistry* mutable_registry() { return &registry_; }
- // Used to keep track of how much memory is being utilized for blob data,
- // we count only the items of TYPE_DATA which are held in memory and not
- // items of TYPE_FILE.
- size_t memory_usage_;
+ BlobMemoryController* mutable_memory_controller() {
+ return &memory_controller_;
+ }
+
+ private:
+ std::unique_ptr<BlobDataHandle> CreateHandle(const std::string& uuid,
+ BlobEntry* entry);
+
+ void NotifyTransportCompleteInternal(BlobEntry* entry);
+
+ void CancelBuildingBlobInternal(BlobEntry* entry, BlobStatus reason);
+
+ void FinishBuilding(BlobEntry* entry);
+
+ void RequestTransport(
+ BlobEntry* entry,
+ std::vector<BlobMemoryController::FileCreationInfo> files);
+
+ void OnEnoughSizeForMemory(const std::string& uuid, bool can_fit);
+
+ void OnDependentBlobFinished(const std::string& owning_blob_uuid,
+ BlobStatus reason);
+
+ void ClearAndFreeMemory(BlobEntry* entry);
+
+ BlobStorageRegistry registry_;
+ BlobMemoryController memory_controller_;
+ base::WeakPtrFactory<BlobStorageContext> ptr_factory_;
DISALLOW_COPY_AND_ASSIGN(BlobStorageContext);
};
diff --git a/chromium/storage/browser/blob/blob_storage_registry.cc b/chromium/storage/browser/blob/blob_storage_registry.cc
index 9ddcb99b555..47b4bd8d602 100644
--- a/chromium/storage/browser/blob/blob_storage_registry.cc
+++ b/chromium/storage/browser/blob/blob_storage_registry.cc
@@ -9,15 +9,13 @@
#include <memory>
#include "base/bind.h"
-#include "base/callback.h"
#include "base/location.h"
#include "base/logging.h"
-#include "base/message_loop/message_loop.h"
#include "base/stl_util.h"
+#include "storage/browser/blob/blob_entry.h"
#include "url/gurl.h"
namespace storage {
-using BlobState = BlobStorageRegistry::BlobState;
namespace {
// We can't use GURL directly for these hash fragment manipulations
@@ -38,18 +36,6 @@ GURL ClearBlobUrlRef(const GURL& url) {
} // namespace
-BlobStorageRegistry::Entry::Entry(int refcount, BlobState state)
- : refcount(refcount), state(state) {}
-
-BlobStorageRegistry::Entry::~Entry() {}
-
-bool BlobStorageRegistry::Entry::TestAndSetState(BlobState expected,
- BlobState set) {
- if (state != expected)
- return false;
- state = set;
- return true;
-}
BlobStorageRegistry::BlobStorageRegistry() {}
@@ -59,15 +45,14 @@ BlobStorageRegistry::~BlobStorageRegistry() {
// So it shouldn't matter.
}
-BlobStorageRegistry::Entry* BlobStorageRegistry::CreateEntry(
+BlobEntry* BlobStorageRegistry::CreateEntry(
const std::string& uuid,
const std::string& content_type,
const std::string& content_disposition) {
- DCHECK(!base::ContainsKey(blob_map_, uuid));
- std::unique_ptr<Entry> entry(new Entry(1, BlobState::PENDING));
- entry->content_type = content_type;
- entry->content_disposition = content_disposition;
- Entry* entry_ptr = entry.get();
+ DCHECK(!ContainsKey(blob_map_, uuid));
+ std::unique_ptr<BlobEntry> entry(
+ new BlobEntry(content_type, content_disposition));
+ BlobEntry* entry_ptr = entry.get();
blob_map_.add(uuid, std::move(entry));
return entry_ptr;
}
@@ -80,16 +65,14 @@ bool BlobStorageRegistry::HasEntry(const std::string& uuid) const {
return blob_map_.find(uuid) != blob_map_.end();
}
-BlobStorageRegistry::Entry* BlobStorageRegistry::GetEntry(
- const std::string& uuid) {
+BlobEntry* BlobStorageRegistry::GetEntry(const std::string& uuid) {
BlobMap::iterator found = blob_map_.find(uuid);
if (found == blob_map_.end())
return nullptr;
return found->second;
}
-const BlobStorageRegistry::Entry* BlobStorageRegistry::GetEntry(
- const std::string& uuid) const {
+const BlobEntry* BlobStorageRegistry::GetEntry(const std::string& uuid) const {
return const_cast<BlobStorageRegistry*>(this)->GetEntry(uuid);
}
@@ -118,14 +101,13 @@ bool BlobStorageRegistry::IsURLMapped(const GURL& blob_url) const {
return base::ContainsKey(url_to_uuid_, blob_url);
}
-BlobStorageRegistry::Entry* BlobStorageRegistry::GetEntryFromURL(
- const GURL& url,
- std::string* uuid) {
+BlobEntry* BlobStorageRegistry::GetEntryFromURL(const GURL& url,
+ std::string* uuid) {
URLMap::iterator found =
url_to_uuid_.find(BlobUrlHasRef(url) ? ClearBlobUrlRef(url) : url);
if (found == url_to_uuid_.end())
return nullptr;
- Entry* entry = GetEntry(found->second);
+ BlobEntry* entry = GetEntry(found->second);
if (entry && uuid)
uuid->assign(found->second);
return entry;
diff --git a/chromium/storage/browser/blob/blob_storage_registry.h b/chromium/storage/browser/blob/blob_storage_registry.h
index 5211094b61a..2b2dbdd13f2 100644
--- a/chromium/storage/browser/blob/blob_storage_registry.h
+++ b/chromium/storage/browser/blob/blob_storage_registry.h
@@ -15,73 +15,31 @@
#include "base/callback_forward.h"
#include "base/containers/scoped_ptr_hash_map.h"
#include "base/macros.h"
-#include "storage/browser/blob/internal_blob_data.h"
#include "storage/browser/storage_browser_export.h"
#include "storage/common/blob_storage/blob_storage_constants.h"
class GURL;
namespace storage {
+class BlobDataHandle;
+class BlobEntry;
+class ShareableBlobDataItem;
// This class stores the blob data in the various states of construction, as
// well as URL mappings to blob uuids.
// Implementation notes:
-// * There is no implicit refcounting in this class, except for setting the
-// refcount to 1 on registration.
// * When removing a uuid registration, we do not check for URL mappings to that
// uuid. The user must keep track of these.
class STORAGE_EXPORT BlobStorageRegistry {
public:
- // True means the blob was constructed successfully, and false means that
- // there was an error, which is reported in the second argument.
- using BlobConstructedCallback =
- base::Callback<void(bool, IPCBlobCreationCancelCode)>;
-
- enum class BlobState {
- // The blob is pending transportation from the renderer. This is the default
- // state on entry construction.
- PENDING,
- // The blob is complete and can be read from.
- COMPLETE,
- // The blob is broken and no longer holds data. This happens when there was
- // a problem constructing the blob, or we've run out of memory.
- BROKEN
- };
-
- struct STORAGE_EXPORT Entry {
- size_t refcount;
- BlobState state;
- std::vector<BlobConstructedCallback> build_completion_callbacks;
-
- // Only applicable if the state == BROKEN.
- IPCBlobCreationCancelCode broken_reason =
- IPCBlobCreationCancelCode::UNKNOWN;
-
- // data and data_builder are mutually exclusive.
- std::unique_ptr<InternalBlobData> data;
- std::unique_ptr<InternalBlobData::Builder> data_builder;
-
- std::string content_type;
- std::string content_disposition;
-
- Entry() = delete;
- Entry(int refcount, BlobState state);
- ~Entry();
-
- // Performs a test-and-set on the state of the given blob. If the state
- // isn't as expected, we return false. Otherwise we set the new state and
- // return true.
- bool TestAndSetState(BlobState expected, BlobState set);
- };
-
BlobStorageRegistry();
~BlobStorageRegistry();
// Creates the blob entry with a refcount of 1 and a state of PENDING. If
// the blob is already in use, we return null.
- Entry* CreateEntry(const std::string& uuid,
- const std::string& content_type,
- const std::string& content_disposition);
+ BlobEntry* CreateEntry(const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition);
// Removes the blob entry with the given uuid. This does not unmap any
// URLs that are pointing to this uuid. Returns if the entry existed.
@@ -91,8 +49,8 @@ class STORAGE_EXPORT BlobStorageRegistry {
// Gets the blob entry for the given uuid. Returns nullptr if the entry
// does not exist.
- Entry* GetEntry(const std::string& uuid);
- const Entry* GetEntry(const std::string& uuid) const;
+ BlobEntry* GetEntry(const std::string& uuid);
+ const BlobEntry* GetEntry(const std::string& uuid) const;
// Creates a url mapping from blob uuid to the given url. Returns false if
// the uuid isn't mapped to an entry or if there already is a map for the URL.
@@ -107,14 +65,15 @@ class STORAGE_EXPORT BlobStorageRegistry {
// Returns the entry from the given url, and optionally populates the uuid for
// that entry. Returns a nullptr if the mapping or entry doesn't exist.
- Entry* GetEntryFromURL(const GURL& url, std::string* uuid);
+ BlobEntry* GetEntryFromURL(const GURL& url, std::string* uuid);
size_t blob_count() const { return blob_map_.size(); }
size_t url_count() const { return url_to_uuid_.size(); }
private:
friend class ViewBlobInternalsJob;
- using BlobMap = base::ScopedPtrHashMap<std::string, std::unique_ptr<Entry>>;
+ using BlobMap =
+ base::ScopedPtrHashMap<std::string, std::unique_ptr<BlobEntry>>;
using URLMap = std::map<GURL, std::string>;
BlobMap blob_map_;
diff --git a/chromium/storage/browser/blob/blob_transport_host.cc b/chromium/storage/browser/blob/blob_transport_host.cc
new file mode 100644
index 00000000000..032b9ba45ed
--- /dev/null
+++ b/chromium/storage/browser/blob/blob_transport_host.cc
@@ -0,0 +1,506 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <memory>
+#include <utility>
+
+#include "base/bind.h"
+#include "base/callback_helpers.h"
+#include "base/memory/ptr_util.h"
+#include "base/memory/shared_memory.h"
+#include "base/stl_util.h"
+#include "storage/browser/blob/blob_data_handle.h"
+#include "storage/browser/blob/blob_memory_controller.h"
+#include "storage/browser/blob/blob_storage_context.h"
+#include "storage/browser/blob/blob_transport_host.h"
+
+namespace storage {
+namespace {
+using MemoryStrategy = BlobMemoryController::Strategy;
+using MemoryItemRequest =
+ BlobTransportRequestBuilder::RendererMemoryItemRequest;
+
+bool CalculateBlobMemorySize(const std::vector<DataElement>& elements,
+ size_t* shortcut_bytes,
+ uint64_t* total_bytes) {
+ DCHECK(shortcut_bytes);
+ DCHECK(total_bytes);
+
+ base::CheckedNumeric<uint64_t> total_size_checked = 0;
+ base::CheckedNumeric<size_t> shortcut_size_checked = 0;
+ for (const auto& e : elements) {
+ if (e.type() == DataElement::TYPE_BYTES) {
+ total_size_checked += e.length();
+ shortcut_size_checked += e.length();
+ } else if (e.type() == DataElement::TYPE_BYTES_DESCRIPTION) {
+ total_size_checked += e.length();
+ } else {
+ continue;
+ }
+ if (!total_size_checked.IsValid() || !shortcut_size_checked.IsValid())
+ return false;
+ }
+ *shortcut_bytes = shortcut_size_checked.ValueOrDie();
+ *total_bytes = total_size_checked.ValueOrDie();
+ return true;
+}
+} // namespace
+
+BlobTransportHost::TransportState::TransportState(
+ const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition,
+ RequestMemoryCallback request_memory_callback,
+ BlobStatusCallback completion_callback)
+ : data_builder(uuid),
+ request_memory_callback(std::move(request_memory_callback)),
+ completion_callback(std::move(completion_callback)) {
+ data_builder.set_content_type(content_type);
+ data_builder.set_content_disposition(content_disposition);
+}
+
+BlobTransportHost::TransportState::TransportState(
+ BlobTransportHost::TransportState&&) = default;
+BlobTransportHost::TransportState& BlobTransportHost::TransportState::operator=(
+ BlobTransportHost::TransportState&&) = default;
+
+BlobTransportHost::TransportState::~TransportState() {}
+
+BlobTransportHost::BlobTransportHost() : ptr_factory_(this) {}
+
+BlobTransportHost::~BlobTransportHost() {}
+
+std::unique_ptr<BlobDataHandle> BlobTransportHost::StartBuildingBlob(
+ const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition,
+ const std::vector<DataElement>& elements,
+ BlobStorageContext* context,
+ const RequestMemoryCallback& request_memory,
+ const BlobStatusCallback& completion_callback) {
+ DCHECK(context);
+ DCHECK(async_blob_map_.find(uuid) == async_blob_map_.end());
+ std::unique_ptr<BlobDataHandle> handle;
+ // Validate that our referenced blobs aren't us.
+ for (const DataElement& e : elements) {
+ if (e.type() == DataElement::TYPE_BLOB && e.blob_uuid() == uuid) {
+ handle = context->AddBrokenBlob(
+ uuid, content_type, content_disposition,
+ BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+ completion_callback.Run(BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+ return handle;
+ }
+ }
+ uint64_t transport_memory_size = 0;
+ size_t shortcut_size = 0;
+ if (!CalculateBlobMemorySize(elements, &shortcut_size,
+ &transport_memory_size)) {
+ handle =
+ context->AddBrokenBlob(uuid, content_type, content_disposition,
+ BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+ completion_callback.Run(BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+ return handle;
+ }
+
+ const BlobMemoryController& memory_controller = context->memory_controller();
+ MemoryStrategy memory_strategy =
+ memory_controller.DetermineStrategy(shortcut_size, transport_memory_size);
+ TransportState state(uuid, content_type, content_disposition, request_memory,
+ completion_callback);
+ switch (memory_strategy) {
+ case MemoryStrategy::TOO_LARGE: {
+ handle = context->AddBrokenBlob(uuid, content_type, content_disposition,
+ BlobStatus::ERR_OUT_OF_MEMORY);
+ completion_callback.Run(BlobStatus::ERR_OUT_OF_MEMORY);
+ return handle;
+ }
+ case MemoryStrategy::NONE_NEEDED: {
+ // This means we don't need to transport anything, so just send the
+ // elements along and tell our callback we're done transporting.
+ for (const DataElement& e : elements) {
+ DCHECK_NE(e.type(), DataElement::TYPE_BYTES_DESCRIPTION);
+ state.data_builder.AppendIPCDataElement(e);
+ }
+ handle = context->BuildBlob(
+ state.data_builder, BlobStorageContext::TransportAllowedCallback());
+ completion_callback.Run(BlobStatus::DONE);
+ return handle;
+ }
+ case MemoryStrategy::IPC:
+ state.strategy = IPCBlobItemRequestStrategy::IPC;
+ state.request_builder.InitializeForIPCRequests(
+ memory_controller.limits().max_ipc_memory_size, transport_memory_size,
+ elements, &(state.data_builder));
+ break;
+ case MemoryStrategy::SHARED_MEMORY:
+ state.strategy = IPCBlobItemRequestStrategy::SHARED_MEMORY;
+ state.request_builder.InitializeForSharedMemoryRequests(
+ memory_controller.limits().max_shared_memory_size,
+ transport_memory_size, elements, &(state.data_builder));
+ break;
+ case MemoryStrategy::FILE:
+ state.strategy = IPCBlobItemRequestStrategy::FILE;
+ state.request_builder.InitializeForFileRequests(
+ memory_controller.limits().max_file_size, transport_memory_size,
+ elements, &(state.data_builder));
+ break;
+ }
+ // We initialize our requests received state now that they are populated.
+ state.request_received.resize(state.request_builder.requests().size(), false);
+ auto it_pair = async_blob_map_.insert(std::make_pair(uuid, std::move(state)));
+
+ // OnReadyForTransport can be called synchronously.
+ handle = context->BuildBlob(
+ it_pair.first->second.data_builder,
+ base::Bind(&BlobTransportHost::OnReadyForTransport,
+ ptr_factory_.GetWeakPtr(), uuid, context->AsWeakPtr()));
+ return handle;
+}
+
+void BlobTransportHost::OnMemoryResponses(
+ const std::string& uuid,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context) {
+ AsyncBlobMap::iterator state_it = async_blob_map_.find(uuid);
+ DCHECK(state_it != async_blob_map_.end()) << "Could not find blob " << uuid;
+ if (responses.empty()) {
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+
+ // Validate response sanity: it should refer to a legal request number, and
+ // we shouldn't have received an answer for that request yet.
+ TransportState& state = state_it->second;
+ const auto& requests = state.request_builder.requests();
+ for (const BlobItemBytesResponse& response : responses) {
+ if (response.request_number >= requests.size()) {
+ // Bad IPC, so we delete our record and ignore.
+ DVLOG(1) << "Invalid request number " << response.request_number;
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+ DCHECK_LT(response.request_number, state.request_received.size());
+ if (state.request_received[response.request_number]) {
+ // Bad IPC, so we delete our record.
+ DVLOG(1) << "Already received response for that request.";
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+ state.request_received[response.request_number] = true;
+ }
+ switch (state.strategy) {
+ case IPCBlobItemRequestStrategy::IPC:
+ OnIPCResponses(uuid, &state, responses, context);
+ break;
+ case IPCBlobItemRequestStrategy::SHARED_MEMORY:
+ OnSharedMemoryResponses(uuid, &state, responses, context);
+ break;
+ case IPCBlobItemRequestStrategy::FILE:
+ OnFileResponses(uuid, &state, responses, context);
+ break;
+ case IPCBlobItemRequestStrategy::UNKNOWN:
+ NOTREACHED();
+ break;
+ }
+}
+
+void BlobTransportHost::CancelBuildingBlob(const std::string& uuid,
+ BlobStatus code,
+ BlobStorageContext* context) {
+ DCHECK(context);
+ DCHECK(BlobStatusIsError(code));
+ AsyncBlobMap::iterator state_it = async_blob_map_.find(uuid);
+ if (state_it == async_blob_map_.end())
+ return;
+ // We can have the blob dereferenced by the renderer, but have it still being
+ // 'built'. In this case, it's destructed in the context, but we still have
+ // it in our map. Hence we make sure the context has the entry before
+ // calling cancel.
+ BlobStatusCallback completion_callback = state_it->second.completion_callback;
+ async_blob_map_.erase(state_it);
+ if (context->registry().HasEntry(uuid))
+ context->CancelBuildingBlob(uuid, code);
+
+ completion_callback.Run(code);
+}
+
+void BlobTransportHost::CancelAll(BlobStorageContext* context) {
+ DCHECK(context);
+ // We cancel all of our blobs just in case another renderer referenced them.
+ std::vector<std::string> pending_blobs;
+ for (const auto& uuid_state_pair : async_blob_map_) {
+ pending_blobs.push_back(uuid_state_pair.second.data_builder.uuid());
+ }
+ // We clear the map before canceling them to prevent any strange reentry into
+ // our class (see OnReadyForTransport) if any blobs were waiting for others
+ // to construct.
+ async_blob_map_.clear();
+ for (auto& uuid : pending_blobs) {
+ if (context->registry().HasEntry(uuid))
+ context->CancelBuildingBlob(uuid, BlobStatus::ERR_SOURCE_DIED_IN_TRANSIT);
+ }
+}
+
+void BlobTransportHost::StartRequests(
+ const std::string& uuid,
+ TransportState* state,
+ BlobStorageContext* context,
+ std::vector<BlobMemoryController::FileCreationInfo> file_infos) {
+ switch (state->strategy) {
+ case IPCBlobItemRequestStrategy::IPC:
+ DCHECK(file_infos.empty());
+ SendIPCRequests(state, context);
+ break;
+ case IPCBlobItemRequestStrategy::SHARED_MEMORY:
+ DCHECK(file_infos.empty());
+ ContinueSharedMemoryRequests(uuid, state, context);
+ break;
+ case IPCBlobItemRequestStrategy::FILE:
+ DCHECK(!file_infos.empty());
+ SendFileRequests(state, context, std::move(file_infos));
+ break;
+ case IPCBlobItemRequestStrategy::UNKNOWN:
+ NOTREACHED();
+ break;
+ }
+}
+
+// Note: This can be called when we cancel a blob in the context.
+void BlobTransportHost::OnReadyForTransport(
+ const std::string& uuid,
+ base::WeakPtr<BlobStorageContext> context,
+ BlobStatus status,
+ std::vector<BlobMemoryController::FileCreationInfo> file_infos) {
+ if (!context) {
+ async_blob_map_.erase(uuid);
+ return;
+ }
+ AsyncBlobMap::iterator state_it = async_blob_map_.find(uuid);
+ if (state_it == async_blob_map_.end())
+ return;
+
+ TransportState& state = state_it->second;
+ if (BlobStatusIsPending(status)) {
+ DCHECK(status == BlobStatus::PENDING_TRANSPORT);
+ StartRequests(uuid, &state, context.get(), std::move(file_infos));
+ return;
+ }
+ // Done or error.
+ BlobStatusCallback completion_callback = state.completion_callback;
+ async_blob_map_.erase(state_it);
+ completion_callback.Run(status);
+}
+
+void BlobTransportHost::SendIPCRequests(TransportState* state,
+ BlobStorageContext* context) {
+ const std::vector<MemoryItemRequest>& requests =
+ state->request_builder.requests();
+ std::vector<BlobItemBytesRequest> byte_requests;
+
+ DCHECK(!requests.empty());
+ for (const MemoryItemRequest& request : requests) {
+ byte_requests.push_back(request.message);
+ }
+
+ state->request_memory_callback.Run(std::move(byte_requests),
+ std::vector<base::SharedMemoryHandle>(),
+ std::vector<base::File>());
+}
+
+void BlobTransportHost::OnIPCResponses(
+ const std::string& uuid,
+ TransportState* state,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context) {
+ const auto& requests = state->request_builder.requests();
+ size_t num_requests = requests.size();
+ for (const BlobItemBytesResponse& response : responses) {
+ const MemoryItemRequest& request = requests[response.request_number];
+ if (response.inline_data.size() < request.message.size) {
+ DVLOG(1) << "Invalid data size " << response.inline_data.size()
+ << " vs requested size of " << request.message.size;
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+ bool success = state->data_builder.PopulateFutureData(
+ request.browser_item_index, response.inline_data.data(),
+ request.browser_item_offset, request.message.size);
+ if (!success) {
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+ state->num_fulfilled_requests++;
+ }
+ if (state->num_fulfilled_requests == num_requests)
+ CompleteTransport(state, context);
+}
+
+void BlobTransportHost::ContinueSharedMemoryRequests(
+ const std::string& uuid,
+ TransportState* state,
+ BlobStorageContext* context) {
+ BlobTransportRequestBuilder& request_builder = state->request_builder;
+ const std::vector<MemoryItemRequest>& requests = request_builder.requests();
+ size_t num_requests = requests.size();
+ DCHECK_LT(state->num_fulfilled_requests, num_requests);
+ if (state->next_request == num_requests) {
+ // We are still waiting on other requests to come back.
+ return;
+ }
+
+ std::vector<BlobItemBytesRequest> byte_requests;
+ std::vector<base::SharedMemoryHandle> shared_memory;
+
+ for (; state->next_request < num_requests; ++state->next_request) {
+ const MemoryItemRequest& request = requests[state->next_request];
+ bool using_shared_memory_handle = state->num_shared_memory_requests > 0;
+ if (using_shared_memory_handle &&
+ state->current_shared_memory_handle_index !=
+ request.message.handle_index) {
+ // We only want one shared memory per requesting blob.
+ break;
+ }
+ state->current_shared_memory_handle_index = request.message.handle_index;
+ state->num_shared_memory_requests++;
+
+ if (!state->shared_memory_block) {
+ state->shared_memory_block.reset(new base::SharedMemory());
+ size_t size =
+ request_builder.shared_memory_sizes()[request.message.handle_index];
+ if (!state->shared_memory_block->CreateAnonymous(size)) {
+ DVLOG(1) << "Unable to allocate shared memory for blob transfer.";
+ CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY, context);
+ return;
+ }
+ }
+ shared_memory.push_back(state->shared_memory_block->handle());
+ byte_requests.push_back(request.message);
+ // Since we are only using one handle at a time, transform our handle
+ // index correctly back to 0.
+ byte_requests.back().handle_index = 0;
+ }
+ DCHECK(!requests.empty());
+
+ state->request_memory_callback.Run(std::move(byte_requests),
+ std::move(shared_memory),
+ std::vector<base::File>());
+ return;
+}
+
+void BlobTransportHost::OnSharedMemoryResponses(
+ const std::string& uuid,
+ TransportState* state,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context) {
+ BlobTransportRequestBuilder& request_builder = state->request_builder;
+ const auto& requests = request_builder.requests();
+ for (const BlobItemBytesResponse& response : responses) {
+ const MemoryItemRequest& request = requests[response.request_number];
+ if (state->num_shared_memory_requests == 0) {
+ DVLOG(1) << "Received too many responses for shared memory.";
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+ state->num_shared_memory_requests--;
+ if (!state->shared_memory_block->memory()) {
+ // We just map the whole block, as we'll probably be accessing the
+ // whole thing in this group of responses.
+ size_t handle_size =
+ request_builder
+ .shared_memory_sizes()[state->current_shared_memory_handle_index];
+ if (!state->shared_memory_block->Map(handle_size)) {
+ DVLOG(1) << "Unable to map memory to size " << handle_size;
+ CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY, context);
+ return;
+ }
+ }
+
+ bool success = state->data_builder.PopulateFutureData(
+ request.browser_item_index,
+ static_cast<const char*>(state->shared_memory_block->memory()) +
+ request.message.handle_offset,
+ request.browser_item_offset, request.message.size);
+
+ if (!success) {
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+ state->num_fulfilled_requests++;
+ }
+ if (state->num_fulfilled_requests == requests.size()) {
+ CompleteTransport(state, context);
+ return;
+ }
+ ContinueSharedMemoryRequests(uuid, state, context);
+}
+
+void BlobTransportHost::SendFileRequests(
+ TransportState* state,
+ BlobStorageContext* context,
+ std::vector<BlobMemoryController::FileCreationInfo> file_infos) {
+ std::vector<base::File> files;
+
+ for (BlobMemoryController::FileCreationInfo& file_info : file_infos) {
+ state->files.push_back(std::move(file_info.file_reference));
+ files.push_back(std::move(file_info.file));
+ }
+
+ const std::vector<MemoryItemRequest>& requests =
+ state->request_builder.requests();
+ std::vector<BlobItemBytesRequest> byte_requests;
+
+ DCHECK(!requests.empty());
+ for (const MemoryItemRequest& request : requests) {
+ byte_requests.push_back(request.message);
+ }
+
+ state->request_memory_callback.Run(std::move(byte_requests),
+ std::vector<base::SharedMemoryHandle>(),
+ std::move(files));
+}
+
+void BlobTransportHost::OnFileResponses(
+ const std::string& uuid,
+ TransportState* state,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context) {
+ BlobTransportRequestBuilder& request_builder = state->request_builder;
+ const auto& requests = request_builder.requests();
+ for (const BlobItemBytesResponse& response : responses) {
+ const MemoryItemRequest& request = requests[response.request_number];
+ const scoped_refptr<ShareableFileReference>& file_ref =
+ state->files[request.message.handle_index];
+ bool success = state->data_builder.PopulateFutureFile(
+ request.browser_item_index, file_ref, response.time_file_modified);
+ if (!success) {
+ CancelBuildingBlob(uuid, BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS,
+ context);
+ return;
+ }
+ state->num_fulfilled_requests++;
+ }
+ if (state->num_fulfilled_requests == requests.size())
+ CompleteTransport(state, context);
+}
+
+void BlobTransportHost::CompleteTransport(TransportState* state,
+ BlobStorageContext* context) {
+ std::string uuid = state->data_builder.uuid();
+ BlobStatusCallback complete_callback = state->completion_callback;
+ async_blob_map_.erase(uuid);
+ context->NotifyTransportComplete(uuid);
+ complete_callback.Run(BlobStatus::DONE);
+}
+
+} // namespace storage
diff --git a/chromium/storage/browser/blob/blob_transport_host.h b/chromium/storage/browser/blob/blob_transport_host.h
new file mode 100644
index 00000000000..f8e952a787a
--- /dev/null
+++ b/chromium/storage/browser/blob/blob_transport_host.h
@@ -0,0 +1,186 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_HOST_H_
+#define STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_HOST_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/files/file.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/shared_memory_handle.h"
+#include "base/memory/weak_ptr.h"
+#include "storage/browser/blob/blob_data_builder.h"
+#include "storage/browser/blob/blob_memory_controller.h"
+#include "storage/browser/blob/blob_transport_request_builder.h"
+#include "storage/browser/storage_browser_export.h"
+#include "storage/common/blob_storage/blob_item_bytes_request.h"
+#include "storage/common/blob_storage/blob_item_bytes_response.h"
+#include "storage/common/blob_storage/blob_storage_constants.h"
+#include "storage/common/data_element.h"
+
+namespace base {
+class SharedMemory;
+}
+
+namespace storage {
+class BlobDataHandle;
+class BlobStorageContext;
+
+// This class facilitates moving memory from the renderer to the browser.
+class STORAGE_EXPORT BlobTransportHost {
+ public:
+ // One is expected to use std::move when calling this callback.
+ using RequestMemoryCallback =
+ base::Callback<void(std::vector<storage::BlobItemBytesRequest>,
+ std::vector<base::SharedMemoryHandle>,
+ std::vector<base::File>)>;
+
+ BlobTransportHost();
+ ~BlobTransportHost();
+
+ // This registers the given blob internally and adds it to the storage with a
+ // refcount of 1. |completion_callback| is called synchronously or
+ // asynchronously with:
+ // * INVALID_CONSTRUCTION_ARGUMENTS if we have invalid input arguments/data.
+ // * REFERENCED_BLOB_BROKEN if one of the referenced blobs is broken or
+ // doesn't exist.
+ // * DONE if we don't need any more data transported and we can clean up.
+ // Returns a blob handle that is never null.
+ std::unique_ptr<BlobDataHandle> StartBuildingBlob(
+ const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition,
+ const std::vector<DataElement>& elements,
+ BlobStorageContext* context,
+ const RequestMemoryCallback& request_memory,
+ const BlobStatusCallback& completion_callback);
+
+ // This is called when we have responses from the Renderer to our calls to
+ // the request_memory callback above. The callbacks given in StartBuildingBlob
+ // will be used to request for more memory or signal completion.
+ // Note: The uuid must be being built in this host (IsBeingBuilt).
+ void OnMemoryResponses(const std::string& uuid,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context);
+
+ // This removes the TransportState from our map and flags the blob as broken
+ // in the context. This can be called both from our own logic to cancel the
+ // blob, or from the DispatcherHost (Renderer). The blob MUST be being built
+ // in this builder. This also calls the |completion_callback|.
+ // Note: if the blob isn't in the context (renderer dereferenced it before we
+ // finished constructing), then we don't bother touching the context.
+ void CancelBuildingBlob(const std::string& uuid,
+ BlobStatus code,
+ BlobStorageContext* context);
+
+ // This clears this object of pending construction and does NOT call transport
+ // complete callbacks.
+ void CancelAll(BlobStorageContext* context);
+
+ bool IsEmpty() const { return async_blob_map_.empty(); }
+
+ size_t blob_building_count() const { return async_blob_map_.size(); }
+
+ bool IsBeingBuilt(const std::string& key) const {
+ return async_blob_map_.find(key) != async_blob_map_.end();
+ }
+
+ private:
+ struct TransportState {
+ TransportState(const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition,
+ RequestMemoryCallback request_memory_callback,
+ BlobStatusCallback completion_callback);
+ TransportState(TransportState&&);
+ TransportState& operator=(TransportState&&);
+ DISALLOW_COPY_AND_ASSIGN(TransportState);
+ ~TransportState();
+
+ IPCBlobItemRequestStrategy strategy = IPCBlobItemRequestStrategy::UNKNOWN;
+ BlobTransportRequestBuilder request_builder;
+ BlobDataBuilder data_builder;
+ std::vector<bool> request_received;
+ size_t num_fulfilled_requests = 0;
+
+ RequestMemoryCallback request_memory_callback;
+ BlobStatusCallback completion_callback;
+
+ // Used by shared memory strategy.
+ size_t next_request = 0;
+ std::unique_ptr<base::SharedMemory> shared_memory_block;
+ // This is the number of requests that have been sent to populate the above
+ // shared data. We won't ask for more data in shared memory until all
+ // requests have been responded to.
+ size_t num_shared_memory_requests = 0;
+ // Only relevant if num_shared_memory_requests is > 0
+ size_t current_shared_memory_handle_index = 0;
+
+ // Used by file strategy.
+ std::vector<scoped_refptr<ShareableFileReference>> files;
+ };
+
+ typedef std::unordered_map<std::string, TransportState> AsyncBlobMap;
+
+ void StartRequests(
+ const std::string& uuid,
+ TransportState* state,
+ BlobStorageContext* context,
+ std::vector<BlobMemoryController::FileCreationInfo> file_infos);
+
+ void OnReadyForTransport(
+ const std::string& uuid,
+ base::WeakPtr<BlobStorageContext> context,
+ BlobStatus status,
+ std::vector<BlobMemoryController::FileCreationInfo> file_infos);
+
+ void SendIPCRequests(TransportState* state, BlobStorageContext* context);
+ void OnIPCResponses(const std::string& uuid,
+ TransportState* state,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context);
+
+ // This is the 'main loop' of our memory requests to the renderer.
+ void ContinueSharedMemoryRequests(const std::string& uuid,
+ TransportState* state,
+ BlobStorageContext* context);
+
+ void OnSharedMemoryResponses(
+ const std::string& uuid,
+ TransportState* state,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context);
+
+ void SendFileRequests(
+ TransportState* state,
+ BlobStorageContext* context,
+ std::vector<BlobMemoryController::FileCreationInfo> files);
+
+ void OnFileResponses(const std::string& uuid,
+ TransportState* state,
+ const std::vector<BlobItemBytesResponse>& responses,
+ BlobStorageContext* context);
+
+ // This finishes creating the blob in the context, decrements blob references
+ // that we were holding during construction, and erases our state.
+ void CompleteTransport(TransportState* state, BlobStorageContext* context);
+
+ AsyncBlobMap async_blob_map_;
+ base::WeakPtrFactory<BlobTransportHost> ptr_factory_;
+
+ DISALLOW_COPY_AND_ASSIGN(BlobTransportHost);
+};
+
+} // namespace storage
+#endif // STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_HOST_H_
diff --git a/chromium/storage/browser/blob/blob_async_transport_request_builder.cc b/chromium/storage/browser/blob/blob_transport_request_builder.cc
index a77d408ca8f..c9e3da0dfe7 100644
--- a/chromium/storage/browser/blob/blob_async_transport_request_builder.cc
+++ b/chromium/storage/browser/blob/blob_transport_request_builder.cc
@@ -8,7 +8,7 @@
#include <algorithm>
#include "base/numerics/safe_math.h"
-#include "storage/browser/blob/blob_async_transport_request_builder.h"
+#include "storage/browser/blob/blob_transport_request_builder.h"
#include "storage/common/blob_storage/blob_storage_constants.h"
namespace storage {
@@ -37,7 +37,7 @@ bool IsBytes(DataElement::Type type) {
class FileStorageStrategy {
public:
FileStorageStrategy(
- std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>*
+ std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>*
requests,
BlobDataBuilder* builder)
: requests(requests), builder(builder), current_item_index(0) {}
@@ -49,7 +49,7 @@ class FileStorageStrategy {
size_t segment_index,
uint64_t segment_offset,
uint64_t size) {
- BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest request;
+ BlobTransportRequestBuilder::RendererMemoryItemRequest request;
request.browser_item_index = current_item_index;
request.browser_item_offset = 0;
request.message.request_number = requests->size();
@@ -61,7 +61,7 @@ class FileStorageStrategy {
request.message.handle_offset = segment_offset;
requests->push_back(request);
- builder->AppendFutureFile(segment_offset, size);
+ builder->AppendFutureFile(segment_offset, size, segment_index);
current_item_index++;
}
@@ -72,8 +72,7 @@ class FileStorageStrategy {
void Done() {}
- std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>*
- requests;
+ std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>* requests;
BlobDataBuilder* builder;
size_t current_item_index;
@@ -85,7 +84,7 @@ class SharedMemoryStorageStrategy {
public:
SharedMemoryStorageStrategy(
size_t max_segment_size,
- std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>*
+ std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>*
requests,
BlobDataBuilder* builder)
: requests(requests),
@@ -105,7 +104,7 @@ class SharedMemoryStorageStrategy {
current_item_index++;
current_item_size = 0;
}
- BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest request;
+ BlobTransportRequestBuilder::RendererMemoryItemRequest request;
request.browser_item_index = current_item_index;
request.browser_item_offset = current_item_size;
request.message.request_number = requests->size();
@@ -137,8 +136,7 @@ class SharedMemoryStorageStrategy {
}
}
- std::vector<BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest>*
- requests;
+ std::vector<BlobTransportRequestBuilder::RendererMemoryItemRequest>* requests;
size_t max_segment_size;
BlobDataBuilder* builder;
@@ -193,17 +191,21 @@ void ForEachWithSegment(const std::vector<DataElement>& elements,
}
} // namespace
-BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest::
+BlobTransportRequestBuilder::RendererMemoryItemRequest::
RendererMemoryItemRequest()
: browser_item_index(0), browser_item_offset(0) {}
-BlobAsyncTransportRequestBuilder::BlobAsyncTransportRequestBuilder()
+BlobTransportRequestBuilder::BlobTransportRequestBuilder()
: total_bytes_size_(0) {}
-BlobAsyncTransportRequestBuilder::~BlobAsyncTransportRequestBuilder() {}
+BlobTransportRequestBuilder::BlobTransportRequestBuilder(
+ BlobTransportRequestBuilder&&) = default;
+BlobTransportRequestBuilder& BlobTransportRequestBuilder::operator=(
+ BlobTransportRequestBuilder&&) = default;
+BlobTransportRequestBuilder::~BlobTransportRequestBuilder() {}
// Initializes the transport strategy for file requests.
-void BlobAsyncTransportRequestBuilder::InitializeForFileRequests(
+void BlobTransportRequestBuilder::InitializeForFileRequests(
size_t max_file_size,
uint64_t blob_total_size,
const std::vector<DataElement>& elements,
@@ -215,7 +217,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForFileRequests(
ForEachWithSegment(elements, static_cast<uint64_t>(max_file_size), &strategy);
}
-void BlobAsyncTransportRequestBuilder::InitializeForSharedMemoryRequests(
+void BlobTransportRequestBuilder::InitializeForSharedMemoryRequests(
size_t max_shared_memory_size,
uint64_t blob_total_size,
const std::vector<DataElement>& elements,
@@ -231,7 +233,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForSharedMemoryRequests(
&strategy);
}
-void BlobAsyncTransportRequestBuilder::InitializeForIPCRequests(
+void BlobTransportRequestBuilder::InitializeForIPCRequests(
size_t max_ipc_memory_size,
uint64_t blob_total_size,
const std::vector<DataElement>& elements,
@@ -247,7 +249,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForIPCRequests(
builder->AppendIPCDataElement(info);
continue;
}
- BlobAsyncTransportRequestBuilder::RendererMemoryItemRequest request;
+ BlobTransportRequestBuilder::RendererMemoryItemRequest request;
request.browser_item_index = i;
request.browser_item_offset = 0;
request.message.request_number = requests_.size();
@@ -261,27 +263,7 @@ void BlobAsyncTransportRequestBuilder::InitializeForIPCRequests(
}
/* static */
-bool BlobAsyncTransportRequestBuilder::ShouldBeShortcut(
- const std::vector<DataElement>& elements,
- size_t memory_available) {
- base::CheckedNumeric<size_t> shortcut_bytes = 0;
- for (const auto& element : elements) {
- DataElement::Type type = element.type();
- if (type == DataElement::TYPE_BYTES_DESCRIPTION) {
- return false;
- }
- if (type == DataElement::TYPE_BYTES) {
- shortcut_bytes += element.length();
- if (!shortcut_bytes.IsValid()) {
- return false;
- }
- }
- }
- return shortcut_bytes.ValueOrDie() <= memory_available;
-}
-
-/* static */
-void BlobAsyncTransportRequestBuilder::ComputeHandleSizes(
+void BlobTransportRequestBuilder::ComputeHandleSizes(
uint64_t total_memory_size,
size_t max_segment_size,
std::vector<size_t>* segment_sizes) {
diff --git a/chromium/storage/browser/blob/blob_async_transport_request_builder.h b/chromium/storage/browser/blob/blob_transport_request_builder.h
index 10dde5ff6ff..4dce9a123a5 100644
--- a/chromium/storage/browser/blob/blob_async_transport_request_builder.h
+++ b/chromium/storage/browser/blob/blob_transport_request_builder.h
@@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#ifndef STORAGE_BROWSER_BLOB_BLOB_ASYNC_TRANSPORT_REQUEST_BUILDER_H_
-#define STORAGE_BROWSER_BLOB_BLOB_ASYNC_TRANSPORT_REQUEST_BUILDER_H_
+#ifndef STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_REQUEST_BUILDER_H_
+#define STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_REQUEST_BUILDER_H_
#include <stddef.h>
#include <stdint.h>
@@ -28,7 +28,7 @@ namespace storage {
// where the data is already present in the blob description, and will
// always give the caller requests for requesting all data from the
// renderer.
-class STORAGE_EXPORT BlobAsyncTransportRequestBuilder {
+class STORAGE_EXPORT BlobTransportRequestBuilder {
public:
struct RendererMemoryItemRequest {
RendererMemoryItemRequest();
@@ -42,8 +42,10 @@ class STORAGE_EXPORT BlobAsyncTransportRequestBuilder {
BlobItemBytesRequest message;
};
- BlobAsyncTransportRequestBuilder();
- virtual ~BlobAsyncTransportRequestBuilder();
+ BlobTransportRequestBuilder();
+ BlobTransportRequestBuilder(BlobTransportRequestBuilder&&);
+ BlobTransportRequestBuilder& operator=(BlobTransportRequestBuilder&&);
+ virtual ~BlobTransportRequestBuilder();
// Initializes the request builder for file requests. One or more files are
// created to hold the given data. Each file can hold data from multiple
@@ -114,9 +116,6 @@ class STORAGE_EXPORT BlobAsyncTransportRequestBuilder {
// The total bytes size of memory items in the blob.
uint64_t total_bytes_size() const { return total_bytes_size_; }
- static bool ShouldBeShortcut(const std::vector<DataElement>& items,
- size_t memory_available);
-
private:
static void ComputeHandleSizes(uint64_t total_memory_size,
size_t max_segment_size,
@@ -130,9 +129,9 @@ class STORAGE_EXPORT BlobAsyncTransportRequestBuilder {
uint64_t total_bytes_size_;
std::vector<RendererMemoryItemRequest> requests_;
- DISALLOW_COPY_AND_ASSIGN(BlobAsyncTransportRequestBuilder);
+ DISALLOW_COPY_AND_ASSIGN(BlobTransportRequestBuilder);
};
} // namespace storage
-#endif // STORAGE_BROWSER_BLOB_BLOB_ASYNC_TRANSPORT_REQUEST_BUILDER_H_
+#endif // STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_REQUEST_BUILDER_H_
diff --git a/chromium/storage/browser/blob/blob_transport_result.h b/chromium/storage/browser/blob/blob_transport_result.h
deleted file mode 100644
index 9aa2f72ad4a..00000000000
--- a/chromium/storage/browser/blob/blob_transport_result.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_RESULT_H_
-#define STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_RESULT_H_
-
-namespace storage {
-
-// This 'result' enum is used in the blob transport logic.
-enum class BlobTransportResult {
- // This means we should flag the incoming IPC as a bad IPC.
- BAD_IPC,
- // Cancel reasons.
- CANCEL_MEMORY_FULL,
- CANCEL_FILE_ERROR,
- CANCEL_REFERENCED_BLOB_BROKEN,
- CANCEL_UNKNOWN,
- // This means we're waiting on responses from the renderer.
- PENDING_RESPONSES,
- // We're done registering or transferring the blob.
- DONE
-};
-
-} // namespace storage
-
-#endif // STORAGE_BROWSER_BLOB_BLOB_TRANSPORT_RESULT_H_
diff --git a/chromium/storage/browser/blob/internal_blob_data.cc b/chromium/storage/browser/blob/internal_blob_data.cc
deleted file mode 100644
index 115c6a65e1f..00000000000
--- a/chromium/storage/browser/blob/internal_blob_data.cc
+++ /dev/null
@@ -1,97 +0,0 @@
-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "storage/browser/blob/internal_blob_data.h"
-
-#include <stddef.h>
-
-#include <memory>
-#include <utility>
-
-#include "base/containers/hash_tables.h"
-#include "base/metrics/histogram.h"
-#include "storage/browser/blob/blob_data_item.h"
-#include "storage/common/data_element.h"
-
-namespace storage {
-
-InternalBlobData::Builder::Builder() : data_(new InternalBlobData()) {
-}
-InternalBlobData::Builder::~Builder() {
-}
-
-void InternalBlobData::Builder::AppendSharedBlobItem(
- scoped_refptr<ShareableBlobDataItem> item) {
- DCHECK(item);
- DCHECK(data_);
- data_->items_.push_back(item);
-}
-
-void InternalBlobData::Builder::RemoveBlobFromShareableItems(
- const std::string& blob_uuid) {
- DCHECK(data_);
- data_->RemoveBlobFromShareableItems(blob_uuid);
-}
-
-size_t InternalBlobData::Builder::GetNonsharedMemoryUsage() const {
- DCHECK(data_);
- return data_->GetUnsharedMemoryUsage();
-}
-
-std::unique_ptr<InternalBlobData> InternalBlobData::Builder::Build() {
- DCHECK(data_);
- return std::move(data_);
-}
-
-InternalBlobData::InternalBlobData() {
-}
-
-InternalBlobData::~InternalBlobData() {
-}
-
-const std::vector<scoped_refptr<ShareableBlobDataItem>>&
-InternalBlobData::items() const {
- return items_;
-}
-
-void InternalBlobData::RemoveBlobFromShareableItems(
- const std::string& blob_uuid) {
- for (auto& data_item : items_) {
- data_item->referencing_blobs().erase(blob_uuid);
- }
-}
-
-size_t InternalBlobData::GetUnsharedMemoryUsage() const {
- size_t memory = 0;
- base::hash_set<void*> seen_items;
- for (const auto& data_item : items_) {
- if (data_item->item()->type() != DataElement::TYPE_BYTES ||
- data_item->referencing_blobs().size() > 1 ||
- seen_items.find(data_item.get()) != seen_items.end()) {
- continue;
- }
- memory += data_item->item()->length();
- seen_items.insert(data_item.get());
- }
- return memory;
-}
-
-void InternalBlobData::GetMemoryUsage(size_t* total_memory,
- size_t* unshared_memory) {
- *total_memory = 0;
- *unshared_memory = 0;
- base::hash_set<void*> seen_items;
- for (const auto& data_item : items_) {
- if (data_item->item()->type() == DataElement::TYPE_BYTES) {
- *total_memory += data_item->item()->length();
- if (data_item->referencing_blobs().size() == 1 &&
- seen_items.find(data_item.get()) == seen_items.end()) {
- *unshared_memory += data_item->item()->length();
- seen_items.insert(data_item.get());
- }
- }
- }
-}
-
-} // namespace storage
diff --git a/chromium/storage/browser/blob/internal_blob_data.h b/chromium/storage/browser/blob/internal_blob_data.h
deleted file mode 100644
index 65db2f3cdc3..00000000000
--- a/chromium/storage/browser/blob/internal_blob_data.h
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef STORAGE_BROWSER_BLOB_INTERNAL_BLOB_DATA_H_
-#define STORAGE_BROWSER_BLOB_INTERNAL_BLOB_DATA_H_
-
-#include <stddef.h>
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "base/macros.h"
-#include "base/memory/ref_counted.h"
-#include "storage/browser/blob/shareable_blob_data_item.h"
-
-namespace storage {
-class ViewBlobInternalsJob;
-
-// This class represents a blob in the BlobStorageContext. It is constructed
-// using the internal Builder class.
-class InternalBlobData {
- public:
- ~InternalBlobData();
-
- protected:
- friend class BlobStorageContext;
- friend class BlobStorageRegistry;
- friend class ViewBlobInternalsJob;
-
- // Removes the given blob uuid from the internal ShareableBlobDataItems.
- // This is called when this blob is being destroyed.
- void RemoveBlobFromShareableItems(const std::string& blob_uuid);
-
- const std::vector<scoped_refptr<ShareableBlobDataItem>>& items() const;
-
- // Gets the memory used by this blob that is not shared by other blobs. This
- // also doesn't count duplicate items.
- size_t GetUnsharedMemoryUsage() const;
-
- // Gets the memory used by this blob. Total memory includes memory of items
- // possibly shared with other blobs, or items that appear multiple times in
- // this blob. Unshared memory is memory used by this blob that is not shared
- // by other blobs.
- void GetMemoryUsage(size_t* total_memory, size_t* unshared_memory);
-
- private:
- friend class Builder;
- InternalBlobData();
-
- std::string content_type_;
- std::string content_disposition_;
- std::vector<scoped_refptr<ShareableBlobDataItem>> items_;
-
- class Builder {
- public:
- Builder();
- ~Builder();
-
- void AppendSharedBlobItem(scoped_refptr<ShareableBlobDataItem> item);
-
- // Gets the memory used by this builder that is not shared with other blobs.
- size_t GetNonsharedMemoryUsage() const;
-
- // Removes the given blob uuid from the internal ShareableBlobDataItems.
- // This is called on destruction of the blob if we're still building it.
- void RemoveBlobFromShareableItems(const std::string& blob_uuid);
-
- // The builder is invalid after calling this method.
- std::unique_ptr<::storage::InternalBlobData> Build();
-
- private:
- std::unique_ptr<::storage::InternalBlobData> data_;
-
- DISALLOW_COPY_AND_ASSIGN(Builder);
- };
-
- DISALLOW_COPY_AND_ASSIGN(InternalBlobData);
-};
-
-} // namespace storage
-#endif // STORAGE_BROWSER_BLOB_INTERNAL_BLOB_DATA_H_
diff --git a/chromium/storage/browser/blob/shareable_blob_data_item.cc b/chromium/storage/browser/blob/shareable_blob_data_item.cc
index 40aaf9c7e10..2b32349085a 100644
--- a/chromium/storage/browser/blob/shareable_blob_data_item.cc
+++ b/chromium/storage/browser/blob/shareable_blob_data_item.cc
@@ -7,20 +7,44 @@
#include "storage/browser/blob/blob_data_item.h"
namespace storage {
+namespace {
+
+uint64_t GetAndIncrementItemId() {
+ static uint64_t sNextItemId = 0;
+ return sNextItemId++;
+}
+
+} // namespace
ShareableBlobDataItem::ShareableBlobDataItem(
- const std::string& blob_uuid,
- const scoped_refptr<BlobDataItem>& item)
- : item_(item) {
+ scoped_refptr<BlobDataItem> item,
+ ShareableBlobDataItem::State state)
+ : item_id_(GetAndIncrementItemId()), state_(state), item_(std::move(item)) {
DCHECK_NE(item_->type(), DataElement::TYPE_BLOB);
- referencing_blobs_.insert(blob_uuid);
}
ShareableBlobDataItem::~ShareableBlobDataItem() {
}
-const scoped_refptr<BlobDataItem>& ShareableBlobDataItem::item() {
- return item_;
+void ShareableBlobDataItem::set_item(scoped_refptr<BlobDataItem> item) {
+ item_ = std::move(item);
+}
+
+void PrintTo(const ShareableBlobDataItem& x, ::std::ostream* os) {
+ *os << "<ShareableBlobDataItem>{ item_id: " << x.item_id_
+ << ", state: " << x.state_ << ", item: ";
+ PrintTo(*x.item_, os);
+ *os << "]}";
+}
+
+bool operator==(const ShareableBlobDataItem& a,
+ const ShareableBlobDataItem& b) {
+ return a.item_id() == b.item_id() && *a.item() == *b.item();
+}
+
+bool operator!=(const ShareableBlobDataItem& a,
+ const ShareableBlobDataItem& b) {
+ return !(a == b);
}
} // namespace storage
diff --git a/chromium/storage/browser/blob/shareable_blob_data_item.h b/chromium/storage/browser/blob/shareable_blob_data_item.h
index 4c2f9c18332..7d6c15f8c2e 100644
--- a/chromium/storage/browser/blob/shareable_blob_data_item.h
+++ b/chromium/storage/browser/blob/shareable_blob_data_item.h
@@ -7,44 +7,94 @@
#include <string>
+#include "base/callback_helpers.h"
#include "base/containers/hash_tables.h"
#include "base/hash.h"
#include "base/macros.h"
#include "base/memory/ref_counted.h"
+#include "storage/browser/blob/blob_memory_controller.h"
+#include "storage/browser/storage_browser_export.h"
#include "storage/common/data_element.h"
namespace storage {
class BlobDataItem;
-class InternalBlobData;
-// This class allows blob items to be shared between blobs, and is only used by
-// BlobStorageContext. This class contains both the blob data item and the uuids
-// of all the blobs using this item.
+// This class allows blob items to be shared between blobs. This class contains
+// both the blob data item and the uuids of all the blobs using this item.
// The data in this class (the item) is immutable, but the item itself can be
// swapped out with an item with the same data but a different backing (think
// RAM vs file backed).
-class ShareableBlobDataItem : public base::RefCounted<ShareableBlobDataItem> {
+// We also allow the storage of a memory quota allocation object which is used
+// for memory quota reclamation.
+class STORAGE_EXPORT ShareableBlobDataItem
+ : public base::RefCounted<ShareableBlobDataItem> {
public:
- ShareableBlobDataItem(const std::string& blob_uuid,
- const scoped_refptr<BlobDataItem>& item);
+ enum State {
+ // We're an item that needs quota (either disk or memory).
+ QUOTA_NEEDED,
+ // We have requested quota from the BlobMemoryController.
+ QUOTA_REQUESTED,
+ // Space has been allocated for this item in the BlobMemoryController, but
+ // it may not yet be populated.
+ QUOTA_GRANTED,
+ // We're a populated item that needed quota.
+ POPULATED_WITH_QUOTA,
+ // We're a populated item that didn't need quota.
+ POPULATED_WITHOUT_QUOTA
+ };
- const scoped_refptr<BlobDataItem>& item();
+ ShareableBlobDataItem(scoped_refptr<BlobDataItem> item, State state);
- base::hash_set<std::string>& referencing_blobs() {
- return referencing_blobs_;
+ const scoped_refptr<BlobDataItem>& item() const { return item_; }
+
+ void set_item(scoped_refptr<BlobDataItem> item);
+
+ // This is a unique auto-incrementing id assigned to this item on
+ // construction. It is used to keep track of this item in an LRU data
+ // structure for eviction to disk.
+ uint64_t item_id() const { return item_id_; }
+
+ State state() const { return state_; }
+ void set_state(State state) { state_ = state; }
+
+ bool IsPopulated() const {
+ return state_ == POPULATED_WITH_QUOTA || state_ == POPULATED_WITHOUT_QUOTA;
+ }
+
+ bool HasGrantedQuota() const {
+ return state_ == POPULATED_WITH_QUOTA || state_ == QUOTA_GRANTED;
}
private:
+ friend class BlobMemoryController;
+ friend class BlobMemoryControllerTest;
+ friend class BlobStorageContext;
friend class base::RefCounted<ShareableBlobDataItem>;
- friend class InternalBlobData;
+ friend STORAGE_EXPORT void PrintTo(const ShareableBlobDataItem& x,
+ ::std::ostream* os);
+
~ShareableBlobDataItem();
- scoped_refptr<BlobDataItem> item_;
+ void set_memory_allocation(
+ std::unique_ptr<BlobMemoryController::MemoryAllocation> allocation) {
+ memory_allocation_ = std::move(allocation);
+ }
- base::hash_set<std::string> referencing_blobs_;
+ bool has_memory_allocation() { return static_cast<bool>(memory_allocation_); }
+
+ // This is a unique identifier for this ShareableBlobDataItem.
+ const uint64_t item_id_;
+ State state_;
+ scoped_refptr<BlobDataItem> item_;
+ std::unique_ptr<BlobMemoryController::MemoryAllocation> memory_allocation_;
DISALLOW_COPY_AND_ASSIGN(ShareableBlobDataItem);
};
+STORAGE_EXPORT bool operator==(const ShareableBlobDataItem& a,
+ const ShareableBlobDataItem& b);
+STORAGE_EXPORT bool operator!=(const ShareableBlobDataItem& a,
+ const ShareableBlobDataItem& b);
+
} // namespace storage
#endif // STORAGE_BROWSER_BLOB_SHAREABLE_BLOB_DATA_ITEM_H_
diff --git a/chromium/storage/browser/blob/view_blob_internals_job.cc b/chromium/storage/browser/blob/view_blob_internals_job.cc
index 269a56036fe..a076321f1ef 100644
--- a/chromium/storage/browser/blob/view_blob_internals_job.cc
+++ b/chromium/storage/browser/blob/view_blob_internals_job.cc
@@ -25,9 +25,10 @@
#include "net/disk_cache/disk_cache.h"
#include "net/url_request/url_request.h"
#include "storage/browser/blob/blob_data_item.h"
+#include "storage/browser/blob/blob_entry.h"
#include "storage/browser/blob/blob_storage_context.h"
#include "storage/browser/blob/blob_storage_registry.h"
-#include "storage/browser/blob/internal_blob_data.h"
+#include "storage/browser/blob/shareable_blob_data_item.h"
namespace {
@@ -143,7 +144,7 @@ int ViewBlobInternalsJob::GetData(
data->clear();
StartHTML(data);
- if (blob_storage_context_->registry_.blob_map_.empty())
+ if (blob_storage_context_->registry().blob_map_.empty())
data->append(kEmptyBlobStorageMessage);
else
GenerateHTML(data);
@@ -153,18 +154,18 @@ int ViewBlobInternalsJob::GetData(
void ViewBlobInternalsJob::GenerateHTML(std::string* out) const {
for (BlobStorageRegistry::BlobMap::const_iterator iter =
- blob_storage_context_->registry_.blob_map_.begin();
- iter != blob_storage_context_->registry_.blob_map_.end(); ++iter) {
+ blob_storage_context_->registry().blob_map_.begin();
+ iter != blob_storage_context_->registry().blob_map_.end(); ++iter) {
AddHTMLBoldText(iter->first, out);
- GenerateHTMLForBlobData(*iter->second->data, iter->second->content_type,
- iter->second->content_disposition,
- iter->second->refcount, out);
+ GenerateHTMLForBlobData(*iter->second, iter->second->content_type(),
+ iter->second->content_disposition(),
+ iter->second->refcount(), out);
}
- if (!blob_storage_context_->registry_.url_to_uuid_.empty()) {
+ if (!blob_storage_context_->registry().url_to_uuid_.empty()) {
AddHorizontalRule(out);
for (BlobStorageRegistry::URLMap::const_iterator iter =
- blob_storage_context_->registry_.url_to_uuid_.begin();
- iter != blob_storage_context_->registry_.url_to_uuid_.end(); ++iter) {
+ blob_storage_context_->registry().url_to_uuid_.begin();
+ iter != blob_storage_context_->registry().url_to_uuid_.end(); ++iter) {
AddHTMLBoldText(iter->first.spec(), out);
StartHTMLList(out);
AddHTMLListItem(kUUID, iter->second, out);
@@ -174,7 +175,7 @@ void ViewBlobInternalsJob::GenerateHTML(std::string* out) const {
}
void ViewBlobInternalsJob::GenerateHTMLForBlobData(
- const InternalBlobData& blob_data,
+ const BlobEntry& blob_data,
const std::string& content_type,
const std::string& content_disposition,
int refcount,
@@ -232,6 +233,7 @@ void ViewBlobInternalsJob::GenerateHTMLForBlobData(
AddHTMLListItem(kURL, item.disk_cache_entry()->GetKey(), out);
break;
case DataElement::TYPE_BYTES_DESCRIPTION:
+ AddHTMLListItem(kType, "pending data", out);
case DataElement::TYPE_UNKNOWN:
NOTREACHED();
break;
diff --git a/chromium/storage/browser/blob/view_blob_internals_job.h b/chromium/storage/browser/blob/view_blob_internals_job.h
index ca6782be333..b1490fb86f6 100644
--- a/chromium/storage/browser/blob/view_blob_internals_job.h
+++ b/chromium/storage/browser/blob/view_blob_internals_job.h
@@ -18,7 +18,7 @@ class URLRequest;
namespace storage {
-class InternalBlobData;
+class BlobEntry;
class BlobStorageContext;
// A job subclass that implements a protocol to inspect the internal
@@ -42,7 +42,7 @@ class STORAGE_EXPORT ViewBlobInternalsJob
~ViewBlobInternalsJob() override;
void GenerateHTML(std::string* out) const;
- static void GenerateHTMLForBlobData(const InternalBlobData& blob_data,
+ static void GenerateHTMLForBlobData(const BlobEntry& blob_data,
const std::string& content_type,
const std::string& content_disposition,
int refcount,
diff --git a/chromium/storage/browser/crbug653751_unittest.cc b/chromium/storage/browser/crbug653751_unittest.cc
new file mode 100644
index 00000000000..44b86fb6ee4
--- /dev/null
+++ b/chromium/storage/browser/crbug653751_unittest.cc
@@ -0,0 +1,13 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+// This file will be removed when the real storage unit tests are migrated over
+// from content_unittests to storage_unittests. Until then, it is needed to keep
+// the win_clang builder happy. See http://crbug.com/653751
+
+TEST(StorageUnittests, EmptyTest) {
+
+}
diff --git a/chromium/storage/browser/database/database_tracker.cc b/chromium/storage/browser/database/database_tracker.cc
index 42041c7869a..1539b0cd643 100644
--- a/chromium/storage/browser/database/database_tracker.cc
+++ b/chromium/storage/browser/database/database_tracker.cc
@@ -614,8 +614,8 @@ int64_t DatabaseTracker::UpdateOpenDatabaseInfoAndNotify(
storage::GetOriginFromIdentifier(origin_id),
storage::kStorageTypeTemporary,
new_size - old_size);
- FOR_EACH_OBSERVER(Observer, observers_, OnDatabaseSizeChanged(
- origin_id, name, new_size));
+ for (auto& observer : observers_)
+ observer.OnDatabaseSizeChanged(origin_id, name, new_size);
}
return new_size;
}
@@ -626,8 +626,8 @@ void DatabaseTracker::ScheduleDatabaseForDeletion(
DCHECK(database_connections_.IsDatabaseOpened(origin_identifier,
database_name));
dbs_to_be_deleted_[origin_identifier].insert(database_name);
- FOR_EACH_OBSERVER(Observer, observers_, OnDatabaseScheduledForDeletion(
- origin_identifier, database_name));
+ for (auto& observer : observers_)
+ observer.OnDatabaseScheduledForDeletion(origin_identifier, database_name);
}
void DatabaseTracker::ScheduleDatabasesForDeletion(
diff --git a/chromium/storage/browser/fileapi/file_system_context.cc b/chromium/storage/browser/fileapi/file_system_context.cc
index 9c1de0ba4e9..f1fcf23eb9d 100644
--- a/chromium/storage/browser/fileapi/file_system_context.cc
+++ b/chromium/storage/browser/fileapi/file_system_context.cc
@@ -103,6 +103,7 @@ int FileSystemContext::GetPermissionPolicy(FileSystemType type) {
return FILE_PERMISSION_USE_FILE_PERMISSION;
case kFileSystemTypeRestrictedNativeLocal:
+ case kFileSystemTypeArcContent:
return FILE_PERMISSION_READ_ONLY |
FILE_PERMISSION_USE_FILE_PERMISSION;
diff --git a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc
index 06e9aefb3f1..efb51f74cb3 100644
--- a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc
+++ b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.cc
@@ -10,6 +10,8 @@
#include <memory>
#include <utility>
+#include "base/files/file_enumerator.h"
+#include "base/files/file_path.h"
#include "base/stl_util.h"
#include "base/synchronization/lock.h"
#include "base/task_runner_util.h"
@@ -266,8 +268,75 @@ int64_t PluginPrivateFileSystemBackend::GetOriginUsageOnFileTaskRunner(
FileSystemContext* context,
const GURL& origin_url,
FileSystemType type) {
- // We don't track usage on this filesystem.
- return 0;
+ DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
+
+ if (!CanHandleType(type))
+ return 0;
+
+ int64_t total_size;
+ base::Time last_modified_time;
+ GetOriginDetailsOnFileTaskRunner(context, origin_url, &total_size,
+ &last_modified_time);
+ return total_size;
+}
+
+void PluginPrivateFileSystemBackend::GetOriginDetailsOnFileTaskRunner(
+ FileSystemContext* context,
+ const GURL& origin_url,
+ int64_t* total_size,
+ base::Time* last_modified_time) {
+ DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
+
+ *total_size = 0;
+ *last_modified_time = base::Time::UnixEpoch();
+ std::string fsid =
+ storage::IsolatedContext::GetInstance()->RegisterFileSystemForVirtualPath(
+ storage::kFileSystemTypePluginPrivate, "pluginprivate",
+ base::FilePath());
+ DCHECK(storage::ValidateIsolatedFileSystemId(fsid));
+
+ std::string root = storage::GetIsolatedFileSystemRootURIString(
+ origin_url, fsid, "pluginprivate");
+
+ std::unique_ptr<FileSystemOperationContext> operation_context(
+ new FileSystemOperationContext(context));
+
+ // Determine the available plugin private filesystem directories for this
+ // origin. Currently the plugin private filesystem is only used by Encrypted
+ // Media Content Decryption Modules. Each CDM gets a directory based on the
+ // mimetype (e.g. plugin application/x-ppapi-widevine-cdm uses directory
+ // application_x-ppapi-widevine-cdm). Enumerate through the set of
+ // directories so that data from any CDM used by this origin is counted.
+ base::File::Error error;
+ base::FilePath path = obfuscated_file_util()->GetDirectoryForOriginAndType(
+ origin_url, "", false, &error);
+ if (error != base::File::FILE_OK) {
+ DLOG(ERROR) << "Unable to read directory for " << origin_url;
+ return;
+ }
+
+ base::FileEnumerator directory_enumerator(path, false,
+ base::FileEnumerator::DIRECTORIES);
+ base::FilePath plugin_path;
+ while (!(plugin_path = directory_enumerator.Next()).empty()) {
+ std::string plugin_name = plugin_path.BaseName().MaybeAsASCII();
+ if (OpenFileSystemOnFileTaskRunner(
+ obfuscated_file_util(), plugin_map_, origin_url, fsid, plugin_name,
+ storage::OPEN_FILE_SYSTEM_FAIL_IF_NONEXISTENT) !=
+ base::File::FILE_OK) {
+ continue;
+ }
+
+ std::unique_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator(
+ obfuscated_file_util()->CreateFileEnumerator(
+ operation_context.get(), context->CrackURL(GURL(root)), true));
+
+ while (!enumerator->Next().empty()) {
+ *total_size += enumerator->Size();
+ if (enumerator->LastModifiedTime() > *last_modified_time)
+ *last_modified_time = enumerator->LastModifiedTime();
+ }
+ }
}
scoped_refptr<QuotaReservation>
diff --git a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h
index 95ffb647147..89da39bf290 100644
--- a/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h
+++ b/chromium/storage/browser/fileapi/plugin_private_file_system_backend.h
@@ -118,6 +118,15 @@ class STORAGE_EXPORT PluginPrivateFileSystemBackend
const GURL& origin_url,
FileSystemType type) override;
+ // Get details on the files saved for the specified |origin_url|. Returns
+ // the total size and last modified time for the set of all files stored
+ // for the particular origin. |total_size| = 0 and |last_modified_time| =
+ // base::Time::UnixEpoch() if no files found.
+ void GetOriginDetailsOnFileTaskRunner(FileSystemContext* context,
+ const GURL& origin_url,
+ int64_t* total_size,
+ base::Time* last_modified_time);
+
private:
friend class content::PluginPrivateFileSystemBackendTest;
diff --git a/chromium/storage/browser/fileapi/sandbox_directory_database.cc b/chromium/storage/browser/fileapi/sandbox_directory_database.cc
index a4e845aa61d..91c4914c7f2 100644
--- a/chromium/storage/browser/fileapi/sandbox_directory_database.cc
+++ b/chromium/storage/browser/fileapi/sandbox_directory_database.cc
@@ -17,7 +17,7 @@
#include "base/files/file_util.h"
#include "base/location.h"
#include "base/macros.h"
-#include "base/metrics/histogram.h"
+#include "base/metrics/histogram_macros.h"
#include "base/pickle.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_util.h"
diff --git a/chromium/storage/browser/fileapi/sandbox_origin_database.cc b/chromium/storage/browser/fileapi/sandbox_origin_database.cc
index 992c5cfbe6b..2cfd3f3da9a 100644
--- a/chromium/storage/browser/fileapi/sandbox_origin_database.cc
+++ b/chromium/storage/browser/fileapi/sandbox_origin_database.cc
@@ -15,7 +15,7 @@
#include "base/format_macros.h"
#include "base/location.h"
#include "base/logging.h"
-#include "base/metrics/histogram.h"
+#include "base/metrics/histogram_macros.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
diff --git a/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc b/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc
index 9174377c8fa..6cce670357c 100644
--- a/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc
+++ b/chromium/storage/browser/quota/quota_temporary_storage_evictor.cc
@@ -9,7 +9,7 @@
#include <algorithm>
#include "base/bind.h"
-#include "base/metrics/histogram.h"
+#include "base/metrics/histogram_macros.h"
#include "storage/browser/quota/quota_manager.h"
#include "url/gurl.h"
diff --git a/chromium/storage/browser/quota/special_storage_policy.cc b/chromium/storage/browser/quota/special_storage_policy.cc
index 6e774505df5..40125ac126e 100644
--- a/chromium/storage/browser/quota/special_storage_policy.cc
+++ b/chromium/storage/browser/quota/special_storage_policy.cc
@@ -22,17 +22,20 @@ void SpecialStoragePolicy::RemoveObserver(Observer* observer) {
void SpecialStoragePolicy::NotifyGranted(const GURL& origin, int change_flags) {
scoped_refptr<SpecialStoragePolicy> protect(this);
- FOR_EACH_OBSERVER(Observer, observers_, OnGranted(origin, change_flags));
+ for (auto& observer : observers_)
+ observer.OnGranted(origin, change_flags);
}
void SpecialStoragePolicy::NotifyRevoked(const GURL& origin, int change_flags) {
scoped_refptr<SpecialStoragePolicy> protect(this);
- FOR_EACH_OBSERVER(Observer, observers_, OnRevoked(origin, change_flags));
+ for (auto& observer : observers_)
+ observer.OnRevoked(origin, change_flags);
}
void SpecialStoragePolicy::NotifyCleared() {
scoped_refptr<SpecialStoragePolicy> protect(this);
- FOR_EACH_OBSERVER(Observer, observers_, OnCleared());
+ for (auto& observer : observers_)
+ observer.OnCleared();
}
} // namespace storage
diff --git a/chromium/storage/browser/quota/usage_tracker.cc b/chromium/storage/browser/quota/usage_tracker.cc
index e324bfe7d28..041f94067ae 100644
--- a/chromium/storage/browser/quota/usage_tracker.cc
+++ b/chromium/storage/browser/quota/usage_tracker.cc
@@ -9,7 +9,7 @@
#include <algorithm>
#include "base/bind.h"
-#include "base/stl_util.h"
+#include "base/memory/ptr_util.h"
#include "storage/browser/quota/client_usage_tracker.h"
#include "storage/browser/quota/storage_monitor.h"
@@ -34,21 +34,18 @@ UsageTracker::UsageTracker(const QuotaClientList& clients,
weak_factory_(this) {
for (auto* client : clients) {
if (client->DoesSupport(type)) {
- client_tracker_map_[client->id()] =
- new ClientUsageTracker(this, client, type, special_storage_policy,
- storage_monitor_);
+ client_tracker_map_[client->id()] = base::MakeUnique<ClientUsageTracker>(
+ this, client, type, special_storage_policy, storage_monitor_);
}
}
}
-UsageTracker::~UsageTracker() {
- base::STLDeleteValues(&client_tracker_map_);
-}
+UsageTracker::~UsageTracker() {}
ClientUsageTracker* UsageTracker::GetClientTracker(QuotaClient::ID client_id) {
- ClientTrackerMap::iterator found = client_tracker_map_.find(client_id);
+ auto found = client_tracker_map_.find(client_id);
if (found != client_tracker_map_.end())
- return found->second;
+ return found->second.get();
return nullptr;
}
diff --git a/chromium/storage/browser/quota/usage_tracker.h b/chromium/storage/browser/quota/usage_tracker.h
index 994fc584635..074b4f54e23 100644
--- a/chromium/storage/browser/quota/usage_tracker.h
+++ b/chromium/storage/browser/quota/usage_tracker.h
@@ -67,8 +67,6 @@ class STORAGE_EXPORT UsageTracker : public QuotaTaskObserver {
int64_t unlimited_usage;
};
- typedef std::map<QuotaClient::ID, ClientUsageTracker*> ClientTrackerMap;
-
typedef CallbackQueue<UsageCallback, int64_t> UsageCallbackQueue;
typedef CallbackQueue<GlobalUsageCallback, int64_t, int64_t>
GlobalUsageCallbackQueue;
@@ -86,7 +84,8 @@ class STORAGE_EXPORT UsageTracker : public QuotaTaskObserver {
int64_t usage);
const StorageType type_;
- ClientTrackerMap client_tracker_map_;
+ std::map<QuotaClient::ID, std::unique_ptr<ClientUsageTracker>>
+ client_tracker_map_;
UsageCallbackQueue global_limited_usage_callbacks_;
GlobalUsageCallbackQueue global_usage_callbacks_;
diff --git a/chromium/storage/common/BUILD.gn b/chromium/storage/common/BUILD.gn
index 03766c3a20b..1f15f489857 100644
--- a/chromium/storage/common/BUILD.gn
+++ b/chromium/storage/common/BUILD.gn
@@ -9,6 +9,7 @@ component("common") {
"blob_storage/blob_item_bytes_request.h",
"blob_storage/blob_item_bytes_response.cc",
"blob_storage/blob_item_bytes_response.h",
+ "blob_storage/blob_storage_constants.cc",
"blob_storage/blob_storage_constants.h",
"data_element.cc",
"data_element.h",
diff --git a/chromium/storage/common/blob_storage/blob_storage_constants.cc b/chromium/storage/common/blob_storage/blob_storage_constants.cc
new file mode 100644
index 00000000000..2d825f4534b
--- /dev/null
+++ b/chromium/storage/common/blob_storage/blob_storage_constants.cc
@@ -0,0 +1,25 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "storage/common/blob_storage/blob_storage_constants.h"
+
+#include "base/logging.h"
+
+namespace storage {
+
+bool BlobStatusIsError(BlobStatus status) {
+ return static_cast<int>(status) <= static_cast<int>(BlobStatus::LAST_ERROR);
+}
+
+bool BlobStatusIsPending(BlobStatus status) {
+ int status_int = static_cast<int>(status);
+ return status_int >= static_cast<int>(BlobStatus::PENDING_QUOTA) &&
+ status_int <= static_cast<int>(BlobStatus::PENDING_INTERNALS);
+}
+
+bool BlobStatusIsBadIPC(BlobStatus status) {
+ return status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+}
+
+} // namespace storage
diff --git a/chromium/storage/common/blob_storage/blob_storage_constants.h b/chromium/storage/common/blob_storage/blob_storage_constants.h
index 0fc10c583ce..f54d20aaa70 100644
--- a/chromium/storage/common/blob_storage/blob_storage_constants.h
+++ b/chromium/storage/common/blob_storage/blob_storage_constants.h
@@ -8,17 +8,35 @@
#include <stddef.h>
#include <stdint.h>
+#include "base/callback_forward.h"
+#include "storage/common/storage_common_export.h"
+
namespace storage {
-// TODO(michaeln): use base::SysInfo::AmountOfPhysicalMemoryMB() in some
-// way to come up with a better limit.
-const int64_t kBlobStorageMaxMemoryUsage = 500 * 1024 * 1024; // Half a gig.
-const size_t kBlobStorageIPCThresholdBytes = 250 * 1024;
-const size_t kBlobStorageMaxSharedMemoryBytes = 10 * 1024 * 1024;
-const uint64_t kBlobStorageMaxFileSizeBytes = 100 * 1024 * 1024;
-const uint64_t kBlobStorageMinFileSizeBytes = 1 * 1024 * 1024;
-const size_t kBlobStorageMaxBlobMemorySize =
- kBlobStorageMaxMemoryUsage - kBlobStorageMinFileSizeBytes;
+// All sizes are in bytes.
+struct BlobStorageLimits {
+ size_t memory_limit_before_paging() const {
+ return max_blob_in_memory_space - min_page_file_size;
+ }
+
+ // This is the maximum amount of memory we can send in an IPC.
+ size_t max_ipc_memory_size = 250 * 1024;
+ // This is the maximum size of a shared memory handle.
+ size_t max_shared_memory_size = 10 * 1024 * 1024;
+
+ // This is the maximum amount of memory we can use to store blobs.
+ size_t max_blob_in_memory_space = 500 * 1024 * 1024;
+
+ // This is the maximum amount of disk space we can use.
+ // TODO(dmurph): Consider storage size of the device.
+ uint64_t max_blob_disk_space = 5ull * 1024 * 1024 * 1024;
+
+ // This is the minimum file size we can use when paging blob items to disk.
+ // We combine items until we reach at least this size.
+ uint64_t min_page_file_size = 5 * 1024 * 1024;
+ // This is the maximum file size we can create.
+ uint64_t max_file_size = 100 * 1024 * 1024;
+};
enum class IPCBlobItemRequestStrategy {
UNKNOWN = 0,
@@ -28,27 +46,54 @@ enum class IPCBlobItemRequestStrategy {
LAST = FILE
};
-// These items cannot be reordered or renumbered because they're recorded to
-// UMA. New items must be added immediately before LAST, and LAST must be set to
-// the the last item.
-enum class IPCBlobCreationCancelCode {
- UNKNOWN = 0,
- OUT_OF_MEMORY = 1,
+// This is the enum to rule them all in the blob system.
+// These values are used in UMA metrics, so they should not be changed. Please
+// update LAST_ERROR if you add an error condition and LAST if you add new
+// state.
+enum class BlobStatus {
+ // Error case section:
+ // The construction arguments are invalid. This is considered a bad ipc.
+ ERR_INVALID_CONSTRUCTION_ARGUMENTS = 0,
+ // We don't have enough memory for the blob.
+ ERR_OUT_OF_MEMORY = 1,
// We couldn't create or write to a file. File system error, like a full disk.
- FILE_WRITE_FAILED = 2,
+ ERR_FILE_WRITE_FAILED = 2,
// The renderer was destroyed while data was in transit.
- SOURCE_DIED_IN_TRANSIT = 3,
+ ERR_SOURCE_DIED_IN_TRANSIT = 3,
// The renderer destructed the blob before it was done transferring, and there
// were no outstanding references (no one is waiting to read) to keep the
// blob alive.
- BLOB_DEREFERENCED_WHILE_BUILDING = 4,
+ ERR_BLOB_DEREFERENCED_WHILE_BUILDING = 4,
// A blob that we referenced during construction is broken, or a browser-side
// builder tries to build a blob with a blob reference that isn't finished
// constructing.
- REFERENCED_BLOB_BROKEN = 5,
- LAST = REFERENCED_BLOB_BROKEN
+ ERR_REFERENCED_BLOB_BROKEN = 5,
+ LAST_ERROR = ERR_REFERENCED_BLOB_BROKEN,
+
+ // Blob state section:
+ // The blob has finished.
+ DONE = 200,
+ // The system is pending on quota being granted, the transport layer
+ // populating pending data, and/or copying data from dependent blobs. See
+ // BlobEntry::BuildingState determine which of these are happening, as they
+ // all can happen concurrently.
+ PENDING_QUOTA = 201,
+ PENDING_TRANSPORT = 202,
+ PENDING_INTERNALS = 203,
+ LAST = PENDING_INTERNALS
};
+using BlobStatusCallback = base::Callback<void(BlobStatus)>;
+
+// Returns if the status is an error code.
+STORAGE_COMMON_EXPORT bool BlobStatusIsError(BlobStatus status);
+
+STORAGE_COMMON_EXPORT bool BlobStatusIsPending(BlobStatus status);
+
+// Returns if the status is a bad enough error to flag the IPC as bad. This is
+// only INVALID_CONSTRUCTION_ARGUMENTS.
+STORAGE_COMMON_EXPORT bool BlobStatusIsBadIPC(BlobStatus status);
+
} // namespace storage
#endif // STORAGE_COMMON_BLOB_STORAGE_BLOB_STORAGE_CONSTANTS_H_
diff --git a/chromium/storage/common/data_element.h b/chromium/storage/common/data_element.h
index 276181b6ec9..744cc99d11c 100644
--- a/chromium/storage/common/data_element.h
+++ b/chromium/storage/common/data_element.h
@@ -26,6 +26,8 @@ namespace storage {
// bytes, file or blob data.
class STORAGE_COMMON_EXPORT DataElement {
public:
+ static const uint64_t kUnknownSize = std::numeric_limits<uint64_t>::max();
+
enum Type {
TYPE_UNKNOWN = -1,
TYPE_BYTES,
diff --git a/chromium/storage/common/fileapi/file_system_types.h b/chromium/storage/common/fileapi/file_system_types.h
index 3f4f942cd9e..6ac262cc4ff 100644
--- a/chromium/storage/common/fileapi/file_system_types.h
+++ b/chromium/storage/common/fileapi/file_system_types.h
@@ -123,6 +123,9 @@ enum FileSystemType {
// limited to media files.
kFileSystemTypeDeviceMediaAsFileStorage,
+ // A filesystem to provide access to contents managed by ARC.
+ kFileSystemTypeArcContent,
+
// --------------------------------------------------------------------
// Marks the end of internal type enum. (This is not the actual fs type)
// New internal filesystem types must be added above this line.
diff --git a/chromium/storage/common/fileapi/file_system_util.cc b/chromium/storage/common/fileapi/file_system_util.cc
index 0472c957d28..1116fcd5631 100644
--- a/chromium/storage/common/fileapi/file_system_util.cc
+++ b/chromium/storage/common/fileapi/file_system_util.cc
@@ -321,6 +321,8 @@ std::string GetFileSystemTypeString(FileSystemType type) {
return "Provided";
case kFileSystemTypeDeviceMediaAsFileStorage:
return "DeviceMediaStorage";
+ case kFileSystemTypeArcContent:
+ return "ArcContent";
case kFileSystemInternalTypeEnumStart:
case kFileSystemInternalTypeEnumEnd:
NOTREACHED();